blob: 7f38d8c3fd5e47b6c99b5beb7af34d29808caac4 [file] [log] [blame]
Mike Frysingerf6013762019-06-13 02:30:51 -04001# -*- coding:utf-8 -*-
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07002#
3# Copyright (C) 2008 The Android Open Source Project
4#
5# Licensed under the Apache License, Version 2.0 (the "License");
6# you may not use this file except in compliance with the License.
7# You may obtain a copy of the License at
8#
9# http://www.apache.org/licenses/LICENSE-2.0
10#
11# Unless required by applicable law or agreed to in writing, software
12# distributed under the License is distributed on an "AS IS" BASIS,
13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14# See the License for the specific language governing permissions and
15# limitations under the License.
16
Sarah Owenscecd1d82012-11-01 22:59:27 -070017from __future__ import print_function
Colin Cross23acdd32012-04-21 00:33:54 -070018import itertools
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070019import os
Conley Owensdb728cd2011-09-26 16:34:01 -070020import re
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070021import sys
David Pursehouse59bbb582013-05-17 10:49:33 +090022import xml.dom.minidom
23
24from pyversion import is_python3
25if is_python3():
Chirayu Desai217ea7d2013-03-01 19:14:38 +053026 import urllib.parse
David Pursehouse59bbb582013-05-17 10:49:33 +090027else:
Chirayu Desai217ea7d2013-03-01 19:14:38 +053028 import imp
29 import urlparse
30 urllib = imp.new_module('urllib')
Chirayu Desaidb2ad9d2013-06-11 13:42:25 +053031 urllib.parse = urlparse
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070032
Simran Basib9a1b732015-08-20 12:19:28 -070033import gitc_utils
David Pursehousee15c65a2012-08-22 10:46:11 +090034from git_config import GitConfig
David Pursehousee00aa6b2012-09-11 14:33:51 +090035from git_refs import R_HEADS, HEAD
Renaud Paquayd5cec5e2016-11-01 11:24:03 -070036import platform_utils
David Pursehousee00aa6b2012-09-11 14:33:51 +090037from project import RemoteSpec, Project, MetaProject
Mike Frysinger04122b72019-07-31 23:32:58 -040038from error import (ManifestParseError, ManifestInvalidPathError,
39 ManifestInvalidRevisionError)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070040
41MANIFEST_FILE_NAME = 'manifest.xml'
Shawn O. Pearce5cc66792008-10-23 16:19:27 -070042LOCAL_MANIFEST_NAME = 'local_manifest.xml'
David Pursehouse2d5a0df2012-11-13 02:50:36 +090043LOCAL_MANIFESTS_DIR_NAME = 'local_manifests'
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070044
Anthony Kingcb07ba72015-03-28 23:26:04 +000045# urljoin gets confused if the scheme is not known.
Joe Kilner6e310792016-10-27 15:53:53 -070046urllib.parse.uses_relative.extend([
47 'ssh',
48 'git',
49 'persistent-https',
50 'sso',
51 'rpc'])
52urllib.parse.uses_netloc.extend([
53 'ssh',
54 'git',
55 'persistent-https',
56 'sso',
57 'rpc'])
Conley Owensdb728cd2011-09-26 16:34:01 -070058
David Pursehouse819827a2020-02-12 15:20:19 +090059
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070060class _Default(object):
61 """Project defaults within the manifest."""
62
Shawn O. Pearce3c8dea12009-05-29 18:38:17 -070063 revisionExpr = None
Conley Owensb6a16e62013-09-25 15:06:09 -070064 destBranchExpr = None
Nasser Grainawida403412018-05-04 12:53:29 -060065 upstreamExpr = None
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070066 remote = None
Shawn O. Pearce6392c872011-09-22 17:44:31 -070067 sync_j = 1
Anatol Pomazau79770d22012-04-20 14:41:59 -070068 sync_c = False
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +080069 sync_s = False
YOUNG HO CHAa32c92c2018-02-14 16:57:31 +090070 sync_tags = True
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070071
Julien Campergue74879922013-10-09 14:38:46 +020072 def __eq__(self, other):
73 return self.__dict__ == other.__dict__
74
75 def __ne__(self, other):
76 return self.__dict__ != other.__dict__
77
David Pursehouse819827a2020-02-12 15:20:19 +090078
Shawn O. Pearced1f70d92009-05-19 14:58:02 -070079class _XmlRemote(object):
80 def __init__(self,
81 name,
Yestin Sunb292b982012-07-02 07:32:50 -070082 alias=None,
Shawn O. Pearced1f70d92009-05-19 14:58:02 -070083 fetch=None,
Steve Raed6480452016-08-10 15:00:00 -070084 pushUrl=None,
Conley Owensdb728cd2011-09-26 16:34:01 -070085 manifestUrl=None,
Anthony King36ea2fb2014-05-06 11:54:01 +010086 review=None,
Jonathan Nieder93719792015-03-17 11:29:58 -070087 revision=None):
Shawn O. Pearced1f70d92009-05-19 14:58:02 -070088 self.name = name
89 self.fetchUrl = fetch
Steve Raed6480452016-08-10 15:00:00 -070090 self.pushUrl = pushUrl
Conley Owensdb728cd2011-09-26 16:34:01 -070091 self.manifestUrl = manifestUrl
Yestin Sunb292b982012-07-02 07:32:50 -070092 self.remoteAlias = alias
Shawn O. Pearced1f70d92009-05-19 14:58:02 -070093 self.reviewUrl = review
Anthony King36ea2fb2014-05-06 11:54:01 +010094 self.revision = revision
Conley Owensceea3682011-10-20 10:45:47 -070095 self.resolvedFetchUrl = self._resolveFetchUrl()
Shawn O. Pearced1f70d92009-05-19 14:58:02 -070096
David Pursehouse717ece92012-11-13 08:49:16 +090097 def __eq__(self, other):
98 return self.__dict__ == other.__dict__
99
100 def __ne__(self, other):
101 return self.__dict__ != other.__dict__
102
Conley Owensceea3682011-10-20 10:45:47 -0700103 def _resolveFetchUrl(self):
104 url = self.fetchUrl.rstrip('/')
Conley Owensdb728cd2011-09-26 16:34:01 -0700105 manifestUrl = self.manifestUrl.rstrip('/')
Conley Owens2d0f5082014-01-31 15:03:51 -0800106 # urljoin will gets confused over quite a few things. The ones we care
107 # about here are:
108 # * no scheme in the base url, like <hostname:port>
Anthony Kingcb07ba72015-03-28 23:26:04 +0000109 # We handle no scheme by replacing it with an obscure protocol, gopher
110 # and then replacing it with the original when we are done.
111
Conley Owensdb728cd2011-09-26 16:34:01 -0700112 if manifestUrl.find(':') != manifestUrl.find('/') - 1:
Conley Owens4ccad752015-04-29 10:45:37 -0700113 url = urllib.parse.urljoin('gopher://' + manifestUrl, url)
114 url = re.sub(r'^gopher://', '', url)
Anthony Kingcb07ba72015-03-28 23:26:04 +0000115 else:
116 url = urllib.parse.urljoin(manifestUrl, url)
Shawn Pearcea9f11b32013-01-02 15:40:48 -0800117 return url
Conley Owensceea3682011-10-20 10:45:47 -0700118
119 def ToRemoteSpec(self, projectName):
David Rileye0684ad2017-04-05 00:02:59 -0700120 fetchUrl = self.resolvedFetchUrl.rstrip('/')
121 url = fetchUrl + '/' + projectName
Yestin Sunb292b982012-07-02 07:32:50 -0700122 remoteName = self.name
Conley Owens1e7ab2a2013-10-08 17:26:57 -0700123 if self.remoteAlias:
David Pursehouse37128b62013-10-15 10:48:40 +0900124 remoteName = self.remoteAlias
Dan Willemsen96c2d652016-04-06 16:03:54 -0700125 return RemoteSpec(remoteName,
126 url=url,
Steve Raed6480452016-08-10 15:00:00 -0700127 pushUrl=self.pushUrl,
Dan Willemsen96c2d652016-04-06 16:03:54 -0700128 review=self.reviewUrl,
David Rileye0684ad2017-04-05 00:02:59 -0700129 orig_name=self.name,
130 fetchUrl=self.fetchUrl)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700131
David Pursehouse819827a2020-02-12 15:20:19 +0900132
Shawn O. Pearcec8a300f2009-05-18 13:19:57 -0700133class XmlManifest(object):
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700134 """manages the repo configuration file"""
135
136 def __init__(self, repodir):
137 self.repodir = os.path.abspath(repodir)
138 self.topdir = os.path.dirname(self.repodir)
139 self.manifestFile = os.path.join(self.repodir, MANIFEST_FILE_NAME)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700140 self.globalConfig = GitConfig.ForUser()
David Pursehouse4eb285c2013-02-14 16:28:44 +0900141 self.localManifestWarning = False
Simran Basib9a1b732015-08-20 12:19:28 -0700142 self.isGitcClient = False
Basil Gelloc7453502018-05-25 20:23:52 +0300143 self._load_local_manifests = True
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700144
145 self.repoProject = MetaProject(self, 'repo',
David Pursehouseabdf7502020-02-12 14:58:39 +0900146 gitdir=os.path.join(repodir, 'repo/.git'),
147 worktree=os.path.join(repodir, 'repo'))
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700148
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700149 self.manifestProject = MetaProject(self, 'manifests',
David Pursehouseabdf7502020-02-12 14:58:39 +0900150 gitdir=os.path.join(repodir, 'manifests.git'),
151 worktree=os.path.join(repodir, 'manifests'))
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700152
153 self._Unload()
154
Basil Gelloc7453502018-05-25 20:23:52 +0300155 def Override(self, name, load_local_manifests=True):
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -0700156 """Use a different manifest, just for the current instantiation.
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700157 """
Basil Gelloc7453502018-05-25 20:23:52 +0300158 path = None
159
160 # Look for a manifest by path in the filesystem (including the cwd).
161 if not load_local_manifests:
162 local_path = os.path.abspath(name)
163 if os.path.isfile(local_path):
164 path = local_path
165
166 # Look for manifests by name from the manifests repo.
167 if path is None:
168 path = os.path.join(self.manifestProject.worktree, name)
169 if not os.path.isfile(path):
170 raise ManifestParseError('manifest %s not found' % name)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700171
172 old = self.manifestFile
173 try:
Basil Gelloc7453502018-05-25 20:23:52 +0300174 self._load_local_manifests = load_local_manifests
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700175 self.manifestFile = path
176 self._Unload()
177 self._Load()
178 finally:
179 self.manifestFile = old
180
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -0700181 def Link(self, name):
182 """Update the repo metadata to use a different manifest.
183 """
184 self.Override(name)
185
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700186 try:
Sebastian Frias223bf962012-11-21 19:09:25 +0100187 if os.path.lexists(self.manifestFile):
Renaud Paquay010fed72016-11-11 14:25:29 -0800188 platform_utils.remove(self.manifestFile)
Renaud Paquayd5cec5e2016-11-01 11:24:03 -0700189 platform_utils.symlink(os.path.join('manifests', name), self.manifestFile)
Sebastian Frias223bf962012-11-21 19:09:25 +0100190 except OSError as e:
191 raise ManifestParseError('cannot link manifest %s: %s' % (name, str(e)))
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700192
Shawn O. Pearcec7a4eef2009-03-05 10:32:38 -0800193 def _RemoteToXml(self, r, doc, root):
194 e = doc.createElement('remote')
195 root.appendChild(e)
196 e.setAttribute('name', r.name)
197 e.setAttribute('fetch', r.fetchUrl)
Steve Raed6480452016-08-10 15:00:00 -0700198 if r.pushUrl is not None:
199 e.setAttribute('pushurl', r.pushUrl)
Conley Owens1e7ab2a2013-10-08 17:26:57 -0700200 if r.remoteAlias is not None:
201 e.setAttribute('alias', r.remoteAlias)
Shawn O. Pearcec7a4eef2009-03-05 10:32:38 -0800202 if r.reviewUrl is not None:
203 e.setAttribute('review', r.reviewUrl)
Anthony King36ea2fb2014-05-06 11:54:01 +0100204 if r.revision is not None:
205 e.setAttribute('revision', r.revision)
Shawn O. Pearcec7a4eef2009-03-05 10:32:38 -0800206
Josh Triplett884a3872014-06-12 14:57:29 -0700207 def _ParseGroups(self, groups):
208 return [x for x in re.split(r'[,\s]+', groups) if x]
209
Dan Willemsen5ea32d12015-09-08 13:27:20 -0700210 def Save(self, fd, peg_rev=False, peg_rev_upstream=True, groups=None):
Shawn O. Pearcec7a4eef2009-03-05 10:32:38 -0800211 """Write the current manifest out to the given file descriptor.
212 """
Colin Cross5acde752012-03-28 20:15:45 -0700213 mp = self.manifestProject
214
Dan Willemsen5ea32d12015-09-08 13:27:20 -0700215 if groups is None:
216 groups = mp.config.GetString('manifest.groups')
Matt Gumbel0c635bb2012-12-21 10:14:53 -0800217 if groups:
Josh Triplett884a3872014-06-12 14:57:29 -0700218 groups = self._ParseGroups(groups)
Colin Cross5acde752012-03-28 20:15:45 -0700219
Shawn O. Pearcec7a4eef2009-03-05 10:32:38 -0800220 doc = xml.dom.minidom.Document()
221 root = doc.createElement('manifest')
222 doc.appendChild(root)
223
Doug Anderson2b8db3c2010-11-01 15:08:06 -0700224 # Save out the notice. There's a little bit of work here to give it the
225 # right whitespace, which assumes that the notice is automatically indented
226 # by 4 by minidom.
227 if self.notice:
228 notice_element = root.appendChild(doc.createElement('notice'))
229 notice_lines = self.notice.splitlines()
David Pursehouse54a4e602020-02-12 14:31:05 +0900230 indented_notice = ('\n'.join(" " * 4 + line for line in notice_lines))[4:]
Doug Anderson2b8db3c2010-11-01 15:08:06 -0700231 notice_element.appendChild(doc.createTextNode(indented_notice))
232
Shawn O. Pearcec7a4eef2009-03-05 10:32:38 -0800233 d = self.default
Shawn O. Pearcec7a4eef2009-03-05 10:32:38 -0800234
Chirayu Desai217ea7d2013-03-01 19:14:38 +0530235 for r in sorted(self.remotes):
Shawn O. Pearcec7a4eef2009-03-05 10:32:38 -0800236 self._RemoteToXml(self.remotes[r], doc, root)
237 if self.remotes:
238 root.appendChild(doc.createTextNode(''))
239
240 have_default = False
241 e = doc.createElement('default')
242 if d.remote:
243 have_default = True
244 e.setAttribute('remote', d.remote.name)
Shawn O. Pearce3c8dea12009-05-29 18:38:17 -0700245 if d.revisionExpr:
Shawn O. Pearcec7a4eef2009-03-05 10:32:38 -0800246 have_default = True
Shawn O. Pearce3c8dea12009-05-29 18:38:17 -0700247 e.setAttribute('revision', d.revisionExpr)
Simon Ruggier7e59de22015-07-24 12:50:06 +0200248 if d.destBranchExpr:
249 have_default = True
250 e.setAttribute('dest-branch', d.destBranchExpr)
Nasser Grainawida403412018-05-04 12:53:29 -0600251 if d.upstreamExpr:
252 have_default = True
253 e.setAttribute('upstream', d.upstreamExpr)
Shawn O. Pearce6392c872011-09-22 17:44:31 -0700254 if d.sync_j > 1:
255 have_default = True
256 e.setAttribute('sync-j', '%d' % d.sync_j)
Anatol Pomazau79770d22012-04-20 14:41:59 -0700257 if d.sync_c:
258 have_default = True
259 e.setAttribute('sync-c', 'true')
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +0800260 if d.sync_s:
261 have_default = True
262 e.setAttribute('sync-s', 'true')
YOUNG HO CHAa32c92c2018-02-14 16:57:31 +0900263 if not d.sync_tags:
264 have_default = True
265 e.setAttribute('sync-tags', 'false')
Shawn O. Pearcec7a4eef2009-03-05 10:32:38 -0800266 if have_default:
267 root.appendChild(e)
268 root.appendChild(doc.createTextNode(''))
269
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -0700270 if self._manifest_server:
271 e = doc.createElement('manifest-server')
272 e.setAttribute('url', self._manifest_server)
273 root.appendChild(e)
274 root.appendChild(doc.createTextNode(''))
275
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +0800276 def output_projects(parent, parent_node, projects):
David James8d201162013-10-11 17:03:19 -0700277 for project_name in projects:
278 for project in self._projects[project_name]:
279 output_project(parent, parent_node, project)
Shawn O. Pearcec7a4eef2009-03-05 10:32:38 -0800280
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +0800281 def output_project(parent, parent_node, p):
Colin Cross5acde752012-03-28 20:15:45 -0700282 if not p.MatchesGroups(groups):
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +0800283 return
284
285 name = p.name
286 relpath = p.relpath
287 if parent:
288 name = self._UnjoinName(parent.name, name)
289 relpath = self._UnjoinRelpath(parent.relpath, relpath)
Colin Cross5acde752012-03-28 20:15:45 -0700290
Shawn O. Pearcec7a4eef2009-03-05 10:32:38 -0800291 e = doc.createElement('project')
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +0800292 parent_node.appendChild(e)
293 e.setAttribute('name', name)
294 if relpath != name:
295 e.setAttribute('path', relpath)
Conley Owensa17d7af2013-10-16 14:38:09 -0700296 remoteName = None
297 if d.remote:
Dan Willemsen96c2d652016-04-06 16:03:54 -0700298 remoteName = d.remote.name
299 if not d.remote or p.remote.orig_name != remoteName:
300 remoteName = p.remote.orig_name
Anthony King36ea2fb2014-05-06 11:54:01 +0100301 e.setAttribute('remote', remoteName)
Shawn O. Pearcec7a4eef2009-03-05 10:32:38 -0800302 if peg_rev:
303 if self.IsMirror:
Brian Harring14a66742012-09-28 20:21:57 -0700304 value = p.bare_git.rev_parse(p.revisionExpr + '^0')
Shawn O. Pearcec7a4eef2009-03-05 10:32:38 -0800305 else:
Brian Harring14a66742012-09-28 20:21:57 -0700306 value = p.work_git.rev_parse(HEAD + '^0')
307 e.setAttribute('revision', value)
Conley Owens551dfec2015-07-10 14:54:54 -0700308 if peg_rev_upstream:
309 if p.upstream:
310 e.setAttribute('upstream', p.upstream)
311 elif value != p.revisionExpr:
312 # Only save the origin if the origin is not a sha1, and the default
313 # isn't our value
314 e.setAttribute('upstream', p.revisionExpr)
Anthony King36ea2fb2014-05-06 11:54:01 +0100315 else:
Dan Willemsen96c2d652016-04-06 16:03:54 -0700316 revision = self.remotes[p.remote.orig_name].revision or d.revisionExpr
Anthony King36ea2fb2014-05-06 11:54:01 +0100317 if not revision or revision != p.revisionExpr:
318 e.setAttribute('revision', p.revisionExpr)
Nasser Grainawida403412018-05-04 12:53:29 -0600319 if (p.upstream and (p.upstream != p.revisionExpr or
320 p.upstream != d.upstreamExpr)):
Mani Chandel7a91d512014-07-24 16:27:08 +0530321 e.setAttribute('upstream', p.upstream)
Shawn O. Pearcec7a4eef2009-03-05 10:32:38 -0800322
Simon Ruggier7e59de22015-07-24 12:50:06 +0200323 if p.dest_branch and p.dest_branch != d.destBranchExpr:
324 e.setAttribute('dest-branch', p.dest_branch)
325
Shawn O. Pearcec7a4eef2009-03-05 10:32:38 -0800326 for c in p.copyfiles:
327 ce = doc.createElement('copyfile')
328 ce.setAttribute('src', c.src)
329 ce.setAttribute('dest', c.dest)
330 e.appendChild(ce)
331
Jeff Hamiltone0df2322014-04-21 17:10:59 -0500332 for l in p.linkfiles:
333 le = doc.createElement('linkfile')
334 le.setAttribute('src', l.src)
335 le.setAttribute('dest', l.dest)
336 e.appendChild(le)
337
Conley Owensbb1b5f52012-08-13 13:11:18 -0700338 default_groups = ['all', 'name:%s' % p.name, 'path:%s' % p.relpath]
Dmitry Fink17f85ea2012-08-06 14:52:29 -0700339 egroups = [g for g in p.groups if g not in default_groups]
Conley Owens971de8e2012-04-16 10:36:08 -0700340 if egroups:
341 e.setAttribute('groups', ','.join(egroups))
Colin Cross5acde752012-03-28 20:15:45 -0700342
James W. Mills24c13082012-04-12 15:04:13 -0500343 for a in p.annotations:
344 if a.keep == "true":
345 ae = doc.createElement('annotation')
346 ae.setAttribute('name', a.name)
347 ae.setAttribute('value', a.value)
348 e.appendChild(ae)
349
Anatol Pomazau79770d22012-04-20 14:41:59 -0700350 if p.sync_c:
351 e.setAttribute('sync-c', 'true')
352
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +0800353 if p.sync_s:
354 e.setAttribute('sync-s', 'true')
355
YOUNG HO CHAa32c92c2018-02-14 16:57:31 +0900356 if not p.sync_tags:
357 e.setAttribute('sync-tags', 'false')
358
Dan Willemsen88409222015-08-17 15:29:10 -0700359 if p.clone_depth:
360 e.setAttribute('clone-depth', str(p.clone_depth))
361
Simran Basib9a1b732015-08-20 12:19:28 -0700362 self._output_manifest_project_extras(p, e)
363
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +0800364 if p.subprojects:
David James8d201162013-10-11 17:03:19 -0700365 subprojects = set(subp.name for subp in p.subprojects)
366 output_projects(p, e, list(sorted(subprojects)))
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +0800367
David James8d201162013-10-11 17:03:19 -0700368 projects = set(p.name for p in self._paths.values() if not p.parent)
369 output_projects(None, root, list(sorted(projects)))
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +0800370
Doug Anderson37282b42011-03-04 11:54:18 -0800371 if self._repo_hooks_project:
372 root.appendChild(doc.createTextNode(''))
373 e = doc.createElement('repo-hooks')
374 e.setAttribute('in-project', self._repo_hooks_project.name)
375 e.setAttribute('enabled-list',
376 ' '.join(self._repo_hooks_project.enabled_repo_hooks))
377 root.appendChild(e)
378
Shawn O. Pearcec7a4eef2009-03-05 10:32:38 -0800379 doc.writexml(fd, '', ' ', '\n', 'UTF-8')
380
Simran Basib9a1b732015-08-20 12:19:28 -0700381 def _output_manifest_project_extras(self, p, e):
382 """Manifests can modify e if they support extra project attributes."""
383 pass
384
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700385 @property
David James8d201162013-10-11 17:03:19 -0700386 def paths(self):
387 self._Load()
388 return self._paths
389
390 @property
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700391 def projects(self):
392 self._Load()
Anthony Kingd58bfe52014-05-05 23:30:49 +0100393 return list(self._paths.values())
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700394
395 @property
396 def remotes(self):
397 self._Load()
398 return self._remotes
399
400 @property
401 def default(self):
402 self._Load()
403 return self._default
404
Shawn O. Pearcee284ad12008-11-04 07:37:10 -0800405 @property
Doug Anderson37282b42011-03-04 11:54:18 -0800406 def repo_hooks_project(self):
407 self._Load()
408 return self._repo_hooks_project
409
410 @property
Doug Anderson2b8db3c2010-11-01 15:08:06 -0700411 def notice(self):
412 self._Load()
413 return self._notice
414
415 @property
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -0700416 def manifest_server(self):
417 self._Load()
Shawn O. Pearce34fb20f2011-11-30 13:41:02 -0800418 return self._manifest_server
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -0700419
420 @property
Xin Li745be2e2019-06-03 11:24:30 -0700421 def CloneFilter(self):
422 if self.manifestProject.config.GetBoolean('repo.partialclone'):
423 return self.manifestProject.config.GetString('repo.clonefilter')
424 return None
425
426 @property
Shawn O. Pearcee284ad12008-11-04 07:37:10 -0800427 def IsMirror(self):
428 return self.manifestProject.config.GetBoolean('repo.mirror')
429
Julien Campergue335f5ef2013-10-16 11:02:35 +0200430 @property
431 def IsArchive(self):
432 return self.manifestProject.config.GetBoolean('repo.archive')
433
Martin Kellye4e94d22017-03-21 16:05:12 -0700434 @property
435 def HasSubmodules(self):
436 return self.manifestProject.config.GetBoolean('repo.submodules')
437
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700438 def _Unload(self):
439 self._loaded = False
440 self._projects = {}
David James8d201162013-10-11 17:03:19 -0700441 self._paths = {}
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700442 self._remotes = {}
443 self._default = None
Doug Anderson37282b42011-03-04 11:54:18 -0800444 self._repo_hooks_project = None
Doug Anderson2b8db3c2010-11-01 15:08:06 -0700445 self._notice = None
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700446 self.branch = None
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -0700447 self._manifest_server = None
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700448
449 def _Load(self):
450 if not self._loaded:
Shawn O. Pearce2450a292008-11-04 08:22:07 -0800451 m = self.manifestProject
452 b = m.GetBranch(m.CurrentBranch).merge
Shawn O. Pearce21c5c342009-06-25 16:47:30 -0700453 if b is not None and b.startswith(R_HEADS):
Shawn O. Pearce2450a292008-11-04 08:22:07 -0800454 b = b[len(R_HEADS):]
455 self.branch = b
456
Colin Cross23acdd32012-04-21 00:33:54 -0700457 nodes = []
Brian Harring475a47d2012-06-07 20:05:35 -0700458 nodes.append(self._ParseManifestXml(self.manifestFile,
459 self.manifestProject.worktree))
Shawn O. Pearce5cc66792008-10-23 16:19:27 -0700460
Basil Gelloc7453502018-05-25 20:23:52 +0300461 if self._load_local_manifests:
462 local = os.path.join(self.repodir, LOCAL_MANIFEST_NAME)
463 if os.path.exists(local):
464 if not self.localManifestWarning:
465 self.localManifestWarning = True
466 print('warning: %s is deprecated; put local manifests '
467 'in `%s` instead' % (LOCAL_MANIFEST_NAME,
David Pursehouseabdf7502020-02-12 14:58:39 +0900468 os.path.join(self.repodir, LOCAL_MANIFESTS_DIR_NAME)),
Basil Gelloc7453502018-05-25 20:23:52 +0300469 file=sys.stderr)
470 nodes.append(self._ParseManifestXml(local, self.repodir))
Colin Cross23acdd32012-04-21 00:33:54 -0700471
Basil Gelloc7453502018-05-25 20:23:52 +0300472 local_dir = os.path.abspath(os.path.join(self.repodir,
David Pursehouseabdf7502020-02-12 14:58:39 +0900473 LOCAL_MANIFESTS_DIR_NAME))
Basil Gelloc7453502018-05-25 20:23:52 +0300474 try:
475 for local_file in sorted(platform_utils.listdir(local_dir)):
476 if local_file.endswith('.xml'):
477 local = os.path.join(local_dir, local_file)
478 nodes.append(self._ParseManifestXml(local, self.repodir))
479 except OSError:
480 pass
David Pursehouse2d5a0df2012-11-13 02:50:36 +0900481
Joe Onorato26e24752013-01-11 12:35:53 -0800482 try:
483 self._ParseManifest(nodes)
484 except ManifestParseError as e:
485 # There was a problem parsing, unload ourselves in case they catch
486 # this error and try again later, we will show the correct error
487 self._Unload()
488 raise e
Shawn O. Pearce5cc66792008-10-23 16:19:27 -0700489
Shawn O. Pearcee284ad12008-11-04 07:37:10 -0800490 if self.IsMirror:
491 self._AddMetaProjectMirror(self.repoProject)
492 self._AddMetaProjectMirror(self.manifestProject)
493
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700494 self._loaded = True
495
Brian Harring475a47d2012-06-07 20:05:35 -0700496 def _ParseManifestXml(self, path, include_root):
David Pursehousef7fc8a92012-11-13 04:00:28 +0900497 try:
498 root = xml.dom.minidom.parse(path)
David Pursehouse2d5a0df2012-11-13 02:50:36 +0900499 except (OSError, xml.parsers.expat.ExpatError) as e:
David Pursehousef7fc8a92012-11-13 04:00:28 +0900500 raise ManifestParseError("error parsing manifest %s: %s" % (path, e))
501
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700502 if not root or not root.childNodes:
Brian Harring26448742011-04-28 05:04:41 -0700503 raise ManifestParseError("no root node in %s" % (path,))
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700504
Jooncheol Park34acdd22012-08-27 02:25:59 +0900505 for manifest in root.childNodes:
506 if manifest.nodeName == 'manifest':
507 break
508 else:
Brian Harring26448742011-04-28 05:04:41 -0700509 raise ManifestParseError("no <manifest> in %s" % (path,))
510
Colin Cross23acdd32012-04-21 00:33:54 -0700511 nodes = []
David Pursehouse65b0ba52018-06-24 16:21:51 +0900512 for node in manifest.childNodes:
David Pursehousec1b86a22012-11-14 11:36:51 +0900513 if node.nodeName == 'include':
514 name = self._reqatt(node, 'name')
515 fp = os.path.join(include_root, name)
516 if not os.path.isfile(fp):
Chirayu Desai217ea7d2013-03-01 19:14:38 +0530517 raise ManifestParseError("include %s doesn't exist or isn't a file"
David Pursehouseabdf7502020-02-12 14:58:39 +0900518 % (name,))
David Pursehousec1b86a22012-11-14 11:36:51 +0900519 try:
520 nodes.extend(self._ParseManifestXml(fp, include_root))
521 # should isolate this to the exact exception, but that's
522 # tricky. actual parsing implementation may vary.
523 except (KeyboardInterrupt, RuntimeError, SystemExit):
524 raise
525 except Exception as e:
526 raise ManifestParseError(
Mike Frysingerec558df2019-07-05 01:38:05 -0400527 "failed parsing included manifest %s: %s" % (name, e))
David Pursehousec1b86a22012-11-14 11:36:51 +0900528 else:
529 nodes.append(node)
Colin Cross23acdd32012-04-21 00:33:54 -0700530 return nodes
Brian Harring26448742011-04-28 05:04:41 -0700531
Colin Cross23acdd32012-04-21 00:33:54 -0700532 def _ParseManifest(self, node_list):
533 for node in itertools.chain(*node_list):
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700534 if node.nodeName == 'remote':
535 remote = self._ParseRemote(node)
David Pursehouse717ece92012-11-13 08:49:16 +0900536 if remote:
537 if remote.name in self._remotes:
538 if remote != self._remotes[remote.name]:
539 raise ManifestParseError(
540 'remote %s already exists with different attributes' %
541 (remote.name))
542 else:
543 self._remotes[remote.name] = remote
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700544
Colin Cross23acdd32012-04-21 00:33:54 -0700545 for node in itertools.chain(*node_list):
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700546 if node.nodeName == 'default':
Julien Campergue74879922013-10-09 14:38:46 +0200547 new_default = self._ParseDefault(node)
548 if self._default is None:
549 self._default = new_default
550 elif new_default != self._default:
David Pursehouse37128b62013-10-15 10:48:40 +0900551 raise ManifestParseError('duplicate default in %s' %
552 (self.manifestFile))
Julien Campergue74879922013-10-09 14:38:46 +0200553
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700554 if self._default is None:
555 self._default = _Default()
556
Colin Cross23acdd32012-04-21 00:33:54 -0700557 for node in itertools.chain(*node_list):
Doug Anderson2b8db3c2010-11-01 15:08:06 -0700558 if node.nodeName == 'notice':
559 if self._notice is not None:
Doug Anderson37282b42011-03-04 11:54:18 -0800560 raise ManifestParseError(
561 'duplicate notice in %s' %
562 (self.manifestFile))
Doug Anderson2b8db3c2010-11-01 15:08:06 -0700563 self._notice = self._ParseNotice(node)
564
Colin Cross23acdd32012-04-21 00:33:54 -0700565 for node in itertools.chain(*node_list):
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -0700566 if node.nodeName == 'manifest-server':
567 url = self._reqatt(node, 'url')
568 if self._manifest_server is not None:
David Pursehousec1b86a22012-11-14 11:36:51 +0900569 raise ManifestParseError(
570 'duplicate manifest-server in %s' %
571 (self.manifestFile))
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -0700572 self._manifest_server = url
573
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +0800574 def recursively_add_projects(project):
David James8d201162013-10-11 17:03:19 -0700575 projects = self._projects.setdefault(project.name, [])
576 if project.relpath is None:
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +0800577 raise ManifestParseError(
David James8d201162013-10-11 17:03:19 -0700578 'missing path for %s in %s' %
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +0800579 (project.name, self.manifestFile))
David James8d201162013-10-11 17:03:19 -0700580 if project.relpath in self._paths:
581 raise ManifestParseError(
582 'duplicate path %s in %s' %
583 (project.relpath, self.manifestFile))
584 self._paths[project.relpath] = project
585 projects.append(project)
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +0800586 for subproject in project.subprojects:
587 recursively_add_projects(subproject)
588
Colin Cross23acdd32012-04-21 00:33:54 -0700589 for node in itertools.chain(*node_list):
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700590 if node.nodeName == 'project':
591 project = self._ParseProject(node)
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +0800592 recursively_add_projects(project)
Josh Triplett884a3872014-06-12 14:57:29 -0700593 if node.nodeName == 'extend-project':
594 name = self._reqatt(node, 'name')
595
596 if name not in self._projects:
597 raise ManifestParseError('extend-project element specifies non-existent '
598 'project: %s' % name)
599
600 path = node.getAttribute('path')
601 groups = node.getAttribute('groups')
602 if groups:
603 groups = self._ParseGroups(groups)
Luis Hector Chavez7d525852018-03-15 09:54:08 -0700604 revision = node.getAttribute('revision')
Kyunam Jobd0aae92020-02-04 11:38:53 +0900605 remote = node.getAttribute('remote')
606 if remote:
607 remote = self._get_remote(node)
Josh Triplett884a3872014-06-12 14:57:29 -0700608
609 for p in self._projects[name]:
610 if path and p.relpath != path:
611 continue
612 if groups:
613 p.groups.extend(groups)
Luis Hector Chavez7d525852018-03-15 09:54:08 -0700614 if revision:
615 p.revisionExpr = revision
Kyunam Jobd0aae92020-02-04 11:38:53 +0900616 if remote:
617 p.remote = remote.ToRemoteSpec(name)
Doug Anderson37282b42011-03-04 11:54:18 -0800618 if node.nodeName == 'repo-hooks':
619 # Get the name of the project and the (space-separated) list of enabled.
620 repo_hooks_project = self._reqatt(node, 'in-project')
621 enabled_repo_hooks = self._reqatt(node, 'enabled-list').split()
622
623 # Only one project can be the hooks project
624 if self._repo_hooks_project is not None:
625 raise ManifestParseError(
626 'duplicate repo-hooks in %s' %
627 (self.manifestFile))
628
629 # Store a reference to the Project.
630 try:
David James8d201162013-10-11 17:03:19 -0700631 repo_hooks_projects = self._projects[repo_hooks_project]
Doug Anderson37282b42011-03-04 11:54:18 -0800632 except KeyError:
633 raise ManifestParseError(
634 'project %s not found for repo-hooks' %
635 (repo_hooks_project))
636
David James8d201162013-10-11 17:03:19 -0700637 if len(repo_hooks_projects) != 1:
638 raise ManifestParseError(
639 'internal error parsing repo-hooks in %s' %
640 (self.manifestFile))
641 self._repo_hooks_project = repo_hooks_projects[0]
642
Doug Anderson37282b42011-03-04 11:54:18 -0800643 # Store the enabled hooks in the Project object.
644 self._repo_hooks_project.enabled_repo_hooks = enabled_repo_hooks
Colin Cross23acdd32012-04-21 00:33:54 -0700645 if node.nodeName == 'remove-project':
646 name = self._reqatt(node, 'name')
David Jamesb8433df2014-01-30 10:11:17 -0800647
648 if name not in self._projects:
David Pursehousef9107482012-11-16 19:12:32 +0900649 raise ManifestParseError('remove-project element specifies non-existent '
650 'project: %s' % name)
Colin Cross23acdd32012-04-21 00:33:54 -0700651
David Jamesb8433df2014-01-30 10:11:17 -0800652 for p in self._projects[name]:
653 del self._paths[p.relpath]
654 del self._projects[name]
655
Colin Cross23acdd32012-04-21 00:33:54 -0700656 # If the manifest removes the hooks project, treat it as if it deleted
657 # the repo-hooks element too.
658 if self._repo_hooks_project and (self._repo_hooks_project.name == name):
659 self._repo_hooks_project = None
660
Shawn O. Pearcee284ad12008-11-04 07:37:10 -0800661 def _AddMetaProjectMirror(self, m):
662 name = None
663 m_url = m.GetRemote(m.remote.name).url
664 if m_url.endswith('/.git'):
Chirayu Desai217ea7d2013-03-01 19:14:38 +0530665 raise ManifestParseError('refusing to mirror %s' % m_url)
Shawn O. Pearcee284ad12008-11-04 07:37:10 -0800666
667 if self._default and self._default.remote:
Conley Owensceea3682011-10-20 10:45:47 -0700668 url = self._default.remote.resolvedFetchUrl
Shawn O. Pearcee284ad12008-11-04 07:37:10 -0800669 if not url.endswith('/'):
670 url += '/'
671 if m_url.startswith(url):
672 remote = self._default.remote
673 name = m_url[len(url):]
674
675 if name is None:
676 s = m_url.rindex('/') + 1
Conley Owensdb728cd2011-09-26 16:34:01 -0700677 manifestUrl = self.manifestProject.config.GetString('remote.origin.url')
Shawn O. Pearcef35b2d92012-08-02 11:46:22 -0700678 remote = _XmlRemote('origin', fetch=m_url[:s], manifestUrl=manifestUrl)
Shawn O. Pearcee284ad12008-11-04 07:37:10 -0800679 name = m_url[s:]
680
681 if name.endswith('.git'):
682 name = name[:-4]
683
684 if name not in self._projects:
685 m.PreSync()
686 gitdir = os.path.join(self.topdir, '%s.git' % name)
David Pursehousee5913ae2020-02-12 13:56:59 +0900687 project = Project(manifest=self,
688 name=name,
689 remote=remote.ToRemoteSpec(name),
690 gitdir=gitdir,
691 objdir=gitdir,
692 worktree=None,
693 relpath=name or None,
694 revisionExpr=m.revisionExpr,
695 revisionId=None)
David James8d201162013-10-11 17:03:19 -0700696 self._projects[project.name] = [project]
Kwanhong Leeccd218c2014-02-17 13:07:32 +0900697 self._paths[project.relpath] = project
Shawn O. Pearcee284ad12008-11-04 07:37:10 -0800698
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700699 def _ParseRemote(self, node):
700 """
701 reads a <remote> element from the manifest file
702 """
703 name = self._reqatt(node, 'name')
Yestin Sunb292b982012-07-02 07:32:50 -0700704 alias = node.getAttribute('alias')
705 if alias == '':
706 alias = None
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700707 fetch = self._reqatt(node, 'fetch')
Steve Raed6480452016-08-10 15:00:00 -0700708 pushUrl = node.getAttribute('pushurl')
709 if pushUrl == '':
710 pushUrl = None
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700711 review = node.getAttribute('review')
Shawn O. Pearceae6e0942008-11-06 10:25:35 -0800712 if review == '':
713 review = None
Anthony King36ea2fb2014-05-06 11:54:01 +0100714 revision = node.getAttribute('revision')
715 if revision == '':
716 revision = None
Conley Owensdb728cd2011-09-26 16:34:01 -0700717 manifestUrl = self.manifestProject.config.GetString('remote.origin.url')
Steve Raed6480452016-08-10 15:00:00 -0700718 return _XmlRemote(name, alias, fetch, pushUrl, manifestUrl, review, revision)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700719
720 def _ParseDefault(self, node):
721 """
722 reads a <default> element from the manifest file
723 """
724 d = _Default()
725 d.remote = self._get_remote(node)
Shawn O. Pearce3c8dea12009-05-29 18:38:17 -0700726 d.revisionExpr = node.getAttribute('revision')
727 if d.revisionExpr == '':
728 d.revisionExpr = None
Anatol Pomazau79770d22012-04-20 14:41:59 -0700729
Bryan Jacobsf609f912013-05-06 13:36:24 -0400730 d.destBranchExpr = node.getAttribute('dest-branch') or None
Nasser Grainawida403412018-05-04 12:53:29 -0600731 d.upstreamExpr = node.getAttribute('upstream') or None
Bryan Jacobsf609f912013-05-06 13:36:24 -0400732
Shawn O. Pearce6392c872011-09-22 17:44:31 -0700733 sync_j = node.getAttribute('sync-j')
734 if sync_j == '' or sync_j is None:
735 d.sync_j = 1
736 else:
737 d.sync_j = int(sync_j)
Anatol Pomazau79770d22012-04-20 14:41:59 -0700738
739 sync_c = node.getAttribute('sync-c')
740 if not sync_c:
741 d.sync_c = False
742 else:
743 d.sync_c = sync_c.lower() in ("yes", "true", "1")
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +0800744
745 sync_s = node.getAttribute('sync-s')
746 if not sync_s:
747 d.sync_s = False
748 else:
749 d.sync_s = sync_s.lower() in ("yes", "true", "1")
YOUNG HO CHAa32c92c2018-02-14 16:57:31 +0900750
751 sync_tags = node.getAttribute('sync-tags')
752 if not sync_tags:
753 d.sync_tags = True
754 else:
755 d.sync_tags = sync_tags.lower() in ("yes", "true", "1")
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700756 return d
757
Doug Anderson2b8db3c2010-11-01 15:08:06 -0700758 def _ParseNotice(self, node):
759 """
760 reads a <notice> element from the manifest file
761
762 The <notice> element is distinct from other tags in the XML in that the
763 data is conveyed between the start and end tag (it's not an empty-element
764 tag).
765
766 The white space (carriage returns, indentation) for the notice element is
767 relevant and is parsed in a way that is based on how python docstrings work.
768 In fact, the code is remarkably similar to here:
769 http://www.python.org/dev/peps/pep-0257/
770 """
771 # Get the data out of the node...
772 notice = node.childNodes[0].data
773
774 # Figure out minimum indentation, skipping the first line (the same line
775 # as the <notice> tag)...
Chirayu Desai217ea7d2013-03-01 19:14:38 +0530776 minIndent = sys.maxsize
Doug Anderson2b8db3c2010-11-01 15:08:06 -0700777 lines = notice.splitlines()
778 for line in lines[1:]:
779 lstrippedLine = line.lstrip()
780 if lstrippedLine:
781 indent = len(line) - len(lstrippedLine)
782 minIndent = min(indent, minIndent)
783
784 # Strip leading / trailing blank lines and also indentation.
785 cleanLines = [lines[0].strip()]
786 for line in lines[1:]:
787 cleanLines.append(line[minIndent:].rstrip())
788
789 # Clear completely blank lines from front and back...
790 while cleanLines and not cleanLines[0]:
791 del cleanLines[0]
792 while cleanLines and not cleanLines[-1]:
793 del cleanLines[-1]
794
795 return '\n'.join(cleanLines)
796
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +0800797 def _JoinName(self, parent_name, name):
798 return os.path.join(parent_name, name)
799
800 def _UnjoinName(self, parent_name, name):
801 return os.path.relpath(name, parent_name)
802
David Pursehousee5913ae2020-02-12 13:56:59 +0900803 def _ParseProject(self, node, parent=None, **extra_proj_attrs):
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700804 """
805 reads a <project> element from the manifest file
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -0700806 """
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700807 name = self._reqatt(node, 'name')
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +0800808 if parent:
809 name = self._JoinName(parent.name, name)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700810
811 remote = self._get_remote(node)
812 if remote is None:
813 remote = self._default.remote
814 if remote is None:
Chirayu Desai217ea7d2013-03-01 19:14:38 +0530815 raise ManifestParseError("no remote for project %s within %s" %
David Pursehouseabdf7502020-02-12 14:58:39 +0900816 (name, self.manifestFile))
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700817
Anthony King36ea2fb2014-05-06 11:54:01 +0100818 revisionExpr = node.getAttribute('revision') or remote.revision
Shawn O. Pearce3c8dea12009-05-29 18:38:17 -0700819 if not revisionExpr:
820 revisionExpr = self._default.revisionExpr
821 if not revisionExpr:
Chirayu Desai217ea7d2013-03-01 19:14:38 +0530822 raise ManifestParseError("no revision for project %s within %s" %
David Pursehouseabdf7502020-02-12 14:58:39 +0900823 (name, self.manifestFile))
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700824
825 path = node.getAttribute('path')
826 if not path:
827 path = name
828 if path.startswith('/'):
Chirayu Desai217ea7d2013-03-01 19:14:38 +0530829 raise ManifestParseError("project %s path cannot be absolute in %s" %
David Pursehouseabdf7502020-02-12 14:58:39 +0900830 (name, self.manifestFile))
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700831
Mike Pontillod3153822012-02-28 11:53:24 -0800832 rebase = node.getAttribute('rebase')
833 if not rebase:
834 rebase = True
835 else:
836 rebase = rebase.lower() in ("yes", "true", "1")
837
Anatol Pomazau79770d22012-04-20 14:41:59 -0700838 sync_c = node.getAttribute('sync-c')
839 if not sync_c:
840 sync_c = False
841 else:
842 sync_c = sync_c.lower() in ("yes", "true", "1")
843
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +0800844 sync_s = node.getAttribute('sync-s')
845 if not sync_s:
846 sync_s = self._default.sync_s
847 else:
848 sync_s = sync_s.lower() in ("yes", "true", "1")
849
YOUNG HO CHAa32c92c2018-02-14 16:57:31 +0900850 sync_tags = node.getAttribute('sync-tags')
851 if not sync_tags:
852 sync_tags = self._default.sync_tags
853 else:
854 sync_tags = sync_tags.lower() in ("yes", "true", "1")
855
David Pursehouseede7f122012-11-27 22:25:30 +0900856 clone_depth = node.getAttribute('clone-depth')
857 if clone_depth:
858 try:
859 clone_depth = int(clone_depth)
David Pursehouse54a4e602020-02-12 14:31:05 +0900860 if clone_depth <= 0:
David Pursehouseede7f122012-11-27 22:25:30 +0900861 raise ValueError()
862 except ValueError:
863 raise ManifestParseError('invalid clone-depth %s in %s' %
864 (clone_depth, self.manifestFile))
865
Bryan Jacobsf609f912013-05-06 13:36:24 -0400866 dest_branch = node.getAttribute('dest-branch') or self._default.destBranchExpr
867
Nasser Grainawida403412018-05-04 12:53:29 -0600868 upstream = node.getAttribute('upstream') or self._default.upstreamExpr
Brian Harring14a66742012-09-28 20:21:57 -0700869
Conley Owens971de8e2012-04-16 10:36:08 -0700870 groups = ''
871 if node.hasAttribute('groups'):
872 groups = node.getAttribute('groups')
Josh Triplett884a3872014-06-12 14:57:29 -0700873 groups = self._ParseGroups(groups)
Brian Harring7da13142012-06-15 02:24:20 -0700874
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +0800875 if parent is None:
David James8d201162013-10-11 17:03:19 -0700876 relpath, worktree, gitdir, objdir = self.GetProjectPaths(name, path)
Shawn O. Pearcecd81dd62012-10-26 12:18:00 -0700877 else:
David James8d201162013-10-11 17:03:19 -0700878 relpath, worktree, gitdir, objdir = \
879 self.GetSubprojectPaths(parent, name, path)
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +0800880
881 default_groups = ['all', 'name:%s' % name, 'path:%s' % relpath]
882 groups.extend(set(default_groups).difference(groups))
Shawn O. Pearcecd81dd62012-10-26 12:18:00 -0700883
Scott Fandb83b1b2013-02-28 09:34:14 +0800884 if self.IsMirror and node.hasAttribute('force-path'):
885 if node.getAttribute('force-path').lower() in ("yes", "true", "1"):
886 gitdir = os.path.join(self.topdir, '%s.git' % path)
887
David Pursehousee5913ae2020-02-12 13:56:59 +0900888 project = Project(manifest=self,
889 name=name,
890 remote=remote.ToRemoteSpec(name),
891 gitdir=gitdir,
892 objdir=objdir,
893 worktree=worktree,
894 relpath=relpath,
895 revisionExpr=revisionExpr,
896 revisionId=None,
897 rebase=rebase,
898 groups=groups,
899 sync_c=sync_c,
900 sync_s=sync_s,
901 sync_tags=sync_tags,
902 clone_depth=clone_depth,
903 upstream=upstream,
904 parent=parent,
905 dest_branch=dest_branch,
Simran Basib9a1b732015-08-20 12:19:28 -0700906 **extra_proj_attrs)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700907
908 for n in node.childNodes:
Shawn O. Pearce242b5262009-05-19 13:00:29 -0700909 if n.nodeName == 'copyfile':
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700910 self._ParseCopyFile(project, n)
Jeff Hamiltone0df2322014-04-21 17:10:59 -0500911 if n.nodeName == 'linkfile':
912 self._ParseLinkFile(project, n)
James W. Mills24c13082012-04-12 15:04:13 -0500913 if n.nodeName == 'annotation':
914 self._ParseAnnotation(project, n)
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +0800915 if n.nodeName == 'project':
David Pursehousee5913ae2020-02-12 13:56:59 +0900916 project.subprojects.append(self._ParseProject(n, parent=project))
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700917
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700918 return project
919
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +0800920 def GetProjectPaths(self, name, path):
921 relpath = path
922 if self.IsMirror:
923 worktree = None
924 gitdir = os.path.join(self.topdir, '%s.git' % name)
David James8d201162013-10-11 17:03:19 -0700925 objdir = gitdir
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +0800926 else:
927 worktree = os.path.join(self.topdir, path).replace('\\', '/')
928 gitdir = os.path.join(self.repodir, 'projects', '%s.git' % path)
David James8d201162013-10-11 17:03:19 -0700929 objdir = os.path.join(self.repodir, 'project-objects', '%s.git' % name)
930 return relpath, worktree, gitdir, objdir
931
932 def GetProjectsWithName(self, name):
933 return self._projects.get(name, [])
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +0800934
935 def GetSubprojectName(self, parent, submodule_path):
936 return os.path.join(parent.name, submodule_path)
937
938 def _JoinRelpath(self, parent_relpath, relpath):
939 return os.path.join(parent_relpath, relpath)
940
941 def _UnjoinRelpath(self, parent_relpath, relpath):
942 return os.path.relpath(relpath, parent_relpath)
943
David James8d201162013-10-11 17:03:19 -0700944 def GetSubprojectPaths(self, parent, name, path):
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +0800945 relpath = self._JoinRelpath(parent.relpath, path)
946 gitdir = os.path.join(parent.gitdir, 'subprojects', '%s.git' % path)
David James8d201162013-10-11 17:03:19 -0700947 objdir = os.path.join(parent.gitdir, 'subproject-objects', '%s.git' % name)
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +0800948 if self.IsMirror:
949 worktree = None
950 else:
951 worktree = os.path.join(parent.worktree, path).replace('\\', '/')
David James8d201162013-10-11 17:03:19 -0700952 return relpath, worktree, gitdir, objdir
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +0800953
Mike Frysinger04122b72019-07-31 23:32:58 -0400954 @staticmethod
955 def _CheckLocalPath(path, symlink=False):
956 """Verify |path| is reasonable for use in <copyfile> & <linkfile>."""
957 if '~' in path:
958 return '~ not allowed (due to 8.3 filenames on Windows filesystems)'
959
960 # Some filesystems (like Apple's HFS+) try to normalize Unicode codepoints
961 # which means there are alternative names for ".git". Reject paths with
962 # these in it as there shouldn't be any reasonable need for them here.
963 # The set of codepoints here was cribbed from jgit's implementation:
964 # https://eclipse.googlesource.com/jgit/jgit/+/9110037e3e9461ff4dac22fee84ef3694ed57648/org.eclipse.jgit/src/org/eclipse/jgit/lib/ObjectChecker.java#884
965 BAD_CODEPOINTS = {
966 u'\u200C', # ZERO WIDTH NON-JOINER
967 u'\u200D', # ZERO WIDTH JOINER
968 u'\u200E', # LEFT-TO-RIGHT MARK
969 u'\u200F', # RIGHT-TO-LEFT MARK
970 u'\u202A', # LEFT-TO-RIGHT EMBEDDING
971 u'\u202B', # RIGHT-TO-LEFT EMBEDDING
972 u'\u202C', # POP DIRECTIONAL FORMATTING
973 u'\u202D', # LEFT-TO-RIGHT OVERRIDE
974 u'\u202E', # RIGHT-TO-LEFT OVERRIDE
975 u'\u206A', # INHIBIT SYMMETRIC SWAPPING
976 u'\u206B', # ACTIVATE SYMMETRIC SWAPPING
977 u'\u206C', # INHIBIT ARABIC FORM SHAPING
978 u'\u206D', # ACTIVATE ARABIC FORM SHAPING
979 u'\u206E', # NATIONAL DIGIT SHAPES
980 u'\u206F', # NOMINAL DIGIT SHAPES
981 u'\uFEFF', # ZERO WIDTH NO-BREAK SPACE
982 }
983 if BAD_CODEPOINTS & set(path):
984 # This message is more expansive than reality, but should be fine.
985 return 'Unicode combining characters not allowed'
986
987 # Assume paths might be used on case-insensitive filesystems.
988 path = path.lower()
989
Mike Frysingerae625412020-02-10 17:10:03 -0500990 # Some people use src="." to create stable links to projects. Lets allow
991 # that but reject all other uses of "." to keep things simple.
992 parts = path.split(os.path.sep)
993 if parts != ['.']:
994 for part in set(parts):
995 if part in {'.', '..', '.git'} or part.startswith('.repo'):
996 return 'bad component: %s' % (part,)
Mike Frysinger04122b72019-07-31 23:32:58 -0400997
998 if not symlink and path.endswith(os.path.sep):
999 return 'dirs not allowed'
1000
1001 norm = os.path.normpath(path)
1002 if norm == '..' or norm.startswith('../') or norm.startswith(os.path.sep):
1003 return 'path cannot be outside'
1004
1005 @classmethod
1006 def _ValidateFilePaths(cls, element, src, dest):
1007 """Verify |src| & |dest| are reasonable for <copyfile> & <linkfile>.
1008
1009 We verify the path independent of any filesystem state as we won't have a
1010 checkout available to compare to. i.e. This is for parsing validation
1011 purposes only.
1012
1013 We'll do full/live sanity checking before we do the actual filesystem
1014 modifications in _CopyFile/_LinkFile/etc...
1015 """
1016 # |dest| is the file we write to or symlink we create.
1017 # It is relative to the top of the repo client checkout.
1018 msg = cls._CheckLocalPath(dest)
1019 if msg:
1020 raise ManifestInvalidPathError(
1021 '<%s> invalid "dest": %s: %s' % (element, dest, msg))
1022
1023 # |src| is the file we read from or path we point to for symlinks.
1024 # It is relative to the top of the git project checkout.
1025 msg = cls._CheckLocalPath(src, symlink=element == 'linkfile')
1026 if msg:
1027 raise ManifestInvalidPathError(
1028 '<%s> invalid "src": %s: %s' % (element, src, msg))
1029
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001030 def _ParseCopyFile(self, project, node):
1031 src = self._reqatt(node, 'src')
1032 dest = self._reqatt(node, 'dest')
Shawn O. Pearcee284ad12008-11-04 07:37:10 -08001033 if not self.IsMirror:
1034 # src is project relative;
Mike Frysinger04122b72019-07-31 23:32:58 -04001035 # dest is relative to the top of the tree.
1036 # We only validate paths if we actually plan to process them.
1037 self._ValidateFilePaths('copyfile', src, dest)
Mike Frysingere6a202f2019-08-02 15:57:57 -04001038 project.AddCopyFile(src, dest, self.topdir)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001039
Jeff Hamiltone0df2322014-04-21 17:10:59 -05001040 def _ParseLinkFile(self, project, node):
1041 src = self._reqatt(node, 'src')
1042 dest = self._reqatt(node, 'dest')
1043 if not self.IsMirror:
1044 # src is project relative;
Mike Frysinger04122b72019-07-31 23:32:58 -04001045 # dest is relative to the top of the tree.
1046 # We only validate paths if we actually plan to process them.
1047 self._ValidateFilePaths('linkfile', src, dest)
Mike Frysingere6a202f2019-08-02 15:57:57 -04001048 project.AddLinkFile(src, dest, self.topdir)
Jeff Hamiltone0df2322014-04-21 17:10:59 -05001049
James W. Mills24c13082012-04-12 15:04:13 -05001050 def _ParseAnnotation(self, project, node):
1051 name = self._reqatt(node, 'name')
1052 value = self._reqatt(node, 'value')
1053 try:
1054 keep = self._reqatt(node, 'keep').lower()
1055 except ManifestParseError:
1056 keep = "true"
1057 if keep != "true" and keep != "false":
Chirayu Desai217ea7d2013-03-01 19:14:38 +05301058 raise ManifestParseError('optional "keep" attribute must be '
David Pursehouseabdf7502020-02-12 14:58:39 +09001059 '"true" or "false"')
James W. Mills24c13082012-04-12 15:04:13 -05001060 project.AddAnnotation(name, value, keep)
1061
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001062 def _get_remote(self, node):
1063 name = node.getAttribute('remote')
1064 if not name:
1065 return None
1066
1067 v = self._remotes.get(name)
1068 if not v:
Chirayu Desai217ea7d2013-03-01 19:14:38 +05301069 raise ManifestParseError("remote %s not defined in %s" %
David Pursehouseabdf7502020-02-12 14:58:39 +09001070 (name, self.manifestFile))
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001071 return v
1072
1073 def _reqatt(self, node, attname):
1074 """
1075 reads a required attribute from the node.
1076 """
1077 v = node.getAttribute(attname)
1078 if not v:
Chirayu Desai217ea7d2013-03-01 19:14:38 +05301079 raise ManifestParseError("no %s in <%s> within %s" %
David Pursehouseabdf7502020-02-12 14:58:39 +09001080 (attname, node.nodeName, self.manifestFile))
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001081 return v
Julien Camperguedd654222014-01-09 16:21:37 +01001082
1083 def projectsDiff(self, manifest):
1084 """return the projects differences between two manifests.
1085
1086 The diff will be from self to given manifest.
1087
1088 """
1089 fromProjects = self.paths
1090 toProjects = manifest.paths
1091
Anthony King7446c592014-05-06 09:19:39 +01001092 fromKeys = sorted(fromProjects.keys())
1093 toKeys = sorted(toProjects.keys())
Julien Camperguedd654222014-01-09 16:21:37 +01001094
1095 diff = {'added': [], 'removed': [], 'changed': [], 'unreachable': []}
1096
1097 for proj in fromKeys:
David Pursehouseeeff3532020-02-12 11:24:10 +09001098 if proj not in toKeys:
Julien Camperguedd654222014-01-09 16:21:37 +01001099 diff['removed'].append(fromProjects[proj])
1100 else:
1101 fromProj = fromProjects[proj]
1102 toProj = toProjects[proj]
1103 try:
1104 fromRevId = fromProj.GetCommitRevisionId()
1105 toRevId = toProj.GetCommitRevisionId()
1106 except ManifestInvalidRevisionError:
1107 diff['unreachable'].append((fromProj, toProj))
1108 else:
1109 if fromRevId != toRevId:
1110 diff['changed'].append((fromProj, toProj))
1111 toKeys.remove(proj)
1112
1113 for proj in toKeys:
1114 diff['added'].append(toProjects[proj])
1115
1116 return diff
Simran Basib9a1b732015-08-20 12:19:28 -07001117
1118
1119class GitcManifest(XmlManifest):
1120
1121 def __init__(self, repodir, gitc_client_name):
1122 """Initialize the GitcManifest object."""
1123 super(GitcManifest, self).__init__(repodir)
1124 self.isGitcClient = True
1125 self.gitc_client_name = gitc_client_name
Simran Basi8ce50412015-08-28 14:25:44 -07001126 self.gitc_client_dir = os.path.join(gitc_utils.get_gitc_manifest_dir(),
Simran Basib9a1b732015-08-20 12:19:28 -07001127 gitc_client_name)
1128 self.manifestFile = os.path.join(self.gitc_client_dir, '.manifest')
1129
David Pursehousee5913ae2020-02-12 13:56:59 +09001130 def _ParseProject(self, node, parent=None):
Simran Basib9a1b732015-08-20 12:19:28 -07001131 """Override _ParseProject and add support for GITC specific attributes."""
1132 return super(GitcManifest, self)._ParseProject(
1133 node, parent=parent, old_revision=node.getAttribute('old-revision'))
1134
1135 def _output_manifest_project_extras(self, p, e):
1136 """Output GITC Specific Project attributes"""
1137 if p.old_revision:
Stefan Beller66851062016-06-17 16:40:08 -07001138 e.setAttribute('old-revision', str(p.old_revision))