blob: 2856806225135d4f5bd030c318e0ef48127210b3 [file] [log] [blame]
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001# Copyright (C) 2008 The Android Open Source Project
2#
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7# http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12# See the License for the specific language governing permissions and
13# limitations under the License.
14
jiajia tanga590e642021-04-25 20:02:02 +080015import errno
Mike Frysingerebf04a42021-02-23 20:48:04 -050016import functools
Mike Frysingeracf63b22019-06-13 02:24:21 -040017import http.cookiejar as cookielib
Mike Frysinger7b586f22021-02-23 18:38:39 -050018import io
Anthony King85b24ac2014-05-06 15:57:48 +010019import json
Mike Frysingerebf04a42021-02-23 20:48:04 -050020import multiprocessing
David Pursehouse86d973d2012-08-24 10:21:02 +090021import netrc
Shawn O. Pearce2a1ccb22009-04-10 16:51:53 -070022from optparse import SUPPRESS_HELP
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070023import os
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -070024import socket
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070025import sys
Dan Willemsen0745bb22015-08-17 13:41:45 -070026import tempfile
Shawn O. Pearcef6906872009-04-18 10:49:00 -070027import time
Mike Frysingeracf63b22019-06-13 02:24:21 -040028import urllib.error
29import urllib.parse
30import urllib.request
31import xmlrpc.client
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070032
Roy Lee18afd7f2010-05-09 04:32:08 +080033try:
34 import threading as _threading
35except ImportError:
36 import dummy_threading as _threading
37
Shawn O. Pearce97d2b2f2011-09-22 17:23:41 -070038try:
39 import resource
David Pursehouse819827a2020-02-12 15:20:19 +090040
Shawn O. Pearce97d2b2f2011-09-22 17:23:41 -070041 def _rlimit_nofile():
42 return resource.getrlimit(resource.RLIMIT_NOFILE)
43except ImportError:
44 def _rlimit_nofile():
45 return (256, 256)
46
David Rileye0684ad2017-04-05 00:02:59 -070047import event_log
Mike Frysinger347f9ed2021-03-15 14:58:52 -040048from git_command import git_require
David Pursehouseba7bc732015-08-20 16:55:42 +090049from git_config import GetUrlCookieFile
David Pursehoused94aaef2012-09-07 09:52:04 +090050from git_refs import R_HEADS, HEAD
Raman Tenneti6a872c92021-01-14 19:17:50 -080051import git_superproject
Simran Basibdb52712015-08-10 13:23:23 -070052import gitc_utils
Jaikumar Ganesh4f2517f2009-06-01 21:10:33 -070053from project import Project
54from project import RemoteSpec
Mike Frysingerd41eed02021-04-20 23:21:29 -040055from command import Command, MirrorSafeCommand, WORKER_BATCH_SIZE
Raman Tenneti1fd7bc22021-02-04 14:39:38 -080056from error import RepoChangedException, GitError, ManifestParseError
Renaud Paquaya65adf72016-11-03 10:37:53 -070057import platform_utils
Shawn O. Pearce350cde42009-04-16 11:21:18 -070058from project import SyncBuffer
Shawn O. Pearce68194f42009-04-10 16:48:52 -070059from progress import Progress
Mike Frysinger19e409c2021-05-05 19:44:35 -040060import ssh
Conley Owens094cdbe2014-01-30 15:09:59 -080061from wrapper import Wrapper
Dan Willemsen5ea32d12015-09-08 13:27:20 -070062from manifest_xml import GitcManifest
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070063
Dave Borowitz67700e92012-10-23 15:00:54 -070064_ONE_DAY_S = 24 * 60 * 60
65
David Pursehouse819827a2020-02-12 15:20:19 +090066
Shawn O. Pearcec95583b2009-03-03 17:47:06 -080067class Sync(Command, MirrorSafeCommand):
Roy Lee18afd7f2010-05-09 04:32:08 +080068 jobs = 1
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070069 common = True
70 helpSummary = "Update working tree to the latest revision"
71 helpUsage = """
72%prog [<project>...]
73"""
74 helpDescription = """
75The '%prog' command synchronizes local project directories
76with the remote repositories specified in the manifest. If a local
77project does not yet exist, it will clone a new local directory from
78the remote repository and set up tracking branches as specified in
79the manifest. If the local project already exists, '%prog'
80will update the remote branches and rebase any new local changes
81on top of the new remote changes.
82
83'%prog' will synchronize all projects listed at the command
84line. Projects can be specified either by name, or by a relative
85or absolute path to the project's local directory. If no projects
86are specified, '%prog' will synchronize all projects listed in
87the manifest.
Shawn O. Pearce3e768c92009-04-10 16:59:36 -070088
89The -d/--detach option can be used to switch specified projects
90back to the manifest revision. This option is especially helpful
91if the project is currently on a topic branch, but the manifest
92revision is temporarily needed.
Shawn O. Pearceeb7af872009-04-21 08:02:04 -070093
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -070094The -s/--smart-sync option can be used to sync to a known good
95build as specified by the manifest-server element in the current
Victor Boivie08c880d2011-04-19 10:32:52 +020096manifest. The -t/--smart-tag option is similar and allows you to
97specify a custom tag/label.
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -070098
David Pursehousecf76b1b2012-09-14 10:31:42 +090099The -u/--manifest-server-username and -p/--manifest-server-password
100options can be used to specify a username and password to authenticate
101with the manifest server when using the -s or -t option.
102
103If -u and -p are not specified when using the -s or -t option, '%prog'
104will attempt to read authentication credentials for the manifest server
105from the user's .netrc file.
106
107'%prog' will not use authentication credentials from -u/-p or .netrc
108if the manifest server specified in the manifest file already includes
109credentials.
110
Mike Frysingerd9e5cf02019-08-26 03:12:55 -0400111By default, all projects will be synced. The --fail-fast option can be used
Mike Frysinger7ae210a2020-05-24 14:56:52 -0400112to halt syncing as soon as possible when the first project fails to sync.
Andrei Warkentin5df6de02010-07-02 17:58:31 -0500113
Kevin Degiabaa7f32014-11-12 11:27:45 -0700114The --force-sync option can be used to overwrite existing git
115directories if they have previously been linked to a different
Roger Shimizuac29ac32020-06-06 02:33:40 +0900116object directory. WARNING: This may cause data to be lost since
Kevin Degiabaa7f32014-11-12 11:27:45 -0700117refs may be removed when overwriting.
118
Oleksii Okolielovd3c0f592018-12-17 19:23:44 -0500119The --force-remove-dirty option can be used to remove previously used
120projects with uncommitted changes. WARNING: This may cause data to be
121lost since uncommitted changes may be removed with projects that no longer
122exist in the manifest.
123
Shawn O. Pearcee02ac0a2012-03-14 15:36:59 -0700124The --no-clone-bundle option disables any attempt to use
125$URL/clone.bundle to bootstrap a new Git repository from a
126resumeable bundle file on a content delivery network. This
127may be necessary if there are problems with the local Python
128HTTP client or proxy configuration, but the Git binary works.
129
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +0800130The --fetch-submodules option enables fetching Git submodules
131of a project from server.
132
David Pursehousef2fad612015-01-29 14:36:28 +0900133The -c/--current-branch option can be used to only fetch objects that
134are on the branch specified by a project's revision.
135
David Pursehouseb1553542014-09-04 21:28:09 +0900136The --optimized-fetch option can be used to only fetch projects that
137are fixed to a sha1 revision if the sha1 revision does not already
138exist locally.
139
David Pursehouse74cfd272015-10-14 10:50:15 +0900140The --prune option can be used to remove any refs that no longer
141exist on the remote.
142
Mike Frysingerb8f7bb02018-10-10 01:05:11 -0400143# SSH Connections
Shawn O. Pearceeb7af872009-04-21 08:02:04 -0700144
145If at least one project remote URL uses an SSH connection (ssh://,
146git+ssh://, or user@host:path syntax) repo will automatically
147enable the SSH ControlMaster option when connecting to that host.
148This feature permits other projects in the same '%prog' session to
149reuse the same SSH tunnel, saving connection setup overheads.
150
151To disable this behavior on UNIX platforms, set the GIT_SSH
152environment variable to 'ssh'. For example:
153
154 export GIT_SSH=ssh
155 %prog
156
Mike Frysingerb8f7bb02018-10-10 01:05:11 -0400157# Compatibility
Shawn O. Pearceeb7af872009-04-21 08:02:04 -0700158
159This feature is automatically disabled on Windows, due to the lack
160of UNIX domain socket support.
161
162This feature is not compatible with url.insteadof rewrites in the
163user's ~/.gitconfig. '%prog' is currently not able to perform the
164rewrite early enough to establish the ControlMaster tunnel.
165
166If the remote SSH daemon is Gerrit Code Review, version 2.0.10 or
167later is required to fix a server side protocol bug.
168
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700169"""
Mike Frysinger6a2400a2021-02-16 01:43:31 -0500170 PARALLEL_JOBS = 1
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700171
Mike Frysinger9180a072021-04-13 14:57:40 -0400172 def _CommonOptions(self, p):
Mike Frysingerc177f942021-05-04 08:06:36 -0400173 if self.manifest:
174 try:
175 self.PARALLEL_JOBS = self.manifest.default.sync_j
176 except ManifestParseError:
177 pass
Mike Frysinger9180a072021-04-13 14:57:40 -0400178 super()._CommonOptions(p)
Shawn O. Pearce6392c872011-09-22 17:44:31 -0700179
Mike Frysinger9180a072021-04-13 14:57:40 -0400180 def _Options(self, p, show_smart=True):
Mike Frysinger49de8ef2021-04-09 00:21:02 -0400181 p.add_option('--jobs-network', default=None, type=int, metavar='JOBS',
182 help='number of network jobs to run in parallel (defaults to --jobs)')
183 p.add_option('--jobs-checkout', default=None, type=int, metavar='JOBS',
184 help='number of local checkout jobs to run in parallel (defaults to --jobs)')
185
Andrei Warkentin5df6de02010-07-02 17:58:31 -0500186 p.add_option('-f', '--force-broken',
Stefan Müller-Klieser46702ed2019-08-30 10:20:15 +0200187 dest='force_broken', action='store_true',
Mike Frysingerd9e5cf02019-08-26 03:12:55 -0400188 help='obsolete option (to be deleted in the future)')
189 p.add_option('--fail-fast',
190 dest='fail_fast', action='store_true',
191 help='stop syncing after first error is hit')
Kevin Degiabaa7f32014-11-12 11:27:45 -0700192 p.add_option('--force-sync',
193 dest='force_sync', action='store_true',
194 help="overwrite an existing git directory if it needs to "
195 "point to a different object directory. WARNING: this "
196 "may cause loss of data")
Oleksii Okolielovd3c0f592018-12-17 19:23:44 -0500197 p.add_option('--force-remove-dirty',
198 dest='force_remove_dirty', action='store_true',
199 help="force remove projects with uncommitted modifications if "
200 "projects no longer exist in the manifest. "
201 "WARNING: this may cause loss of data")
David Pursehouse8f62fb72012-11-14 12:09:38 +0900202 p.add_option('-l', '--local-only',
Shawn O. Pearceb1562fa2009-04-10 17:04:08 -0700203 dest='local_only', action='store_true',
204 help="only update working tree, don't fetch")
David Pursehouse54a4e602020-02-12 14:31:05 +0900205 p.add_option('--no-manifest-update', '--nmu',
Fredrik de Grootcc960972019-11-22 09:04:31 +0100206 dest='mp_update', action='store_false', default='true',
207 help='use the existing manifest checkout as-is. '
208 '(do not update to the latest revision)')
David Pursehouse8f62fb72012-11-14 12:09:38 +0900209 p.add_option('-n', '--network-only',
Shawn O. Pearce96fdcef2009-04-10 16:29:20 -0700210 dest='network_only', action='store_true',
211 help="fetch only, don't update working tree")
David Pursehouse8f62fb72012-11-14 12:09:38 +0900212 p.add_option('-d', '--detach',
Shawn O. Pearce3e768c92009-04-10 16:59:36 -0700213 dest='detach_head', action='store_true',
214 help='detach projects back to manifest revision')
David Pursehouse8f62fb72012-11-14 12:09:38 +0900215 p.add_option('-c', '--current-branch',
Anatol Pomazau53d6f4d2011-08-25 17:21:47 -0700216 dest='current_branch_only', action='store_true',
217 help='fetch only current branch from server')
Mike Frysinger73561142021-05-03 01:10:09 -0400218 p.add_option('--no-current-branch',
219 dest='current_branch_only', action='store_false',
220 help='fetch all branches from server')
Chris Wolfee9dc3b32012-01-26 11:36:18 -0500221 p.add_option('-m', '--manifest-name',
222 dest='manifest_name',
223 help='temporary manifest to use for this sync', metavar='NAME.xml')
Xin Lid79a4bc2020-05-20 16:03:45 -0700224 p.add_option('--clone-bundle', action='store_true',
225 help='enable use of /clone.bundle on HTTP/HTTPS')
226 p.add_option('--no-clone-bundle', dest='clone_bundle', action='store_false',
Shawn O. Pearcee02ac0a2012-03-14 15:36:59 -0700227 help='disable use of /clone.bundle on HTTP/HTTPS')
Conley Owens8d070cf2012-11-06 13:14:31 -0800228 p.add_option('-u', '--manifest-server-username', action='store',
229 dest='manifest_server_username',
230 help='username to authenticate with the manifest server')
231 p.add_option('-p', '--manifest-server-password', action='store',
232 dest='manifest_server_password',
233 help='password to authenticate with the manifest server')
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +0800234 p.add_option('--fetch-submodules',
235 dest='fetch_submodules', action='store_true',
236 help='fetch submodules from server')
Raman Tenneti6a872c92021-01-14 19:17:50 -0800237 p.add_option('--use-superproject', action='store_true',
238 help='use the manifest superproject to sync projects')
Raman Tenneti23ea7542021-05-07 14:01:54 -0700239 p.add_option('--no-use-superproject', action='store_false',
240 dest='use_superproject',
241 help='disable use of manifest superprojects')
Mike Frysingerd68ed632021-05-03 01:21:35 -0400242 p.add_option('--tags',
243 action='store_false',
244 help='fetch tags')
Mitchel Humpherys597868b2012-10-29 10:18:34 -0700245 p.add_option('--no-tags',
Mike Frysingerd68ed632021-05-03 01:21:35 -0400246 dest='tags', action='store_false',
Mitchel Humpherys597868b2012-10-29 10:18:34 -0700247 help="don't fetch tags")
David Pursehouseb1553542014-09-04 21:28:09 +0900248 p.add_option('--optimized-fetch',
249 dest='optimized_fetch', action='store_true',
250 help='only fetch projects fixed to sha1 if revision does not exist locally')
George Engelbrecht9bc283e2020-04-02 12:36:09 -0600251 p.add_option('--retry-fetches',
252 default=0, action='store', type='int',
253 help='number of times to retry fetches on transient errors')
David Pursehouse74cfd272015-10-14 10:50:15 +0900254 p.add_option('--prune', dest='prune', action='store_true',
255 help='delete refs that no longer exist on the remote')
Nico Sallembien6623b212010-05-11 12:57:01 -0700256 if show_smart:
257 p.add_option('-s', '--smart-sync',
258 dest='smart_sync', action='store_true',
David Pursehouse79fba682016-04-13 18:03:00 +0900259 help='smart sync using manifest from the latest known good build')
Victor Boivie08c880d2011-04-19 10:32:52 +0200260 p.add_option('-t', '--smart-tag',
261 dest='smart_tag', action='store',
262 help='smart sync using manifest from a known tag')
Shawn O. Pearce3e768c92009-04-10 16:59:36 -0700263
Shawn O. Pearcefd89b672009-04-18 11:28:57 -0700264 g = p.add_option_group('repo Version options')
265 g.add_option('--no-repo-verify',
Mike Frysingerc58ec4d2020-02-17 14:36:08 -0500266 dest='repo_verify', default=True, action='store_false',
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700267 help='do not verify repo source code')
Shawn O. Pearcefd89b672009-04-18 11:28:57 -0700268 g.add_option('--repo-upgraded',
Shawn O. Pearcec9ef7442008-11-03 10:32:09 -0800269 dest='repo_upgraded', action='store_true',
Shawn O. Pearce2a1ccb22009-04-10 16:51:53 -0700270 help=SUPPRESS_HELP)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700271
Raman Tenneti8d43dea2021-02-07 16:30:27 -0800272 def _GetBranch(self):
273 """Returns the branch name for getting the approved manifest."""
274 p = self.manifest.manifestProject
275 b = p.GetBranch(p.CurrentBranch)
276 branch = b.merge
277 if branch.startswith(R_HEADS):
278 branch = branch[len(R_HEADS):]
279 return branch
280
Raman Tenneti2ae44d72021-03-23 15:12:27 -0700281 def _UseSuperproject(self, opt):
282 """Returns True if use-superproject option is enabled"""
Raman Tenneti23ea7542021-05-07 14:01:54 -0700283 if opt.use_superproject is not None:
284 return opt.use_superproject
285 else:
286 return self.manifest.manifestProject.config.GetBoolean('repo.superproject')
Raman Tenneti2ae44d72021-03-23 15:12:27 -0700287
288 def _GetCurrentBranchOnly(self, opt):
289 """Returns True if current-branch or use-superproject options are enabled."""
290 return opt.current_branch_only or self._UseSuperproject(opt)
291
Raman Tennetifeb28912021-05-02 19:47:29 -0700292 def _UpdateProjectsRevisionId(self, opt, args, load_local_manifests):
Raman Tenneti1fd7bc22021-02-04 14:39:38 -0800293 """Update revisionId of every project with the SHA from superproject.
294
295 This function updates each project's revisionId with SHA from superproject.
296 It writes the updated manifest into a file and reloads the manifest from it.
297
298 Args:
299 opt: Program options returned from optparse. See _Options().
300 args: Arguments to pass to GetProjects. See the GetProjects
301 docstring for details.
Raman Tennetifeb28912021-05-02 19:47:29 -0700302 load_local_manifests: Whether to load local manifests.
Raman Tenneti1fd7bc22021-02-04 14:39:38 -0800303
304 Returns:
305 Returns path to the overriding manifest file.
306 """
Raman Tenneti21dce3d2021-02-09 00:26:31 -0800307 superproject = git_superproject.Superproject(self.manifest,
Raman Tennetief99ec02021-03-04 10:29:40 -0800308 self.repodir,
309 quiet=opt.quiet)
Raman Tenneti1fd7bc22021-02-04 14:39:38 -0800310 all_projects = self.GetProjects(args,
311 missing_ok=True,
312 submodules_ok=opt.fetch_submodules)
Raman Tenneti21dce3d2021-02-09 00:26:31 -0800313 manifest_path = superproject.UpdateProjectsRevisionId(all_projects)
Raman Tenneti1fd7bc22021-02-04 14:39:38 -0800314 if not manifest_path:
Raman Tennetif907ced2021-05-07 10:22:16 -0700315 print('error: Update of revsionId from superproject has failed. '
316 'Please resync with --no-use-superproject option',
Raman Tenneti1fd7bc22021-02-04 14:39:38 -0800317 file=sys.stderr)
318 sys.exit(1)
Raman Tennetifeb28912021-05-02 19:47:29 -0700319 self._ReloadManifest(manifest_path, load_local_manifests)
Raman Tenneti1fd7bc22021-02-04 14:39:38 -0800320 return manifest_path
321
Mike Frysingerb2fa30a2021-02-24 00:15:32 -0500322 def _FetchProjectList(self, opt, projects):
323 """Main function of the fetch worker.
324
325 The projects we're given share the same underlying git object store, so we
326 have to fetch them in serial.
Roy Lee18afd7f2010-05-09 04:32:08 +0800327
David James8d201162013-10-11 17:03:19 -0700328 Delegates most of the work to _FetchHelper.
329
330 Args:
331 opt: Program options returned from optparse. See _Options().
332 projects: Projects to fetch.
David James8d201162013-10-11 17:03:19 -0700333 """
Mike Frysingerb2fa30a2021-02-24 00:15:32 -0500334 return [self._FetchOne(opt, x) for x in projects]
David James8d201162013-10-11 17:03:19 -0700335
Mike Frysingerb2fa30a2021-02-24 00:15:32 -0500336 def _FetchOne(self, opt, project):
David James8d201162013-10-11 17:03:19 -0700337 """Fetch git objects for a single project.
338
David Pursehousec1b86a22012-11-14 11:36:51 +0900339 Args:
340 opt: Program options returned from optparse. See _Options().
341 project: Project object for the project to fetch.
David James8d201162013-10-11 17:03:19 -0700342
343 Returns:
344 Whether the fetch was successful.
David Pursehousec1b86a22012-11-14 11:36:51 +0900345 """
David Rileye0684ad2017-04-05 00:02:59 -0700346 start = time.time()
347 success = False
Mike Frysinger7b586f22021-02-23 18:38:39 -0500348 buf = io.StringIO()
David Pursehousec1b86a22012-11-14 11:36:51 +0900349 try:
Mike Frysingerb2fa30a2021-02-24 00:15:32 -0500350 success = project.Sync_NetworkHalf(
351 quiet=opt.quiet,
352 verbose=opt.verbose,
353 output_redir=buf,
354 current_branch_only=self._GetCurrentBranchOnly(opt),
355 force_sync=opt.force_sync,
356 clone_bundle=opt.clone_bundle,
357 tags=opt.tags, archive=self.manifest.IsArchive,
358 optimized_fetch=opt.optimized_fetch,
359 retry_fetches=opt.retry_fetches,
360 prune=opt.prune,
Mike Frysinger19e409c2021-05-05 19:44:35 -0400361 ssh_proxy=True,
Raman Tennetif32f2432021-04-12 20:57:25 -0700362 clone_filter=self.manifest.CloneFilter,
363 partial_clone_exclude=self.manifest.PartialCloneExclude)
Doug Andersonfc06ced2011-03-16 15:49:18 -0700364
Mike Frysingerb2fa30a2021-02-24 00:15:32 -0500365 output = buf.getvalue()
366 if opt.verbose and output:
367 print('\n' + output.rstrip())
Doug Andersonfc06ced2011-03-16 15:49:18 -0700368
Mike Frysingerb2fa30a2021-02-24 00:15:32 -0500369 if not success:
370 print('error: Cannot fetch %s from %s'
371 % (project.name, project.remote.url),
372 file=sys.stderr)
Raman Tennetiad8aa692021-04-15 09:20:51 -0700373 except GitError as e:
374 print('error.GitError: Cannot fetch %s' % str(e), file=sys.stderr)
Mike Frysingerb2fa30a2021-02-24 00:15:32 -0500375 except Exception as e:
376 print('error: Cannot fetch %s (%s: %s)'
377 % (project.name, type(e).__name__, str(e)), file=sys.stderr)
378 raise
Mike Frysinger7b586f22021-02-23 18:38:39 -0500379
Mike Frysingerb2fa30a2021-02-24 00:15:32 -0500380 finish = time.time()
381 return (success, project, start, finish)
David James8d201162013-10-11 17:03:19 -0700382
Mike Frysinger5a033082019-09-23 19:21:20 -0400383 def _Fetch(self, projects, opt, err_event):
Mike Frysingerb2fa30a2021-02-24 00:15:32 -0500384 ret = True
385
Mike Frysinger49de8ef2021-04-09 00:21:02 -0400386 jobs = opt.jobs_network if opt.jobs_network else self.jobs
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700387 fetched = set()
Mike Frysinger151701e2021-04-13 15:07:21 -0400388 pm = Progress('Fetching', len(projects), delay=False, quiet=opt.quiet)
Roy Lee18afd7f2010-05-09 04:32:08 +0800389
David James89ece422014-01-09 18:51:58 -0800390 objdir_project_map = dict()
391 for project in projects:
392 objdir_project_map.setdefault(project.objdir, []).append(project)
Mike Frysingerb2fa30a2021-02-24 00:15:32 -0500393 projects_list = list(objdir_project_map.values())
David James8d201162013-10-11 17:03:19 -0700394
Mike Frysingerb2fa30a2021-02-24 00:15:32 -0500395 def _ProcessResults(results_sets):
396 ret = True
397 for results in results_sets:
398 for (success, project, start, finish) in results:
399 self._fetch_times.Set(project, finish - start)
400 self.event_log.AddSync(project, event_log.TASK_SYNC_NETWORK,
401 start, finish, success)
402 # Check for any errors before running any more tasks.
403 # ...we'll let existing jobs finish, though.
404 if not success:
405 ret = False
406 else:
407 fetched.add(project.gitdir)
408 pm.update(msg=project.name)
409 if not ret and opt.fail_fast:
410 break
411 return ret
Doug Andersonfc06ced2011-03-16 15:49:18 -0700412
Mike Frysingerb2fa30a2021-02-24 00:15:32 -0500413 # NB: Multiprocessing is heavy, so don't spin it up for one job.
Mike Frysinger49de8ef2021-04-09 00:21:02 -0400414 if len(projects_list) == 1 or jobs == 1:
Mike Frysingerb2fa30a2021-02-24 00:15:32 -0500415 if not _ProcessResults(self._FetchProjectList(opt, x) for x in projects_list):
416 ret = False
417 else:
418 # Favor throughput over responsiveness when quiet. It seems that imap()
419 # will yield results in batches relative to chunksize, so even as the
420 # children finish a sync, we won't see the result until one child finishes
421 # ~chunksize jobs. When using a large --jobs with large chunksize, this
422 # can be jarring as there will be a large initial delay where repo looks
423 # like it isn't doing anything and sits at 0%, but then suddenly completes
424 # a lot of jobs all at once. Since this code is more network bound, we
425 # can accept a bit more CPU overhead with a smaller chunksize so that the
426 # user sees more immediate & continuous feedback.
427 if opt.quiet:
428 chunksize = WORKER_BATCH_SIZE
David James89ece422014-01-09 18:51:58 -0800429 else:
Mike Frysingerb2fa30a2021-02-24 00:15:32 -0500430 pm.update(inc=0, msg='warming up')
431 chunksize = 4
Mike Frysinger49de8ef2021-04-09 00:21:02 -0400432 with multiprocessing.Pool(jobs) as pool:
Mike Frysingerb2fa30a2021-02-24 00:15:32 -0500433 results = pool.imap_unordered(
434 functools.partial(self._FetchProjectList, opt),
435 projects_list,
436 chunksize=chunksize)
437 if not _ProcessResults(results):
438 ret = False
439 pool.close()
Roy Lee18afd7f2010-05-09 04:32:08 +0800440
Shawn O. Pearce68194f42009-04-10 16:48:52 -0700441 pm.end()
Dave Borowitz67700e92012-10-23 15:00:54 -0700442 self._fetch_times.Save()
Dave Borowitz18857212012-10-23 17:02:59 -0700443
Julien Campergue335f5ef2013-10-16 11:02:35 +0200444 if not self.manifest.IsArchive:
Mike Frysinger5a033082019-09-23 19:21:20 -0400445 self._GCProjects(projects, opt, err_event)
Julien Campergue335f5ef2013-10-16 11:02:35 +0200446
Mike Frysingerb2fa30a2021-02-24 00:15:32 -0500447 return (ret, fetched)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700448
Mike Frysingerb4429432021-05-05 20:03:26 -0400449 def _FetchMain(self, opt, args, all_projects, err_event, manifest_name,
450 load_local_manifests):
451 """The main network fetch loop.
452
453 Args:
454 opt: Program options returned from optparse. See _Options().
455 args: Command line args used to filter out projects.
456 all_projects: List of all projects that should be checked out.
457 err_event: Whether an error was hit while processing.
458 manifest_name: Manifest file to be reloaded.
459 load_local_manifests: Whether to load local manifests.
460 """
461 rp = self.manifest.repoProject
462
463 to_fetch = []
464 now = time.time()
465 if _ONE_DAY_S <= (now - rp.LastFetch):
466 to_fetch.append(rp)
467 to_fetch.extend(all_projects)
468 to_fetch.sort(key=self._fetch_times.Get, reverse=True)
469
470 success, fetched = self._Fetch(to_fetch, opt, err_event)
471 if not success:
472 err_event.set()
473
474 _PostRepoFetch(rp, opt.repo_verify)
475 if opt.network_only:
476 # bail out now; the rest touches the working tree
477 if err_event.is_set():
478 print('\nerror: Exited sync due to fetch errors.\n', file=sys.stderr)
479 sys.exit(1)
480 return
481
482 # Iteratively fetch missing and/or nested unregistered submodules
483 previously_missing_set = set()
484 while True:
485 self._ReloadManifest(manifest_name, load_local_manifests)
486 all_projects = self.GetProjects(args,
487 missing_ok=True,
488 submodules_ok=opt.fetch_submodules)
489 missing = []
490 for project in all_projects:
491 if project.gitdir not in fetched:
492 missing.append(project)
493 if not missing:
494 break
495 # Stop us from non-stopped fetching actually-missing repos: If set of
496 # missing repos has not been changed from last fetch, we break.
497 missing_set = set(p.name for p in missing)
498 if previously_missing_set == missing_set:
499 break
500 previously_missing_set = missing_set
501 success, new_fetched = self._Fetch(missing, opt, err_event)
502 if not success:
503 err_event.set()
504 fetched.update(new_fetched)
505
Mike Frysingerb5d075d2021-03-01 00:56:38 -0500506 def _CheckoutOne(self, detach_head, force_sync, project):
Xin Li745be2e2019-06-03 11:24:30 -0700507 """Checkout work tree for one project
508
509 Args:
Mike Frysingerb5d075d2021-03-01 00:56:38 -0500510 detach_head: Whether to leave a detached HEAD.
511 force_sync: Force checking out of the repo.
Xin Li745be2e2019-06-03 11:24:30 -0700512 project: Project object for the project to checkout.
Xin Li745be2e2019-06-03 11:24:30 -0700513
514 Returns:
515 Whether the fetch was successful.
516 """
Xin Li745be2e2019-06-03 11:24:30 -0700517 start = time.time()
518 syncbuf = SyncBuffer(self.manifest.manifestProject.config,
Mike Frysingerb5d075d2021-03-01 00:56:38 -0500519 detach_head=detach_head)
Xin Li745be2e2019-06-03 11:24:30 -0700520 success = False
521 try:
Mike Frysingerb5d075d2021-03-01 00:56:38 -0500522 project.Sync_LocalHalf(syncbuf, force_sync=force_sync)
Mike Frysingerebf04a42021-02-23 20:48:04 -0500523 success = syncbuf.Finish()
Raman Tennetiad8aa692021-04-15 09:20:51 -0700524 except GitError as e:
525 print('error.GitError: Cannot checkout %s: %s' %
526 (project.name, str(e)), file=sys.stderr)
Mike Frysingerebf04a42021-02-23 20:48:04 -0500527 except Exception as e:
528 print('error: Cannot checkout %s: %s: %s' %
529 (project.name, type(e).__name__, str(e)),
530 file=sys.stderr)
531 raise
Xin Li745be2e2019-06-03 11:24:30 -0700532
Mike Frysingerebf04a42021-02-23 20:48:04 -0500533 if not success:
534 print('error: Cannot checkout %s' % (project.name), file=sys.stderr)
535 finish = time.time()
536 return (success, project, start, finish)
Xin Li745be2e2019-06-03 11:24:30 -0700537
Mike Frysingerebf04a42021-02-23 20:48:04 -0500538 def _Checkout(self, all_projects, opt, err_results):
Xin Li745be2e2019-06-03 11:24:30 -0700539 """Checkout projects listed in all_projects
540
541 Args:
542 all_projects: List of all projects that should be checked out.
543 opt: Program options returned from optparse. See _Options().
Mike Frysingerebf04a42021-02-23 20:48:04 -0500544 err_results: A list of strings, paths to git repos where checkout failed.
Xin Li745be2e2019-06-03 11:24:30 -0700545 """
Mike Frysingerebf04a42021-02-23 20:48:04 -0500546 # Only checkout projects with worktrees.
547 all_projects = [x for x in all_projects if x.worktree]
Xin Li745be2e2019-06-03 11:24:30 -0700548
Mike Frysingerb5d075d2021-03-01 00:56:38 -0500549 def _ProcessResults(pool, pm, results):
550 ret = True
Mike Frysingerebf04a42021-02-23 20:48:04 -0500551 for (success, project, start, finish) in results:
552 self.event_log.AddSync(project, event_log.TASK_SYNC_LOCAL,
553 start, finish, success)
554 # Check for any errors before running any more tasks.
Mike Frysingerb2fa30a2021-02-24 00:15:32 -0500555 # ...we'll let existing jobs finish, though.
Mike Frysingerebf04a42021-02-23 20:48:04 -0500556 if not success:
Mike Frysingerb5d075d2021-03-01 00:56:38 -0500557 ret = False
Mike Frysingerebf04a42021-02-23 20:48:04 -0500558 err_results.append(project.relpath)
559 if opt.fail_fast:
Mike Frysingerb5d075d2021-03-01 00:56:38 -0500560 if pool:
561 pool.close()
562 return ret
Mike Frysingerebf04a42021-02-23 20:48:04 -0500563 pm.update(msg=project.name)
Mike Frysingerb5d075d2021-03-01 00:56:38 -0500564 return ret
Xin Li745be2e2019-06-03 11:24:30 -0700565
Mike Frysingerb5d075d2021-03-01 00:56:38 -0500566 return self.ExecuteInParallel(
567 opt.jobs_checkout if opt.jobs_checkout else self.jobs,
568 functools.partial(self._CheckoutOne, opt.detach_head, opt.force_sync),
569 all_projects,
570 callback=_ProcessResults,
571 output=Progress('Checking out', len(all_projects), quiet=opt.quiet)) and not err_results
Mike Frysingerebf04a42021-02-23 20:48:04 -0500572
Mike Frysinger5a033082019-09-23 19:21:20 -0400573 def _GCProjects(self, projects, opt, err_event):
Mike Frysinger151701e2021-04-13 15:07:21 -0400574 pm = Progress('Garbage collecting', len(projects), delay=False, quiet=opt.quiet)
Mike Frysinger65af2602021-04-08 22:47:44 -0400575 pm.update(inc=0, msg='prescan')
576
Gabe Black2ff30292014-10-09 17:54:35 -0700577 gc_gitdirs = {}
David James8d201162013-10-11 17:03:19 -0700578 for project in projects:
Mike Frysinger5f2b0452020-02-11 03:23:24 -0500579 # Make sure pruning never kicks in with shared projects.
Mike Frysinger979d5bd2020-02-09 02:28:34 -0500580 if (not project.use_git_worktrees and
David Pursehouseaa611a22020-02-20 10:47:26 +0900581 len(project.manifest.GetProjectsWithName(project.name)) > 1):
Anders Björklund2a2da802021-01-18 10:32:36 +0100582 if not opt.quiet:
Mike Frysinger65af2602021-04-08 22:47:44 -0400583 print('\r%s: Shared project %s found, disabling pruning.' %
Anders Björklund2a2da802021-01-18 10:32:36 +0100584 (project.relpath, project.name))
Mike Frysinger5f2b0452020-02-11 03:23:24 -0500585 if git_require((2, 7, 0)):
Mike Frysingerf81c72e2020-02-19 15:50:00 -0500586 project.EnableRepositoryExtension('preciousObjects')
Mike Frysinger5f2b0452020-02-11 03:23:24 -0500587 else:
588 # This isn't perfect, but it's the best we can do with old git.
Mike Frysinger65af2602021-04-08 22:47:44 -0400589 print('\r%s: WARNING: shared projects are unreliable when using old '
Mike Frysinger5f2b0452020-02-11 03:23:24 -0500590 'versions of git; please upgrade to git-2.7.0+.'
591 % (project.relpath,),
592 file=sys.stderr)
593 project.config.SetString('gc.pruneExpire', 'never')
Gabe Black2ff30292014-10-09 17:54:35 -0700594 gc_gitdirs[project.gitdir] = project.bare_git
David James8d201162013-10-11 17:03:19 -0700595
Mike Frysinger65af2602021-04-08 22:47:44 -0400596 pm.update(inc=len(projects) - len(gc_gitdirs), msg='warming up')
597
598 cpu_count = os.cpu_count()
Dave Borowitz18857212012-10-23 17:02:59 -0700599 jobs = min(self.jobs, cpu_count)
600
601 if jobs < 2:
Gabe Black2ff30292014-10-09 17:54:35 -0700602 for bare_git in gc_gitdirs.values():
Mike Frysinger65af2602021-04-08 22:47:44 -0400603 pm.update(msg=bare_git._project.name)
David James8d201162013-10-11 17:03:19 -0700604 bare_git.gc('--auto')
Mike Frysinger65af2602021-04-08 22:47:44 -0400605 pm.end()
Dave Borowitz18857212012-10-23 17:02:59 -0700606 return
607
Mike Frysinger0c0e9342019-06-13 12:42:39 -0400608 config = {'pack.threads': cpu_count // jobs if cpu_count > jobs else 1}
Dave Borowitz18857212012-10-23 17:02:59 -0700609
610 threads = set()
611 sem = _threading.Semaphore(jobs)
Dave Borowitz18857212012-10-23 17:02:59 -0700612
David James8d201162013-10-11 17:03:19 -0700613 def GC(bare_git):
Mike Frysinger65af2602021-04-08 22:47:44 -0400614 pm.start(bare_git._project.name)
Dave Borowitz18857212012-10-23 17:02:59 -0700615 try:
616 try:
David James8d201162013-10-11 17:03:19 -0700617 bare_git.gc('--auto', config=config)
Dave Borowitz18857212012-10-23 17:02:59 -0700618 except GitError:
619 err_event.set()
David Pursehouse145e35b2020-02-12 15:40:47 +0900620 except Exception:
Dave Borowitz18857212012-10-23 17:02:59 -0700621 err_event.set()
622 raise
623 finally:
Mike Frysinger65af2602021-04-08 22:47:44 -0400624 pm.finish(bare_git._project.name)
Dave Borowitz18857212012-10-23 17:02:59 -0700625 sem.release()
626
Gabe Black2ff30292014-10-09 17:54:35 -0700627 for bare_git in gc_gitdirs.values():
Mike Frysingerbe24a542021-02-23 03:24:12 -0500628 if err_event.is_set() and opt.fail_fast:
Dave Borowitz18857212012-10-23 17:02:59 -0700629 break
630 sem.acquire()
David James8d201162013-10-11 17:03:19 -0700631 t = _threading.Thread(target=GC, args=(bare_git,))
Dave Borowitz18857212012-10-23 17:02:59 -0700632 t.daemon = True
633 threads.add(t)
634 t.start()
635
636 for t in threads:
637 t.join()
Mike Frysinger65af2602021-04-08 22:47:44 -0400638 pm.end()
Dave Borowitz18857212012-10-23 17:02:59 -0700639
Raman Tennetifeb28912021-05-02 19:47:29 -0700640 def _ReloadManifest(self, manifest_name=None, load_local_manifests=True):
641 """Reload the manfiest from the file specified by the |manifest_name|.
642
643 It unloads the manifest if |manifest_name| is None.
644
645 Args:
646 manifest_name: Manifest file to be reloaded.
647 load_local_manifests: Whether to load local manifests.
648 """
Tim Kilbourn07669002013-03-08 15:02:49 -0800649 if manifest_name:
650 # Override calls _Unload already
Raman Tennetifeb28912021-05-02 19:47:29 -0700651 self.manifest.Override(manifest_name, load_local_manifests=load_local_manifests)
Tim Kilbourn07669002013-03-08 15:02:49 -0800652 else:
653 self.manifest._Unload()
654
Oleksii Okolielovd3c0f592018-12-17 19:23:44 -0500655 def UpdateProjectList(self, opt):
Jaikumar Ganesh4f2517f2009-06-01 21:10:33 -0700656 new_project_paths = []
Colin Cross5acde752012-03-28 20:15:45 -0700657 for project in self.GetProjects(None, missing_ok=True):
Shawn O. Pearce3a68bb42009-06-04 16:18:09 -0700658 if project.relpath:
659 new_project_paths.append(project.relpath)
Jaikumar Ganesh4f2517f2009-06-01 21:10:33 -0700660 file_name = 'project.list'
Mike Frysingere3315bb2021-02-09 23:45:28 -0500661 file_path = os.path.join(self.repodir, file_name)
Jaikumar Ganesh4f2517f2009-06-01 21:10:33 -0700662 old_project_paths = []
663
664 if os.path.exists(file_path):
Mike Frysinger3164d402019-11-11 05:40:22 -0500665 with open(file_path, 'r') as fd:
Jaikumar Ganesh4f2517f2009-06-01 21:10:33 -0700666 old_project_paths = fd.read().split('\n')
Kuang-che Wu0d9b16d2019-04-06 00:49:47 +0800667 # In reversed order, so subfolders are deleted before parent folder.
668 for path in sorted(old_project_paths, reverse=True):
Shawn O. Pearce3a68bb42009-06-04 16:18:09 -0700669 if not path:
670 continue
Jaikumar Ganesh4f2517f2009-06-01 21:10:33 -0700671 if path not in new_project_paths:
David Pursehouse8a68ff92012-09-24 12:15:13 +0900672 # If the path has already been deleted, we don't need to do it
Dan Willemsen43507912016-09-01 16:26:02 -0700673 gitdir = os.path.join(self.manifest.topdir, path, '.git')
674 if os.path.exists(gitdir):
David Pursehousec1b86a22012-11-14 11:36:51 +0900675 project = Project(
David Pursehouseabdf7502020-02-12 14:58:39 +0900676 manifest=self.manifest,
677 name=path,
678 remote=RemoteSpec('origin'),
679 gitdir=gitdir,
680 objdir=gitdir,
Mike Frysingerc0d18662020-02-19 19:19:18 -0500681 use_git_worktrees=os.path.isfile(gitdir),
David Pursehouseabdf7502020-02-12 14:58:39 +0900682 worktree=os.path.join(self.manifest.topdir, path),
683 relpath=path,
684 revisionExpr='HEAD',
685 revisionId=None,
686 groups=None)
Mike Frysingerc0d18662020-02-19 19:19:18 -0500687 if not project.DeleteWorktree(
David Pursehouseaa611a22020-02-20 10:47:26 +0900688 quiet=opt.quiet,
689 force=opt.force_remove_dirty):
Mike Frysingera850ca22019-08-07 17:19:24 -0400690 return 1
Jaikumar Ganesh4f2517f2009-06-01 21:10:33 -0700691
Shawn O. Pearce9fb29ce2009-06-04 20:41:02 -0700692 new_project_paths.sort()
Mike Frysinger3164d402019-11-11 05:40:22 -0500693 with open(file_path, 'w') as fd:
Jaikumar Ganesh4f2517f2009-06-01 21:10:33 -0700694 fd.write('\n'.join(new_project_paths))
Shawn O. Pearce3a68bb42009-06-04 16:18:09 -0700695 fd.write('\n')
Jaikumar Ganesh4f2517f2009-06-01 21:10:33 -0700696 return 0
697
jiajia tanga590e642021-04-25 20:02:02 +0800698 def UpdateCopyLinkfileList(self):
699 """Save all dests of copyfile and linkfile, and update them if needed.
700
701 Returns:
702 Whether update was successful.
703 """
704 new_paths = {}
705 new_linkfile_paths = []
706 new_copyfile_paths = []
707 for project in self.GetProjects(None, missing_ok=True):
708 new_linkfile_paths.extend(x.dest for x in project.linkfiles)
709 new_copyfile_paths.extend(x.dest for x in project.copyfiles)
710
711 new_paths = {
712 'linkfile': new_linkfile_paths,
713 'copyfile': new_copyfile_paths,
714 }
715
716 copylinkfile_name = 'copy-link-files.json'
717 copylinkfile_path = os.path.join(self.manifest.repodir, copylinkfile_name)
718 old_copylinkfile_paths = {}
719
720 if os.path.exists(copylinkfile_path):
721 with open(copylinkfile_path, 'rb') as fp:
722 try:
723 old_copylinkfile_paths = json.load(fp)
724 except:
725 print('error: %s is not a json formatted file.' %
726 copylinkfile_path, file=sys.stderr)
727 platform_utils.remove(copylinkfile_path)
728 return False
729
730 need_remove_files = []
731 need_remove_files.extend(
732 set(old_copylinkfile_paths.get('linkfile', [])) -
733 set(new_linkfile_paths))
734 need_remove_files.extend(
735 set(old_copylinkfile_paths.get('copyfile', [])) -
736 set(new_copyfile_paths))
737
738 for need_remove_file in need_remove_files:
739 try:
740 platform_utils.remove(need_remove_file)
741 except OSError as e:
742 if e.errno == errno.ENOENT:
743 # Try to remove the updated copyfile or linkfile.
744 # So, if the file is not exist, nothing need to do.
745 pass
746
747 # Create copy-link-files.json, save dest path of "copyfile" and "linkfile".
748 with open(copylinkfile_path, 'w', encoding='utf-8') as fp:
749 json.dump(new_paths, fp)
750 return True
751
Mike Frysinger01d6c3c2019-08-27 01:56:43 -0400752 def _SmartSyncSetup(self, opt, smart_sync_manifest_path):
753 if not self.manifest.manifest_server:
754 print('error: cannot smart sync: no manifest server defined in '
755 'manifest', file=sys.stderr)
756 sys.exit(1)
757
758 manifest_server = self.manifest.manifest_server
759 if not opt.quiet:
760 print('Using manifest server %s' % manifest_server)
761
David Pursehouseeeff3532020-02-12 11:24:10 +0900762 if '@' not in manifest_server:
Mike Frysinger01d6c3c2019-08-27 01:56:43 -0400763 username = None
764 password = None
765 if opt.manifest_server_username and opt.manifest_server_password:
766 username = opt.manifest_server_username
767 password = opt.manifest_server_password
768 else:
769 try:
770 info = netrc.netrc()
771 except IOError:
772 # .netrc file does not exist or could not be opened
773 pass
774 else:
775 try:
776 parse_result = urllib.parse.urlparse(manifest_server)
777 if parse_result.hostname:
778 auth = info.authenticators(parse_result.hostname)
779 if auth:
780 username, _account, password = auth
781 else:
782 print('No credentials found for %s in .netrc'
783 % parse_result.hostname, file=sys.stderr)
784 except netrc.NetrcParseError as e:
785 print('Error parsing .netrc file: %s' % e, file=sys.stderr)
786
787 if (username and password):
788 manifest_server = manifest_server.replace('://', '://%s:%s@' %
789 (username, password),
790 1)
791
792 transport = PersistentTransport(manifest_server)
793 if manifest_server.startswith('persistent-'):
794 manifest_server = manifest_server[len('persistent-'):]
795
796 try:
797 server = xmlrpc.client.Server(manifest_server, transport=transport)
798 if opt.smart_sync:
Raman Tenneti8d43dea2021-02-07 16:30:27 -0800799 branch = self._GetBranch()
Mike Frysinger01d6c3c2019-08-27 01:56:43 -0400800
Mike Frysinger56ce3462019-12-04 19:30:48 -0500801 if 'SYNC_TARGET' in os.environ:
Mike Frysingere20da3e2020-03-07 01:53:53 -0500802 target = os.environ['SYNC_TARGET']
Mike Frysinger01d6c3c2019-08-27 01:56:43 -0400803 [success, manifest_str] = server.GetApprovedManifest(branch, target)
Mike Frysinger56ce3462019-12-04 19:30:48 -0500804 elif ('TARGET_PRODUCT' in os.environ and
805 'TARGET_BUILD_VARIANT' in os.environ):
Mike Frysingere20da3e2020-03-07 01:53:53 -0500806 target = '%s-%s' % (os.environ['TARGET_PRODUCT'],
807 os.environ['TARGET_BUILD_VARIANT'])
Mike Frysinger01d6c3c2019-08-27 01:56:43 -0400808 [success, manifest_str] = server.GetApprovedManifest(branch, target)
809 else:
810 [success, manifest_str] = server.GetApprovedManifest(branch)
811 else:
812 assert(opt.smart_tag)
813 [success, manifest_str] = server.GetManifest(opt.smart_tag)
814
815 if success:
816 manifest_name = os.path.basename(smart_sync_manifest_path)
817 try:
Mike Frysinger3164d402019-11-11 05:40:22 -0500818 with open(smart_sync_manifest_path, 'w') as f:
Mike Frysinger01d6c3c2019-08-27 01:56:43 -0400819 f.write(manifest_str)
Mike Frysinger01d6c3c2019-08-27 01:56:43 -0400820 except IOError as e:
821 print('error: cannot write manifest to %s:\n%s'
822 % (smart_sync_manifest_path, e),
823 file=sys.stderr)
824 sys.exit(1)
825 self._ReloadManifest(manifest_name)
826 else:
827 print('error: manifest server RPC call failed: %s' %
828 manifest_str, file=sys.stderr)
829 sys.exit(1)
830 except (socket.error, IOError, xmlrpc.client.Fault) as e:
831 print('error: cannot connect to manifest server %s:\n%s'
832 % (self.manifest.manifest_server, e), file=sys.stderr)
833 sys.exit(1)
834 except xmlrpc.client.ProtocolError as e:
835 print('error: cannot connect to manifest server %s:\n%d %s'
836 % (self.manifest.manifest_server, e.errcode, e.errmsg),
837 file=sys.stderr)
838 sys.exit(1)
839
840 return manifest_name
841
Mike Frysingerfb527e32019-08-27 02:34:32 -0400842 def _UpdateManifestProject(self, opt, mp, manifest_name):
843 """Fetch & update the local manifest project."""
844 if not opt.local_only:
845 start = time.time()
Mike Frysinger521d01b2020-02-17 01:51:49 -0500846 success = mp.Sync_NetworkHalf(quiet=opt.quiet, verbose=opt.verbose,
Raman Tenneti2ae44d72021-03-23 15:12:27 -0700847 current_branch_only=self._GetCurrentBranchOnly(opt),
Erwan Yvindc5c4d12019-06-18 13:49:12 +0200848 force_sync=opt.force_sync,
Mike Frysingerc58ec4d2020-02-17 14:36:08 -0500849 tags=opt.tags,
Mike Frysingerfb527e32019-08-27 02:34:32 -0400850 optimized_fetch=opt.optimized_fetch,
George Engelbrecht9bc283e2020-04-02 12:36:09 -0600851 retry_fetches=opt.retry_fetches,
Mike Frysingerfb527e32019-08-27 02:34:32 -0400852 submodules=self.manifest.HasSubmodules,
Raman Tennetif32f2432021-04-12 20:57:25 -0700853 clone_filter=self.manifest.CloneFilter,
854 partial_clone_exclude=self.manifest.PartialCloneExclude)
Mike Frysingerfb527e32019-08-27 02:34:32 -0400855 finish = time.time()
856 self.event_log.AddSync(mp, event_log.TASK_SYNC_NETWORK,
857 start, finish, success)
858
859 if mp.HasChanges:
860 syncbuf = SyncBuffer(mp.config)
861 start = time.time()
862 mp.Sync_LocalHalf(syncbuf, submodules=self.manifest.HasSubmodules)
863 clean = syncbuf.Finish()
864 self.event_log.AddSync(mp, event_log.TASK_SYNC_LOCAL,
865 start, time.time(), clean)
866 if not clean:
867 sys.exit(1)
Mike Frysinger05638bf2021-05-04 15:33:31 -0400868 self._ReloadManifest(manifest_name)
Mike Frysingerfb527e32019-08-27 02:34:32 -0400869 if opt.jobs is None:
870 self.jobs = self.manifest.default.sync_j
871
Mike Frysingerae6cb082019-08-27 01:10:59 -0400872 def ValidateOptions(self, opt, args):
873 if opt.force_broken:
874 print('warning: -f/--force-broken is now the default behavior, and the '
875 'options are deprecated', file=sys.stderr)
876 if opt.network_only and opt.detach_head:
877 self.OptionParser.error('cannot combine -n and -d')
878 if opt.network_only and opt.local_only:
879 self.OptionParser.error('cannot combine -n and -l')
880 if opt.manifest_name and opt.smart_sync:
881 self.OptionParser.error('cannot combine -m and -s')
882 if opt.manifest_name and opt.smart_tag:
883 self.OptionParser.error('cannot combine -m and -t')
884 if opt.manifest_server_username or opt.manifest_server_password:
885 if not (opt.smart_sync or opt.smart_tag):
886 self.OptionParser.error('-u and -p may only be combined with -s or -t')
887 if None in [opt.manifest_server_username, opt.manifest_server_password]:
888 self.OptionParser.error('both -u and -p must be given')
889
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700890 def Execute(self, opt, args):
Roy Lee18afd7f2010-05-09 04:32:08 +0800891 if opt.jobs:
892 self.jobs = opt.jobs
Shawn O. Pearce97d2b2f2011-09-22 17:23:41 -0700893 if self.jobs > 1:
894 soft_limit, _ = _rlimit_nofile()
Mike Frysinger0c0e9342019-06-13 12:42:39 -0400895 self.jobs = min(self.jobs, (soft_limit - 5) // 3)
Shawn O. Pearce97d2b2f2011-09-22 17:23:41 -0700896
Chris Wolfee9dc3b32012-01-26 11:36:18 -0500897 if opt.manifest_name:
898 self.manifest.Override(opt.manifest_name)
Shawn O. Pearce3e768c92009-04-10 16:59:36 -0700899
Chirayu Desaia892b102013-06-11 14:18:46 +0530900 manifest_name = opt.manifest_name
David Pursehouse59b41742015-05-07 14:36:09 +0900901 smart_sync_manifest_path = os.path.join(
David Pursehouseabdf7502020-02-12 14:58:39 +0900902 self.manifest.manifestProject.worktree, 'smart_sync_override.xml')
Chirayu Desaia892b102013-06-11 14:18:46 +0530903
Xin Lid79a4bc2020-05-20 16:03:45 -0700904 if opt.clone_bundle is None:
905 opt.clone_bundle = self.manifest.CloneBundle
906
Victor Boivie08c880d2011-04-19 10:32:52 +0200907 if opt.smart_sync or opt.smart_tag:
Mike Frysinger01d6c3c2019-08-27 01:56:43 -0400908 manifest_name = self._SmartSyncSetup(opt, smart_sync_manifest_path)
909 else:
David Pursehouse59b41742015-05-07 14:36:09 +0900910 if os.path.isfile(smart_sync_manifest_path):
911 try:
Renaud Paquay010fed72016-11-11 14:25:29 -0800912 platform_utils.remove(smart_sync_manifest_path)
David Pursehouse59b41742015-05-07 14:36:09 +0900913 except OSError as e:
914 print('error: failed to remove existing smart sync override manifest: %s' %
915 e, file=sys.stderr)
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -0700916
Mike Frysingerc99322a2021-05-04 15:32:43 -0400917 err_event = multiprocessing.Event()
Mike Frysinger5a033082019-09-23 19:21:20 -0400918
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700919 rp = self.manifest.repoProject
920 rp.PreSync()
Mike Frysinger23c900f2020-02-29 02:52:44 -0500921 cb = rp.CurrentBranch
922 if cb:
923 base = rp.GetBranch(cb).merge
924 if not base or not base.startswith('refs/heads/'):
925 print('warning: repo is not tracking a remote branch, so it will not '
Mike Frysinger58ac1672020-03-14 14:35:26 -0400926 'receive updates; run `repo init --repo-rev=stable` to fix.',
Mike Frysinger23c900f2020-02-29 02:52:44 -0500927 file=sys.stderr)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700928
929 mp = self.manifest.manifestProject
930 mp.PreSync()
931
Shawn O. Pearcec9ef7442008-11-03 10:32:09 -0800932 if opt.repo_upgraded:
Shawn O. Pearce80d2ceb2012-10-26 12:23:05 -0700933 _PostRepoUpgrade(self.manifest, quiet=opt.quiet)
Shawn O. Pearcec9ef7442008-11-03 10:32:09 -0800934
Fredrik de Grootcc960972019-11-22 09:04:31 +0100935 if not opt.mp_update:
936 print('Skipping update of local manifest project.')
937 else:
938 self._UpdateManifestProject(opt, mp, manifest_name)
Simran Basib9a1b732015-08-20 12:19:28 -0700939
Raman Tennetifeb28912021-05-02 19:47:29 -0700940 load_local_manifests = not self.manifest.HasLocalManifests
Raman Tenneti2ae44d72021-03-23 15:12:27 -0700941 if self._UseSuperproject(opt):
Raman Tennetifeb28912021-05-02 19:47:29 -0700942 manifest_name = self._UpdateProjectsRevisionId(opt, args, load_local_manifests)
Raman Tenneti1fd7bc22021-02-04 14:39:38 -0800943
Simran Basib9a1b732015-08-20 12:19:28 -0700944 if self.gitc_manifest:
945 gitc_manifest_projects = self.GetProjects(args,
Simran Basib9a1b732015-08-20 12:19:28 -0700946 missing_ok=True)
947 gitc_projects = []
948 opened_projects = []
949 for project in gitc_manifest_projects:
Dan Willemsen5ea32d12015-09-08 13:27:20 -0700950 if project.relpath in self.gitc_manifest.paths and \
951 self.gitc_manifest.paths[project.relpath].old_revision:
952 opened_projects.append(project.relpath)
Simran Basib9a1b732015-08-20 12:19:28 -0700953 else:
Dan Willemsen5ea32d12015-09-08 13:27:20 -0700954 gitc_projects.append(project.relpath)
Simran Basib9a1b732015-08-20 12:19:28 -0700955
Dan Willemsen5ea32d12015-09-08 13:27:20 -0700956 if not args:
957 gitc_projects = None
958
959 if gitc_projects != [] and not opt.local_only:
Simran Basib9a1b732015-08-20 12:19:28 -0700960 print('Updating GITC client: %s' % self.gitc_manifest.gitc_client_name)
Dan Willemsen5ea32d12015-09-08 13:27:20 -0700961 manifest = GitcManifest(self.repodir, self.gitc_manifest.gitc_client_name)
962 if manifest_name:
963 manifest.Override(manifest_name)
964 else:
965 manifest.Override(self.manifest.manifestFile)
966 gitc_utils.generate_gitc_manifest(self.gitc_manifest,
967 manifest,
Simran Basib9a1b732015-08-20 12:19:28 -0700968 gitc_projects)
969 print('GITC client successfully synced.')
970
971 # The opened projects need to be synced as normal, therefore we
972 # generate a new args list to represent the opened projects.
Dan Willemsen5ea32d12015-09-08 13:27:20 -0700973 # TODO: make this more reliable -- if there's a project name/path overlap,
974 # this may choose the wrong project.
David Pursehouse3bcd3052017-07-10 22:42:22 +0900975 args = [os.path.relpath(self.manifest.paths[path].worktree, os.getcwd())
976 for path in opened_projects]
Simran Basib9a1b732015-08-20 12:19:28 -0700977 if not args:
978 return
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +0800979 all_projects = self.GetProjects(args,
980 missing_ok=True,
981 submodules_ok=opt.fetch_submodules)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700982
Mike Frysinger5a033082019-09-23 19:21:20 -0400983 err_network_sync = False
984 err_update_projects = False
Mike Frysinger5a033082019-09-23 19:21:20 -0400985
Dave Borowitz67700e92012-10-23 15:00:54 -0700986 self._fetch_times = _FetchTimes(self.manifest)
Shawn O. Pearceb1562fa2009-04-10 17:04:08 -0700987 if not opt.local_only:
Mike Frysinger19e409c2021-05-05 19:44:35 -0400988 try:
989 ssh.init()
990 self._FetchMain(opt, args, all_projects, err_event, manifest_name,
991 load_local_manifests)
992 finally:
993 ssh.close()
Mike Frysinger5a033082019-09-23 19:21:20 -0400994
995 # If we saw an error, exit with code 1 so that other scripts can check.
Mike Frysingerbe24a542021-02-23 03:24:12 -0500996 if err_event.is_set():
Mike Frysinger5a033082019-09-23 19:21:20 -0400997 err_network_sync = True
998 if opt.fail_fast:
999 print('\nerror: Exited sync due to fetch errors.\n'
1000 'Local checkouts *not* updated. Resolve network issues & '
1001 'retry.\n'
1002 '`repo sync -l` will update some local checkouts.',
1003 file=sys.stderr)
1004 sys.exit(1)
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +08001005
Julien Campergue335f5ef2013-10-16 11:02:35 +02001006 if self.manifest.IsMirror or self.manifest.IsArchive:
Shawn O. Pearcecd1d7ff2009-06-04 16:15:53 -07001007 # bail out now, we have no working tree
1008 return
1009
Oleksii Okolielovd3c0f592018-12-17 19:23:44 -05001010 if self.UpdateProjectList(opt):
Mike Frysinger5a033082019-09-23 19:21:20 -04001011 err_event.set()
1012 err_update_projects = True
1013 if opt.fail_fast:
1014 print('\nerror: Local checkouts *not* updated.', file=sys.stderr)
1015 sys.exit(1)
Jaikumar Ganesh4f2517f2009-06-01 21:10:33 -07001016
Mike Frysinger14208f42021-05-04 15:31:51 -04001017 err_update_linkfiles = not self.UpdateCopyLinkfileList()
1018 if err_update_linkfiles:
jiajia tanga590e642021-04-25 20:02:02 +08001019 err_event.set()
jiajia tanga590e642021-04-25 20:02:02 +08001020 if opt.fail_fast:
1021 print('\nerror: Local update copyfile or linkfile failed.', file=sys.stderr)
1022 sys.exit(1)
1023
Mike Frysinger5a033082019-09-23 19:21:20 -04001024 err_results = []
Mike Frysingerebf04a42021-02-23 20:48:04 -05001025 # NB: We don't exit here because this is the last step.
1026 err_checkout = not self._Checkout(all_projects, opt, err_results)
1027 if err_checkout:
1028 err_event.set()
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001029
Doug Anderson2b8db3c2010-11-01 15:08:06 -07001030 # If there's a notice that's supposed to print at the end of the sync, print
1031 # it now...
1032 if self.manifest.notice:
Sarah Owenscecd1d82012-11-01 22:59:27 -07001033 print(self.manifest.notice)
Doug Anderson2b8db3c2010-11-01 15:08:06 -07001034
Mike Frysinger5a033082019-09-23 19:21:20 -04001035 # If we saw an error, exit with code 1 so that other scripts can check.
Mike Frysingerbe24a542021-02-23 03:24:12 -05001036 if err_event.is_set():
Mike Frysinger5a033082019-09-23 19:21:20 -04001037 print('\nerror: Unable to fully sync the tree.', file=sys.stderr)
1038 if err_network_sync:
1039 print('error: Downloading network changes failed.', file=sys.stderr)
1040 if err_update_projects:
1041 print('error: Updating local project lists failed.', file=sys.stderr)
jiajia tanga590e642021-04-25 20:02:02 +08001042 if err_update_linkfiles:
1043 print('error: Updating copyfiles or linkfiles failed.', file=sys.stderr)
Mike Frysinger5a033082019-09-23 19:21:20 -04001044 if err_checkout:
1045 print('error: Checking out local projects failed.', file=sys.stderr)
1046 if err_results:
1047 print('Failing repos:\n%s' % '\n'.join(err_results), file=sys.stderr)
1048 print('Try re-running with "-j1 --fail-fast" to exit at the first error.',
1049 file=sys.stderr)
1050 sys.exit(1)
1051
Mike Frysingere19d9e12020-02-12 11:23:32 -05001052 if not opt.quiet:
1053 print('repo sync has finished successfully.')
1054
David Pursehouse819827a2020-02-12 15:20:19 +09001055
Shawn O. Pearce80d2ceb2012-10-26 12:23:05 -07001056def _PostRepoUpgrade(manifest, quiet=False):
Conley Owens094cdbe2014-01-30 15:09:59 -08001057 wrapper = Wrapper()
Conley Owensc9129d92012-10-01 16:12:28 -07001058 if wrapper.NeedSetupGnuPG():
Shawn O. Pearce80d2ceb2012-10-26 12:23:05 -07001059 wrapper.SetupGnuPG(quiet)
Conley Owensf2fe2d92014-01-29 13:53:43 -08001060 for project in manifest.projects:
Shawn O. Pearcee756c412009-04-13 11:51:15 -07001061 if project.Exists:
1062 project.PostRepoUpgrade()
1063
David Pursehouse819827a2020-02-12 15:20:19 +09001064
Mike Frysingerc58ec4d2020-02-17 14:36:08 -05001065def _PostRepoFetch(rp, repo_verify=True, verbose=False):
Shawn O. Pearcee756c412009-04-13 11:51:15 -07001066 if rp.HasChanges:
Sarah Owenscecd1d82012-11-01 22:59:27 -07001067 print('info: A new version of repo is available', file=sys.stderr)
Mike Frysinger347f9ed2021-03-15 14:58:52 -04001068 wrapper = Wrapper()
1069 try:
1070 rev = rp.bare_git.describe(rp.GetRevisionId())
1071 except GitError:
1072 rev = None
1073 _, new_rev = wrapper.check_repo_rev(rp.gitdir, rev, repo_verify=repo_verify)
1074 # See if we're held back due to missing signed tag.
1075 current_revid = rp.bare_git.rev_parse('HEAD')
1076 new_revid = rp.bare_git.rev_parse('--verify', new_rev)
1077 if current_revid != new_revid:
1078 # We want to switch to the new rev, but also not trash any uncommitted
1079 # changes. This helps with local testing/hacking.
1080 # If a local change has been made, we will throw that away.
1081 # We also have to make sure this will switch to an older commit if that's
1082 # the latest tag in order to support release rollback.
1083 try:
1084 rp.work_git.reset('--keep', new_rev)
1085 except GitError as e:
1086 sys.exit(str(e))
Sarah Owenscecd1d82012-11-01 22:59:27 -07001087 print('info: Restarting repo with latest version', file=sys.stderr)
Shawn O. Pearcee756c412009-04-13 11:51:15 -07001088 raise RepoChangedException(['--repo-upgraded'])
1089 else:
Sarah Owenscecd1d82012-11-01 22:59:27 -07001090 print('warning: Skipped upgrade to unverified version', file=sys.stderr)
Shawn O. Pearcee756c412009-04-13 11:51:15 -07001091 else:
1092 if verbose:
Sarah Owenscecd1d82012-11-01 22:59:27 -07001093 print('repo version %s is current' % rp.work_git.describe(HEAD),
1094 file=sys.stderr)
Shawn O. Pearcee756c412009-04-13 11:51:15 -07001095
David Pursehouse819827a2020-02-12 15:20:19 +09001096
Dave Borowitz67700e92012-10-23 15:00:54 -07001097class _FetchTimes(object):
Dave Borowitzd9478582012-10-23 16:35:39 -07001098 _ALPHA = 0.5
1099
Dave Borowitz67700e92012-10-23 15:00:54 -07001100 def __init__(self, manifest):
Anthony King85b24ac2014-05-06 15:57:48 +01001101 self._path = os.path.join(manifest.repodir, '.repo_fetchtimes.json')
Dave Borowitz67700e92012-10-23 15:00:54 -07001102 self._times = None
Dave Borowitzd9478582012-10-23 16:35:39 -07001103 self._seen = set()
Dave Borowitz67700e92012-10-23 15:00:54 -07001104
1105 def Get(self, project):
1106 self._Load()
1107 return self._times.get(project.name, _ONE_DAY_S)
1108
1109 def Set(self, project, t):
Dave Borowitzd9478582012-10-23 16:35:39 -07001110 self._Load()
1111 name = project.name
1112 old = self._times.get(name, t)
1113 self._seen.add(name)
1114 a = self._ALPHA
David Pursehouse54a4e602020-02-12 14:31:05 +09001115 self._times[name] = (a * t) + ((1 - a) * old)
Dave Borowitz67700e92012-10-23 15:00:54 -07001116
1117 def _Load(self):
1118 if self._times is None:
1119 try:
Mike Frysinger3164d402019-11-11 05:40:22 -05001120 with open(self._path) as f:
Anthony King85b24ac2014-05-06 15:57:48 +01001121 self._times = json.load(f)
Anthony King85b24ac2014-05-06 15:57:48 +01001122 except (IOError, ValueError):
1123 try:
Renaud Paquay010fed72016-11-11 14:25:29 -08001124 platform_utils.remove(self._path)
Anthony King85b24ac2014-05-06 15:57:48 +01001125 except OSError:
1126 pass
1127 self._times = {}
Dave Borowitz67700e92012-10-23 15:00:54 -07001128
1129 def Save(self):
1130 if self._times is None:
1131 return
Dave Borowitzd9478582012-10-23 16:35:39 -07001132
1133 to_delete = []
1134 for name in self._times:
1135 if name not in self._seen:
1136 to_delete.append(name)
1137 for name in to_delete:
1138 del self._times[name]
1139
Dave Borowitz67700e92012-10-23 15:00:54 -07001140 try:
Mike Frysinger3164d402019-11-11 05:40:22 -05001141 with open(self._path, 'w') as f:
Anthony King85b24ac2014-05-06 15:57:48 +01001142 json.dump(self._times, f, indent=2)
Anthony King85b24ac2014-05-06 15:57:48 +01001143 except (IOError, TypeError):
1144 try:
Renaud Paquay010fed72016-11-11 14:25:29 -08001145 platform_utils.remove(self._path)
Anthony King85b24ac2014-05-06 15:57:48 +01001146 except OSError:
1147 pass
Dan Willemsen0745bb22015-08-17 13:41:45 -07001148
1149# This is a replacement for xmlrpc.client.Transport using urllib2
1150# and supporting persistent-http[s]. It cannot change hosts from
1151# request to request like the normal transport, the real url
1152# is passed during initialization.
David Pursehouse819827a2020-02-12 15:20:19 +09001153
1154
Dan Willemsen0745bb22015-08-17 13:41:45 -07001155class PersistentTransport(xmlrpc.client.Transport):
1156 def __init__(self, orig_host):
1157 self.orig_host = orig_host
1158
1159 def request(self, host, handler, request_body, verbose=False):
1160 with GetUrlCookieFile(self.orig_host, not verbose) as (cookiefile, proxy):
1161 # Python doesn't understand cookies with the #HttpOnly_ prefix
1162 # Since we're only using them for HTTP, copy the file temporarily,
1163 # stripping those prefixes away.
Dan Willemsen3010e5b2015-08-20 10:09:20 -07001164 if cookiefile:
Collin Fijalkoviche1191b32020-02-18 10:57:32 -08001165 tmpcookiefile = tempfile.NamedTemporaryFile(mode='w')
David Pursehouse4c5f74e2015-10-02 11:10:10 +09001166 tmpcookiefile.write("# HTTP Cookie File")
Dan Willemsen3010e5b2015-08-20 10:09:20 -07001167 try:
1168 with open(cookiefile) as f:
1169 for line in f:
1170 if line.startswith("#HttpOnly_"):
1171 line = line[len("#HttpOnly_"):]
1172 tmpcookiefile.write(line)
1173 tmpcookiefile.flush()
Dan Willemsen0745bb22015-08-17 13:41:45 -07001174
Dan Willemsen3010e5b2015-08-20 10:09:20 -07001175 cookiejar = cookielib.MozillaCookieJar(tmpcookiefile.name)
David Pursehouseb1ad2192015-09-30 10:35:43 +09001176 try:
1177 cookiejar.load()
1178 except cookielib.LoadError:
1179 cookiejar = cookielib.CookieJar()
Dan Willemsen3010e5b2015-08-20 10:09:20 -07001180 finally:
1181 tmpcookiefile.close()
1182 else:
1183 cookiejar = cookielib.CookieJar()
Dan Willemsen0745bb22015-08-17 13:41:45 -07001184
1185 proxyhandler = urllib.request.ProxyHandler
1186 if proxy:
1187 proxyhandler = urllib.request.ProxyHandler({
1188 "http": proxy,
David Pursehouse54a4e602020-02-12 14:31:05 +09001189 "https": proxy})
Dan Willemsen0745bb22015-08-17 13:41:45 -07001190
1191 opener = urllib.request.build_opener(
1192 urllib.request.HTTPCookieProcessor(cookiejar),
1193 proxyhandler)
1194
1195 url = urllib.parse.urljoin(self.orig_host, handler)
1196 parse_results = urllib.parse.urlparse(url)
1197
1198 scheme = parse_results.scheme
1199 if scheme == 'persistent-http':
1200 scheme = 'http'
1201 if scheme == 'persistent-https':
1202 # If we're proxying through persistent-https, use http. The
1203 # proxy itself will do the https.
1204 if proxy:
1205 scheme = 'http'
1206 else:
1207 scheme = 'https'
1208
1209 # Parse out any authentication information using the base class
1210 host, extra_headers, _ = self.get_host_info(parse_results.netloc)
1211
1212 url = urllib.parse.urlunparse((
1213 scheme,
1214 host,
1215 parse_results.path,
1216 parse_results.params,
1217 parse_results.query,
1218 parse_results.fragment))
1219
1220 request = urllib.request.Request(url, request_body)
1221 if extra_headers is not None:
1222 for (name, header) in extra_headers:
1223 request.add_header(name, header)
1224 request.add_header('Content-Type', 'text/xml')
1225 try:
1226 response = opener.open(request)
1227 except urllib.error.HTTPError as e:
1228 if e.code == 501:
1229 # We may have been redirected through a login process
1230 # but our POST turned into a GET. Retry.
1231 response = opener.open(request)
1232 else:
1233 raise
1234
1235 p, u = xmlrpc.client.getparser()
1236 while 1:
1237 data = response.read(1024)
1238 if not data:
1239 break
1240 p.feed(data)
1241 p.close()
1242 return u.close()
1243
1244 def close(self):
1245 pass