blob: 1e1f2fc16092fa7ac13eaeb1ec2508440f737a6d [file] [log] [blame]
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001# Copyright (C) 2008 The Android Open Source Project
2#
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7# http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12# See the License for the specific language governing permissions and
13# limitations under the License.
14
Anthony King85b24ac2014-05-06 15:57:48 +010015import json
David Pursehouse86d973d2012-08-24 10:21:02 +090016import netrc
Shawn O. Pearce2a1ccb22009-04-10 16:51:53 -070017from optparse import SUPPRESS_HELP
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070018import os
19import re
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -070020import socket
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070021import subprocess
22import sys
Dan Willemsen0745bb22015-08-17 13:41:45 -070023import tempfile
Shawn O. Pearcef6906872009-04-18 10:49:00 -070024import time
David Pursehouse59bbb582013-05-17 10:49:33 +090025
26from pyversion import is_python3
27if is_python3():
Dan Willemsen0745bb22015-08-17 13:41:45 -070028 import http.cookiejar as cookielib
29 import urllib.error
Chirayu Desai217ea7d2013-03-01 19:14:38 +053030 import urllib.parse
Dan Willemsen0745bb22015-08-17 13:41:45 -070031 import urllib.request
David Pursehouse59bbb582013-05-17 10:49:33 +090032 import xmlrpc.client
33else:
Dan Willemsen0745bb22015-08-17 13:41:45 -070034 import cookielib
Chirayu Desai217ea7d2013-03-01 19:14:38 +053035 import imp
Dan Willemsen0745bb22015-08-17 13:41:45 -070036 import urllib2
Chirayu Desai217ea7d2013-03-01 19:14:38 +053037 import urlparse
David Pursehouse59bbb582013-05-17 10:49:33 +090038 import xmlrpclib
Chirayu Desai217ea7d2013-03-01 19:14:38 +053039 urllib = imp.new_module('urllib')
Dan Willemsen0745bb22015-08-17 13:41:45 -070040 urllib.error = urllib2
Chirayu Desaidb2ad9d2013-06-11 13:42:25 +053041 urllib.parse = urlparse
Dan Willemsen0745bb22015-08-17 13:41:45 -070042 urllib.request = urllib2
Chirayu Desai217ea7d2013-03-01 19:14:38 +053043 xmlrpc = imp.new_module('xmlrpc')
44 xmlrpc.client = xmlrpclib
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070045
Roy Lee18afd7f2010-05-09 04:32:08 +080046try:
47 import threading as _threading
48except ImportError:
49 import dummy_threading as _threading
50
Shawn O. Pearce97d2b2f2011-09-22 17:23:41 -070051try:
52 import resource
David Pursehouse819827a2020-02-12 15:20:19 +090053
Shawn O. Pearce97d2b2f2011-09-22 17:23:41 -070054 def _rlimit_nofile():
55 return resource.getrlimit(resource.RLIMIT_NOFILE)
56except ImportError:
57 def _rlimit_nofile():
58 return (256, 256)
59
Dave Borowitz18857212012-10-23 17:02:59 -070060try:
61 import multiprocessing
62except ImportError:
63 multiprocessing = None
64
David Rileye0684ad2017-04-05 00:02:59 -070065import event_log
Dave Borowitze2152672012-10-31 12:24:38 -070066from git_command import GIT, git_require
David Pursehouseba7bc732015-08-20 16:55:42 +090067from git_config import GetUrlCookieFile
David Pursehoused94aaef2012-09-07 09:52:04 +090068from git_refs import R_HEADS, HEAD
Simran Basibdb52712015-08-10 13:23:23 -070069import gitc_utils
Jaikumar Ganesh4f2517f2009-06-01 21:10:33 -070070from project import Project
71from project import RemoteSpec
Shawn O. Pearcec95583b2009-03-03 17:47:06 -080072from command import Command, MirrorSafeCommand
Torne (Richard Coles)7bdbde72012-12-05 10:58:06 +000073from error import RepoChangedException, GitError, ManifestParseError
Renaud Paquaya65adf72016-11-03 10:37:53 -070074import platform_utils
Shawn O. Pearce350cde42009-04-16 11:21:18 -070075from project import SyncBuffer
Shawn O. Pearce68194f42009-04-10 16:48:52 -070076from progress import Progress
Conley Owens094cdbe2014-01-30 15:09:59 -080077from wrapper import Wrapper
Dan Willemsen5ea32d12015-09-08 13:27:20 -070078from manifest_xml import GitcManifest
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070079
Dave Borowitz67700e92012-10-23 15:00:54 -070080_ONE_DAY_S = 24 * 60 * 60
81
David Pursehouse819827a2020-02-12 15:20:19 +090082
Doug Andersonfc06ced2011-03-16 15:49:18 -070083class _FetchError(Exception):
84 """Internal error thrown in _FetchHelper() when we don't want stack trace."""
85 pass
86
David Pursehouse819827a2020-02-12 15:20:19 +090087
Xin Li745be2e2019-06-03 11:24:30 -070088class _CheckoutError(Exception):
89 """Internal error thrown in _CheckoutOne() when we don't want stack trace."""
90
David Pursehouse819827a2020-02-12 15:20:19 +090091
Shawn O. Pearcec95583b2009-03-03 17:47:06 -080092class Sync(Command, MirrorSafeCommand):
Roy Lee18afd7f2010-05-09 04:32:08 +080093 jobs = 1
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -070094 common = True
95 helpSummary = "Update working tree to the latest revision"
96 helpUsage = """
97%prog [<project>...]
98"""
99 helpDescription = """
100The '%prog' command synchronizes local project directories
101with the remote repositories specified in the manifest. If a local
102project does not yet exist, it will clone a new local directory from
103the remote repository and set up tracking branches as specified in
104the manifest. If the local project already exists, '%prog'
105will update the remote branches and rebase any new local changes
106on top of the new remote changes.
107
108'%prog' will synchronize all projects listed at the command
109line. Projects can be specified either by name, or by a relative
110or absolute path to the project's local directory. If no projects
111are specified, '%prog' will synchronize all projects listed in
112the manifest.
Shawn O. Pearce3e768c92009-04-10 16:59:36 -0700113
114The -d/--detach option can be used to switch specified projects
115back to the manifest revision. This option is especially helpful
116if the project is currently on a topic branch, but the manifest
117revision is temporarily needed.
Shawn O. Pearceeb7af872009-04-21 08:02:04 -0700118
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -0700119The -s/--smart-sync option can be used to sync to a known good
120build as specified by the manifest-server element in the current
Victor Boivie08c880d2011-04-19 10:32:52 +0200121manifest. The -t/--smart-tag option is similar and allows you to
122specify a custom tag/label.
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -0700123
David Pursehousecf76b1b2012-09-14 10:31:42 +0900124The -u/--manifest-server-username and -p/--manifest-server-password
125options can be used to specify a username and password to authenticate
126with the manifest server when using the -s or -t option.
127
128If -u and -p are not specified when using the -s or -t option, '%prog'
129will attempt to read authentication credentials for the manifest server
130from the user's .netrc file.
131
132'%prog' will not use authentication credentials from -u/-p or .netrc
133if the manifest server specified in the manifest file already includes
134credentials.
135
Mike Frysingerd9e5cf02019-08-26 03:12:55 -0400136By default, all projects will be synced. The --fail-fast option can be used
Mike Frysinger7ae210a2020-05-24 14:56:52 -0400137to halt syncing as soon as possible when the first project fails to sync.
Andrei Warkentin5df6de02010-07-02 17:58:31 -0500138
Kevin Degiabaa7f32014-11-12 11:27:45 -0700139The --force-sync option can be used to overwrite existing git
140directories if they have previously been linked to a different
Roger Shimizuac29ac32020-06-06 02:33:40 +0900141object directory. WARNING: This may cause data to be lost since
Kevin Degiabaa7f32014-11-12 11:27:45 -0700142refs may be removed when overwriting.
143
Oleksii Okolielovd3c0f592018-12-17 19:23:44 -0500144The --force-remove-dirty option can be used to remove previously used
145projects with uncommitted changes. WARNING: This may cause data to be
146lost since uncommitted changes may be removed with projects that no longer
147exist in the manifest.
148
Shawn O. Pearcee02ac0a2012-03-14 15:36:59 -0700149The --no-clone-bundle option disables any attempt to use
150$URL/clone.bundle to bootstrap a new Git repository from a
151resumeable bundle file on a content delivery network. This
152may be necessary if there are problems with the local Python
153HTTP client or proxy configuration, but the Git binary works.
154
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +0800155The --fetch-submodules option enables fetching Git submodules
156of a project from server.
157
David Pursehousef2fad612015-01-29 14:36:28 +0900158The -c/--current-branch option can be used to only fetch objects that
159are on the branch specified by a project's revision.
160
David Pursehouseb1553542014-09-04 21:28:09 +0900161The --optimized-fetch option can be used to only fetch projects that
162are fixed to a sha1 revision if the sha1 revision does not already
163exist locally.
164
David Pursehouse74cfd272015-10-14 10:50:15 +0900165The --prune option can be used to remove any refs that no longer
166exist on the remote.
167
Mike Frysingerb8f7bb02018-10-10 01:05:11 -0400168# SSH Connections
Shawn O. Pearceeb7af872009-04-21 08:02:04 -0700169
170If at least one project remote URL uses an SSH connection (ssh://,
171git+ssh://, or user@host:path syntax) repo will automatically
172enable the SSH ControlMaster option when connecting to that host.
173This feature permits other projects in the same '%prog' session to
174reuse the same SSH tunnel, saving connection setup overheads.
175
176To disable this behavior on UNIX platforms, set the GIT_SSH
177environment variable to 'ssh'. For example:
178
179 export GIT_SSH=ssh
180 %prog
181
Mike Frysingerb8f7bb02018-10-10 01:05:11 -0400182# Compatibility
Shawn O. Pearceeb7af872009-04-21 08:02:04 -0700183
184This feature is automatically disabled on Windows, due to the lack
185of UNIX domain socket support.
186
187This feature is not compatible with url.insteadof rewrites in the
188user's ~/.gitconfig. '%prog' is currently not able to perform the
189rewrite early enough to establish the ControlMaster tunnel.
190
191If the remote SSH daemon is Gerrit Code Review, version 2.0.10 or
192later is required to fix a server side protocol bug.
193
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700194"""
195
Nico Sallembien6623b212010-05-11 12:57:01 -0700196 def _Options(self, p, show_smart=True):
Torne (Richard Coles)7bdbde72012-12-05 10:58:06 +0000197 try:
198 self.jobs = self.manifest.default.sync_j
199 except ManifestParseError:
200 self.jobs = 1
Shawn O. Pearce6392c872011-09-22 17:44:31 -0700201
Andrei Warkentin5df6de02010-07-02 17:58:31 -0500202 p.add_option('-f', '--force-broken',
Stefan Müller-Klieser46702ed2019-08-30 10:20:15 +0200203 dest='force_broken', action='store_true',
Mike Frysingerd9e5cf02019-08-26 03:12:55 -0400204 help='obsolete option (to be deleted in the future)')
205 p.add_option('--fail-fast',
206 dest='fail_fast', action='store_true',
207 help='stop syncing after first error is hit')
Kevin Degiabaa7f32014-11-12 11:27:45 -0700208 p.add_option('--force-sync',
209 dest='force_sync', action='store_true',
210 help="overwrite an existing git directory if it needs to "
211 "point to a different object directory. WARNING: this "
212 "may cause loss of data")
Oleksii Okolielovd3c0f592018-12-17 19:23:44 -0500213 p.add_option('--force-remove-dirty',
214 dest='force_remove_dirty', action='store_true',
215 help="force remove projects with uncommitted modifications if "
216 "projects no longer exist in the manifest. "
217 "WARNING: this may cause loss of data")
David Pursehouse8f62fb72012-11-14 12:09:38 +0900218 p.add_option('-l', '--local-only',
Shawn O. Pearceb1562fa2009-04-10 17:04:08 -0700219 dest='local_only', action='store_true',
220 help="only update working tree, don't fetch")
David Pursehouse54a4e602020-02-12 14:31:05 +0900221 p.add_option('--no-manifest-update', '--nmu',
Fredrik de Grootcc960972019-11-22 09:04:31 +0100222 dest='mp_update', action='store_false', default='true',
223 help='use the existing manifest checkout as-is. '
224 '(do not update to the latest revision)')
David Pursehouse8f62fb72012-11-14 12:09:38 +0900225 p.add_option('-n', '--network-only',
Shawn O. Pearce96fdcef2009-04-10 16:29:20 -0700226 dest='network_only', action='store_true',
227 help="fetch only, don't update working tree")
David Pursehouse8f62fb72012-11-14 12:09:38 +0900228 p.add_option('-d', '--detach',
Shawn O. Pearce3e768c92009-04-10 16:59:36 -0700229 dest='detach_head', action='store_true',
230 help='detach projects back to manifest revision')
David Pursehouse8f62fb72012-11-14 12:09:38 +0900231 p.add_option('-c', '--current-branch',
Anatol Pomazau53d6f4d2011-08-25 17:21:47 -0700232 dest='current_branch_only', action='store_true',
233 help='fetch only current branch from server')
Mike Frysinger521d01b2020-02-17 01:51:49 -0500234 p.add_option('-v', '--verbose',
235 dest='output_mode', action='store_true',
236 help='show all sync output')
David Pursehouse8f62fb72012-11-14 12:09:38 +0900237 p.add_option('-q', '--quiet',
Mike Frysinger521d01b2020-02-17 01:51:49 -0500238 dest='output_mode', action='store_false',
239 help='only show errors')
David Pursehouse8f62fb72012-11-14 12:09:38 +0900240 p.add_option('-j', '--jobs',
Roy Lee18afd7f2010-05-09 04:32:08 +0800241 dest='jobs', action='store', type='int',
Shawn O. Pearce6392c872011-09-22 17:44:31 -0700242 help="projects to fetch simultaneously (default %d)" % self.jobs)
Chris Wolfee9dc3b32012-01-26 11:36:18 -0500243 p.add_option('-m', '--manifest-name',
244 dest='manifest_name',
245 help='temporary manifest to use for this sync', metavar='NAME.xml')
Xin Lid79a4bc2020-05-20 16:03:45 -0700246 p.add_option('--clone-bundle', action='store_true',
247 help='enable use of /clone.bundle on HTTP/HTTPS')
248 p.add_option('--no-clone-bundle', dest='clone_bundle', action='store_false',
Shawn O. Pearcee02ac0a2012-03-14 15:36:59 -0700249 help='disable use of /clone.bundle on HTTP/HTTPS')
Conley Owens8d070cf2012-11-06 13:14:31 -0800250 p.add_option('-u', '--manifest-server-username', action='store',
251 dest='manifest_server_username',
252 help='username to authenticate with the manifest server')
253 p.add_option('-p', '--manifest-server-password', action='store',
254 dest='manifest_server_password',
255 help='password to authenticate with the manifest server')
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +0800256 p.add_option('--fetch-submodules',
257 dest='fetch_submodules', action='store_true',
258 help='fetch submodules from server')
Mitchel Humpherys597868b2012-10-29 10:18:34 -0700259 p.add_option('--no-tags',
Mike Frysingerc58ec4d2020-02-17 14:36:08 -0500260 dest='tags', default=True, action='store_false',
Mitchel Humpherys597868b2012-10-29 10:18:34 -0700261 help="don't fetch tags")
David Pursehouseb1553542014-09-04 21:28:09 +0900262 p.add_option('--optimized-fetch',
263 dest='optimized_fetch', action='store_true',
264 help='only fetch projects fixed to sha1 if revision does not exist locally')
George Engelbrecht9bc283e2020-04-02 12:36:09 -0600265 p.add_option('--retry-fetches',
266 default=0, action='store', type='int',
267 help='number of times to retry fetches on transient errors')
David Pursehouse74cfd272015-10-14 10:50:15 +0900268 p.add_option('--prune', dest='prune', action='store_true',
269 help='delete refs that no longer exist on the remote')
Nico Sallembien6623b212010-05-11 12:57:01 -0700270 if show_smart:
271 p.add_option('-s', '--smart-sync',
272 dest='smart_sync', action='store_true',
David Pursehouse79fba682016-04-13 18:03:00 +0900273 help='smart sync using manifest from the latest known good build')
Victor Boivie08c880d2011-04-19 10:32:52 +0200274 p.add_option('-t', '--smart-tag',
275 dest='smart_tag', action='store',
276 help='smart sync using manifest from a known tag')
Shawn O. Pearce3e768c92009-04-10 16:59:36 -0700277
Shawn O. Pearcefd89b672009-04-18 11:28:57 -0700278 g = p.add_option_group('repo Version options')
279 g.add_option('--no-repo-verify',
Mike Frysingerc58ec4d2020-02-17 14:36:08 -0500280 dest='repo_verify', default=True, action='store_false',
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700281 help='do not verify repo source code')
Shawn O. Pearcefd89b672009-04-18 11:28:57 -0700282 g.add_option('--repo-upgraded',
Shawn O. Pearcec9ef7442008-11-03 10:32:09 -0800283 dest='repo_upgraded', action='store_true',
Shawn O. Pearce2a1ccb22009-04-10 16:51:53 -0700284 help=SUPPRESS_HELP)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700285
Andrew Wheeler7f1ccfb2016-06-17 16:51:07 -0500286 def _FetchProjectList(self, opt, projects, sem, *args, **kwargs):
Xin Li745be2e2019-06-03 11:24:30 -0700287 """Main function of the fetch threads.
Roy Lee18afd7f2010-05-09 04:32:08 +0800288
David James8d201162013-10-11 17:03:19 -0700289 Delegates most of the work to _FetchHelper.
290
291 Args:
292 opt: Program options returned from optparse. See _Options().
293 projects: Projects to fetch.
Andrew Wheeler7f1ccfb2016-06-17 16:51:07 -0500294 sem: We'll release() this semaphore when we exit so that another thread
295 can be started up.
David James89ece422014-01-09 18:51:58 -0800296 *args, **kwargs: Remaining arguments to pass to _FetchHelper. See the
David James8d201162013-10-11 17:03:19 -0700297 _FetchHelper docstring for details.
298 """
Andrew Wheeler7f1ccfb2016-06-17 16:51:07 -0500299 try:
300 for project in projects:
301 success = self._FetchHelper(opt, project, *args, **kwargs)
Mike Frysingerd9e5cf02019-08-26 03:12:55 -0400302 if not success and opt.fail_fast:
Andrew Wheeler7f1ccfb2016-06-17 16:51:07 -0500303 break
304 finally:
305 sem.release()
David James8d201162013-10-11 17:03:19 -0700306
Xin Li745be2e2019-06-03 11:24:30 -0700307 def _FetchHelper(self, opt, project, lock, fetched, pm, err_event,
308 clone_filter):
David James8d201162013-10-11 17:03:19 -0700309 """Fetch git objects for a single project.
310
David Pursehousec1b86a22012-11-14 11:36:51 +0900311 Args:
312 opt: Program options returned from optparse. See _Options().
313 project: Project object for the project to fetch.
314 lock: Lock for accessing objects that are shared amongst multiple
315 _FetchHelper() threads.
316 fetched: set object that we will add project.gitdir to when we're done
317 (with our lock held).
318 pm: Instance of a Project object. We will call pm.update() (with our
319 lock held).
David Pursehousec1b86a22012-11-14 11:36:51 +0900320 err_event: We'll set this event in the case of an error (after printing
321 out info about the error).
Xin Li745be2e2019-06-03 11:24:30 -0700322 clone_filter: Filter for use in a partial clone.
David James8d201162013-10-11 17:03:19 -0700323
324 Returns:
325 Whether the fetch was successful.
David Pursehousec1b86a22012-11-14 11:36:51 +0900326 """
327 # We'll set to true once we've locked the lock.
328 did_lock = False
Doug Andersonfc06ced2011-03-16 15:49:18 -0700329
David Pursehousec1b86a22012-11-14 11:36:51 +0900330 # Encapsulate everything in a try/except/finally so that:
331 # - We always set err_event in the case of an exception.
David Pursehousec1b86a22012-11-14 11:36:51 +0900332 # - We always make sure we unlock the lock if we locked it.
David Rileye0684ad2017-04-05 00:02:59 -0700333 start = time.time()
334 success = False
David Pursehousec1b86a22012-11-14 11:36:51 +0900335 try:
Doug Andersonfc06ced2011-03-16 15:49:18 -0700336 try:
David Pursehousec1b86a22012-11-14 11:36:51 +0900337 success = project.Sync_NetworkHalf(
David Pursehouseabdf7502020-02-12 14:58:39 +0900338 quiet=opt.quiet,
Mike Frysinger521d01b2020-02-17 01:51:49 -0500339 verbose=opt.verbose,
David Pursehouseabdf7502020-02-12 14:58:39 +0900340 current_branch_only=opt.current_branch_only,
341 force_sync=opt.force_sync,
Mike Frysingerc58ec4d2020-02-17 14:36:08 -0500342 clone_bundle=opt.clone_bundle,
343 tags=opt.tags, archive=self.manifest.IsArchive,
David Pursehouseabdf7502020-02-12 14:58:39 +0900344 optimized_fetch=opt.optimized_fetch,
George Engelbrecht9bc283e2020-04-02 12:36:09 -0600345 retry_fetches=opt.retry_fetches,
David Pursehouseabdf7502020-02-12 14:58:39 +0900346 prune=opt.prune,
347 clone_filter=clone_filter)
David Pursehousec1b86a22012-11-14 11:36:51 +0900348 self._fetch_times.Set(project, time.time() - start)
Doug Andersonfc06ced2011-03-16 15:49:18 -0700349
David Pursehousec1b86a22012-11-14 11:36:51 +0900350 # Lock around all the rest of the code, since printing, updating a set
351 # and Progress.update() are not thread safe.
352 lock.acquire()
353 did_lock = True
Doug Andersonfc06ced2011-03-16 15:49:18 -0700354
David Pursehousec1b86a22012-11-14 11:36:51 +0900355 if not success:
Hu Xiuyune9becc02015-11-25 15:52:26 +0800356 err_event.set()
Marc Herbertffb4b892017-04-04 22:03:53 -0700357 print('error: Cannot fetch %s from %s'
358 % (project.name, project.remote.url),
359 file=sys.stderr)
Mike Frysingerd9e5cf02019-08-26 03:12:55 -0400360 if opt.fail_fast:
David Pursehousec1b86a22012-11-14 11:36:51 +0900361 raise _FetchError()
Doug Andersonfc06ced2011-03-16 15:49:18 -0700362
David Pursehousec1b86a22012-11-14 11:36:51 +0900363 fetched.add(project.gitdir)
Mike Frysinger3538dd22019-08-26 15:32:06 -0400364 pm.update(msg=project.name)
David Pursehousec1b86a22012-11-14 11:36:51 +0900365 except _FetchError:
Hu Xiuyune9becc02015-11-25 15:52:26 +0800366 pass
Dan Sandlerc5cd4332015-07-31 09:37:53 -0400367 except Exception as e:
David Pursehouse42339d72020-02-12 14:37:15 +0900368 print('error: Cannot fetch %s (%s: %s)'
David Pursehouseabdf7502020-02-12 14:58:39 +0900369 % (project.name, type(e).__name__, str(e)), file=sys.stderr)
David Pursehousec1b86a22012-11-14 11:36:51 +0900370 err_event.set()
371 raise
372 finally:
373 if did_lock:
374 lock.release()
David Rileye0684ad2017-04-05 00:02:59 -0700375 finish = time.time()
376 self.event_log.AddSync(project, event_log.TASK_SYNC_NETWORK,
377 start, finish, success)
Roy Lee18afd7f2010-05-09 04:32:08 +0800378
David James8d201162013-10-11 17:03:19 -0700379 return success
380
Mike Frysinger5a033082019-09-23 19:21:20 -0400381 def _Fetch(self, projects, opt, err_event):
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700382 fetched = set()
David James89ece422014-01-09 18:51:58 -0800383 lock = _threading.Lock()
Tim Schumacher913327f2017-06-05 15:01:41 +0200384 pm = Progress('Fetching projects', len(projects),
Tim Schumacher7be072e2017-06-28 18:29:23 +0200385 always_print_percentage=opt.quiet)
Roy Lee18afd7f2010-05-09 04:32:08 +0800386
David James89ece422014-01-09 18:51:58 -0800387 objdir_project_map = dict()
388 for project in projects:
389 objdir_project_map.setdefault(project.objdir, []).append(project)
David James8d201162013-10-11 17:03:19 -0700390
David James89ece422014-01-09 18:51:58 -0800391 threads = set()
392 sem = _threading.Semaphore(self.jobs)
David James89ece422014-01-09 18:51:58 -0800393 for project_list in objdir_project_map.values():
394 # Check for any errors before running any more tasks.
395 # ...we'll let existing threads finish, though.
Mike Frysingerd9e5cf02019-08-26 03:12:55 -0400396 if err_event.isSet() and opt.fail_fast:
David James89ece422014-01-09 18:51:58 -0800397 break
Doug Andersonfc06ced2011-03-16 15:49:18 -0700398
David James89ece422014-01-09 18:51:58 -0800399 sem.acquire()
400 kwargs = dict(opt=opt,
401 projects=project_list,
Andrew Wheeler7f1ccfb2016-06-17 16:51:07 -0500402 sem=sem,
David James89ece422014-01-09 18:51:58 -0800403 lock=lock,
404 fetched=fetched,
405 pm=pm,
Xin Li745be2e2019-06-03 11:24:30 -0700406 err_event=err_event,
407 clone_filter=self.manifest.CloneFilter)
David James89ece422014-01-09 18:51:58 -0800408 if self.jobs > 1:
David Pursehousee5913ae2020-02-12 13:56:59 +0900409 t = _threading.Thread(target=self._FetchProjectList,
410 kwargs=kwargs)
David 'Digit' Turnere2126652012-09-05 10:35:06 +0200411 # Ensure that Ctrl-C will not freeze the repo process.
412 t.daemon = True
Roy Lee18afd7f2010-05-09 04:32:08 +0800413 threads.add(t)
414 t.start()
David James89ece422014-01-09 18:51:58 -0800415 else:
416 self._FetchProjectList(**kwargs)
Roy Lee18afd7f2010-05-09 04:32:08 +0800417
David James89ece422014-01-09 18:51:58 -0800418 for t in threads:
419 t.join()
Roy Lee18afd7f2010-05-09 04:32:08 +0800420
Shawn O. Pearce68194f42009-04-10 16:48:52 -0700421 pm.end()
Dave Borowitz67700e92012-10-23 15:00:54 -0700422 self._fetch_times.Save()
Dave Borowitz18857212012-10-23 17:02:59 -0700423
Julien Campergue335f5ef2013-10-16 11:02:35 +0200424 if not self.manifest.IsArchive:
Mike Frysinger5a033082019-09-23 19:21:20 -0400425 self._GCProjects(projects, opt, err_event)
Julien Campergue335f5ef2013-10-16 11:02:35 +0200426
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700427 return fetched
428
Xin Li745be2e2019-06-03 11:24:30 -0700429 def _CheckoutWorker(self, opt, sem, project, *args, **kwargs):
430 """Main function of the fetch threads.
431
432 Delegates most of the work to _CheckoutOne.
433
434 Args:
435 opt: Program options returned from optparse. See _Options().
436 projects: Projects to fetch.
437 sem: We'll release() this semaphore when we exit so that another thread
438 can be started up.
439 *args, **kwargs: Remaining arguments to pass to _CheckoutOne. See the
440 _CheckoutOne docstring for details.
441 """
442 try:
Mike Frysingera34186e2019-08-07 18:07:31 -0400443 return self._CheckoutOne(opt, project, *args, **kwargs)
Xin Li745be2e2019-06-03 11:24:30 -0700444 finally:
445 sem.release()
446
Vadim Bendeburydff91942019-11-06 11:05:00 -0800447 def _CheckoutOne(self, opt, project, lock, pm, err_event, err_results):
Xin Li745be2e2019-06-03 11:24:30 -0700448 """Checkout work tree for one project
449
450 Args:
451 opt: Program options returned from optparse. See _Options().
452 project: Project object for the project to checkout.
453 lock: Lock for accessing objects that are shared amongst multiple
454 _CheckoutWorker() threads.
455 pm: Instance of a Project object. We will call pm.update() (with our
456 lock held).
457 err_event: We'll set this event in the case of an error (after printing
458 out info about the error).
Vadim Bendeburydff91942019-11-06 11:05:00 -0800459 err_results: A list of strings, paths to git repos where checkout
460 failed.
Xin Li745be2e2019-06-03 11:24:30 -0700461
462 Returns:
463 Whether the fetch was successful.
464 """
465 # We'll set to true once we've locked the lock.
466 did_lock = False
467
Xin Li745be2e2019-06-03 11:24:30 -0700468 # Encapsulate everything in a try/except/finally so that:
469 # - We always set err_event in the case of an exception.
470 # - We always make sure we unlock the lock if we locked it.
471 start = time.time()
472 syncbuf = SyncBuffer(self.manifest.manifestProject.config,
473 detach_head=opt.detach_head)
474 success = False
475 try:
476 try:
477 project.Sync_LocalHalf(syncbuf, force_sync=opt.force_sync)
Xin Li745be2e2019-06-03 11:24:30 -0700478
479 # Lock around all the rest of the code, since printing, updating a set
480 # and Progress.update() are not thread safe.
481 lock.acquire()
Mike Frysinger3538dd22019-08-26 15:32:06 -0400482 success = syncbuf.Finish()
Xin Li745be2e2019-06-03 11:24:30 -0700483 did_lock = True
484
485 if not success:
486 err_event.set()
487 print('error: Cannot checkout %s' % (project.name),
488 file=sys.stderr)
489 raise _CheckoutError()
490
Mike Frysinger3538dd22019-08-26 15:32:06 -0400491 pm.update(msg=project.name)
Xin Li745be2e2019-06-03 11:24:30 -0700492 except _CheckoutError:
493 pass
494 except Exception as e:
495 print('error: Cannot checkout %s: %s: %s' %
496 (project.name, type(e).__name__, str(e)),
497 file=sys.stderr)
498 err_event.set()
499 raise
500 finally:
501 if did_lock:
Vadim Bendeburydff91942019-11-06 11:05:00 -0800502 if not success:
503 err_results.append(project.relpath)
Xin Li745be2e2019-06-03 11:24:30 -0700504 lock.release()
505 finish = time.time()
506 self.event_log.AddSync(project, event_log.TASK_SYNC_LOCAL,
507 start, finish, success)
508
509 return success
510
Mike Frysinger5a033082019-09-23 19:21:20 -0400511 def _Checkout(self, all_projects, opt, err_event, err_results):
Xin Li745be2e2019-06-03 11:24:30 -0700512 """Checkout projects listed in all_projects
513
514 Args:
515 all_projects: List of all projects that should be checked out.
516 opt: Program options returned from optparse. See _Options().
Mike Frysinger5a033082019-09-23 19:21:20 -0400517 err_event: We'll set this event in the case of an error (after printing
518 out info about the error).
519 err_results: A list of strings, paths to git repos where checkout
520 failed.
Xin Li745be2e2019-06-03 11:24:30 -0700521 """
522
523 # Perform checkouts in multiple threads when we are using partial clone.
524 # Without partial clone, all needed git objects are already downloaded,
525 # in this situation it's better to use only one process because the checkout
526 # would be mostly disk I/O; with partial clone, the objects are only
527 # downloaded when demanded (at checkout time), which is similar to the
528 # Sync_NetworkHalf case and parallelism would be helpful.
529 if self.manifest.CloneFilter:
530 syncjobs = self.jobs
531 else:
532 syncjobs = 1
533
534 lock = _threading.Lock()
Mike Frysinger3538dd22019-08-26 15:32:06 -0400535 pm = Progress('Checking out projects', len(all_projects))
Xin Li745be2e2019-06-03 11:24:30 -0700536
537 threads = set()
538 sem = _threading.Semaphore(syncjobs)
Xin Li745be2e2019-06-03 11:24:30 -0700539
540 for project in all_projects:
541 # Check for any errors before running any more tasks.
542 # ...we'll let existing threads finish, though.
Mike Frysingerd9e5cf02019-08-26 03:12:55 -0400543 if err_event.isSet() and opt.fail_fast:
Xin Li745be2e2019-06-03 11:24:30 -0700544 break
545
546 sem.acquire()
547 if project.worktree:
548 kwargs = dict(opt=opt,
549 sem=sem,
550 project=project,
551 lock=lock,
552 pm=pm,
Vadim Bendeburydff91942019-11-06 11:05:00 -0800553 err_event=err_event,
554 err_results=err_results)
Xin Li745be2e2019-06-03 11:24:30 -0700555 if syncjobs > 1:
556 t = _threading.Thread(target=self._CheckoutWorker,
557 kwargs=kwargs)
558 # Ensure that Ctrl-C will not freeze the repo process.
559 t.daemon = True
560 threads.add(t)
561 t.start()
562 else:
563 self._CheckoutWorker(**kwargs)
564
565 for t in threads:
566 t.join()
567
568 pm.end()
Xin Li745be2e2019-06-03 11:24:30 -0700569
Mike Frysinger5a033082019-09-23 19:21:20 -0400570 def _GCProjects(self, projects, opt, err_event):
Gabe Black2ff30292014-10-09 17:54:35 -0700571 gc_gitdirs = {}
David James8d201162013-10-11 17:03:19 -0700572 for project in projects:
Mike Frysinger5f2b0452020-02-11 03:23:24 -0500573 # Make sure pruning never kicks in with shared projects.
Mike Frysinger979d5bd2020-02-09 02:28:34 -0500574 if (not project.use_git_worktrees and
David Pursehouseaa611a22020-02-20 10:47:26 +0900575 len(project.manifest.GetProjectsWithName(project.name)) > 1):
Mike Frysinger5f2b0452020-02-11 03:23:24 -0500576 print('%s: Shared project %s found, disabling pruning.' %
577 (project.relpath, project.name))
578 if git_require((2, 7, 0)):
Mike Frysingerf81c72e2020-02-19 15:50:00 -0500579 project.EnableRepositoryExtension('preciousObjects')
Mike Frysinger5f2b0452020-02-11 03:23:24 -0500580 else:
581 # This isn't perfect, but it's the best we can do with old git.
582 print('%s: WARNING: shared projects are unreliable when using old '
583 'versions of git; please upgrade to git-2.7.0+.'
584 % (project.relpath,),
585 file=sys.stderr)
586 project.config.SetString('gc.pruneExpire', 'never')
Gabe Black2ff30292014-10-09 17:54:35 -0700587 gc_gitdirs[project.gitdir] = project.bare_git
David James8d201162013-10-11 17:03:19 -0700588
Mike Frysinger6f1c6262020-02-04 00:09:23 -0500589 if multiprocessing:
Dave Borowitz18857212012-10-23 17:02:59 -0700590 cpu_count = multiprocessing.cpu_count()
591 else:
592 cpu_count = 1
593 jobs = min(self.jobs, cpu_count)
594
595 if jobs < 2:
Gabe Black2ff30292014-10-09 17:54:35 -0700596 for bare_git in gc_gitdirs.values():
David James8d201162013-10-11 17:03:19 -0700597 bare_git.gc('--auto')
Dave Borowitz18857212012-10-23 17:02:59 -0700598 return
599
Mike Frysinger0c0e9342019-06-13 12:42:39 -0400600 config = {'pack.threads': cpu_count // jobs if cpu_count > jobs else 1}
Dave Borowitz18857212012-10-23 17:02:59 -0700601
602 threads = set()
603 sem = _threading.Semaphore(jobs)
Dave Borowitz18857212012-10-23 17:02:59 -0700604
David James8d201162013-10-11 17:03:19 -0700605 def GC(bare_git):
Dave Borowitz18857212012-10-23 17:02:59 -0700606 try:
607 try:
David James8d201162013-10-11 17:03:19 -0700608 bare_git.gc('--auto', config=config)
Dave Borowitz18857212012-10-23 17:02:59 -0700609 except GitError:
610 err_event.set()
David Pursehouse145e35b2020-02-12 15:40:47 +0900611 except Exception:
Dave Borowitz18857212012-10-23 17:02:59 -0700612 err_event.set()
613 raise
614 finally:
615 sem.release()
616
Gabe Black2ff30292014-10-09 17:54:35 -0700617 for bare_git in gc_gitdirs.values():
Mike Frysinger5a033082019-09-23 19:21:20 -0400618 if err_event.isSet() and opt.fail_fast:
Dave Borowitz18857212012-10-23 17:02:59 -0700619 break
620 sem.acquire()
David James8d201162013-10-11 17:03:19 -0700621 t = _threading.Thread(target=GC, args=(bare_git,))
Dave Borowitz18857212012-10-23 17:02:59 -0700622 t.daemon = True
623 threads.add(t)
624 t.start()
625
626 for t in threads:
627 t.join()
628
Tim Kilbourn07669002013-03-08 15:02:49 -0800629 def _ReloadManifest(self, manifest_name=None):
630 if manifest_name:
631 # Override calls _Unload already
632 self.manifest.Override(manifest_name)
633 else:
634 self.manifest._Unload()
635
Oleksii Okolielovd3c0f592018-12-17 19:23:44 -0500636 def UpdateProjectList(self, opt):
Jaikumar Ganesh4f2517f2009-06-01 21:10:33 -0700637 new_project_paths = []
Colin Cross5acde752012-03-28 20:15:45 -0700638 for project in self.GetProjects(None, missing_ok=True):
Shawn O. Pearce3a68bb42009-06-04 16:18:09 -0700639 if project.relpath:
640 new_project_paths.append(project.relpath)
Jaikumar Ganesh4f2517f2009-06-01 21:10:33 -0700641 file_name = 'project.list'
642 file_path = os.path.join(self.manifest.repodir, file_name)
643 old_project_paths = []
644
645 if os.path.exists(file_path):
Mike Frysinger3164d402019-11-11 05:40:22 -0500646 with open(file_path, 'r') as fd:
Jaikumar Ganesh4f2517f2009-06-01 21:10:33 -0700647 old_project_paths = fd.read().split('\n')
Kuang-che Wu0d9b16d2019-04-06 00:49:47 +0800648 # In reversed order, so subfolders are deleted before parent folder.
649 for path in sorted(old_project_paths, reverse=True):
Shawn O. Pearce3a68bb42009-06-04 16:18:09 -0700650 if not path:
651 continue
Jaikumar Ganesh4f2517f2009-06-01 21:10:33 -0700652 if path not in new_project_paths:
David Pursehouse8a68ff92012-09-24 12:15:13 +0900653 # If the path has already been deleted, we don't need to do it
Dan Willemsen43507912016-09-01 16:26:02 -0700654 gitdir = os.path.join(self.manifest.topdir, path, '.git')
655 if os.path.exists(gitdir):
David Pursehousec1b86a22012-11-14 11:36:51 +0900656 project = Project(
David Pursehouseabdf7502020-02-12 14:58:39 +0900657 manifest=self.manifest,
658 name=path,
659 remote=RemoteSpec('origin'),
660 gitdir=gitdir,
661 objdir=gitdir,
Mike Frysingerc0d18662020-02-19 19:19:18 -0500662 use_git_worktrees=os.path.isfile(gitdir),
David Pursehouseabdf7502020-02-12 14:58:39 +0900663 worktree=os.path.join(self.manifest.topdir, path),
664 relpath=path,
665 revisionExpr='HEAD',
666 revisionId=None,
667 groups=None)
Mike Frysingerc0d18662020-02-19 19:19:18 -0500668 if not project.DeleteWorktree(
David Pursehouseaa611a22020-02-20 10:47:26 +0900669 quiet=opt.quiet,
670 force=opt.force_remove_dirty):
Mike Frysingera850ca22019-08-07 17:19:24 -0400671 return 1
Jaikumar Ganesh4f2517f2009-06-01 21:10:33 -0700672
Shawn O. Pearce9fb29ce2009-06-04 20:41:02 -0700673 new_project_paths.sort()
Mike Frysinger3164d402019-11-11 05:40:22 -0500674 with open(file_path, 'w') as fd:
Jaikumar Ganesh4f2517f2009-06-01 21:10:33 -0700675 fd.write('\n'.join(new_project_paths))
Shawn O. Pearce3a68bb42009-06-04 16:18:09 -0700676 fd.write('\n')
Jaikumar Ganesh4f2517f2009-06-01 21:10:33 -0700677 return 0
678
Mike Frysinger01d6c3c2019-08-27 01:56:43 -0400679 def _SmartSyncSetup(self, opt, smart_sync_manifest_path):
680 if not self.manifest.manifest_server:
681 print('error: cannot smart sync: no manifest server defined in '
682 'manifest', file=sys.stderr)
683 sys.exit(1)
684
685 manifest_server = self.manifest.manifest_server
686 if not opt.quiet:
687 print('Using manifest server %s' % manifest_server)
688
David Pursehouseeeff3532020-02-12 11:24:10 +0900689 if '@' not in manifest_server:
Mike Frysinger01d6c3c2019-08-27 01:56:43 -0400690 username = None
691 password = None
692 if opt.manifest_server_username and opt.manifest_server_password:
693 username = opt.manifest_server_username
694 password = opt.manifest_server_password
695 else:
696 try:
697 info = netrc.netrc()
698 except IOError:
699 # .netrc file does not exist or could not be opened
700 pass
701 else:
702 try:
703 parse_result = urllib.parse.urlparse(manifest_server)
704 if parse_result.hostname:
705 auth = info.authenticators(parse_result.hostname)
706 if auth:
707 username, _account, password = auth
708 else:
709 print('No credentials found for %s in .netrc'
710 % parse_result.hostname, file=sys.stderr)
711 except netrc.NetrcParseError as e:
712 print('Error parsing .netrc file: %s' % e, file=sys.stderr)
713
714 if (username and password):
715 manifest_server = manifest_server.replace('://', '://%s:%s@' %
716 (username, password),
717 1)
718
719 transport = PersistentTransport(manifest_server)
720 if manifest_server.startswith('persistent-'):
721 manifest_server = manifest_server[len('persistent-'):]
722
723 try:
724 server = xmlrpc.client.Server(manifest_server, transport=transport)
725 if opt.smart_sync:
726 p = self.manifest.manifestProject
727 b = p.GetBranch(p.CurrentBranch)
728 branch = b.merge
729 if branch.startswith(R_HEADS):
730 branch = branch[len(R_HEADS):]
731
Mike Frysinger56ce3462019-12-04 19:30:48 -0500732 if 'SYNC_TARGET' in os.environ:
Mike Frysingere20da3e2020-03-07 01:53:53 -0500733 target = os.environ['SYNC_TARGET']
Mike Frysinger01d6c3c2019-08-27 01:56:43 -0400734 [success, manifest_str] = server.GetApprovedManifest(branch, target)
Mike Frysinger56ce3462019-12-04 19:30:48 -0500735 elif ('TARGET_PRODUCT' in os.environ and
736 'TARGET_BUILD_VARIANT' in os.environ):
Mike Frysingere20da3e2020-03-07 01:53:53 -0500737 target = '%s-%s' % (os.environ['TARGET_PRODUCT'],
738 os.environ['TARGET_BUILD_VARIANT'])
Mike Frysinger01d6c3c2019-08-27 01:56:43 -0400739 [success, manifest_str] = server.GetApprovedManifest(branch, target)
740 else:
741 [success, manifest_str] = server.GetApprovedManifest(branch)
742 else:
743 assert(opt.smart_tag)
744 [success, manifest_str] = server.GetManifest(opt.smart_tag)
745
746 if success:
747 manifest_name = os.path.basename(smart_sync_manifest_path)
748 try:
Mike Frysinger3164d402019-11-11 05:40:22 -0500749 with open(smart_sync_manifest_path, 'w') as f:
Mike Frysinger01d6c3c2019-08-27 01:56:43 -0400750 f.write(manifest_str)
Mike Frysinger01d6c3c2019-08-27 01:56:43 -0400751 except IOError as e:
752 print('error: cannot write manifest to %s:\n%s'
753 % (smart_sync_manifest_path, e),
754 file=sys.stderr)
755 sys.exit(1)
756 self._ReloadManifest(manifest_name)
757 else:
758 print('error: manifest server RPC call failed: %s' %
759 manifest_str, file=sys.stderr)
760 sys.exit(1)
761 except (socket.error, IOError, xmlrpc.client.Fault) as e:
762 print('error: cannot connect to manifest server %s:\n%s'
763 % (self.manifest.manifest_server, e), file=sys.stderr)
764 sys.exit(1)
765 except xmlrpc.client.ProtocolError as e:
766 print('error: cannot connect to manifest server %s:\n%d %s'
767 % (self.manifest.manifest_server, e.errcode, e.errmsg),
768 file=sys.stderr)
769 sys.exit(1)
770
771 return manifest_name
772
Mike Frysingerfb527e32019-08-27 02:34:32 -0400773 def _UpdateManifestProject(self, opt, mp, manifest_name):
774 """Fetch & update the local manifest project."""
775 if not opt.local_only:
776 start = time.time()
Mike Frysinger521d01b2020-02-17 01:51:49 -0500777 success = mp.Sync_NetworkHalf(quiet=opt.quiet, verbose=opt.verbose,
Mike Frysingerfb527e32019-08-27 02:34:32 -0400778 current_branch_only=opt.current_branch_only,
Erwan Yvindc5c4d12019-06-18 13:49:12 +0200779 force_sync=opt.force_sync,
Mike Frysingerc58ec4d2020-02-17 14:36:08 -0500780 tags=opt.tags,
Mike Frysingerfb527e32019-08-27 02:34:32 -0400781 optimized_fetch=opt.optimized_fetch,
George Engelbrecht9bc283e2020-04-02 12:36:09 -0600782 retry_fetches=opt.retry_fetches,
Mike Frysingerfb527e32019-08-27 02:34:32 -0400783 submodules=self.manifest.HasSubmodules,
784 clone_filter=self.manifest.CloneFilter)
785 finish = time.time()
786 self.event_log.AddSync(mp, event_log.TASK_SYNC_NETWORK,
787 start, finish, success)
788
789 if mp.HasChanges:
790 syncbuf = SyncBuffer(mp.config)
791 start = time.time()
792 mp.Sync_LocalHalf(syncbuf, submodules=self.manifest.HasSubmodules)
793 clean = syncbuf.Finish()
794 self.event_log.AddSync(mp, event_log.TASK_SYNC_LOCAL,
795 start, time.time(), clean)
796 if not clean:
797 sys.exit(1)
798 self._ReloadManifest(opt.manifest_name)
799 if opt.jobs is None:
800 self.jobs = self.manifest.default.sync_j
801
Mike Frysingerae6cb082019-08-27 01:10:59 -0400802 def ValidateOptions(self, opt, args):
803 if opt.force_broken:
804 print('warning: -f/--force-broken is now the default behavior, and the '
805 'options are deprecated', file=sys.stderr)
806 if opt.network_only and opt.detach_head:
807 self.OptionParser.error('cannot combine -n and -d')
808 if opt.network_only and opt.local_only:
809 self.OptionParser.error('cannot combine -n and -l')
810 if opt.manifest_name and opt.smart_sync:
811 self.OptionParser.error('cannot combine -m and -s')
812 if opt.manifest_name and opt.smart_tag:
813 self.OptionParser.error('cannot combine -m and -t')
814 if opt.manifest_server_username or opt.manifest_server_password:
815 if not (opt.smart_sync or opt.smart_tag):
816 self.OptionParser.error('-u and -p may only be combined with -s or -t')
817 if None in [opt.manifest_server_username, opt.manifest_server_password]:
818 self.OptionParser.error('both -u and -p must be given')
819
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700820 def Execute(self, opt, args):
Roy Lee18afd7f2010-05-09 04:32:08 +0800821 if opt.jobs:
822 self.jobs = opt.jobs
Shawn O. Pearce97d2b2f2011-09-22 17:23:41 -0700823 if self.jobs > 1:
824 soft_limit, _ = _rlimit_nofile()
Mike Frysinger0c0e9342019-06-13 12:42:39 -0400825 self.jobs = min(self.jobs, (soft_limit - 5) // 3)
Shawn O. Pearce97d2b2f2011-09-22 17:23:41 -0700826
Mike Frysinger521d01b2020-02-17 01:51:49 -0500827 opt.quiet = opt.output_mode is False
828 opt.verbose = opt.output_mode is True
829
Chris Wolfee9dc3b32012-01-26 11:36:18 -0500830 if opt.manifest_name:
831 self.manifest.Override(opt.manifest_name)
Shawn O. Pearce3e768c92009-04-10 16:59:36 -0700832
Chirayu Desaia892b102013-06-11 14:18:46 +0530833 manifest_name = opt.manifest_name
David Pursehouse59b41742015-05-07 14:36:09 +0900834 smart_sync_manifest_path = os.path.join(
David Pursehouseabdf7502020-02-12 14:58:39 +0900835 self.manifest.manifestProject.worktree, 'smart_sync_override.xml')
Chirayu Desaia892b102013-06-11 14:18:46 +0530836
Xin Lid79a4bc2020-05-20 16:03:45 -0700837 if opt.clone_bundle is None:
838 opt.clone_bundle = self.manifest.CloneBundle
839
Victor Boivie08c880d2011-04-19 10:32:52 +0200840 if opt.smart_sync or opt.smart_tag:
Mike Frysinger01d6c3c2019-08-27 01:56:43 -0400841 manifest_name = self._SmartSyncSetup(opt, smart_sync_manifest_path)
842 else:
David Pursehouse59b41742015-05-07 14:36:09 +0900843 if os.path.isfile(smart_sync_manifest_path):
844 try:
Renaud Paquay010fed72016-11-11 14:25:29 -0800845 platform_utils.remove(smart_sync_manifest_path)
David Pursehouse59b41742015-05-07 14:36:09 +0900846 except OSError as e:
847 print('error: failed to remove existing smart sync override manifest: %s' %
848 e, file=sys.stderr)
Nico Sallembiena1bfd2c2010-04-06 10:40:01 -0700849
Mike Frysinger5a033082019-09-23 19:21:20 -0400850 err_event = _threading.Event()
851
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700852 rp = self.manifest.repoProject
853 rp.PreSync()
Mike Frysinger23c900f2020-02-29 02:52:44 -0500854 cb = rp.CurrentBranch
855 if cb:
856 base = rp.GetBranch(cb).merge
857 if not base or not base.startswith('refs/heads/'):
858 print('warning: repo is not tracking a remote branch, so it will not '
Mike Frysinger58ac1672020-03-14 14:35:26 -0400859 'receive updates; run `repo init --repo-rev=stable` to fix.',
Mike Frysinger23c900f2020-02-29 02:52:44 -0500860 file=sys.stderr)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700861
862 mp = self.manifest.manifestProject
863 mp.PreSync()
864
Shawn O. Pearcec9ef7442008-11-03 10:32:09 -0800865 if opt.repo_upgraded:
Shawn O. Pearce80d2ceb2012-10-26 12:23:05 -0700866 _PostRepoUpgrade(self.manifest, quiet=opt.quiet)
Shawn O. Pearcec9ef7442008-11-03 10:32:09 -0800867
Fredrik de Grootcc960972019-11-22 09:04:31 +0100868 if not opt.mp_update:
869 print('Skipping update of local manifest project.')
870 else:
871 self._UpdateManifestProject(opt, mp, manifest_name)
Simran Basib9a1b732015-08-20 12:19:28 -0700872
Simran Basib9a1b732015-08-20 12:19:28 -0700873 if self.gitc_manifest:
874 gitc_manifest_projects = self.GetProjects(args,
Simran Basib9a1b732015-08-20 12:19:28 -0700875 missing_ok=True)
876 gitc_projects = []
877 opened_projects = []
878 for project in gitc_manifest_projects:
Dan Willemsen5ea32d12015-09-08 13:27:20 -0700879 if project.relpath in self.gitc_manifest.paths and \
880 self.gitc_manifest.paths[project.relpath].old_revision:
881 opened_projects.append(project.relpath)
Simran Basib9a1b732015-08-20 12:19:28 -0700882 else:
Dan Willemsen5ea32d12015-09-08 13:27:20 -0700883 gitc_projects.append(project.relpath)
Simran Basib9a1b732015-08-20 12:19:28 -0700884
Dan Willemsen5ea32d12015-09-08 13:27:20 -0700885 if not args:
886 gitc_projects = None
887
888 if gitc_projects != [] and not opt.local_only:
Simran Basib9a1b732015-08-20 12:19:28 -0700889 print('Updating GITC client: %s' % self.gitc_manifest.gitc_client_name)
Dan Willemsen5ea32d12015-09-08 13:27:20 -0700890 manifest = GitcManifest(self.repodir, self.gitc_manifest.gitc_client_name)
891 if manifest_name:
892 manifest.Override(manifest_name)
893 else:
894 manifest.Override(self.manifest.manifestFile)
895 gitc_utils.generate_gitc_manifest(self.gitc_manifest,
896 manifest,
Simran Basib9a1b732015-08-20 12:19:28 -0700897 gitc_projects)
898 print('GITC client successfully synced.')
899
900 # The opened projects need to be synced as normal, therefore we
901 # generate a new args list to represent the opened projects.
Dan Willemsen5ea32d12015-09-08 13:27:20 -0700902 # TODO: make this more reliable -- if there's a project name/path overlap,
903 # this may choose the wrong project.
David Pursehouse3bcd3052017-07-10 22:42:22 +0900904 args = [os.path.relpath(self.manifest.paths[path].worktree, os.getcwd())
905 for path in opened_projects]
Simran Basib9a1b732015-08-20 12:19:28 -0700906 if not args:
907 return
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +0800908 all_projects = self.GetProjects(args,
909 missing_ok=True,
910 submodules_ok=opt.fetch_submodules)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700911
Mike Frysinger5a033082019-09-23 19:21:20 -0400912 err_network_sync = False
913 err_update_projects = False
914 err_checkout = False
915
Dave Borowitz67700e92012-10-23 15:00:54 -0700916 self._fetch_times = _FetchTimes(self.manifest)
Shawn O. Pearceb1562fa2009-04-10 17:04:08 -0700917 if not opt.local_only:
Shawn O. Pearcef6906872009-04-18 10:49:00 -0700918 to_fetch = []
919 now = time.time()
Dave Borowitz67700e92012-10-23 15:00:54 -0700920 if _ONE_DAY_S <= (now - rp.LastFetch):
Shawn O. Pearcef6906872009-04-18 10:49:00 -0700921 to_fetch.append(rp)
David Pursehouse8a68ff92012-09-24 12:15:13 +0900922 to_fetch.extend(all_projects)
Dave Borowitz67700e92012-10-23 15:00:54 -0700923 to_fetch.sort(key=self._fetch_times.Get, reverse=True)
Shawn O. Pearcef6906872009-04-18 10:49:00 -0700924
Mike Frysinger5a033082019-09-23 19:21:20 -0400925 fetched = self._Fetch(to_fetch, opt, err_event)
926
Mike Frysingerc58ec4d2020-02-17 14:36:08 -0500927 _PostRepoFetch(rp, opt.repo_verify)
Shawn O. Pearceb1562fa2009-04-10 17:04:08 -0700928 if opt.network_only:
929 # bail out now; the rest touches the working tree
Mike Frysinger5a033082019-09-23 19:21:20 -0400930 if err_event.isSet():
931 print('\nerror: Exited sync due to fetch errors.\n', file=sys.stderr)
932 sys.exit(1)
Shawn O. Pearceb1562fa2009-04-10 17:04:08 -0700933 return
934
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +0800935 # Iteratively fetch missing and/or nested unregistered submodules
936 previously_missing_set = set()
937 while True:
Victor Boivie53a6c5d2013-03-19 12:20:52 +0100938 self._ReloadManifest(manifest_name)
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +0800939 all_projects = self.GetProjects(args,
940 missing_ok=True,
941 submodules_ok=opt.fetch_submodules)
942 missing = []
943 for project in all_projects:
944 if project.gitdir not in fetched:
945 missing.append(project)
946 if not missing:
947 break
948 # Stop us from non-stopped fetching actually-missing repos: If set of
949 # missing repos has not been changed from last fetch, we break.
950 missing_set = set(p.name for p in missing)
951 if previously_missing_set == missing_set:
952 break
953 previously_missing_set = missing_set
Mike Frysinger5a033082019-09-23 19:21:20 -0400954 fetched.update(self._Fetch(missing, opt, err_event))
955
956 # If we saw an error, exit with code 1 so that other scripts can check.
957 if err_event.isSet():
958 err_network_sync = True
959 if opt.fail_fast:
960 print('\nerror: Exited sync due to fetch errors.\n'
961 'Local checkouts *not* updated. Resolve network issues & '
962 'retry.\n'
963 '`repo sync -l` will update some local checkouts.',
964 file=sys.stderr)
965 sys.exit(1)
Che-Liang Chioub2bd91c2012-01-11 11:28:42 +0800966
Julien Campergue335f5ef2013-10-16 11:02:35 +0200967 if self.manifest.IsMirror or self.manifest.IsArchive:
Shawn O. Pearcecd1d7ff2009-06-04 16:15:53 -0700968 # bail out now, we have no working tree
969 return
970
Oleksii Okolielovd3c0f592018-12-17 19:23:44 -0500971 if self.UpdateProjectList(opt):
Mike Frysinger5a033082019-09-23 19:21:20 -0400972 err_event.set()
973 err_update_projects = True
974 if opt.fail_fast:
975 print('\nerror: Local checkouts *not* updated.', file=sys.stderr)
976 sys.exit(1)
Jaikumar Ganesh4f2517f2009-06-01 21:10:33 -0700977
Mike Frysinger5a033082019-09-23 19:21:20 -0400978 err_results = []
979 self._Checkout(all_projects, opt, err_event, err_results)
980 if err_event.isSet():
981 err_checkout = True
982 # NB: We don't exit here because this is the last step.
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -0700983
Doug Anderson2b8db3c2010-11-01 15:08:06 -0700984 # If there's a notice that's supposed to print at the end of the sync, print
985 # it now...
986 if self.manifest.notice:
Sarah Owenscecd1d82012-11-01 22:59:27 -0700987 print(self.manifest.notice)
Doug Anderson2b8db3c2010-11-01 15:08:06 -0700988
Mike Frysinger5a033082019-09-23 19:21:20 -0400989 # If we saw an error, exit with code 1 so that other scripts can check.
990 if err_event.isSet():
991 print('\nerror: Unable to fully sync the tree.', file=sys.stderr)
992 if err_network_sync:
993 print('error: Downloading network changes failed.', file=sys.stderr)
994 if err_update_projects:
995 print('error: Updating local project lists failed.', file=sys.stderr)
996 if err_checkout:
997 print('error: Checking out local projects failed.', file=sys.stderr)
998 if err_results:
999 print('Failing repos:\n%s' % '\n'.join(err_results), file=sys.stderr)
1000 print('Try re-running with "-j1 --fail-fast" to exit at the first error.',
1001 file=sys.stderr)
1002 sys.exit(1)
1003
Mike Frysingere19d9e12020-02-12 11:23:32 -05001004 if not opt.quiet:
1005 print('repo sync has finished successfully.')
1006
David Pursehouse819827a2020-02-12 15:20:19 +09001007
Shawn O. Pearce80d2ceb2012-10-26 12:23:05 -07001008def _PostRepoUpgrade(manifest, quiet=False):
Conley Owens094cdbe2014-01-30 15:09:59 -08001009 wrapper = Wrapper()
Conley Owensc9129d92012-10-01 16:12:28 -07001010 if wrapper.NeedSetupGnuPG():
Shawn O. Pearce80d2ceb2012-10-26 12:23:05 -07001011 wrapper.SetupGnuPG(quiet)
Conley Owensf2fe2d92014-01-29 13:53:43 -08001012 for project in manifest.projects:
Shawn O. Pearcee756c412009-04-13 11:51:15 -07001013 if project.Exists:
1014 project.PostRepoUpgrade()
1015
David Pursehouse819827a2020-02-12 15:20:19 +09001016
Mike Frysingerc58ec4d2020-02-17 14:36:08 -05001017def _PostRepoFetch(rp, repo_verify=True, verbose=False):
Shawn O. Pearcee756c412009-04-13 11:51:15 -07001018 if rp.HasChanges:
Sarah Owenscecd1d82012-11-01 22:59:27 -07001019 print('info: A new version of repo is available', file=sys.stderr)
1020 print(file=sys.stderr)
Mike Frysingerc58ec4d2020-02-17 14:36:08 -05001021 if not repo_verify or _VerifyTag(rp):
Shawn O. Pearce350cde42009-04-16 11:21:18 -07001022 syncbuf = SyncBuffer(rp.config)
1023 rp.Sync_LocalHalf(syncbuf)
1024 if not syncbuf.Finish():
Shawn O. Pearcee756c412009-04-13 11:51:15 -07001025 sys.exit(1)
Sarah Owenscecd1d82012-11-01 22:59:27 -07001026 print('info: Restarting repo with latest version', file=sys.stderr)
Shawn O. Pearcee756c412009-04-13 11:51:15 -07001027 raise RepoChangedException(['--repo-upgraded'])
1028 else:
Sarah Owenscecd1d82012-11-01 22:59:27 -07001029 print('warning: Skipped upgrade to unverified version', file=sys.stderr)
Shawn O. Pearcee756c412009-04-13 11:51:15 -07001030 else:
1031 if verbose:
Sarah Owenscecd1d82012-11-01 22:59:27 -07001032 print('repo version %s is current' % rp.work_git.describe(HEAD),
1033 file=sys.stderr)
Shawn O. Pearcee756c412009-04-13 11:51:15 -07001034
David Pursehouse819827a2020-02-12 15:20:19 +09001035
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001036def _VerifyTag(project):
1037 gpg_dir = os.path.expanduser('~/.repoconfig/gnupg')
1038 if not os.path.exists(gpg_dir):
Sarah Owenscecd1d82012-11-01 22:59:27 -07001039 print('warning: GnuPG was not available during last "repo init"\n'
1040 'warning: Cannot automatically authenticate repo."""',
1041 file=sys.stderr)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001042 return True
1043
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001044 try:
Shawn O. Pearce3c8dea12009-05-29 18:38:17 -07001045 cur = project.bare_git.describe(project.GetRevisionId())
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001046 except GitError:
1047 cur = None
1048
1049 if not cur \
1050 or re.compile(r'^.*-[0-9]{1,}-g[0-9a-f]{1,}$').match(cur):
Shawn O. Pearce3c8dea12009-05-29 18:38:17 -07001051 rev = project.revisionExpr
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001052 if rev.startswith(R_HEADS):
1053 rev = rev[len(R_HEADS):]
1054
Sarah Owenscecd1d82012-11-01 22:59:27 -07001055 print(file=sys.stderr)
1056 print("warning: project '%s' branch '%s' is not signed"
1057 % (project.name, rev), file=sys.stderr)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001058 return False
1059
Shawn O. Pearcef18cb762010-12-07 11:41:05 -08001060 env = os.environ.copy()
Mike Frysinger56ce3462019-12-04 19:30:48 -05001061 env['GIT_DIR'] = project.gitdir
1062 env['GNUPGHOME'] = gpg_dir
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001063
1064 cmd = [GIT, 'tag', '-v', cur]
1065 proc = subprocess.Popen(cmd,
David Pursehousee5913ae2020-02-12 13:56:59 +09001066 stdout=subprocess.PIPE,
1067 stderr=subprocess.PIPE,
1068 env=env)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001069 out = proc.stdout.read()
1070 proc.stdout.close()
1071
1072 err = proc.stderr.read()
1073 proc.stderr.close()
1074
1075 if proc.wait() != 0:
Sarah Owenscecd1d82012-11-01 22:59:27 -07001076 print(file=sys.stderr)
1077 print(out, file=sys.stderr)
1078 print(err, file=sys.stderr)
1079 print(file=sys.stderr)
The Android Open Source Projectcf31fe92008-10-21 07:00:00 -07001080 return False
1081 return True
Dave Borowitz67700e92012-10-23 15:00:54 -07001082
David Rileye0684ad2017-04-05 00:02:59 -07001083
Dave Borowitz67700e92012-10-23 15:00:54 -07001084class _FetchTimes(object):
Dave Borowitzd9478582012-10-23 16:35:39 -07001085 _ALPHA = 0.5
1086
Dave Borowitz67700e92012-10-23 15:00:54 -07001087 def __init__(self, manifest):
Anthony King85b24ac2014-05-06 15:57:48 +01001088 self._path = os.path.join(manifest.repodir, '.repo_fetchtimes.json')
Dave Borowitz67700e92012-10-23 15:00:54 -07001089 self._times = None
Dave Borowitzd9478582012-10-23 16:35:39 -07001090 self._seen = set()
Dave Borowitz67700e92012-10-23 15:00:54 -07001091
1092 def Get(self, project):
1093 self._Load()
1094 return self._times.get(project.name, _ONE_DAY_S)
1095
1096 def Set(self, project, t):
Dave Borowitzd9478582012-10-23 16:35:39 -07001097 self._Load()
1098 name = project.name
1099 old = self._times.get(name, t)
1100 self._seen.add(name)
1101 a = self._ALPHA
David Pursehouse54a4e602020-02-12 14:31:05 +09001102 self._times[name] = (a * t) + ((1 - a) * old)
Dave Borowitz67700e92012-10-23 15:00:54 -07001103
1104 def _Load(self):
1105 if self._times is None:
1106 try:
Mike Frysinger3164d402019-11-11 05:40:22 -05001107 with open(self._path) as f:
Anthony King85b24ac2014-05-06 15:57:48 +01001108 self._times = json.load(f)
Anthony King85b24ac2014-05-06 15:57:48 +01001109 except (IOError, ValueError):
1110 try:
Renaud Paquay010fed72016-11-11 14:25:29 -08001111 platform_utils.remove(self._path)
Anthony King85b24ac2014-05-06 15:57:48 +01001112 except OSError:
1113 pass
1114 self._times = {}
Dave Borowitz67700e92012-10-23 15:00:54 -07001115
1116 def Save(self):
1117 if self._times is None:
1118 return
Dave Borowitzd9478582012-10-23 16:35:39 -07001119
1120 to_delete = []
1121 for name in self._times:
1122 if name not in self._seen:
1123 to_delete.append(name)
1124 for name in to_delete:
1125 del self._times[name]
1126
Dave Borowitz67700e92012-10-23 15:00:54 -07001127 try:
Mike Frysinger3164d402019-11-11 05:40:22 -05001128 with open(self._path, 'w') as f:
Anthony King85b24ac2014-05-06 15:57:48 +01001129 json.dump(self._times, f, indent=2)
Anthony King85b24ac2014-05-06 15:57:48 +01001130 except (IOError, TypeError):
1131 try:
Renaud Paquay010fed72016-11-11 14:25:29 -08001132 platform_utils.remove(self._path)
Anthony King85b24ac2014-05-06 15:57:48 +01001133 except OSError:
1134 pass
Dan Willemsen0745bb22015-08-17 13:41:45 -07001135
1136# This is a replacement for xmlrpc.client.Transport using urllib2
1137# and supporting persistent-http[s]. It cannot change hosts from
1138# request to request like the normal transport, the real url
1139# is passed during initialization.
David Pursehouse819827a2020-02-12 15:20:19 +09001140
1141
Dan Willemsen0745bb22015-08-17 13:41:45 -07001142class PersistentTransport(xmlrpc.client.Transport):
1143 def __init__(self, orig_host):
1144 self.orig_host = orig_host
1145
1146 def request(self, host, handler, request_body, verbose=False):
1147 with GetUrlCookieFile(self.orig_host, not verbose) as (cookiefile, proxy):
1148 # Python doesn't understand cookies with the #HttpOnly_ prefix
1149 # Since we're only using them for HTTP, copy the file temporarily,
1150 # stripping those prefixes away.
Dan Willemsen3010e5b2015-08-20 10:09:20 -07001151 if cookiefile:
Collin Fijalkoviche1191b32020-02-18 10:57:32 -08001152 tmpcookiefile = tempfile.NamedTemporaryFile(mode='w')
David Pursehouse4c5f74e2015-10-02 11:10:10 +09001153 tmpcookiefile.write("# HTTP Cookie File")
Dan Willemsen3010e5b2015-08-20 10:09:20 -07001154 try:
1155 with open(cookiefile) as f:
1156 for line in f:
1157 if line.startswith("#HttpOnly_"):
1158 line = line[len("#HttpOnly_"):]
1159 tmpcookiefile.write(line)
1160 tmpcookiefile.flush()
Dan Willemsen0745bb22015-08-17 13:41:45 -07001161
Dan Willemsen3010e5b2015-08-20 10:09:20 -07001162 cookiejar = cookielib.MozillaCookieJar(tmpcookiefile.name)
David Pursehouseb1ad2192015-09-30 10:35:43 +09001163 try:
1164 cookiejar.load()
1165 except cookielib.LoadError:
1166 cookiejar = cookielib.CookieJar()
Dan Willemsen3010e5b2015-08-20 10:09:20 -07001167 finally:
1168 tmpcookiefile.close()
1169 else:
1170 cookiejar = cookielib.CookieJar()
Dan Willemsen0745bb22015-08-17 13:41:45 -07001171
1172 proxyhandler = urllib.request.ProxyHandler
1173 if proxy:
1174 proxyhandler = urllib.request.ProxyHandler({
1175 "http": proxy,
David Pursehouse54a4e602020-02-12 14:31:05 +09001176 "https": proxy})
Dan Willemsen0745bb22015-08-17 13:41:45 -07001177
1178 opener = urllib.request.build_opener(
1179 urllib.request.HTTPCookieProcessor(cookiejar),
1180 proxyhandler)
1181
1182 url = urllib.parse.urljoin(self.orig_host, handler)
1183 parse_results = urllib.parse.urlparse(url)
1184
1185 scheme = parse_results.scheme
1186 if scheme == 'persistent-http':
1187 scheme = 'http'
1188 if scheme == 'persistent-https':
1189 # If we're proxying through persistent-https, use http. The
1190 # proxy itself will do the https.
1191 if proxy:
1192 scheme = 'http'
1193 else:
1194 scheme = 'https'
1195
1196 # Parse out any authentication information using the base class
1197 host, extra_headers, _ = self.get_host_info(parse_results.netloc)
1198
1199 url = urllib.parse.urlunparse((
1200 scheme,
1201 host,
1202 parse_results.path,
1203 parse_results.params,
1204 parse_results.query,
1205 parse_results.fragment))
1206
1207 request = urllib.request.Request(url, request_body)
1208 if extra_headers is not None:
1209 for (name, header) in extra_headers:
1210 request.add_header(name, header)
1211 request.add_header('Content-Type', 'text/xml')
1212 try:
1213 response = opener.open(request)
1214 except urllib.error.HTTPError as e:
1215 if e.code == 501:
1216 # We may have been redirected through a login process
1217 # but our POST turned into a GET. Retry.
1218 response = opener.open(request)
1219 else:
1220 raise
1221
1222 p, u = xmlrpc.client.getparser()
1223 while 1:
1224 data = response.read(1024)
1225 if not data:
1226 break
1227 p.feed(data)
1228 p.close()
1229 return u.close()
1230
1231 def close(self):
1232 pass