blob: 22effe13e47e6c6a4881d541ff5222f16b70b6d0 [file] [log] [blame]
Mike Frysinger2de7f042012-07-10 04:45:03 -04001# Copyright (c) 2012 The Chromium OS Authors. All rights reserved.
Brian Harringb938c782012-02-29 15:14:38 -08002# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
Mike Frysinger80dfce92014-04-21 10:58:53 -04005"""This script fetches and prepares an SDK chroot."""
Brian Harringb938c782012-02-29 15:14:38 -08006
Mike Frysinger383367e2014-09-16 15:06:17 -04007from __future__ import print_function
8
Mike Frysinger80dfce92014-04-21 10:58:53 -04009import errno
Josh Triplett472a4182013-03-08 11:48:57 -080010import glob
Brian Harringb938c782012-02-29 15:14:38 -080011import os
Josh Triplett472a4182013-03-08 11:48:57 -080012import pwd
Mike Frysinger80dfce92014-04-21 10:58:53 -040013import signal
David James56e6c2c2012-10-24 23:54:41 -070014import sys
Mike Frysinger80dfce92014-04-21 10:58:53 -040015import time
Brian Harringb938c782012-02-29 15:14:38 -080016import urlparse
17
Don Garrett88b8d782014-05-13 17:30:55 -070018from chromite.cbuildbot import constants
Brian Harringcfe762a2012-02-29 13:03:53 -080019from chromite.lib import cgroups
Brian Harringb6cf9142012-09-01 20:43:17 -070020from chromite.lib import commandline
Brian Harringb938c782012-02-29 15:14:38 -080021from chromite.lib import cros_build_lib
Brian Harringb938c782012-02-29 15:14:38 -080022from chromite.lib import locking
Josh Triplette759b232013-03-08 13:03:43 -080023from chromite.lib import namespaces
Brian Harringae0a5322012-09-15 01:46:51 -070024from chromite.lib import osutils
David Jamesc93e6a4d2014-01-13 11:37:36 -080025from chromite.lib import retry_util
Mike Frysinger8e727a32013-01-16 16:57:53 -050026from chromite.lib import toolchain
Brian Harringb938c782012-02-29 15:14:38 -080027
28cros_build_lib.STRICT_SUDO = True
29
30
Zdenek Behanaa52cea2012-05-30 01:31:11 +020031COMPRESSION_PREFERENCE = ('xz', 'bz2')
Zdenek Behanfd0efe42012-04-13 04:36:40 +020032
Brian Harringb938c782012-02-29 15:14:38 -080033# TODO(zbehan): Remove the dependency on these, reimplement them in python
Mike Frysinger648ba2d2013-01-08 14:19:34 -050034MAKE_CHROOT = [os.path.join(constants.SOURCE_ROOT,
35 'src/scripts/sdk_lib/make_chroot.sh')]
36ENTER_CHROOT = [os.path.join(constants.SOURCE_ROOT,
37 'src/scripts/sdk_lib/enter_chroot.sh')]
Brian Harringb938c782012-02-29 15:14:38 -080038
Josh Triplett472a4182013-03-08 11:48:57 -080039# Proxy simulator configuration.
40PROXY_HOST_IP = '192.168.240.1'
41PROXY_PORT = 8080
42PROXY_GUEST_IP = '192.168.240.2'
43PROXY_NETMASK = 30
44PROXY_VETH_PREFIX = 'veth'
45PROXY_CONNECT_PORTS = (80, 443, 9418)
46PROXY_APACHE_FALLBACK_USERS = ('www-data', 'apache', 'nobody')
47PROXY_APACHE_MPMS = ('event', 'worker', 'prefork')
48PROXY_APACHE_FALLBACK_PATH = ':'.join(
49 '/usr/lib/apache2/mpm-%s' % mpm for mpm in PROXY_APACHE_MPMS
50)
51PROXY_APACHE_MODULE_GLOBS = ('/usr/lib*/apache2/modules', '/usr/lib*/apache2')
52
Josh Triplett9a495f62013-03-15 18:06:55 -070053# We need these tools to run. Very common tools (tar,..) are omitted.
Josh Triplette759b232013-03-08 13:03:43 -080054NEEDED_TOOLS = ('curl', 'xz')
Brian Harringb938c782012-02-29 15:14:38 -080055
Josh Triplett472a4182013-03-08 11:48:57 -080056# Tools needed for --proxy-sim only.
57PROXY_NEEDED_TOOLS = ('ip',)
Brian Harringb938c782012-02-29 15:14:38 -080058
Mike Frysingercc838832014-05-24 13:10:30 -040059
Brian Harring1790ac42012-09-23 08:53:33 -070060def GetArchStageTarballs(version):
Brian Harringb938c782012-02-29 15:14:38 -080061 """Returns the URL for a given arch/version"""
Brian Harring1790ac42012-09-23 08:53:33 -070062 extension = {'bz2':'tbz2', 'xz':'tar.xz'}
Mike Frysinger8e727a32013-01-16 16:57:53 -050063 return [toolchain.GetSdkURL(suburl='cros-sdk-%s.%s'
64 % (version, extension[compressor]))
Brian Harring1790ac42012-09-23 08:53:33 -070065 for compressor in COMPRESSION_PREFERENCE]
66
67
68def GetStage3Urls(version):
Mike Frysinger8e727a32013-01-16 16:57:53 -050069 return [toolchain.GetSdkURL(suburl='stage3-amd64-%s.tar.%s' % (version, ext))
Brian Harring1790ac42012-09-23 08:53:33 -070070 for ext in COMPRESSION_PREFERENCE]
Brian Harringb938c782012-02-29 15:14:38 -080071
72
Brian Harringae0a5322012-09-15 01:46:51 -070073def FetchRemoteTarballs(storage_dir, urls):
Mike Frysinger34db8692013-11-11 14:54:08 -050074 """Fetches a tarball given by url, and place it in |storage_dir|.
Zdenek Behanfd0efe42012-04-13 04:36:40 +020075
76 Args:
Mike Frysinger34db8692013-11-11 14:54:08 -050077 storage_dir: Path where to save the tarball.
Zdenek Behanfd0efe42012-04-13 04:36:40 +020078 urls: List of URLs to try to download. Download will stop on first success.
79
80 Returns:
81 Full path to the downloaded file
82 """
Zdenek Behanfd0efe42012-04-13 04:36:40 +020083
Brian Harring1790ac42012-09-23 08:53:33 -070084 # Note we track content length ourselves since certain versions of curl
85 # fail if asked to resume a complete file.
86 # pylint: disable=C0301,W0631
87 # https://sourceforge.net/tracker/?func=detail&atid=100976&aid=3482927&group_id=976
Zdenek Behanfd0efe42012-04-13 04:36:40 +020088 for url in urls:
Brian Harring1790ac42012-09-23 08:53:33 -070089 # http://www.logilab.org/ticket/8766
90 # pylint: disable=E1101
91 parsed = urlparse.urlparse(url)
92 tarball_name = os.path.basename(parsed.path)
93 if parsed.scheme in ('', 'file'):
94 if os.path.exists(parsed.path):
95 return parsed.path
96 continue
97 content_length = 0
Mike Frysinger383367e2014-09-16 15:06:17 -040098 print('Attempting download: %s' % url)
David Jamesc93e6a4d2014-01-13 11:37:36 -080099 result = retry_util.RunCurl(
Brian Harring1790ac42012-09-23 08:53:33 -0700100 ['-I', url], redirect_stdout=True, redirect_stderr=True,
101 print_cmd=False)
102 successful = False
103 for header in result.output.splitlines():
104 # We must walk the output to find the string '200 OK' for use cases where
105 # a proxy is involved and may have pushed down the actual header.
106 if header.find('200 OK') != -1:
107 successful = True
108 elif header.lower().startswith("content-length:"):
109 content_length = int(header.split(":", 1)[-1].strip())
110 if successful:
111 break
112 if successful:
Zdenek Behanfd0efe42012-04-13 04:36:40 +0200113 break
114 else:
115 raise Exception('No valid URLs found!')
116
Brian Harringae0a5322012-09-15 01:46:51 -0700117 tarball_dest = os.path.join(storage_dir, tarball_name)
Brian Harring1790ac42012-09-23 08:53:33 -0700118 current_size = 0
119 if os.path.exists(tarball_dest):
120 current_size = os.path.getsize(tarball_dest)
121 if current_size > content_length:
David James56e6c2c2012-10-24 23:54:41 -0700122 osutils.SafeUnlink(tarball_dest)
Brian Harring1790ac42012-09-23 08:53:33 -0700123 current_size = 0
Zdenek Behanb2fa72e2012-03-16 04:49:30 +0100124
Brian Harring1790ac42012-09-23 08:53:33 -0700125 if current_size < content_length:
David Jamesc93e6a4d2014-01-13 11:37:36 -0800126 retry_util.RunCurl(
Brian Harring1790ac42012-09-23 08:53:33 -0700127 ['-f', '-L', '-y', '30', '-C', '-', '--output', tarball_dest, url],
128 print_cmd=False)
Brian Harringb938c782012-02-29 15:14:38 -0800129
Brian Harring1790ac42012-09-23 08:53:33 -0700130 # Cleanup old tarballs now since we've successfull fetched; only cleanup
131 # the tarballs for our prefix, or unknown ones.
132 ignored_prefix = ('stage3-' if tarball_name.startswith('cros-sdk-')
133 else 'cros-sdk-')
134 for filename in os.listdir(storage_dir):
135 if filename == tarball_name or filename.startswith(ignored_prefix):
136 continue
Brian Harringb938c782012-02-29 15:14:38 -0800137
Mike Frysinger383367e2014-09-16 15:06:17 -0400138 print('Cleaning up old tarball: %s' % (filename,))
David James56e6c2c2012-10-24 23:54:41 -0700139 osutils.SafeUnlink(os.path.join(storage_dir, filename))
Zdenek Behan9c644dd2012-04-05 06:24:02 +0200140
Brian Harringb938c782012-02-29 15:14:38 -0800141 return tarball_dest
142
143
Brian Harring1790ac42012-09-23 08:53:33 -0700144def CreateChroot(chroot_path, sdk_tarball, cache_dir, nousepkg=False):
Brian Harringb938c782012-02-29 15:14:38 -0800145 """Creates a new chroot from a given SDK"""
Brian Harringb938c782012-02-29 15:14:38 -0800146
Brian Harring1790ac42012-09-23 08:53:33 -0700147 cmd = MAKE_CHROOT + ['--stage3_path', sdk_tarball,
Brian Harringae0a5322012-09-15 01:46:51 -0700148 '--chroot', chroot_path,
149 '--cache_dir', cache_dir]
Mike Frysinger2de7f042012-07-10 04:45:03 -0400150 if nousepkg:
151 cmd.append('--nousepkg')
Brian Harringb938c782012-02-29 15:14:38 -0800152
153 try:
154 cros_build_lib.RunCommand(cmd, print_cmd=False)
155 except cros_build_lib.RunCommandError:
Brian Harring98b54902012-03-23 04:05:42 -0700156 raise SystemExit('Running %r failed!' % cmd)
Brian Harringb938c782012-02-29 15:14:38 -0800157
158
159def DeleteChroot(chroot_path):
160 """Deletes an existing chroot"""
161 cmd = MAKE_CHROOT + ['--chroot', chroot_path,
162 '--delete']
163 try:
164 cros_build_lib.RunCommand(cmd, print_cmd=False)
165 except cros_build_lib.RunCommandError:
Brian Harring98b54902012-03-23 04:05:42 -0700166 raise SystemExit('Running %r failed!' % cmd)
Brian Harringb938c782012-02-29 15:14:38 -0800167
168
Brian Harringae0a5322012-09-15 01:46:51 -0700169def EnterChroot(chroot_path, cache_dir, chrome_root, chrome_root_mount,
170 additional_args):
Brian Harringb938c782012-02-29 15:14:38 -0800171 """Enters an existing SDK chroot"""
Mike Frysingere5456972013-06-13 00:07:23 -0400172 st = os.statvfs(os.path.join(chroot_path, 'usr', 'bin', 'sudo'))
173 # The os.ST_NOSUID constant wasn't added until python-3.2.
174 if st.f_flag & 0x2:
175 cros_build_lib.Die('chroot cannot be in a nosuid mount')
176
Brian Harringae0a5322012-09-15 01:46:51 -0700177 cmd = ENTER_CHROOT + ['--chroot', chroot_path, '--cache_dir', cache_dir]
Brian Harringb938c782012-02-29 15:14:38 -0800178 if chrome_root:
179 cmd.extend(['--chrome_root', chrome_root])
180 if chrome_root_mount:
181 cmd.extend(['--chrome_root_mount', chrome_root_mount])
182 if len(additional_args) > 0:
183 cmd.append('--')
184 cmd.extend(additional_args)
Brian Harring7199e7d2012-03-23 04:10:08 -0700185
186 ret = cros_build_lib.RunCommand(cmd, print_cmd=False, error_code_ok=True)
187 # If we were in interactive mode, ignore the exit code; it'll be whatever
188 # they last ran w/in the chroot and won't matter to us one way or another.
189 # Note this does allow chroot entrance to fail and be ignored during
190 # interactive; this is however a rare case and the user will immediately
191 # see it (nor will they be checking the exit code manually).
192 if ret.returncode != 0 and additional_args:
193 raise SystemExit('Running %r failed with exit code %i'
194 % (cmd, ret.returncode))
Brian Harringb938c782012-02-29 15:14:38 -0800195
196
David James56e6c2c2012-10-24 23:54:41 -0700197def _SudoCommand():
198 """Get the 'sudo' command, along with all needed environment variables."""
199
David James5a73b4d2013-03-07 10:23:40 -0800200 # Pass in the ENVIRONMENT_WHITELIST and ENV_PASSTHRU variables so that
201 # scripts in the chroot know what variables to pass through.
David James56e6c2c2012-10-24 23:54:41 -0700202 cmd = ['sudo']
David James5a73b4d2013-03-07 10:23:40 -0800203 for key in constants.CHROOT_ENVIRONMENT_WHITELIST + constants.ENV_PASSTHRU:
David James56e6c2c2012-10-24 23:54:41 -0700204 value = os.environ.get(key)
205 if value is not None:
206 cmd += ['%s=%s' % (key, value)]
207
208 # Pass in the path to the depot_tools so that users can access them from
209 # within the chroot.
Mike Frysinger08e75f12014-08-13 01:30:09 -0400210 cmd += ['DEPOT_TOOLS=%s' % constants.DEPOT_TOOLS_DIR]
Mike Frysinger749251e2014-01-29 05:04:27 -0500211
David James56e6c2c2012-10-24 23:54:41 -0700212 return cmd
213
214
Josh Triplett472a4182013-03-08 11:48:57 -0800215def _ReportMissing(missing):
216 """Report missing utilities, then exit.
217
218 Args:
219 missing: List of missing utilities, as returned by
220 osutils.FindMissingBinaries. If non-empty, will not return.
221 """
222
223 if missing:
224 raise SystemExit(
225 'The tool(s) %s were not found.\n'
226 'Please install the appropriate package in your host.\n'
227 'Example(ubuntu):\n'
228 ' sudo apt-get install <packagename>'
229 % ', '.join(missing))
230
231
232def _ProxySimSetup(options):
233 """Set up proxy simulator, and return only in the child environment.
234
235 TODO: Ideally, this should support multiple concurrent invocations of
236 cros_sdk --proxy-sim; currently, such invocations will conflict with each
237 other due to the veth device names and IP addresses. Either this code would
238 need to generate fresh, unused names for all of these before forking, or it
239 would need to support multiple concurrent cros_sdk invocations sharing one
240 proxy and allowing it to exit when unused (without counting on any local
241 service-management infrastructure on the host).
242 """
243
244 may_need_mpm = False
245 apache_bin = osutils.Which('apache2')
246 if apache_bin is None:
247 apache_bin = osutils.Which('apache2', PROXY_APACHE_FALLBACK_PATH)
248 if apache_bin is None:
249 _ReportMissing(('apache2',))
250 else:
251 may_need_mpm = True
252
253 # Module names and .so names included for ease of grepping.
254 apache_modules = [('proxy_module', 'mod_proxy.so'),
255 ('proxy_connect_module', 'mod_proxy_connect.so'),
256 ('proxy_http_module', 'mod_proxy_http.so'),
257 ('proxy_ftp_module', 'mod_proxy_ftp.so')]
258
259 # Find the apache module directory, and make sure it has the modules we need.
260 module_dirs = {}
261 for g in PROXY_APACHE_MODULE_GLOBS:
262 for mod, so in apache_modules:
263 for f in glob.glob(os.path.join(g, so)):
264 module_dirs.setdefault(os.path.dirname(f), []).append(so)
265 for apache_module_path, modules_found in module_dirs.iteritems():
266 if len(modules_found) == len(apache_modules):
267 break
268 else:
269 # Appease cros lint, which doesn't understand that this else block will not
270 # fall through to the subsequent code which relies on apache_module_path.
271 apache_module_path = None
272 raise SystemExit(
273 'Could not find apache module path containing all required modules: %s'
274 % ', '.join(so for mod, so in apache_modules))
275
276 def check_add_module(name):
277 so = 'mod_%s.so' % name
278 if os.access(os.path.join(apache_module_path, so), os.F_OK):
279 mod = '%s_module' % name
280 apache_modules.append((mod, so))
281 return True
282 return False
283
284 check_add_module('authz_core')
285 if may_need_mpm:
286 for mpm in PROXY_APACHE_MPMS:
287 if check_add_module('mpm_%s' % mpm):
288 break
289
290 veth_host = '%s-host' % PROXY_VETH_PREFIX
291 veth_guest = '%s-guest' % PROXY_VETH_PREFIX
292
293 # Set up pipes from parent to child and vice versa.
294 # The child writes a byte to the parent after calling unshare, so that the
295 # parent can then assign the guest end of the veth interface to the child's
296 # new network namespace. The parent then writes a byte to the child after
297 # assigning the guest interface, so that the child can then configure that
298 # interface. In both cases, if we get back an EOF when reading from the
299 # pipe, we assume the other end exited with an error message, so just exit.
300 parent_readfd, child_writefd = os.pipe()
301 child_readfd, parent_writefd = os.pipe()
302 SUCCESS_FLAG = '+'
303
304 pid = os.fork()
305 if not pid:
306 os.close(parent_readfd)
307 os.close(parent_writefd)
308
309 namespaces.Unshare(namespaces.CLONE_NEWNET)
310 os.write(child_writefd, SUCCESS_FLAG)
311 os.close(child_writefd)
312 if os.read(child_readfd, 1) != SUCCESS_FLAG:
313 # Parent failed; it will have already have outputted an error message.
314 sys.exit(1)
315 os.close(child_readfd)
316
317 # Set up child side of the network.
318 commands = (
Mike Frysinger7e82d9a2014-10-09 23:14:55 -0400319 ('ip', 'link', 'set', 'up', 'lo'),
Josh Triplett472a4182013-03-08 11:48:57 -0800320 ('ip', 'address', 'add',
321 '%s/%u' % (PROXY_GUEST_IP, PROXY_NETMASK),
322 'dev', veth_guest),
323 ('ip', 'link', 'set', veth_guest, 'up'),
324 )
325 try:
326 for cmd in commands:
327 cros_build_lib.RunCommand(cmd, print_cmd=False)
328 except cros_build_lib.RunCommandError:
329 raise SystemExit('Running %r failed!' % (cmd,))
330
331 proxy_url = 'http://%s:%u' % (PROXY_HOST_IP, PROXY_PORT)
332 for proto in ('http', 'https', 'ftp'):
333 os.environ[proto + '_proxy'] = proxy_url
334 for v in ('all_proxy', 'RSYNC_PROXY', 'no_proxy'):
335 os.environ.pop(v, None)
336 return
337
338 os.close(child_readfd)
339 os.close(child_writefd)
340
341 if os.read(parent_readfd, 1) != SUCCESS_FLAG:
342 # Child failed; it will have already have outputted an error message.
343 sys.exit(1)
344 os.close(parent_readfd)
345
346 # Set up parent side of the network.
347 uid = int(os.environ.get('SUDO_UID', '0'))
348 gid = int(os.environ.get('SUDO_GID', '0'))
349 if uid == 0 or gid == 0:
350 for username in PROXY_APACHE_FALLBACK_USERS:
351 try:
352 pwnam = pwd.getpwnam(username)
353 uid, gid = pwnam.pw_uid, pwnam.pw_gid
354 break
355 except KeyError:
356 continue
357 if uid == 0 or gid == 0:
358 raise SystemExit('Could not find a non-root user to run Apache as')
359
360 chroot_parent, chroot_base = os.path.split(options.chroot)
361 pid_file = os.path.join(chroot_parent, '.%s-apache-proxy.pid' % chroot_base)
362 log_file = os.path.join(chroot_parent, '.%s-apache-proxy.log' % chroot_base)
363
364 apache_directives = [
365 'User #%u' % uid,
366 'Group #%u' % gid,
367 'PidFile %s' % pid_file,
368 'ErrorLog %s' % log_file,
369 'Listen %s:%u' % (PROXY_HOST_IP, PROXY_PORT),
370 'ServerName %s' % PROXY_HOST_IP,
371 'ProxyRequests On',
372 'AllowCONNECT %s' % ' '.join(map(str, PROXY_CONNECT_PORTS)),
373 ] + [
374 'LoadModule %s %s' % (mod, os.path.join(apache_module_path, so))
375 for (mod, so) in apache_modules
376 ]
377 commands = (
378 ('ip', 'link', 'add', 'name', veth_host,
379 'type', 'veth', 'peer', 'name', veth_guest),
380 ('ip', 'address', 'add',
381 '%s/%u' % (PROXY_HOST_IP, PROXY_NETMASK),
382 'dev', veth_host),
383 ('ip', 'link', 'set', veth_host, 'up'),
384 [apache_bin, '-f', '/dev/null']
385 + [arg for d in apache_directives for arg in ('-C', d)],
386 ('ip', 'link', 'set', veth_guest, 'netns', str(pid)),
387 )
388 cmd = None # Make cros lint happy.
389 try:
390 for cmd in commands:
391 cros_build_lib.RunCommand(cmd, print_cmd=False)
392 except cros_build_lib.RunCommandError:
393 # Clean up existing interfaces, if any.
394 cmd_cleanup = ('ip', 'link', 'del', veth_host)
395 try:
396 cros_build_lib.RunCommand(cmd_cleanup, print_cmd=False)
397 except cros_build_lib.RunCommandError:
398 cros_build_lib.Error('running %r failed', cmd_cleanup)
399 raise SystemExit('Running %r failed!' % (cmd,))
400 os.write(parent_writefd, SUCCESS_FLAG)
401 os.close(parent_writefd)
402
Mike Frysinger80dfce92014-04-21 10:58:53 -0400403 _ExitAsStatus(os.waitpid(pid, 0)[1])
404
405
406def _ExitAsStatus(status):
407 """Exit the same way as |status|.
408
409 If the status field says it was killed by a signal, then we'll do that to
410 ourselves. Otherwise we'll exit with the exit code.
411
412 See http://www.cons.org/cracauer/sigint.html for more details.
413
414 Args:
415 status: A status as returned by os.wait type funcs.
416 """
417 sig_status = status & 0xff
418 exit_status = (status >> 8) & 0xff
419
420 if sig_status:
421 # Kill ourselves with the same signal.
422 pid = os.getpid()
423 os.kill(pid, sig_status)
424 time.sleep(0.1)
425
426 # Still here? Maybe the signal was masked.
427 signal.signal(sig_status, signal.SIG_DFL)
428 os.kill(pid, sig_status)
429 time.sleep(0.1)
430
431 # Still here? Just exit.
432 exit_status = 127
433
434 # Exit with the code we want.
435 sys.exit(exit_status)
436
437
438def _ReapChildren(pid):
439 """Reap all children that get reparented to us until we see |pid| exit.
440
441 Args:
442 pid: The main child to watch for.
443
444 Returns:
445 The wait status of the |pid| child.
446 """
447 pid_status = 0
448
449 while True:
450 try:
451 (wpid, status) = os.wait()
452 if pid == wpid:
453 # Save the status of our main child so we can exit with it below.
454 pid_status = status
455 except OSError as e:
456 if e.errno == errno.ECHILD:
457 break
458 else:
459 raise
460
461 return pid_status
462
463
464def _CreatePidNamespace():
465 """Start a new pid namespace
466
467 This will launch all the right manager processes. The child that returns
468 will be isolated in a new pid namespace.
469
470 If functionality is not available, then it will return w/out doing anything.
471
472 Returns:
473 The last pid outside of the namespace.
474 """
475 first_pid = os.getpid()
476
477 try:
478 # First create the namespace.
479 namespaces.Unshare(namespaces.CLONE_NEWPID)
480 except OSError as e:
481 if e.errno == errno.EINVAL:
482 # For older kernels, or the functionality is disabled in the config,
483 # return silently. We don't want to hard require this stuff.
484 return first_pid
485 else:
486 # For all other errors, abort. They shouldn't happen.
487 raise
488
489 # Now that we're in the new pid namespace, fork. The parent is the master
490 # of it in the original namespace, so it only monitors the child inside it.
491 # It is only allowed to fork once too.
492 pid = os.fork()
493 if pid:
494 # Reap the children as the parent of the new namespace.
495 _ExitAsStatus(_ReapChildren(pid))
496 else:
497 # The child needs its own proc mount as it'll be different.
498 osutils.Mount('proc', '/proc', 'proc',
499 osutils.MS_NOSUID | osutils.MS_NODEV | osutils.MS_NOEXEC |
500 osutils.MS_RELATIME)
501
502 pid = os.fork()
503 if pid:
504 # Watch all of the children. We need to act as the master inside the
505 # namespace and reap old processes.
506 _ExitAsStatus(_ReapChildren(pid))
507
508 # The grandchild will return and take over the rest of the sdk steps.
509 return first_pid
Josh Triplett472a4182013-03-08 11:48:57 -0800510
511
Mike Frysingera78a56e2012-11-20 06:02:30 -0500512def _ReExecuteIfNeeded(argv):
David James56e6c2c2012-10-24 23:54:41 -0700513 """Re-execute cros_sdk as root.
514
515 Also unshare the mount namespace so as to ensure that processes outside
516 the chroot can't mess with our mounts.
517 """
518 if os.geteuid() != 0:
Mike Frysingera78a56e2012-11-20 06:02:30 -0500519 cmd = _SudoCommand() + ['--'] + argv
520 os.execvp(cmd[0], cmd)
Mike Frysingera78a56e2012-11-20 06:02:30 -0500521 else:
Mike Frysinger80dfce92014-04-21 10:58:53 -0400522 # We must set up the cgroups mounts before we enter our own namespace.
523 # This way it is a shared resource in the root mount namespace.
Josh Triplette759b232013-03-08 13:03:43 -0800524 cgroups.Cgroup.InitSystem()
Mike Frysinger6da18852014-04-20 21:16:25 -0400525 namespaces.Unshare(namespaces.CLONE_NEWNS | namespaces.CLONE_NEWUTS)
David James56e6c2c2012-10-24 23:54:41 -0700526
527
Mike Frysinger34db8692013-11-11 14:54:08 -0500528def _CreateParser(sdk_latest_version, bootstrap_latest_version):
529 """Generate and return the parser with all the options."""
Brian Harring218e13c2012-10-10 16:21:26 -0700530 usage = """usage: %prog [options] [VAR1=val1 .. VARn=valn -- args]
Brian Harringb938c782012-02-29 15:14:38 -0800531
Brian Harring218e13c2012-10-10 16:21:26 -0700532This script is used for manipulating local chroot environments; creating,
533deleting, downloading, etc. If given --enter (or no args), it defaults
534to an interactive bash shell within the chroot.
Brian Harringb938c782012-02-29 15:14:38 -0800535
Brian Harring218e13c2012-10-10 16:21:26 -0700536If given args those are passed to the chroot environment, and executed."""
Brian Harring1790ac42012-09-23 08:53:33 -0700537
Brian Harring218e13c2012-10-10 16:21:26 -0700538 parser = commandline.OptionParser(usage=usage, caching=True)
539
Mike Frysinger34db8692013-11-11 14:54:08 -0500540 # Global options.
Mike Frysinger648ba2d2013-01-08 14:19:34 -0500541 default_chroot = os.path.join(constants.SOURCE_ROOT,
542 constants.DEFAULT_CHROOT_DIR)
Brian Harring218e13c2012-10-10 16:21:26 -0700543 parser.add_option(
544 '--chroot', dest='chroot', default=default_chroot, type='path',
545 help=('SDK chroot dir name [%s]' % constants.DEFAULT_CHROOT_DIR))
Brian Harringb938c782012-02-29 15:14:38 -0800546
Brian Harring218e13c2012-10-10 16:21:26 -0700547 parser.add_option('--chrome_root', default=None, type='path',
548 help='Mount this chrome root into the SDK chroot')
549 parser.add_option('--chrome_root_mount', default=None, type='path',
550 help='Mount chrome into this path inside SDK chroot')
551 parser.add_option('--nousepkg', action='store_true', default=False,
552 help='Do not use binary packages when creating a chroot.')
Brian Harringb938c782012-02-29 15:14:38 -0800553 parser.add_option('-u', '--url',
Brian Harringb6cf9142012-09-01 20:43:17 -0700554 dest='sdk_url', default=None,
Brian Harringb938c782012-02-29 15:14:38 -0800555 help=('''Use sdk tarball located at this url.
556 Use file:// for local files.'''))
Brian Harring1790ac42012-09-23 08:53:33 -0700557 parser.add_option('--sdk-version', default=None,
558 help='Use this sdk version. For prebuilt, current is %r'
Mike Frysinger34db8692013-11-11 14:54:08 -0500559 ', for bootstrapping it is %r.'
Brian Harring1790ac42012-09-23 08:53:33 -0700560 % (sdk_latest_version, bootstrap_latest_version))
Mike Frysinger34db8692013-11-11 14:54:08 -0500561
562 # Commands.
563 group = parser.add_option_group('Commands')
564 group.add_option(
565 '--enter', action='store_true', default=False,
566 help='Enter the SDK chroot. Implies --create.')
567 group.add_option(
568 '--create', action='store_true',default=False,
569 help='Create the chroot only if it does not already exist. '
570 'Implies --download.')
571 group.add_option(
572 '--bootstrap', action='store_true', default=False,
573 help='Build everything from scratch, including the sdk. '
574 'Use this only if you need to validate a change '
575 'that affects SDK creation itself (toolchain and '
576 'build are typically the only folk who need this). '
577 'Note this will quite heavily slow down the build. '
578 'This option implies --create --nousepkg.')
579 group.add_option(
580 '-r', '--replace', action='store_true', default=False,
581 help='Replace an existing SDK chroot. Basically an alias '
582 'for --delete --create.')
583 group.add_option(
584 '--delete', action='store_true', default=False,
585 help='Delete the current SDK chroot if it exists.')
586 group.add_option(
587 '--download', action='store_true', default=False,
588 help='Download the sdk.')
589 commands = group
590
Mike Frysinger80dfce92014-04-21 10:58:53 -0400591 # Namespace options.
592 group = parser.add_option_group('Namespaces')
593 group.add_option('--proxy-sim', action='store_true', default=False,
594 help='Simulate a restrictive network requiring an outbound'
595 ' proxy.')
Mike Frysinger94b32a12014-09-12 14:28:13 -0400596 # TODO(vapier): Turn off pid ns until ccache issues can be fixed.
597 # http://crbug.com/411984
Mike Frysinger80dfce92014-04-21 10:58:53 -0400598 group.add_option('--no-ns-pid', dest='ns_pid',
Mike Frysinger94b32a12014-09-12 14:28:13 -0400599 default=False, action='store_false',
Mike Frysinger80dfce92014-04-21 10:58:53 -0400600 help='Do not create a new PID namespace.')
601
Mike Frysinger34db8692013-11-11 14:54:08 -0500602 # Internal options.
603 group = parser.add_option_group(
604 'Internal Chromium OS Build Team Options',
605 'Caution: these are for meant for the Chromium OS build team only')
606 group.add_option('--buildbot-log-version', default=False, action='store_true',
607 help='Log SDK version for buildbot consumption')
608
609 return (parser, commands)
610
611
612def main(argv):
613 conf = cros_build_lib.LoadKeyValueFile(
614 os.path.join(constants.SOURCE_ROOT, constants.SDK_VERSION_FILE),
615 ignore_missing=True)
616 sdk_latest_version = conf.get('SDK_LATEST_VERSION', '<unknown>')
617 bootstrap_latest_version = conf.get('BOOTSTRAP_LATEST_VERSION', '<unknown>')
618 parser, commands = _CreateParser(sdk_latest_version, bootstrap_latest_version)
Brian Harring218e13c2012-10-10 16:21:26 -0700619 options, chroot_command = parser.parse_args(argv)
Brian Harringb938c782012-02-29 15:14:38 -0800620
621 # Some sanity checks first, before we ask for sudo credentials.
Mike Frysinger8fd67dc2012-12-03 23:51:18 -0500622 cros_build_lib.AssertOutsideChroot()
Brian Harringb938c782012-02-29 15:14:38 -0800623
Brian Harring1790ac42012-09-23 08:53:33 -0700624 host = os.uname()[4]
Brian Harring1790ac42012-09-23 08:53:33 -0700625 if host != 'x86_64':
626 parser.error(
627 "cros_sdk is currently only supported on x86_64; you're running"
628 " %s. Please find a x86_64 machine." % (host,))
629
Josh Triplett472a4182013-03-08 11:48:57 -0800630 _ReportMissing(osutils.FindMissingBinaries(NEEDED_TOOLS))
631 if options.proxy_sim:
632 _ReportMissing(osutils.FindMissingBinaries(PROXY_NEEDED_TOOLS))
Brian Harringb938c782012-02-29 15:14:38 -0800633
David James471532c2013-01-21 10:23:31 -0800634 _ReExecuteIfNeeded([sys.argv[0]] + argv)
Mike Frysinger80dfce92014-04-21 10:58:53 -0400635 if options.ns_pid:
636 first_pid = _CreatePidNamespace()
637 else:
638 first_pid = None
David James471532c2013-01-21 10:23:31 -0800639
Brian Harring218e13c2012-10-10 16:21:26 -0700640 # Expand out the aliases...
641 if options.replace:
642 options.delete = options.create = True
Brian Harringb938c782012-02-29 15:14:38 -0800643
Brian Harring218e13c2012-10-10 16:21:26 -0700644 if options.bootstrap:
645 options.create = True
Brian Harringb938c782012-02-29 15:14:38 -0800646
Brian Harring218e13c2012-10-10 16:21:26 -0700647 # If a command is not given, default to enter.
648 options.enter |= not any(getattr(options, x.dest)
649 for x in commands.option_list)
650 options.enter |= bool(chroot_command)
651
652 if options.enter and options.delete and not options.create:
653 parser.error("Trying to enter the chroot when --delete "
654 "was specified makes no sense.")
655
656 # Finally, discern if we need to create the chroot.
657 chroot_exists = os.path.exists(options.chroot)
658 if options.create or options.enter:
659 # Only create if it's being wiped, or if it doesn't exist.
660 if not options.delete and chroot_exists:
661 options.create = False
662 else:
663 options.download = True
664
665 # Finally, flip create if necessary.
666 if options.enter:
667 options.create |= not chroot_exists
Brian Harringb938c782012-02-29 15:14:38 -0800668
Brian Harringb938c782012-02-29 15:14:38 -0800669 if not options.sdk_version:
Brian Harring1790ac42012-09-23 08:53:33 -0700670 sdk_version = (bootstrap_latest_version if options.bootstrap
671 else sdk_latest_version)
Brian Harringb938c782012-02-29 15:14:38 -0800672 else:
673 sdk_version = options.sdk_version
Mike Frysinger34db8692013-11-11 14:54:08 -0500674 if options.buildbot_log_version:
675 cros_build_lib.PrintBuildbotStepText(sdk_version)
Brian Harringb938c782012-02-29 15:14:38 -0800676
Brian Harring1790ac42012-09-23 08:53:33 -0700677 # Based on selections, fetch the tarball.
678 if options.sdk_url:
679 urls = [options.sdk_url]
680 elif options.bootstrap:
681 urls = GetStage3Urls(sdk_version)
682 else:
683 urls = GetArchStageTarballs(sdk_version)
684
Brian Harringb6cf9142012-09-01 20:43:17 -0700685 lock_path = os.path.dirname(options.chroot)
Brian Harringb938c782012-02-29 15:14:38 -0800686 lock_path = os.path.join(lock_path,
Brian Harringb6cf9142012-09-01 20:43:17 -0700687 '.%s_lock' % os.path.basename(options.chroot))
Mike Frysinger80dfce92014-04-21 10:58:53 -0400688 with cgroups.SimpleContainChildren('cros_sdk', pid=first_pid):
David James56e6c2c2012-10-24 23:54:41 -0700689 with locking.FileLock(lock_path, 'chroot lock') as lock:
Brian Harring1790ac42012-09-23 08:53:33 -0700690
Josh Triplett472a4182013-03-08 11:48:57 -0800691 if options.proxy_sim:
692 _ProxySimSetup(options)
693
David James56e6c2c2012-10-24 23:54:41 -0700694 if options.delete and os.path.exists(options.chroot):
695 lock.write_lock()
696 DeleteChroot(options.chroot)
Brian Harringb938c782012-02-29 15:14:38 -0800697
David James56e6c2c2012-10-24 23:54:41 -0700698 sdk_cache = os.path.join(options.cache_dir, 'sdks')
699 distfiles_cache = os.path.join(options.cache_dir, 'distfiles')
Yu-Ju Hong2c066762013-10-28 14:05:08 -0700700 osutils.SafeMakedirsNonRoot(options.cache_dir)
Brian Harringae0a5322012-09-15 01:46:51 -0700701
David James56e6c2c2012-10-24 23:54:41 -0700702 for target in (sdk_cache, distfiles_cache):
Mike Frysinger648ba2d2013-01-08 14:19:34 -0500703 src = os.path.join(constants.SOURCE_ROOT, os.path.basename(target))
David James56e6c2c2012-10-24 23:54:41 -0700704 if not os.path.exists(src):
705 osutils.SafeMakedirs(target)
706 continue
707 lock.write_lock(
708 "Upgrade to %r needed but chroot is locked; please exit "
709 "all instances so this upgrade can finish." % src)
710 if not os.path.exists(src):
711 # Note that while waiting for the write lock, src may've vanished;
712 # it's a rare race during the upgrade process that's a byproduct
713 # of us avoiding taking a write lock to do the src check. If we
714 # took a write lock for that check, it would effectively limit
715 # all cros_sdk for a chroot to a single instance.
716 osutils.SafeMakedirs(target)
717 elif not os.path.exists(target):
718 # Upgrade occurred, but a reversion, or something whacky
719 # occurred writing to the old location. Wipe and continue.
720 os.rename(src, target)
721 else:
722 # Upgrade occurred once already, but either a reversion or
723 # some before/after separate cros_sdk usage is at play.
724 # Wipe and continue.
725 osutils.RmDir(src)
Brian Harringae0a5322012-09-15 01:46:51 -0700726
David James56e6c2c2012-10-24 23:54:41 -0700727 if options.download:
728 lock.write_lock()
729 sdk_tarball = FetchRemoteTarballs(sdk_cache, urls)
Brian Harring218e13c2012-10-10 16:21:26 -0700730
David James56e6c2c2012-10-24 23:54:41 -0700731 if options.create:
732 lock.write_lock()
733 CreateChroot(options.chroot, sdk_tarball, options.cache_dir,
734 nousepkg=(options.bootstrap or options.nousepkg))
Brian Harring1790ac42012-09-23 08:53:33 -0700735
David James56e6c2c2012-10-24 23:54:41 -0700736 if options.enter:
737 lock.read_lock()
738 EnterChroot(options.chroot, options.cache_dir, options.chrome_root,
739 options.chrome_root_mount, chroot_command)