blob: 0de2ed5dd37ddf80cd70c3ef6c8936092b4b7c7c [file] [log] [blame]
Mike Frysinger0a647fc2012-08-06 14:36:05 -04001# Copyright (c) 2012 The Chromium OS Authors. All rights reserved.
David Jamesfcb70ef2011-02-02 16:02:30 -08002# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
5"""Program to run emerge in parallel, for significant speedup.
6
7Usage:
David James386ccd12011-05-04 20:17:42 -07008 ./parallel_emerge [--board=BOARD] [--workon=PKGS]
David Jamesfcb70ef2011-02-02 16:02:30 -08009 [--force-remote-binary=PKGS] [emerge args] package
10
David James78b6cd92012-04-02 21:36:12 -070011This script runs multiple emerge processes in parallel, using appropriate
12Portage APIs. It is faster than standard emerge because it has a
13multiprocess model instead of an asynchronous model.
David Jamesfcb70ef2011-02-02 16:02:30 -080014"""
15
Mike Frysinger383367e2014-09-16 15:06:17 -040016from __future__ import print_function
17
David Jamesfcb70ef2011-02-02 16:02:30 -080018import codecs
19import copy
20import errno
Brian Harring8294d652012-05-23 02:20:52 -070021import gc
David James8c7e5e32011-06-28 11:26:03 -070022import heapq
David Jamesfcb70ef2011-02-02 16:02:30 -080023import multiprocessing
24import os
Mike Frysinger1ae28092013-10-17 17:17:22 -040025try:
26 import Queue
27except ImportError:
28 # Python-3 renamed to "queue". We still use Queue to avoid collisions
29 # with naming variables as "queue". Maybe we'll transition at some point.
30 # pylint: disable=F0401
31 import queue as Queue
David Jamesfcb70ef2011-02-02 16:02:30 -080032import signal
Bertrand SIMONNET19d789e2014-12-09 13:36:31 -080033import subprocess
David Jamesfcb70ef2011-02-02 16:02:30 -080034import sys
35import tempfile
Brian Harring8294d652012-05-23 02:20:52 -070036import threading
David Jamesfcb70ef2011-02-02 16:02:30 -080037import time
38import traceback
David Jamesfcb70ef2011-02-02 16:02:30 -080039
Thiago Goncalesf4acc422013-07-17 10:26:35 -070040from chromite.lib import cros_build_lib
Chris Ching5fcbd622016-11-28 09:22:15 -070041from chromite.lib import cros_event
Mike Frysingere2d8f0d2014-11-01 13:09:26 -040042from chromite.lib import process_util
Mike Frysingerd74fe4a2014-04-24 11:43:38 -040043from chromite.lib import proctitle
Thiago Goncalesf4acc422013-07-17 10:26:35 -070044
David Jamesfcb70ef2011-02-02 16:02:30 -080045# If PORTAGE_USERNAME isn't specified, scrape it from the $HOME variable. On
46# Chromium OS, the default "portage" user doesn't have the necessary
47# permissions. It'd be easier if we could default to $USERNAME, but $USERNAME
48# is "root" here because we get called through sudo.
49#
50# We need to set this before importing any portage modules, because portage
51# looks up "PORTAGE_USERNAME" at import time.
52#
53# NOTE: .bashrc sets PORTAGE_USERNAME = $USERNAME, so most people won't
54# encounter this case unless they have an old chroot or blow away the
55# environment by running sudo without the -E specifier.
56if "PORTAGE_USERNAME" not in os.environ:
57 homedir = os.environ.get("HOME")
58 if homedir:
59 os.environ["PORTAGE_USERNAME"] = os.path.basename(homedir)
60
Bertrand SIMONNET19d789e2014-12-09 13:36:31 -080061# Wrap Popen with a lock to ensure no two Popen are executed simultaneously in
62# the same process.
63# Two Popen call at the same time might be the cause for crbug.com/433482.
64_popen_lock = threading.Lock()
65_old_popen = subprocess.Popen
66
67def _LockedPopen(*args, **kwargs):
68 with _popen_lock:
69 return _old_popen(*args, **kwargs)
70
71subprocess.Popen = _LockedPopen
72
David Jamesfcb70ef2011-02-02 16:02:30 -080073# Portage doesn't expose dependency trees in its public API, so we have to
74# make use of some private APIs here. These modules are found under
75# /usr/lib/portage/pym/.
76#
77# TODO(davidjames): Update Portage to expose public APIs for these features.
Don Garrett25f309a2014-03-19 14:02:12 -070078# pylint: disable=F0401
David Jamesfcb70ef2011-02-02 16:02:30 -080079from _emerge.actions import adjust_configs
80from _emerge.actions import load_emerge_config
81from _emerge.create_depgraph_params import create_depgraph_params
David James386ccd12011-05-04 20:17:42 -070082from _emerge.depgraph import backtrack_depgraph
David Jamesfcb70ef2011-02-02 16:02:30 -080083from _emerge.main import emerge_main
84from _emerge.main import parse_opts
85from _emerge.Package import Package
Bertrand SIMONNETa15b5072014-10-23 15:27:52 -070086from _emerge.post_emerge import clean_logs
David Jamesfcb70ef2011-02-02 16:02:30 -080087from _emerge.Scheduler import Scheduler
David Jamesfcb70ef2011-02-02 16:02:30 -080088from _emerge.stdout_spinner import stdout_spinner
David James386ccd12011-05-04 20:17:42 -070089from portage._global_updates import _global_updates
David Jamesfcb70ef2011-02-02 16:02:30 -080090import portage
91import portage.debug
Don Garrettf8bf7842014-03-20 17:03:42 -070092# pylint: enable=F0401
Mike Frysinger91d7da92013-02-19 15:53:46 -050093
David Jamesfcb70ef2011-02-02 16:02:30 -080094
David Jamesfcb70ef2011-02-02 16:02:30 -080095def Usage():
96 """Print usage."""
Mike Frysinger383367e2014-09-16 15:06:17 -040097 print("Usage:")
Chris Ching5fcbd622016-11-28 09:22:15 -070098 print(" ./parallel_emerge [--board=BOARD] [--workon=PKGS] [--rebuild]")
99 print(" [--eventlogfile=FILE] [emerge args] package")
Mike Frysinger383367e2014-09-16 15:06:17 -0400100 print()
101 print("Packages specified as workon packages are always built from source.")
102 print()
103 print("The --workon argument is mainly useful when you want to build and")
104 print("install packages that you are working on unconditionally, but do not")
105 print("to have to rev the package to indicate you want to build it from")
106 print("source. The build_packages script will automatically supply the")
107 print("workon argument to emerge, ensuring that packages selected using")
108 print("cros-workon are rebuilt.")
109 print()
110 print("The --rebuild option rebuilds packages whenever their dependencies")
111 print("are changed. This ensures that your build is correct.")
Chris Ching5fcbd622016-11-28 09:22:15 -0700112 print()
113 print("The --eventlogfile writes events to the given file. File is")
114 print("is overwritten if it exists.")
David Jamesfcb70ef2011-02-02 16:02:30 -0800115
116
David Jamesfcb70ef2011-02-02 16:02:30 -0800117# Global start time
118GLOBAL_START = time.time()
119
David James7358d032011-05-19 10:40:03 -0700120# Whether process has been killed by a signal.
121KILLED = multiprocessing.Event()
122
David Jamesfcb70ef2011-02-02 16:02:30 -0800123
124class EmergeData(object):
125 """This simple struct holds various emerge variables.
126
127 This struct helps us easily pass emerge variables around as a unit.
128 These variables are used for calculating dependencies and installing
129 packages.
130 """
131
David Jamesbf1e3442011-05-28 07:44:20 -0700132 __slots__ = ["action", "cmdline_packages", "depgraph", "favorites",
133 "mtimedb", "opts", "root_config", "scheduler_graph",
134 "settings", "spinner", "trees"]
David Jamesfcb70ef2011-02-02 16:02:30 -0800135
136 def __init__(self):
137 # The action the user requested. If the user is installing packages, this
138 # is None. If the user is doing anything other than installing packages,
139 # this will contain the action name, which will map exactly to the
140 # long-form name of the associated emerge option.
141 #
142 # Example: If you call parallel_emerge --unmerge package, the action name
143 # will be "unmerge"
144 self.action = None
145
146 # The list of packages the user passed on the command-line.
147 self.cmdline_packages = None
148
149 # The emerge dependency graph. It'll contain all the packages involved in
150 # this merge, along with their versions.
151 self.depgraph = None
152
David Jamesbf1e3442011-05-28 07:44:20 -0700153 # The list of candidates to add to the world file.
154 self.favorites = None
155
David Jamesfcb70ef2011-02-02 16:02:30 -0800156 # A dict of the options passed to emerge. This dict has been cleaned up
157 # a bit by parse_opts, so that it's a bit easier for the emerge code to
158 # look at the options.
159 #
160 # Emerge takes a few shortcuts in its cleanup process to make parsing of
161 # the options dict easier. For example, if you pass in "--usepkg=n", the
162 # "--usepkg" flag is just left out of the dictionary altogether. Because
163 # --usepkg=n is the default, this makes parsing easier, because emerge
164 # can just assume that if "--usepkg" is in the dictionary, it's enabled.
165 #
166 # These cleanup processes aren't applied to all options. For example, the
167 # --with-bdeps flag is passed in as-is. For a full list of the cleanups
168 # applied by emerge, see the parse_opts function in the _emerge.main
169 # package.
170 self.opts = None
171
172 # A dictionary used by portage to maintain global state. This state is
173 # loaded from disk when portage starts up, and saved to disk whenever we
174 # call mtimedb.commit().
175 #
176 # This database contains information about global updates (i.e., what
177 # version of portage we have) and what we're currently doing. Portage
178 # saves what it is currently doing in this database so that it can be
179 # resumed when you call it with the --resume option.
180 #
181 # parallel_emerge does not save what it is currently doing in the mtimedb,
182 # so we do not support the --resume option.
183 self.mtimedb = None
184
185 # The portage configuration for our current root. This contains the portage
186 # settings (see below) and the three portage trees for our current root.
187 # (The three portage trees are explained below, in the documentation for
188 # the "trees" member.)
189 self.root_config = None
190
191 # The scheduler graph is used by emerge to calculate what packages to
192 # install. We don't actually install any deps, so this isn't really used,
193 # but we pass it in to the Scheduler object anyway.
194 self.scheduler_graph = None
195
196 # Portage settings for our current session. Most of these settings are set
197 # in make.conf inside our current install root.
198 self.settings = None
199
200 # The spinner, which spews stuff to stdout to indicate that portage is
201 # doing something. We maintain our own spinner, so we set the portage
202 # spinner to "silent" mode.
203 self.spinner = None
204
205 # The portage trees. There are separate portage trees for each root. To get
206 # the portage tree for the current root, you can look in self.trees[root],
207 # where root = self.settings["ROOT"].
208 #
209 # In each root, there are three trees: vartree, porttree, and bintree.
210 # - vartree: A database of the currently-installed packages.
211 # - porttree: A database of ebuilds, that can be used to build packages.
212 # - bintree: A database of binary packages.
213 self.trees = None
214
215
216class DepGraphGenerator(object):
217 """Grab dependency information about packages from portage.
218
219 Typical usage:
220 deps = DepGraphGenerator()
221 deps.Initialize(sys.argv[1:])
222 deps_tree, deps_info = deps.GenDependencyTree()
223 deps_graph = deps.GenDependencyGraph(deps_tree, deps_info)
224 deps.PrintTree(deps_tree)
225 PrintDepsMap(deps_graph)
226 """
227
Bertrand SIMONNET0625e1a2015-04-07 11:41:16 -0700228 __slots__ = ["board", "emerge", "package_db", "show_output", "sysroot",
Bertrand SIMONNET411945d2015-05-20 17:23:28 -0700229 "unpack_only", "max_retries"]
David Jamesfcb70ef2011-02-02 16:02:30 -0800230
231 def __init__(self):
232 self.board = None
233 self.emerge = EmergeData()
David Jamesfcb70ef2011-02-02 16:02:30 -0800234 self.package_db = {}
David Jamesfcb70ef2011-02-02 16:02:30 -0800235 self.show_output = False
Bertrand SIMONNET0625e1a2015-04-07 11:41:16 -0700236 self.sysroot = None
Thiago Goncalesf4acc422013-07-17 10:26:35 -0700237 self.unpack_only = False
Bertrand SIMONNET411945d2015-05-20 17:23:28 -0700238 self.max_retries = 1
David Jamesfcb70ef2011-02-02 16:02:30 -0800239
240 def ParseParallelEmergeArgs(self, argv):
241 """Read the parallel emerge arguments from the command-line.
242
243 We need to be compatible with emerge arg format. We scrape arguments that
244 are specific to parallel_emerge, and pass through the rest directly to
245 emerge.
Mike Frysinger1a736a82013-12-12 01:50:59 -0500246
David Jamesfcb70ef2011-02-02 16:02:30 -0800247 Args:
248 argv: arguments list
Mike Frysinger1a736a82013-12-12 01:50:59 -0500249
David Jamesfcb70ef2011-02-02 16:02:30 -0800250 Returns:
251 Arguments that don't belong to parallel_emerge
252 """
253 emerge_args = []
254 for arg in argv:
255 # Specifically match arguments that are specific to parallel_emerge, and
256 # pass through the rest.
257 if arg.startswith("--board="):
258 self.board = arg.replace("--board=", "")
Bertrand SIMONNET0625e1a2015-04-07 11:41:16 -0700259 elif arg.startswith("--sysroot="):
260 self.sysroot = arg.replace("--sysroot=", "")
David Jamesfcb70ef2011-02-02 16:02:30 -0800261 elif arg.startswith("--workon="):
262 workon_str = arg.replace("--workon=", "")
David James7a1ea4b2011-10-13 15:06:41 -0700263 emerge_args.append("--reinstall-atoms=%s" % workon_str)
264 emerge_args.append("--usepkg-exclude=%s" % workon_str)
David Jamesfcb70ef2011-02-02 16:02:30 -0800265 elif arg.startswith("--force-remote-binary="):
266 force_remote_binary = arg.replace("--force-remote-binary=", "")
David James7a1ea4b2011-10-13 15:06:41 -0700267 emerge_args.append("--useoldpkg-atoms=%s" % force_remote_binary)
Bertrand SIMONNET411945d2015-05-20 17:23:28 -0700268 elif arg.startswith("--retries="):
269 self.max_retries = int(arg.replace("--retries=", ""))
David Jamesfcb70ef2011-02-02 16:02:30 -0800270 elif arg == "--show-output":
271 self.show_output = True
David James386ccd12011-05-04 20:17:42 -0700272 elif arg == "--rebuild":
David James7a1ea4b2011-10-13 15:06:41 -0700273 emerge_args.append("--rebuild-if-unbuilt")
Thiago Goncalesf4acc422013-07-17 10:26:35 -0700274 elif arg == "--unpackonly":
275 emerge_args.append("--fetchonly")
276 self.unpack_only = True
Chris Ching5fcbd622016-11-28 09:22:15 -0700277 elif arg.startswith("--eventlogfile="):
278 log_file_name = arg.replace("--eventlogfile=", "")
279 cros_event.setEventLogger(cros_event.getEventFileLogger(log_file_name))
David Jamesfcb70ef2011-02-02 16:02:30 -0800280 else:
281 # Not one of our options, so pass through to emerge.
282 emerge_args.append(arg)
283
David James386ccd12011-05-04 20:17:42 -0700284 # These packages take a really long time to build, so, for expediency, we
285 # are blacklisting them from automatic rebuilds because one of their
286 # dependencies needs to be recompiled.
Mike Frysinger5c2a9052014-04-15 15:52:04 -0400287 for pkg in ("chromeos-base/chromeos-chrome",):
David James7a1ea4b2011-10-13 15:06:41 -0700288 emerge_args.append("--rebuild-exclude=%s" % pkg)
David Jamesfcb70ef2011-02-02 16:02:30 -0800289
290 return emerge_args
291
292 def Initialize(self, args):
293 """Initializer. Parses arguments and sets up portage state."""
294
295 # Parse and strip out args that are just intended for parallel_emerge.
296 emerge_args = self.ParseParallelEmergeArgs(args)
297
Bertrand SIMONNET0625e1a2015-04-07 11:41:16 -0700298 if self.sysroot and self.board:
299 cros_build_lib.Die("--sysroot and --board are incompatible.")
300
David Jamesfcb70ef2011-02-02 16:02:30 -0800301 # Setup various environment variables based on our current board. These
302 # variables are normally setup inside emerge-${BOARD}, but since we don't
303 # call that script, we have to set it up here. These variables serve to
304 # point our tools at /build/BOARD and to setup cross compiles to the
305 # appropriate board as configured in toolchain.conf.
306 if self.board:
Bertrand SIMONNET0625e1a2015-04-07 11:41:16 -0700307 self.sysroot = os.environ.get('SYSROOT',
308 cros_build_lib.GetSysroot(self.board))
309
310 if self.sysroot:
311 os.environ["PORTAGE_CONFIGROOT"] = self.sysroot
312 os.environ["SYSROOT"] = self.sysroot
David Jamesfcb70ef2011-02-02 16:02:30 -0800313
David Jamesfcb70ef2011-02-02 16:02:30 -0800314 # Turn off interactive delays
315 os.environ["EBEEP_IGNORE"] = "1"
316 os.environ["EPAUSE_IGNORE"] = "1"
Mike Frysinger0a647fc2012-08-06 14:36:05 -0400317 os.environ["CLEAN_DELAY"] = "0"
David Jamesfcb70ef2011-02-02 16:02:30 -0800318
319 # Parse the emerge options.
David Jamesea3ca332011-05-26 11:48:29 -0700320 action, opts, cmdline_packages = parse_opts(emerge_args, silent=True)
David Jamesfcb70ef2011-02-02 16:02:30 -0800321
322 # Set environment variables based on options. Portage normally sets these
323 # environment variables in emerge_main, but we can't use that function,
324 # because it also does a bunch of other stuff that we don't want.
325 # TODO(davidjames): Patch portage to move this logic into a function we can
326 # reuse here.
327 if "--debug" in opts:
328 os.environ["PORTAGE_DEBUG"] = "1"
329 if "--config-root" in opts:
330 os.environ["PORTAGE_CONFIGROOT"] = opts["--config-root"]
331 if "--root" in opts:
332 os.environ["ROOT"] = opts["--root"]
333 if "--accept-properties" in opts:
334 os.environ["ACCEPT_PROPERTIES"] = opts["--accept-properties"]
335
David James88d780c2014-02-05 13:03:29 -0800336 # If we're installing packages to the board, we can disable vardb locks.
337 # This is safe because we only run up to one instance of parallel_emerge in
338 # parallel.
339 # TODO(davidjames): Enable this for the host too.
Bertrand SIMONNET0625e1a2015-04-07 11:41:16 -0700340 if self.sysroot:
David Jamesfcb70ef2011-02-02 16:02:30 -0800341 os.environ.setdefault("PORTAGE_LOCKS", "false")
David Jamesfcb70ef2011-02-02 16:02:30 -0800342
343 # Now that we've setup the necessary environment variables, we can load the
344 # emerge config from disk.
Gilad Arnold94758762015-05-22 12:23:23 -0700345 # pylint: disable=unpacking-non-sequence
David Jamesfcb70ef2011-02-02 16:02:30 -0800346 settings, trees, mtimedb = load_emerge_config()
347
David Jamesea3ca332011-05-26 11:48:29 -0700348 # Add in EMERGE_DEFAULT_OPTS, if specified.
349 tmpcmdline = []
350 if "--ignore-default-opts" not in opts:
351 tmpcmdline.extend(settings["EMERGE_DEFAULT_OPTS"].split())
352 tmpcmdline.extend(emerge_args)
353 action, opts, cmdline_packages = parse_opts(tmpcmdline)
354
355 # If we're installing to the board, we want the --root-deps option so that
356 # portage will install the build dependencies to that location as well.
Bertrand SIMONNET0625e1a2015-04-07 11:41:16 -0700357 if self.sysroot:
David Jamesea3ca332011-05-26 11:48:29 -0700358 opts.setdefault("--root-deps", True)
359
David Jamesfcb70ef2011-02-02 16:02:30 -0800360 # Check whether our portage tree is out of date. Typically, this happens
361 # when you're setting up a new portage tree, such as in setup_board and
362 # make_chroot. In that case, portage applies a bunch of global updates
363 # here. Once the updates are finished, we need to commit any changes
364 # that the global update made to our mtimedb, and reload the config.
365 #
366 # Portage normally handles this logic in emerge_main, but again, we can't
367 # use that function here.
368 if _global_updates(trees, mtimedb["updates"]):
369 mtimedb.commit()
Gilad Arnold94758762015-05-22 12:23:23 -0700370 # pylint: disable=unpacking-non-sequence
David Jamesfcb70ef2011-02-02 16:02:30 -0800371 settings, trees, mtimedb = load_emerge_config(trees=trees)
372
373 # Setup implied options. Portage normally handles this logic in
374 # emerge_main.
375 if "--buildpkgonly" in opts or "buildpkg" in settings.features:
376 opts.setdefault("--buildpkg", True)
377 if "--getbinpkgonly" in opts:
378 opts.setdefault("--usepkgonly", True)
379 opts.setdefault("--getbinpkg", True)
380 if "getbinpkg" in settings.features:
381 # Per emerge_main, FEATURES=getbinpkg overrides --getbinpkg=n
382 opts["--getbinpkg"] = True
383 if "--getbinpkg" in opts or "--usepkgonly" in opts:
384 opts.setdefault("--usepkg", True)
385 if "--fetch-all-uri" in opts:
386 opts.setdefault("--fetchonly", True)
387 if "--skipfirst" in opts:
388 opts.setdefault("--resume", True)
389 if "--buildpkgonly" in opts:
390 # --buildpkgonly will not merge anything, so it overrides all binary
391 # package options.
392 for opt in ("--getbinpkg", "--getbinpkgonly",
393 "--usepkg", "--usepkgonly"):
394 opts.pop(opt, None)
395 if (settings.get("PORTAGE_DEBUG", "") == "1" and
396 "python-trace" in settings.features):
397 portage.debug.set_trace(True)
398
399 # Complain about unsupported options
David James386ccd12011-05-04 20:17:42 -0700400 for opt in ("--ask", "--ask-enter-invalid", "--resume", "--skipfirst"):
David Jamesfcb70ef2011-02-02 16:02:30 -0800401 if opt in opts:
Mike Frysinger383367e2014-09-16 15:06:17 -0400402 print("%s is not supported by parallel_emerge" % opt)
David Jamesfcb70ef2011-02-02 16:02:30 -0800403 sys.exit(1)
404
405 # Make emerge specific adjustments to the config (e.g. colors!)
406 adjust_configs(opts, trees)
407
408 # Save our configuration so far in the emerge object
409 emerge = self.emerge
410 emerge.action, emerge.opts = action, opts
411 emerge.settings, emerge.trees, emerge.mtimedb = settings, trees, mtimedb
412 emerge.cmdline_packages = cmdline_packages
413 root = settings["ROOT"]
414 emerge.root_config = trees[root]["root_config"]
415
David James386ccd12011-05-04 20:17:42 -0700416 if "--usepkg" in opts:
David Jamesfcb70ef2011-02-02 16:02:30 -0800417 emerge.trees[root]["bintree"].populate("--getbinpkg" in opts)
418
David Jamesfcb70ef2011-02-02 16:02:30 -0800419 def CreateDepgraph(self, emerge, packages):
420 """Create an emerge depgraph object."""
421 # Setup emerge options.
422 emerge_opts = emerge.opts.copy()
423
David James386ccd12011-05-04 20:17:42 -0700424 # Ask portage to build a dependency graph. with the options we specified
425 # above.
David Jamesfcb70ef2011-02-02 16:02:30 -0800426 params = create_depgraph_params(emerge_opts, emerge.action)
David Jamesbf1e3442011-05-28 07:44:20 -0700427 success, depgraph, favorites = backtrack_depgraph(
David James386ccd12011-05-04 20:17:42 -0700428 emerge.settings, emerge.trees, emerge_opts, params, emerge.action,
429 packages, emerge.spinner)
430 emerge.depgraph = depgraph
David Jamesfcb70ef2011-02-02 16:02:30 -0800431
David James386ccd12011-05-04 20:17:42 -0700432 # Is it impossible to honor the user's request? Bail!
433 if not success:
434 depgraph.display_problems()
435 sys.exit(1)
David Jamesfcb70ef2011-02-02 16:02:30 -0800436
437 emerge.depgraph = depgraph
David Jamesbf1e3442011-05-28 07:44:20 -0700438 emerge.favorites = favorites
David Jamesfcb70ef2011-02-02 16:02:30 -0800439
David Jamesdeebd692011-05-09 17:02:52 -0700440 # Prime and flush emerge caches.
441 root = emerge.settings["ROOT"]
442 vardb = emerge.trees[root]["vartree"].dbapi
David James0bdc5de2011-05-12 16:22:26 -0700443 if "--pretend" not in emerge.opts:
444 vardb.counter_tick()
David Jamesdeebd692011-05-09 17:02:52 -0700445 vardb.flush_cache()
446
David James386ccd12011-05-04 20:17:42 -0700447 def GenDependencyTree(self):
David Jamesfcb70ef2011-02-02 16:02:30 -0800448 """Get dependency tree info from emerge.
449
David Jamesfcb70ef2011-02-02 16:02:30 -0800450 Returns:
451 Dependency tree
452 """
453 start = time.time()
454
455 emerge = self.emerge
456
457 # Create a list of packages to merge
458 packages = set(emerge.cmdline_packages[:])
David Jamesfcb70ef2011-02-02 16:02:30 -0800459
460 # Tell emerge to be quiet. We print plenty of info ourselves so we don't
461 # need any extra output from portage.
462 portage.util.noiselimit = -1
463
464 # My favorite feature: The silent spinner. It doesn't spin. Ever.
465 # I'd disable the colors by default too, but they look kind of cool.
466 emerge.spinner = stdout_spinner()
467 emerge.spinner.update = emerge.spinner.update_quiet
468
469 if "--quiet" not in emerge.opts:
Mike Frysinger383367e2014-09-16 15:06:17 -0400470 print("Calculating deps...")
David Jamesfcb70ef2011-02-02 16:02:30 -0800471
Chris Ching151617c2017-01-25 16:57:06 -0700472 with cros_event.newEvent(kind="GenerateDepTree"):
Chris Ching5fcbd622016-11-28 09:22:15 -0700473 self.CreateDepgraph(emerge, packages)
474 depgraph = emerge.depgraph
David Jamesfcb70ef2011-02-02 16:02:30 -0800475
476 # Build our own tree from the emerge digraph.
477 deps_tree = {}
Don Garrett25f309a2014-03-19 14:02:12 -0700478 # pylint: disable=W0212
David Jamesfcb70ef2011-02-02 16:02:30 -0800479 digraph = depgraph._dynamic_config.digraph
David James3f778802011-08-25 19:31:45 -0700480 root = emerge.settings["ROOT"]
Bertrand SIMONNETa15b5072014-10-23 15:27:52 -0700481 final_db = depgraph._dynamic_config._filtered_trees[root]['graph_db']
David Jamesfcb70ef2011-02-02 16:02:30 -0800482 for node, node_deps in digraph.nodes.items():
483 # Calculate dependency packages that need to be installed first. Each
484 # child on the digraph is a dependency. The "operation" field specifies
485 # what we're doing (e.g. merge, uninstall, etc.). The "priorities" array
486 # contains the type of dependency (e.g. build, runtime, runtime_post,
487 # etc.)
488 #
David Jamesfcb70ef2011-02-02 16:02:30 -0800489 # Portage refers to the identifiers for packages as a CPV. This acronym
490 # stands for Component/Path/Version.
491 #
492 # Here's an example CPV: chromeos-base/power_manager-0.0.1-r1
493 # Split up, this CPV would be:
494 # C -- Component: chromeos-base
495 # P -- Path: power_manager
496 # V -- Version: 0.0.1-r1
497 #
498 # We just refer to CPVs as packages here because it's easier.
499 deps = {}
500 for child, priorities in node_deps[0].items():
David James3f778802011-08-25 19:31:45 -0700501 if isinstance(child, Package) and child.root == root:
502 cpv = str(child.cpv)
503 action = str(child.operation)
504
505 # If we're uninstalling a package, check whether Portage is
506 # installing a replacement. If so, just depend on the installation
507 # of the new package, because the old package will automatically
508 # be uninstalled at that time.
509 if action == "uninstall":
510 for pkg in final_db.match_pkgs(child.slot_atom):
511 cpv = str(pkg.cpv)
512 action = "merge"
513 break
514
515 deps[cpv] = dict(action=action,
516 deptypes=[str(x) for x in priorities],
517 deps={})
David Jamesfcb70ef2011-02-02 16:02:30 -0800518
519 # We've built our list of deps, so we can add our package to the tree.
David James3f778802011-08-25 19:31:45 -0700520 if isinstance(node, Package) and node.root == root:
David Jamesfcb70ef2011-02-02 16:02:30 -0800521 deps_tree[str(node.cpv)] = dict(action=str(node.operation),
522 deps=deps)
523
David Jamesfcb70ef2011-02-02 16:02:30 -0800524 # Ask portage for its install plan, so that we can only throw out
David James386ccd12011-05-04 20:17:42 -0700525 # dependencies that portage throws out.
David Jamesfcb70ef2011-02-02 16:02:30 -0800526 deps_info = {}
527 for pkg in depgraph.altlist():
528 if isinstance(pkg, Package):
David James3f778802011-08-25 19:31:45 -0700529 assert pkg.root == root
David Jamesfcb70ef2011-02-02 16:02:30 -0800530 self.package_db[pkg.cpv] = pkg
531
David Jamesfcb70ef2011-02-02 16:02:30 -0800532 # Save off info about the package
David James386ccd12011-05-04 20:17:42 -0700533 deps_info[str(pkg.cpv)] = {"idx": len(deps_info)}
David Jamesfcb70ef2011-02-02 16:02:30 -0800534
535 seconds = time.time() - start
536 if "--quiet" not in emerge.opts:
Mike Frysinger383367e2014-09-16 15:06:17 -0400537 print("Deps calculated in %dm%.1fs" % (seconds / 60, seconds % 60))
David Jamesfcb70ef2011-02-02 16:02:30 -0800538
539 return deps_tree, deps_info
540
541 def PrintTree(self, deps, depth=""):
542 """Print the deps we have seen in the emerge output.
543
544 Args:
Mike Frysinger6f3c48e2015-05-06 02:38:51 -0400545 deps: Dependency tree structure.
546 depth: Allows printing the tree recursively, with indentation.
David Jamesfcb70ef2011-02-02 16:02:30 -0800547 """
548 for entry in sorted(deps):
549 action = deps[entry]["action"]
Mike Frysinger383367e2014-09-16 15:06:17 -0400550 print("%s %s (%s)" % (depth, entry, action))
David Jamesfcb70ef2011-02-02 16:02:30 -0800551 self.PrintTree(deps[entry]["deps"], depth=depth + " ")
552
David James386ccd12011-05-04 20:17:42 -0700553 def GenDependencyGraph(self, deps_tree, deps_info):
David Jamesfcb70ef2011-02-02 16:02:30 -0800554 """Generate a doubly linked dependency graph.
555
556 Args:
557 deps_tree: Dependency tree structure.
558 deps_info: More details on the dependencies.
Mike Frysinger1a736a82013-12-12 01:50:59 -0500559
David Jamesfcb70ef2011-02-02 16:02:30 -0800560 Returns:
561 Deps graph in the form of a dict of packages, with each package
562 specifying a "needs" list and "provides" list.
563 """
564 emerge = self.emerge
David Jamesfcb70ef2011-02-02 16:02:30 -0800565
David Jamesfcb70ef2011-02-02 16:02:30 -0800566 # deps_map is the actual dependency graph.
567 #
568 # Each package specifies a "needs" list and a "provides" list. The "needs"
569 # list indicates which packages we depend on. The "provides" list
570 # indicates the reverse dependencies -- what packages need us.
571 #
572 # We also provide some other information in the dependency graph:
573 # - action: What we're planning on doing with this package. Generally,
574 # "merge", "nomerge", or "uninstall"
David Jamesfcb70ef2011-02-02 16:02:30 -0800575 deps_map = {}
576
577 def ReverseTree(packages):
578 """Convert tree to digraph.
579
580 Take the tree of package -> requirements and reverse it to a digraph of
581 buildable packages -> packages they unblock.
Mike Frysinger1a736a82013-12-12 01:50:59 -0500582
David Jamesfcb70ef2011-02-02 16:02:30 -0800583 Args:
584 packages: Tree(s) of dependencies.
Mike Frysinger1a736a82013-12-12 01:50:59 -0500585
David Jamesfcb70ef2011-02-02 16:02:30 -0800586 Returns:
587 Unsanitized digraph.
588 """
David James8c7e5e32011-06-28 11:26:03 -0700589 binpkg_phases = set(["setup", "preinst", "postinst"])
David Jamese5e1c0a2014-09-29 17:19:41 -0700590 needed_dep_types = set(["blocker", "buildtime", "buildtime_slot_op",
591 "runtime", "runtime_slot_op"])
592 ignored_dep_types = set(["ignored", "optional", "runtime_post", "soft"])
593 all_dep_types = ignored_dep_types | needed_dep_types
David Jamesfcb70ef2011-02-02 16:02:30 -0800594 for pkg in packages:
595
596 # Create an entry for the package
597 action = packages[pkg]["action"]
David James8c7e5e32011-06-28 11:26:03 -0700598 default_pkg = {"needs": {}, "provides": set(), "action": action,
599 "nodeps": False, "binary": False}
David Jamesfcb70ef2011-02-02 16:02:30 -0800600 this_pkg = deps_map.setdefault(pkg, default_pkg)
601
David James8c7e5e32011-06-28 11:26:03 -0700602 if pkg in deps_info:
603 this_pkg["idx"] = deps_info[pkg]["idx"]
604
605 # If a package doesn't have any defined phases that might use the
606 # dependent packages (i.e. pkg_setup, pkg_preinst, or pkg_postinst),
607 # we can install this package before its deps are ready.
608 emerge_pkg = self.package_db.get(pkg)
609 if emerge_pkg and emerge_pkg.type_name == "binary":
610 this_pkg["binary"] = True
Mike Frysinger66652ec2014-04-24 11:42:25 -0400611 defined_phases = emerge_pkg.defined_phases
David James8c7e5e32011-06-28 11:26:03 -0700612 defined_binpkg_phases = binpkg_phases.intersection(defined_phases)
613 if not defined_binpkg_phases:
614 this_pkg["nodeps"] = True
615
David Jamesfcb70ef2011-02-02 16:02:30 -0800616 # Create entries for dependencies of this package first.
617 ReverseTree(packages[pkg]["deps"])
618
619 # Add dependencies to this package.
620 for dep, dep_item in packages[pkg]["deps"].iteritems():
David James8c7e5e32011-06-28 11:26:03 -0700621 # We only need to enforce strict ordering of dependencies if the
David James3f778802011-08-25 19:31:45 -0700622 # dependency is a blocker, or is a buildtime or runtime dependency.
623 # (I.e., ignored, optional, and runtime_post dependencies don't
624 # depend on ordering.)
David James8c7e5e32011-06-28 11:26:03 -0700625 dep_types = dep_item["deptypes"]
626 if needed_dep_types.intersection(dep_types):
627 deps_map[dep]["provides"].add(pkg)
628 this_pkg["needs"][dep] = "/".join(dep_types)
David Jamesfcb70ef2011-02-02 16:02:30 -0800629
David Jamese5e1c0a2014-09-29 17:19:41 -0700630 # Verify we processed all appropriate dependency types.
631 unknown_dep_types = set(dep_types) - all_dep_types
632 if unknown_dep_types:
633 print("Unknown dependency types found:")
634 print(" %s -> %s (%s)" % (pkg, dep, "/".join(unknown_dep_types)))
635 sys.exit(1)
636
David James3f778802011-08-25 19:31:45 -0700637 # If there's a blocker, Portage may need to move files from one
638 # package to another, which requires editing the CONTENTS files of
639 # both packages. To avoid race conditions while editing this file,
640 # the two packages must not be installed in parallel, so we can't
641 # safely ignore dependencies. See http://crosbug.com/19328
642 if "blocker" in dep_types:
643 this_pkg["nodeps"] = False
644
David Jamesfcb70ef2011-02-02 16:02:30 -0800645 def FindCycles():
646 """Find cycles in the dependency tree.
647
648 Returns:
649 A dict mapping cyclic packages to a dict of the deps that cause
650 cycles. For each dep that causes cycles, it returns an example
651 traversal of the graph that shows the cycle.
652 """
653
654 def FindCyclesAtNode(pkg, cycles, unresolved, resolved):
655 """Find cycles in cyclic dependencies starting at specified package.
656
657 Args:
658 pkg: Package identifier.
659 cycles: A dict mapping cyclic packages to a dict of the deps that
660 cause cycles. For each dep that causes cycles, it returns an
661 example traversal of the graph that shows the cycle.
662 unresolved: Nodes that have been visited but are not fully processed.
663 resolved: Nodes that have been visited and are fully processed.
664 """
665 pkg_cycles = cycles.get(pkg)
666 if pkg in resolved and not pkg_cycles:
667 # If we already looked at this package, and found no cyclic
668 # dependencies, we can stop now.
669 return
670 unresolved.append(pkg)
671 for dep in deps_map[pkg]["needs"]:
672 if dep in unresolved:
673 idx = unresolved.index(dep)
674 mycycle = unresolved[idx:] + [dep]
David James321490a2012-12-17 12:05:56 -0800675 for i in xrange(len(mycycle) - 1):
David Jamesfcb70ef2011-02-02 16:02:30 -0800676 pkg1, pkg2 = mycycle[i], mycycle[i+1]
677 cycles.setdefault(pkg1, {}).setdefault(pkg2, mycycle)
678 elif not pkg_cycles or dep not in pkg_cycles:
679 # Looks like we haven't seen this edge before.
680 FindCyclesAtNode(dep, cycles, unresolved, resolved)
681 unresolved.pop()
682 resolved.add(pkg)
683
684 cycles, unresolved, resolved = {}, [], set()
685 for pkg in deps_map:
686 FindCyclesAtNode(pkg, cycles, unresolved, resolved)
687 return cycles
688
David James386ccd12011-05-04 20:17:42 -0700689 def RemoveUnusedPackages():
David Jamesfcb70ef2011-02-02 16:02:30 -0800690 """Remove installed packages, propagating dependencies."""
David Jamesfcb70ef2011-02-02 16:02:30 -0800691 # Schedule packages that aren't on the install list for removal
692 rm_pkgs = set(deps_map.keys()) - set(deps_info.keys())
693
David Jamesfcb70ef2011-02-02 16:02:30 -0800694 # Remove the packages we don't want, simplifying the graph and making
695 # it easier for us to crack cycles.
696 for pkg in sorted(rm_pkgs):
697 this_pkg = deps_map[pkg]
698 needs = this_pkg["needs"]
699 provides = this_pkg["provides"]
700 for dep in needs:
701 dep_provides = deps_map[dep]["provides"]
702 dep_provides.update(provides)
703 dep_provides.discard(pkg)
704 dep_provides.discard(dep)
705 for target in provides:
706 target_needs = deps_map[target]["needs"]
707 target_needs.update(needs)
708 target_needs.pop(pkg, None)
709 target_needs.pop(target, None)
710 del deps_map[pkg]
711
712 def PrintCycleBreak(basedep, dep, mycycle):
713 """Print details about a cycle that we are planning on breaking.
714
Mike Frysinger02e1e072013-11-10 22:11:34 -0500715 We are breaking a cycle where dep needs basedep. mycycle is an
716 example cycle which contains dep -> basedep.
717 """
David Jamesfcb70ef2011-02-02 16:02:30 -0800718
David Jamesfcb70ef2011-02-02 16:02:30 -0800719 needs = deps_map[dep]["needs"]
720 depinfo = needs.get(basedep, "deleted")
David Jamesfcb70ef2011-02-02 16:02:30 -0800721
David James3f778802011-08-25 19:31:45 -0700722 # It's OK to swap install order for blockers, as long as the two
723 # packages aren't installed in parallel. If there is a cycle, then
724 # we know the packages depend on each other already, so we can drop the
725 # blocker safely without printing a warning.
726 if depinfo == "blocker":
727 return
728
David Jamesfcb70ef2011-02-02 16:02:30 -0800729 # Notify the user that we're breaking a cycle.
Mike Frysinger383367e2014-09-16 15:06:17 -0400730 print("Breaking %s -> %s (%s)" % (dep, basedep, depinfo))
David Jamesfcb70ef2011-02-02 16:02:30 -0800731
732 # Show cycle.
David James321490a2012-12-17 12:05:56 -0800733 for i in xrange(len(mycycle) - 1):
David Jamesfcb70ef2011-02-02 16:02:30 -0800734 pkg1, pkg2 = mycycle[i], mycycle[i+1]
735 needs = deps_map[pkg1]["needs"]
736 depinfo = needs.get(pkg2, "deleted")
737 if pkg1 == dep and pkg2 == basedep:
738 depinfo = depinfo + ", deleting"
Mike Frysinger383367e2014-09-16 15:06:17 -0400739 print(" %s -> %s (%s)" % (pkg1, pkg2, depinfo))
David Jamesfcb70ef2011-02-02 16:02:30 -0800740
741 def SanitizeTree():
742 """Remove circular dependencies.
743
744 We prune all dependencies involved in cycles that go against the emerge
745 ordering. This has a nice property: we're guaranteed to merge
746 dependencies in the same order that portage does.
747
748 Because we don't treat any dependencies as "soft" unless they're killed
749 by a cycle, we pay attention to a larger number of dependencies when
750 merging. This hurts performance a bit, but helps reliability.
751 """
752 start = time.time()
753 cycles = FindCycles()
754 while cycles:
755 for dep, mycycles in cycles.iteritems():
756 for basedep, mycycle in mycycles.iteritems():
757 if deps_info[basedep]["idx"] >= deps_info[dep]["idx"]:
Matt Tennant08797302011-10-17 16:18:45 -0700758 if "--quiet" not in emerge.opts:
759 PrintCycleBreak(basedep, dep, mycycle)
David Jamesfcb70ef2011-02-02 16:02:30 -0800760 del deps_map[dep]["needs"][basedep]
761 deps_map[basedep]["provides"].remove(dep)
762 cycles = FindCycles()
763 seconds = time.time() - start
764 if "--quiet" not in emerge.opts and seconds >= 0.1:
Mike Frysinger383367e2014-09-16 15:06:17 -0400765 print("Tree sanitized in %dm%.1fs" % (seconds / 60, seconds % 60))
David Jamesfcb70ef2011-02-02 16:02:30 -0800766
David James8c7e5e32011-06-28 11:26:03 -0700767 def FindRecursiveProvides(pkg, seen):
768 """Find all nodes that require a particular package.
769
770 Assumes that graph is acyclic.
771
772 Args:
773 pkg: Package identifier.
774 seen: Nodes that have been visited so far.
775 """
776 if pkg in seen:
777 return
778 seen.add(pkg)
779 info = deps_map[pkg]
780 info["tprovides"] = info["provides"].copy()
781 for dep in info["provides"]:
782 FindRecursiveProvides(dep, seen)
783 info["tprovides"].update(deps_map[dep]["tprovides"])
784
David Jamesa22906f2011-05-04 19:53:26 -0700785 ReverseTree(deps_tree)
David Jamesa22906f2011-05-04 19:53:26 -0700786
David James386ccd12011-05-04 20:17:42 -0700787 # We need to remove unused packages so that we can use the dependency
788 # ordering of the install process to show us what cycles to crack.
789 RemoveUnusedPackages()
David Jamesfcb70ef2011-02-02 16:02:30 -0800790 SanitizeTree()
David James8c7e5e32011-06-28 11:26:03 -0700791 seen = set()
792 for pkg in deps_map:
793 FindRecursiveProvides(pkg, seen)
David Jamesfcb70ef2011-02-02 16:02:30 -0800794 return deps_map
795
796 def PrintInstallPlan(self, deps_map):
797 """Print an emerge-style install plan.
798
799 The install plan lists what packages we're installing, in order.
800 It's useful for understanding what parallel_emerge is doing.
801
802 Args:
803 deps_map: The dependency graph.
804 """
805
806 def InstallPlanAtNode(target, deps_map):
807 nodes = []
808 nodes.append(target)
809 for dep in deps_map[target]["provides"]:
810 del deps_map[dep]["needs"][target]
811 if not deps_map[dep]["needs"]:
812 nodes.extend(InstallPlanAtNode(dep, deps_map))
813 return nodes
814
815 deps_map = copy.deepcopy(deps_map)
816 install_plan = []
817 plan = set()
818 for target, info in deps_map.iteritems():
819 if not info["needs"] and target not in plan:
820 for item in InstallPlanAtNode(target, deps_map):
821 plan.add(item)
822 install_plan.append(self.package_db[item])
823
824 for pkg in plan:
825 del deps_map[pkg]
826
827 if deps_map:
Mike Frysinger383367e2014-09-16 15:06:17 -0400828 print("Cyclic dependencies:", " ".join(deps_map))
David Jamesfcb70ef2011-02-02 16:02:30 -0800829 PrintDepsMap(deps_map)
830 sys.exit(1)
831
832 self.emerge.depgraph.display(install_plan)
833
834
835def PrintDepsMap(deps_map):
836 """Print dependency graph, for each package list it's prerequisites."""
837 for i in sorted(deps_map):
Mike Frysinger383367e2014-09-16 15:06:17 -0400838 print("%s: (%s) needs" % (i, deps_map[i]["action"]))
David Jamesfcb70ef2011-02-02 16:02:30 -0800839 needs = deps_map[i]["needs"]
840 for j in sorted(needs):
Mike Frysinger383367e2014-09-16 15:06:17 -0400841 print(" %s" % (j))
David Jamesfcb70ef2011-02-02 16:02:30 -0800842 if not needs:
Mike Frysinger383367e2014-09-16 15:06:17 -0400843 print(" no dependencies")
David Jamesfcb70ef2011-02-02 16:02:30 -0800844
845
846class EmergeJobState(object):
Don Garrett25f309a2014-03-19 14:02:12 -0700847 """Structure describing the EmergeJobState."""
848
David Jamesfcb70ef2011-02-02 16:02:30 -0800849 __slots__ = ["done", "filename", "last_notify_timestamp", "last_output_seek",
850 "last_output_timestamp", "pkgname", "retcode", "start_timestamp",
Thiago Goncalesf4acc422013-07-17 10:26:35 -0700851 "target", "fetch_only", "unpack_only"]
David Jamesfcb70ef2011-02-02 16:02:30 -0800852
853 def __init__(self, target, pkgname, done, filename, start_timestamp,
Thiago Goncalesf4acc422013-07-17 10:26:35 -0700854 retcode=None, fetch_only=False, unpack_only=False):
David Jamesfcb70ef2011-02-02 16:02:30 -0800855
856 # The full name of the target we're building (e.g.
Mike Frysingerfd969312014-04-02 22:16:42 -0400857 # virtual/target-os-1-r60)
David Jamesfcb70ef2011-02-02 16:02:30 -0800858 self.target = target
859
Mike Frysingerfd969312014-04-02 22:16:42 -0400860 # The short name of the target we're building (e.g. target-os-1-r60)
David Jamesfcb70ef2011-02-02 16:02:30 -0800861 self.pkgname = pkgname
862
863 # Whether the job is done. (True if the job is done; false otherwise.)
864 self.done = done
865
866 # The filename where output is currently stored.
867 self.filename = filename
868
869 # The timestamp of the last time we printed the name of the log file. We
870 # print this at the beginning of the job, so this starts at
871 # start_timestamp.
872 self.last_notify_timestamp = start_timestamp
873
874 # The location (in bytes) of the end of the last complete line we printed.
875 # This starts off at zero. We use this to jump to the right place when we
876 # print output from the same ebuild multiple times.
877 self.last_output_seek = 0
878
879 # The timestamp of the last time we printed output. Since we haven't
880 # printed output yet, this starts at zero.
881 self.last_output_timestamp = 0
882
883 # The return code of our job, if the job is actually finished.
884 self.retcode = retcode
885
Brian Harring0be85c62012-03-17 19:52:12 -0700886 # Was this just a fetch job?
887 self.fetch_only = fetch_only
888
David Jamesfcb70ef2011-02-02 16:02:30 -0800889 # The timestamp when our job started.
890 self.start_timestamp = start_timestamp
891
Thiago Goncalesf4acc422013-07-17 10:26:35 -0700892 # No emerge, only unpack packages.
893 self.unpack_only = unpack_only
894
David Jamesfcb70ef2011-02-02 16:02:30 -0800895
David James321490a2012-12-17 12:05:56 -0800896def KillHandler(_signum, _frame):
David James7358d032011-05-19 10:40:03 -0700897 # Kill self and all subprocesses.
898 os.killpg(0, signal.SIGKILL)
899
Mike Frysingercc838832014-05-24 13:10:30 -0400900
David Jamesfcb70ef2011-02-02 16:02:30 -0800901def SetupWorkerSignals():
David James321490a2012-12-17 12:05:56 -0800902 def ExitHandler(_signum, _frame):
David James7358d032011-05-19 10:40:03 -0700903 # Set KILLED flag.
904 KILLED.set()
David James13cead42011-05-18 16:22:01 -0700905
David James7358d032011-05-19 10:40:03 -0700906 # Remove our signal handlers so we don't get called recursively.
907 signal.signal(signal.SIGINT, KillHandler)
908 signal.signal(signal.SIGTERM, KillHandler)
David Jamesfcb70ef2011-02-02 16:02:30 -0800909
910 # Ensure that we exit quietly and cleanly, if possible, when we receive
911 # SIGTERM or SIGINT signals. By default, when the user hits CTRL-C, all
912 # of the child processes will print details about KeyboardInterrupt
913 # exceptions, which isn't very helpful.
914 signal.signal(signal.SIGINT, ExitHandler)
915 signal.signal(signal.SIGTERM, ExitHandler)
916
Mike Frysingerd74fe4a2014-04-24 11:43:38 -0400917
918def EmergeProcess(output, target, *args, **kwargs):
David James1ed3e252011-10-05 20:26:15 -0700919 """Merge a package in a subprocess.
920
921 Args:
David James1ed3e252011-10-05 20:26:15 -0700922 output: Temporary file to write output.
Mike Frysingerd74fe4a2014-04-24 11:43:38 -0400923 target: The package we'll be processing (for display purposes).
David James6b29d052012-11-02 10:27:27 -0700924 *args: Arguments to pass to Scheduler constructor.
925 **kwargs: Keyword arguments to pass to Scheduler constructor.
David James1ed3e252011-10-05 20:26:15 -0700926
927 Returns:
928 The exit code returned by the subprocess.
929 """
Chris Ching151617c2017-01-25 16:57:06 -0700930 event = cros_event.newEvent(kind="EmergePackage", package=target)
David James1ed3e252011-10-05 20:26:15 -0700931 pid = os.fork()
932 if pid == 0:
933 try:
Mike Frysingerd74fe4a2014-04-24 11:43:38 -0400934 proctitle.settitle('EmergeProcess', target)
935
David James1ed3e252011-10-05 20:26:15 -0700936 # Sanity checks.
Mike Frysingerf02736e2013-11-08 15:27:00 -0500937 if sys.stdout.fileno() != 1:
938 raise Exception("sys.stdout.fileno() != 1")
939 if sys.stderr.fileno() != 2:
940 raise Exception("sys.stderr.fileno() != 2")
David James1ed3e252011-10-05 20:26:15 -0700941
942 # - Redirect 1 (stdout) and 2 (stderr) at our temporary file.
943 # - Redirect 0 to point at sys.stdin. In this case, sys.stdin
944 # points at a file reading os.devnull, because multiprocessing mucks
945 # with sys.stdin.
946 # - Leave the sys.stdin and output filehandles alone.
947 fd_pipes = {0: sys.stdin.fileno(),
948 1: output.fileno(),
949 2: output.fileno(),
950 sys.stdin.fileno(): sys.stdin.fileno(),
951 output.fileno(): output.fileno()}
Mike Frysinger66652ec2014-04-24 11:42:25 -0400952 # pylint: disable=W0212
953 portage.process._setup_pipes(fd_pipes, close_fds=False)
David James1ed3e252011-10-05 20:26:15 -0700954
955 # Portage doesn't like when sys.stdin.fileno() != 0, so point sys.stdin
956 # at the filehandle we just created in _setup_pipes.
957 if sys.stdin.fileno() != 0:
David James6b29d052012-11-02 10:27:27 -0700958 sys.__stdin__ = sys.stdin = os.fdopen(0, "r")
959
960 scheduler = Scheduler(*args, **kwargs)
961
962 # Enable blocker handling even though we're in --nodeps mode. This
963 # allows us to unmerge the blocker after we've merged the replacement.
964 scheduler._opts_ignore_blockers = frozenset()
David James1ed3e252011-10-05 20:26:15 -0700965
966 # Actually do the merge.
Chris Ching5fcbd622016-11-28 09:22:15 -0700967 with event:
968 retval = scheduler.merge()
969 if retval != 0:
970 event.fail(message="non-zero value returned")
David James1ed3e252011-10-05 20:26:15 -0700971
972 # We catch all exceptions here (including SystemExit, KeyboardInterrupt,
973 # etc) so as to ensure that we don't confuse the multiprocessing module,
974 # which expects that all forked children exit with os._exit().
David James321490a2012-12-17 12:05:56 -0800975 # pylint: disable=W0702
David James1ed3e252011-10-05 20:26:15 -0700976 except:
977 traceback.print_exc(file=output)
978 retval = 1
979 sys.stdout.flush()
980 sys.stderr.flush()
981 output.flush()
Don Garrett25f309a2014-03-19 14:02:12 -0700982 # pylint: disable=W0212
David James1ed3e252011-10-05 20:26:15 -0700983 os._exit(retval)
984 else:
985 # Return the exit code of the subprocess.
986 return os.waitpid(pid, 0)[1]
David Jamesfcb70ef2011-02-02 16:02:30 -0800987
Thiago Goncalesf4acc422013-07-17 10:26:35 -0700988
989def UnpackPackage(pkg_state):
990 """Unpacks package described by pkg_state.
991
992 Args:
993 pkg_state: EmergeJobState object describing target.
994
995 Returns:
996 Exit code returned by subprocess.
997 """
998 pkgdir = os.environ.get("PKGDIR",
999 os.path.join(os.environ["SYSROOT"], "packages"))
1000 root = os.environ.get("ROOT", os.environ["SYSROOT"])
1001 path = os.path.join(pkgdir, pkg_state.target + ".tbz2")
1002 comp = cros_build_lib.FindCompressor(cros_build_lib.COMP_BZIP2)
1003 cmd = [comp, "-dc"]
1004 if comp.endswith("pbzip2"):
1005 cmd.append("--ignore-trailing-garbage=1")
1006 cmd.append(path)
1007
Chris Ching5fcbd622016-11-28 09:22:15 -07001008 with cros_event.newEvent(step="unpack package", **pkg_state) as event:
1009 result = cros_build_lib.RunCommand(cmd, cwd=root, stdout_to_pipe=True,
1010 print_cmd=False, error_code_ok=True)
Thiago Goncalesf4acc422013-07-17 10:26:35 -07001011
Chris Ching5fcbd622016-11-28 09:22:15 -07001012 # If we were not successful, return now and don't attempt untar.
1013 if result.returncode != 0:
1014 event.fail("error compressing: returned {}".format(result.returncode))
1015 return result.returncode
1016
1017 cmd = ["sudo", "tar", "-xf", "-", "-C", root]
1018
1019 result = cros_build_lib.RunCommand(cmd, cwd=root, input=result.output,
1020 print_cmd=False, error_code_ok=True)
1021 if result.returncode != 0:
1022 event.fail("error extracting:returned {}".format(result.returncode))
1023
Thiago Goncalesf4acc422013-07-17 10:26:35 -07001024 return result.returncode
1025
Thiago Goncalesf4acc422013-07-17 10:26:35 -07001026
1027def EmergeWorker(task_queue, job_queue, emerge, package_db, fetch_only=False,
1028 unpack_only=False):
David Jamesfcb70ef2011-02-02 16:02:30 -08001029 """This worker emerges any packages given to it on the task_queue.
1030
1031 Args:
1032 task_queue: The queue of tasks for this worker to do.
1033 job_queue: The queue of results from the worker.
1034 emerge: An EmergeData() object.
1035 package_db: A dict, mapping package ids to portage Package objects.
Brian Harring0be85c62012-03-17 19:52:12 -07001036 fetch_only: A bool, indicating if we should just fetch the target.
Thiago Goncalesf4acc422013-07-17 10:26:35 -07001037 unpack_only: A bool, indicating if we should just unpack the target.
David Jamesfcb70ef2011-02-02 16:02:30 -08001038
1039 It expects package identifiers to be passed to it via task_queue. When
1040 a task is started, it pushes the (target, filename) to the started_queue.
1041 The output is stored in filename. When a merge starts or finishes, we push
1042 EmergeJobState objects to the job_queue.
1043 """
Mike Frysingerd74fe4a2014-04-24 11:43:38 -04001044 if fetch_only:
1045 mode = 'fetch'
1046 elif unpack_only:
1047 mode = 'unpack'
1048 else:
1049 mode = 'emerge'
1050 proctitle.settitle('EmergeWorker', mode, '[idle]')
David Jamesfcb70ef2011-02-02 16:02:30 -08001051
1052 SetupWorkerSignals()
1053 settings, trees, mtimedb = emerge.settings, emerge.trees, emerge.mtimedb
David Jamesdeebd692011-05-09 17:02:52 -07001054
1055 # Disable flushing of caches to save on I/O.
David James7a1ea4b2011-10-13 15:06:41 -07001056 root = emerge.settings["ROOT"]
1057 vardb = emerge.trees[root]["vartree"].dbapi
Mike Frysingere56debd2014-11-19 01:54:36 -05001058 vardb._flush_cache_enabled = False # pylint: disable=protected-access
Brian Harring0be85c62012-03-17 19:52:12 -07001059 bindb = emerge.trees[root]["bintree"].dbapi
1060 # Might be a set, might be a list, might be None; no clue, just use shallow
1061 # copy to ensure we can roll it back.
Don Garrett25f309a2014-03-19 14:02:12 -07001062 # pylint: disable=W0212
Brian Harring0be85c62012-03-17 19:52:12 -07001063 original_remotepkgs = copy.copy(bindb.bintree._remotepkgs)
David Jamesdeebd692011-05-09 17:02:52 -07001064
David Jamesfcb70ef2011-02-02 16:02:30 -08001065 opts, spinner = emerge.opts, emerge.spinner
1066 opts["--nodeps"] = True
Brian Harring0be85c62012-03-17 19:52:12 -07001067 if fetch_only:
1068 opts["--fetchonly"] = True
1069
David Jamesfcb70ef2011-02-02 16:02:30 -08001070 while True:
1071 # Wait for a new item to show up on the queue. This is a blocking wait,
1072 # so if there's nothing to do, we just sit here.
Brian Harring0be85c62012-03-17 19:52:12 -07001073 pkg_state = task_queue.get()
1074 if pkg_state is None:
David Jamesfcb70ef2011-02-02 16:02:30 -08001075 # If target is None, this means that the main thread wants us to quit.
1076 # The other workers need to exit too, so we'll push the message back on
1077 # to the queue so they'll get it too.
Brian Harring0be85c62012-03-17 19:52:12 -07001078 task_queue.put(None)
David Jamesfcb70ef2011-02-02 16:02:30 -08001079 return
David James7358d032011-05-19 10:40:03 -07001080 if KILLED.is_set():
1081 return
1082
Brian Harring0be85c62012-03-17 19:52:12 -07001083 target = pkg_state.target
Mike Frysingerd74fe4a2014-04-24 11:43:38 -04001084 proctitle.settitle('EmergeWorker', mode, target)
Brian Harring0be85c62012-03-17 19:52:12 -07001085
David Jamesfcb70ef2011-02-02 16:02:30 -08001086 db_pkg = package_db[target]
Brian Harring0be85c62012-03-17 19:52:12 -07001087
1088 if db_pkg.type_name == "binary":
1089 if not fetch_only and pkg_state.fetched_successfully:
1090 # Ensure portage doesn't think our pkg is remote- else it'll force
1091 # a redownload of it (even if the on-disk file is fine). In-memory
1092 # caching basically, implemented dumbly.
1093 bindb.bintree._remotepkgs = None
1094 else:
1095 bindb.bintree_remotepkgs = original_remotepkgs
1096
David Jamesfcb70ef2011-02-02 16:02:30 -08001097 db_pkg.root_config = emerge.root_config
1098 install_list = [db_pkg]
1099 pkgname = db_pkg.pf
1100 output = tempfile.NamedTemporaryFile(prefix=pkgname + "-", delete=False)
David James01b1e0f2012-06-07 17:18:05 -07001101 os.chmod(output.name, 644)
David Jamesfcb70ef2011-02-02 16:02:30 -08001102 start_timestamp = time.time()
Brian Harring0be85c62012-03-17 19:52:12 -07001103 job = EmergeJobState(target, pkgname, False, output.name, start_timestamp,
Thiago Goncalesf4acc422013-07-17 10:26:35 -07001104 fetch_only=fetch_only, unpack_only=unpack_only)
David Jamesfcb70ef2011-02-02 16:02:30 -08001105 job_queue.put(job)
1106 if "--pretend" in opts:
1107 retcode = 0
1108 else:
David Jamesfcb70ef2011-02-02 16:02:30 -08001109 try:
David James386ccd12011-05-04 20:17:42 -07001110 emerge.scheduler_graph.mergelist = install_list
Thiago Goncalesf4acc422013-07-17 10:26:35 -07001111 if unpack_only:
1112 retcode = UnpackPackage(pkg_state)
1113 else:
Mike Frysingerd74fe4a2014-04-24 11:43:38 -04001114 retcode = EmergeProcess(output, target, settings, trees, mtimedb,
1115 opts, spinner, favorites=emerge.favorites,
Thiago Goncalesf4acc422013-07-17 10:26:35 -07001116 graph_config=emerge.scheduler_graph)
David Jamesfcb70ef2011-02-02 16:02:30 -08001117 except Exception:
1118 traceback.print_exc(file=output)
1119 retcode = 1
David James1ed3e252011-10-05 20:26:15 -07001120 output.close()
David Jamesfcb70ef2011-02-02 16:02:30 -08001121
David James7358d032011-05-19 10:40:03 -07001122 if KILLED.is_set():
1123 return
1124
David Jamesfcb70ef2011-02-02 16:02:30 -08001125 job = EmergeJobState(target, pkgname, True, output.name, start_timestamp,
Thiago Goncalesf4acc422013-07-17 10:26:35 -07001126 retcode, fetch_only=fetch_only,
1127 unpack_only=unpack_only)
David Jamesfcb70ef2011-02-02 16:02:30 -08001128 job_queue.put(job)
1129
Mike Frysingerd74fe4a2014-04-24 11:43:38 -04001130 # Set the title back to idle as the multiprocess pool won't destroy us;
1131 # when another job comes up, it'll re-use this process.
1132 proctitle.settitle('EmergeWorker', mode, '[idle]')
1133
David Jamesfcb70ef2011-02-02 16:02:30 -08001134
1135class LinePrinter(object):
1136 """Helper object to print a single line."""
1137
1138 def __init__(self, line):
1139 self.line = line
1140
David James321490a2012-12-17 12:05:56 -08001141 def Print(self, _seek_locations):
Mike Frysinger383367e2014-09-16 15:06:17 -04001142 print(self.line)
David Jamesfcb70ef2011-02-02 16:02:30 -08001143
1144
1145class JobPrinter(object):
1146 """Helper object to print output of a job."""
1147
1148 def __init__(self, job, unlink=False):
1149 """Print output of job.
1150
Mike Frysinger02e1e072013-11-10 22:11:34 -05001151 If unlink is True, unlink the job output file when done.
1152 """
David Jamesfcb70ef2011-02-02 16:02:30 -08001153 self.current_time = time.time()
1154 self.job = job
1155 self.unlink = unlink
1156
1157 def Print(self, seek_locations):
1158
1159 job = self.job
1160
1161 # Calculate how long the job has been running.
1162 seconds = self.current_time - job.start_timestamp
1163
1164 # Note that we've printed out the job so far.
1165 job.last_output_timestamp = self.current_time
1166
1167 # Note that we're starting the job
1168 info = "job %s (%dm%.1fs)" % (job.pkgname, seconds / 60, seconds % 60)
1169 last_output_seek = seek_locations.get(job.filename, 0)
1170 if last_output_seek:
Mike Frysinger383367e2014-09-16 15:06:17 -04001171 print("=== Continue output for %s ===" % info)
David Jamesfcb70ef2011-02-02 16:02:30 -08001172 else:
Mike Frysinger383367e2014-09-16 15:06:17 -04001173 print("=== Start output for %s ===" % info)
David Jamesfcb70ef2011-02-02 16:02:30 -08001174
1175 # Print actual output from job
1176 f = codecs.open(job.filename, encoding='utf-8', errors='replace')
1177 f.seek(last_output_seek)
1178 prefix = job.pkgname + ":"
1179 for line in f:
1180
1181 # Save off our position in the file
1182 if line and line[-1] == "\n":
1183 last_output_seek = f.tell()
1184 line = line[:-1]
1185
1186 # Print our line
Mike Frysinger383367e2014-09-16 15:06:17 -04001187 print(prefix, line.encode('utf-8', 'replace'))
David Jamesfcb70ef2011-02-02 16:02:30 -08001188 f.close()
1189
1190 # Save our last spot in the file so that we don't print out the same
1191 # location twice.
1192 seek_locations[job.filename] = last_output_seek
1193
1194 # Note end of output section
1195 if job.done:
Mike Frysinger383367e2014-09-16 15:06:17 -04001196 print("=== Complete: %s ===" % info)
David Jamesfcb70ef2011-02-02 16:02:30 -08001197 else:
Mike Frysinger383367e2014-09-16 15:06:17 -04001198 print("=== Still running: %s ===" % info)
David Jamesfcb70ef2011-02-02 16:02:30 -08001199
1200 if self.unlink:
1201 os.unlink(job.filename)
1202
1203
1204def PrintWorker(queue):
1205 """A worker that prints stuff to the screen as requested."""
Mike Frysingerd74fe4a2014-04-24 11:43:38 -04001206 proctitle.settitle('PrintWorker')
David Jamesfcb70ef2011-02-02 16:02:30 -08001207
David James321490a2012-12-17 12:05:56 -08001208 def ExitHandler(_signum, _frame):
David James7358d032011-05-19 10:40:03 -07001209 # Set KILLED flag.
1210 KILLED.set()
1211
David Jamesfcb70ef2011-02-02 16:02:30 -08001212 # Switch to default signal handlers so that we'll die after two signals.
David James7358d032011-05-19 10:40:03 -07001213 signal.signal(signal.SIGINT, KillHandler)
1214 signal.signal(signal.SIGTERM, KillHandler)
David Jamesfcb70ef2011-02-02 16:02:30 -08001215
1216 # Don't exit on the first SIGINT / SIGTERM, because the parent worker will
1217 # handle it and tell us when we need to exit.
1218 signal.signal(signal.SIGINT, ExitHandler)
1219 signal.signal(signal.SIGTERM, ExitHandler)
1220
1221 # seek_locations is a map indicating the position we are at in each file.
1222 # It starts off empty, but is set by the various Print jobs as we go along
1223 # to indicate where we left off in each file.
1224 seek_locations = {}
1225 while True:
1226 try:
1227 job = queue.get()
1228 if job:
1229 job.Print(seek_locations)
David Jamesbccf8eb2011-07-27 14:06:06 -07001230 sys.stdout.flush()
David Jamesfcb70ef2011-02-02 16:02:30 -08001231 else:
1232 break
1233 except IOError as ex:
1234 if ex.errno == errno.EINTR:
1235 # Looks like we received a signal. Keep printing.
1236 continue
1237 raise
1238
Brian Harring867e2362012-03-17 04:05:17 -07001239
Brian Harring0be85c62012-03-17 19:52:12 -07001240class TargetState(object):
Chris Ching5fcbd622016-11-28 09:22:15 -07001241 """Structure describing the TargetState."""
Brian Harring867e2362012-03-17 04:05:17 -07001242
Brian Harring0be85c62012-03-17 19:52:12 -07001243 __slots__ = ("target", "info", "score", "prefetched", "fetched_successfully")
Brian Harring867e2362012-03-17 04:05:17 -07001244
David James321490a2012-12-17 12:05:56 -08001245 def __init__(self, target, info):
Brian Harring867e2362012-03-17 04:05:17 -07001246 self.target, self.info = target, info
Brian Harring0be85c62012-03-17 19:52:12 -07001247 self.fetched_successfully = False
1248 self.prefetched = False
David James321490a2012-12-17 12:05:56 -08001249 self.score = None
Brian Harring867e2362012-03-17 04:05:17 -07001250 self.update_score()
1251
1252 def __cmp__(self, other):
1253 return cmp(self.score, other.score)
1254
1255 def update_score(self):
1256 self.score = (
1257 -len(self.info["tprovides"]),
Brian Harring0be85c62012-03-17 19:52:12 -07001258 len(self.info["needs"]),
Brian Harring11c5eeb2012-03-18 11:02:39 -07001259 not self.info["binary"],
Brian Harring867e2362012-03-17 04:05:17 -07001260 -len(self.info["provides"]),
1261 self.info["idx"],
1262 self.target,
1263 )
1264
1265
1266class ScoredHeap(object):
Don Garrett25f309a2014-03-19 14:02:12 -07001267 """Implementation of a general purpose scored heap."""
Brian Harring867e2362012-03-17 04:05:17 -07001268
Brian Harring0be85c62012-03-17 19:52:12 -07001269 __slots__ = ("heap", "_heap_set")
1270
Brian Harring867e2362012-03-17 04:05:17 -07001271 def __init__(self, initial=()):
Brian Harring0be85c62012-03-17 19:52:12 -07001272 self.heap = list()
1273 self._heap_set = set()
1274 if initial:
1275 self.multi_put(initial)
Brian Harring867e2362012-03-17 04:05:17 -07001276
1277 def get(self):
Brian Harring0be85c62012-03-17 19:52:12 -07001278 item = heapq.heappop(self.heap)
1279 self._heap_set.remove(item.target)
1280 return item
Brian Harring867e2362012-03-17 04:05:17 -07001281
Brian Harring0be85c62012-03-17 19:52:12 -07001282 def put(self, item):
1283 if not isinstance(item, TargetState):
1284 raise ValueError("Item %r isn't a TargetState" % (item,))
1285 heapq.heappush(self.heap, item)
1286 self._heap_set.add(item.target)
Brian Harring867e2362012-03-17 04:05:17 -07001287
Brian Harring0be85c62012-03-17 19:52:12 -07001288 def multi_put(self, sequence):
1289 sequence = list(sequence)
1290 self.heap.extend(sequence)
1291 self._heap_set.update(x.target for x in sequence)
Brian Harring867e2362012-03-17 04:05:17 -07001292 self.sort()
1293
David James5c9996d2012-03-24 10:50:46 -07001294 def sort(self):
1295 heapq.heapify(self.heap)
1296
Brian Harring0be85c62012-03-17 19:52:12 -07001297 def __contains__(self, target):
1298 return target in self._heap_set
1299
1300 def __nonzero__(self):
1301 return bool(self.heap)
1302
Brian Harring867e2362012-03-17 04:05:17 -07001303 def __len__(self):
1304 return len(self.heap)
1305
1306
David Jamesfcb70ef2011-02-02 16:02:30 -08001307class EmergeQueue(object):
1308 """Class to schedule emerge jobs according to a dependency graph."""
1309
Bertrand SIMONNET411945d2015-05-20 17:23:28 -07001310 def __init__(self, deps_map, emerge, package_db, show_output, unpack_only,
1311 max_retries):
David Jamesfcb70ef2011-02-02 16:02:30 -08001312 # Store the dependency graph.
1313 self._deps_map = deps_map
Brian Harring0be85c62012-03-17 19:52:12 -07001314 self._state_map = {}
David Jamesfcb70ef2011-02-02 16:02:30 -08001315 # Initialize the running queue to empty
Brian Harring0be85c62012-03-17 19:52:12 -07001316 self._build_jobs = {}
1317 self._build_ready = ScoredHeap()
1318 self._fetch_jobs = {}
1319 self._fetch_ready = ScoredHeap()
Thiago Goncalesf4acc422013-07-17 10:26:35 -07001320 self._unpack_jobs = {}
1321 self._unpack_ready = ScoredHeap()
David Jamesfcb70ef2011-02-02 16:02:30 -08001322 # List of total package installs represented in deps_map.
1323 install_jobs = [x for x in deps_map if deps_map[x]["action"] == "merge"]
1324 self._total_jobs = len(install_jobs)
1325 self._show_output = show_output
Thiago Goncalesf4acc422013-07-17 10:26:35 -07001326 self._unpack_only = unpack_only
Bertrand SIMONNET411945d2015-05-20 17:23:28 -07001327 self._max_retries = max_retries
David Jamesfcb70ef2011-02-02 16:02:30 -08001328
1329 if "--pretend" in emerge.opts:
Mike Frysinger383367e2014-09-16 15:06:17 -04001330 print("Skipping merge because of --pretend mode.")
David Jamesfcb70ef2011-02-02 16:02:30 -08001331 sys.exit(0)
1332
David Jamesaaf49e42014-04-24 09:40:05 -07001333 # Set up a session so we can easily terminate all children.
1334 self._SetupSession()
David James7358d032011-05-19 10:40:03 -07001335
David Jamesfcb70ef2011-02-02 16:02:30 -08001336 # Setup scheduler graph object. This is used by the child processes
1337 # to help schedule jobs.
1338 emerge.scheduler_graph = emerge.depgraph.schedulerGraph()
1339
1340 # Calculate how many jobs we can run in parallel. We don't want to pass
1341 # the --jobs flag over to emerge itself, because that'll tell emerge to
1342 # hide its output, and said output is quite useful for debugging hung
1343 # jobs.
1344 procs = min(self._total_jobs,
1345 emerge.opts.pop("--jobs", multiprocessing.cpu_count()))
Nam T. Nguyenf7098b32015-05-08 11:04:48 -07001346 self._build_procs = self._unpack_procs = max(1, procs)
1347 # Fetch is IO bound, we can use more processes.
1348 self._fetch_procs = max(4, procs)
David James8c7e5e32011-06-28 11:26:03 -07001349 self._load_avg = emerge.opts.pop("--load-average", None)
David Jamesfcb70ef2011-02-02 16:02:30 -08001350 self._job_queue = multiprocessing.Queue()
1351 self._print_queue = multiprocessing.Queue()
Brian Harring0be85c62012-03-17 19:52:12 -07001352
1353 self._fetch_queue = multiprocessing.Queue()
1354 args = (self._fetch_queue, self._job_queue, emerge, package_db, True)
1355 self._fetch_pool = multiprocessing.Pool(self._fetch_procs, EmergeWorker,
1356 args)
1357
1358 self._build_queue = multiprocessing.Queue()
1359 args = (self._build_queue, self._job_queue, emerge, package_db)
1360 self._build_pool = multiprocessing.Pool(self._build_procs, EmergeWorker,
1361 args)
1362
Thiago Goncalesf4acc422013-07-17 10:26:35 -07001363 if self._unpack_only:
1364 # Unpack pool only required on unpack_only jobs.
1365 self._unpack_queue = multiprocessing.Queue()
1366 args = (self._unpack_queue, self._job_queue, emerge, package_db, False,
1367 True)
1368 self._unpack_pool = multiprocessing.Pool(self._unpack_procs, EmergeWorker,
1369 args)
1370
David Jamesfcb70ef2011-02-02 16:02:30 -08001371 self._print_worker = multiprocessing.Process(target=PrintWorker,
1372 args=[self._print_queue])
1373 self._print_worker.start()
1374
1375 # Initialize the failed queue to empty.
1376 self._retry_queue = []
Bertrand SIMONNET411945d2015-05-20 17:23:28 -07001377 self._failed_count = dict()
David Jamesfcb70ef2011-02-02 16:02:30 -08001378
David Jamesfcb70ef2011-02-02 16:02:30 -08001379 # Setup an exit handler so that we print nice messages if we are
1380 # terminated.
1381 self._SetupExitHandler()
1382
1383 # Schedule our jobs.
Brian Harring0be85c62012-03-17 19:52:12 -07001384 self._state_map.update(
1385 (pkg, TargetState(pkg, data)) for pkg, data in deps_map.iteritems())
1386 self._fetch_ready.multi_put(self._state_map.itervalues())
David Jamesfcb70ef2011-02-02 16:02:30 -08001387
David Jamesaaf49e42014-04-24 09:40:05 -07001388 def _SetupSession(self):
1389 """Set up a session so we can easily terminate all children."""
1390 # When we call os.setsid(), this sets up a session / process group for this
1391 # process and all children. These session groups are needed so that we can
1392 # easily kill all children (including processes launched by emerge) before
1393 # we exit.
1394 #
1395 # One unfortunate side effect of os.setsid() is that it blocks CTRL-C from
1396 # being received. To work around this, we only call os.setsid() in a forked
1397 # process, so that the parent can still watch for CTRL-C. The parent will
1398 # just sit around, watching for signals and propagating them to the child,
1399 # until the child exits.
1400 #
1401 # TODO(davidjames): It would be nice if we could replace this with cgroups.
1402 pid = os.fork()
1403 if pid == 0:
1404 os.setsid()
1405 else:
Mike Frysingerd74fe4a2014-04-24 11:43:38 -04001406 proctitle.settitle('SessionManager')
1407
David Jamesaaf49e42014-04-24 09:40:05 -07001408 def PropagateToChildren(signum, _frame):
1409 # Just propagate the signals down to the child. We'll exit when the
1410 # child does.
1411 try:
1412 os.kill(pid, signum)
1413 except OSError as ex:
1414 if ex.errno != errno.ESRCH:
1415 raise
1416 signal.signal(signal.SIGINT, PropagateToChildren)
1417 signal.signal(signal.SIGTERM, PropagateToChildren)
1418
1419 def StopGroup(_signum, _frame):
1420 # When we get stopped, stop the children.
1421 try:
1422 os.killpg(pid, signal.SIGSTOP)
1423 os.kill(0, signal.SIGSTOP)
1424 except OSError as ex:
1425 if ex.errno != errno.ESRCH:
1426 raise
1427 signal.signal(signal.SIGTSTP, StopGroup)
1428
1429 def ContinueGroup(_signum, _frame):
1430 # Launch the children again after being stopped.
1431 try:
1432 os.killpg(pid, signal.SIGCONT)
1433 except OSError as ex:
1434 if ex.errno != errno.ESRCH:
1435 raise
1436 signal.signal(signal.SIGCONT, ContinueGroup)
1437
1438 # Loop until the children exit. We exit with os._exit to be sure we
1439 # don't run any finalizers (those will be run by the child process.)
1440 # pylint: disable=W0212
1441 while True:
1442 try:
1443 # Wait for the process to exit. When it does, exit with the return
1444 # value of the subprocess.
Mike Frysingere2d8f0d2014-11-01 13:09:26 -04001445 os._exit(process_util.GetExitStatus(os.waitpid(pid, 0)[1]))
David Jamesaaf49e42014-04-24 09:40:05 -07001446 except OSError as ex:
1447 if ex.errno == errno.EINTR:
1448 continue
1449 traceback.print_exc()
1450 os._exit(1)
1451 except BaseException:
1452 traceback.print_exc()
1453 os._exit(1)
1454
David Jamesfcb70ef2011-02-02 16:02:30 -08001455 def _SetupExitHandler(self):
1456
David James321490a2012-12-17 12:05:56 -08001457 def ExitHandler(signum, _frame):
David James7358d032011-05-19 10:40:03 -07001458 # Set KILLED flag.
1459 KILLED.set()
David Jamesfcb70ef2011-02-02 16:02:30 -08001460
1461 # Kill our signal handlers so we don't get called recursively
David James7358d032011-05-19 10:40:03 -07001462 signal.signal(signal.SIGINT, KillHandler)
1463 signal.signal(signal.SIGTERM, KillHandler)
David Jamesfcb70ef2011-02-02 16:02:30 -08001464
1465 # Print our current job status
Brian Harring0be85c62012-03-17 19:52:12 -07001466 for job in self._build_jobs.itervalues():
David Jamesfcb70ef2011-02-02 16:02:30 -08001467 if job:
1468 self._print_queue.put(JobPrinter(job, unlink=True))
1469
1470 # Notify the user that we are exiting
1471 self._Print("Exiting on signal %s" % signum)
David James7358d032011-05-19 10:40:03 -07001472 self._print_queue.put(None)
1473 self._print_worker.join()
David Jamesfcb70ef2011-02-02 16:02:30 -08001474
1475 # Kill child threads, then exit.
David James7358d032011-05-19 10:40:03 -07001476 os.killpg(0, signal.SIGKILL)
David Jamesfcb70ef2011-02-02 16:02:30 -08001477 sys.exit(1)
1478
1479 # Print out job status when we are killed
1480 signal.signal(signal.SIGINT, ExitHandler)
1481 signal.signal(signal.SIGTERM, ExitHandler)
1482
Thiago Goncalesf4acc422013-07-17 10:26:35 -07001483 def _ScheduleUnpack(self, pkg_state):
1484 self._unpack_jobs[pkg_state.target] = None
1485 self._unpack_queue.put(pkg_state)
1486
Brian Harring0be85c62012-03-17 19:52:12 -07001487 def _Schedule(self, pkg_state):
David Jamesfcb70ef2011-02-02 16:02:30 -08001488 # We maintain a tree of all deps, if this doesn't need
David James8c7e5e32011-06-28 11:26:03 -07001489 # to be installed just free up its children and continue.
David Jamesfcb70ef2011-02-02 16:02:30 -08001490 # It is possible to reinstall deps of deps, without reinstalling
1491 # first level deps, like so:
Mike Frysingerfd969312014-04-02 22:16:42 -04001492 # virtual/target-os (merge) -> eselect (nomerge) -> python (merge)
Brian Harring0be85c62012-03-17 19:52:12 -07001493 this_pkg = pkg_state.info
1494 target = pkg_state.target
1495 if pkg_state.info is not None:
1496 if this_pkg["action"] == "nomerge":
1497 self._Finish(target)
1498 elif target not in self._build_jobs:
1499 # Kick off the build if it's marked to be built.
1500 self._build_jobs[target] = None
1501 self._build_queue.put(pkg_state)
1502 return True
David Jamesfcb70ef2011-02-02 16:02:30 -08001503
Thiago Goncalesf4acc422013-07-17 10:26:35 -07001504 def _ScheduleLoop(self, unpack_only=False):
1505 if unpack_only:
1506 ready_queue = self._unpack_ready
1507 jobs_queue = self._unpack_jobs
1508 procs = self._unpack_procs
1509 else:
1510 ready_queue = self._build_ready
1511 jobs_queue = self._build_jobs
1512 procs = self._build_procs
1513
David James8c7e5e32011-06-28 11:26:03 -07001514 # If the current load exceeds our desired load average, don't schedule
1515 # more than one job.
1516 if self._load_avg and os.getloadavg()[0] > self._load_avg:
1517 needed_jobs = 1
1518 else:
Thiago Goncalesf4acc422013-07-17 10:26:35 -07001519 needed_jobs = procs
David James8c7e5e32011-06-28 11:26:03 -07001520
1521 # Schedule more jobs.
Thiago Goncalesf4acc422013-07-17 10:26:35 -07001522 while ready_queue and len(jobs_queue) < needed_jobs:
1523 state = ready_queue.get()
1524 if unpack_only:
1525 self._ScheduleUnpack(state)
1526 else:
Bertrand SIMONNET411945d2015-05-20 17:23:28 -07001527 if state.target not in self._failed_count:
Thiago Goncalesf4acc422013-07-17 10:26:35 -07001528 self._Schedule(state)
David Jamesfcb70ef2011-02-02 16:02:30 -08001529
1530 def _Print(self, line):
1531 """Print a single line."""
1532 self._print_queue.put(LinePrinter(line))
1533
1534 def _Status(self):
1535 """Print status."""
1536 current_time = time.time()
Aviv Keshet3b381682015-11-12 13:15:06 -08001537 current_time_struct = time.localtime(current_time)
David Jamesfcb70ef2011-02-02 16:02:30 -08001538 no_output = True
1539
1540 # Print interim output every minute if --show-output is used. Otherwise,
1541 # print notifications about running packages every 2 minutes, and print
1542 # full output for jobs that have been running for 60 minutes or more.
1543 if self._show_output:
1544 interval = 60
1545 notify_interval = 0
1546 else:
1547 interval = 60 * 60
1548 notify_interval = 60 * 2
David James321490a2012-12-17 12:05:56 -08001549 for job in self._build_jobs.itervalues():
David Jamesfcb70ef2011-02-02 16:02:30 -08001550 if job:
1551 last_timestamp = max(job.start_timestamp, job.last_output_timestamp)
1552 if last_timestamp + interval < current_time:
1553 self._print_queue.put(JobPrinter(job))
1554 job.last_output_timestamp = current_time
1555 no_output = False
1556 elif (notify_interval and
1557 job.last_notify_timestamp + notify_interval < current_time):
1558 job_seconds = current_time - job.start_timestamp
1559 args = (job.pkgname, job_seconds / 60, job_seconds % 60, job.filename)
1560 info = "Still building %s (%dm%.1fs). Logs in %s" % args
1561 job.last_notify_timestamp = current_time
1562 self._Print(info)
1563 no_output = False
1564
1565 # If we haven't printed any messages yet, print a general status message
1566 # here.
1567 if no_output:
1568 seconds = current_time - GLOBAL_START
Brian Harring0be85c62012-03-17 19:52:12 -07001569 fjobs, fready = len(self._fetch_jobs), len(self._fetch_ready)
Thiago Goncalesf4acc422013-07-17 10:26:35 -07001570 ujobs, uready = len(self._unpack_jobs), len(self._unpack_ready)
Brian Harring0be85c62012-03-17 19:52:12 -07001571 bjobs, bready = len(self._build_jobs), len(self._build_ready)
1572 retries = len(self._retry_queue)
1573 pending = max(0, len(self._deps_map) - fjobs - bjobs)
1574 line = "Pending %s/%s, " % (pending, self._total_jobs)
1575 if fjobs or fready:
1576 line += "Fetching %s/%s, " % (fjobs, fready + fjobs)
Thiago Goncalesf4acc422013-07-17 10:26:35 -07001577 if ujobs or uready:
1578 line += "Unpacking %s/%s, " % (ujobs, uready + ujobs)
Brian Harring0be85c62012-03-17 19:52:12 -07001579 if bjobs or bready or retries:
1580 line += "Building %s/%s, " % (bjobs, bready + bjobs)
1581 if retries:
1582 line += "Retrying %s, " % (retries,)
Mike Frysingerd6e2df02014-11-26 02:55:04 -05001583 load = " ".join(str(x) for x in os.getloadavg())
Aviv Keshet3b381682015-11-12 13:15:06 -08001584 line += ("[Time %s | Elapsed %dm%.1fs | Load %s]" % (
1585 time.strftime('%H:%M:%S', current_time_struct), seconds / 60,
1586 seconds % 60, load))
Brian Harring0be85c62012-03-17 19:52:12 -07001587 self._Print(line)
David Jamesfcb70ef2011-02-02 16:02:30 -08001588
1589 def _Finish(self, target):
David James8c7e5e32011-06-28 11:26:03 -07001590 """Mark a target as completed and unblock dependencies."""
1591 this_pkg = self._deps_map[target]
1592 if this_pkg["needs"] and this_pkg["nodeps"]:
1593 # We got installed, but our deps have not been installed yet. Dependent
1594 # packages should only be installed when our needs have been fully met.
1595 this_pkg["action"] = "nomerge"
1596 else:
David James8c7e5e32011-06-28 11:26:03 -07001597 for dep in this_pkg["provides"]:
1598 dep_pkg = self._deps_map[dep]
Brian Harring0be85c62012-03-17 19:52:12 -07001599 state = self._state_map[dep]
David James8c7e5e32011-06-28 11:26:03 -07001600 del dep_pkg["needs"][target]
Brian Harring0be85c62012-03-17 19:52:12 -07001601 state.update_score()
1602 if not state.prefetched:
1603 if dep in self._fetch_ready:
1604 # If it's not currently being fetched, update the prioritization
1605 self._fetch_ready.sort()
1606 elif not dep_pkg["needs"]:
David James8c7e5e32011-06-28 11:26:03 -07001607 if dep_pkg["nodeps"] and dep_pkg["action"] == "nomerge":
1608 self._Finish(dep)
1609 else:
Brian Harring0be85c62012-03-17 19:52:12 -07001610 self._build_ready.put(self._state_map[dep])
David James8c7e5e32011-06-28 11:26:03 -07001611 self._deps_map.pop(target)
David Jamesfcb70ef2011-02-02 16:02:30 -08001612
1613 def _Retry(self):
David James8c7e5e32011-06-28 11:26:03 -07001614 while self._retry_queue:
Brian Harring0be85c62012-03-17 19:52:12 -07001615 state = self._retry_queue.pop(0)
1616 if self._Schedule(state):
1617 self._Print("Retrying emerge of %s." % state.target)
David James8c7e5e32011-06-28 11:26:03 -07001618 break
David Jamesfcb70ef2011-02-02 16:02:30 -08001619
Brian Harringa43f5952012-04-12 01:19:34 -07001620 def _Shutdown(self):
David Jamesfcb70ef2011-02-02 16:02:30 -08001621 # Tell emerge workers to exit. They all exit when 'None' is pushed
1622 # to the queue.
Brian Harring0be85c62012-03-17 19:52:12 -07001623
Brian Harringa43f5952012-04-12 01:19:34 -07001624 # Shutdown the workers first; then jobs (which is how they feed things back)
1625 # then finally the print queue.
Brian Harring0be85c62012-03-17 19:52:12 -07001626
Brian Harringa43f5952012-04-12 01:19:34 -07001627 def _stop(queue, pool):
1628 if pool is None:
1629 return
1630 try:
1631 queue.put(None)
1632 pool.close()
1633 pool.join()
1634 finally:
1635 pool.terminate()
Brian Harring0be85c62012-03-17 19:52:12 -07001636
Brian Harringa43f5952012-04-12 01:19:34 -07001637 _stop(self._fetch_queue, self._fetch_pool)
1638 self._fetch_queue = self._fetch_pool = None
Brian Harring0be85c62012-03-17 19:52:12 -07001639
Brian Harringa43f5952012-04-12 01:19:34 -07001640 _stop(self._build_queue, self._build_pool)
1641 self._build_queue = self._build_pool = None
1642
Thiago Goncalesf4acc422013-07-17 10:26:35 -07001643 if self._unpack_only:
1644 _stop(self._unpack_queue, self._unpack_pool)
1645 self._unpack_queue = self._unpack_pool = None
1646
Brian Harringa43f5952012-04-12 01:19:34 -07001647 if self._job_queue is not None:
1648 self._job_queue.close()
1649 self._job_queue = None
David Jamesfcb70ef2011-02-02 16:02:30 -08001650
1651 # Now that our workers are finished, we can kill the print queue.
Brian Harringa43f5952012-04-12 01:19:34 -07001652 if self._print_worker is not None:
1653 try:
1654 self._print_queue.put(None)
1655 self._print_queue.close()
1656 self._print_worker.join()
1657 finally:
1658 self._print_worker.terminate()
1659 self._print_queue = self._print_worker = None
David Jamesfcb70ef2011-02-02 16:02:30 -08001660
1661 def Run(self):
1662 """Run through the scheduled ebuilds.
1663
1664 Keep running so long as we have uninstalled packages in the
1665 dependency graph to merge.
1666 """
Brian Harringa43f5952012-04-12 01:19:34 -07001667 if not self._deps_map:
1668 return
1669
Brian Harring0be85c62012-03-17 19:52:12 -07001670 # Start the fetchers.
1671 for _ in xrange(min(self._fetch_procs, len(self._fetch_ready))):
1672 state = self._fetch_ready.get()
1673 self._fetch_jobs[state.target] = None
1674 self._fetch_queue.put(state)
1675
1676 # Print an update, then get going.
1677 self._Status()
1678
David Jamesfcb70ef2011-02-02 16:02:30 -08001679 while self._deps_map:
1680 # Check here that we are actually waiting for something.
Brian Harring0be85c62012-03-17 19:52:12 -07001681 if (self._build_queue.empty() and
David Jamesfcb70ef2011-02-02 16:02:30 -08001682 self._job_queue.empty() and
Brian Harring0be85c62012-03-17 19:52:12 -07001683 not self._fetch_jobs and
1684 not self._fetch_ready and
Thiago Goncalesf4acc422013-07-17 10:26:35 -07001685 not self._unpack_jobs and
1686 not self._unpack_ready and
Brian Harring0be85c62012-03-17 19:52:12 -07001687 not self._build_jobs and
1688 not self._build_ready and
David Jamesfcb70ef2011-02-02 16:02:30 -08001689 self._deps_map):
1690 # If we have failed on a package, retry it now.
1691 if self._retry_queue:
1692 self._Retry()
1693 else:
David Jamesfcb70ef2011-02-02 16:02:30 -08001694 # Tell the user why we're exiting.
Bertrand SIMONNET411945d2015-05-20 17:23:28 -07001695 if self._failed_count:
1696 print('Packages failed:\n\t%s' %
1697 '\n\t'.join(self._failed_count.iterkeys()))
David James0eae23e2012-07-03 15:04:25 -07001698 status_file = os.environ.get("PARALLEL_EMERGE_STATUS_FILE")
1699 if status_file:
David James321490a2012-12-17 12:05:56 -08001700 failed_pkgs = set(portage.versions.cpv_getkey(x)
Bertrand SIMONNET411945d2015-05-20 17:23:28 -07001701 for x in self._failed_count.iterkeys())
David James0eae23e2012-07-03 15:04:25 -07001702 with open(status_file, "a") as f:
1703 f.write("%s\n" % " ".join(failed_pkgs))
David Jamesfcb70ef2011-02-02 16:02:30 -08001704 else:
Mike Frysinger383367e2014-09-16 15:06:17 -04001705 print("Deadlock! Circular dependencies!")
David Jamesfcb70ef2011-02-02 16:02:30 -08001706 sys.exit(1)
1707
David James321490a2012-12-17 12:05:56 -08001708 for _ in xrange(12):
David Jamesa74289a2011-08-12 10:41:24 -07001709 try:
1710 job = self._job_queue.get(timeout=5)
1711 break
1712 except Queue.Empty:
1713 # Check if any more jobs can be scheduled.
1714 self._ScheduleLoop()
1715 else:
Brian Harring706747c2012-03-16 03:04:31 -07001716 # Print an update every 60 seconds.
David Jamesfcb70ef2011-02-02 16:02:30 -08001717 self._Status()
1718 continue
1719
1720 target = job.target
1721
Brian Harring0be85c62012-03-17 19:52:12 -07001722 if job.fetch_only:
1723 if not job.done:
1724 self._fetch_jobs[job.target] = job
1725 else:
1726 state = self._state_map[job.target]
1727 state.prefetched = True
1728 state.fetched_successfully = (job.retcode == 0)
1729 del self._fetch_jobs[job.target]
1730 self._Print("Fetched %s in %2.2fs"
1731 % (target, time.time() - job.start_timestamp))
1732
1733 if self._show_output or job.retcode != 0:
1734 self._print_queue.put(JobPrinter(job, unlink=True))
1735 else:
1736 os.unlink(job.filename)
1737 # Failure or not, let build work with it next.
1738 if not self._deps_map[job.target]["needs"]:
1739 self._build_ready.put(state)
1740 self._ScheduleLoop()
1741
Thiago Goncalesf4acc422013-07-17 10:26:35 -07001742 if self._unpack_only and job.retcode == 0:
1743 self._unpack_ready.put(state)
1744 self._ScheduleLoop(unpack_only=True)
1745
Brian Harring0be85c62012-03-17 19:52:12 -07001746 if self._fetch_ready:
1747 state = self._fetch_ready.get()
1748 self._fetch_queue.put(state)
1749 self._fetch_jobs[state.target] = None
1750 else:
1751 # Minor optimization; shut down fetchers early since we know
1752 # the queue is empty.
1753 self._fetch_queue.put(None)
1754 continue
1755
Thiago Goncalesf4acc422013-07-17 10:26:35 -07001756 if job.unpack_only:
1757 if not job.done:
1758 self._unpack_jobs[target] = job
1759 else:
1760 del self._unpack_jobs[target]
1761 self._Print("Unpacked %s in %2.2fs"
1762 % (target, time.time() - job.start_timestamp))
1763 if self._show_output or job.retcode != 0:
1764 self._print_queue.put(JobPrinter(job, unlink=True))
1765 else:
1766 os.unlink(job.filename)
1767 if self._unpack_ready:
1768 state = self._unpack_ready.get()
1769 self._unpack_queue.put(state)
1770 self._unpack_jobs[state.target] = None
1771 continue
1772
David Jamesfcb70ef2011-02-02 16:02:30 -08001773 if not job.done:
Brian Harring0be85c62012-03-17 19:52:12 -07001774 self._build_jobs[target] = job
David Jamesfcb70ef2011-02-02 16:02:30 -08001775 self._Print("Started %s (logged in %s)" % (target, job.filename))
1776 continue
1777
1778 # Print output of job
1779 if self._show_output or job.retcode != 0:
1780 self._print_queue.put(JobPrinter(job, unlink=True))
1781 else:
1782 os.unlink(job.filename)
Brian Harring0be85c62012-03-17 19:52:12 -07001783 del self._build_jobs[target]
David Jamesfcb70ef2011-02-02 16:02:30 -08001784
1785 seconds = time.time() - job.start_timestamp
1786 details = "%s (in %dm%.1fs)" % (target, seconds / 60, seconds % 60)
1787
1788 # Complain if necessary.
1789 if job.retcode != 0:
1790 # Handle job failure.
Bertrand SIMONNET411945d2015-05-20 17:23:28 -07001791 failed_count = self._failed_count.get(target, 0)
1792 if failed_count >= self._max_retries:
1793 # If this job has failed and can't be retried, give up.
David Jamesfcb70ef2011-02-02 16:02:30 -08001794 self._Print("Failed %s. Your build has failed." % details)
1795 else:
1796 # Queue up this build to try again after a long while.
Brian Harring0be85c62012-03-17 19:52:12 -07001797 self._retry_queue.append(self._state_map[target])
Bertrand SIMONNET411945d2015-05-20 17:23:28 -07001798 self._failed_count[target] = failed_count + 1
David Jamesfcb70ef2011-02-02 16:02:30 -08001799 self._Print("Failed %s, retrying later." % details)
1800 else:
David James32420cc2011-08-25 21:32:46 -07001801 self._Print("Completed %s" % details)
1802
1803 # Mark as completed and unblock waiting ebuilds.
1804 self._Finish(target)
1805
Bertrand SIMONNET411945d2015-05-20 17:23:28 -07001806 if target in self._failed_count and self._retry_queue:
David Jamesfcb70ef2011-02-02 16:02:30 -08001807 # If we have successfully retried a failed package, and there
1808 # are more failed packages, try the next one. We will only have
1809 # one retrying package actively running at a time.
1810 self._Retry()
1811
David Jamesfcb70ef2011-02-02 16:02:30 -08001812
David James8c7e5e32011-06-28 11:26:03 -07001813 # Schedule pending jobs and print an update.
1814 self._ScheduleLoop()
1815 self._Status()
David Jamesfcb70ef2011-02-02 16:02:30 -08001816
David Jamese703d0f2012-01-12 16:27:45 -08001817 # If packages were retried, output a warning.
Bertrand SIMONNET411945d2015-05-20 17:23:28 -07001818 if self._failed_count:
David Jamese703d0f2012-01-12 16:27:45 -08001819 self._Print("")
Bertrand SIMONNET411945d2015-05-20 17:23:28 -07001820 self._Print("WARNING: The following packages failed once or more,")
David Jamese703d0f2012-01-12 16:27:45 -08001821 self._Print("but succeeded upon retry. This might indicate incorrect")
1822 self._Print("dependencies.")
Bertrand SIMONNET411945d2015-05-20 17:23:28 -07001823 for pkg in self._failed_count.iterkeys():
David Jamese703d0f2012-01-12 16:27:45 -08001824 self._Print(" %s" % pkg)
1825 self._Print("@@@STEP_WARNINGS@@@")
1826 self._Print("")
1827
David Jamesfcb70ef2011-02-02 16:02:30 -08001828 # Tell child threads to exit.
1829 self._Print("Merge complete")
David Jamesfcb70ef2011-02-02 16:02:30 -08001830
1831
Brian Harring30675052012-02-29 12:18:22 -08001832def main(argv):
Brian Harring8294d652012-05-23 02:20:52 -07001833 try:
1834 return real_main(argv)
1835 finally:
1836 # Work around multiprocessing sucking and not cleaning up after itself.
1837 # http://bugs.python.org/issue4106;
1838 # Step one; ensure GC is ran *prior* to the VM starting shutdown.
1839 gc.collect()
1840 # Step two; go looking for those threads and try to manually reap
1841 # them if we can.
1842 for x in threading.enumerate():
1843 # Filter on the name, and ident; if ident is None, the thread
1844 # wasn't started.
1845 if x.name == 'QueueFeederThread' and x.ident is not None:
1846 x.join(1)
David Jamesfcb70ef2011-02-02 16:02:30 -08001847
Brian Harring8294d652012-05-23 02:20:52 -07001848
1849def real_main(argv):
Brian Harring30675052012-02-29 12:18:22 -08001850 parallel_emerge_args = argv[:]
David Jamesfcb70ef2011-02-02 16:02:30 -08001851 deps = DepGraphGenerator()
Brian Harring30675052012-02-29 12:18:22 -08001852 deps.Initialize(parallel_emerge_args)
David Jamesfcb70ef2011-02-02 16:02:30 -08001853 emerge = deps.emerge
1854
1855 if emerge.action is not None:
Brian Harring30675052012-02-29 12:18:22 -08001856 argv = deps.ParseParallelEmergeArgs(argv)
Brian Harring8294d652012-05-23 02:20:52 -07001857 return emerge_main(argv)
David Jamesfcb70ef2011-02-02 16:02:30 -08001858 elif not emerge.cmdline_packages:
1859 Usage()
Brian Harring8294d652012-05-23 02:20:52 -07001860 return 1
David Jamesfcb70ef2011-02-02 16:02:30 -08001861
1862 # Unless we're in pretend mode, there's not much point running without
1863 # root access. We need to be able to install packages.
1864 #
1865 # NOTE: Even if you're running --pretend, it's a good idea to run
1866 # parallel_emerge with root access so that portage can write to the
1867 # dependency cache. This is important for performance.
David James321490a2012-12-17 12:05:56 -08001868 if "--pretend" not in emerge.opts and portage.data.secpass < 2:
Mike Frysinger383367e2014-09-16 15:06:17 -04001869 print("parallel_emerge: superuser access is required.")
Brian Harring8294d652012-05-23 02:20:52 -07001870 return 1
David Jamesfcb70ef2011-02-02 16:02:30 -08001871
1872 if "--quiet" not in emerge.opts:
1873 cmdline_packages = " ".join(emerge.cmdline_packages)
Mike Frysinger383367e2014-09-16 15:06:17 -04001874 print("Starting fast-emerge.")
1875 print(" Building package %s on %s" % (cmdline_packages,
Gilad Arnold05f94b02015-05-22 10:41:05 -07001876 deps.sysroot or "root"))
David Jamesfcb70ef2011-02-02 16:02:30 -08001877
David James386ccd12011-05-04 20:17:42 -07001878 deps_tree, deps_info = deps.GenDependencyTree()
David Jamesfcb70ef2011-02-02 16:02:30 -08001879
1880 # You want me to be verbose? I'll give you two trees! Twice as much value.
1881 if "--tree" in emerge.opts and "--verbose" in emerge.opts:
1882 deps.PrintTree(deps_tree)
1883
David James386ccd12011-05-04 20:17:42 -07001884 deps_graph = deps.GenDependencyGraph(deps_tree, deps_info)
David Jamesfcb70ef2011-02-02 16:02:30 -08001885
1886 # OK, time to print out our progress so far.
1887 deps.PrintInstallPlan(deps_graph)
1888 if "--tree" in emerge.opts:
1889 PrintDepsMap(deps_graph)
1890
1891 # Are we upgrading portage? If so, and there are more packages to merge,
1892 # schedule a restart of parallel_emerge to merge the rest. This ensures that
1893 # we pick up all updates to portage settings before merging any more
1894 # packages.
1895 portage_upgrade = False
1896 root = emerge.settings["ROOT"]
Don Garrett25f309a2014-03-19 14:02:12 -07001897 # pylint: disable=W0212
David Jamesfcb70ef2011-02-02 16:02:30 -08001898 if root == "/":
Gilad Arnoldcead28a2015-05-22 10:45:02 -07001899 final_db = emerge.depgraph._dynamic_config._filtered_trees[root]['graph_db']
David Jamesfcb70ef2011-02-02 16:02:30 -08001900 for db_pkg in final_db.match_pkgs("sys-apps/portage"):
1901 portage_pkg = deps_graph.get(db_pkg.cpv)
David James0ff16f22012-11-02 14:18:07 -07001902 if portage_pkg:
David Jamesfcb70ef2011-02-02 16:02:30 -08001903 portage_upgrade = True
1904 if "--quiet" not in emerge.opts:
Mike Frysinger383367e2014-09-16 15:06:17 -04001905 print("Upgrading portage first, then restarting...")
David Jamesfcb70ef2011-02-02 16:02:30 -08001906
David James0ff16f22012-11-02 14:18:07 -07001907 # Upgrade Portage first, then the rest of the packages.
1908 #
1909 # In order to grant the child permission to run setsid, we need to run sudo
1910 # again. We preserve SUDO_USER here in case an ebuild depends on it.
1911 if portage_upgrade:
1912 # Calculate what arguments to use when re-invoking.
1913 args = ["sudo", "-E", "SUDO_USER=%s" % os.environ.get("SUDO_USER", "")]
1914 args += [os.path.abspath(sys.argv[0])] + parallel_emerge_args
1915 args += ["--exclude=sys-apps/portage"]
1916
1917 # First upgrade Portage.
1918 passthrough_args = ("--quiet", "--pretend", "--verbose")
1919 emerge_args = [k for k in emerge.opts if k in passthrough_args]
1920 ret = emerge_main(emerge_args + ["portage"])
1921 if ret != 0:
1922 return ret
1923
1924 # Now upgrade the rest.
1925 os.execvp(args[0], args)
1926
Bertrand SIMONNETc03c8ee2014-12-10 17:02:55 -08001927 # Attempt to solve crbug.com/433482
1928 # The file descriptor error appears only when getting userpriv_groups
1929 # (lazily generated). Loading userpriv_groups here will reduce the number of
1930 # calls from few hundreds to one.
1931 portage.data._get_global('userpriv_groups')
1932
David Jamesfcb70ef2011-02-02 16:02:30 -08001933 # Run the queued emerges.
Thiago Goncalesf4acc422013-07-17 10:26:35 -07001934 scheduler = EmergeQueue(deps_graph, emerge, deps.package_db, deps.show_output,
Bertrand SIMONNET411945d2015-05-20 17:23:28 -07001935 deps.unpack_only, deps.max_retries)
Brian Harringa43f5952012-04-12 01:19:34 -07001936 try:
1937 scheduler.Run()
1938 finally:
Don Garrett25f309a2014-03-19 14:02:12 -07001939 # pylint: disable=W0212
Brian Harringa43f5952012-04-12 01:19:34 -07001940 scheduler._Shutdown()
David James97ce8902011-08-16 09:51:05 -07001941 scheduler = None
David Jamesfcb70ef2011-02-02 16:02:30 -08001942
Mike Frysingerd20a6e22012-10-04 19:01:10 -04001943 clean_logs(emerge.settings)
1944
Mike Frysinger383367e2014-09-16 15:06:17 -04001945 print("Done")
Brian Harring8294d652012-05-23 02:20:52 -07001946 return 0