blob: abf012192d1294f9bf819a155aa6a894bfd9f36e [file] [log] [blame]
David Jamesfcb70ef2011-02-02 16:02:30 -08001#!/usr/bin/python2.6
2# Copyright (c) 2010 The Chromium OS Authors. All rights reserved.
3# Use of this source code is governed by a BSD-style license that can be
4# found in the LICENSE file.
5
6"""Program to run emerge in parallel, for significant speedup.
7
8Usage:
David James386ccd12011-05-04 20:17:42 -07009 ./parallel_emerge [--board=BOARD] [--workon=PKGS]
David Jamesfcb70ef2011-02-02 16:02:30 -080010 [--force-remote-binary=PKGS] [emerge args] package
11
12Basic operation:
13 Runs 'emerge -p --debug' to display dependencies, and stores a
14 dependency graph. All non-blocked packages are launched in parallel,
15 as 'emerge --nodeps package' with any blocked packages being emerged
16 immediately upon deps being met.
17
18 For this to work effectively, /usr/lib/portage/pym/portage/locks.py
19 must be stubbed out, preventing portage from slowing itself with
20 unneccesary locking, as this script ensures that emerge is run in such
21 a way that common resources are never in conflict. This is controlled
22 by an environment variable PORTAGE_LOCKS set in parallel emerge
23 subprocesses.
24
25 Parallel Emerge unlocks two things during operation, here's what you
26 must do to keep this safe:
27 * Storage dir containing binary packages. - Don't emerge new
28 packages while installing the existing ones.
29 * Portage database - You must not examine deps while modifying the
30 database. Therefore you may only parallelize "-p" read only access,
31 or "--nodeps" write only access.
32 Caveats:
33 * Some ebuild packages have incorrectly specified deps, and running
34 them in parallel is more likely to bring out these failures.
35 * Some ebuilds (especially the build part) have complex dependencies
36 that are not captured well by this script (it may be necessary to
37 install an old package to build, but then install a newer version
38 of the same package for a runtime dep).
39"""
40
41import codecs
42import copy
43import errno
44import multiprocessing
45import os
46import Queue
David Jamesfcb70ef2011-02-02 16:02:30 -080047import signal
48import sys
49import tempfile
50import time
51import traceback
David Jamesfcb70ef2011-02-02 16:02:30 -080052
53# If PORTAGE_USERNAME isn't specified, scrape it from the $HOME variable. On
54# Chromium OS, the default "portage" user doesn't have the necessary
55# permissions. It'd be easier if we could default to $USERNAME, but $USERNAME
56# is "root" here because we get called through sudo.
57#
58# We need to set this before importing any portage modules, because portage
59# looks up "PORTAGE_USERNAME" at import time.
60#
61# NOTE: .bashrc sets PORTAGE_USERNAME = $USERNAME, so most people won't
62# encounter this case unless they have an old chroot or blow away the
63# environment by running sudo without the -E specifier.
64if "PORTAGE_USERNAME" not in os.environ:
65 homedir = os.environ.get("HOME")
66 if homedir:
67 os.environ["PORTAGE_USERNAME"] = os.path.basename(homedir)
68
69# Portage doesn't expose dependency trees in its public API, so we have to
70# make use of some private APIs here. These modules are found under
71# /usr/lib/portage/pym/.
72#
73# TODO(davidjames): Update Portage to expose public APIs for these features.
74from _emerge.actions import adjust_configs
75from _emerge.actions import load_emerge_config
76from _emerge.create_depgraph_params import create_depgraph_params
David James386ccd12011-05-04 20:17:42 -070077from _emerge.depgraph import backtrack_depgraph
David Jamesfcb70ef2011-02-02 16:02:30 -080078from _emerge.main import emerge_main
79from _emerge.main import parse_opts
80from _emerge.Package import Package
81from _emerge.Scheduler import Scheduler
82from _emerge.SetArg import SetArg
83from _emerge.stdout_spinner import stdout_spinner
David James386ccd12011-05-04 20:17:42 -070084from portage._global_updates import _global_updates
85from portage.versions import vercmp
David Jamesfcb70ef2011-02-02 16:02:30 -080086import portage
87import portage.debug
David Jamesfcb70ef2011-02-02 16:02:30 -080088
David James386ccd12011-05-04 20:17:42 -070089NEW_PORTAGE = (0 <= vercmp(portage.VERSION, "2.1.9.46-r2"))
David Jamesfcb70ef2011-02-02 16:02:30 -080090
91def Usage():
92 """Print usage."""
93 print "Usage:"
David James386ccd12011-05-04 20:17:42 -070094 print " ./parallel_emerge [--board=BOARD] [--workon=PKGS]"
David Jamesfcb70ef2011-02-02 16:02:30 -080095 print " [--rebuild] [emerge args] package"
96 print
97 print "Packages specified as workon packages are always built from source."
David Jamesfcb70ef2011-02-02 16:02:30 -080098 print
99 print "The --workon argument is mainly useful when you want to build and"
100 print "install packages that you are working on unconditionally, but do not"
101 print "to have to rev the package to indicate you want to build it from"
102 print "source. The build_packages script will automatically supply the"
103 print "workon argument to emerge, ensuring that packages selected using"
104 print "cros-workon are rebuilt."
105 print
106 print "The --rebuild option rebuilds packages whenever their dependencies"
107 print "are changed. This ensures that your build is correct."
108 sys.exit(1)
109
110
David Jamesfcb70ef2011-02-02 16:02:30 -0800111# Global start time
112GLOBAL_START = time.time()
113
David James7358d032011-05-19 10:40:03 -0700114# Whether process has been killed by a signal.
115KILLED = multiprocessing.Event()
116
David Jamesfcb70ef2011-02-02 16:02:30 -0800117
118class EmergeData(object):
119 """This simple struct holds various emerge variables.
120
121 This struct helps us easily pass emerge variables around as a unit.
122 These variables are used for calculating dependencies and installing
123 packages.
124 """
125
126 __slots__ = ["action", "cmdline_packages", "depgraph", "mtimedb", "opts",
127 "root_config", "scheduler_graph", "settings", "spinner",
128 "trees"]
129
130 def __init__(self):
131 # The action the user requested. If the user is installing packages, this
132 # is None. If the user is doing anything other than installing packages,
133 # this will contain the action name, which will map exactly to the
134 # long-form name of the associated emerge option.
135 #
136 # Example: If you call parallel_emerge --unmerge package, the action name
137 # will be "unmerge"
138 self.action = None
139
140 # The list of packages the user passed on the command-line.
141 self.cmdline_packages = None
142
143 # The emerge dependency graph. It'll contain all the packages involved in
144 # this merge, along with their versions.
145 self.depgraph = None
146
147 # A dict of the options passed to emerge. This dict has been cleaned up
148 # a bit by parse_opts, so that it's a bit easier for the emerge code to
149 # look at the options.
150 #
151 # Emerge takes a few shortcuts in its cleanup process to make parsing of
152 # the options dict easier. For example, if you pass in "--usepkg=n", the
153 # "--usepkg" flag is just left out of the dictionary altogether. Because
154 # --usepkg=n is the default, this makes parsing easier, because emerge
155 # can just assume that if "--usepkg" is in the dictionary, it's enabled.
156 #
157 # These cleanup processes aren't applied to all options. For example, the
158 # --with-bdeps flag is passed in as-is. For a full list of the cleanups
159 # applied by emerge, see the parse_opts function in the _emerge.main
160 # package.
161 self.opts = None
162
163 # A dictionary used by portage to maintain global state. This state is
164 # loaded from disk when portage starts up, and saved to disk whenever we
165 # call mtimedb.commit().
166 #
167 # This database contains information about global updates (i.e., what
168 # version of portage we have) and what we're currently doing. Portage
169 # saves what it is currently doing in this database so that it can be
170 # resumed when you call it with the --resume option.
171 #
172 # parallel_emerge does not save what it is currently doing in the mtimedb,
173 # so we do not support the --resume option.
174 self.mtimedb = None
175
176 # The portage configuration for our current root. This contains the portage
177 # settings (see below) and the three portage trees for our current root.
178 # (The three portage trees are explained below, in the documentation for
179 # the "trees" member.)
180 self.root_config = None
181
182 # The scheduler graph is used by emerge to calculate what packages to
183 # install. We don't actually install any deps, so this isn't really used,
184 # but we pass it in to the Scheduler object anyway.
185 self.scheduler_graph = None
186
187 # Portage settings for our current session. Most of these settings are set
188 # in make.conf inside our current install root.
189 self.settings = None
190
191 # The spinner, which spews stuff to stdout to indicate that portage is
192 # doing something. We maintain our own spinner, so we set the portage
193 # spinner to "silent" mode.
194 self.spinner = None
195
196 # The portage trees. There are separate portage trees for each root. To get
197 # the portage tree for the current root, you can look in self.trees[root],
198 # where root = self.settings["ROOT"].
199 #
200 # In each root, there are three trees: vartree, porttree, and bintree.
201 # - vartree: A database of the currently-installed packages.
202 # - porttree: A database of ebuilds, that can be used to build packages.
203 # - bintree: A database of binary packages.
204 self.trees = None
205
206
207class DepGraphGenerator(object):
208 """Grab dependency information about packages from portage.
209
210 Typical usage:
211 deps = DepGraphGenerator()
212 deps.Initialize(sys.argv[1:])
213 deps_tree, deps_info = deps.GenDependencyTree()
214 deps_graph = deps.GenDependencyGraph(deps_tree, deps_info)
215 deps.PrintTree(deps_tree)
216 PrintDepsMap(deps_graph)
217 """
218
David James386ccd12011-05-04 20:17:42 -0700219 __slots__ = ["board", "emerge", "package_db", "show_output"]
David Jamesfcb70ef2011-02-02 16:02:30 -0800220
221 def __init__(self):
222 self.board = None
223 self.emerge = EmergeData()
David Jamesfcb70ef2011-02-02 16:02:30 -0800224 self.package_db = {}
David Jamesfcb70ef2011-02-02 16:02:30 -0800225 self.show_output = False
David Jamesfcb70ef2011-02-02 16:02:30 -0800226
227 def ParseParallelEmergeArgs(self, argv):
228 """Read the parallel emerge arguments from the command-line.
229
230 We need to be compatible with emerge arg format. We scrape arguments that
231 are specific to parallel_emerge, and pass through the rest directly to
232 emerge.
233 Args:
234 argv: arguments list
235 Returns:
236 Arguments that don't belong to parallel_emerge
237 """
238 emerge_args = []
239 for arg in argv:
240 # Specifically match arguments that are specific to parallel_emerge, and
241 # pass through the rest.
242 if arg.startswith("--board="):
243 self.board = arg.replace("--board=", "")
244 elif arg.startswith("--workon="):
245 workon_str = arg.replace("--workon=", "")
David James386ccd12011-05-04 20:17:42 -0700246 if NEW_PORTAGE:
247 emerge_args.append("--reinstall-atoms=%s" % workon_str)
248 emerge_args.append("--usepkg-exclude=%s" % workon_str)
David Jamesfcb70ef2011-02-02 16:02:30 -0800249 elif arg.startswith("--force-remote-binary="):
250 force_remote_binary = arg.replace("--force-remote-binary=", "")
David James386ccd12011-05-04 20:17:42 -0700251 if NEW_PORTAGE:
252 emerge_args.append("--useoldpkg-atoms=%s" % force_remote_binary)
David Jamesfcb70ef2011-02-02 16:02:30 -0800253 elif arg == "--show-output":
254 self.show_output = True
David James386ccd12011-05-04 20:17:42 -0700255 elif arg == "--rebuild":
256 if NEW_PORTAGE:
257 emerge_args.append("--rebuild-if-unbuilt")
David Jamesfcb70ef2011-02-02 16:02:30 -0800258 else:
259 # Not one of our options, so pass through to emerge.
260 emerge_args.append(arg)
261
David James386ccd12011-05-04 20:17:42 -0700262 # These packages take a really long time to build, so, for expediency, we
263 # are blacklisting them from automatic rebuilds because one of their
264 # dependencies needs to be recompiled.
265 for pkg in ("chromeos-base/chromeos-chrome", "media-plugins/o3d",
266 "dev-java/icedtea"):
267 if NEW_PORTAGE:
268 emerge_args.append("--rebuild-exclude=%s" % pkg)
David Jamesfcb70ef2011-02-02 16:02:30 -0800269
270 return emerge_args
271
272 def Initialize(self, args):
273 """Initializer. Parses arguments and sets up portage state."""
274
275 # Parse and strip out args that are just intended for parallel_emerge.
276 emerge_args = self.ParseParallelEmergeArgs(args)
277
278 # Setup various environment variables based on our current board. These
279 # variables are normally setup inside emerge-${BOARD}, but since we don't
280 # call that script, we have to set it up here. These variables serve to
281 # point our tools at /build/BOARD and to setup cross compiles to the
282 # appropriate board as configured in toolchain.conf.
283 if self.board:
284 os.environ["PORTAGE_CONFIGROOT"] = "/build/" + self.board
285 os.environ["PORTAGE_SYSROOT"] = "/build/" + self.board
286 os.environ["SYSROOT"] = "/build/" + self.board
287 srcroot = "%s/../../src" % os.path.dirname(os.path.realpath(__file__))
288 # Strip the variant out of the board name to look for the toolchain. This
289 # is similar to what setup_board does.
290 board_no_variant = self.board.split('_')[0]
291 public_toolchain_path = ("%s/overlays/overlay-%s/toolchain.conf" %
292 (srcroot, board_no_variant))
293 private_toolchain_path = (
294 "%s/private-overlays/overlay-%s-private/toolchain.conf" %
295 (srcroot, board_no_variant))
296 if os.path.isfile(public_toolchain_path):
297 toolchain_path = public_toolchain_path
298 elif os.path.isfile(private_toolchain_path):
299 toolchain_path = private_toolchain_path
300 else:
301 print "Not able to locate toolchain.conf in board overlays"
302 sys.exit(1)
303
304 f = open(toolchain_path)
305 os.environ["CHOST"] = f.readline().strip()
306 f.close()
307
308 # Although CHROMEOS_ROOT isn't specific to boards, it's normally setup
309 # inside emerge-${BOARD}, so we set it up here for compatibility. It
310 # will be going away soon as we migrate to CROS_WORKON_SRCROOT.
311 os.environ.setdefault("CHROMEOS_ROOT", os.environ["HOME"] + "/trunk")
312
313 # Turn off interactive delays
314 os.environ["EBEEP_IGNORE"] = "1"
315 os.environ["EPAUSE_IGNORE"] = "1"
316 os.environ["UNMERGE_DELAY"] = "0"
317
318 # Parse the emerge options.
David Jamesea3ca332011-05-26 11:48:29 -0700319 action, opts, cmdline_packages = parse_opts(emerge_args, silent=True)
David Jamesfcb70ef2011-02-02 16:02:30 -0800320
321 # Set environment variables based on options. Portage normally sets these
322 # environment variables in emerge_main, but we can't use that function,
323 # because it also does a bunch of other stuff that we don't want.
324 # TODO(davidjames): Patch portage to move this logic into a function we can
325 # reuse here.
326 if "--debug" in opts:
327 os.environ["PORTAGE_DEBUG"] = "1"
328 if "--config-root" in opts:
329 os.environ["PORTAGE_CONFIGROOT"] = opts["--config-root"]
330 if "--root" in opts:
331 os.environ["ROOT"] = opts["--root"]
332 if "--accept-properties" in opts:
333 os.environ["ACCEPT_PROPERTIES"] = opts["--accept-properties"]
334
335 # Portage has two flags for doing collision protection: collision-protect
336 # and protect-owned. The protect-owned feature is enabled by default and
337 # is quite useful: it checks to make sure that we don't have multiple
338 # packages that own the same file. The collision-protect feature is more
339 # strict, and less useful: it fails if it finds a conflicting file, even
340 # if that file was created by an earlier ebuild that failed to install.
341 #
342 # We want to disable collision-protect here because we don't handle
343 # failures during the merge step very well. Sometimes we leave old files
344 # lying around and they cause problems, so for now we disable the flag.
345 # TODO(davidjames): Look for a better solution.
346 features = os.environ.get("FEATURES", "") + " -collision-protect"
347
David Jamesdeebd692011-05-09 17:02:52 -0700348 # Install packages in parallel.
349 features = features + " parallel-install"
350
David Jamesfcb70ef2011-02-02 16:02:30 -0800351 # If we're installing packages to the board, and we're not using the
352 # official flag, we can enable the following optimizations:
353 # 1) Don't lock during install step. This allows multiple packages to be
354 # installed at once. This is safe because our board packages do not
355 # muck with each other during the post-install step.
356 # 2) Don't update the environment until the end of the build. This is
357 # safe because board packages don't need to run during the build --
358 # they're cross-compiled, so our CPU architecture doesn't support them
359 # anyway.
360 if self.board and os.environ.get("CHROMEOS_OFFICIAL") != "1":
361 os.environ.setdefault("PORTAGE_LOCKS", "false")
David Jamesdeebd692011-05-09 17:02:52 -0700362 features = features + " -ebuild-locks no-env-update"
David Jamesfcb70ef2011-02-02 16:02:30 -0800363
364 os.environ["FEATURES"] = features
365
366 # Now that we've setup the necessary environment variables, we can load the
367 # emerge config from disk.
368 settings, trees, mtimedb = load_emerge_config()
369
David Jamesea3ca332011-05-26 11:48:29 -0700370 # Add in EMERGE_DEFAULT_OPTS, if specified.
371 tmpcmdline = []
372 if "--ignore-default-opts" not in opts:
373 tmpcmdline.extend(settings["EMERGE_DEFAULT_OPTS"].split())
374 tmpcmdline.extend(emerge_args)
375 action, opts, cmdline_packages = parse_opts(tmpcmdline)
376
377 # If we're installing to the board, we want the --root-deps option so that
378 # portage will install the build dependencies to that location as well.
379 if self.board:
380 opts.setdefault("--root-deps", True)
381
David Jamesfcb70ef2011-02-02 16:02:30 -0800382 # Check whether our portage tree is out of date. Typically, this happens
383 # when you're setting up a new portage tree, such as in setup_board and
384 # make_chroot. In that case, portage applies a bunch of global updates
385 # here. Once the updates are finished, we need to commit any changes
386 # that the global update made to our mtimedb, and reload the config.
387 #
388 # Portage normally handles this logic in emerge_main, but again, we can't
389 # use that function here.
390 if _global_updates(trees, mtimedb["updates"]):
391 mtimedb.commit()
392 settings, trees, mtimedb = load_emerge_config(trees=trees)
393
394 # Setup implied options. Portage normally handles this logic in
395 # emerge_main.
396 if "--buildpkgonly" in opts or "buildpkg" in settings.features:
397 opts.setdefault("--buildpkg", True)
398 if "--getbinpkgonly" in opts:
399 opts.setdefault("--usepkgonly", True)
400 opts.setdefault("--getbinpkg", True)
401 if "getbinpkg" in settings.features:
402 # Per emerge_main, FEATURES=getbinpkg overrides --getbinpkg=n
403 opts["--getbinpkg"] = True
404 if "--getbinpkg" in opts or "--usepkgonly" in opts:
405 opts.setdefault("--usepkg", True)
406 if "--fetch-all-uri" in opts:
407 opts.setdefault("--fetchonly", True)
408 if "--skipfirst" in opts:
409 opts.setdefault("--resume", True)
410 if "--buildpkgonly" in opts:
411 # --buildpkgonly will not merge anything, so it overrides all binary
412 # package options.
413 for opt in ("--getbinpkg", "--getbinpkgonly",
414 "--usepkg", "--usepkgonly"):
415 opts.pop(opt, None)
416 if (settings.get("PORTAGE_DEBUG", "") == "1" and
417 "python-trace" in settings.features):
418 portage.debug.set_trace(True)
419
420 # Complain about unsupported options
David James386ccd12011-05-04 20:17:42 -0700421 for opt in ("--ask", "--ask-enter-invalid", "--resume", "--skipfirst"):
David Jamesfcb70ef2011-02-02 16:02:30 -0800422 if opt in opts:
423 print "%s is not supported by parallel_emerge" % opt
424 sys.exit(1)
425
426 # Make emerge specific adjustments to the config (e.g. colors!)
427 adjust_configs(opts, trees)
428
429 # Save our configuration so far in the emerge object
430 emerge = self.emerge
431 emerge.action, emerge.opts = action, opts
432 emerge.settings, emerge.trees, emerge.mtimedb = settings, trees, mtimedb
433 emerge.cmdline_packages = cmdline_packages
434 root = settings["ROOT"]
435 emerge.root_config = trees[root]["root_config"]
436
David James386ccd12011-05-04 20:17:42 -0700437 if "--usepkg" in opts:
David Jamesfcb70ef2011-02-02 16:02:30 -0800438 emerge.trees[root]["bintree"].populate("--getbinpkg" in opts)
439
David Jamesfcb70ef2011-02-02 16:02:30 -0800440 def CreateDepgraph(self, emerge, packages):
441 """Create an emerge depgraph object."""
442 # Setup emerge options.
443 emerge_opts = emerge.opts.copy()
444
David James386ccd12011-05-04 20:17:42 -0700445 # Ask portage to build a dependency graph. with the options we specified
446 # above.
David Jamesfcb70ef2011-02-02 16:02:30 -0800447 params = create_depgraph_params(emerge_opts, emerge.action)
David James386ccd12011-05-04 20:17:42 -0700448 success, depgraph, _ = backtrack_depgraph(
449 emerge.settings, emerge.trees, emerge_opts, params, emerge.action,
450 packages, emerge.spinner)
451 emerge.depgraph = depgraph
David Jamesfcb70ef2011-02-02 16:02:30 -0800452
David James386ccd12011-05-04 20:17:42 -0700453 # Is it impossible to honor the user's request? Bail!
454 if not success:
455 depgraph.display_problems()
456 sys.exit(1)
David Jamesfcb70ef2011-02-02 16:02:30 -0800457
458 emerge.depgraph = depgraph
459
460 # Is it impossible to honor the user's request? Bail!
461 if not success:
462 depgraph.display_problems()
463 sys.exit(1)
464
David Jamesdeebd692011-05-09 17:02:52 -0700465 # Prime and flush emerge caches.
466 root = emerge.settings["ROOT"]
467 vardb = emerge.trees[root]["vartree"].dbapi
David James0bdc5de2011-05-12 16:22:26 -0700468 if "--pretend" not in emerge.opts:
469 vardb.counter_tick()
David Jamesdeebd692011-05-09 17:02:52 -0700470 vardb.flush_cache()
471
David James386ccd12011-05-04 20:17:42 -0700472 def GenDependencyTree(self):
David Jamesfcb70ef2011-02-02 16:02:30 -0800473 """Get dependency tree info from emerge.
474
David Jamesfcb70ef2011-02-02 16:02:30 -0800475 Returns:
476 Dependency tree
477 """
478 start = time.time()
479
480 emerge = self.emerge
481
482 # Create a list of packages to merge
483 packages = set(emerge.cmdline_packages[:])
David Jamesfcb70ef2011-02-02 16:02:30 -0800484
485 # Tell emerge to be quiet. We print plenty of info ourselves so we don't
486 # need any extra output from portage.
487 portage.util.noiselimit = -1
488
489 # My favorite feature: The silent spinner. It doesn't spin. Ever.
490 # I'd disable the colors by default too, but they look kind of cool.
491 emerge.spinner = stdout_spinner()
492 emerge.spinner.update = emerge.spinner.update_quiet
493
494 if "--quiet" not in emerge.opts:
495 print "Calculating deps..."
496
497 self.CreateDepgraph(emerge, packages)
498 depgraph = emerge.depgraph
499
500 # Build our own tree from the emerge digraph.
501 deps_tree = {}
502 digraph = depgraph._dynamic_config.digraph
503 for node, node_deps in digraph.nodes.items():
504 # Calculate dependency packages that need to be installed first. Each
505 # child on the digraph is a dependency. The "operation" field specifies
506 # what we're doing (e.g. merge, uninstall, etc.). The "priorities" array
507 # contains the type of dependency (e.g. build, runtime, runtime_post,
508 # etc.)
509 #
510 # Emerge itself actually treats some dependencies as "soft" dependencies
511 # and sometimes ignores them. We don't do that -- we honor all
512 # dependencies unless we're forced to prune them because they're cyclic.
513 #
514 # Portage refers to the identifiers for packages as a CPV. This acronym
515 # stands for Component/Path/Version.
516 #
517 # Here's an example CPV: chromeos-base/power_manager-0.0.1-r1
518 # Split up, this CPV would be:
519 # C -- Component: chromeos-base
520 # P -- Path: power_manager
521 # V -- Version: 0.0.1-r1
522 #
523 # We just refer to CPVs as packages here because it's easier.
524 deps = {}
525 for child, priorities in node_deps[0].items():
526 if isinstance(child, SetArg): continue
527 deps[str(child.cpv)] = dict(action=str(child.operation),
528 deptype=str(priorities[-1]),
529 deps={})
530
531 # We've built our list of deps, so we can add our package to the tree.
David James386ccd12011-05-04 20:17:42 -0700532 if isinstance(node, Package) and node.root == emerge.settings["ROOT"]:
David Jamesfcb70ef2011-02-02 16:02:30 -0800533 deps_tree[str(node.cpv)] = dict(action=str(node.operation),
534 deps=deps)
535
David Jamesfcb70ef2011-02-02 16:02:30 -0800536 # Ask portage for its install plan, so that we can only throw out
David James386ccd12011-05-04 20:17:42 -0700537 # dependencies that portage throws out.
David Jamesfcb70ef2011-02-02 16:02:30 -0800538 deps_info = {}
539 for pkg in depgraph.altlist():
540 if isinstance(pkg, Package):
David James386ccd12011-05-04 20:17:42 -0700541 assert pkg.root == emerge.settings["ROOT"]
David Jamesfcb70ef2011-02-02 16:02:30 -0800542 self.package_db[pkg.cpv] = pkg
543
David Jamesfcb70ef2011-02-02 16:02:30 -0800544 # Save off info about the package
David James386ccd12011-05-04 20:17:42 -0700545 deps_info[str(pkg.cpv)] = {"idx": len(deps_info)}
David Jamesfcb70ef2011-02-02 16:02:30 -0800546
547 seconds = time.time() - start
548 if "--quiet" not in emerge.opts:
549 print "Deps calculated in %dm%.1fs" % (seconds / 60, seconds % 60)
550
551 return deps_tree, deps_info
552
553 def PrintTree(self, deps, depth=""):
554 """Print the deps we have seen in the emerge output.
555
556 Args:
557 deps: Dependency tree structure.
558 depth: Allows printing the tree recursively, with indentation.
559 """
560 for entry in sorted(deps):
561 action = deps[entry]["action"]
562 print "%s %s (%s)" % (depth, entry, action)
563 self.PrintTree(deps[entry]["deps"], depth=depth + " ")
564
David James386ccd12011-05-04 20:17:42 -0700565 def GenDependencyGraph(self, deps_tree, deps_info):
David Jamesfcb70ef2011-02-02 16:02:30 -0800566 """Generate a doubly linked dependency graph.
567
568 Args:
569 deps_tree: Dependency tree structure.
570 deps_info: More details on the dependencies.
571 Returns:
572 Deps graph in the form of a dict of packages, with each package
573 specifying a "needs" list and "provides" list.
574 """
575 emerge = self.emerge
576 root = emerge.settings["ROOT"]
577
David Jamesfcb70ef2011-02-02 16:02:30 -0800578 # deps_map is the actual dependency graph.
579 #
580 # Each package specifies a "needs" list and a "provides" list. The "needs"
581 # list indicates which packages we depend on. The "provides" list
582 # indicates the reverse dependencies -- what packages need us.
583 #
584 # We also provide some other information in the dependency graph:
585 # - action: What we're planning on doing with this package. Generally,
586 # "merge", "nomerge", or "uninstall"
David Jamesfcb70ef2011-02-02 16:02:30 -0800587 deps_map = {}
588
589 def ReverseTree(packages):
590 """Convert tree to digraph.
591
592 Take the tree of package -> requirements and reverse it to a digraph of
593 buildable packages -> packages they unblock.
594 Args:
595 packages: Tree(s) of dependencies.
596 Returns:
597 Unsanitized digraph.
598 """
599 for pkg in packages:
600
601 # Create an entry for the package
602 action = packages[pkg]["action"]
David James386ccd12011-05-04 20:17:42 -0700603 default_pkg = {"needs": {}, "provides": set(), "action": action}
David Jamesfcb70ef2011-02-02 16:02:30 -0800604 this_pkg = deps_map.setdefault(pkg, default_pkg)
605
606 # Create entries for dependencies of this package first.
607 ReverseTree(packages[pkg]["deps"])
608
609 # Add dependencies to this package.
610 for dep, dep_item in packages[pkg]["deps"].iteritems():
611 dep_pkg = deps_map[dep]
612 dep_type = dep_item["deptype"]
613 if dep_type != "runtime_post":
614 dep_pkg["provides"].add(pkg)
615 this_pkg["needs"][dep] = dep_type
616
David Jamesfcb70ef2011-02-02 16:02:30 -0800617 def FindCycles():
618 """Find cycles in the dependency tree.
619
620 Returns:
621 A dict mapping cyclic packages to a dict of the deps that cause
622 cycles. For each dep that causes cycles, it returns an example
623 traversal of the graph that shows the cycle.
624 """
625
626 def FindCyclesAtNode(pkg, cycles, unresolved, resolved):
627 """Find cycles in cyclic dependencies starting at specified package.
628
629 Args:
630 pkg: Package identifier.
631 cycles: A dict mapping cyclic packages to a dict of the deps that
632 cause cycles. For each dep that causes cycles, it returns an
633 example traversal of the graph that shows the cycle.
634 unresolved: Nodes that have been visited but are not fully processed.
635 resolved: Nodes that have been visited and are fully processed.
636 """
637 pkg_cycles = cycles.get(pkg)
638 if pkg in resolved and not pkg_cycles:
639 # If we already looked at this package, and found no cyclic
640 # dependencies, we can stop now.
641 return
642 unresolved.append(pkg)
643 for dep in deps_map[pkg]["needs"]:
644 if dep in unresolved:
645 idx = unresolved.index(dep)
646 mycycle = unresolved[idx:] + [dep]
647 for i in range(len(mycycle) - 1):
648 pkg1, pkg2 = mycycle[i], mycycle[i+1]
649 cycles.setdefault(pkg1, {}).setdefault(pkg2, mycycle)
650 elif not pkg_cycles or dep not in pkg_cycles:
651 # Looks like we haven't seen this edge before.
652 FindCyclesAtNode(dep, cycles, unresolved, resolved)
653 unresolved.pop()
654 resolved.add(pkg)
655
656 cycles, unresolved, resolved = {}, [], set()
657 for pkg in deps_map:
658 FindCyclesAtNode(pkg, cycles, unresolved, resolved)
659 return cycles
660
David James386ccd12011-05-04 20:17:42 -0700661 def RemoveUnusedPackages():
David Jamesfcb70ef2011-02-02 16:02:30 -0800662 """Remove installed packages, propagating dependencies."""
David Jamesfcb70ef2011-02-02 16:02:30 -0800663 # Schedule packages that aren't on the install list for removal
664 rm_pkgs = set(deps_map.keys()) - set(deps_info.keys())
665
David Jamesfcb70ef2011-02-02 16:02:30 -0800666 # Remove the packages we don't want, simplifying the graph and making
667 # it easier for us to crack cycles.
668 for pkg in sorted(rm_pkgs):
669 this_pkg = deps_map[pkg]
670 needs = this_pkg["needs"]
671 provides = this_pkg["provides"]
672 for dep in needs:
673 dep_provides = deps_map[dep]["provides"]
674 dep_provides.update(provides)
675 dep_provides.discard(pkg)
676 dep_provides.discard(dep)
677 for target in provides:
678 target_needs = deps_map[target]["needs"]
679 target_needs.update(needs)
680 target_needs.pop(pkg, None)
681 target_needs.pop(target, None)
682 del deps_map[pkg]
683
684 def PrintCycleBreak(basedep, dep, mycycle):
685 """Print details about a cycle that we are planning on breaking.
686
687 We are breaking a cycle where dep needs basedep. mycycle is an
688 example cycle which contains dep -> basedep."""
689
690 # If it's an optional dependency, there's no need to spam the user with
691 # warning messages.
692 needs = deps_map[dep]["needs"]
693 depinfo = needs.get(basedep, "deleted")
694 if depinfo == "optional":
695 return
696
697 # Notify the user that we're breaking a cycle.
698 print "Breaking %s -> %s (%s)" % (dep, basedep, depinfo)
699
700 # Show cycle.
701 for i in range(len(mycycle) - 1):
702 pkg1, pkg2 = mycycle[i], mycycle[i+1]
703 needs = deps_map[pkg1]["needs"]
704 depinfo = needs.get(pkg2, "deleted")
705 if pkg1 == dep and pkg2 == basedep:
706 depinfo = depinfo + ", deleting"
707 print " %s -> %s (%s)" % (pkg1, pkg2, depinfo)
708
709 def SanitizeTree():
710 """Remove circular dependencies.
711
712 We prune all dependencies involved in cycles that go against the emerge
713 ordering. This has a nice property: we're guaranteed to merge
714 dependencies in the same order that portage does.
715
716 Because we don't treat any dependencies as "soft" unless they're killed
717 by a cycle, we pay attention to a larger number of dependencies when
718 merging. This hurts performance a bit, but helps reliability.
719 """
720 start = time.time()
721 cycles = FindCycles()
722 while cycles:
723 for dep, mycycles in cycles.iteritems():
724 for basedep, mycycle in mycycles.iteritems():
725 if deps_info[basedep]["idx"] >= deps_info[dep]["idx"]:
726 PrintCycleBreak(basedep, dep, mycycle)
727 del deps_map[dep]["needs"][basedep]
728 deps_map[basedep]["provides"].remove(dep)
729 cycles = FindCycles()
730 seconds = time.time() - start
731 if "--quiet" not in emerge.opts and seconds >= 0.1:
732 print "Tree sanitized in %dm%.1fs" % (seconds / 60, seconds % 60)
733
David Jamesa22906f2011-05-04 19:53:26 -0700734 ReverseTree(deps_tree)
David Jamesa22906f2011-05-04 19:53:26 -0700735
David James386ccd12011-05-04 20:17:42 -0700736 # We need to remove unused packages so that we can use the dependency
737 # ordering of the install process to show us what cycles to crack.
738 RemoveUnusedPackages()
David Jamesfcb70ef2011-02-02 16:02:30 -0800739 SanitizeTree()
David Jamesfcb70ef2011-02-02 16:02:30 -0800740 return deps_map
741
742 def PrintInstallPlan(self, deps_map):
743 """Print an emerge-style install plan.
744
745 The install plan lists what packages we're installing, in order.
746 It's useful for understanding what parallel_emerge is doing.
747
748 Args:
749 deps_map: The dependency graph.
750 """
751
752 def InstallPlanAtNode(target, deps_map):
753 nodes = []
754 nodes.append(target)
755 for dep in deps_map[target]["provides"]:
756 del deps_map[dep]["needs"][target]
757 if not deps_map[dep]["needs"]:
758 nodes.extend(InstallPlanAtNode(dep, deps_map))
759 return nodes
760
761 deps_map = copy.deepcopy(deps_map)
762 install_plan = []
763 plan = set()
764 for target, info in deps_map.iteritems():
765 if not info["needs"] and target not in plan:
766 for item in InstallPlanAtNode(target, deps_map):
767 plan.add(item)
768 install_plan.append(self.package_db[item])
769
770 for pkg in plan:
771 del deps_map[pkg]
772
773 if deps_map:
774 print "Cyclic dependencies:", " ".join(deps_map)
775 PrintDepsMap(deps_map)
776 sys.exit(1)
777
778 self.emerge.depgraph.display(install_plan)
779
780
781def PrintDepsMap(deps_map):
782 """Print dependency graph, for each package list it's prerequisites."""
783 for i in sorted(deps_map):
784 print "%s: (%s) needs" % (i, deps_map[i]["action"])
785 needs = deps_map[i]["needs"]
786 for j in sorted(needs):
787 print " %s" % (j)
788 if not needs:
789 print " no dependencies"
790
791
792class EmergeJobState(object):
793 __slots__ = ["done", "filename", "last_notify_timestamp", "last_output_seek",
794 "last_output_timestamp", "pkgname", "retcode", "start_timestamp",
795 "target"]
796
797 def __init__(self, target, pkgname, done, filename, start_timestamp,
798 retcode=None):
799
800 # The full name of the target we're building (e.g.
801 # chromeos-base/chromeos-0.0.1-r60)
802 self.target = target
803
804 # The short name of the target we're building (e.g. chromeos-0.0.1-r60)
805 self.pkgname = pkgname
806
807 # Whether the job is done. (True if the job is done; false otherwise.)
808 self.done = done
809
810 # The filename where output is currently stored.
811 self.filename = filename
812
813 # The timestamp of the last time we printed the name of the log file. We
814 # print this at the beginning of the job, so this starts at
815 # start_timestamp.
816 self.last_notify_timestamp = start_timestamp
817
818 # The location (in bytes) of the end of the last complete line we printed.
819 # This starts off at zero. We use this to jump to the right place when we
820 # print output from the same ebuild multiple times.
821 self.last_output_seek = 0
822
823 # The timestamp of the last time we printed output. Since we haven't
824 # printed output yet, this starts at zero.
825 self.last_output_timestamp = 0
826
827 # The return code of our job, if the job is actually finished.
828 self.retcode = retcode
829
830 # The timestamp when our job started.
831 self.start_timestamp = start_timestamp
832
833
David James7358d032011-05-19 10:40:03 -0700834def KillHandler(signum, frame):
835 # Kill self and all subprocesses.
836 os.killpg(0, signal.SIGKILL)
837
David Jamesfcb70ef2011-02-02 16:02:30 -0800838def SetupWorkerSignals():
839 def ExitHandler(signum, frame):
David James7358d032011-05-19 10:40:03 -0700840 # Set KILLED flag.
841 KILLED.set()
David James13cead42011-05-18 16:22:01 -0700842
David James7358d032011-05-19 10:40:03 -0700843 # Remove our signal handlers so we don't get called recursively.
844 signal.signal(signal.SIGINT, KillHandler)
845 signal.signal(signal.SIGTERM, KillHandler)
David Jamesfcb70ef2011-02-02 16:02:30 -0800846
847 # Ensure that we exit quietly and cleanly, if possible, when we receive
848 # SIGTERM or SIGINT signals. By default, when the user hits CTRL-C, all
849 # of the child processes will print details about KeyboardInterrupt
850 # exceptions, which isn't very helpful.
851 signal.signal(signal.SIGINT, ExitHandler)
852 signal.signal(signal.SIGTERM, ExitHandler)
853
854
855def EmergeWorker(task_queue, job_queue, emerge, package_db):
856 """This worker emerges any packages given to it on the task_queue.
857
858 Args:
859 task_queue: The queue of tasks for this worker to do.
860 job_queue: The queue of results from the worker.
861 emerge: An EmergeData() object.
862 package_db: A dict, mapping package ids to portage Package objects.
863
864 It expects package identifiers to be passed to it via task_queue. When
865 a task is started, it pushes the (target, filename) to the started_queue.
866 The output is stored in filename. When a merge starts or finishes, we push
867 EmergeJobState objects to the job_queue.
868 """
869
870 SetupWorkerSignals()
871 settings, trees, mtimedb = emerge.settings, emerge.trees, emerge.mtimedb
David Jamesdeebd692011-05-09 17:02:52 -0700872
873 # Disable flushing of caches to save on I/O.
874 if 0 <= vercmp(portage.VERSION, "2.1.9.48"):
875 root = emerge.settings["ROOT"]
876 vardb = emerge.trees[root]["vartree"].dbapi
877 vardb._flush_cache_enabled = False
878
David Jamesfcb70ef2011-02-02 16:02:30 -0800879 opts, spinner = emerge.opts, emerge.spinner
880 opts["--nodeps"] = True
David James386ccd12011-05-04 20:17:42 -0700881 # When Portage launches new processes, it goes on a rampage and closes all
882 # open file descriptors. Ask Portage not to do that, as it breaks us.
883 portage.process.get_open_fds = lambda: []
David Jamesfcb70ef2011-02-02 16:02:30 -0800884 while True:
885 # Wait for a new item to show up on the queue. This is a blocking wait,
886 # so if there's nothing to do, we just sit here.
887 target = task_queue.get()
888 if not target:
889 # If target is None, this means that the main thread wants us to quit.
890 # The other workers need to exit too, so we'll push the message back on
891 # to the queue so they'll get it too.
892 task_queue.put(target)
893 return
David James7358d032011-05-19 10:40:03 -0700894 if KILLED.is_set():
895 return
896
David Jamesfcb70ef2011-02-02 16:02:30 -0800897 db_pkg = package_db[target]
898 db_pkg.root_config = emerge.root_config
899 install_list = [db_pkg]
900 pkgname = db_pkg.pf
901 output = tempfile.NamedTemporaryFile(prefix=pkgname + "-", delete=False)
902 start_timestamp = time.time()
903 job = EmergeJobState(target, pkgname, False, output.name, start_timestamp)
904 job_queue.put(job)
905 if "--pretend" in opts:
906 retcode = 0
907 else:
908 save_stdout = sys.stdout
909 save_stderr = sys.stderr
910 try:
911 sys.stdout = output
912 sys.stderr = output
David James386ccd12011-05-04 20:17:42 -0700913 emerge.scheduler_graph.mergelist = install_list
914 scheduler = Scheduler(settings, trees, mtimedb, opts, spinner,
915 favorites=[], graph_config=emerge.scheduler_graph)
David Jamesfcb70ef2011-02-02 16:02:30 -0800916 retcode = scheduler.merge()
917 except Exception:
918 traceback.print_exc(file=output)
919 retcode = 1
920 finally:
921 sys.stdout = save_stdout
922 sys.stderr = save_stderr
923 output.close()
924 if retcode is None:
925 retcode = 0
926
David James7358d032011-05-19 10:40:03 -0700927 if KILLED.is_set():
928 return
929
David Jamesfcb70ef2011-02-02 16:02:30 -0800930 job = EmergeJobState(target, pkgname, True, output.name, start_timestamp,
931 retcode)
932 job_queue.put(job)
933
934
935class LinePrinter(object):
936 """Helper object to print a single line."""
937
938 def __init__(self, line):
939 self.line = line
940
941 def Print(self, seek_locations):
942 print self.line
943
944
945class JobPrinter(object):
946 """Helper object to print output of a job."""
947
948 def __init__(self, job, unlink=False):
949 """Print output of job.
950
951 If unlink is True, unlink the job output file when done."""
952 self.current_time = time.time()
953 self.job = job
954 self.unlink = unlink
955
956 def Print(self, seek_locations):
957
958 job = self.job
959
960 # Calculate how long the job has been running.
961 seconds = self.current_time - job.start_timestamp
962
963 # Note that we've printed out the job so far.
964 job.last_output_timestamp = self.current_time
965
966 # Note that we're starting the job
967 info = "job %s (%dm%.1fs)" % (job.pkgname, seconds / 60, seconds % 60)
968 last_output_seek = seek_locations.get(job.filename, 0)
969 if last_output_seek:
970 print "=== Continue output for %s ===" % info
971 else:
972 print "=== Start output for %s ===" % info
973
974 # Print actual output from job
975 f = codecs.open(job.filename, encoding='utf-8', errors='replace')
976 f.seek(last_output_seek)
977 prefix = job.pkgname + ":"
978 for line in f:
979
980 # Save off our position in the file
981 if line and line[-1] == "\n":
982 last_output_seek = f.tell()
983 line = line[:-1]
984
985 # Print our line
986 print prefix, line.encode('utf-8', 'replace')
987 f.close()
988
989 # Save our last spot in the file so that we don't print out the same
990 # location twice.
991 seek_locations[job.filename] = last_output_seek
992
993 # Note end of output section
994 if job.done:
995 print "=== Complete: %s ===" % info
996 else:
997 print "=== Still running: %s ===" % info
998
999 if self.unlink:
1000 os.unlink(job.filename)
1001
1002
1003def PrintWorker(queue):
1004 """A worker that prints stuff to the screen as requested."""
1005
1006 def ExitHandler(signum, frame):
David James7358d032011-05-19 10:40:03 -07001007 # Set KILLED flag.
1008 KILLED.set()
1009
David Jamesfcb70ef2011-02-02 16:02:30 -08001010 # Switch to default signal handlers so that we'll die after two signals.
David James7358d032011-05-19 10:40:03 -07001011 signal.signal(signal.SIGINT, KillHandler)
1012 signal.signal(signal.SIGTERM, KillHandler)
David Jamesfcb70ef2011-02-02 16:02:30 -08001013
1014 # Don't exit on the first SIGINT / SIGTERM, because the parent worker will
1015 # handle it and tell us when we need to exit.
1016 signal.signal(signal.SIGINT, ExitHandler)
1017 signal.signal(signal.SIGTERM, ExitHandler)
1018
1019 # seek_locations is a map indicating the position we are at in each file.
1020 # It starts off empty, but is set by the various Print jobs as we go along
1021 # to indicate where we left off in each file.
1022 seek_locations = {}
1023 while True:
1024 try:
1025 job = queue.get()
1026 if job:
1027 job.Print(seek_locations)
1028 else:
1029 break
1030 except IOError as ex:
1031 if ex.errno == errno.EINTR:
1032 # Looks like we received a signal. Keep printing.
1033 continue
1034 raise
1035
David Jamesfcb70ef2011-02-02 16:02:30 -08001036class EmergeQueue(object):
1037 """Class to schedule emerge jobs according to a dependency graph."""
1038
1039 def __init__(self, deps_map, emerge, package_db, show_output):
1040 # Store the dependency graph.
1041 self._deps_map = deps_map
1042 # Initialize the running queue to empty
1043 self._jobs = {}
1044 # List of total package installs represented in deps_map.
1045 install_jobs = [x for x in deps_map if deps_map[x]["action"] == "merge"]
1046 self._total_jobs = len(install_jobs)
1047 self._show_output = show_output
1048
1049 if "--pretend" in emerge.opts:
1050 print "Skipping merge because of --pretend mode."
1051 sys.exit(0)
1052
David James7358d032011-05-19 10:40:03 -07001053 # Set a process group so we can easily terminate all children.
1054 os.setsid()
1055
David Jamesfcb70ef2011-02-02 16:02:30 -08001056 # Setup scheduler graph object. This is used by the child processes
1057 # to help schedule jobs.
1058 emerge.scheduler_graph = emerge.depgraph.schedulerGraph()
1059
1060 # Calculate how many jobs we can run in parallel. We don't want to pass
1061 # the --jobs flag over to emerge itself, because that'll tell emerge to
1062 # hide its output, and said output is quite useful for debugging hung
1063 # jobs.
1064 procs = min(self._total_jobs,
1065 emerge.opts.pop("--jobs", multiprocessing.cpu_count()))
1066 self._emerge_queue = multiprocessing.Queue()
1067 self._job_queue = multiprocessing.Queue()
1068 self._print_queue = multiprocessing.Queue()
1069 args = (self._emerge_queue, self._job_queue, emerge, package_db)
1070 self._pool = multiprocessing.Pool(procs, EmergeWorker, args)
1071 self._print_worker = multiprocessing.Process(target=PrintWorker,
1072 args=[self._print_queue])
1073 self._print_worker.start()
1074
1075 # Initialize the failed queue to empty.
1076 self._retry_queue = []
1077 self._failed = set()
1078
1079 # Print an update before we launch the merges.
1080 self._Status()
1081
1082 # Setup an exit handler so that we print nice messages if we are
1083 # terminated.
1084 self._SetupExitHandler()
1085
1086 # Schedule our jobs.
1087 for target, info in deps_map.items():
1088 if not info["needs"]:
1089 self._Schedule(target)
1090
1091 def _SetupExitHandler(self):
1092
1093 def ExitHandler(signum, frame):
David James7358d032011-05-19 10:40:03 -07001094 # Set KILLED flag.
1095 KILLED.set()
David Jamesfcb70ef2011-02-02 16:02:30 -08001096
1097 # Kill our signal handlers so we don't get called recursively
David James7358d032011-05-19 10:40:03 -07001098 signal.signal(signal.SIGINT, KillHandler)
1099 signal.signal(signal.SIGTERM, KillHandler)
David Jamesfcb70ef2011-02-02 16:02:30 -08001100
1101 # Print our current job status
1102 for target, job in self._jobs.iteritems():
1103 if job:
1104 self._print_queue.put(JobPrinter(job, unlink=True))
1105
1106 # Notify the user that we are exiting
1107 self._Print("Exiting on signal %s" % signum)
David James7358d032011-05-19 10:40:03 -07001108 self._print_queue.put(None)
1109 self._print_worker.join()
David Jamesfcb70ef2011-02-02 16:02:30 -08001110
1111 # Kill child threads, then exit.
David James7358d032011-05-19 10:40:03 -07001112 os.killpg(0, signal.SIGKILL)
David Jamesfcb70ef2011-02-02 16:02:30 -08001113 sys.exit(1)
1114
1115 # Print out job status when we are killed
1116 signal.signal(signal.SIGINT, ExitHandler)
1117 signal.signal(signal.SIGTERM, ExitHandler)
1118
1119 def _Schedule(self, target):
1120 # We maintain a tree of all deps, if this doesn't need
1121 # to be installed just free up it's children and continue.
1122 # It is possible to reinstall deps of deps, without reinstalling
1123 # first level deps, like so:
1124 # chromeos (merge) -> eselect (nomerge) -> python (merge)
David James386ccd12011-05-04 20:17:42 -07001125 if target not in self._deps_map:
1126 pass
1127 elif self._deps_map[target]["action"] == "nomerge":
David Jamesfcb70ef2011-02-02 16:02:30 -08001128 self._Finish(target)
David Jamesd20a6d92011-04-26 16:11:59 -07001129 elif target not in self._jobs:
David Jamesfcb70ef2011-02-02 16:02:30 -08001130 # Kick off the build if it's marked to be built.
1131 self._jobs[target] = None
1132 self._emerge_queue.put(target)
1133
1134 def _LoadAvg(self):
1135 loads = open("/proc/loadavg", "r").readline().split()[:3]
1136 return " ".join(loads)
1137
1138 def _Print(self, line):
1139 """Print a single line."""
1140 self._print_queue.put(LinePrinter(line))
1141
1142 def _Status(self):
1143 """Print status."""
1144 current_time = time.time()
1145 no_output = True
1146
1147 # Print interim output every minute if --show-output is used. Otherwise,
1148 # print notifications about running packages every 2 minutes, and print
1149 # full output for jobs that have been running for 60 minutes or more.
1150 if self._show_output:
1151 interval = 60
1152 notify_interval = 0
1153 else:
1154 interval = 60 * 60
1155 notify_interval = 60 * 2
1156 for target, job in self._jobs.iteritems():
1157 if job:
1158 last_timestamp = max(job.start_timestamp, job.last_output_timestamp)
1159 if last_timestamp + interval < current_time:
1160 self._print_queue.put(JobPrinter(job))
1161 job.last_output_timestamp = current_time
1162 no_output = False
1163 elif (notify_interval and
1164 job.last_notify_timestamp + notify_interval < current_time):
1165 job_seconds = current_time - job.start_timestamp
1166 args = (job.pkgname, job_seconds / 60, job_seconds % 60, job.filename)
1167 info = "Still building %s (%dm%.1fs). Logs in %s" % args
1168 job.last_notify_timestamp = current_time
1169 self._Print(info)
1170 no_output = False
1171
1172 # If we haven't printed any messages yet, print a general status message
1173 # here.
1174 if no_output:
1175 seconds = current_time - GLOBAL_START
1176 line = ("Pending %s, Ready %s, Running %s, Retrying %s, Total %s "
1177 "[Time %dm%.1fs Load %s]")
1178 qsize = self._emerge_queue.qsize()
1179 self._Print(line % (len(self._deps_map), qsize, len(self._jobs) - qsize,
1180 len(self._retry_queue), self._total_jobs,
1181 seconds / 60, seconds % 60, self._LoadAvg()))
1182
1183 def _Finish(self, target):
1184 """Mark a target as completed and unblock dependecies."""
1185 for dep in self._deps_map[target]["provides"]:
1186 del self._deps_map[dep]["needs"][target]
1187 if not self._deps_map[dep]["needs"]:
1188 self._Schedule(dep)
1189 self._deps_map.pop(target)
1190
1191 def _Retry(self):
1192 if self._retry_queue:
1193 target = self._retry_queue.pop(0)
1194 self._Schedule(target)
1195 self._Print("Retrying emerge of %s." % target)
1196
1197 def _Exit(self):
1198 # Tell emerge workers to exit. They all exit when 'None' is pushed
1199 # to the queue.
1200 self._emerge_queue.put(None)
1201 self._pool.close()
1202 self._pool.join()
1203
1204 # Now that our workers are finished, we can kill the print queue.
1205 self._print_queue.put(None)
1206 self._print_worker.join()
1207
1208 def Run(self):
1209 """Run through the scheduled ebuilds.
1210
1211 Keep running so long as we have uninstalled packages in the
1212 dependency graph to merge.
1213 """
1214 while self._deps_map:
1215 # Check here that we are actually waiting for something.
1216 if (self._emerge_queue.empty() and
1217 self._job_queue.empty() and
1218 not self._jobs and
1219 self._deps_map):
1220 # If we have failed on a package, retry it now.
1221 if self._retry_queue:
1222 self._Retry()
1223 else:
1224 # Tell child threads to exit.
1225 self._Exit()
1226
1227 # The dependency map is helpful for debugging failures.
1228 PrintDepsMap(self._deps_map)
1229
1230 # Tell the user why we're exiting.
1231 if self._failed:
1232 print "Packages failed: %s" % " ,".join(self._failed)
1233 else:
1234 print "Deadlock! Circular dependencies!"
1235 sys.exit(1)
1236
1237 try:
1238 job = self._job_queue.get(timeout=5)
1239 except Queue.Empty:
1240 # Print an update.
1241 self._Status()
1242 continue
1243
1244 target = job.target
1245
1246 if not job.done:
1247 self._jobs[target] = job
1248 self._Print("Started %s (logged in %s)" % (target, job.filename))
1249 continue
1250
1251 # Print output of job
1252 if self._show_output or job.retcode != 0:
1253 self._print_queue.put(JobPrinter(job, unlink=True))
1254 else:
1255 os.unlink(job.filename)
1256 del self._jobs[target]
1257
1258 seconds = time.time() - job.start_timestamp
1259 details = "%s (in %dm%.1fs)" % (target, seconds / 60, seconds % 60)
1260
1261 # Complain if necessary.
1262 if job.retcode != 0:
1263 # Handle job failure.
1264 if target in self._failed:
1265 # If this job has failed previously, give up.
1266 self._Print("Failed %s. Your build has failed." % details)
1267 else:
1268 # Queue up this build to try again after a long while.
1269 self._retry_queue.append(target)
1270 self._failed.add(target)
1271 self._Print("Failed %s, retrying later." % details)
1272 else:
1273 if target in self._failed and self._retry_queue:
1274 # If we have successfully retried a failed package, and there
1275 # are more failed packages, try the next one. We will only have
1276 # one retrying package actively running at a time.
1277 self._Retry()
1278
1279 self._Print("Completed %s" % details)
1280 # Mark as completed and unblock waiting ebuilds.
1281 self._Finish(target)
1282
1283 # Print an update.
1284 self._Status()
1285
1286 # Tell child threads to exit.
1287 self._Print("Merge complete")
1288 self._Exit()
1289
1290
1291def main():
1292
David James57437532011-05-06 15:51:21 -07001293 parallel_emerge_args = sys.argv[:]
David Jamesfcb70ef2011-02-02 16:02:30 -08001294 deps = DepGraphGenerator()
David James57437532011-05-06 15:51:21 -07001295 deps.Initialize(parallel_emerge_args[1:])
David Jamesfcb70ef2011-02-02 16:02:30 -08001296 emerge = deps.emerge
1297
1298 if emerge.action is not None:
1299 sys.argv = deps.ParseParallelEmergeArgs(sys.argv)
1300 sys.exit(emerge_main())
1301 elif not emerge.cmdline_packages:
1302 Usage()
1303 sys.exit(1)
1304
1305 # Unless we're in pretend mode, there's not much point running without
1306 # root access. We need to be able to install packages.
1307 #
1308 # NOTE: Even if you're running --pretend, it's a good idea to run
1309 # parallel_emerge with root access so that portage can write to the
1310 # dependency cache. This is important for performance.
1311 if "--pretend" not in emerge.opts and portage.secpass < 2:
1312 print "parallel_emerge: superuser access is required."
1313 sys.exit(1)
1314
1315 if "--quiet" not in emerge.opts:
1316 cmdline_packages = " ".join(emerge.cmdline_packages)
David Jamesfcb70ef2011-02-02 16:02:30 -08001317 print "Starting fast-emerge."
1318 print " Building package %s on %s" % (cmdline_packages,
1319 deps.board or "root")
David Jamesfcb70ef2011-02-02 16:02:30 -08001320
David James386ccd12011-05-04 20:17:42 -07001321 deps_tree, deps_info = deps.GenDependencyTree()
David Jamesfcb70ef2011-02-02 16:02:30 -08001322
1323 # You want me to be verbose? I'll give you two trees! Twice as much value.
1324 if "--tree" in emerge.opts and "--verbose" in emerge.opts:
1325 deps.PrintTree(deps_tree)
1326
David James386ccd12011-05-04 20:17:42 -07001327 deps_graph = deps.GenDependencyGraph(deps_tree, deps_info)
David Jamesfcb70ef2011-02-02 16:02:30 -08001328
1329 # OK, time to print out our progress so far.
1330 deps.PrintInstallPlan(deps_graph)
1331 if "--tree" in emerge.opts:
1332 PrintDepsMap(deps_graph)
1333
1334 # Are we upgrading portage? If so, and there are more packages to merge,
1335 # schedule a restart of parallel_emerge to merge the rest. This ensures that
1336 # we pick up all updates to portage settings before merging any more
1337 # packages.
1338 portage_upgrade = False
1339 root = emerge.settings["ROOT"]
1340 final_db = emerge.depgraph._dynamic_config.mydbapi[root]
1341 if root == "/":
1342 for db_pkg in final_db.match_pkgs("sys-apps/portage"):
1343 portage_pkg = deps_graph.get(db_pkg.cpv)
1344 if portage_pkg and len(deps_graph) > 1:
1345 portage_pkg["needs"].clear()
1346 portage_pkg["provides"].clear()
1347 deps_graph = { str(db_pkg.cpv): portage_pkg }
1348 portage_upgrade = True
1349 if "--quiet" not in emerge.opts:
1350 print "Upgrading portage first, then restarting..."
1351
1352 # Run the queued emerges.
1353 scheduler = EmergeQueue(deps_graph, emerge, deps.package_db, deps.show_output)
1354 scheduler.Run()
1355
1356 # Update world.
1357 if ("--oneshot" not in emerge.opts and
1358 "--pretend" not in emerge.opts):
1359 world_set = emerge.root_config.sets["selected"]
1360 new_world_pkgs = []
1361 for pkg in emerge.cmdline_packages:
1362 for db_pkg in final_db.match_pkgs(pkg):
1363 print "Adding %s to world" % db_pkg.cp
1364 new_world_pkgs.append(db_pkg.cp)
1365 if new_world_pkgs:
1366 world_set.update(new_world_pkgs)
1367
1368 # Update environment (library cache, symlinks, etc.)
1369 if deps.board and "--pretend" not in emerge.opts:
1370 portage.env_update()
1371
1372 # If we already upgraded portage, we don't need to do so again. But we do
1373 # need to upgrade the rest of the packages. So we'll go ahead and do that.
David Jamesebc3ae02011-05-21 20:46:10 -07001374 #
1375 # In order to grant the child permission to run setsid, we need to run sudo
1376 # again. We preserve SUDO_USER here in case an ebuild depends on it.
David Jamesfcb70ef2011-02-02 16:02:30 -08001377 if portage_upgrade:
David Jamesebc3ae02011-05-21 20:46:10 -07001378 sudo = ["sudo", "-E", "SUDO_USER=%s" % os.environ.get("SUDO_USER", "")]
1379 args = sudo + parallel_emerge_args + ["--exclude=sys-apps/portage"]
1380 os.execvp("sudo", args)
David Jamesfcb70ef2011-02-02 16:02:30 -08001381
1382 print "Done"
1383 sys.exit(0)
1384
1385if __name__ == "__main__":
1386 main()