David James | fcb70ef | 2011-02-02 16:02:30 -0800 | [diff] [blame] | 1 | #!/usr/bin/python2.6 |
| 2 | # Copyright (c) 2010 The Chromium OS Authors. All rights reserved. |
| 3 | # Use of this source code is governed by a BSD-style license that can be |
| 4 | # found in the LICENSE file. |
| 5 | |
| 6 | """Program to run emerge in parallel, for significant speedup. |
| 7 | |
| 8 | Usage: |
David James | 386ccd1 | 2011-05-04 20:17:42 -0700 | [diff] [blame] | 9 | ./parallel_emerge [--board=BOARD] [--workon=PKGS] |
David James | fcb70ef | 2011-02-02 16:02:30 -0800 | [diff] [blame] | 10 | [--force-remote-binary=PKGS] [emerge args] package |
| 11 | |
| 12 | Basic operation: |
| 13 | Runs 'emerge -p --debug' to display dependencies, and stores a |
| 14 | dependency graph. All non-blocked packages are launched in parallel, |
| 15 | as 'emerge --nodeps package' with any blocked packages being emerged |
| 16 | immediately upon deps being met. |
| 17 | |
| 18 | For this to work effectively, /usr/lib/portage/pym/portage/locks.py |
| 19 | must be stubbed out, preventing portage from slowing itself with |
| 20 | unneccesary locking, as this script ensures that emerge is run in such |
| 21 | a way that common resources are never in conflict. This is controlled |
| 22 | by an environment variable PORTAGE_LOCKS set in parallel emerge |
| 23 | subprocesses. |
| 24 | |
| 25 | Parallel Emerge unlocks two things during operation, here's what you |
| 26 | must do to keep this safe: |
| 27 | * Storage dir containing binary packages. - Don't emerge new |
| 28 | packages while installing the existing ones. |
| 29 | * Portage database - You must not examine deps while modifying the |
| 30 | database. Therefore you may only parallelize "-p" read only access, |
| 31 | or "--nodeps" write only access. |
| 32 | Caveats: |
| 33 | * Some ebuild packages have incorrectly specified deps, and running |
| 34 | them in parallel is more likely to bring out these failures. |
| 35 | * Some ebuilds (especially the build part) have complex dependencies |
| 36 | that are not captured well by this script (it may be necessary to |
| 37 | install an old package to build, but then install a newer version |
| 38 | of the same package for a runtime dep). |
| 39 | """ |
| 40 | |
| 41 | import codecs |
| 42 | import copy |
| 43 | import errno |
| 44 | import multiprocessing |
| 45 | import os |
| 46 | import Queue |
David James | fcb70ef | 2011-02-02 16:02:30 -0800 | [diff] [blame] | 47 | import signal |
| 48 | import sys |
| 49 | import tempfile |
| 50 | import time |
| 51 | import traceback |
David James | fcb70ef | 2011-02-02 16:02:30 -0800 | [diff] [blame] | 52 | |
| 53 | # If PORTAGE_USERNAME isn't specified, scrape it from the $HOME variable. On |
| 54 | # Chromium OS, the default "portage" user doesn't have the necessary |
| 55 | # permissions. It'd be easier if we could default to $USERNAME, but $USERNAME |
| 56 | # is "root" here because we get called through sudo. |
| 57 | # |
| 58 | # We need to set this before importing any portage modules, because portage |
| 59 | # looks up "PORTAGE_USERNAME" at import time. |
| 60 | # |
| 61 | # NOTE: .bashrc sets PORTAGE_USERNAME = $USERNAME, so most people won't |
| 62 | # encounter this case unless they have an old chroot or blow away the |
| 63 | # environment by running sudo without the -E specifier. |
| 64 | if "PORTAGE_USERNAME" not in os.environ: |
| 65 | homedir = os.environ.get("HOME") |
| 66 | if homedir: |
| 67 | os.environ["PORTAGE_USERNAME"] = os.path.basename(homedir) |
| 68 | |
| 69 | # Portage doesn't expose dependency trees in its public API, so we have to |
| 70 | # make use of some private APIs here. These modules are found under |
| 71 | # /usr/lib/portage/pym/. |
| 72 | # |
| 73 | # TODO(davidjames): Update Portage to expose public APIs for these features. |
| 74 | from _emerge.actions import adjust_configs |
| 75 | from _emerge.actions import load_emerge_config |
| 76 | from _emerge.create_depgraph_params import create_depgraph_params |
David James | 386ccd1 | 2011-05-04 20:17:42 -0700 | [diff] [blame] | 77 | from _emerge.depgraph import backtrack_depgraph |
David James | fcb70ef | 2011-02-02 16:02:30 -0800 | [diff] [blame] | 78 | from _emerge.main import emerge_main |
| 79 | from _emerge.main import parse_opts |
| 80 | from _emerge.Package import Package |
| 81 | from _emerge.Scheduler import Scheduler |
| 82 | from _emerge.SetArg import SetArg |
| 83 | from _emerge.stdout_spinner import stdout_spinner |
David James | 386ccd1 | 2011-05-04 20:17:42 -0700 | [diff] [blame] | 84 | from portage._global_updates import _global_updates |
| 85 | from portage.versions import vercmp |
David James | fcb70ef | 2011-02-02 16:02:30 -0800 | [diff] [blame] | 86 | import portage |
| 87 | import portage.debug |
David James | fcb70ef | 2011-02-02 16:02:30 -0800 | [diff] [blame] | 88 | |
David James | 386ccd1 | 2011-05-04 20:17:42 -0700 | [diff] [blame] | 89 | NEW_PORTAGE = (0 <= vercmp(portage.VERSION, "2.1.9.46-r2")) |
David James | fcb70ef | 2011-02-02 16:02:30 -0800 | [diff] [blame] | 90 | |
| 91 | def Usage(): |
| 92 | """Print usage.""" |
| 93 | print "Usage:" |
David James | 386ccd1 | 2011-05-04 20:17:42 -0700 | [diff] [blame] | 94 | print " ./parallel_emerge [--board=BOARD] [--workon=PKGS]" |
David James | fcb70ef | 2011-02-02 16:02:30 -0800 | [diff] [blame] | 95 | print " [--rebuild] [emerge args] package" |
| 96 | print |
| 97 | print "Packages specified as workon packages are always built from source." |
David James | fcb70ef | 2011-02-02 16:02:30 -0800 | [diff] [blame] | 98 | print |
| 99 | print "The --workon argument is mainly useful when you want to build and" |
| 100 | print "install packages that you are working on unconditionally, but do not" |
| 101 | print "to have to rev the package to indicate you want to build it from" |
| 102 | print "source. The build_packages script will automatically supply the" |
| 103 | print "workon argument to emerge, ensuring that packages selected using" |
| 104 | print "cros-workon are rebuilt." |
| 105 | print |
| 106 | print "The --rebuild option rebuilds packages whenever their dependencies" |
| 107 | print "are changed. This ensures that your build is correct." |
| 108 | sys.exit(1) |
| 109 | |
| 110 | |
David James | fcb70ef | 2011-02-02 16:02:30 -0800 | [diff] [blame] | 111 | # Global start time |
| 112 | GLOBAL_START = time.time() |
| 113 | |
David James | 7358d03 | 2011-05-19 10:40:03 -0700 | [diff] [blame] | 114 | # Whether process has been killed by a signal. |
| 115 | KILLED = multiprocessing.Event() |
| 116 | |
David James | fcb70ef | 2011-02-02 16:02:30 -0800 | [diff] [blame] | 117 | |
| 118 | class EmergeData(object): |
| 119 | """This simple struct holds various emerge variables. |
| 120 | |
| 121 | This struct helps us easily pass emerge variables around as a unit. |
| 122 | These variables are used for calculating dependencies and installing |
| 123 | packages. |
| 124 | """ |
| 125 | |
| 126 | __slots__ = ["action", "cmdline_packages", "depgraph", "mtimedb", "opts", |
| 127 | "root_config", "scheduler_graph", "settings", "spinner", |
| 128 | "trees"] |
| 129 | |
| 130 | def __init__(self): |
| 131 | # The action the user requested. If the user is installing packages, this |
| 132 | # is None. If the user is doing anything other than installing packages, |
| 133 | # this will contain the action name, which will map exactly to the |
| 134 | # long-form name of the associated emerge option. |
| 135 | # |
| 136 | # Example: If you call parallel_emerge --unmerge package, the action name |
| 137 | # will be "unmerge" |
| 138 | self.action = None |
| 139 | |
| 140 | # The list of packages the user passed on the command-line. |
| 141 | self.cmdline_packages = None |
| 142 | |
| 143 | # The emerge dependency graph. It'll contain all the packages involved in |
| 144 | # this merge, along with their versions. |
| 145 | self.depgraph = None |
| 146 | |
| 147 | # A dict of the options passed to emerge. This dict has been cleaned up |
| 148 | # a bit by parse_opts, so that it's a bit easier for the emerge code to |
| 149 | # look at the options. |
| 150 | # |
| 151 | # Emerge takes a few shortcuts in its cleanup process to make parsing of |
| 152 | # the options dict easier. For example, if you pass in "--usepkg=n", the |
| 153 | # "--usepkg" flag is just left out of the dictionary altogether. Because |
| 154 | # --usepkg=n is the default, this makes parsing easier, because emerge |
| 155 | # can just assume that if "--usepkg" is in the dictionary, it's enabled. |
| 156 | # |
| 157 | # These cleanup processes aren't applied to all options. For example, the |
| 158 | # --with-bdeps flag is passed in as-is. For a full list of the cleanups |
| 159 | # applied by emerge, see the parse_opts function in the _emerge.main |
| 160 | # package. |
| 161 | self.opts = None |
| 162 | |
| 163 | # A dictionary used by portage to maintain global state. This state is |
| 164 | # loaded from disk when portage starts up, and saved to disk whenever we |
| 165 | # call mtimedb.commit(). |
| 166 | # |
| 167 | # This database contains information about global updates (i.e., what |
| 168 | # version of portage we have) and what we're currently doing. Portage |
| 169 | # saves what it is currently doing in this database so that it can be |
| 170 | # resumed when you call it with the --resume option. |
| 171 | # |
| 172 | # parallel_emerge does not save what it is currently doing in the mtimedb, |
| 173 | # so we do not support the --resume option. |
| 174 | self.mtimedb = None |
| 175 | |
| 176 | # The portage configuration for our current root. This contains the portage |
| 177 | # settings (see below) and the three portage trees for our current root. |
| 178 | # (The three portage trees are explained below, in the documentation for |
| 179 | # the "trees" member.) |
| 180 | self.root_config = None |
| 181 | |
| 182 | # The scheduler graph is used by emerge to calculate what packages to |
| 183 | # install. We don't actually install any deps, so this isn't really used, |
| 184 | # but we pass it in to the Scheduler object anyway. |
| 185 | self.scheduler_graph = None |
| 186 | |
| 187 | # Portage settings for our current session. Most of these settings are set |
| 188 | # in make.conf inside our current install root. |
| 189 | self.settings = None |
| 190 | |
| 191 | # The spinner, which spews stuff to stdout to indicate that portage is |
| 192 | # doing something. We maintain our own spinner, so we set the portage |
| 193 | # spinner to "silent" mode. |
| 194 | self.spinner = None |
| 195 | |
| 196 | # The portage trees. There are separate portage trees for each root. To get |
| 197 | # the portage tree for the current root, you can look in self.trees[root], |
| 198 | # where root = self.settings["ROOT"]. |
| 199 | # |
| 200 | # In each root, there are three trees: vartree, porttree, and bintree. |
| 201 | # - vartree: A database of the currently-installed packages. |
| 202 | # - porttree: A database of ebuilds, that can be used to build packages. |
| 203 | # - bintree: A database of binary packages. |
| 204 | self.trees = None |
| 205 | |
| 206 | |
| 207 | class DepGraphGenerator(object): |
| 208 | """Grab dependency information about packages from portage. |
| 209 | |
| 210 | Typical usage: |
| 211 | deps = DepGraphGenerator() |
| 212 | deps.Initialize(sys.argv[1:]) |
| 213 | deps_tree, deps_info = deps.GenDependencyTree() |
| 214 | deps_graph = deps.GenDependencyGraph(deps_tree, deps_info) |
| 215 | deps.PrintTree(deps_tree) |
| 216 | PrintDepsMap(deps_graph) |
| 217 | """ |
| 218 | |
David James | 386ccd1 | 2011-05-04 20:17:42 -0700 | [diff] [blame] | 219 | __slots__ = ["board", "emerge", "package_db", "show_output"] |
David James | fcb70ef | 2011-02-02 16:02:30 -0800 | [diff] [blame] | 220 | |
| 221 | def __init__(self): |
| 222 | self.board = None |
| 223 | self.emerge = EmergeData() |
David James | fcb70ef | 2011-02-02 16:02:30 -0800 | [diff] [blame] | 224 | self.package_db = {} |
David James | fcb70ef | 2011-02-02 16:02:30 -0800 | [diff] [blame] | 225 | self.show_output = False |
David James | fcb70ef | 2011-02-02 16:02:30 -0800 | [diff] [blame] | 226 | |
| 227 | def ParseParallelEmergeArgs(self, argv): |
| 228 | """Read the parallel emerge arguments from the command-line. |
| 229 | |
| 230 | We need to be compatible with emerge arg format. We scrape arguments that |
| 231 | are specific to parallel_emerge, and pass through the rest directly to |
| 232 | emerge. |
| 233 | Args: |
| 234 | argv: arguments list |
| 235 | Returns: |
| 236 | Arguments that don't belong to parallel_emerge |
| 237 | """ |
| 238 | emerge_args = [] |
| 239 | for arg in argv: |
| 240 | # Specifically match arguments that are specific to parallel_emerge, and |
| 241 | # pass through the rest. |
| 242 | if arg.startswith("--board="): |
| 243 | self.board = arg.replace("--board=", "") |
| 244 | elif arg.startswith("--workon="): |
| 245 | workon_str = arg.replace("--workon=", "") |
David James | 386ccd1 | 2011-05-04 20:17:42 -0700 | [diff] [blame] | 246 | if NEW_PORTAGE: |
| 247 | emerge_args.append("--reinstall-atoms=%s" % workon_str) |
| 248 | emerge_args.append("--usepkg-exclude=%s" % workon_str) |
David James | fcb70ef | 2011-02-02 16:02:30 -0800 | [diff] [blame] | 249 | elif arg.startswith("--force-remote-binary="): |
| 250 | force_remote_binary = arg.replace("--force-remote-binary=", "") |
David James | 386ccd1 | 2011-05-04 20:17:42 -0700 | [diff] [blame] | 251 | if NEW_PORTAGE: |
| 252 | emerge_args.append("--useoldpkg-atoms=%s" % force_remote_binary) |
David James | fcb70ef | 2011-02-02 16:02:30 -0800 | [diff] [blame] | 253 | elif arg == "--show-output": |
| 254 | self.show_output = True |
David James | 386ccd1 | 2011-05-04 20:17:42 -0700 | [diff] [blame] | 255 | elif arg == "--rebuild": |
| 256 | if NEW_PORTAGE: |
| 257 | emerge_args.append("--rebuild-if-unbuilt") |
David James | fcb70ef | 2011-02-02 16:02:30 -0800 | [diff] [blame] | 258 | else: |
| 259 | # Not one of our options, so pass through to emerge. |
| 260 | emerge_args.append(arg) |
| 261 | |
David James | 386ccd1 | 2011-05-04 20:17:42 -0700 | [diff] [blame] | 262 | # These packages take a really long time to build, so, for expediency, we |
| 263 | # are blacklisting them from automatic rebuilds because one of their |
| 264 | # dependencies needs to be recompiled. |
| 265 | for pkg in ("chromeos-base/chromeos-chrome", "media-plugins/o3d", |
| 266 | "dev-java/icedtea"): |
| 267 | if NEW_PORTAGE: |
| 268 | emerge_args.append("--rebuild-exclude=%s" % pkg) |
David James | fcb70ef | 2011-02-02 16:02:30 -0800 | [diff] [blame] | 269 | |
| 270 | return emerge_args |
| 271 | |
| 272 | def Initialize(self, args): |
| 273 | """Initializer. Parses arguments and sets up portage state.""" |
| 274 | |
| 275 | # Parse and strip out args that are just intended for parallel_emerge. |
| 276 | emerge_args = self.ParseParallelEmergeArgs(args) |
| 277 | |
| 278 | # Setup various environment variables based on our current board. These |
| 279 | # variables are normally setup inside emerge-${BOARD}, but since we don't |
| 280 | # call that script, we have to set it up here. These variables serve to |
| 281 | # point our tools at /build/BOARD and to setup cross compiles to the |
| 282 | # appropriate board as configured in toolchain.conf. |
| 283 | if self.board: |
| 284 | os.environ["PORTAGE_CONFIGROOT"] = "/build/" + self.board |
| 285 | os.environ["PORTAGE_SYSROOT"] = "/build/" + self.board |
| 286 | os.environ["SYSROOT"] = "/build/" + self.board |
| 287 | srcroot = "%s/../../src" % os.path.dirname(os.path.realpath(__file__)) |
| 288 | # Strip the variant out of the board name to look for the toolchain. This |
| 289 | # is similar to what setup_board does. |
| 290 | board_no_variant = self.board.split('_')[0] |
| 291 | public_toolchain_path = ("%s/overlays/overlay-%s/toolchain.conf" % |
| 292 | (srcroot, board_no_variant)) |
| 293 | private_toolchain_path = ( |
| 294 | "%s/private-overlays/overlay-%s-private/toolchain.conf" % |
| 295 | (srcroot, board_no_variant)) |
| 296 | if os.path.isfile(public_toolchain_path): |
| 297 | toolchain_path = public_toolchain_path |
| 298 | elif os.path.isfile(private_toolchain_path): |
| 299 | toolchain_path = private_toolchain_path |
| 300 | else: |
| 301 | print "Not able to locate toolchain.conf in board overlays" |
| 302 | sys.exit(1) |
| 303 | |
| 304 | f = open(toolchain_path) |
| 305 | os.environ["CHOST"] = f.readline().strip() |
| 306 | f.close() |
| 307 | |
| 308 | # Although CHROMEOS_ROOT isn't specific to boards, it's normally setup |
| 309 | # inside emerge-${BOARD}, so we set it up here for compatibility. It |
| 310 | # will be going away soon as we migrate to CROS_WORKON_SRCROOT. |
| 311 | os.environ.setdefault("CHROMEOS_ROOT", os.environ["HOME"] + "/trunk") |
| 312 | |
| 313 | # Turn off interactive delays |
| 314 | os.environ["EBEEP_IGNORE"] = "1" |
| 315 | os.environ["EPAUSE_IGNORE"] = "1" |
| 316 | os.environ["UNMERGE_DELAY"] = "0" |
| 317 | |
| 318 | # Parse the emerge options. |
David James | ea3ca33 | 2011-05-26 11:48:29 -0700 | [diff] [blame] | 319 | action, opts, cmdline_packages = parse_opts(emerge_args, silent=True) |
David James | fcb70ef | 2011-02-02 16:02:30 -0800 | [diff] [blame] | 320 | |
| 321 | # Set environment variables based on options. Portage normally sets these |
| 322 | # environment variables in emerge_main, but we can't use that function, |
| 323 | # because it also does a bunch of other stuff that we don't want. |
| 324 | # TODO(davidjames): Patch portage to move this logic into a function we can |
| 325 | # reuse here. |
| 326 | if "--debug" in opts: |
| 327 | os.environ["PORTAGE_DEBUG"] = "1" |
| 328 | if "--config-root" in opts: |
| 329 | os.environ["PORTAGE_CONFIGROOT"] = opts["--config-root"] |
| 330 | if "--root" in opts: |
| 331 | os.environ["ROOT"] = opts["--root"] |
| 332 | if "--accept-properties" in opts: |
| 333 | os.environ["ACCEPT_PROPERTIES"] = opts["--accept-properties"] |
| 334 | |
| 335 | # Portage has two flags for doing collision protection: collision-protect |
| 336 | # and protect-owned. The protect-owned feature is enabled by default and |
| 337 | # is quite useful: it checks to make sure that we don't have multiple |
| 338 | # packages that own the same file. The collision-protect feature is more |
| 339 | # strict, and less useful: it fails if it finds a conflicting file, even |
| 340 | # if that file was created by an earlier ebuild that failed to install. |
| 341 | # |
| 342 | # We want to disable collision-protect here because we don't handle |
| 343 | # failures during the merge step very well. Sometimes we leave old files |
| 344 | # lying around and they cause problems, so for now we disable the flag. |
| 345 | # TODO(davidjames): Look for a better solution. |
| 346 | features = os.environ.get("FEATURES", "") + " -collision-protect" |
| 347 | |
David James | deebd69 | 2011-05-09 17:02:52 -0700 | [diff] [blame] | 348 | # Install packages in parallel. |
| 349 | features = features + " parallel-install" |
| 350 | |
David James | fcb70ef | 2011-02-02 16:02:30 -0800 | [diff] [blame] | 351 | # If we're installing packages to the board, and we're not using the |
| 352 | # official flag, we can enable the following optimizations: |
| 353 | # 1) Don't lock during install step. This allows multiple packages to be |
| 354 | # installed at once. This is safe because our board packages do not |
| 355 | # muck with each other during the post-install step. |
| 356 | # 2) Don't update the environment until the end of the build. This is |
| 357 | # safe because board packages don't need to run during the build -- |
| 358 | # they're cross-compiled, so our CPU architecture doesn't support them |
| 359 | # anyway. |
| 360 | if self.board and os.environ.get("CHROMEOS_OFFICIAL") != "1": |
| 361 | os.environ.setdefault("PORTAGE_LOCKS", "false") |
David James | deebd69 | 2011-05-09 17:02:52 -0700 | [diff] [blame] | 362 | features = features + " -ebuild-locks no-env-update" |
David James | fcb70ef | 2011-02-02 16:02:30 -0800 | [diff] [blame] | 363 | |
| 364 | os.environ["FEATURES"] = features |
| 365 | |
| 366 | # Now that we've setup the necessary environment variables, we can load the |
| 367 | # emerge config from disk. |
| 368 | settings, trees, mtimedb = load_emerge_config() |
| 369 | |
David James | ea3ca33 | 2011-05-26 11:48:29 -0700 | [diff] [blame] | 370 | # Add in EMERGE_DEFAULT_OPTS, if specified. |
| 371 | tmpcmdline = [] |
| 372 | if "--ignore-default-opts" not in opts: |
| 373 | tmpcmdline.extend(settings["EMERGE_DEFAULT_OPTS"].split()) |
| 374 | tmpcmdline.extend(emerge_args) |
| 375 | action, opts, cmdline_packages = parse_opts(tmpcmdline) |
| 376 | |
| 377 | # If we're installing to the board, we want the --root-deps option so that |
| 378 | # portage will install the build dependencies to that location as well. |
| 379 | if self.board: |
| 380 | opts.setdefault("--root-deps", True) |
| 381 | |
David James | fcb70ef | 2011-02-02 16:02:30 -0800 | [diff] [blame] | 382 | # Check whether our portage tree is out of date. Typically, this happens |
| 383 | # when you're setting up a new portage tree, such as in setup_board and |
| 384 | # make_chroot. In that case, portage applies a bunch of global updates |
| 385 | # here. Once the updates are finished, we need to commit any changes |
| 386 | # that the global update made to our mtimedb, and reload the config. |
| 387 | # |
| 388 | # Portage normally handles this logic in emerge_main, but again, we can't |
| 389 | # use that function here. |
| 390 | if _global_updates(trees, mtimedb["updates"]): |
| 391 | mtimedb.commit() |
| 392 | settings, trees, mtimedb = load_emerge_config(trees=trees) |
| 393 | |
| 394 | # Setup implied options. Portage normally handles this logic in |
| 395 | # emerge_main. |
| 396 | if "--buildpkgonly" in opts or "buildpkg" in settings.features: |
| 397 | opts.setdefault("--buildpkg", True) |
| 398 | if "--getbinpkgonly" in opts: |
| 399 | opts.setdefault("--usepkgonly", True) |
| 400 | opts.setdefault("--getbinpkg", True) |
| 401 | if "getbinpkg" in settings.features: |
| 402 | # Per emerge_main, FEATURES=getbinpkg overrides --getbinpkg=n |
| 403 | opts["--getbinpkg"] = True |
| 404 | if "--getbinpkg" in opts or "--usepkgonly" in opts: |
| 405 | opts.setdefault("--usepkg", True) |
| 406 | if "--fetch-all-uri" in opts: |
| 407 | opts.setdefault("--fetchonly", True) |
| 408 | if "--skipfirst" in opts: |
| 409 | opts.setdefault("--resume", True) |
| 410 | if "--buildpkgonly" in opts: |
| 411 | # --buildpkgonly will not merge anything, so it overrides all binary |
| 412 | # package options. |
| 413 | for opt in ("--getbinpkg", "--getbinpkgonly", |
| 414 | "--usepkg", "--usepkgonly"): |
| 415 | opts.pop(opt, None) |
| 416 | if (settings.get("PORTAGE_DEBUG", "") == "1" and |
| 417 | "python-trace" in settings.features): |
| 418 | portage.debug.set_trace(True) |
| 419 | |
| 420 | # Complain about unsupported options |
David James | 386ccd1 | 2011-05-04 20:17:42 -0700 | [diff] [blame] | 421 | for opt in ("--ask", "--ask-enter-invalid", "--resume", "--skipfirst"): |
David James | fcb70ef | 2011-02-02 16:02:30 -0800 | [diff] [blame] | 422 | if opt in opts: |
| 423 | print "%s is not supported by parallel_emerge" % opt |
| 424 | sys.exit(1) |
| 425 | |
| 426 | # Make emerge specific adjustments to the config (e.g. colors!) |
| 427 | adjust_configs(opts, trees) |
| 428 | |
| 429 | # Save our configuration so far in the emerge object |
| 430 | emerge = self.emerge |
| 431 | emerge.action, emerge.opts = action, opts |
| 432 | emerge.settings, emerge.trees, emerge.mtimedb = settings, trees, mtimedb |
| 433 | emerge.cmdline_packages = cmdline_packages |
| 434 | root = settings["ROOT"] |
| 435 | emerge.root_config = trees[root]["root_config"] |
| 436 | |
David James | 386ccd1 | 2011-05-04 20:17:42 -0700 | [diff] [blame] | 437 | if "--usepkg" in opts: |
David James | fcb70ef | 2011-02-02 16:02:30 -0800 | [diff] [blame] | 438 | emerge.trees[root]["bintree"].populate("--getbinpkg" in opts) |
| 439 | |
David James | fcb70ef | 2011-02-02 16:02:30 -0800 | [diff] [blame] | 440 | def CreateDepgraph(self, emerge, packages): |
| 441 | """Create an emerge depgraph object.""" |
| 442 | # Setup emerge options. |
| 443 | emerge_opts = emerge.opts.copy() |
| 444 | |
David James | 386ccd1 | 2011-05-04 20:17:42 -0700 | [diff] [blame] | 445 | # Ask portage to build a dependency graph. with the options we specified |
| 446 | # above. |
David James | fcb70ef | 2011-02-02 16:02:30 -0800 | [diff] [blame] | 447 | params = create_depgraph_params(emerge_opts, emerge.action) |
David James | 386ccd1 | 2011-05-04 20:17:42 -0700 | [diff] [blame] | 448 | success, depgraph, _ = backtrack_depgraph( |
| 449 | emerge.settings, emerge.trees, emerge_opts, params, emerge.action, |
| 450 | packages, emerge.spinner) |
| 451 | emerge.depgraph = depgraph |
David James | fcb70ef | 2011-02-02 16:02:30 -0800 | [diff] [blame] | 452 | |
David James | 386ccd1 | 2011-05-04 20:17:42 -0700 | [diff] [blame] | 453 | # Is it impossible to honor the user's request? Bail! |
| 454 | if not success: |
| 455 | depgraph.display_problems() |
| 456 | sys.exit(1) |
David James | fcb70ef | 2011-02-02 16:02:30 -0800 | [diff] [blame] | 457 | |
| 458 | emerge.depgraph = depgraph |
| 459 | |
| 460 | # Is it impossible to honor the user's request? Bail! |
| 461 | if not success: |
| 462 | depgraph.display_problems() |
| 463 | sys.exit(1) |
| 464 | |
David James | deebd69 | 2011-05-09 17:02:52 -0700 | [diff] [blame] | 465 | # Prime and flush emerge caches. |
| 466 | root = emerge.settings["ROOT"] |
| 467 | vardb = emerge.trees[root]["vartree"].dbapi |
David James | 0bdc5de | 2011-05-12 16:22:26 -0700 | [diff] [blame] | 468 | if "--pretend" not in emerge.opts: |
| 469 | vardb.counter_tick() |
David James | deebd69 | 2011-05-09 17:02:52 -0700 | [diff] [blame] | 470 | vardb.flush_cache() |
| 471 | |
David James | 386ccd1 | 2011-05-04 20:17:42 -0700 | [diff] [blame] | 472 | def GenDependencyTree(self): |
David James | fcb70ef | 2011-02-02 16:02:30 -0800 | [diff] [blame] | 473 | """Get dependency tree info from emerge. |
| 474 | |
David James | fcb70ef | 2011-02-02 16:02:30 -0800 | [diff] [blame] | 475 | Returns: |
| 476 | Dependency tree |
| 477 | """ |
| 478 | start = time.time() |
| 479 | |
| 480 | emerge = self.emerge |
| 481 | |
| 482 | # Create a list of packages to merge |
| 483 | packages = set(emerge.cmdline_packages[:]) |
David James | fcb70ef | 2011-02-02 16:02:30 -0800 | [diff] [blame] | 484 | |
| 485 | # Tell emerge to be quiet. We print plenty of info ourselves so we don't |
| 486 | # need any extra output from portage. |
| 487 | portage.util.noiselimit = -1 |
| 488 | |
| 489 | # My favorite feature: The silent spinner. It doesn't spin. Ever. |
| 490 | # I'd disable the colors by default too, but they look kind of cool. |
| 491 | emerge.spinner = stdout_spinner() |
| 492 | emerge.spinner.update = emerge.spinner.update_quiet |
| 493 | |
| 494 | if "--quiet" not in emerge.opts: |
| 495 | print "Calculating deps..." |
| 496 | |
| 497 | self.CreateDepgraph(emerge, packages) |
| 498 | depgraph = emerge.depgraph |
| 499 | |
| 500 | # Build our own tree from the emerge digraph. |
| 501 | deps_tree = {} |
| 502 | digraph = depgraph._dynamic_config.digraph |
| 503 | for node, node_deps in digraph.nodes.items(): |
| 504 | # Calculate dependency packages that need to be installed first. Each |
| 505 | # child on the digraph is a dependency. The "operation" field specifies |
| 506 | # what we're doing (e.g. merge, uninstall, etc.). The "priorities" array |
| 507 | # contains the type of dependency (e.g. build, runtime, runtime_post, |
| 508 | # etc.) |
| 509 | # |
| 510 | # Emerge itself actually treats some dependencies as "soft" dependencies |
| 511 | # and sometimes ignores them. We don't do that -- we honor all |
| 512 | # dependencies unless we're forced to prune them because they're cyclic. |
| 513 | # |
| 514 | # Portage refers to the identifiers for packages as a CPV. This acronym |
| 515 | # stands for Component/Path/Version. |
| 516 | # |
| 517 | # Here's an example CPV: chromeos-base/power_manager-0.0.1-r1 |
| 518 | # Split up, this CPV would be: |
| 519 | # C -- Component: chromeos-base |
| 520 | # P -- Path: power_manager |
| 521 | # V -- Version: 0.0.1-r1 |
| 522 | # |
| 523 | # We just refer to CPVs as packages here because it's easier. |
| 524 | deps = {} |
| 525 | for child, priorities in node_deps[0].items(): |
| 526 | if isinstance(child, SetArg): continue |
| 527 | deps[str(child.cpv)] = dict(action=str(child.operation), |
| 528 | deptype=str(priorities[-1]), |
| 529 | deps={}) |
| 530 | |
| 531 | # We've built our list of deps, so we can add our package to the tree. |
David James | 386ccd1 | 2011-05-04 20:17:42 -0700 | [diff] [blame] | 532 | if isinstance(node, Package) and node.root == emerge.settings["ROOT"]: |
David James | fcb70ef | 2011-02-02 16:02:30 -0800 | [diff] [blame] | 533 | deps_tree[str(node.cpv)] = dict(action=str(node.operation), |
| 534 | deps=deps) |
| 535 | |
David James | fcb70ef | 2011-02-02 16:02:30 -0800 | [diff] [blame] | 536 | # Ask portage for its install plan, so that we can only throw out |
David James | 386ccd1 | 2011-05-04 20:17:42 -0700 | [diff] [blame] | 537 | # dependencies that portage throws out. |
David James | fcb70ef | 2011-02-02 16:02:30 -0800 | [diff] [blame] | 538 | deps_info = {} |
| 539 | for pkg in depgraph.altlist(): |
| 540 | if isinstance(pkg, Package): |
David James | 386ccd1 | 2011-05-04 20:17:42 -0700 | [diff] [blame] | 541 | assert pkg.root == emerge.settings["ROOT"] |
David James | fcb70ef | 2011-02-02 16:02:30 -0800 | [diff] [blame] | 542 | self.package_db[pkg.cpv] = pkg |
| 543 | |
David James | fcb70ef | 2011-02-02 16:02:30 -0800 | [diff] [blame] | 544 | # Save off info about the package |
David James | 386ccd1 | 2011-05-04 20:17:42 -0700 | [diff] [blame] | 545 | deps_info[str(pkg.cpv)] = {"idx": len(deps_info)} |
David James | fcb70ef | 2011-02-02 16:02:30 -0800 | [diff] [blame] | 546 | |
| 547 | seconds = time.time() - start |
| 548 | if "--quiet" not in emerge.opts: |
| 549 | print "Deps calculated in %dm%.1fs" % (seconds / 60, seconds % 60) |
| 550 | |
| 551 | return deps_tree, deps_info |
| 552 | |
| 553 | def PrintTree(self, deps, depth=""): |
| 554 | """Print the deps we have seen in the emerge output. |
| 555 | |
| 556 | Args: |
| 557 | deps: Dependency tree structure. |
| 558 | depth: Allows printing the tree recursively, with indentation. |
| 559 | """ |
| 560 | for entry in sorted(deps): |
| 561 | action = deps[entry]["action"] |
| 562 | print "%s %s (%s)" % (depth, entry, action) |
| 563 | self.PrintTree(deps[entry]["deps"], depth=depth + " ") |
| 564 | |
David James | 386ccd1 | 2011-05-04 20:17:42 -0700 | [diff] [blame] | 565 | def GenDependencyGraph(self, deps_tree, deps_info): |
David James | fcb70ef | 2011-02-02 16:02:30 -0800 | [diff] [blame] | 566 | """Generate a doubly linked dependency graph. |
| 567 | |
| 568 | Args: |
| 569 | deps_tree: Dependency tree structure. |
| 570 | deps_info: More details on the dependencies. |
| 571 | Returns: |
| 572 | Deps graph in the form of a dict of packages, with each package |
| 573 | specifying a "needs" list and "provides" list. |
| 574 | """ |
| 575 | emerge = self.emerge |
| 576 | root = emerge.settings["ROOT"] |
| 577 | |
David James | fcb70ef | 2011-02-02 16:02:30 -0800 | [diff] [blame] | 578 | # deps_map is the actual dependency graph. |
| 579 | # |
| 580 | # Each package specifies a "needs" list and a "provides" list. The "needs" |
| 581 | # list indicates which packages we depend on. The "provides" list |
| 582 | # indicates the reverse dependencies -- what packages need us. |
| 583 | # |
| 584 | # We also provide some other information in the dependency graph: |
| 585 | # - action: What we're planning on doing with this package. Generally, |
| 586 | # "merge", "nomerge", or "uninstall" |
David James | fcb70ef | 2011-02-02 16:02:30 -0800 | [diff] [blame] | 587 | deps_map = {} |
| 588 | |
| 589 | def ReverseTree(packages): |
| 590 | """Convert tree to digraph. |
| 591 | |
| 592 | Take the tree of package -> requirements and reverse it to a digraph of |
| 593 | buildable packages -> packages they unblock. |
| 594 | Args: |
| 595 | packages: Tree(s) of dependencies. |
| 596 | Returns: |
| 597 | Unsanitized digraph. |
| 598 | """ |
| 599 | for pkg in packages: |
| 600 | |
| 601 | # Create an entry for the package |
| 602 | action = packages[pkg]["action"] |
David James | 386ccd1 | 2011-05-04 20:17:42 -0700 | [diff] [blame] | 603 | default_pkg = {"needs": {}, "provides": set(), "action": action} |
David James | fcb70ef | 2011-02-02 16:02:30 -0800 | [diff] [blame] | 604 | this_pkg = deps_map.setdefault(pkg, default_pkg) |
| 605 | |
| 606 | # Create entries for dependencies of this package first. |
| 607 | ReverseTree(packages[pkg]["deps"]) |
| 608 | |
| 609 | # Add dependencies to this package. |
| 610 | for dep, dep_item in packages[pkg]["deps"].iteritems(): |
| 611 | dep_pkg = deps_map[dep] |
| 612 | dep_type = dep_item["deptype"] |
| 613 | if dep_type != "runtime_post": |
| 614 | dep_pkg["provides"].add(pkg) |
| 615 | this_pkg["needs"][dep] = dep_type |
| 616 | |
David James | fcb70ef | 2011-02-02 16:02:30 -0800 | [diff] [blame] | 617 | def FindCycles(): |
| 618 | """Find cycles in the dependency tree. |
| 619 | |
| 620 | Returns: |
| 621 | A dict mapping cyclic packages to a dict of the deps that cause |
| 622 | cycles. For each dep that causes cycles, it returns an example |
| 623 | traversal of the graph that shows the cycle. |
| 624 | """ |
| 625 | |
| 626 | def FindCyclesAtNode(pkg, cycles, unresolved, resolved): |
| 627 | """Find cycles in cyclic dependencies starting at specified package. |
| 628 | |
| 629 | Args: |
| 630 | pkg: Package identifier. |
| 631 | cycles: A dict mapping cyclic packages to a dict of the deps that |
| 632 | cause cycles. For each dep that causes cycles, it returns an |
| 633 | example traversal of the graph that shows the cycle. |
| 634 | unresolved: Nodes that have been visited but are not fully processed. |
| 635 | resolved: Nodes that have been visited and are fully processed. |
| 636 | """ |
| 637 | pkg_cycles = cycles.get(pkg) |
| 638 | if pkg in resolved and not pkg_cycles: |
| 639 | # If we already looked at this package, and found no cyclic |
| 640 | # dependencies, we can stop now. |
| 641 | return |
| 642 | unresolved.append(pkg) |
| 643 | for dep in deps_map[pkg]["needs"]: |
| 644 | if dep in unresolved: |
| 645 | idx = unresolved.index(dep) |
| 646 | mycycle = unresolved[idx:] + [dep] |
| 647 | for i in range(len(mycycle) - 1): |
| 648 | pkg1, pkg2 = mycycle[i], mycycle[i+1] |
| 649 | cycles.setdefault(pkg1, {}).setdefault(pkg2, mycycle) |
| 650 | elif not pkg_cycles or dep not in pkg_cycles: |
| 651 | # Looks like we haven't seen this edge before. |
| 652 | FindCyclesAtNode(dep, cycles, unresolved, resolved) |
| 653 | unresolved.pop() |
| 654 | resolved.add(pkg) |
| 655 | |
| 656 | cycles, unresolved, resolved = {}, [], set() |
| 657 | for pkg in deps_map: |
| 658 | FindCyclesAtNode(pkg, cycles, unresolved, resolved) |
| 659 | return cycles |
| 660 | |
David James | 386ccd1 | 2011-05-04 20:17:42 -0700 | [diff] [blame] | 661 | def RemoveUnusedPackages(): |
David James | fcb70ef | 2011-02-02 16:02:30 -0800 | [diff] [blame] | 662 | """Remove installed packages, propagating dependencies.""" |
David James | fcb70ef | 2011-02-02 16:02:30 -0800 | [diff] [blame] | 663 | # Schedule packages that aren't on the install list for removal |
| 664 | rm_pkgs = set(deps_map.keys()) - set(deps_info.keys()) |
| 665 | |
David James | fcb70ef | 2011-02-02 16:02:30 -0800 | [diff] [blame] | 666 | # Remove the packages we don't want, simplifying the graph and making |
| 667 | # it easier for us to crack cycles. |
| 668 | for pkg in sorted(rm_pkgs): |
| 669 | this_pkg = deps_map[pkg] |
| 670 | needs = this_pkg["needs"] |
| 671 | provides = this_pkg["provides"] |
| 672 | for dep in needs: |
| 673 | dep_provides = deps_map[dep]["provides"] |
| 674 | dep_provides.update(provides) |
| 675 | dep_provides.discard(pkg) |
| 676 | dep_provides.discard(dep) |
| 677 | for target in provides: |
| 678 | target_needs = deps_map[target]["needs"] |
| 679 | target_needs.update(needs) |
| 680 | target_needs.pop(pkg, None) |
| 681 | target_needs.pop(target, None) |
| 682 | del deps_map[pkg] |
| 683 | |
| 684 | def PrintCycleBreak(basedep, dep, mycycle): |
| 685 | """Print details about a cycle that we are planning on breaking. |
| 686 | |
| 687 | We are breaking a cycle where dep needs basedep. mycycle is an |
| 688 | example cycle which contains dep -> basedep.""" |
| 689 | |
| 690 | # If it's an optional dependency, there's no need to spam the user with |
| 691 | # warning messages. |
| 692 | needs = deps_map[dep]["needs"] |
| 693 | depinfo = needs.get(basedep, "deleted") |
| 694 | if depinfo == "optional": |
| 695 | return |
| 696 | |
| 697 | # Notify the user that we're breaking a cycle. |
| 698 | print "Breaking %s -> %s (%s)" % (dep, basedep, depinfo) |
| 699 | |
| 700 | # Show cycle. |
| 701 | for i in range(len(mycycle) - 1): |
| 702 | pkg1, pkg2 = mycycle[i], mycycle[i+1] |
| 703 | needs = deps_map[pkg1]["needs"] |
| 704 | depinfo = needs.get(pkg2, "deleted") |
| 705 | if pkg1 == dep and pkg2 == basedep: |
| 706 | depinfo = depinfo + ", deleting" |
| 707 | print " %s -> %s (%s)" % (pkg1, pkg2, depinfo) |
| 708 | |
| 709 | def SanitizeTree(): |
| 710 | """Remove circular dependencies. |
| 711 | |
| 712 | We prune all dependencies involved in cycles that go against the emerge |
| 713 | ordering. This has a nice property: we're guaranteed to merge |
| 714 | dependencies in the same order that portage does. |
| 715 | |
| 716 | Because we don't treat any dependencies as "soft" unless they're killed |
| 717 | by a cycle, we pay attention to a larger number of dependencies when |
| 718 | merging. This hurts performance a bit, but helps reliability. |
| 719 | """ |
| 720 | start = time.time() |
| 721 | cycles = FindCycles() |
| 722 | while cycles: |
| 723 | for dep, mycycles in cycles.iteritems(): |
| 724 | for basedep, mycycle in mycycles.iteritems(): |
| 725 | if deps_info[basedep]["idx"] >= deps_info[dep]["idx"]: |
| 726 | PrintCycleBreak(basedep, dep, mycycle) |
| 727 | del deps_map[dep]["needs"][basedep] |
| 728 | deps_map[basedep]["provides"].remove(dep) |
| 729 | cycles = FindCycles() |
| 730 | seconds = time.time() - start |
| 731 | if "--quiet" not in emerge.opts and seconds >= 0.1: |
| 732 | print "Tree sanitized in %dm%.1fs" % (seconds / 60, seconds % 60) |
| 733 | |
David James | a22906f | 2011-05-04 19:53:26 -0700 | [diff] [blame] | 734 | ReverseTree(deps_tree) |
David James | a22906f | 2011-05-04 19:53:26 -0700 | [diff] [blame] | 735 | |
David James | 386ccd1 | 2011-05-04 20:17:42 -0700 | [diff] [blame] | 736 | # We need to remove unused packages so that we can use the dependency |
| 737 | # ordering of the install process to show us what cycles to crack. |
| 738 | RemoveUnusedPackages() |
David James | fcb70ef | 2011-02-02 16:02:30 -0800 | [diff] [blame] | 739 | SanitizeTree() |
David James | fcb70ef | 2011-02-02 16:02:30 -0800 | [diff] [blame] | 740 | return deps_map |
| 741 | |
| 742 | def PrintInstallPlan(self, deps_map): |
| 743 | """Print an emerge-style install plan. |
| 744 | |
| 745 | The install plan lists what packages we're installing, in order. |
| 746 | It's useful for understanding what parallel_emerge is doing. |
| 747 | |
| 748 | Args: |
| 749 | deps_map: The dependency graph. |
| 750 | """ |
| 751 | |
| 752 | def InstallPlanAtNode(target, deps_map): |
| 753 | nodes = [] |
| 754 | nodes.append(target) |
| 755 | for dep in deps_map[target]["provides"]: |
| 756 | del deps_map[dep]["needs"][target] |
| 757 | if not deps_map[dep]["needs"]: |
| 758 | nodes.extend(InstallPlanAtNode(dep, deps_map)) |
| 759 | return nodes |
| 760 | |
| 761 | deps_map = copy.deepcopy(deps_map) |
| 762 | install_plan = [] |
| 763 | plan = set() |
| 764 | for target, info in deps_map.iteritems(): |
| 765 | if not info["needs"] and target not in plan: |
| 766 | for item in InstallPlanAtNode(target, deps_map): |
| 767 | plan.add(item) |
| 768 | install_plan.append(self.package_db[item]) |
| 769 | |
| 770 | for pkg in plan: |
| 771 | del deps_map[pkg] |
| 772 | |
| 773 | if deps_map: |
| 774 | print "Cyclic dependencies:", " ".join(deps_map) |
| 775 | PrintDepsMap(deps_map) |
| 776 | sys.exit(1) |
| 777 | |
| 778 | self.emerge.depgraph.display(install_plan) |
| 779 | |
| 780 | |
| 781 | def PrintDepsMap(deps_map): |
| 782 | """Print dependency graph, for each package list it's prerequisites.""" |
| 783 | for i in sorted(deps_map): |
| 784 | print "%s: (%s) needs" % (i, deps_map[i]["action"]) |
| 785 | needs = deps_map[i]["needs"] |
| 786 | for j in sorted(needs): |
| 787 | print " %s" % (j) |
| 788 | if not needs: |
| 789 | print " no dependencies" |
| 790 | |
| 791 | |
| 792 | class EmergeJobState(object): |
| 793 | __slots__ = ["done", "filename", "last_notify_timestamp", "last_output_seek", |
| 794 | "last_output_timestamp", "pkgname", "retcode", "start_timestamp", |
| 795 | "target"] |
| 796 | |
| 797 | def __init__(self, target, pkgname, done, filename, start_timestamp, |
| 798 | retcode=None): |
| 799 | |
| 800 | # The full name of the target we're building (e.g. |
| 801 | # chromeos-base/chromeos-0.0.1-r60) |
| 802 | self.target = target |
| 803 | |
| 804 | # The short name of the target we're building (e.g. chromeos-0.0.1-r60) |
| 805 | self.pkgname = pkgname |
| 806 | |
| 807 | # Whether the job is done. (True if the job is done; false otherwise.) |
| 808 | self.done = done |
| 809 | |
| 810 | # The filename where output is currently stored. |
| 811 | self.filename = filename |
| 812 | |
| 813 | # The timestamp of the last time we printed the name of the log file. We |
| 814 | # print this at the beginning of the job, so this starts at |
| 815 | # start_timestamp. |
| 816 | self.last_notify_timestamp = start_timestamp |
| 817 | |
| 818 | # The location (in bytes) of the end of the last complete line we printed. |
| 819 | # This starts off at zero. We use this to jump to the right place when we |
| 820 | # print output from the same ebuild multiple times. |
| 821 | self.last_output_seek = 0 |
| 822 | |
| 823 | # The timestamp of the last time we printed output. Since we haven't |
| 824 | # printed output yet, this starts at zero. |
| 825 | self.last_output_timestamp = 0 |
| 826 | |
| 827 | # The return code of our job, if the job is actually finished. |
| 828 | self.retcode = retcode |
| 829 | |
| 830 | # The timestamp when our job started. |
| 831 | self.start_timestamp = start_timestamp |
| 832 | |
| 833 | |
David James | 7358d03 | 2011-05-19 10:40:03 -0700 | [diff] [blame] | 834 | def KillHandler(signum, frame): |
| 835 | # Kill self and all subprocesses. |
| 836 | os.killpg(0, signal.SIGKILL) |
| 837 | |
David James | fcb70ef | 2011-02-02 16:02:30 -0800 | [diff] [blame] | 838 | def SetupWorkerSignals(): |
| 839 | def ExitHandler(signum, frame): |
David James | 7358d03 | 2011-05-19 10:40:03 -0700 | [diff] [blame] | 840 | # Set KILLED flag. |
| 841 | KILLED.set() |
David James | 13cead4 | 2011-05-18 16:22:01 -0700 | [diff] [blame] | 842 | |
David James | 7358d03 | 2011-05-19 10:40:03 -0700 | [diff] [blame] | 843 | # Remove our signal handlers so we don't get called recursively. |
| 844 | signal.signal(signal.SIGINT, KillHandler) |
| 845 | signal.signal(signal.SIGTERM, KillHandler) |
David James | fcb70ef | 2011-02-02 16:02:30 -0800 | [diff] [blame] | 846 | |
| 847 | # Ensure that we exit quietly and cleanly, if possible, when we receive |
| 848 | # SIGTERM or SIGINT signals. By default, when the user hits CTRL-C, all |
| 849 | # of the child processes will print details about KeyboardInterrupt |
| 850 | # exceptions, which isn't very helpful. |
| 851 | signal.signal(signal.SIGINT, ExitHandler) |
| 852 | signal.signal(signal.SIGTERM, ExitHandler) |
| 853 | |
| 854 | |
| 855 | def EmergeWorker(task_queue, job_queue, emerge, package_db): |
| 856 | """This worker emerges any packages given to it on the task_queue. |
| 857 | |
| 858 | Args: |
| 859 | task_queue: The queue of tasks for this worker to do. |
| 860 | job_queue: The queue of results from the worker. |
| 861 | emerge: An EmergeData() object. |
| 862 | package_db: A dict, mapping package ids to portage Package objects. |
| 863 | |
| 864 | It expects package identifiers to be passed to it via task_queue. When |
| 865 | a task is started, it pushes the (target, filename) to the started_queue. |
| 866 | The output is stored in filename. When a merge starts or finishes, we push |
| 867 | EmergeJobState objects to the job_queue. |
| 868 | """ |
| 869 | |
| 870 | SetupWorkerSignals() |
| 871 | settings, trees, mtimedb = emerge.settings, emerge.trees, emerge.mtimedb |
David James | deebd69 | 2011-05-09 17:02:52 -0700 | [diff] [blame] | 872 | |
| 873 | # Disable flushing of caches to save on I/O. |
| 874 | if 0 <= vercmp(portage.VERSION, "2.1.9.48"): |
| 875 | root = emerge.settings["ROOT"] |
| 876 | vardb = emerge.trees[root]["vartree"].dbapi |
| 877 | vardb._flush_cache_enabled = False |
| 878 | |
David James | fcb70ef | 2011-02-02 16:02:30 -0800 | [diff] [blame] | 879 | opts, spinner = emerge.opts, emerge.spinner |
| 880 | opts["--nodeps"] = True |
David James | 386ccd1 | 2011-05-04 20:17:42 -0700 | [diff] [blame] | 881 | # When Portage launches new processes, it goes on a rampage and closes all |
| 882 | # open file descriptors. Ask Portage not to do that, as it breaks us. |
| 883 | portage.process.get_open_fds = lambda: [] |
David James | fcb70ef | 2011-02-02 16:02:30 -0800 | [diff] [blame] | 884 | while True: |
| 885 | # Wait for a new item to show up on the queue. This is a blocking wait, |
| 886 | # so if there's nothing to do, we just sit here. |
| 887 | target = task_queue.get() |
| 888 | if not target: |
| 889 | # If target is None, this means that the main thread wants us to quit. |
| 890 | # The other workers need to exit too, so we'll push the message back on |
| 891 | # to the queue so they'll get it too. |
| 892 | task_queue.put(target) |
| 893 | return |
David James | 7358d03 | 2011-05-19 10:40:03 -0700 | [diff] [blame] | 894 | if KILLED.is_set(): |
| 895 | return |
| 896 | |
David James | fcb70ef | 2011-02-02 16:02:30 -0800 | [diff] [blame] | 897 | db_pkg = package_db[target] |
| 898 | db_pkg.root_config = emerge.root_config |
| 899 | install_list = [db_pkg] |
| 900 | pkgname = db_pkg.pf |
| 901 | output = tempfile.NamedTemporaryFile(prefix=pkgname + "-", delete=False) |
| 902 | start_timestamp = time.time() |
| 903 | job = EmergeJobState(target, pkgname, False, output.name, start_timestamp) |
| 904 | job_queue.put(job) |
| 905 | if "--pretend" in opts: |
| 906 | retcode = 0 |
| 907 | else: |
| 908 | save_stdout = sys.stdout |
| 909 | save_stderr = sys.stderr |
| 910 | try: |
| 911 | sys.stdout = output |
| 912 | sys.stderr = output |
David James | 386ccd1 | 2011-05-04 20:17:42 -0700 | [diff] [blame] | 913 | emerge.scheduler_graph.mergelist = install_list |
| 914 | scheduler = Scheduler(settings, trees, mtimedb, opts, spinner, |
| 915 | favorites=[], graph_config=emerge.scheduler_graph) |
David James | fcb70ef | 2011-02-02 16:02:30 -0800 | [diff] [blame] | 916 | retcode = scheduler.merge() |
| 917 | except Exception: |
| 918 | traceback.print_exc(file=output) |
| 919 | retcode = 1 |
| 920 | finally: |
| 921 | sys.stdout = save_stdout |
| 922 | sys.stderr = save_stderr |
| 923 | output.close() |
| 924 | if retcode is None: |
| 925 | retcode = 0 |
| 926 | |
David James | 7358d03 | 2011-05-19 10:40:03 -0700 | [diff] [blame] | 927 | if KILLED.is_set(): |
| 928 | return |
| 929 | |
David James | fcb70ef | 2011-02-02 16:02:30 -0800 | [diff] [blame] | 930 | job = EmergeJobState(target, pkgname, True, output.name, start_timestamp, |
| 931 | retcode) |
| 932 | job_queue.put(job) |
| 933 | |
| 934 | |
| 935 | class LinePrinter(object): |
| 936 | """Helper object to print a single line.""" |
| 937 | |
| 938 | def __init__(self, line): |
| 939 | self.line = line |
| 940 | |
| 941 | def Print(self, seek_locations): |
| 942 | print self.line |
| 943 | |
| 944 | |
| 945 | class JobPrinter(object): |
| 946 | """Helper object to print output of a job.""" |
| 947 | |
| 948 | def __init__(self, job, unlink=False): |
| 949 | """Print output of job. |
| 950 | |
| 951 | If unlink is True, unlink the job output file when done.""" |
| 952 | self.current_time = time.time() |
| 953 | self.job = job |
| 954 | self.unlink = unlink |
| 955 | |
| 956 | def Print(self, seek_locations): |
| 957 | |
| 958 | job = self.job |
| 959 | |
| 960 | # Calculate how long the job has been running. |
| 961 | seconds = self.current_time - job.start_timestamp |
| 962 | |
| 963 | # Note that we've printed out the job so far. |
| 964 | job.last_output_timestamp = self.current_time |
| 965 | |
| 966 | # Note that we're starting the job |
| 967 | info = "job %s (%dm%.1fs)" % (job.pkgname, seconds / 60, seconds % 60) |
| 968 | last_output_seek = seek_locations.get(job.filename, 0) |
| 969 | if last_output_seek: |
| 970 | print "=== Continue output for %s ===" % info |
| 971 | else: |
| 972 | print "=== Start output for %s ===" % info |
| 973 | |
| 974 | # Print actual output from job |
| 975 | f = codecs.open(job.filename, encoding='utf-8', errors='replace') |
| 976 | f.seek(last_output_seek) |
| 977 | prefix = job.pkgname + ":" |
| 978 | for line in f: |
| 979 | |
| 980 | # Save off our position in the file |
| 981 | if line and line[-1] == "\n": |
| 982 | last_output_seek = f.tell() |
| 983 | line = line[:-1] |
| 984 | |
| 985 | # Print our line |
| 986 | print prefix, line.encode('utf-8', 'replace') |
| 987 | f.close() |
| 988 | |
| 989 | # Save our last spot in the file so that we don't print out the same |
| 990 | # location twice. |
| 991 | seek_locations[job.filename] = last_output_seek |
| 992 | |
| 993 | # Note end of output section |
| 994 | if job.done: |
| 995 | print "=== Complete: %s ===" % info |
| 996 | else: |
| 997 | print "=== Still running: %s ===" % info |
| 998 | |
| 999 | if self.unlink: |
| 1000 | os.unlink(job.filename) |
| 1001 | |
| 1002 | |
| 1003 | def PrintWorker(queue): |
| 1004 | """A worker that prints stuff to the screen as requested.""" |
| 1005 | |
| 1006 | def ExitHandler(signum, frame): |
David James | 7358d03 | 2011-05-19 10:40:03 -0700 | [diff] [blame] | 1007 | # Set KILLED flag. |
| 1008 | KILLED.set() |
| 1009 | |
David James | fcb70ef | 2011-02-02 16:02:30 -0800 | [diff] [blame] | 1010 | # Switch to default signal handlers so that we'll die after two signals. |
David James | 7358d03 | 2011-05-19 10:40:03 -0700 | [diff] [blame] | 1011 | signal.signal(signal.SIGINT, KillHandler) |
| 1012 | signal.signal(signal.SIGTERM, KillHandler) |
David James | fcb70ef | 2011-02-02 16:02:30 -0800 | [diff] [blame] | 1013 | |
| 1014 | # Don't exit on the first SIGINT / SIGTERM, because the parent worker will |
| 1015 | # handle it and tell us when we need to exit. |
| 1016 | signal.signal(signal.SIGINT, ExitHandler) |
| 1017 | signal.signal(signal.SIGTERM, ExitHandler) |
| 1018 | |
| 1019 | # seek_locations is a map indicating the position we are at in each file. |
| 1020 | # It starts off empty, but is set by the various Print jobs as we go along |
| 1021 | # to indicate where we left off in each file. |
| 1022 | seek_locations = {} |
| 1023 | while True: |
| 1024 | try: |
| 1025 | job = queue.get() |
| 1026 | if job: |
| 1027 | job.Print(seek_locations) |
| 1028 | else: |
| 1029 | break |
| 1030 | except IOError as ex: |
| 1031 | if ex.errno == errno.EINTR: |
| 1032 | # Looks like we received a signal. Keep printing. |
| 1033 | continue |
| 1034 | raise |
| 1035 | |
David James | fcb70ef | 2011-02-02 16:02:30 -0800 | [diff] [blame] | 1036 | class EmergeQueue(object): |
| 1037 | """Class to schedule emerge jobs according to a dependency graph.""" |
| 1038 | |
| 1039 | def __init__(self, deps_map, emerge, package_db, show_output): |
| 1040 | # Store the dependency graph. |
| 1041 | self._deps_map = deps_map |
| 1042 | # Initialize the running queue to empty |
| 1043 | self._jobs = {} |
| 1044 | # List of total package installs represented in deps_map. |
| 1045 | install_jobs = [x for x in deps_map if deps_map[x]["action"] == "merge"] |
| 1046 | self._total_jobs = len(install_jobs) |
| 1047 | self._show_output = show_output |
| 1048 | |
| 1049 | if "--pretend" in emerge.opts: |
| 1050 | print "Skipping merge because of --pretend mode." |
| 1051 | sys.exit(0) |
| 1052 | |
David James | 7358d03 | 2011-05-19 10:40:03 -0700 | [diff] [blame] | 1053 | # Set a process group so we can easily terminate all children. |
| 1054 | os.setsid() |
| 1055 | |
David James | fcb70ef | 2011-02-02 16:02:30 -0800 | [diff] [blame] | 1056 | # Setup scheduler graph object. This is used by the child processes |
| 1057 | # to help schedule jobs. |
| 1058 | emerge.scheduler_graph = emerge.depgraph.schedulerGraph() |
| 1059 | |
| 1060 | # Calculate how many jobs we can run in parallel. We don't want to pass |
| 1061 | # the --jobs flag over to emerge itself, because that'll tell emerge to |
| 1062 | # hide its output, and said output is quite useful for debugging hung |
| 1063 | # jobs. |
| 1064 | procs = min(self._total_jobs, |
| 1065 | emerge.opts.pop("--jobs", multiprocessing.cpu_count())) |
| 1066 | self._emerge_queue = multiprocessing.Queue() |
| 1067 | self._job_queue = multiprocessing.Queue() |
| 1068 | self._print_queue = multiprocessing.Queue() |
| 1069 | args = (self._emerge_queue, self._job_queue, emerge, package_db) |
| 1070 | self._pool = multiprocessing.Pool(procs, EmergeWorker, args) |
| 1071 | self._print_worker = multiprocessing.Process(target=PrintWorker, |
| 1072 | args=[self._print_queue]) |
| 1073 | self._print_worker.start() |
| 1074 | |
| 1075 | # Initialize the failed queue to empty. |
| 1076 | self._retry_queue = [] |
| 1077 | self._failed = set() |
| 1078 | |
| 1079 | # Print an update before we launch the merges. |
| 1080 | self._Status() |
| 1081 | |
| 1082 | # Setup an exit handler so that we print nice messages if we are |
| 1083 | # terminated. |
| 1084 | self._SetupExitHandler() |
| 1085 | |
| 1086 | # Schedule our jobs. |
| 1087 | for target, info in deps_map.items(): |
| 1088 | if not info["needs"]: |
| 1089 | self._Schedule(target) |
| 1090 | |
| 1091 | def _SetupExitHandler(self): |
| 1092 | |
| 1093 | def ExitHandler(signum, frame): |
David James | 7358d03 | 2011-05-19 10:40:03 -0700 | [diff] [blame] | 1094 | # Set KILLED flag. |
| 1095 | KILLED.set() |
David James | fcb70ef | 2011-02-02 16:02:30 -0800 | [diff] [blame] | 1096 | |
| 1097 | # Kill our signal handlers so we don't get called recursively |
David James | 7358d03 | 2011-05-19 10:40:03 -0700 | [diff] [blame] | 1098 | signal.signal(signal.SIGINT, KillHandler) |
| 1099 | signal.signal(signal.SIGTERM, KillHandler) |
David James | fcb70ef | 2011-02-02 16:02:30 -0800 | [diff] [blame] | 1100 | |
| 1101 | # Print our current job status |
| 1102 | for target, job in self._jobs.iteritems(): |
| 1103 | if job: |
| 1104 | self._print_queue.put(JobPrinter(job, unlink=True)) |
| 1105 | |
| 1106 | # Notify the user that we are exiting |
| 1107 | self._Print("Exiting on signal %s" % signum) |
David James | 7358d03 | 2011-05-19 10:40:03 -0700 | [diff] [blame] | 1108 | self._print_queue.put(None) |
| 1109 | self._print_worker.join() |
David James | fcb70ef | 2011-02-02 16:02:30 -0800 | [diff] [blame] | 1110 | |
| 1111 | # Kill child threads, then exit. |
David James | 7358d03 | 2011-05-19 10:40:03 -0700 | [diff] [blame] | 1112 | os.killpg(0, signal.SIGKILL) |
David James | fcb70ef | 2011-02-02 16:02:30 -0800 | [diff] [blame] | 1113 | sys.exit(1) |
| 1114 | |
| 1115 | # Print out job status when we are killed |
| 1116 | signal.signal(signal.SIGINT, ExitHandler) |
| 1117 | signal.signal(signal.SIGTERM, ExitHandler) |
| 1118 | |
| 1119 | def _Schedule(self, target): |
| 1120 | # We maintain a tree of all deps, if this doesn't need |
| 1121 | # to be installed just free up it's children and continue. |
| 1122 | # It is possible to reinstall deps of deps, without reinstalling |
| 1123 | # first level deps, like so: |
| 1124 | # chromeos (merge) -> eselect (nomerge) -> python (merge) |
David James | 386ccd1 | 2011-05-04 20:17:42 -0700 | [diff] [blame] | 1125 | if target not in self._deps_map: |
| 1126 | pass |
| 1127 | elif self._deps_map[target]["action"] == "nomerge": |
David James | fcb70ef | 2011-02-02 16:02:30 -0800 | [diff] [blame] | 1128 | self._Finish(target) |
David James | d20a6d9 | 2011-04-26 16:11:59 -0700 | [diff] [blame] | 1129 | elif target not in self._jobs: |
David James | fcb70ef | 2011-02-02 16:02:30 -0800 | [diff] [blame] | 1130 | # Kick off the build if it's marked to be built. |
| 1131 | self._jobs[target] = None |
| 1132 | self._emerge_queue.put(target) |
| 1133 | |
| 1134 | def _LoadAvg(self): |
| 1135 | loads = open("/proc/loadavg", "r").readline().split()[:3] |
| 1136 | return " ".join(loads) |
| 1137 | |
| 1138 | def _Print(self, line): |
| 1139 | """Print a single line.""" |
| 1140 | self._print_queue.put(LinePrinter(line)) |
| 1141 | |
| 1142 | def _Status(self): |
| 1143 | """Print status.""" |
| 1144 | current_time = time.time() |
| 1145 | no_output = True |
| 1146 | |
| 1147 | # Print interim output every minute if --show-output is used. Otherwise, |
| 1148 | # print notifications about running packages every 2 minutes, and print |
| 1149 | # full output for jobs that have been running for 60 minutes or more. |
| 1150 | if self._show_output: |
| 1151 | interval = 60 |
| 1152 | notify_interval = 0 |
| 1153 | else: |
| 1154 | interval = 60 * 60 |
| 1155 | notify_interval = 60 * 2 |
| 1156 | for target, job in self._jobs.iteritems(): |
| 1157 | if job: |
| 1158 | last_timestamp = max(job.start_timestamp, job.last_output_timestamp) |
| 1159 | if last_timestamp + interval < current_time: |
| 1160 | self._print_queue.put(JobPrinter(job)) |
| 1161 | job.last_output_timestamp = current_time |
| 1162 | no_output = False |
| 1163 | elif (notify_interval and |
| 1164 | job.last_notify_timestamp + notify_interval < current_time): |
| 1165 | job_seconds = current_time - job.start_timestamp |
| 1166 | args = (job.pkgname, job_seconds / 60, job_seconds % 60, job.filename) |
| 1167 | info = "Still building %s (%dm%.1fs). Logs in %s" % args |
| 1168 | job.last_notify_timestamp = current_time |
| 1169 | self._Print(info) |
| 1170 | no_output = False |
| 1171 | |
| 1172 | # If we haven't printed any messages yet, print a general status message |
| 1173 | # here. |
| 1174 | if no_output: |
| 1175 | seconds = current_time - GLOBAL_START |
| 1176 | line = ("Pending %s, Ready %s, Running %s, Retrying %s, Total %s " |
| 1177 | "[Time %dm%.1fs Load %s]") |
| 1178 | qsize = self._emerge_queue.qsize() |
| 1179 | self._Print(line % (len(self._deps_map), qsize, len(self._jobs) - qsize, |
| 1180 | len(self._retry_queue), self._total_jobs, |
| 1181 | seconds / 60, seconds % 60, self._LoadAvg())) |
| 1182 | |
| 1183 | def _Finish(self, target): |
| 1184 | """Mark a target as completed and unblock dependecies.""" |
| 1185 | for dep in self._deps_map[target]["provides"]: |
| 1186 | del self._deps_map[dep]["needs"][target] |
| 1187 | if not self._deps_map[dep]["needs"]: |
| 1188 | self._Schedule(dep) |
| 1189 | self._deps_map.pop(target) |
| 1190 | |
| 1191 | def _Retry(self): |
| 1192 | if self._retry_queue: |
| 1193 | target = self._retry_queue.pop(0) |
| 1194 | self._Schedule(target) |
| 1195 | self._Print("Retrying emerge of %s." % target) |
| 1196 | |
| 1197 | def _Exit(self): |
| 1198 | # Tell emerge workers to exit. They all exit when 'None' is pushed |
| 1199 | # to the queue. |
| 1200 | self._emerge_queue.put(None) |
| 1201 | self._pool.close() |
| 1202 | self._pool.join() |
| 1203 | |
| 1204 | # Now that our workers are finished, we can kill the print queue. |
| 1205 | self._print_queue.put(None) |
| 1206 | self._print_worker.join() |
| 1207 | |
| 1208 | def Run(self): |
| 1209 | """Run through the scheduled ebuilds. |
| 1210 | |
| 1211 | Keep running so long as we have uninstalled packages in the |
| 1212 | dependency graph to merge. |
| 1213 | """ |
| 1214 | while self._deps_map: |
| 1215 | # Check here that we are actually waiting for something. |
| 1216 | if (self._emerge_queue.empty() and |
| 1217 | self._job_queue.empty() and |
| 1218 | not self._jobs and |
| 1219 | self._deps_map): |
| 1220 | # If we have failed on a package, retry it now. |
| 1221 | if self._retry_queue: |
| 1222 | self._Retry() |
| 1223 | else: |
| 1224 | # Tell child threads to exit. |
| 1225 | self._Exit() |
| 1226 | |
| 1227 | # The dependency map is helpful for debugging failures. |
| 1228 | PrintDepsMap(self._deps_map) |
| 1229 | |
| 1230 | # Tell the user why we're exiting. |
| 1231 | if self._failed: |
| 1232 | print "Packages failed: %s" % " ,".join(self._failed) |
| 1233 | else: |
| 1234 | print "Deadlock! Circular dependencies!" |
| 1235 | sys.exit(1) |
| 1236 | |
| 1237 | try: |
| 1238 | job = self._job_queue.get(timeout=5) |
| 1239 | except Queue.Empty: |
| 1240 | # Print an update. |
| 1241 | self._Status() |
| 1242 | continue |
| 1243 | |
| 1244 | target = job.target |
| 1245 | |
| 1246 | if not job.done: |
| 1247 | self._jobs[target] = job |
| 1248 | self._Print("Started %s (logged in %s)" % (target, job.filename)) |
| 1249 | continue |
| 1250 | |
| 1251 | # Print output of job |
| 1252 | if self._show_output or job.retcode != 0: |
| 1253 | self._print_queue.put(JobPrinter(job, unlink=True)) |
| 1254 | else: |
| 1255 | os.unlink(job.filename) |
| 1256 | del self._jobs[target] |
| 1257 | |
| 1258 | seconds = time.time() - job.start_timestamp |
| 1259 | details = "%s (in %dm%.1fs)" % (target, seconds / 60, seconds % 60) |
| 1260 | |
| 1261 | # Complain if necessary. |
| 1262 | if job.retcode != 0: |
| 1263 | # Handle job failure. |
| 1264 | if target in self._failed: |
| 1265 | # If this job has failed previously, give up. |
| 1266 | self._Print("Failed %s. Your build has failed." % details) |
| 1267 | else: |
| 1268 | # Queue up this build to try again after a long while. |
| 1269 | self._retry_queue.append(target) |
| 1270 | self._failed.add(target) |
| 1271 | self._Print("Failed %s, retrying later." % details) |
| 1272 | else: |
| 1273 | if target in self._failed and self._retry_queue: |
| 1274 | # If we have successfully retried a failed package, and there |
| 1275 | # are more failed packages, try the next one. We will only have |
| 1276 | # one retrying package actively running at a time. |
| 1277 | self._Retry() |
| 1278 | |
| 1279 | self._Print("Completed %s" % details) |
| 1280 | # Mark as completed and unblock waiting ebuilds. |
| 1281 | self._Finish(target) |
| 1282 | |
| 1283 | # Print an update. |
| 1284 | self._Status() |
| 1285 | |
| 1286 | # Tell child threads to exit. |
| 1287 | self._Print("Merge complete") |
| 1288 | self._Exit() |
| 1289 | |
| 1290 | |
| 1291 | def main(): |
| 1292 | |
David James | 5743753 | 2011-05-06 15:51:21 -0700 | [diff] [blame] | 1293 | parallel_emerge_args = sys.argv[:] |
David James | fcb70ef | 2011-02-02 16:02:30 -0800 | [diff] [blame] | 1294 | deps = DepGraphGenerator() |
David James | 5743753 | 2011-05-06 15:51:21 -0700 | [diff] [blame] | 1295 | deps.Initialize(parallel_emerge_args[1:]) |
David James | fcb70ef | 2011-02-02 16:02:30 -0800 | [diff] [blame] | 1296 | emerge = deps.emerge |
| 1297 | |
| 1298 | if emerge.action is not None: |
| 1299 | sys.argv = deps.ParseParallelEmergeArgs(sys.argv) |
| 1300 | sys.exit(emerge_main()) |
| 1301 | elif not emerge.cmdline_packages: |
| 1302 | Usage() |
| 1303 | sys.exit(1) |
| 1304 | |
| 1305 | # Unless we're in pretend mode, there's not much point running without |
| 1306 | # root access. We need to be able to install packages. |
| 1307 | # |
| 1308 | # NOTE: Even if you're running --pretend, it's a good idea to run |
| 1309 | # parallel_emerge with root access so that portage can write to the |
| 1310 | # dependency cache. This is important for performance. |
| 1311 | if "--pretend" not in emerge.opts and portage.secpass < 2: |
| 1312 | print "parallel_emerge: superuser access is required." |
| 1313 | sys.exit(1) |
| 1314 | |
| 1315 | if "--quiet" not in emerge.opts: |
| 1316 | cmdline_packages = " ".join(emerge.cmdline_packages) |
David James | fcb70ef | 2011-02-02 16:02:30 -0800 | [diff] [blame] | 1317 | print "Starting fast-emerge." |
| 1318 | print " Building package %s on %s" % (cmdline_packages, |
| 1319 | deps.board or "root") |
David James | fcb70ef | 2011-02-02 16:02:30 -0800 | [diff] [blame] | 1320 | |
David James | 386ccd1 | 2011-05-04 20:17:42 -0700 | [diff] [blame] | 1321 | deps_tree, deps_info = deps.GenDependencyTree() |
David James | fcb70ef | 2011-02-02 16:02:30 -0800 | [diff] [blame] | 1322 | |
| 1323 | # You want me to be verbose? I'll give you two trees! Twice as much value. |
| 1324 | if "--tree" in emerge.opts and "--verbose" in emerge.opts: |
| 1325 | deps.PrintTree(deps_tree) |
| 1326 | |
David James | 386ccd1 | 2011-05-04 20:17:42 -0700 | [diff] [blame] | 1327 | deps_graph = deps.GenDependencyGraph(deps_tree, deps_info) |
David James | fcb70ef | 2011-02-02 16:02:30 -0800 | [diff] [blame] | 1328 | |
| 1329 | # OK, time to print out our progress so far. |
| 1330 | deps.PrintInstallPlan(deps_graph) |
| 1331 | if "--tree" in emerge.opts: |
| 1332 | PrintDepsMap(deps_graph) |
| 1333 | |
| 1334 | # Are we upgrading portage? If so, and there are more packages to merge, |
| 1335 | # schedule a restart of parallel_emerge to merge the rest. This ensures that |
| 1336 | # we pick up all updates to portage settings before merging any more |
| 1337 | # packages. |
| 1338 | portage_upgrade = False |
| 1339 | root = emerge.settings["ROOT"] |
| 1340 | final_db = emerge.depgraph._dynamic_config.mydbapi[root] |
| 1341 | if root == "/": |
| 1342 | for db_pkg in final_db.match_pkgs("sys-apps/portage"): |
| 1343 | portage_pkg = deps_graph.get(db_pkg.cpv) |
| 1344 | if portage_pkg and len(deps_graph) > 1: |
| 1345 | portage_pkg["needs"].clear() |
| 1346 | portage_pkg["provides"].clear() |
| 1347 | deps_graph = { str(db_pkg.cpv): portage_pkg } |
| 1348 | portage_upgrade = True |
| 1349 | if "--quiet" not in emerge.opts: |
| 1350 | print "Upgrading portage first, then restarting..." |
| 1351 | |
| 1352 | # Run the queued emerges. |
| 1353 | scheduler = EmergeQueue(deps_graph, emerge, deps.package_db, deps.show_output) |
| 1354 | scheduler.Run() |
| 1355 | |
| 1356 | # Update world. |
| 1357 | if ("--oneshot" not in emerge.opts and |
| 1358 | "--pretend" not in emerge.opts): |
| 1359 | world_set = emerge.root_config.sets["selected"] |
| 1360 | new_world_pkgs = [] |
| 1361 | for pkg in emerge.cmdline_packages: |
| 1362 | for db_pkg in final_db.match_pkgs(pkg): |
| 1363 | print "Adding %s to world" % db_pkg.cp |
| 1364 | new_world_pkgs.append(db_pkg.cp) |
| 1365 | if new_world_pkgs: |
| 1366 | world_set.update(new_world_pkgs) |
| 1367 | |
| 1368 | # Update environment (library cache, symlinks, etc.) |
| 1369 | if deps.board and "--pretend" not in emerge.opts: |
| 1370 | portage.env_update() |
| 1371 | |
| 1372 | # If we already upgraded portage, we don't need to do so again. But we do |
| 1373 | # need to upgrade the rest of the packages. So we'll go ahead and do that. |
David James | ebc3ae0 | 2011-05-21 20:46:10 -0700 | [diff] [blame] | 1374 | # |
| 1375 | # In order to grant the child permission to run setsid, we need to run sudo |
| 1376 | # again. We preserve SUDO_USER here in case an ebuild depends on it. |
David James | fcb70ef | 2011-02-02 16:02:30 -0800 | [diff] [blame] | 1377 | if portage_upgrade: |
David James | ebc3ae0 | 2011-05-21 20:46:10 -0700 | [diff] [blame] | 1378 | sudo = ["sudo", "-E", "SUDO_USER=%s" % os.environ.get("SUDO_USER", "")] |
| 1379 | args = sudo + parallel_emerge_args + ["--exclude=sys-apps/portage"] |
| 1380 | os.execvp("sudo", args) |
David James | fcb70ef | 2011-02-02 16:02:30 -0800 | [diff] [blame] | 1381 | |
| 1382 | print "Done" |
| 1383 | sys.exit(0) |
| 1384 | |
| 1385 | if __name__ == "__main__": |
| 1386 | main() |