Update gmerge to support deep upgrades.

With --deep mode, gmerge updates all necessary packages. This is useful for
ensuring dependencies are up to date.

Other features added in this CL:
 - /usr/local/portage is rm -rf'd at beginning of gmerge as well in case
   leftover state is there from a failed merge.
 - Improve error message when devserver is not running.
 - The --extra flag allows you to specify extra flags to emerge. This is useful
   for example to exclude certain packages from the upgrade. E.g
    gmerge -Dn chromeos -x '--usepkg-exclude=trousers'

BUG=chromium-os:15606, chromium-os:11332, chromium-os:14748
TEST=Try gmerging a deep upgrade of power manager. If packages that use enewuser
     / enewgroup are excluded, we can upgrade the rest of the packages. If the
     latest toolchain is used which includes getent, we can upgrade those
     packages.

Change-Id: Id01b35133486ec9448b62c80194d61ff1360e87e
Reviewed-on: http://gerrit.chromium.org/gerrit/1320
Reviewed-by: Chris Sosa <sosa@chromium.org>
Tested-by: David James <davidjames@chromium.org>
diff --git a/builder.py b/builder.py
index 2f93141..df93d20 100644
--- a/builder.py
+++ b/builder.py
@@ -86,7 +86,7 @@
   xpak.tbz2(out_path).recompose_mem(x)
 
 
-def _UpdateGmergeBinhost(board, pkg):
+def _UpdateGmergeBinhost(board, pkg, deep):
   """Add pkg to our gmerge-specific binhost.
 
   Files matching DEFAULT_INSTALL_MASK are not included in the tarball.
@@ -110,10 +110,16 @@
                                          settings=bintree.settings)
   gmerge_tree.populate()
 
-  # Create lists of matching packages.
-  gmerge_matches = set(gmerge_tree.dbapi.match(pkg))
-  bindb_matches = set(bintree.dbapi.match(pkg))
-  installed_matches = set(vardb.match(pkg)) & bindb_matches
+  if deep:
+    # If we're in deep mode, fill in the binhost completely.
+    gmerge_matches = set(gmerge_tree.dbapi.cpv_all())
+    bindb_matches = set(bintree.dbapi.cpv_all())
+    installed_matches = set(vardb.cpv_all()) & bindb_matches
+  else:
+    # Otherwise, just fill in the requested package.
+    gmerge_matches = set(gmerge_tree.dbapi.match(pkg))
+    bindb_matches = set(bintree.dbapi.match(pkg))
+    installed_matches = set(vardb.match(pkg)) & bindb_matches
 
   # Remove any stale packages that exist in the local binhost but are not
   # installed anymore.
@@ -142,6 +148,7 @@
       if old_build_time == build_time:
         continue
 
+    cherrypy.log('Filtering install mask from %s' % pkg, 'BUILD')
     _FilterInstallMaskFromPackage(build_path, gmerge_path)
     changed = True
 
@@ -206,7 +213,8 @@
           return self.SetError('Could not emerge ' + pkg)
 
       # Sync gmerge binhost.
-      if not _UpdateGmergeBinhost(board, pkg):
+      deep = additional_args.get('deep')
+      if not _UpdateGmergeBinhost(board, pkg, deep):
         return self.SetError('Package %s is not installed' % pkg)
 
       return 'Success\n'