Update devserver to filter packages in a separate root.
The devserver wants to clean debug symbols and build-time only files
from packages before installing to the device, but doing this directly
in the build root destroys our build environment. To prevent this
destruction, I've moved the filtering to occur in a subdirectory.
This task requires some extra copying, as we need to copy the files to
a separate root before editing them, but it makes it possible to easily
gmerge huge packages like chromeos-chrome to a small root partition.
BUG=chromium-os:12677
TEST=gmerge chromeos-chrome and a few other packages to a VM.
Change-Id: I085025d8ea908bf1801beece938d08064bf1ad2e
Reviewed-on: http://gerrit.chromium.org/gerrit/1138
Reviewed-by: Chris Sosa <sosa@chromium.org>
Tested-by: David James <davidjames@chromium.org>
diff --git a/builder.py b/builder.py
index 7a6af9c..2f93141 100644
--- a/builder.py
+++ b/builder.py
@@ -6,8 +6,12 @@
"""Package builder for the dev server."""
import os
+from portage import dbapi
+from portage import xpak
+import portage
import subprocess
import sys
+import tempfile
import cherrypy
@@ -32,6 +36,127 @@
return output_blob
+def _FilterInstallMaskFromPackage(in_path, out_path):
+ """Filter files matching DEFAULT_INSTALL_MASK out of a tarball.
+
+ Args:
+ in_path: Unfiltered tarball.
+ out_path: Location to write filtered tarball.
+ """
+
+ # Grab metadata about package in xpak format.
+ x = xpak.xpak_mem(xpak.tbz2(in_path).get_data())
+
+ # Build list of files to exclude. The tar command uses a slightly
+ # different exclude format than gmerge, so it needs to be adjusted
+ # appropriately.
+ #
+ # 1. tar matches against relative paths instead of absolute paths,
+ # so we need to prepend '.' to any paths that don't start with
+ # a wildcard.
+ # 2. tar expects the full filename to match (instead of a prefix),
+ # so we need to append a wildcard to any paths that don't already
+ # end with a wildcard.
+ excludes = []
+ for pattern in os.environ['DEFAULT_INSTALL_MASK'].split():
+ if not pattern.startswith('*'):
+ pattern = '.' + pattern
+ elif not pattern.endswith('*'):
+ pattern = pattern + '*'
+ excludes.append('--exclude="%s"' % pattern)
+ excludes = ' '.join(excludes)
+
+ gmerge_dir = os.path.dirname(out_path)
+ subprocess.check_call(['mkdir', '-p', gmerge_dir])
+
+ tmpd = tempfile.mkdtemp()
+ try:
+ # Extract package to temporary directory (excluding masked files).
+ cmd = ('pbzip2 -dc --ignore-trailing-garbage=1 %s'
+ ' | sudo tar -x -C %s %s --wildcards')
+ subprocess.check_call(cmd % (in_path, tmpd, excludes), shell=True)
+
+ # Build filtered version of package.
+ cmd = 'sudo tar -c --use-compress-program=pbzip2 -C %s . > %s'
+ subprocess.check_call(cmd % (tmpd, out_path), shell=True)
+ finally:
+ subprocess.check_call(['sudo', 'rm', '-rf', tmpd])
+
+ # Copy package metadata over to new package file.
+ xpak.tbz2(out_path).recompose_mem(x)
+
+
+def _UpdateGmergeBinhost(board, pkg):
+ """Add pkg to our gmerge-specific binhost.
+
+ Files matching DEFAULT_INSTALL_MASK are not included in the tarball.
+ """
+
+ root = '/build/%s/' % board
+ pkgdir = '/build/%s/packages' % board
+ gmerge_pkgdir = '/build/%s/gmerge-packages' % board
+
+ # Create gmerge pkgdir and give us permission to write to it.
+ subprocess.check_call(['sudo', 'mkdir', '-p', gmerge_pkgdir])
+ username = os.environ['PORTAGE_USERNAME']
+ subprocess.check_call(['sudo', 'chown', username, gmerge_pkgdir])
+
+ # Load databases.
+ trees = portage.create_trees(config_root=root, target_root=root)
+ vardb = trees[root]['vartree'].dbapi
+ bintree = trees[root]['bintree']
+ bintree.populate()
+ gmerge_tree = dbapi.bintree.binarytree(root, gmerge_pkgdir,
+ settings=bintree.settings)
+ gmerge_tree.populate()
+
+ # Create lists of matching packages.
+ gmerge_matches = set(gmerge_tree.dbapi.match(pkg))
+ bindb_matches = set(bintree.dbapi.match(pkg))
+ installed_matches = set(vardb.match(pkg)) & bindb_matches
+
+ # Remove any stale packages that exist in the local binhost but are not
+ # installed anymore.
+ if bindb_matches - installed_matches:
+ subprocess.check_call(['eclean-%s' % board, '-d', 'packages'])
+
+ # Remove any stale packages that exist in the gmerge binhost but are not
+ # installed anymore.
+ changed = False
+ for pkg in gmerge_matches - installed_matches:
+ gmerge_path = gmerge_tree.getname(pkg)
+ if os.path.exists(gmerge_path):
+ os.unlink(gmerge_path)
+ changed = True
+
+ # Copy any installed packages that have been rebuilt to the gmerge binhost.
+ for pkg in installed_matches:
+ build_time, = bintree.dbapi.aux_get(pkg, ['BUILD_TIME'])
+ build_path = bintree.getname(pkg)
+ gmerge_path = gmerge_tree.getname(pkg)
+
+ # If a package exists in the gmerge binhost with the same build time,
+ # don't rebuild it.
+ if pkg in gmerge_matches and os.path.exists(gmerge_path):
+ old_build_time, = gmerge_tree.dbapi.aux_get(pkg, ['BUILD_TIME'])
+ if old_build_time == build_time:
+ continue
+
+ _FilterInstallMaskFromPackage(build_path, gmerge_path)
+ changed = True
+
+ # If the gmerge binhost was changed, update the Packages file to match.
+ if changed:
+ env_copy = os.environ.copy()
+ env_copy['PKGDIR'] = gmerge_pkgdir
+ env_copy['ROOT'] = root
+ env_copy['PORTAGE_CONFIGROOT'] = root
+ cmd = ['/usr/lib/portage/bin/emaint', '-f', 'binhost']
+ subprocess.check_call(cmd, env=env_copy)
+
+ return bool(installed_matches)
+
+
class Builder(object):
"""Builds packages for the devserver."""
@@ -73,16 +198,17 @@
'Either start working on the package or pass --accept_stable '
'to gmerge')
- rc = subprocess.call(['emerge-%s' % board, pkg], env=env_copy)
- if rc != 0:
- return self.SetError('Could not emerge ' + pkg)
+ # If user did not supply -n, we want to rebuild the package.
+ usepkg = additional_args.get('usepkg')
+ if not usepkg:
+ rc = subprocess.call(['emerge-%s' % board, pkg], env=env_copy)
+ if rc != 0:
+ return self.SetError('Could not emerge ' + pkg)
- cherrypy.log('ecleaning %s' % pkg, 'BUILD')
- rc = subprocess.call(['eclean-' + board, '-d', 'packages'])
- if rc != 0:
- return self.SetError('eclean failed')
+ # Sync gmerge binhost.
+ if not _UpdateGmergeBinhost(board, pkg):
+ return self.SetError('Package %s is not installed' % pkg)
- cherrypy.log('eclean complete %s' % pkg, 'BUILD')
return 'Success\n'
except OSError, e:
return self.SetError('Could not execute build command: ' + str(e))