scripts/gconv_strip.py

# Copyright 2014 The ChromiumOS Authors
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

"""Script to remove unused gconv charset modules from a build."""

import functools
import glob
import logging
import operator
import os
import stat

from chromite.third_party import lddtree

from chromite.lib import commandline
from chromite.lib import cros_build_lib
from chromite.lib import osutils


try:
    import pytest  # pylint: disable=import-error

    ahocorasick = pytest.importorskip("ahocorasick")
except ImportError:
    import ahocorasick


# Path pattern to search for the gconv-modules file.
GCONV_MODULES_PATH = "usr/*/gconv/gconv-modules"

# Sticky modules. These charsets modules are always included even if they
# aren't used. You can specify any charset name as supported by 'iconv_open',
# for example, 'LATIN1' or 'ISO-8859-1'.
STICKY_MODULES = ("UTF-16", "UTF-32", "UNICODE")

# List of function names (symbols) known to use a charset as a parameter.
GCONV_SYMBOLS = (
    # glibc
    "iconv_open",
    "iconv",
    # glib
    "g_convert",
    "g_convert_with_fallback",
    "g_iconv",
    "g_locale_to_utf8",
    "g_get_charset",
)


class GconvModules:
    """Class to manipulate the gconv/gconv-modules file and referenced modules.

    This class parses the contents of the gconv-modules file installed by glibc
    which provides the definition of the charsets supported by iconv_open(3). It
    allows to load the current gconv-modules file and rewrite it to include only
    a subset of the supported modules, removing the other modules.

    Each charset is involved on some transformation between that charset and an
    internal representation. This transformation is defined on a .so file loaded
    dynamically with dlopen(3) when the charset defined in this file is
    requested to iconv_open(3).

    See the comments on gconv-modules file for syntax details.
    """

    def __init__(self, gconv_modules_file, modules_dir):
        """Initialize the class.

        Args:
            gconv_modules_file: Path to gconv/gconv-modules file.
            modules_dir: Path to the directory that contains the gconv modules.
        """
        self._filename = gconv_modules_file
        self._modules_dir = modules_dir

        # An alias map of charsets. The key (fromcharset) is the alias name and
        # the value (tocharset) is the real charset name. We also support a
        # value that is an alias for another charset.
        self._alias = {}

        # The modules dict goes from charset to module names (the filenames
        # without the .so extension). Since several transformations involving
        # the same charset could be defined in different files, the values of
        # this dict are a set of module names.
        self._modules = {}

    def Load(self):
        """Load the charsets from gconv-modules."""
        with open(self._filename, encoding="utf-8") as fp:
            for line in fp:
                line = line.split("#", 1)[0].strip()
                if not line:
                    # Ignore blank lines & comments.
                    continue

                lst = line.split()
                if lst[0] == "module":
                    _, fromset, toset, filename = lst[:4]
                    for charset in (fromset, toset):
                        charset = charset.rstrip("/")
                        mods = self._modules.get(charset, set())
                        mods.add(filename)
                        self._modules[charset] = mods
                elif lst[0] == "alias":
                    _, fromset, toset = lst
                    fromset = fromset.rstrip("/")
                    toset = toset.rstrip("/")
                    # Warn if the same charset is defined as two different
                    # aliases.
                    if self._alias.get(fromset, toset) != toset:
                        logging.error(
                            'charset "%s" already defined as "%s".',
                            fromset,
                            self._alias[fromset],
                        )
                    self._alias[fromset] = toset
                else:
                    cros_build_lib.Die("Unknown line: %s", line)

        logging.debug(
            "Found %d modules and %d alias in %s",
            len(self._modules),
            len(self._alias),
            self._filename,
        )
        charsets = sorted(list(self._alias) + list(self._modules))
        # Remove the 'INTERNAL' charset from the list, since it is not a charset
        # but an internal representation used to convert to and from other
        # charsets.
        if "INTERNAL" in charsets:
            charsets.remove("INTERNAL")
        return charsets

    def Rewrite(self, used_charsets, dryrun=False):
        """Rewrite gconv-modules file with only the used charsets.

        Args:
            used_charsets: A list of used charsets. This should be a subset of
                the list returned by Load().
            dryrun: Whether this function should not change any file.
        """

        # Compute the used modules.
        used_modules = set()
        for charset in used_charsets:
            while charset in self._alias:
                charset = self._alias[charset]
            used_modules.update(self._modules[charset])
        unused_modules = (
            functools.reduce(set.union, list(self._modules.values()))
            - used_modules
        )

        all_modules = set.union(used_modules, unused_modules)
        # The list of charsets that depend on a given library. For example,
        # libdeps['libCNS.so'] is the set of all the modules that require that
        # library. These libraries live in the same directory as the modules.
        libdeps = {}
        for module in all_modules:
            deps = lddtree.ParseELF(
                os.path.join(self._modules_dir, "%s.so" % module),
                self._modules_dir,
                [],
            )
            if "needed" not in deps:
                continue
            for lib in deps["needed"]:
                # Ignore the libs without a path defined (outside the
                # modules_dir).
                if deps["libs"][lib]["path"]:
                    libdeps[lib] = libdeps.get(lib, set()).union([module])

        used_libdeps = set(
            lib
            for lib, deps in libdeps.items()
            if deps.intersection(used_modules)
        )
        unused_libdeps = set(libdeps).difference(used_libdeps)

        logging.debug("Used modules: %s", ", ".join(sorted(used_modules)))
        logging.debug("Used dependency libs: %s, ".join(sorted(used_libdeps)))

        unused_size = 0
        for module in sorted(unused_modules):
            module_path = os.path.join(self._modules_dir, "%s.so" % module)
            unused_size += os.lstat(module_path).st_size
            logging.debug("rm %s", module_path)
            if not dryrun:
                os.unlink(module_path)

        unused_libdeps_size = 0
        for lib in sorted(unused_libdeps):
            lib_path = os.path.join(self._modules_dir, lib)
            unused_libdeps_size += os.lstat(lib_path).st_size
            logging.debug("rm %s", lib_path)
            if not dryrun:
                os.unlink(lib_path)

        logging.info(
            "Done. Using %d gconv modules. Removed %d unused modules"
            " (%.1f KiB) and %d unused dependencies (%.1f KiB)",
            len(used_modules),
            len(unused_modules),
            unused_size / 1024.0,
            len(unused_libdeps),
            unused_libdeps_size / 1024.0,
        )

        # Recompute the gconv-modules file with only the included gconv modules.
        result = []
        with open(self._filename, encoding="utf-8") as fp:
            for line in fp:
                lst = line.split("#", 1)[0].strip().split()

                if not lst:
                    # Keep comments and copyright headers.
                    result.append(line)
                elif lst[0] == "module":
                    _, _, _, filename = lst[:4]
                    if filename in used_modules:
                        # Used module
                        result.append(line)
                elif lst[0] == "alias":
                    _, charset, _ = lst
                    charset = charset.rstrip("/")
                    while charset in self._alias:
                        charset = self._alias[charset]
                    if used_modules.intersection(self._modules[charset]):
                        # Alias to an used module
                        result.append(line)
                else:
                    cros_build_lib.Die("Unknown line: %s", line)

        if not dryrun:
            osutils.WriteFile(self._filename, "".join(result))


def MultipleStringMatch(patterns, corpus):
    """Search a list of strings in a corpus string.

    Args:
        patterns: A list of strings.
        corpus: The text where to search for the strings.

    Returns:
        A list of Booleans stating whether each pattern string was found in the
        corpus or not.
    """
    result = [False] * len(patterns)

    tree = ahocorasick.Automaton()
    for i, word in enumerate(patterns):
        tree.add_word(word, i)
    tree.make_automaton()

    for _, i in tree.iter(corpus):
        result[i] = True

    return result


def GconvStrip(opts):
    """Process gconv-modules and remove unused modules.

    Args:
        opts: The command-line args passed to the script.

    Returns:
        The exit code number indicating whether the process succeeded.
    """
    root_st = os.lstat(opts.root)
    if not stat.S_ISDIR(root_st.st_mode):
        cros_build_lib.Die("root (%s) must be a directory.", opts.root)

    # Detect the possible locations of the gconv-modules file.
    gconv_modules_files = glob.glob(os.path.join(opts.root, GCONV_MODULES_PATH))

    if not gconv_modules_files:
        logging.warning("gconv-modules file not found.")
        return 1

    # Only one gconv-modules files should be present, either on /usr/lib or
    # /usr/lib64, but not both.
    if len(gconv_modules_files) > 1:
        cros_build_lib.Die("Found several gconv-modules files.")

    gconv_modules_file = gconv_modules_files[0]
    logging.info(
        "Searching for unused gconv files defined in %s", gconv_modules_file
    )

    # Additional gconv-modules configuration files can be present in the
    # co-located gconv-modules.d. glibc installs a gconv-modules-extra.conf
    # here by default.
    modules_dir = os.path.dirname(gconv_modules_file)
    extras = glob.glob(
        os.path.join(
            modules_dir,
            os.path.basename(gconv_modules_file) + ".d",
            "*.conf",
        )
    )
    gmods_groups = [GconvModules(gconv_modules_file, modules_dir)]
    gmods_groups.extend(GconvModules(x, modules_dir) for x in extras)

    # Use scanelf to search for all the binary files on the rootfs that require
    # or define the symbol iconv_open. We also include the binaries that define
    # it since there could be internal calls to it from other functions.
    symbols = ",".join(GCONV_SYMBOLS)
    cmd = [
        "scanelf",
        "--mount",
        "--quiet",
        "--recursive",
        "--format",
        "#s%F",
        "--symbol",
        symbols,
        opts.root,
    ]
    result = cros_build_lib.run(
        cmd, stdout=True, print_cmd=False, encoding="utf-8"
    )
    files = set(result.stdout.splitlines())
    logging.debug("Symbols %s found on %d files.", symbols, len(files))

    for gmods in gmods_groups:
        charsets = gmods.Load()
        # The charsets are represented as nul-terminated strings in the binary
        # files, so we append the '\0' to each string. This prevents some false
        # positives when the name of the charset is a substring of some other
        # string. It doesn't prevent false positives when the charset name is
        # the suffix of another string, for example a binary with the string
        # "DON'T DO IT\0" will match the 'IT' charset. Empirical test on
        # ChromeOS images suggests that only 4 charsets could fall in category.
        strings = [s.encode("utf-8") + b"x\00" for s in charsets]
        logging.info(
            "Will search for %d strings in %d files", len(strings), len(files)
        )

        # Charsets listed in STICKY_MOUDLES are initialized as used. Note that
        # those strings should be listed in the gconv-modules file.
        unknown_sticky_modules = set(STICKY_MODULES) - set(charsets)
        if unknown_sticky_modules:
            logging.warning(
                "The following charsets were explicitly requested in "
                "STICKY_MODULES even though they don't exist: %s",
                ", ".join(unknown_sticky_modules),
            )
        global_used = [charset in STICKY_MODULES for charset in charsets]

        for filename in files:
            used_filenames = MultipleStringMatch(
                strings, osutils.ReadFile(filename, mode="rb")
            )

            global_used = [
                operator.or_(*x) for x in zip(global_used, used_filenames)
            ]
            # Check the debug flag to avoid running a useless loop.
            if opts.debug and any(used_filenames):
                logging.debug("File %s:", filename)
                for i, used_filename in enumerate(used_filenames):
                    if used_filename:
                        logging.debug(" - %s", strings[i])

        used_charsets = [cs for cs, used in zip(charsets, global_used) if used]
        gmods.Rewrite(used_charsets, opts.dryrun)
    return 0


def ParseArgs(argv):
    """Return parsed commandline arguments."""

    parser = commandline.ArgumentParser(description=__doc__, dryrun=True)
    parser.add_argument(
        "root",
        type="path",
        help="path to the directory where the rootfs is mounted.",
    )

    opts = parser.parse_args(argv)
    opts.Freeze()
    return opts


def main(argv):
    """Main function to start the script."""
    opts = ParseArgs(argv)
    logging.debug("Options are %s", opts)

    return GconvStrip(opts)