MB: Copy MB from Chromium repo
Essentially a copy of https://codereview.chromium.org/2299953002/
plus changes to WebRTC's license, changed OWNERS and additional
MB updates up to Chromium revision http://crrev.com/f1e2718a3ff.
The PRESUBMIT.py check was updated to use the existing
webrtc/build/mb_config.pyl to avoid breaking bots (that have
this path hardcoded).
This replaces the previously symlinked MB, which already
runs validation of the WebRTC configs as part of
webrtc/build/PRESUBMIT.py.
BUG=chromium:664425
NOTRY=True
TESTED=Ran:
tools/mb/mb.py gen -m client.webrtc -b 'Mac64 Release' --config-file webrtc/build/mb_config.pyl --isolate-map-file=webrtc/build/gn_isolate_map.pyl --gyp-script=webrtc/build/gyp_webrtc.py //out/Release
Review-Url: https://codereview.webrtc.org/2306163002
Cr-Commit-Position: refs/heads/master@{#15068}
diff --git a/tools/mb/mb_unittest.py b/tools/mb/mb_unittest.py
new file mode 100755
index 0000000..b70af00
--- /dev/null
+++ b/tools/mb/mb_unittest.py
@@ -0,0 +1,567 @@
+#!/usr/bin/python
+# Copyright (c) 2016 The WebRTC project authors. All Rights Reserved.
+#
+# Use of this source code is governed by a BSD-style license
+# that can be found in the LICENSE file in the root of the source
+# tree. An additional intellectual property rights grant can be found
+# in the file PATENTS. All contributing project authors may
+# be found in the AUTHORS file in the root of the source tree.
+
+"""Tests for mb.py."""
+
+import json
+import StringIO
+import os
+import sys
+import unittest
+
+import mb
+
+
+class FakeMBW(mb.MetaBuildWrapper):
+ def __init__(self, win32=False):
+ super(FakeMBW, self).__init__()
+
+ # Override vars for test portability.
+ if win32:
+ self.chromium_src_dir = 'c:\\fake_src'
+ self.default_config = 'c:\\fake_src\\tools\\mb\\mb_config.pyl'
+ self.default_isolate_map = ('c:\\fake_src\\testing\\buildbot\\'
+ 'gn_isolate_map.pyl')
+ self.platform = 'win32'
+ self.executable = 'c:\\python\\python.exe'
+ self.sep = '\\'
+ else:
+ self.chromium_src_dir = '/fake_src'
+ self.default_config = '/fake_src/tools/mb/mb_config.pyl'
+ self.default_isolate_map = '/fake_src/testing/buildbot/gn_isolate_map.pyl'
+ self.executable = '/usr/bin/python'
+ self.platform = 'linux2'
+ self.sep = '/'
+
+ self.files = {}
+ self.calls = []
+ self.cmds = []
+ self.cross_compile = None
+ self.out = ''
+ self.err = ''
+ self.rmdirs = []
+
+ def ExpandUser(self, path):
+ return '$HOME/%s' % path
+
+ def Exists(self, path):
+ return self.files.get(path) is not None
+
+ def MaybeMakeDirectory(self, path):
+ self.files[path] = True
+
+ def PathJoin(self, *comps):
+ return self.sep.join(comps)
+
+ def ReadFile(self, path):
+ return self.files[path]
+
+ def WriteFile(self, path, contents, force_verbose=False):
+ if self.args.dryrun or self.args.verbose or force_verbose:
+ self.Print('\nWriting """\\\n%s""" to %s.\n' % (contents, path))
+ self.files[path] = contents
+
+ def Call(self, cmd, env=None, buffer_output=True):
+ if env:
+ self.cross_compile = env.get('GYP_CROSSCOMPILE')
+ self.calls.append(cmd)
+ if self.cmds:
+ return self.cmds.pop(0)
+ return 0, '', ''
+
+ def Print(self, *args, **kwargs):
+ sep = kwargs.get('sep', ' ')
+ end = kwargs.get('end', '\n')
+ f = kwargs.get('file', sys.stdout)
+ if f == sys.stderr:
+ self.err += sep.join(args) + end
+ else:
+ self.out += sep.join(args) + end
+
+ def TempFile(self, mode='w'):
+ return FakeFile(self.files)
+
+ def RemoveFile(self, path):
+ del self.files[path]
+
+ def RemoveDirectory(self, path):
+ self.rmdirs.append(path)
+ files_to_delete = [f for f in self.files if f.startswith(path)]
+ for f in files_to_delete:
+ self.files[f] = None
+
+
+class FakeFile(object):
+ def __init__(self, files):
+ self.name = '/tmp/file'
+ self.buf = ''
+ self.files = files
+
+ def write(self, contents):
+ self.buf += contents
+
+ def close(self):
+ self.files[self.name] = self.buf
+
+
+TEST_CONFIG = """\
+{
+ 'masters': {
+ 'chromium': {},
+ 'fake_master': {
+ 'fake_builder': 'gyp_rel_bot',
+ 'fake_gn_builder': 'gn_rel_bot',
+ 'fake_gyp_crosscompile_builder': 'gyp_crosscompile',
+ 'fake_gn_debug_builder': 'gn_debug_goma',
+ 'fake_gyp_builder': 'gyp_debug',
+ 'fake_gn_args_bot': '//build/args/bots/fake_master/fake_gn_args_bot.gn',
+ 'fake_multi_phase': { 'phase_1': 'gn_phase_1', 'phase_2': 'gn_phase_2'},
+ },
+ },
+ 'configs': {
+ 'gyp_rel_bot': ['gyp', 'rel', 'goma'],
+ 'gn_debug_goma': ['gn', 'debug', 'goma'],
+ 'gyp_debug': ['gyp', 'debug', 'fake_feature1'],
+ 'gn_rel_bot': ['gn', 'rel', 'goma'],
+ 'gyp_crosscompile': ['gyp', 'crosscompile'],
+ 'gn_phase_1': ['gn', 'phase_1'],
+ 'gn_phase_2': ['gn', 'phase_2'],
+ },
+ 'mixins': {
+ 'crosscompile': {
+ 'gyp_crosscompile': True,
+ },
+ 'fake_feature1': {
+ 'gn_args': 'enable_doom_melon=true',
+ 'gyp_defines': 'doom_melon=1',
+ },
+ 'gyp': {'type': 'gyp'},
+ 'gn': {'type': 'gn'},
+ 'goma': {
+ 'gn_args': 'use_goma=true',
+ 'gyp_defines': 'goma=1',
+ },
+ 'phase_1': {
+ 'gn_args': 'phase=1',
+ 'gyp_args': 'phase=1',
+ },
+ 'phase_2': {
+ 'gn_args': 'phase=2',
+ 'gyp_args': 'phase=2',
+ },
+ 'rel': {
+ 'gn_args': 'is_debug=false',
+ },
+ 'debug': {
+ 'gn_args': 'is_debug=true',
+ },
+ },
+}
+"""
+
+
+TEST_BAD_CONFIG = """\
+{
+ 'configs': {
+ 'gn_rel_bot_1': ['gn', 'rel', 'chrome_with_codecs'],
+ 'gn_rel_bot_2': ['gn', 'rel', 'bad_nested_config'],
+ },
+ 'masters': {
+ 'chromium': {
+ 'a': 'gn_rel_bot_1',
+ 'b': 'gn_rel_bot_2',
+ },
+ },
+ 'mixins': {
+ 'gn': {'type': 'gn'},
+ 'chrome_with_codecs': {
+ 'gn_args': 'proprietary_codecs=true',
+ },
+ 'bad_nested_config': {
+ 'mixins': ['chrome_with_codecs'],
+ },
+ 'rel': {
+ 'gn_args': 'is_debug=false',
+ },
+ },
+}
+"""
+
+
+GYP_HACKS_CONFIG = """\
+{
+ 'masters': {
+ 'chromium': {},
+ 'fake_master': {
+ 'fake_builder': 'fake_config',
+ },
+ },
+ 'configs': {
+ 'fake_config': ['fake_mixin'],
+ },
+ 'mixins': {
+ 'fake_mixin': {
+ 'type': 'gyp',
+ 'gn_args': '',
+ 'gyp_defines':
+ ('foo=bar llvm_force_head_revision=1 '
+ 'gyp_link_concurrency=1 baz=1'),
+ },
+ },
+}
+"""
+
+
+class UnitTest(unittest.TestCase):
+ def fake_mbw(self, files=None, win32=False):
+ mbw = FakeMBW(win32=win32)
+ mbw.files.setdefault(mbw.default_config, TEST_CONFIG)
+ mbw.files.setdefault(
+ mbw.ToAbsPath('//testing/buildbot/gn_isolate_map.pyl'),
+ '''{
+ "foo_unittests": {
+ "label": "//foo:foo_unittests",
+ "type": "console_test_launcher",
+ "args": [],
+ },
+ }''')
+ mbw.files.setdefault(
+ mbw.ToAbsPath('//build/args/bots/fake_master/fake_gn_args_bot.gn'),
+ 'is_debug = false\n')
+ if files:
+ for path, contents in files.items():
+ mbw.files[path] = contents
+ return mbw
+
+ def check(self, args, mbw=None, files=None, out=None, err=None, ret=None):
+ if not mbw:
+ mbw = self.fake_mbw(files)
+
+ actual_ret = mbw.Main(args)
+
+ self.assertEqual(actual_ret, ret)
+ if out is not None:
+ self.assertEqual(mbw.out, out)
+ if err is not None:
+ self.assertEqual(mbw.err, err)
+ return mbw
+
+ def test_clobber(self):
+ files = {
+ '/fake_src/out/Debug': None,
+ '/fake_src/out/Debug/mb_type': None,
+ }
+ mbw = self.fake_mbw(files)
+
+ # The first time we run this, the build dir doesn't exist, so no clobber.
+ self.check(['gen', '-c', 'gn_debug_goma', '//out/Debug'], mbw=mbw, ret=0)
+ self.assertEqual(mbw.rmdirs, [])
+ self.assertEqual(mbw.files['/fake_src/out/Debug/mb_type'], 'gn')
+
+ # The second time we run this, the build dir exists and matches, so no
+ # clobber.
+ self.check(['gen', '-c', 'gn_debug_goma', '//out/Debug'], mbw=mbw, ret=0)
+ self.assertEqual(mbw.rmdirs, [])
+ self.assertEqual(mbw.files['/fake_src/out/Debug/mb_type'], 'gn')
+
+ # Now we switch build types; this should result in a clobber.
+ self.check(['gen', '-c', 'gyp_debug', '//out/Debug'], mbw=mbw, ret=0)
+ self.assertEqual(mbw.rmdirs, ['/fake_src/out/Debug'])
+ self.assertEqual(mbw.files['/fake_src/out/Debug/mb_type'], 'gyp')
+
+ # Now we delete mb_type; this checks the case where the build dir
+ # exists but wasn't populated by mb; this should also result in a clobber.
+ del mbw.files['/fake_src/out/Debug/mb_type']
+ self.check(['gen', '-c', 'gyp_debug', '//out/Debug'], mbw=mbw, ret=0)
+ self.assertEqual(mbw.rmdirs,
+ ['/fake_src/out/Debug', '/fake_src/out/Debug'])
+ self.assertEqual(mbw.files['/fake_src/out/Debug/mb_type'], 'gyp')
+
+ def test_gn_analyze(self):
+ files = {'/tmp/in.json': '''{\
+ "files": ["foo/foo_unittest.cc"],
+ "test_targets": ["foo_unittests"],
+ "additional_compile_targets": ["all"]
+ }''',
+ '/tmp/out.json.gn': '''{\
+ "status": "Found dependency",
+ "compile_targets": ["//foo:foo_unittests"],
+ "test_targets": ["//foo:foo_unittests"]
+ }'''}
+
+ mbw = self.fake_mbw(files)
+ mbw.Call = lambda cmd, env=None, buffer_output=True: (0, '', '')
+
+ self.check(['analyze', '-c', 'gn_debug_goma', '//out/Default',
+ '/tmp/in.json', '/tmp/out.json'], mbw=mbw, ret=0)
+ out = json.loads(mbw.files['/tmp/out.json'])
+ self.assertEqual(out, {
+ 'status': 'Found dependency',
+ 'compile_targets': ['foo:foo_unittests'],
+ 'test_targets': ['foo_unittests']
+ })
+
+ def test_gn_gen(self):
+ mbw = self.fake_mbw()
+ self.check(['gen', '-c', 'gn_debug_goma', '//out/Default', '-g', '/goma'],
+ mbw=mbw, ret=0)
+ self.assertMultiLineEqual(mbw.files['/fake_src/out/Default/args.gn'],
+ ('goma_dir = "/goma"\n'
+ 'is_debug = true\n'
+ 'use_goma = true\n'))
+
+ # Make sure we log both what is written to args.gn and the command line.
+ self.assertIn('Writing """', mbw.out)
+ self.assertIn('/fake_src/buildtools/linux64/gn gen //out/Default --check',
+ mbw.out)
+
+ mbw = self.fake_mbw(win32=True)
+ self.check(['gen', '-c', 'gn_debug_goma', '-g', 'c:\\goma', '//out/Debug'],
+ mbw=mbw, ret=0)
+ self.assertMultiLineEqual(mbw.files['c:\\fake_src\\out\\Debug\\args.gn'],
+ ('goma_dir = "c:\\\\goma"\n'
+ 'is_debug = true\n'
+ 'use_goma = true\n'))
+ self.assertIn('c:\\fake_src\\buildtools\\win\\gn.exe gen //out/Debug '
+ '--check\n', mbw.out)
+
+ mbw = self.fake_mbw()
+ self.check(['gen', '-m', 'fake_master', '-b', 'fake_gn_args_bot',
+ '//out/Debug'],
+ mbw=mbw, ret=0)
+ self.assertEqual(
+ mbw.files['/fake_src/out/Debug/args.gn'],
+ 'import("//build/args/bots/fake_master/fake_gn_args_bot.gn")\n')
+
+
+ def test_gn_gen_fails(self):
+ mbw = self.fake_mbw()
+ mbw.Call = lambda cmd, env=None, buffer_output=True: (1, '', '')
+ self.check(['gen', '-c', 'gn_debug_goma', '//out/Default'], mbw=mbw, ret=1)
+
+ def test_gn_gen_swarming(self):
+ files = {
+ '/tmp/swarming_targets': 'base_unittests\n',
+ '/fake_src/testing/buildbot/gn_isolate_map.pyl': (
+ "{'base_unittests': {"
+ " 'label': '//base:base_unittests',"
+ " 'type': 'raw',"
+ " 'args': [],"
+ "}}\n"
+ ),
+ '/fake_src/out/Default/base_unittests.runtime_deps': (
+ "base_unittests\n"
+ ),
+ }
+ mbw = self.fake_mbw(files)
+ self.check(['gen',
+ '-c', 'gn_debug_goma',
+ '--swarming-targets-file', '/tmp/swarming_targets',
+ '//out/Default'], mbw=mbw, ret=0)
+ self.assertIn('/fake_src/out/Default/base_unittests.isolate',
+ mbw.files)
+ self.assertIn('/fake_src/out/Default/base_unittests.isolated.gen.json',
+ mbw.files)
+
+ def test_gn_gen_swarming_script(self):
+ files = {
+ '/tmp/swarming_targets': 'cc_perftests\n',
+ '/fake_src/testing/buildbot/gn_isolate_map.pyl': (
+ "{'cc_perftests': {"
+ " 'label': '//cc:cc_perftests',"
+ " 'type': 'script',"
+ " 'script': '/fake_src/out/Default/test_script.py',"
+ " 'args': [],"
+ "}}\n"
+ ),
+ 'c:\\fake_src\out\Default\cc_perftests.exe.runtime_deps': (
+ "cc_perftests\n"
+ ),
+ }
+ mbw = self.fake_mbw(files=files, win32=True)
+ self.check(['gen',
+ '-c', 'gn_debug_goma',
+ '--swarming-targets-file', '/tmp/swarming_targets',
+ '--isolate-map-file',
+ '/fake_src/testing/buildbot/gn_isolate_map.pyl',
+ '//out/Default'], mbw=mbw, ret=0)
+ self.assertIn('c:\\fake_src\\out\\Default\\cc_perftests.isolate',
+ mbw.files)
+ self.assertIn('c:\\fake_src\\out\\Default\\cc_perftests.isolated.gen.json',
+ mbw.files)
+
+
+ def test_gn_isolate(self):
+ files = {
+ '/fake_src/out/Default/toolchain.ninja': "",
+ '/fake_src/testing/buildbot/gn_isolate_map.pyl': (
+ "{'base_unittests': {"
+ " 'label': '//base:base_unittests',"
+ " 'type': 'raw',"
+ " 'args': [],"
+ "}}\n"
+ ),
+ '/fake_src/out/Default/base_unittests.runtime_deps': (
+ "base_unittests\n"
+ ),
+ }
+ self.check(['isolate', '-c', 'gn_debug_goma', '//out/Default',
+ 'base_unittests'], files=files, ret=0)
+
+ # test running isolate on an existing build_dir
+ files['/fake_src/out/Default/args.gn'] = 'is_debug = True\n'
+ self.check(['isolate', '//out/Default', 'base_unittests'],
+ files=files, ret=0)
+
+ files['/fake_src/out/Default/mb_type'] = 'gn\n'
+ self.check(['isolate', '//out/Default', 'base_unittests'],
+ files=files, ret=0)
+
+ def test_gn_run(self):
+ files = {
+ '/fake_src/testing/buildbot/gn_isolate_map.pyl': (
+ "{'base_unittests': {"
+ " 'label': '//base:base_unittests',"
+ " 'type': 'raw',"
+ " 'args': [],"
+ "}}\n"
+ ),
+ '/fake_src/out/Default/base_unittests.runtime_deps': (
+ "base_unittests\n"
+ ),
+ }
+ self.check(['run', '-c', 'gn_debug_goma', '//out/Default',
+ 'base_unittests'], files=files, ret=0)
+
+ def test_gn_lookup(self):
+ self.check(['lookup', '-c', 'gn_debug_goma'], ret=0)
+
+ def test_gn_lookup_goma_dir_expansion(self):
+ self.check(['lookup', '-c', 'gn_rel_bot', '-g', '/foo'], ret=0,
+ out=('\n'
+ 'Writing """\\\n'
+ 'goma_dir = "/foo"\n'
+ 'is_debug = false\n'
+ 'use_goma = true\n'
+ '""" to _path_/args.gn.\n\n'
+ '/fake_src/buildtools/linux64/gn gen _path_\n'))
+
+ def test_gyp_analyze(self):
+ mbw = self.check(['analyze', '-c', 'gyp_rel_bot', '//out/Release',
+ '/tmp/in.json', '/tmp/out.json'], ret=0)
+ self.assertIn('analyzer', mbw.calls[0])
+
+ def test_gyp_crosscompile(self):
+ mbw = self.fake_mbw()
+ self.check(['gen', '-c', 'gyp_crosscompile', '//out/Release'],
+ mbw=mbw, ret=0)
+ self.assertTrue(mbw.cross_compile)
+
+ def test_gyp_gen(self):
+ self.check(['gen', '-c', 'gyp_rel_bot', '-g', '/goma', '//out/Release'],
+ ret=0,
+ out=("GYP_DEFINES='goma=1 gomadir=/goma'\n"
+ "python build/gyp_chromium -G output_dir=out\n"))
+
+ mbw = self.fake_mbw(win32=True)
+ self.check(['gen', '-c', 'gyp_rel_bot', '-g', 'c:\\goma', '//out/Release'],
+ mbw=mbw, ret=0,
+ out=("set GYP_DEFINES=goma=1 gomadir='c:\\goma'\n"
+ "python build\\gyp_chromium -G output_dir=out\n"))
+
+ def test_gyp_gen_fails(self):
+ mbw = self.fake_mbw()
+ mbw.Call = lambda cmd, env=None, buffer_output=True: (1, '', '')
+ self.check(['gen', '-c', 'gyp_rel_bot', '//out/Release'], mbw=mbw, ret=1)
+
+ def test_gyp_lookup_goma_dir_expansion(self):
+ self.check(['lookup', '-c', 'gyp_rel_bot', '-g', '/foo'], ret=0,
+ out=("GYP_DEFINES='goma=1 gomadir=/foo'\n"
+ "python build/gyp_chromium -G output_dir=_path_\n"))
+
+ def test_help(self):
+ orig_stdout = sys.stdout
+ try:
+ sys.stdout = StringIO.StringIO()
+ self.assertRaises(SystemExit, self.check, ['-h'])
+ self.assertRaises(SystemExit, self.check, ['help'])
+ self.assertRaises(SystemExit, self.check, ['help', 'gen'])
+ finally:
+ sys.stdout = orig_stdout
+
+ def test_multiple_phases(self):
+ # Check that not passing a --phase to a multi-phase builder fails.
+ mbw = self.check(['lookup', '-m', 'fake_master', '-b', 'fake_multi_phase'],
+ ret=1)
+ self.assertIn('Must specify a build --phase', mbw.out)
+
+ # Check that passing a --phase to a single-phase builder fails.
+ mbw = self.check(['lookup', '-m', 'fake_master', '-b', 'fake_gn_builder',
+ '--phase', 'phase_1'], ret=1)
+ self.assertIn('Must not specify a build --phase', mbw.out)
+
+ # Check that passing a wrong phase key to a multi-phase builder fails.
+ mbw = self.check(['lookup', '-m', 'fake_master', '-b', 'fake_multi_phase',
+ '--phase', 'wrong_phase'], ret=1)
+ self.assertIn('Phase wrong_phase doesn\'t exist', mbw.out)
+
+ # Check that passing a correct phase key to a multi-phase builder passes.
+ mbw = self.check(['lookup', '-m', 'fake_master', '-b', 'fake_multi_phase',
+ '--phase', 'phase_1'], ret=0)
+ self.assertIn('phase = 1', mbw.out)
+
+ mbw = self.check(['lookup', '-m', 'fake_master', '-b', 'fake_multi_phase',
+ '--phase', 'phase_2'], ret=0)
+ self.assertIn('phase = 2', mbw.out)
+
+ def test_validate(self):
+ mbw = self.fake_mbw()
+ self.check(['validate'], mbw=mbw, ret=0)
+
+ def test_bad_validate(self):
+ mbw = self.fake_mbw()
+ mbw.files[mbw.default_config] = TEST_BAD_CONFIG
+ self.check(['validate'], mbw=mbw, ret=1)
+
+ def test_gyp_env_hacks(self):
+ mbw = self.fake_mbw()
+ mbw.files[mbw.default_config] = GYP_HACKS_CONFIG
+ self.check(['lookup', '-c', 'fake_config'], mbw=mbw,
+ ret=0,
+ out=("GYP_DEFINES='foo=bar baz=1'\n"
+ "GYP_LINK_CONCURRENCY=1\n"
+ "LLVM_FORCE_HEAD_REVISION=1\n"
+ "python build/gyp_chromium -G output_dir=_path_\n"))
+
+
+if __name__ == '__main__':
+ unittest.main()
+
+ def test_validate(self):
+ mbw = self.fake_mbw()
+ self.check(['validate'], mbw=mbw, ret=0)
+
+ def test_bad_validate(self):
+ mbw = self.fake_mbw()
+ mbw.files[mbw.default_config] = TEST_BAD_CONFIG
+ self.check(['validate'], mbw=mbw, ret=1)
+
+ def test_gyp_env_hacks(self):
+ mbw = self.fake_mbw()
+ mbw.files[mbw.default_config] = GYP_HACKS_CONFIG
+ self.check(['lookup', '-c', 'fake_config'], mbw=mbw,
+ ret=0,
+ out=("GYP_DEFINES='foo=bar baz=1'\n"
+ "GYP_LINK_CONCURRENCY=1\n"
+ "LLVM_FORCE_HEAD_REVISION=1\n"
+ "python build/gyp_chromium -G output_dir=_path_\n"))
+
+
+if __name__ == '__main__':
+ unittest.main()