[TestRunner] create new test runner that takes configuration file
This CL lands the initial new run_test_suite.js that can take its
configuration from a config JSON file, or from flags.
This new script is only used to run `npm run auto-interactionstest`
and nothing more; I want to roll it out slowly and ensure that
everyone is aware of it before removing the old Python script. I will
create a doc with all the various steps, and required documentation,
as I've taken the chance to rename some options to make them clearer.
Bug: chromium:1186163
Change-Id: I5a199f82ac7ab0f323988acacaa7aae2d64e6349
Reviewed-on: https://chromium-review.googlesource.com/c/devtools/devtools-frontend/+/2763866
Commit-Queue: Jack Franklin <jacktfranklin@chromium.org>
Reviewed-by: Paul Lewis <aerotwist@chromium.org>
diff --git a/scripts/devtools_paths.js b/scripts/devtools_paths.js
index e8025ac..a433eb6 100644
--- a/scripts/devtools_paths.js
+++ b/scripts/devtools_paths.js
@@ -145,6 +145,10 @@
return path.join(nodeModulesPath(), 'stylelint', 'bin', 'stylelint.js');
}
+function mochaExecutablePath() {
+ return path.join(nodeModulesPath(), 'mocha', 'bin', 'mocha');
+}
+
function downloadedChromeBinaryPath() {
const paths = {
'linux': path.join('chrome-linux', 'chrome'),
@@ -159,6 +163,7 @@
nodePath,
devtoolsRootPath,
nodeModulesPath,
+ mochaExecutablePath,
stylelintExecutablePath,
downloadedChromeBinaryPath
};
diff --git a/scripts/test/run_test_suite.js b/scripts/test/run_test_suite.js
new file mode 100644
index 0000000..73508ad
--- /dev/null
+++ b/scripts/test/run_test_suite.js
@@ -0,0 +1,189 @@
+#!/usr/bin/env node
+
+// Copyright 2020 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+const path = require('path');
+const fs = require('fs');
+const childProcess = require('child_process');
+const {
+ nodePath,
+ mochaExecutablePath,
+ downloadedChromeBinaryPath,
+ devtoolsRootPath,
+} = require('../devtools_paths.js');
+
+function log(...msg) {
+ console.log('[run_test_suite.js]', ...msg);
+}
+function err(msg) {
+ console.error('[run_test_suite.js]', ...msg);
+}
+
+const yargsObject =
+ require('yargs')
+ .option(
+ 'test-suite-path', {type: 'string', desc: 'Path to the test suite, starting from out/Target directory.'})
+ .option('target', {type: 'string', default: 'Default', desc: 'Name of the Ninja output directory.'})
+ .option('node-modules-path', {
+ type: 'string',
+ desc:
+ 'Path to the node_modules directory for Node to use, relative to the current working directory. Defaults to local node_modules folder.'
+ })
+ .option('test-file-pattern', {
+ type: 'string',
+ desc: 'A comma separated glob (or just a file path) to select specific test files to execute.'
+ })
+ .option(
+ 'chrome-binary-path',
+ {type: 'string', desc: 'Path to the Chromium binary.', default: downloadedChromeBinaryPath()})
+ .option('chrome-features', {
+ type: 'string',
+ desc: 'Comma separated list of strings passed to --enable-features on the Chromium command line.'
+ })
+ .option('jobs', {
+ type: 'number',
+ desc: 'Number of parallel runners to use (if supported). Defaults to 1.',
+ default: 1,
+ })
+ .option('cwd', {
+ type: 'string',
+ desc: 'Path to the directory containing the out/TARGET folder.',
+ default: devtoolsRootPath()
+ })
+ .parserConfiguration({
+ // So that if we pass --foo-bar, Yargs only populates
+ // argv with '--foo-bar', not '--foo-bar' and 'fooBar'.
+ 'camel-case-expansion': false
+ })
+ .demandOption(['test-suite-path'])
+ // Take options via --config config.json
+ .config()
+ // Fail on any unknown arguments
+ .strict()
+ .argv;
+
+
+function getAbsoluteTestSuitePath(target) {
+ const pathInput = yargsObject['test-suite-path'];
+ // We take the input with Linux path separators, but need to split and join to make sure this works on Windows.
+ const testSuitePathParts = pathInput.split('/');
+ log(`Using test suite ${path.join(pathInput, path.sep)}`);
+
+ const fullPath = path.join(yargsObject['cwd'], 'out', target, ...testSuitePathParts);
+ return fullPath;
+}
+
+function setEnvValueIfValuePresent(name, value) {
+ if (value) {
+ process.env[name] = value;
+ }
+}
+
+function setNodeModulesPath(nodeModulesPath) {
+ if (nodeModulesPath) {
+ // Node requires the path to be absolute
+ if (path.isAbsolute(nodeModulesPath)) {
+ setEnvValueIfValuePresent('NODE_PATH', nodeModulesPath);
+ } else {
+ setEnvValueIfValuePresent('NODE_PATH', path.join(yargsObject['cwd'], nodeModulesPath));
+ }
+ }
+}
+
+function executeTestSuite(
+ {absoluteTestSuitePath, jobs, target, nodeModulesPath, chromeBinaryPath, chromeFeatures, testFilePattern, cwd}) {
+ /**
+ * Internally within various scripts (Mocha configs, Conductor, etc), we rely on
+ * process.env.FOO. We are moving to exposing the entire configuration to
+ * process.env.TEST_CONFIG_JSON but for now we need to still expose the values
+ * directly on the environment whilst we roll out this script and make all the
+ * required changes.
+ */
+ setEnvValueIfValuePresent('CHROME_BIN', chromeBinaryPath);
+ setEnvValueIfValuePresent('CHROME_FEATURES', chromeFeatures);
+ setEnvValueIfValuePresent('JOBS', jobs);
+ setEnvValueIfValuePresent('TARGET', target);
+ setEnvValueIfValuePresent('TEST_PATTERNS', testFilePattern);
+
+ /**
+ * This one has to be set as an ENV variable as Node looks for the NODE_PATH environment variable.
+ */
+ setNodeModulesPath(nodeModulesPath);
+
+ const argumentsForNode = [
+ mochaExecutablePath(),
+ ];
+ if (process.env.DEBUG) {
+ argumentsForNode.unshift('--inspect');
+ }
+
+ const testSuiteConfig = path.join(absoluteTestSuitePath, '.mocharc.js');
+ argumentsForNode.push('--config', testSuiteConfig);
+ const result = childProcess.spawnSync(nodePath(), argumentsForNode, {encoding: 'utf-8', stdio: 'inherit', cwd});
+ return result.status;
+}
+
+function fileIsExecutable(filePath) {
+ try {
+ fs.accessSync(filePath, fs.constants.X_OK);
+ return true;
+ } catch (e) {
+ return false;
+ }
+}
+
+function validateChromeBinaryExistsAndExecutable(chromeBinaryPath) {
+ return (
+ fs.existsSync(chromeBinaryPath) && fs.statSync(chromeBinaryPath).isFile(chromeBinaryPath) &&
+ fileIsExecutable(chromeBinaryPath));
+}
+
+function main() {
+ const chromeBinaryPath = yargsObject['chrome-binary-path'];
+
+ if (!validateChromeBinaryExistsAndExecutable(chromeBinaryPath)) {
+ err(`Chrome binary path ${chromeBinaryPath} is not valid`);
+ }
+
+ const chromeFeatures = yargsObject['chrome-features'] ? `--enable-features=${yargsObject['chrome-features']}` : '';
+
+ const target = yargsObject['target'];
+ // eslint-disable-next-line no-unused-vars
+ const {$0, _, ...namedConfigFlags} = yargsObject;
+
+ /**
+ * Expose the configuration to any downstream test runners (Mocha, Conductor,
+ * Test servers, etc).
+ */
+ process.env.TEST_RUNNER_JSON_CONFIG = JSON.stringify(namedConfigFlags);
+
+ log(`Using Chromium binary ${chromeBinaryPath} ${chromeFeatures}`);
+ log(`Using target ${target}`);
+
+ const testSuitePath = getAbsoluteTestSuitePath(target);
+
+ let resultStatusCode = -1;
+ try {
+ resultStatusCode = executeTestSuite({
+ absoluteTestSuitePath: testSuitePath,
+ chromeBinaryPath,
+ chromeFeatures,
+ nodeModulesPath: yargsObject['node-modules-path'],
+ jobs: yargsObject['jobs'],
+ testFilePattern: yargsObject['test-file-pattern'],
+ target,
+ cwd: yargsObject['cwd']
+ });
+ } catch (error) {
+ log('Unexpected error when running test suite', error);
+ resultStatusCode = 1;
+ }
+ if (resultStatusCode !== 0) {
+ log('ERRORS DETECTED');
+ }
+ process.exit(resultStatusCode);
+}
+
+main();