Move test scripts to scripts/test
R=aerotwist@chromium.org, liviurau@chromium.org, machenbach@chromium.org
Bug: 1018122
Change-Id: I7cc3a1a9ab01c649b3c7fe8ba5d5254adc631634
Reviewed-on: https://chromium-review.googlesource.com/c/devtools/devtools-frontend/+/1880033
Commit-Queue: Yang Guo <yangguo@chromium.org>
Reviewed-by: Liviu Rau <liviurau@chromium.org>
Reviewed-by: Michael Achenbach <machenbach@chromium.org>
diff --git a/scripts/compile_frontend.py b/scripts/compile_frontend.py
index 58f1bd6..564071d 100755
--- a/scripts/compile_frontend.py
+++ b/scripts/compile_frontend.py
@@ -27,381 +27,6 @@
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-import argparse
-import os
-import os.path as path
-import re
-import shutil
-import subprocess
-import sys
-import tempfile
+from test import run_type_check
-from build import modular_build
-from build import generate_protocol_externs
-
-import dependency_preprocessor
-import devtools_paths
-import utils
-
-try:
- import simplejson as json
-except ImportError:
- import json
-
-is_cygwin = sys.platform == 'cygwin'
-
-
-def popen(arguments):
- return subprocess.Popen(arguments, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
-
-
-def to_platform_path(filepath):
- if not is_cygwin:
- return filepath
- return re.sub(r'^/cygdrive/(\w)', '\\1:', filepath)
-
-
-def to_platform_path_exact(filepath):
- if not is_cygwin:
- return filepath
- output, _ = popen(['cygpath', '-w', filepath]).communicate()
- # pylint: disable=E1103
- return output.strip().replace('\\', '\\\\')
-
-
-SCRIPTS_PATH = path.dirname(path.abspath(__file__))
-DEVTOOLS_PATH = path.dirname(SCRIPTS_PATH)
-ROOT_PATH = devtools_paths.root_path()
-BROWSER_PROTOCOL_PATH = path.join(ROOT_PATH, 'third_party', 'blink', 'renderer', 'core', 'inspector', 'browser_protocol.pdl')
-# TODO(dgozman): move these checks to v8.
-JS_PROTOCOL_PATH = path.join(ROOT_PATH, 'v8', 'include', 'js_protocol.pdl')
-DEVTOOLS_FRONTEND_PATH = path.join(DEVTOOLS_PATH, 'front_end')
-GLOBAL_EXTERNS_FILE = to_platform_path(path.join(DEVTOOLS_FRONTEND_PATH, 'externs.js'))
-DEFAULT_PROTOCOL_EXTERNS_FILE = path.join(DEVTOOLS_FRONTEND_PATH, 'protocol_externs.js')
-RUNTIME_FILE = to_platform_path(path.join(DEVTOOLS_FRONTEND_PATH, 'Runtime.js'))
-ROOT_MODULE_FILE = to_platform_path(path.join(DEVTOOLS_FRONTEND_PATH, 'root.js'))
-
-CLOSURE_COMPILER_JAR = to_platform_path(path.join(SCRIPTS_PATH, 'closure', 'compiler.jar'))
-CLOSURE_RUNNER_JAR = to_platform_path(path.join(SCRIPTS_PATH, 'closure', 'closure_runner', 'closure_runner.jar'))
-JSDOC_VALIDATOR_JAR = to_platform_path(path.join(SCRIPTS_PATH, 'jsdoc_validator', 'jsdoc_validator.jar'))
-
-TYPE_CHECKED_JSDOC_TAGS_LIST = ['param', 'return', 'type', 'enum']
-TYPE_CHECKED_JSDOC_TAGS_OR = '|'.join(TYPE_CHECKED_JSDOC_TAGS_LIST)
-
-# Basic regex for invalid JsDoc types: an object type name ([A-Z][_A-Za-z0-9.]+[A-Za-z0-9]) not preceded by '!', '?', ':' (this, new), or '.' (object property).
-INVALID_TYPE_REGEX = re.compile(r'@(?:' + TYPE_CHECKED_JSDOC_TAGS_OR +
- r')\s*\{.*(?<![!?:._A-Za-z0-9])([A-Z][_A-Za-z0-9.]+[A-Za-z0-9])[^/]*\}')
-INVALID_TYPE_DESIGNATOR_REGEX = re.compile(r'@(?:' + TYPE_CHECKED_JSDOC_TAGS_OR + r')\s*.*(?<![{: ])([?!])=?\}')
-INVALID_NON_OBJECT_TYPE_REGEX = re.compile(r'@(?:' + TYPE_CHECKED_JSDOC_TAGS_OR + r')\s*\{.*(![a-z]+)[^/]*\}')
-ERROR_WARNING_REGEX = re.compile(r'WARNING|ERROR')
-LOADED_CSS_REGEX = re.compile(r'(?:registerRequiredCSS|WebInspector\.View\.createStyleElement)\s*\(\s*"(.+)"\s*\)')
-
-JAVA_BUILD_REGEX = re.compile(r'\w+ version "(\d+)\.(\d+)')
-
-
-def log_error(message):
- print 'ERROR: ' + message
-
-
-def error_excepthook(exctype, value, traceback):
- print 'ERROR:'
- sys.__excepthook__(exctype, value, traceback)
-
-
-sys.excepthook = error_excepthook
-
-APPLICATION_DESCRIPTORS = [
- 'inspector',
- 'toolbox',
- 'integration_test_runner',
- 'formatter_worker',
- 'heap_snapshot_worker',
-]
-
-SKIPPED_NAMESPACES = {
- 'Console', # Closure uses Console as a namespace item so we cannot override it right now.
- 'Gonzales', # third party module defined in front_end/externs.js
- 'Terminal', # third party module defined in front_end/externs.js
-}
-
-
-def has_errors(output):
- return re.search(ERROR_WARNING_REGEX, output) is not None
-
-
-class JSDocChecker:
-
- def __init__(self, descriptors, java_exec):
- self._error_found = False
- self._all_files = descriptors.all_compiled_files()
- self._java_exec = java_exec
-
- def check(self):
- print 'Verifying JSDoc comments...'
- self._verify_jsdoc()
- self._run_jsdoc_validator()
- return self._error_found
-
- def _run_jsdoc_validator(self):
- files = [to_platform_path(f) for f in self._all_files]
- file_list = tempfile.NamedTemporaryFile(mode='wt', delete=False)
- try:
- file_list.write('\n'.join(files))
- finally:
- file_list.close()
- proc = popen(self._java_exec + ['-jar', JSDOC_VALIDATOR_JAR, '--files-list-name', to_platform_path_exact(file_list.name)])
- (out, _) = proc.communicate()
- if out:
- print('JSDoc validator output:%s%s' % (os.linesep, out))
- self._error_found = True
- os.remove(file_list.name)
-
- def _verify_jsdoc(self):
- for full_file_name in self._all_files:
- line_index = 0
- with open(full_file_name, 'r') as sourceFile:
- for line in sourceFile:
- line_index += 1
- if line.rstrip():
- self._verify_jsdoc_line(full_file_name, line_index, line)
-
- def _verify_jsdoc_line(self, file_name, line_index, line):
-
- def print_error(message, error_position):
- print '%s:%s: ERROR - %s%s%s%s%s%s' % (file_name, line_index, message, os.linesep, line, os.linesep,
- ' ' * error_position + '^', os.linesep)
-
- known_css = {}
- match = re.search(INVALID_TYPE_REGEX, line)
- if match:
- print_error('Type "%s" nullability not marked explicitly with "?" (nullable) or "!" (non-nullable)' % match.group(1),
- match.start(1))
- self._error_found = True
-
- match = re.search(INVALID_NON_OBJECT_TYPE_REGEX, line)
- if match:
- print_error('Non-object type explicitly marked with "!" (non-nullable), which is the default and should be omitted',
- match.start(1))
- self._error_found = True
-
- match = re.search(INVALID_TYPE_DESIGNATOR_REGEX, line)
- if match:
- print_error('Type nullability indicator misplaced, should precede type', match.start(1))
- self._error_found = True
-
- match = re.search(LOADED_CSS_REGEX, line)
- if match:
- css_file = path.join(DEVTOOLS_FRONTEND_PATH, match.group(1))
- exists = known_css.get(css_file)
- if exists is None:
- exists = path.isfile(css_file)
- known_css[css_file] = exists
- if not exists:
- print_error('Dynamically loaded CSS stylesheet is missing in the source tree', match.start(1))
- self._error_found = True
-
-
-def find_java():
- required_major = 1
- required_minor = 7
- exec_command = None
- has_server_jvm = True
- java_path = utils.which('java')
-
- if not java_path:
- print 'NOTE: No Java executable found in $PATH.'
- sys.exit(1)
-
- is_ok = False
- java_version_out, _ = popen([java_path, '-version']).communicate()
- # pylint: disable=E1103
- match = re.search(JAVA_BUILD_REGEX, java_version_out)
- if match:
- major = int(match.group(1))
- minor = int(match.group(2))
- is_ok = major > required_major or major == required_major and minor >= required_minor
- if is_ok:
- exec_command = [java_path, '-Xms1024m', '-server', '-XX:+TieredCompilation']
- check_server_proc = popen(exec_command + ['-version'])
- check_server_proc.communicate()
- if check_server_proc.returncode != 0:
- # Not all Java installs have server JVMs.
- exec_command = exec_command.remove('-server')
- has_server_jvm = False
-
- if not is_ok:
- print 'NOTE: Java executable version %d.%d or above not found in $PATH.' % (required_major, required_minor)
- sys.exit(1)
- print 'Java executable: %s%s' % (java_path, '' if has_server_jvm else ' (no server JVM)')
- return exec_command
-
-
-common_closure_args = [
- '--summary_detail_level',
- '3',
- '--jscomp_error',
- 'visibility',
- '--jscomp_warning',
- 'missingOverride',
- '--compilation_level',
- 'SIMPLE_OPTIMIZATIONS',
- '--warning_level',
- 'VERBOSE',
- '--language_in=ECMASCRIPT_NEXT',
- '--language_out=ES5_STRICT',
- '--extra_annotation_name',
- 'suppressReceiverCheck',
- '--extra_annotation_name',
- 'suppressGlobalPropertiesCheck',
- '--checks-only',
-]
-
-GENERATED_SKIP_COMPILATION_FILES = [
- 'SupportedCSSProperties.js',
- 'InspectorBackendCommands.js',
-]
-
-
-def check_conditional_dependencies(modules_by_name):
- errors_found = False
- for name in modules_by_name:
- if 'test_runner' in name:
- continue
- for dep_name in modules_by_name[name].get('dependencies', []):
- dependency = modules_by_name[dep_name]
- if dependency.get('experiment') or dependency.get('condition'):
- log_error('Module "%s" may not depend on the conditional module "%s"' % (name, dep_name))
- errors_found = True
- return errors_found
-
-
-def prepare_closure_frontend_compile(temp_devtools_path, descriptors, namespace_externs_path, protocol_externs_file):
- temp_frontend_path = path.join(temp_devtools_path, 'front_end')
- checker = dependency_preprocessor.DependencyPreprocessor(descriptors, temp_frontend_path, DEVTOOLS_FRONTEND_PATH)
- checker.enforce_dependencies()
-
- command = common_closure_args + [
- '--externs',
- to_platform_path(GLOBAL_EXTERNS_FILE),
- '--externs',
- namespace_externs_path,
- '--js',
- RUNTIME_FILE,
- '--js',
- ROOT_MODULE_FILE,
- ]
-
- all_files = descriptors.all_compiled_files()
- args = []
- for file in all_files:
- args.extend(['--js', file])
- if "InspectorBackend.js" in file:
- args.extend(['--js', protocol_externs_file])
-
- for file in GENERATED_SKIP_COMPILATION_FILES:
- # Write a dummy file for skipped compilation files that are autogenerated.
- # We don't type-check this file, but we import them via ES modules
- generated_file = path.join(temp_frontend_path, file)
- modular_build.write_file(generated_file, '')
- args.extend(['--js', generated_file])
-
- command += args
- command = [arg.replace(DEVTOOLS_FRONTEND_PATH, temp_frontend_path) for arg in command]
- compiler_args_file = tempfile.NamedTemporaryFile(mode='wt', delete=False)
- try:
- compiler_args_file.write('devtools_frontend %s' % (' '.join(command)))
- finally:
- compiler_args_file.close()
- return compiler_args_file.name
-
-
-def generate_namespace_externs(modules_by_name):
- special_case_namespaces_path = path.join(path.dirname(path.abspath(__file__)), 'special_case_namespaces.json')
- with open(special_case_namespaces_path) as json_file:
- special_case_namespaces = json.load(json_file)
-
- def map_module_to_namespace(module):
- return special_case_namespaces.get(module, to_camel_case(module))
-
- def to_camel_case(snake_string):
- components = snake_string.split('_')
- return ''.join(x.title() for x in components)
-
- all_namespaces = [map_module_to_namespace(module) for module in modules_by_name]
- namespaces = [namespace for namespace in all_namespaces if namespace not in SKIPPED_NAMESPACES]
- namespaces.sort()
- namespace_externs_file = tempfile.NamedTemporaryFile(mode='wt', delete=False)
- try:
- namespace_externs_file.write('var Root = {};\n')
- for namespace in namespaces:
- namespace_externs_file.write('var %s = {};\n' % namespace)
- finally:
- namespace_externs_file.close()
- namespace_externs_path = to_platform_path(namespace_externs_file.name)
- return namespace_externs_path
-
-
-def main():
- protocol_externs_file = DEFAULT_PROTOCOL_EXTERNS_FILE
- errors_found = False
- parser = argparse.ArgumentParser()
- parser.add_argument('--protocol-externs-file')
- args, _ = parser.parse_known_args()
- if args.protocol_externs_file:
- protocol_externs_file = args.protocol_externs_file
- else:
- generate_protocol_externs.generate_protocol_externs(protocol_externs_file, BROWSER_PROTOCOL_PATH, JS_PROTOCOL_PATH)
- loader = modular_build.DescriptorLoader(DEVTOOLS_FRONTEND_PATH)
- descriptors = loader.load_applications(APPLICATION_DESCRIPTORS)
- modules_by_name = descriptors.modules
-
- java_exec = find_java()
- errors_found |= check_conditional_dependencies(modules_by_name)
-
- print 'Compiling frontend...'
- temp_devtools_path = tempfile.mkdtemp()
- namespace_externs_path = generate_namespace_externs(modules_by_name)
- compiler_args_file_path = prepare_closure_frontend_compile(temp_devtools_path, descriptors, namespace_externs_path,
- protocol_externs_file)
- frontend_compile_proc = popen(
- java_exec + ['-jar', CLOSURE_RUNNER_JAR, '--compiler-args-file',
- to_platform_path_exact(compiler_args_file_path)])
-
- print 'Compiling devtools_compatibility.js...'
-
- closure_compiler_command = java_exec + ['-jar', CLOSURE_COMPILER_JAR] + common_closure_args
-
- devtools_js_compile_command = closure_compiler_command + [
- '--externs',
- to_platform_path(GLOBAL_EXTERNS_FILE), '--jscomp_off=externsValidation', '--js',
- to_platform_path(path.join(DEVTOOLS_FRONTEND_PATH, 'devtools_compatibility.js'))
- ]
- devtools_js_compile_proc = popen(devtools_js_compile_command)
-
- errors_found |= JSDocChecker(descriptors, java_exec).check()
-
- (devtools_js_compile_out, _) = devtools_js_compile_proc.communicate()
- print 'devtools_compatibility.js compilation output:%s' % os.linesep, devtools_js_compile_out
- errors_found |= has_errors(devtools_js_compile_out)
-
- (frontend_compile_out, _) = frontend_compile_proc.communicate()
- print 'devtools frontend compilation output:'
- for line in frontend_compile_out.splitlines():
- if "@@ START_MODULE" in line or "@@ END_MODULE" in line:
- continue
- print line
- errors_found |= has_errors(frontend_compile_out)
-
- os.remove(protocol_externs_file)
- os.remove(namespace_externs_path)
- os.remove(compiler_args_file_path)
- shutil.rmtree(temp_devtools_path, True)
-
- if errors_found:
- print 'ERRORS DETECTED'
- sys.exit(1)
- print 'DONE - compiled without errors'
-
-
-if __name__ == "__main__":
- main()
+run_type_check.main()