diff --git a/.circleci/config.yml b/.circleci/config.yml index 587fd502e1..d8cb06589b 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -26,12 +26,14 @@ env-release-build: &env-release-build GN_CONFIG: //electron/build/args/release.gn env-browsertests: &env-browsertests + GN_EXTRA_ARGS: 'is_component_ffmpeg = false' BUILD_TARGET: electron:chromium_browsertests - TESTS_CONFIG: src/electron/script/browsertests.yml + TESTS_CONFIG: src/electron/spec/configs/browsertests.yml env-unittests: &env-unittests + GN_EXTRA_ARGS: 'is_component_ffmpeg = false' BUILD_TARGET: electron:chromium_unittests - TESTS_CONFIG: src/electron/script/unittests.yml + TESTS_CONFIG: src/electron/spec/configs/unittests.yml # Build targets options. env-ia32: &env-ia32 @@ -948,8 +950,9 @@ workflows: - linux-arm-checkout - linux-arm64-checkout -# TODO(alexeykuzmin): The only suite there fails to start, so skip -# the job for now. Fix the "content_browsertests" and enable the job back. + # TODO(alexeykuzmin): Enable it back. + # Tons of crashes right now, see + # https://circleci.com/gh/electron/electron/67463 # - linux-x64-browsertests: # requires: # - linux-checkout diff --git a/script/lib/native_tests.py b/script/lib/native_tests.py new file mode 100644 index 0000000000..119d1f5516 --- /dev/null +++ b/script/lib/native_tests.py @@ -0,0 +1,274 @@ +from __future__ import print_function + +import os +import subprocess +import sys + +SOURCE_ROOT = os.path.abspath( + os.path.dirname(os.path.dirname(os.path.dirname(__file__)))) +VENDOR_DIR = os.path.join(SOURCE_ROOT, 'vendor') +PYYAML_LIB_DIR = os.path.join(VENDOR_DIR, 'pyyaml', 'lib') +sys.path.append(PYYAML_LIB_DIR) +import yaml #pylint: disable=wrong-import-position,wrong-import-order + + +class Verbosity: + CHATTY = 'chatty' # stdout and stderr + ERRORS = 'errors' # stderr only + SILENT = 'silent' # no output + + @staticmethod + def get_all(): + return Verbosity.__get_all_in_order() + + @staticmethod + def __get_all_in_order(): + return [Verbosity.SILENT, Verbosity.ERRORS, Verbosity.CHATTY] + + @staticmethod + def __get_indices(*values): + ordered = Verbosity.__get_all_in_order() + indices = map(ordered.index, values) + return indices + + @staticmethod + def ge(a, b): + """Greater or equal""" + a_index, b_index = Verbosity.__get_indices(a, b) + return a_index >= b_index + + @staticmethod + def le(a, b): + """Less or equal""" + a_index, b_index = Verbosity.__get_indices(a, b) + return a_index <= b_index + + +class Platform: + LINUX = 'linux' + MAC = 'mac' + WINDOWS = 'windows' + + @staticmethod + def get_current(): + platform = sys.platform + if platform in ('linux', 'linux2'): + return Platform.LINUX + if platform == 'darwin': + return Platform.MAC + if platform in ('cygwin', 'win32'): + return Platform.WINDOWS + + assert False, "unexpected current platform '{}'".format(platform) + + @staticmethod + def get_all(): + return [Platform.LINUX, Platform.MAC, Platform.WINDOWS] + + @staticmethod + def is_valid(platform): + return platform in Platform.get_all() + + +class TestsList(): + def __init__(self, config_path, tests_dir): + self.config_path = config_path + self.tests_dir = tests_dir + + # A dict with binary names (e.g. 'base_unittests') as keys + # and various test data as values of dict type. + self.tests = TestsList.__get_tests_list(config_path) + + def __len__(self): + return len(self.tests) + + def get_for_current_platform(self): + all_binaries = self.tests.keys() + supported_binaries = filter(self.__platform_supports, all_binaries) + return supported_binaries + + def run(self, binaries, output_dir=None, verbosity=Verbosity.CHATTY): + # Don't run anything twice. + binaries = set(binaries) + + # First check that all names are present in the config. + for binary_name in binaries: + if binary_name not in self.tests: + raise Exception("binary {0} not found in config '{1}'".format( + binary_name, self.config_path)) + + # Respect the "platform" setting. + for binary_name in binaries: + if not self.__platform_supports(binary_name): + raise Exception( + "binary {0} cannot be run on {1}, check the config".format( + binary_name, Platform.get_current())) + + suite_returncode = sum( + [self.__run(binary, output_dir, verbosity) for binary in binaries]) + return suite_returncode + + def run_only(self, binary_name, output_dir=None, verbosity=Verbosity.CHATTY): + return self.run([binary_name], output_dir, verbosity) + + def run_all(self, output_dir=None, verbosity=Verbosity.CHATTY): + return self.run(self.get_for_current_platform(), output_dir, verbosity) + + @staticmethod + def __get_tests_list(config_path): + tests_list = {} + config_data = TestsList.__get_config_data(config_path) + + for data_item in config_data['tests']: + (binary_name, test_data) = TestsList.__get_test_data(data_item) + tests_list[binary_name] = test_data + + return tests_list + + @staticmethod + def __get_config_data(config_path): + with open(config_path, 'r') as stream: + return yaml.load(stream) + + @staticmethod + def __expand_shorthand(value): + """ Treat a string as {'string_value': None}.""" + if isinstance(value, dict): + return value + + if isinstance(value, basestring): + return {value: None} + + assert False, "unexpected shorthand type: {}".format(type(value)) + + @staticmethod + def __make_a_list(value): + """Make a list if not already a list.""" + if isinstance(value, list): + return value + return [value] + + @staticmethod + def __merge_nested_lists(value): + """Converts a dict of lists to a list.""" + if isinstance(value, list): + return value + + if isinstance(value, dict): + # It looks ugly as hell, but it does the job. + return [list_item for key in value for list_item in value[key]] + + assert False, "unexpected type for list merging: {}".format(type(value)) + + def __platform_supports(self, binary_name): + return Platform.get_current() in self.tests[binary_name]['platforms'] + + @staticmethod + def __get_test_data(data_item): + data_item = TestsList.__expand_shorthand(data_item) + + binary_name = data_item.keys()[0] + test_data = { + 'excluded_tests': None, + 'platforms': Platform.get_all() + } + + configs = data_item[binary_name] + if configs is not None: + # List of excluded tests. + if 'disabled' in configs: + excluded_tests = TestsList.__merge_nested_lists(configs['disabled']) + test_data['excluded_tests'] = excluded_tests + + # List of platforms to run the tests on. + if 'platform' in configs: + platforms = TestsList.__make_a_list(configs['platform']) + + for platform in platforms: + assert Platform.is_valid(platform), \ + "platform '{0}' is not supported, check {1} config" \ + .format(platform, binary_name) + + test_data['platforms'] = platforms + + return (binary_name, test_data) + + def __run(self, binary_name, output_dir, verbosity): + binary_path = os.path.join(self.tests_dir, binary_name) + test_binary = TestBinary(binary_path) + + test_data = self.tests[binary_name] + excluded_tests = test_data['excluded_tests'] + + output_file_path = TestsList.__get_output_path(binary_name, output_dir) + + return test_binary.run(excluded_tests=excluded_tests, + output_file_path=output_file_path, + verbosity=verbosity) + + @staticmethod + def __get_output_path(binary_name, output_dir=None): + if output_dir is None: + return None + + return os.path.join(output_dir, "results_{}.xml".format(binary_name)) + + +class TestBinary(): + # Is only used when writing to a file. + output_format = 'xml' + + def __init__(self, binary_path): + self.binary_path = binary_path + + def run(self, excluded_tests=None, output_file_path=None, + verbosity=Verbosity.CHATTY): + gtest_filter = TestBinary.__get_gtest_filter(excluded_tests) + gtest_output = TestBinary.__get_gtest_output(output_file_path) + + args = [self.binary_path, gtest_filter, gtest_output] + stdout, stderr = TestBinary.__get_stdout_and_stderr(verbosity) + + returncode = 0 + try: + returncode = subprocess.call(args, stdout=stdout, stderr=stderr) + except Exception as exception: + if Verbosity.ge(verbosity, Verbosity.ERRORS): + print("An error occurred while running '{}':".format(self.binary_path), + '\n', exception, file=sys.stderr) + returncode = 1 + + return returncode + + @staticmethod + def __get_gtest_filter(excluded_tests): + gtest_filter = "" + if excluded_tests is not None and len(excluded_tests) > 0: + excluded_tests_string = TestBinary.__format_excluded_tests( + excluded_tests) + gtest_filter = "--gtest_filter={}".format(excluded_tests_string) + return gtest_filter + + @staticmethod + def __get_gtest_output(output_file_path): + gtest_output = "" + if output_file_path is not None: + gtest_output = "--gtest_output={0}:{1}".format(TestBinary.output_format, + output_file_path) + return gtest_output + + @staticmethod + def __format_excluded_tests(excluded_tests): + return "-" + ":".join(excluded_tests) + + @staticmethod + def __get_stdout_and_stderr(verbosity): + stdout = stderr = None + + if Verbosity.le(verbosity, Verbosity.ERRORS): + devnull = open(os.devnull, 'w') + stdout = devnull + if verbosity == Verbosity.SILENT: + stderr = devnull + + return (stdout, stderr) diff --git a/script/native-tests.py b/script/native-tests.py index 7b4f05492b..cb98e93543 100755 --- a/script/native-tests.py +++ b/script/native-tests.py @@ -4,53 +4,15 @@ from __future__ import print_function import argparse import os -import subprocess import sys -SOURCE_ROOT = os.path.abspath(os.path.dirname(os.path.dirname(__file__))) -VENDOR_DIR = os.path.join(SOURCE_ROOT, 'vendor') -PYYAML_LIB_DIR = os.path.join(VENDOR_DIR, 'pyyaml', 'lib') -sys.path.append(PYYAML_LIB_DIR) -import yaml #pylint: disable=wrong-import-position,wrong-import-order +from lib.native_tests import TestsList, Verbosity class Command: LIST = 'list' RUN = 'run' -class Verbosity: - CHATTY = 'chatty' # stdout and stderr - ERRORS = 'errors' # stderr only - SILENT = 'silent' # no output - - @staticmethod - def get_all(): - return [Verbosity.CHATTY, Verbosity.ERRORS, Verbosity.SILENT] - -class Platform: - LINUX = 'linux' - MAC = 'mac' - WINDOWS = 'windows' - - @staticmethod - def get_current(): - platform = sys.platform - if platform in ('linux', 'linux2'): - return Platform.LINUX - if platform == 'darwin': - return Platform.MAC - if platform in ('cygwin', 'win32'): - return Platform.WINDOWS - - assert False, "unexpected current platform '{}'".format(platform) - - @staticmethod - def get_all(): - return [Platform.LINUX, Platform.MAC, Platform.WINDOWS] - - @staticmethod - def is_valid(platform): - return platform in Platform.get_all() def parse_args(): parser = argparse.ArgumentParser(description='Run Google Test binaries') @@ -127,202 +89,5 @@ def main(): assert False, "unexpected command '{}'".format(args.command) -class TestsList(): - def __init__(self, config_path, tests_dir): - self.config_path = config_path - self.tests_dir = tests_dir - - # A dict with binary names (e.g. 'base_unittests') as keys - # and various test data as values of dict type. - self.tests = TestsList.__get_tests_list(config_path) - - def __len__(self): - return len(self.tests) - - def get_for_current_platform(self): - all_binaries = self.tests.keys() - - supported_binaries = filter(self.__platform_supports, all_binaries) - - return supported_binaries - - def run(self, binaries, output_dir=None, verbosity=Verbosity.CHATTY): - # Don't run anything twice. - binaries = set(binaries) - - # First check that all names are present in the config. - for binary_name in binaries: - if binary_name not in self.tests: - raise Exception("binary {0} not found in config '{1}'".format( - binary_name, self.config_path)) - - # Respect the "platform" setting. - for binary_name in binaries: - if not self.__platform_supports(binary_name): - raise Exception( - "binary {0} cannot be run on {1}, check the config".format( - binary_name, Platform.get_current())) - - suite_returncode = sum( - [self.__run(binary, output_dir, verbosity) for binary in binaries]) - return suite_returncode - - def run_only(self, binary_name, output_dir=None, verbosity=Verbosity.CHATTY): - return self.run([binary_name], output_dir, verbosity) - - def run_all(self, output_dir=None, verbosity=Verbosity.CHATTY): - return self.run(self.get_for_current_platform(), output_dir, verbosity) - - @staticmethod - def __get_tests_list(config_path): - tests_list = {} - config_data = TestsList.__get_config_data(config_path) - - for data_item in config_data['tests']: - (binary_name, test_data) = TestsList.__get_test_data(data_item) - tests_list[binary_name] = test_data - - return tests_list - - @staticmethod - def __get_config_data(config_path): - with open(config_path, 'r') as stream: - return yaml.load(stream) - - @staticmethod - def __expand_shorthand(value): - """ Treat a string as {'string_value': None}.""" - if isinstance(value, dict): - return value - - if isinstance(value, basestring): - return {value: None} - - assert False, "unexpected shorthand type: {}".format(type(value)) - - @staticmethod - def __make_a_list(value): - """Make a list if not already a list.""" - if isinstance(value, list): - return value - return [value] - - @staticmethod - def __merge_nested_lists(value): - """Converts a dict of lists to a list.""" - if isinstance(value, list): - return value - - if isinstance(value, dict): - # It looks ugly as hell, but it does the job. - return [list_item for key in value for list_item in value[key]] - - assert False, "unexpected type for list merging: {}".format(type(value)) - - def __platform_supports(self, binary_name): - return Platform.get_current() in self.tests[binary_name]['platforms'] - - @staticmethod - def __get_test_data(data_item): - data_item = TestsList.__expand_shorthand(data_item) - - binary_name = data_item.keys()[0] - test_data = { - 'excluded_tests': None, - 'platforms': Platform.get_all() - } - - configs = data_item[binary_name] - if configs is not None: - # List of excluded tests. - if 'disabled' in configs: - excluded_tests = TestsList.__merge_nested_lists(configs['disabled']) - test_data['excluded_tests'] = excluded_tests - - # List of platforms to run the tests on. - if 'platform' in configs: - platforms = TestsList.__make_a_list(configs['platform']) - - for platform in platforms: - assert Platform.is_valid(platform), \ - "platform '{0}' is not supported, check {1} config" \ - .format(platform, binary_name) - - test_data['platforms'] = platforms - - return (binary_name, test_data) - - def __run(self, binary_name, output_dir, verbosity): - binary_path = os.path.join(self.tests_dir, binary_name) - test_binary = TestBinary(binary_path) - - test_data = self.tests[binary_name] - excluded_tests = test_data['excluded_tests'] - - output_file_path = TestsList.__get_output_path(binary_name, output_dir) - - return test_binary.run(excluded_tests=excluded_tests, - output_file_path=output_file_path, - verbosity=verbosity) - - @staticmethod - def __get_output_path(binary_name, output_dir=None): - if output_dir is None: - return None - - return os.path.join(output_dir, "results_{}.xml".format(binary_name)) - - -class TestBinary(): - # Is only used when writing to a file. - output_format = 'xml' - - def __init__(self, binary_path): - self.binary_path = binary_path - - def run(self, excluded_tests=None, output_file_path=None, - verbosity=Verbosity.CHATTY): - gtest_filter = "" - if excluded_tests is not None and len(excluded_tests) > 0: - excluded_tests_string = TestBinary.__format_excluded_tests( - excluded_tests) - gtest_filter = "--gtest_filter={}".format(excluded_tests_string) - - gtest_output = "" - if output_file_path is not None: - gtest_output = "--gtest_output={0}:{1}".format(TestBinary.output_format, - output_file_path) - - args = [self.binary_path, gtest_filter, gtest_output] - stdout, stderr = TestBinary.__get_stdout_and_stderr(verbosity) - - returncode = 0 - try: - returncode = subprocess.call(args, stdout=stdout, stderr=stderr) - except Exception as exception: - if verbosity in (Verbosity.CHATTY, Verbosity.ERRORS): - print("An error occurred while running '{}':".format(self.binary_path), - '\n', exception, file=sys.stderr) - returncode = 1 - - return returncode - - @staticmethod - def __format_excluded_tests(excluded_tests): - return "-" + ":".join(excluded_tests) - - @staticmethod - def __get_stdout_and_stderr(verbosity): - stdout = stderr = None - - if verbosity in (Verbosity.ERRORS, Verbosity.SILENT): - devnull = open(os.devnull, 'w') - stdout = devnull - if verbosity == Verbosity.SILENT: - stderr = devnull - - return (stdout, stderr) - - if __name__ == '__main__': sys.exit(main()) diff --git a/script/browsertests.yml b/spec/configs/browsertests.yml similarity index 100% rename from script/browsertests.yml rename to spec/configs/browsertests.yml diff --git a/script/unittests.yml b/spec/configs/unittests.yml similarity index 77% rename from script/unittests.yml rename to spec/configs/unittests.yml index 675d318abb..825cb13faf 100644 --- a/script/unittests.yml +++ b/spec/configs/unittests.yml @@ -15,29 +15,24 @@ tests: - RTLTest* - SysStrings* - UTFOffsetStringConversionsTest* -# TODO(alexeykuzmin): The binary can't be launched, fix it and enable it back. -# - cc_unittests -# TODO(alexeykuzmin): The binary can't be launched, fix it and enable it back. -# - cc_blink_unittests -# TODO(alexeykuzmin): The binary can't be launched, fix it and enable it back. -# - content_unittests: -# disabled: + - cc_unittests + - cc_blink_unittests + - content_unittests: + disabled: # TODO(alexeykuzmin): Should those be fixed? -# - _/DOMStorageMapParamTest.EnforcesQuota/0 -# - _/DOMStorageMapParamTest.EnforcesQuota/1 + - _/DOMStorageMapParamTest.EnforcesQuota/0 + - _/DOMStorageMapParamTest.EnforcesQuota/1 - crypto_unittests - device_unittests - gin_unittests - gpu_unittests - ipc_tests -# TODO(alexeykuzmin): The binary can't be launched, fix it and enable it back. -# - media_unittests + - media_unittests - capture_unittests - midi_unittests - media_mojo_unittests - mojo_unittests -# TODO(alexeykuzmin): The binary can't be launched, fix it and enable it back. -# - mojo_common_unittests + - mojo_common_unittests - net_unittests: disabled: # TODO(alexeykuzmin): Should those be fixed? @@ -85,10 +80,9 @@ tests: # TODO(alexeykuzmin): Should it be fixed? - LiveRangeUnitTest* # Blink -# TODO(alexeykuzmin): Doesn't exist in Ch67. +# TODO: Enable in Ch68. # - blink_common_unittests -# TODO(alexeykuzmin): The binary can't be launched, fix it and enable it back. -# - blink_heap_unittests + - blink_heap_unittests - blink_platform_unittests: disabled: # TODO(alexeykuzmin): Should those be fixed? @@ -108,8 +102,7 @@ tests: - PNGTests* - StaticPNGTests* - StaticWebPTests* -# TODO(alexeykuzmin): The binary can't be launched, fix it and enable it back. -# - webkit_unit_tests + - webkit_unit_tests - wtf_unittests # Third party - angle_unittests @@ -120,7 +113,13 @@ tests: - sandbox_linux_unittests: platform: linux disabled: - # TODO(alexeykuzmin): Should it be fixed? + # TODO(alexeykuzmin): Should these be fixed? - BrokerProcess* + - SandboxBPF.SigBus - dbus_unittests: platform: linux + disabled: + # TODO(alexeykuzmin): Should these be fixed? + - EndToEndAsyncTest.InvalidServiceName + - EndToEndSyncTest.InvalidServiceName + - MessageTest.SetInvalidHeaders