16ebf5866SFelix Guo#!/usr/bin/python3 26ebf5866SFelix Guo# SPDX-License-Identifier: GPL-2.0 36ebf5866SFelix Guo# 46ebf5866SFelix Guo# A thin wrapper on top of the KUnit Kernel 56ebf5866SFelix Guo# 66ebf5866SFelix Guo# Copyright (C) 2019, Google LLC. 76ebf5866SFelix Guo# Author: Felix Guo <felixguoxiuping@gmail.com> 86ebf5866SFelix Guo# Author: Brendan Higgins <brendanhiggins@google.com> 96ebf5866SFelix Guo 106ebf5866SFelix Guoimport argparse 116ebf5866SFelix Guoimport sys 126ebf5866SFelix Guoimport os 136ebf5866SFelix Guoimport time 14ff7b437fSBrendan Higginsimport shutil 156ebf5866SFelix Guo 166ebf5866SFelix Guofrom collections import namedtuple 176ebf5866SFelix Guofrom enum import Enum, auto 186ebf5866SFelix Guo 196ebf5866SFelix Guoimport kunit_config 206ebf5866SFelix Guoimport kunit_kernel 216ebf5866SFelix Guoimport kunit_parser 226ebf5866SFelix Guo 2345ba7a89SDavid GowKunitResult = namedtuple('KunitResult', ['status','result','elapsed_time']) 246ebf5866SFelix Guo 2545ba7a89SDavid GowKunitConfigRequest = namedtuple('KunitConfigRequest', 2601397e82SVitor Massaru Iha ['build_dir', 'make_options']) 2745ba7a89SDavid GowKunitBuildRequest = namedtuple('KunitBuildRequest', 2845ba7a89SDavid Gow ['jobs', 'build_dir', 'alltests', 2945ba7a89SDavid Gow 'make_options']) 3045ba7a89SDavid GowKunitExecRequest = namedtuple('KunitExecRequest', 3145ba7a89SDavid Gow ['timeout', 'build_dir', 'alltests']) 3245ba7a89SDavid GowKunitParseRequest = namedtuple('KunitParseRequest', 3345ba7a89SDavid Gow ['raw_output', 'input_data']) 34021ed9f5SHeidi FahimKunitRequest = namedtuple('KunitRequest', ['raw_output','timeout', 'jobs', 359bdf64b3SVitor Massaru Iha 'build_dir', 'alltests', 369bdf64b3SVitor Massaru Iha 'make_options']) 376ebf5866SFelix Guo 38be886ba9SHeidi FahimKernelDirectoryPath = sys.argv[0].split('tools/testing/kunit/')[0] 39be886ba9SHeidi Fahim 406ebf5866SFelix Guoclass KunitStatus(Enum): 416ebf5866SFelix Guo SUCCESS = auto() 426ebf5866SFelix Guo CONFIG_FAILURE = auto() 436ebf5866SFelix Guo BUILD_FAILURE = auto() 446ebf5866SFelix Guo TEST_FAILURE = auto() 456ebf5866SFelix Guo 46ff7b437fSBrendan Higginsdef create_default_kunitconfig(): 47e3212513SSeongJae Park if not os.path.exists(kunit_kernel.kunitconfig_path): 48ff7b437fSBrendan Higgins shutil.copyfile('arch/um/configs/kunit_defconfig', 49e3212513SSeongJae Park kunit_kernel.kunitconfig_path) 50ff7b437fSBrendan Higgins 51be886ba9SHeidi Fahimdef get_kernel_root_path(): 52be886ba9SHeidi Fahim parts = sys.argv[0] if not __file__ else __file__ 53be886ba9SHeidi Fahim parts = os.path.realpath(parts).split('tools/testing/kunit') 54be886ba9SHeidi Fahim if len(parts) != 2: 55be886ba9SHeidi Fahim sys.exit(1) 56be886ba9SHeidi Fahim return parts[0] 57be886ba9SHeidi Fahim 5845ba7a89SDavid Gowdef config_tests(linux: kunit_kernel.LinuxSourceTree, 5945ba7a89SDavid Gow request: KunitConfigRequest) -> KunitResult: 6045ba7a89SDavid Gow kunit_parser.print_with_timestamp('Configuring KUnit Kernel ...') 6145ba7a89SDavid Gow 626ebf5866SFelix Guo config_start = time.time() 6345ba7a89SDavid Gow create_default_kunitconfig() 640476e69fSGreg Thelen success = linux.build_reconfig(request.build_dir, request.make_options) 656ebf5866SFelix Guo config_end = time.time() 666ebf5866SFelix Guo if not success: 6745ba7a89SDavid Gow return KunitResult(KunitStatus.CONFIG_FAILURE, 6845ba7a89SDavid Gow 'could not configure kernel', 6945ba7a89SDavid Gow config_end - config_start) 7045ba7a89SDavid Gow return KunitResult(KunitStatus.SUCCESS, 7145ba7a89SDavid Gow 'configured kernel successfully', 7245ba7a89SDavid Gow config_end - config_start) 736ebf5866SFelix Guo 7445ba7a89SDavid Gowdef build_tests(linux: kunit_kernel.LinuxSourceTree, 7545ba7a89SDavid Gow request: KunitBuildRequest) -> KunitResult: 766ebf5866SFelix Guo kunit_parser.print_with_timestamp('Building KUnit Kernel ...') 776ebf5866SFelix Guo 786ebf5866SFelix Guo build_start = time.time() 79021ed9f5SHeidi Fahim success = linux.build_um_kernel(request.alltests, 80021ed9f5SHeidi Fahim request.jobs, 810476e69fSGreg Thelen request.build_dir, 820476e69fSGreg Thelen request.make_options) 836ebf5866SFelix Guo build_end = time.time() 846ebf5866SFelix Guo if not success: 85ee61492aSDavid Gow return KunitResult(KunitStatus.BUILD_FAILURE, 86ee61492aSDavid Gow 'could not build kernel', 87ee61492aSDavid Gow build_end - build_start) 8845ba7a89SDavid Gow if not success: 8945ba7a89SDavid Gow return KunitResult(KunitStatus.BUILD_FAILURE, 9045ba7a89SDavid Gow 'could not build kernel', 9145ba7a89SDavid Gow build_end - build_start) 9245ba7a89SDavid Gow return KunitResult(KunitStatus.SUCCESS, 9345ba7a89SDavid Gow 'built kernel successfully', 9445ba7a89SDavid Gow build_end - build_start) 956ebf5866SFelix Guo 9645ba7a89SDavid Gowdef exec_tests(linux: kunit_kernel.LinuxSourceTree, 9745ba7a89SDavid Gow request: KunitExecRequest) -> KunitResult: 986ebf5866SFelix Guo kunit_parser.print_with_timestamp('Starting KUnit Kernel ...') 996ebf5866SFelix Guo test_start = time.time() 10045ba7a89SDavid Gow result = linux.run_kernel( 101021ed9f5SHeidi Fahim timeout=None if request.alltests else request.timeout, 1026ec1b81dSSeongJae Park build_dir=request.build_dir) 10345ba7a89SDavid Gow 1046ebf5866SFelix Guo test_end = time.time() 1056ebf5866SFelix Guo 10645ba7a89SDavid Gow return KunitResult(KunitStatus.SUCCESS, 10745ba7a89SDavid Gow result, 10845ba7a89SDavid Gow test_end - test_start) 10945ba7a89SDavid Gow 11045ba7a89SDavid Gowdef parse_tests(request: KunitParseRequest) -> KunitResult: 11145ba7a89SDavid Gow parse_start = time.time() 11245ba7a89SDavid Gow 11345ba7a89SDavid Gow test_result = kunit_parser.TestResult(kunit_parser.TestStatus.SUCCESS, 11445ba7a89SDavid Gow [], 11545ba7a89SDavid Gow 'Tests not Parsed.') 11645ba7a89SDavid Gow if request.raw_output: 11745ba7a89SDavid Gow kunit_parser.raw_output(request.input_data) 11845ba7a89SDavid Gow else: 11945ba7a89SDavid Gow test_result = kunit_parser.parse_run_tests(request.input_data) 12045ba7a89SDavid Gow parse_end = time.time() 12145ba7a89SDavid Gow 12245ba7a89SDavid Gow if test_result.status != kunit_parser.TestStatus.SUCCESS: 12345ba7a89SDavid Gow return KunitResult(KunitStatus.TEST_FAILURE, test_result, 12445ba7a89SDavid Gow parse_end - parse_start) 12545ba7a89SDavid Gow 12645ba7a89SDavid Gow return KunitResult(KunitStatus.SUCCESS, test_result, 12745ba7a89SDavid Gow parse_end - parse_start) 12845ba7a89SDavid Gow 12945ba7a89SDavid Gow 13045ba7a89SDavid Gowdef run_tests(linux: kunit_kernel.LinuxSourceTree, 13145ba7a89SDavid Gow request: KunitRequest) -> KunitResult: 13245ba7a89SDavid Gow run_start = time.time() 13345ba7a89SDavid Gow 13445ba7a89SDavid Gow config_request = KunitConfigRequest(request.build_dir, 13545ba7a89SDavid Gow request.make_options) 13645ba7a89SDavid Gow config_result = config_tests(linux, config_request) 13745ba7a89SDavid Gow if config_result.status != KunitStatus.SUCCESS: 13845ba7a89SDavid Gow return config_result 13945ba7a89SDavid Gow 14045ba7a89SDavid Gow build_request = KunitBuildRequest(request.jobs, request.build_dir, 14145ba7a89SDavid Gow request.alltests, 14245ba7a89SDavid Gow request.make_options) 14345ba7a89SDavid Gow build_result = build_tests(linux, build_request) 14445ba7a89SDavid Gow if build_result.status != KunitStatus.SUCCESS: 14545ba7a89SDavid Gow return build_result 14645ba7a89SDavid Gow 14745ba7a89SDavid Gow exec_request = KunitExecRequest(request.timeout, request.build_dir, 14845ba7a89SDavid Gow request.alltests) 14945ba7a89SDavid Gow exec_result = exec_tests(linux, exec_request) 15045ba7a89SDavid Gow if exec_result.status != KunitStatus.SUCCESS: 15145ba7a89SDavid Gow return exec_result 15245ba7a89SDavid Gow 15345ba7a89SDavid Gow parse_request = KunitParseRequest(request.raw_output, 15445ba7a89SDavid Gow exec_result.result) 15545ba7a89SDavid Gow parse_result = parse_tests(parse_request) 15645ba7a89SDavid Gow 15745ba7a89SDavid Gow run_end = time.time() 15845ba7a89SDavid Gow 1596ebf5866SFelix Guo kunit_parser.print_with_timestamp(( 1606ebf5866SFelix Guo 'Elapsed time: %.3fs total, %.3fs configuring, %.3fs ' + 1616ebf5866SFelix Guo 'building, %.3fs running\n') % ( 16245ba7a89SDavid Gow run_end - run_start, 16345ba7a89SDavid Gow config_result.elapsed_time, 16445ba7a89SDavid Gow build_result.elapsed_time, 16545ba7a89SDavid Gow exec_result.elapsed_time)) 16645ba7a89SDavid Gow return parse_result 1676ebf5866SFelix Guo 16845ba7a89SDavid Gowdef add_common_opts(parser): 16945ba7a89SDavid Gow parser.add_argument('--build_dir', 17045ba7a89SDavid Gow help='As in the make command, it specifies the build ' 17145ba7a89SDavid Gow 'directory.', 172ddbd60c7SVitor Massaru Iha type=str, default='.kunit', metavar='build_dir') 17345ba7a89SDavid Gow parser.add_argument('--make_options', 17445ba7a89SDavid Gow help='X=Y make option, can be repeated.', 17545ba7a89SDavid Gow action='append') 17645ba7a89SDavid Gow parser.add_argument('--alltests', 17745ba7a89SDavid Gow help='Run all KUnit tests through allyesconfig', 1786ebf5866SFelix Guo action='store_true') 1796ebf5866SFelix Guo 18045ba7a89SDavid Gowdef add_build_opts(parser): 18145ba7a89SDavid Gow parser.add_argument('--jobs', 18245ba7a89SDavid Gow help='As in the make command, "Specifies the number of ' 18345ba7a89SDavid Gow 'jobs (commands) to run simultaneously."', 18445ba7a89SDavid Gow type=int, default=8, metavar='jobs') 18545ba7a89SDavid Gow 18645ba7a89SDavid Gowdef add_exec_opts(parser): 18745ba7a89SDavid Gow parser.add_argument('--timeout', 1886ebf5866SFelix Guo help='maximum number of seconds to allow for all tests ' 1896ebf5866SFelix Guo 'to run. This does not include time taken to build the ' 1906ebf5866SFelix Guo 'tests.', 1916ebf5866SFelix Guo type=int, 1926ebf5866SFelix Guo default=300, 1936ebf5866SFelix Guo metavar='timeout') 1946ebf5866SFelix Guo 19545ba7a89SDavid Gowdef add_parse_opts(parser): 19645ba7a89SDavid Gow parser.add_argument('--raw_output', help='don\'t format output from kernel', 197ff7b437fSBrendan Higgins action='store_true') 198ff7b437fSBrendan Higgins 199021ed9f5SHeidi Fahim 20045ba7a89SDavid Gowdef main(argv, linux=None): 20145ba7a89SDavid Gow parser = argparse.ArgumentParser( 20245ba7a89SDavid Gow description='Helps writing and running KUnit tests.') 20345ba7a89SDavid Gow subparser = parser.add_subparsers(dest='subcommand') 20445ba7a89SDavid Gow 20545ba7a89SDavid Gow # The 'run' command will config, build, exec, and parse in one go. 20645ba7a89SDavid Gow run_parser = subparser.add_parser('run', help='Runs KUnit tests.') 20745ba7a89SDavid Gow add_common_opts(run_parser) 20845ba7a89SDavid Gow add_build_opts(run_parser) 20945ba7a89SDavid Gow add_exec_opts(run_parser) 21045ba7a89SDavid Gow add_parse_opts(run_parser) 21145ba7a89SDavid Gow 21245ba7a89SDavid Gow config_parser = subparser.add_parser('config', 21345ba7a89SDavid Gow help='Ensures that .config contains all of ' 21445ba7a89SDavid Gow 'the options in .kunitconfig') 21545ba7a89SDavid Gow add_common_opts(config_parser) 21645ba7a89SDavid Gow 21745ba7a89SDavid Gow build_parser = subparser.add_parser('build', help='Builds a kernel with KUnit tests') 21845ba7a89SDavid Gow add_common_opts(build_parser) 21945ba7a89SDavid Gow add_build_opts(build_parser) 22045ba7a89SDavid Gow 22145ba7a89SDavid Gow exec_parser = subparser.add_parser('exec', help='Run a kernel with KUnit tests') 22245ba7a89SDavid Gow add_common_opts(exec_parser) 22345ba7a89SDavid Gow add_exec_opts(exec_parser) 22445ba7a89SDavid Gow add_parse_opts(exec_parser) 22545ba7a89SDavid Gow 22645ba7a89SDavid Gow # The 'parse' option is special, as it doesn't need the kernel source 22745ba7a89SDavid Gow # (therefore there is no need for a build_dir, hence no add_common_opts) 22845ba7a89SDavid Gow # and the '--file' argument is not relevant to 'run', so isn't in 22945ba7a89SDavid Gow # add_parse_opts() 23045ba7a89SDavid Gow parse_parser = subparser.add_parser('parse', 23145ba7a89SDavid Gow help='Parses KUnit results from a file, ' 23245ba7a89SDavid Gow 'and parses formatted results.') 23345ba7a89SDavid Gow add_parse_opts(parse_parser) 23445ba7a89SDavid Gow parse_parser.add_argument('file', 23545ba7a89SDavid Gow help='Specifies the file to read results from.', 23645ba7a89SDavid Gow type=str, nargs='?', metavar='input_file') 2370476e69fSGreg Thelen 2386ebf5866SFelix Guo cli_args = parser.parse_args(argv) 2396ebf5866SFelix Guo 240*5578d008SBrendan Higgins if get_kernel_root_path(): 241*5578d008SBrendan Higgins os.chdir(get_kernel_root_path()) 242*5578d008SBrendan Higgins 2436ebf5866SFelix Guo if cli_args.subcommand == 'run': 244e3212513SSeongJae Park if not os.path.exists(cli_args.build_dir): 245e3212513SSeongJae Park os.mkdir(cli_args.build_dir) 246*5578d008SBrendan Higgins create_default_kunitconfig() 24701397e82SVitor Massaru Iha 248ff7b437fSBrendan Higgins if not linux: 249ff7b437fSBrendan Higgins linux = kunit_kernel.LinuxSourceTree() 250ff7b437fSBrendan Higgins 2516ebf5866SFelix Guo request = KunitRequest(cli_args.raw_output, 2526ebf5866SFelix Guo cli_args.timeout, 2536ebf5866SFelix Guo cli_args.jobs, 254ff7b437fSBrendan Higgins cli_args.build_dir, 2550476e69fSGreg Thelen cli_args.alltests, 2560476e69fSGreg Thelen cli_args.make_options) 2576ebf5866SFelix Guo result = run_tests(linux, request) 2586ebf5866SFelix Guo if result.status != KunitStatus.SUCCESS: 2596ebf5866SFelix Guo sys.exit(1) 26045ba7a89SDavid Gow elif cli_args.subcommand == 'config': 26145ba7a89SDavid Gow if cli_args.build_dir: 26245ba7a89SDavid Gow if not os.path.exists(cli_args.build_dir): 26345ba7a89SDavid Gow os.mkdir(cli_args.build_dir) 264*5578d008SBrendan Higgins create_default_kunitconfig() 26501397e82SVitor Massaru Iha 26645ba7a89SDavid Gow if not linux: 26745ba7a89SDavid Gow linux = kunit_kernel.LinuxSourceTree() 26845ba7a89SDavid Gow 26945ba7a89SDavid Gow request = KunitConfigRequest(cli_args.build_dir, 27045ba7a89SDavid Gow cli_args.make_options) 27145ba7a89SDavid Gow result = config_tests(linux, request) 27245ba7a89SDavid Gow kunit_parser.print_with_timestamp(( 27345ba7a89SDavid Gow 'Elapsed time: %.3fs\n') % ( 27445ba7a89SDavid Gow result.elapsed_time)) 27545ba7a89SDavid Gow if result.status != KunitStatus.SUCCESS: 27645ba7a89SDavid Gow sys.exit(1) 27745ba7a89SDavid Gow elif cli_args.subcommand == 'build': 27845ba7a89SDavid Gow if not linux: 27945ba7a89SDavid Gow linux = kunit_kernel.LinuxSourceTree() 28045ba7a89SDavid Gow 28145ba7a89SDavid Gow request = KunitBuildRequest(cli_args.jobs, 28245ba7a89SDavid Gow cli_args.build_dir, 28345ba7a89SDavid Gow cli_args.alltests, 28445ba7a89SDavid Gow cli_args.make_options) 28545ba7a89SDavid Gow result = build_tests(linux, request) 28645ba7a89SDavid Gow kunit_parser.print_with_timestamp(( 28745ba7a89SDavid Gow 'Elapsed time: %.3fs\n') % ( 28845ba7a89SDavid Gow result.elapsed_time)) 28945ba7a89SDavid Gow if result.status != KunitStatus.SUCCESS: 29045ba7a89SDavid Gow sys.exit(1) 29145ba7a89SDavid Gow elif cli_args.subcommand == 'exec': 29245ba7a89SDavid Gow if not linux: 29345ba7a89SDavid Gow linux = kunit_kernel.LinuxSourceTree() 29445ba7a89SDavid Gow 29545ba7a89SDavid Gow exec_request = KunitExecRequest(cli_args.timeout, 29645ba7a89SDavid Gow cli_args.build_dir, 29745ba7a89SDavid Gow cli_args.alltests) 29845ba7a89SDavid Gow exec_result = exec_tests(linux, exec_request) 29945ba7a89SDavid Gow parse_request = KunitParseRequest(cli_args.raw_output, 30045ba7a89SDavid Gow exec_result.result) 30145ba7a89SDavid Gow result = parse_tests(parse_request) 30245ba7a89SDavid Gow kunit_parser.print_with_timestamp(( 30345ba7a89SDavid Gow 'Elapsed time: %.3fs\n') % ( 30445ba7a89SDavid Gow exec_result.elapsed_time)) 30545ba7a89SDavid Gow if result.status != KunitStatus.SUCCESS: 30645ba7a89SDavid Gow sys.exit(1) 30745ba7a89SDavid Gow elif cli_args.subcommand == 'parse': 30845ba7a89SDavid Gow if cli_args.file == None: 30945ba7a89SDavid Gow kunit_output = sys.stdin 31045ba7a89SDavid Gow else: 31145ba7a89SDavid Gow with open(cli_args.file, 'r') as f: 31245ba7a89SDavid Gow kunit_output = f.read().splitlines() 31345ba7a89SDavid Gow request = KunitParseRequest(cli_args.raw_output, 31445ba7a89SDavid Gow kunit_output) 31545ba7a89SDavid Gow result = parse_tests(request) 31645ba7a89SDavid Gow if result.status != KunitStatus.SUCCESS: 31745ba7a89SDavid Gow sys.exit(1) 3186ebf5866SFelix Guo else: 3196ebf5866SFelix Guo parser.print_help() 3206ebf5866SFelix Guo 3216ebf5866SFelix Guoif __name__ == '__main__': 322ff7b437fSBrendan Higgins main(sys.argv[1:]) 323