1c25ce589SFinn Behrens#!/usr/bin/env python3 26ebf5866SFelix Guo# SPDX-License-Identifier: GPL-2.0 36ebf5866SFelix Guo# 46ebf5866SFelix Guo# A thin wrapper on top of the KUnit Kernel 56ebf5866SFelix Guo# 66ebf5866SFelix Guo# Copyright (C) 2019, Google LLC. 76ebf5866SFelix Guo# Author: Felix Guo <felixguoxiuping@gmail.com> 86ebf5866SFelix Guo# Author: Brendan Higgins <brendanhiggins@google.com> 96ebf5866SFelix Guo 106ebf5866SFelix Guoimport argparse 116ebf5866SFelix Guoimport sys 126ebf5866SFelix Guoimport os 136ebf5866SFelix Guoimport time 146ebf5866SFelix Guo 156ebf5866SFelix Guofrom collections import namedtuple 166ebf5866SFelix Guofrom enum import Enum, auto 176ebf5866SFelix Guo 186ebf5866SFelix Guoimport kunit_config 1921a6d178SHeidi Fahimimport kunit_json 206ebf5866SFelix Guoimport kunit_kernel 216ebf5866SFelix Guoimport kunit_parser 226ebf5866SFelix Guo 2345ba7a89SDavid GowKunitResult = namedtuple('KunitResult', ['status','result','elapsed_time']) 246ebf5866SFelix Guo 2545ba7a89SDavid GowKunitConfigRequest = namedtuple('KunitConfigRequest', 2601397e82SVitor Massaru Iha ['build_dir', 'make_options']) 2745ba7a89SDavid GowKunitBuildRequest = namedtuple('KunitBuildRequest', 2845ba7a89SDavid Gow ['jobs', 'build_dir', 'alltests', 2945ba7a89SDavid Gow 'make_options']) 3045ba7a89SDavid GowKunitExecRequest = namedtuple('KunitExecRequest', 3145ba7a89SDavid Gow ['timeout', 'build_dir', 'alltests']) 3245ba7a89SDavid GowKunitParseRequest = namedtuple('KunitParseRequest', 3321a6d178SHeidi Fahim ['raw_output', 'input_data', 'build_dir', 'json']) 34021ed9f5SHeidi FahimKunitRequest = namedtuple('KunitRequest', ['raw_output','timeout', 'jobs', 3521a6d178SHeidi Fahim 'build_dir', 'alltests', 'json', 369bdf64b3SVitor Massaru Iha 'make_options']) 376ebf5866SFelix Guo 38be886ba9SHeidi FahimKernelDirectoryPath = sys.argv[0].split('tools/testing/kunit/')[0] 39be886ba9SHeidi Fahim 406ebf5866SFelix Guoclass KunitStatus(Enum): 416ebf5866SFelix Guo SUCCESS = auto() 426ebf5866SFelix Guo CONFIG_FAILURE = auto() 436ebf5866SFelix Guo BUILD_FAILURE = auto() 446ebf5866SFelix Guo TEST_FAILURE = auto() 456ebf5866SFelix Guo 46*09641f7cSDaniel Latypovdef get_kernel_root_path() -> str: 47*09641f7cSDaniel Latypov path = sys.argv[0] if not __file__ else __file__ 48*09641f7cSDaniel Latypov parts = os.path.realpath(path).split('tools/testing/kunit') 49be886ba9SHeidi Fahim if len(parts) != 2: 50be886ba9SHeidi Fahim sys.exit(1) 51be886ba9SHeidi Fahim return parts[0] 52be886ba9SHeidi Fahim 5345ba7a89SDavid Gowdef config_tests(linux: kunit_kernel.LinuxSourceTree, 5445ba7a89SDavid Gow request: KunitConfigRequest) -> KunitResult: 5545ba7a89SDavid Gow kunit_parser.print_with_timestamp('Configuring KUnit Kernel ...') 5645ba7a89SDavid Gow 576ebf5866SFelix Guo config_start = time.time() 580476e69fSGreg Thelen success = linux.build_reconfig(request.build_dir, request.make_options) 596ebf5866SFelix Guo config_end = time.time() 606ebf5866SFelix Guo if not success: 6145ba7a89SDavid Gow return KunitResult(KunitStatus.CONFIG_FAILURE, 6245ba7a89SDavid Gow 'could not configure kernel', 6345ba7a89SDavid Gow config_end - config_start) 6445ba7a89SDavid Gow return KunitResult(KunitStatus.SUCCESS, 6545ba7a89SDavid Gow 'configured kernel successfully', 6645ba7a89SDavid Gow config_end - config_start) 676ebf5866SFelix Guo 6845ba7a89SDavid Gowdef build_tests(linux: kunit_kernel.LinuxSourceTree, 6945ba7a89SDavid Gow request: KunitBuildRequest) -> KunitResult: 706ebf5866SFelix Guo kunit_parser.print_with_timestamp('Building KUnit Kernel ...') 716ebf5866SFelix Guo 726ebf5866SFelix Guo build_start = time.time() 73021ed9f5SHeidi Fahim success = linux.build_um_kernel(request.alltests, 74021ed9f5SHeidi Fahim request.jobs, 750476e69fSGreg Thelen request.build_dir, 760476e69fSGreg Thelen request.make_options) 776ebf5866SFelix Guo build_end = time.time() 786ebf5866SFelix Guo if not success: 79ee61492aSDavid Gow return KunitResult(KunitStatus.BUILD_FAILURE, 80ee61492aSDavid Gow 'could not build kernel', 81ee61492aSDavid Gow build_end - build_start) 8245ba7a89SDavid Gow if not success: 8345ba7a89SDavid Gow return KunitResult(KunitStatus.BUILD_FAILURE, 8445ba7a89SDavid Gow 'could not build kernel', 8545ba7a89SDavid Gow build_end - build_start) 8645ba7a89SDavid Gow return KunitResult(KunitStatus.SUCCESS, 8745ba7a89SDavid Gow 'built kernel successfully', 8845ba7a89SDavid Gow build_end - build_start) 896ebf5866SFelix Guo 9045ba7a89SDavid Gowdef exec_tests(linux: kunit_kernel.LinuxSourceTree, 9145ba7a89SDavid Gow request: KunitExecRequest) -> KunitResult: 926ebf5866SFelix Guo kunit_parser.print_with_timestamp('Starting KUnit Kernel ...') 936ebf5866SFelix Guo test_start = time.time() 9445ba7a89SDavid Gow result = linux.run_kernel( 95021ed9f5SHeidi Fahim timeout=None if request.alltests else request.timeout, 966ec1b81dSSeongJae Park build_dir=request.build_dir) 9745ba7a89SDavid Gow 986ebf5866SFelix Guo test_end = time.time() 996ebf5866SFelix Guo 10045ba7a89SDavid Gow return KunitResult(KunitStatus.SUCCESS, 10145ba7a89SDavid Gow result, 10245ba7a89SDavid Gow test_end - test_start) 10345ba7a89SDavid Gow 10445ba7a89SDavid Gowdef parse_tests(request: KunitParseRequest) -> KunitResult: 10545ba7a89SDavid Gow parse_start = time.time() 10645ba7a89SDavid Gow 10745ba7a89SDavid Gow test_result = kunit_parser.TestResult(kunit_parser.TestStatus.SUCCESS, 10845ba7a89SDavid Gow [], 10945ba7a89SDavid Gow 'Tests not Parsed.') 11021a6d178SHeidi Fahim 11145ba7a89SDavid Gow if request.raw_output: 11245ba7a89SDavid Gow kunit_parser.raw_output(request.input_data) 11345ba7a89SDavid Gow else: 11445ba7a89SDavid Gow test_result = kunit_parser.parse_run_tests(request.input_data) 11545ba7a89SDavid Gow parse_end = time.time() 11645ba7a89SDavid Gow 11721a6d178SHeidi Fahim if request.json: 11821a6d178SHeidi Fahim json_obj = kunit_json.get_json_result( 11921a6d178SHeidi Fahim test_result=test_result, 12021a6d178SHeidi Fahim def_config='kunit_defconfig', 12121a6d178SHeidi Fahim build_dir=request.build_dir, 12221a6d178SHeidi Fahim json_path=request.json) 12321a6d178SHeidi Fahim if request.json == 'stdout': 12421a6d178SHeidi Fahim print(json_obj) 12521a6d178SHeidi Fahim 12645ba7a89SDavid Gow if test_result.status != kunit_parser.TestStatus.SUCCESS: 12745ba7a89SDavid Gow return KunitResult(KunitStatus.TEST_FAILURE, test_result, 12845ba7a89SDavid Gow parse_end - parse_start) 12945ba7a89SDavid Gow 13045ba7a89SDavid Gow return KunitResult(KunitStatus.SUCCESS, test_result, 13145ba7a89SDavid Gow parse_end - parse_start) 13245ba7a89SDavid Gow 13345ba7a89SDavid Gow 13445ba7a89SDavid Gowdef run_tests(linux: kunit_kernel.LinuxSourceTree, 13545ba7a89SDavid Gow request: KunitRequest) -> KunitResult: 13645ba7a89SDavid Gow run_start = time.time() 13745ba7a89SDavid Gow 13845ba7a89SDavid Gow config_request = KunitConfigRequest(request.build_dir, 13945ba7a89SDavid Gow request.make_options) 14045ba7a89SDavid Gow config_result = config_tests(linux, config_request) 14145ba7a89SDavid Gow if config_result.status != KunitStatus.SUCCESS: 14245ba7a89SDavid Gow return config_result 14345ba7a89SDavid Gow 14445ba7a89SDavid Gow build_request = KunitBuildRequest(request.jobs, request.build_dir, 14545ba7a89SDavid Gow request.alltests, 14645ba7a89SDavid Gow request.make_options) 14745ba7a89SDavid Gow build_result = build_tests(linux, build_request) 14845ba7a89SDavid Gow if build_result.status != KunitStatus.SUCCESS: 14945ba7a89SDavid Gow return build_result 15045ba7a89SDavid Gow 15145ba7a89SDavid Gow exec_request = KunitExecRequest(request.timeout, request.build_dir, 15245ba7a89SDavid Gow request.alltests) 15345ba7a89SDavid Gow exec_result = exec_tests(linux, exec_request) 15445ba7a89SDavid Gow if exec_result.status != KunitStatus.SUCCESS: 15545ba7a89SDavid Gow return exec_result 15645ba7a89SDavid Gow 15745ba7a89SDavid Gow parse_request = KunitParseRequest(request.raw_output, 15821a6d178SHeidi Fahim exec_result.result, 15921a6d178SHeidi Fahim request.build_dir, 16021a6d178SHeidi Fahim request.json) 16145ba7a89SDavid Gow parse_result = parse_tests(parse_request) 16245ba7a89SDavid Gow 16345ba7a89SDavid Gow run_end = time.time() 16445ba7a89SDavid Gow 1656ebf5866SFelix Guo kunit_parser.print_with_timestamp(( 1666ebf5866SFelix Guo 'Elapsed time: %.3fs total, %.3fs configuring, %.3fs ' + 1676ebf5866SFelix Guo 'building, %.3fs running\n') % ( 16845ba7a89SDavid Gow run_end - run_start, 16945ba7a89SDavid Gow config_result.elapsed_time, 17045ba7a89SDavid Gow build_result.elapsed_time, 17145ba7a89SDavid Gow exec_result.elapsed_time)) 17245ba7a89SDavid Gow return parse_result 1736ebf5866SFelix Guo 174*09641f7cSDaniel Latypovdef add_common_opts(parser) -> None: 17545ba7a89SDavid Gow parser.add_argument('--build_dir', 17645ba7a89SDavid Gow help='As in the make command, it specifies the build ' 17745ba7a89SDavid Gow 'directory.', 178ddbd60c7SVitor Massaru Iha type=str, default='.kunit', metavar='build_dir') 17945ba7a89SDavid Gow parser.add_argument('--make_options', 18045ba7a89SDavid Gow help='X=Y make option, can be repeated.', 18145ba7a89SDavid Gow action='append') 18245ba7a89SDavid Gow parser.add_argument('--alltests', 18345ba7a89SDavid Gow help='Run all KUnit tests through allyesconfig', 1846ebf5866SFelix Guo action='store_true') 1856ebf5866SFelix Guo 186*09641f7cSDaniel Latypovdef add_build_opts(parser) -> None: 18745ba7a89SDavid Gow parser.add_argument('--jobs', 18845ba7a89SDavid Gow help='As in the make command, "Specifies the number of ' 18945ba7a89SDavid Gow 'jobs (commands) to run simultaneously."', 19045ba7a89SDavid Gow type=int, default=8, metavar='jobs') 19145ba7a89SDavid Gow 192*09641f7cSDaniel Latypovdef add_exec_opts(parser) -> None: 19345ba7a89SDavid Gow parser.add_argument('--timeout', 1946ebf5866SFelix Guo help='maximum number of seconds to allow for all tests ' 1956ebf5866SFelix Guo 'to run. This does not include time taken to build the ' 1966ebf5866SFelix Guo 'tests.', 1976ebf5866SFelix Guo type=int, 1986ebf5866SFelix Guo default=300, 1996ebf5866SFelix Guo metavar='timeout') 2006ebf5866SFelix Guo 201*09641f7cSDaniel Latypovdef add_parse_opts(parser) -> None: 20245ba7a89SDavid Gow parser.add_argument('--raw_output', help='don\'t format output from kernel', 203ff7b437fSBrendan Higgins action='store_true') 20421a6d178SHeidi Fahim parser.add_argument('--json', 20521a6d178SHeidi Fahim nargs='?', 20621a6d178SHeidi Fahim help='Stores test results in a JSON, and either ' 20721a6d178SHeidi Fahim 'prints to stdout or saves to file if a ' 20821a6d178SHeidi Fahim 'filename is specified', 20921a6d178SHeidi Fahim type=str, const='stdout', default=None) 210021ed9f5SHeidi Fahim 21145ba7a89SDavid Gowdef main(argv, linux=None): 21245ba7a89SDavid Gow parser = argparse.ArgumentParser( 21345ba7a89SDavid Gow description='Helps writing and running KUnit tests.') 21445ba7a89SDavid Gow subparser = parser.add_subparsers(dest='subcommand') 21545ba7a89SDavid Gow 21645ba7a89SDavid Gow # The 'run' command will config, build, exec, and parse in one go. 21745ba7a89SDavid Gow run_parser = subparser.add_parser('run', help='Runs KUnit tests.') 21845ba7a89SDavid Gow add_common_opts(run_parser) 21945ba7a89SDavid Gow add_build_opts(run_parser) 22045ba7a89SDavid Gow add_exec_opts(run_parser) 22145ba7a89SDavid Gow add_parse_opts(run_parser) 22245ba7a89SDavid Gow 22345ba7a89SDavid Gow config_parser = subparser.add_parser('config', 22445ba7a89SDavid Gow help='Ensures that .config contains all of ' 22545ba7a89SDavid Gow 'the options in .kunitconfig') 22645ba7a89SDavid Gow add_common_opts(config_parser) 22745ba7a89SDavid Gow 22845ba7a89SDavid Gow build_parser = subparser.add_parser('build', help='Builds a kernel with KUnit tests') 22945ba7a89SDavid Gow add_common_opts(build_parser) 23045ba7a89SDavid Gow add_build_opts(build_parser) 23145ba7a89SDavid Gow 23245ba7a89SDavid Gow exec_parser = subparser.add_parser('exec', help='Run a kernel with KUnit tests') 23345ba7a89SDavid Gow add_common_opts(exec_parser) 23445ba7a89SDavid Gow add_exec_opts(exec_parser) 23545ba7a89SDavid Gow add_parse_opts(exec_parser) 23645ba7a89SDavid Gow 23745ba7a89SDavid Gow # The 'parse' option is special, as it doesn't need the kernel source 23845ba7a89SDavid Gow # (therefore there is no need for a build_dir, hence no add_common_opts) 23945ba7a89SDavid Gow # and the '--file' argument is not relevant to 'run', so isn't in 24045ba7a89SDavid Gow # add_parse_opts() 24145ba7a89SDavid Gow parse_parser = subparser.add_parser('parse', 24245ba7a89SDavid Gow help='Parses KUnit results from a file, ' 24345ba7a89SDavid Gow 'and parses formatted results.') 24445ba7a89SDavid Gow add_parse_opts(parse_parser) 24545ba7a89SDavid Gow parse_parser.add_argument('file', 24645ba7a89SDavid Gow help='Specifies the file to read results from.', 24745ba7a89SDavid Gow type=str, nargs='?', metavar='input_file') 2480476e69fSGreg Thelen 2496ebf5866SFelix Guo cli_args = parser.parse_args(argv) 2506ebf5866SFelix Guo 2515578d008SBrendan Higgins if get_kernel_root_path(): 2525578d008SBrendan Higgins os.chdir(get_kernel_root_path()) 2535578d008SBrendan Higgins 2546ebf5866SFelix Guo if cli_args.subcommand == 'run': 255e3212513SSeongJae Park if not os.path.exists(cli_args.build_dir): 256e3212513SSeongJae Park os.mkdir(cli_args.build_dir) 25782206a0cSBrendan Higgins 258ff7b437fSBrendan Higgins if not linux: 259ff7b437fSBrendan Higgins linux = kunit_kernel.LinuxSourceTree() 260ff7b437fSBrendan Higgins 261fcdb0bc0SAndy Shevchenko linux.create_kunitconfig(cli_args.build_dir) 262fcdb0bc0SAndy Shevchenko linux.read_kunitconfig(cli_args.build_dir) 263fcdb0bc0SAndy Shevchenko 2646ebf5866SFelix Guo request = KunitRequest(cli_args.raw_output, 2656ebf5866SFelix Guo cli_args.timeout, 2666ebf5866SFelix Guo cli_args.jobs, 267ff7b437fSBrendan Higgins cli_args.build_dir, 2680476e69fSGreg Thelen cli_args.alltests, 26921a6d178SHeidi Fahim cli_args.json, 2700476e69fSGreg Thelen cli_args.make_options) 2716ebf5866SFelix Guo result = run_tests(linux, request) 2726ebf5866SFelix Guo if result.status != KunitStatus.SUCCESS: 2736ebf5866SFelix Guo sys.exit(1) 27445ba7a89SDavid Gow elif cli_args.subcommand == 'config': 27582206a0cSBrendan Higgins if cli_args.build_dir and ( 27682206a0cSBrendan Higgins not os.path.exists(cli_args.build_dir)): 27745ba7a89SDavid Gow os.mkdir(cli_args.build_dir) 27882206a0cSBrendan Higgins 27945ba7a89SDavid Gow if not linux: 28045ba7a89SDavid Gow linux = kunit_kernel.LinuxSourceTree() 28145ba7a89SDavid Gow 282fcdb0bc0SAndy Shevchenko linux.create_kunitconfig(cli_args.build_dir) 283fcdb0bc0SAndy Shevchenko linux.read_kunitconfig(cli_args.build_dir) 284fcdb0bc0SAndy Shevchenko 28545ba7a89SDavid Gow request = KunitConfigRequest(cli_args.build_dir, 28645ba7a89SDavid Gow cli_args.make_options) 28745ba7a89SDavid Gow result = config_tests(linux, request) 28845ba7a89SDavid Gow kunit_parser.print_with_timestamp(( 28945ba7a89SDavid Gow 'Elapsed time: %.3fs\n') % ( 29045ba7a89SDavid Gow result.elapsed_time)) 29145ba7a89SDavid Gow if result.status != KunitStatus.SUCCESS: 29245ba7a89SDavid Gow sys.exit(1) 29345ba7a89SDavid Gow elif cli_args.subcommand == 'build': 29445ba7a89SDavid Gow if not linux: 29545ba7a89SDavid Gow linux = kunit_kernel.LinuxSourceTree() 29645ba7a89SDavid Gow 297fcdb0bc0SAndy Shevchenko linux.create_kunitconfig(cli_args.build_dir) 298fcdb0bc0SAndy Shevchenko linux.read_kunitconfig(cli_args.build_dir) 299fcdb0bc0SAndy Shevchenko 30045ba7a89SDavid Gow request = KunitBuildRequest(cli_args.jobs, 30145ba7a89SDavid Gow cli_args.build_dir, 30245ba7a89SDavid Gow cli_args.alltests, 30345ba7a89SDavid Gow cli_args.make_options) 30445ba7a89SDavid Gow result = build_tests(linux, request) 30545ba7a89SDavid Gow kunit_parser.print_with_timestamp(( 30645ba7a89SDavid Gow 'Elapsed time: %.3fs\n') % ( 30745ba7a89SDavid Gow result.elapsed_time)) 30845ba7a89SDavid Gow if result.status != KunitStatus.SUCCESS: 30945ba7a89SDavid Gow sys.exit(1) 31045ba7a89SDavid Gow elif cli_args.subcommand == 'exec': 31145ba7a89SDavid Gow if not linux: 31245ba7a89SDavid Gow linux = kunit_kernel.LinuxSourceTree() 31345ba7a89SDavid Gow 314fcdb0bc0SAndy Shevchenko linux.create_kunitconfig(cli_args.build_dir) 315fcdb0bc0SAndy Shevchenko linux.read_kunitconfig(cli_args.build_dir) 316fcdb0bc0SAndy Shevchenko 31745ba7a89SDavid Gow exec_request = KunitExecRequest(cli_args.timeout, 31845ba7a89SDavid Gow cli_args.build_dir, 31945ba7a89SDavid Gow cli_args.alltests) 32045ba7a89SDavid Gow exec_result = exec_tests(linux, exec_request) 32145ba7a89SDavid Gow parse_request = KunitParseRequest(cli_args.raw_output, 32221a6d178SHeidi Fahim exec_result.result, 32321a6d178SHeidi Fahim cli_args.build_dir, 32421a6d178SHeidi Fahim cli_args.json) 32545ba7a89SDavid Gow result = parse_tests(parse_request) 32645ba7a89SDavid Gow kunit_parser.print_with_timestamp(( 32745ba7a89SDavid Gow 'Elapsed time: %.3fs\n') % ( 32845ba7a89SDavid Gow exec_result.elapsed_time)) 32945ba7a89SDavid Gow if result.status != KunitStatus.SUCCESS: 33045ba7a89SDavid Gow sys.exit(1) 33145ba7a89SDavid Gow elif cli_args.subcommand == 'parse': 33245ba7a89SDavid Gow if cli_args.file == None: 33345ba7a89SDavid Gow kunit_output = sys.stdin 33445ba7a89SDavid Gow else: 33545ba7a89SDavid Gow with open(cli_args.file, 'r') as f: 33645ba7a89SDavid Gow kunit_output = f.read().splitlines() 33745ba7a89SDavid Gow request = KunitParseRequest(cli_args.raw_output, 33821a6d178SHeidi Fahim kunit_output, 3393959d0a6SDavid Gow None, 34021a6d178SHeidi Fahim cli_args.json) 34145ba7a89SDavid Gow result = parse_tests(request) 34245ba7a89SDavid Gow if result.status != KunitStatus.SUCCESS: 34345ba7a89SDavid Gow sys.exit(1) 3446ebf5866SFelix Guo else: 3456ebf5866SFelix Guo parser.print_help() 3466ebf5866SFelix Guo 3476ebf5866SFelix Guoif __name__ == '__main__': 348ff7b437fSBrendan Higgins main(sys.argv[1:]) 349