1c25ce589SFinn Behrens#!/usr/bin/env python3 26ebf5866SFelix Guo# SPDX-License-Identifier: GPL-2.0 36ebf5866SFelix Guo# 46ebf5866SFelix Guo# A thin wrapper on top of the KUnit Kernel 56ebf5866SFelix Guo# 66ebf5866SFelix Guo# Copyright (C) 2019, Google LLC. 76ebf5866SFelix Guo# Author: Felix Guo <felixguoxiuping@gmail.com> 86ebf5866SFelix Guo# Author: Brendan Higgins <brendanhiggins@google.com> 96ebf5866SFelix Guo 106ebf5866SFelix Guoimport argparse 116ebf5866SFelix Guoimport os 12ff9e09a3SDaniel Latypovimport re 13ff9e09a3SDaniel Latypovimport sys 146ebf5866SFelix Guoimport time 156ebf5866SFelix Guo 16df4b0807SSeongJae Parkassert sys.version_info >= (3, 7), "Python version is too old" 17df4b0807SSeongJae Park 18db167981SDaniel Latypovfrom dataclasses import dataclass 196ebf5866SFelix Guofrom enum import Enum, auto 2095dcbc55SDaniel Latypovfrom typing import Iterable, List, Optional, Sequence, Tuple 216ebf5866SFelix Guo 2221a6d178SHeidi Fahimimport kunit_json 236ebf5866SFelix Guoimport kunit_kernel 246ebf5866SFelix Guoimport kunit_parser 256ebf5866SFelix Guo 266ebf5866SFelix Guoclass KunitStatus(Enum): 276ebf5866SFelix Guo SUCCESS = auto() 286ebf5866SFelix Guo CONFIG_FAILURE = auto() 296ebf5866SFelix Guo BUILD_FAILURE = auto() 306ebf5866SFelix Guo TEST_FAILURE = auto() 316ebf5866SFelix Guo 32db167981SDaniel Latypov@dataclass 33db167981SDaniel Latypovclass KunitResult: 34db167981SDaniel Latypov status: KunitStatus 35db167981SDaniel Latypov elapsed_time: float 36db167981SDaniel Latypov 37db167981SDaniel Latypov@dataclass 38db167981SDaniel Latypovclass KunitConfigRequest: 39db167981SDaniel Latypov build_dir: str 40db167981SDaniel Latypov make_options: Optional[List[str]] 41db167981SDaniel Latypov 42db167981SDaniel Latypov@dataclass 43db167981SDaniel Latypovclass KunitBuildRequest(KunitConfigRequest): 44db167981SDaniel Latypov jobs: int 45db167981SDaniel Latypov alltests: bool 46db167981SDaniel Latypov 47db167981SDaniel Latypov@dataclass 48db167981SDaniel Latypovclass KunitParseRequest: 49db167981SDaniel Latypov raw_output: Optional[str] 50db167981SDaniel Latypov json: Optional[str] 51db167981SDaniel Latypov 52db167981SDaniel Latypov@dataclass 53db167981SDaniel Latypovclass KunitExecRequest(KunitParseRequest): 54ee96d25fSDaniel Latypov build_dir: str 55db167981SDaniel Latypov timeout: int 56db167981SDaniel Latypov alltests: bool 57db167981SDaniel Latypov filter_glob: str 58db167981SDaniel Latypov kernel_args: Optional[List[str]] 59db167981SDaniel Latypov run_isolated: Optional[str] 60db167981SDaniel Latypov 61db167981SDaniel Latypov@dataclass 62db167981SDaniel Latypovclass KunitRequest(KunitExecRequest, KunitBuildRequest): 63db167981SDaniel Latypov pass 64db167981SDaniel Latypov 65db167981SDaniel Latypov 6609641f7cSDaniel Latypovdef get_kernel_root_path() -> str: 6709641f7cSDaniel Latypov path = sys.argv[0] if not __file__ else __file__ 6809641f7cSDaniel Latypov parts = os.path.realpath(path).split('tools/testing/kunit') 69be886ba9SHeidi Fahim if len(parts) != 2: 70be886ba9SHeidi Fahim sys.exit(1) 71be886ba9SHeidi Fahim return parts[0] 72be886ba9SHeidi Fahim 7345ba7a89SDavid Gowdef config_tests(linux: kunit_kernel.LinuxSourceTree, 7445ba7a89SDavid Gow request: KunitConfigRequest) -> KunitResult: 7545ba7a89SDavid Gow kunit_parser.print_with_timestamp('Configuring KUnit Kernel ...') 7645ba7a89SDavid Gow 776ebf5866SFelix Guo config_start = time.time() 780476e69fSGreg Thelen success = linux.build_reconfig(request.build_dir, request.make_options) 796ebf5866SFelix Guo config_end = time.time() 806ebf5866SFelix Guo if not success: 8145ba7a89SDavid Gow return KunitResult(KunitStatus.CONFIG_FAILURE, 8245ba7a89SDavid Gow config_end - config_start) 8345ba7a89SDavid Gow return KunitResult(KunitStatus.SUCCESS, 8445ba7a89SDavid Gow config_end - config_start) 856ebf5866SFelix Guo 8645ba7a89SDavid Gowdef build_tests(linux: kunit_kernel.LinuxSourceTree, 8745ba7a89SDavid Gow request: KunitBuildRequest) -> KunitResult: 886ebf5866SFelix Guo kunit_parser.print_with_timestamp('Building KUnit Kernel ...') 896ebf5866SFelix Guo 906ebf5866SFelix Guo build_start = time.time() 9187c9c163SBrendan Higgins success = linux.build_kernel(request.alltests, 92021ed9f5SHeidi Fahim request.jobs, 930476e69fSGreg Thelen request.build_dir, 940476e69fSGreg Thelen request.make_options) 956ebf5866SFelix Guo build_end = time.time() 966ebf5866SFelix Guo if not success: 97ee61492aSDavid Gow return KunitResult(KunitStatus.BUILD_FAILURE, 98ee61492aSDavid Gow build_end - build_start) 9945ba7a89SDavid Gow if not success: 10045ba7a89SDavid Gow return KunitResult(KunitStatus.BUILD_FAILURE, 10145ba7a89SDavid Gow build_end - build_start) 10245ba7a89SDavid Gow return KunitResult(KunitStatus.SUCCESS, 10345ba7a89SDavid Gow build_end - build_start) 1046ebf5866SFelix Guo 1051ee2ba89SDaniel Latypovdef config_and_build_tests(linux: kunit_kernel.LinuxSourceTree, 1061ee2ba89SDaniel Latypov request: KunitBuildRequest) -> KunitResult: 1071ee2ba89SDaniel Latypov config_result = config_tests(linux, request) 1081ee2ba89SDaniel Latypov if config_result.status != KunitStatus.SUCCESS: 1091ee2ba89SDaniel Latypov return config_result 1101ee2ba89SDaniel Latypov 1111ee2ba89SDaniel Latypov return build_tests(linux, request) 1121ee2ba89SDaniel Latypov 113ff9e09a3SDaniel Latypovdef _list_tests(linux: kunit_kernel.LinuxSourceTree, request: KunitExecRequest) -> List[str]: 114ff9e09a3SDaniel Latypov args = ['kunit.action=list'] 115ff9e09a3SDaniel Latypov if request.kernel_args: 116ff9e09a3SDaniel Latypov args.extend(request.kernel_args) 117ff9e09a3SDaniel Latypov 118ff9e09a3SDaniel Latypov output = linux.run_kernel(args=args, 119ff9e09a3SDaniel Latypov timeout=None if request.alltests else request.timeout, 120ff9e09a3SDaniel Latypov filter_glob=request.filter_glob, 121ff9e09a3SDaniel Latypov build_dir=request.build_dir) 122ff9e09a3SDaniel Latypov lines = kunit_parser.extract_tap_lines(output) 123ff9e09a3SDaniel Latypov # Hack! Drop the dummy TAP version header that the executor prints out. 124ff9e09a3SDaniel Latypov lines.pop() 125ff9e09a3SDaniel Latypov 126ff9e09a3SDaniel Latypov # Filter out any extraneous non-test output that might have gotten mixed in. 127ff9e09a3SDaniel Latypov return [l for l in lines if re.match('^[^\s.]+\.[^\s.]+$', l)] 128ff9e09a3SDaniel Latypov 129ff9e09a3SDaniel Latypovdef _suites_from_test_list(tests: List[str]) -> List[str]: 130ff9e09a3SDaniel Latypov """Extracts all the suites from an ordered list of tests.""" 131ff9e09a3SDaniel Latypov suites = [] # type: List[str] 132ff9e09a3SDaniel Latypov for t in tests: 133ff9e09a3SDaniel Latypov parts = t.split('.', maxsplit=2) 134ff9e09a3SDaniel Latypov if len(parts) != 2: 135ff9e09a3SDaniel Latypov raise ValueError(f'internal KUnit error, test name should be of the form "<suite>.<test>", got "{t}"') 136ff9e09a3SDaniel Latypov suite, case = parts 137ff9e09a3SDaniel Latypov if not suites or suites[-1] != suite: 138ff9e09a3SDaniel Latypov suites.append(suite) 139ff9e09a3SDaniel Latypov return suites 140ff9e09a3SDaniel Latypov 141ff9e09a3SDaniel Latypov 142ff9e09a3SDaniel Latypov 143db167981SDaniel Latypovdef exec_tests(linux: kunit_kernel.LinuxSourceTree, request: KunitExecRequest) -> KunitResult: 144ff9e09a3SDaniel Latypov filter_globs = [request.filter_glob] 145ff9e09a3SDaniel Latypov if request.run_isolated: 146ff9e09a3SDaniel Latypov tests = _list_tests(linux, request) 147ff9e09a3SDaniel Latypov if request.run_isolated == 'test': 148ff9e09a3SDaniel Latypov filter_globs = tests 149ff9e09a3SDaniel Latypov if request.run_isolated == 'suite': 150ff9e09a3SDaniel Latypov filter_globs = _suites_from_test_list(tests) 151ff9e09a3SDaniel Latypov # Apply the test-part of the user's glob, if present. 152ff9e09a3SDaniel Latypov if '.' in request.filter_glob: 153ff9e09a3SDaniel Latypov test_glob = request.filter_glob.split('.', maxsplit=2)[1] 154ff9e09a3SDaniel Latypov filter_globs = [g + '.'+ test_glob for g in filter_globs] 155ff9e09a3SDaniel Latypov 156885210d3SDaniel Latypov metadata = kunit_json.Metadata(arch=linux.arch(), build_dir=request.build_dir, def_config='kunit_defconfig') 157ee96d25fSDaniel Latypov 158d65d07cbSRae Moar test_counts = kunit_parser.TestCounts() 159ff9e09a3SDaniel Latypov exec_time = 0.0 160ff9e09a3SDaniel Latypov for i, filter_glob in enumerate(filter_globs): 161ff9e09a3SDaniel Latypov kunit_parser.print_with_timestamp('Starting KUnit Kernel ({}/{})...'.format(i+1, len(filter_globs))) 162ff9e09a3SDaniel Latypov 1636ebf5866SFelix Guo test_start = time.time() 1647ef925eaSDaniel Latypov run_result = linux.run_kernel( 1656cb51a18SDaniel Latypov args=request.kernel_args, 166021ed9f5SHeidi Fahim timeout=None if request.alltests else request.timeout, 167ff9e09a3SDaniel Latypov filter_glob=filter_glob, 1686ec1b81dSSeongJae Park build_dir=request.build_dir) 16945ba7a89SDavid Gow 170ee96d25fSDaniel Latypov _, test_result = parse_tests(request, metadata, run_result) 1715f6aa6d8SDaniel Latypov # run_kernel() doesn't block on the kernel exiting. 1725f6aa6d8SDaniel Latypov # That only happens after we get the last line of output from `run_result`. 1735f6aa6d8SDaniel Latypov # So exec_time here actually contains parsing + execution time, which is fine. 1746ebf5866SFelix Guo test_end = time.time() 175ff9e09a3SDaniel Latypov exec_time += test_end - test_start 176ff9e09a3SDaniel Latypov 17795dcbc55SDaniel Latypov test_counts.add_subtest_counts(test_result.counts) 1786ebf5866SFelix Guo 1797fa7ffcfSDaniel Latypov if len(filter_globs) == 1 and test_counts.crashed > 0: 1807fa7ffcfSDaniel Latypov bd = request.build_dir 1817fa7ffcfSDaniel Latypov print('The kernel seems to have crashed; you can decode the stack traces with:') 1827fa7ffcfSDaniel Latypov print('$ scripts/decode_stacktrace.sh {}/vmlinux {} < {} | tee {}/decoded.log | {} parse'.format( 1837fa7ffcfSDaniel Latypov bd, bd, kunit_kernel.get_outfile_path(bd), bd, sys.argv[0])) 1847fa7ffcfSDaniel Latypov 185d65d07cbSRae Moar kunit_status = _map_to_overall_status(test_counts.get_status()) 18695dcbc55SDaniel Latypov return KunitResult(status=kunit_status, elapsed_time=exec_time) 187d65d07cbSRae Moar 188d65d07cbSRae Moardef _map_to_overall_status(test_status: kunit_parser.TestStatus) -> KunitStatus: 189d65d07cbSRae Moar if test_status in (kunit_parser.TestStatus.SUCCESS, kunit_parser.TestStatus.SKIPPED): 190d65d07cbSRae Moar return KunitStatus.SUCCESS 191d65d07cbSRae Moar else: 192d65d07cbSRae Moar return KunitStatus.TEST_FAILURE 1937ef925eaSDaniel Latypov 194ee96d25fSDaniel Latypovdef parse_tests(request: KunitParseRequest, metadata: kunit_json.Metadata, input_data: Iterable[str]) -> Tuple[KunitResult, kunit_parser.Test]: 19545ba7a89SDavid Gow parse_start = time.time() 19645ba7a89SDavid Gow 197e0cc8c05SDaniel Latypov test_result = kunit_parser.Test() 19821a6d178SHeidi Fahim 19945ba7a89SDavid Gow if request.raw_output: 200d65d07cbSRae Moar # Treat unparsed results as one passing test. 201e0cc8c05SDaniel Latypov test_result.status = kunit_parser.TestStatus.SUCCESS 202e0cc8c05SDaniel Latypov test_result.counts.passed = 1 203d65d07cbSRae Moar 2047ef925eaSDaniel Latypov output: Iterable[str] = input_data 2056a499c9cSDaniel Latypov if request.raw_output == 'all': 2066a499c9cSDaniel Latypov pass 2076a499c9cSDaniel Latypov elif request.raw_output == 'kunit': 2086a499c9cSDaniel Latypov output = kunit_parser.extract_tap_lines(output) 2096a499c9cSDaniel Latypov for line in output: 2106a499c9cSDaniel Latypov print(line.rstrip()) 2116a499c9cSDaniel Latypov 21245ba7a89SDavid Gow else: 2137ef925eaSDaniel Latypov test_result = kunit_parser.parse_run_tests(input_data) 21445ba7a89SDavid Gow parse_end = time.time() 21545ba7a89SDavid Gow 21621a6d178SHeidi Fahim if request.json: 21700f75043SDaniel Latypov json_str = kunit_json.get_json_result( 218e0cc8c05SDaniel Latypov test=test_result, 219ee96d25fSDaniel Latypov metadata=metadata) 22021a6d178SHeidi Fahim if request.json == 'stdout': 22100f75043SDaniel Latypov print(json_str) 22200f75043SDaniel Latypov else: 22300f75043SDaniel Latypov with open(request.json, 'w') as f: 22400f75043SDaniel Latypov f.write(json_str) 22500f75043SDaniel Latypov kunit_parser.print_with_timestamp("Test results stored in %s" % 22600f75043SDaniel Latypov os.path.abspath(request.json)) 22721a6d178SHeidi Fahim 22845ba7a89SDavid Gow if test_result.status != kunit_parser.TestStatus.SUCCESS: 22995dcbc55SDaniel Latypov return KunitResult(KunitStatus.TEST_FAILURE, parse_end - parse_start), test_result 23045ba7a89SDavid Gow 23195dcbc55SDaniel Latypov return KunitResult(KunitStatus.SUCCESS, parse_end - parse_start), test_result 23245ba7a89SDavid Gow 23345ba7a89SDavid Gowdef run_tests(linux: kunit_kernel.LinuxSourceTree, 23445ba7a89SDavid Gow request: KunitRequest) -> KunitResult: 23545ba7a89SDavid Gow run_start = time.time() 23645ba7a89SDavid Gow 237db167981SDaniel Latypov config_result = config_tests(linux, request) 23845ba7a89SDavid Gow if config_result.status != KunitStatus.SUCCESS: 23945ba7a89SDavid Gow return config_result 24045ba7a89SDavid Gow 241db167981SDaniel Latypov build_result = build_tests(linux, request) 24245ba7a89SDavid Gow if build_result.status != KunitStatus.SUCCESS: 24345ba7a89SDavid Gow return build_result 24445ba7a89SDavid Gow 245db167981SDaniel Latypov exec_result = exec_tests(linux, request) 24645ba7a89SDavid Gow 24745ba7a89SDavid Gow run_end = time.time() 24845ba7a89SDavid Gow 2496ebf5866SFelix Guo kunit_parser.print_with_timestamp(( 2506ebf5866SFelix Guo 'Elapsed time: %.3fs total, %.3fs configuring, %.3fs ' + 2516ebf5866SFelix Guo 'building, %.3fs running\n') % ( 25245ba7a89SDavid Gow run_end - run_start, 25345ba7a89SDavid Gow config_result.elapsed_time, 25445ba7a89SDavid Gow build_result.elapsed_time, 25545ba7a89SDavid Gow exec_result.elapsed_time)) 2567ef925eaSDaniel Latypov return exec_result 2576ebf5866SFelix Guo 258d8c23eadSDaniel Latypov# Problem: 259d8c23eadSDaniel Latypov# $ kunit.py run --json 260d8c23eadSDaniel Latypov# works as one would expect and prints the parsed test results as JSON. 261d8c23eadSDaniel Latypov# $ kunit.py run --json suite_name 262d8c23eadSDaniel Latypov# would *not* pass suite_name as the filter_glob and print as json. 263d8c23eadSDaniel Latypov# argparse will consider it to be another way of writing 264d8c23eadSDaniel Latypov# $ kunit.py run --json=suite_name 265d8c23eadSDaniel Latypov# i.e. it would run all tests, and dump the json to a `suite_name` file. 266d8c23eadSDaniel Latypov# So we hackily automatically rewrite --json => --json=stdout 267d8c23eadSDaniel Latypovpseudo_bool_flag_defaults = { 268d8c23eadSDaniel Latypov '--json': 'stdout', 269d8c23eadSDaniel Latypov '--raw_output': 'kunit', 270d8c23eadSDaniel Latypov} 271d8c23eadSDaniel Latypovdef massage_argv(argv: Sequence[str]) -> Sequence[str]: 272d8c23eadSDaniel Latypov def massage_arg(arg: str) -> str: 273d8c23eadSDaniel Latypov if arg not in pseudo_bool_flag_defaults: 274d8c23eadSDaniel Latypov return arg 275d8c23eadSDaniel Latypov return f'{arg}={pseudo_bool_flag_defaults[arg]}' 276d8c23eadSDaniel Latypov return list(map(massage_arg, argv)) 277d8c23eadSDaniel Latypov 278ad659ccbSDavid Gowdef get_default_jobs() -> int: 279ad659ccbSDavid Gow return len(os.sched_getaffinity(0)) 280ad659ccbSDavid Gow 28109641f7cSDaniel Latypovdef add_common_opts(parser) -> None: 28245ba7a89SDavid Gow parser.add_argument('--build_dir', 28345ba7a89SDavid Gow help='As in the make command, it specifies the build ' 28445ba7a89SDavid Gow 'directory.', 285*baa33315SDaniel Latypov type=str, default='.kunit', metavar='DIR') 28645ba7a89SDavid Gow parser.add_argument('--make_options', 28745ba7a89SDavid Gow help='X=Y make option, can be repeated.', 288*baa33315SDaniel Latypov action='append', metavar='X=Y') 28945ba7a89SDavid Gow parser.add_argument('--alltests', 29045ba7a89SDavid Gow help='Run all KUnit tests through allyesconfig', 2916ebf5866SFelix Guo action='store_true') 292243180f5SDaniel Latypov parser.add_argument('--kunitconfig', 2939854781dSDaniel Latypov help='Path to Kconfig fragment that enables KUnit tests.' 2949854781dSDaniel Latypov ' If given a directory, (e.g. lib/kunit), "/.kunitconfig" ' 2959854781dSDaniel Latypov 'will get automatically appended.', 296*baa33315SDaniel Latypov metavar='PATH') 2979f57cc76SDaniel Latypov parser.add_argument('--kconfig_add', 2989f57cc76SDaniel Latypov help='Additional Kconfig options to append to the ' 2999f57cc76SDaniel Latypov '.kunitconfig, e.g. CONFIG_KASAN=y. Can be repeated.', 300*baa33315SDaniel Latypov action='append', metavar='CONFIG_X=Y') 3016ebf5866SFelix Guo 30287c9c163SBrendan Higgins parser.add_argument('--arch', 30387c9c163SBrendan Higgins help=('Specifies the architecture to run tests under. ' 30487c9c163SBrendan Higgins 'The architecture specified here must match the ' 30587c9c163SBrendan Higgins 'string passed to the ARCH make param, ' 30687c9c163SBrendan Higgins 'e.g. i386, x86_64, arm, um, etc. Non-UML ' 30787c9c163SBrendan Higgins 'architectures run on QEMU.'), 308*baa33315SDaniel Latypov type=str, default='um', metavar='ARCH') 30987c9c163SBrendan Higgins 31087c9c163SBrendan Higgins parser.add_argument('--cross_compile', 31187c9c163SBrendan Higgins help=('Sets make\'s CROSS_COMPILE variable; it should ' 31287c9c163SBrendan Higgins 'be set to a toolchain path prefix (the prefix ' 31387c9c163SBrendan Higgins 'of gcc and other tools in your toolchain, for ' 31487c9c163SBrendan Higgins 'example `sparc64-linux-gnu-` if you have the ' 31587c9c163SBrendan Higgins 'sparc toolchain installed on your system, or ' 31687c9c163SBrendan Higgins '`$HOME/toolchains/microblaze/gcc-9.2.0-nolibc/microblaze-linux/bin/microblaze-linux-` ' 31787c9c163SBrendan Higgins 'if you have downloaded the microblaze toolchain ' 31887c9c163SBrendan Higgins 'from the 0-day website to a directory in your ' 31987c9c163SBrendan Higgins 'home directory called `toolchains`).'), 320*baa33315SDaniel Latypov metavar='PREFIX') 32187c9c163SBrendan Higgins 32287c9c163SBrendan Higgins parser.add_argument('--qemu_config', 32387c9c163SBrendan Higgins help=('Takes a path to a path to a file containing ' 32487c9c163SBrendan Higgins 'a QemuArchParams object.'), 325*baa33315SDaniel Latypov type=str, metavar='FILE') 32687c9c163SBrendan Higgins 32709641f7cSDaniel Latypovdef add_build_opts(parser) -> None: 32845ba7a89SDavid Gow parser.add_argument('--jobs', 32945ba7a89SDavid Gow help='As in the make command, "Specifies the number of ' 33045ba7a89SDavid Gow 'jobs (commands) to run simultaneously."', 331*baa33315SDaniel Latypov type=int, default=get_default_jobs(), metavar='N') 33245ba7a89SDavid Gow 33309641f7cSDaniel Latypovdef add_exec_opts(parser) -> None: 33445ba7a89SDavid Gow parser.add_argument('--timeout', 3356ebf5866SFelix Guo help='maximum number of seconds to allow for all tests ' 3366ebf5866SFelix Guo 'to run. This does not include time taken to build the ' 3376ebf5866SFelix Guo 'tests.', 3386ebf5866SFelix Guo type=int, 3396ebf5866SFelix Guo default=300, 340*baa33315SDaniel Latypov metavar='SECONDS') 341d992880bSDaniel Latypov parser.add_argument('filter_glob', 342a127b154SDaniel Latypov help='Filter which KUnit test suites/tests run at ' 343a127b154SDaniel Latypov 'boot-time, e.g. list* or list*.*del_test', 344d992880bSDaniel Latypov type=str, 345d992880bSDaniel Latypov nargs='?', 346d992880bSDaniel Latypov default='', 347d992880bSDaniel Latypov metavar='filter_glob') 3486cb51a18SDaniel Latypov parser.add_argument('--kernel_args', 3496cb51a18SDaniel Latypov help='Kernel command-line parameters. Maybe be repeated', 350*baa33315SDaniel Latypov action='append', metavar='') 351ff9e09a3SDaniel Latypov parser.add_argument('--run_isolated', help='If set, boot the kernel for each ' 352ff9e09a3SDaniel Latypov 'individual suite/test. This is can be useful for debugging ' 353ff9e09a3SDaniel Latypov 'a non-hermetic test, one that might pass/fail based on ' 354ff9e09a3SDaniel Latypov 'what ran before it.', 355ff9e09a3SDaniel Latypov type=str, 356ff9e09a3SDaniel Latypov choices=['suite', 'test']), 3576ebf5866SFelix Guo 35809641f7cSDaniel Latypovdef add_parse_opts(parser) -> None: 3596a499c9cSDaniel Latypov parser.add_argument('--raw_output', help='If set don\'t format output from kernel. ' 3606a499c9cSDaniel Latypov 'If set to --raw_output=kunit, filters to just KUnit output.', 361*baa33315SDaniel Latypov type=str, nargs='?', const='all', default=None, choices=['all', 'kunit']) 36221a6d178SHeidi Fahim parser.add_argument('--json', 36321a6d178SHeidi Fahim nargs='?', 36421a6d178SHeidi Fahim help='Stores test results in a JSON, and either ' 36521a6d178SHeidi Fahim 'prints to stdout or saves to file if a ' 36621a6d178SHeidi Fahim 'filename is specified', 367*baa33315SDaniel Latypov type=str, const='stdout', default=None, metavar='FILE') 368021ed9f5SHeidi Fahim 36945ba7a89SDavid Gowdef main(argv, linux=None): 37045ba7a89SDavid Gow parser = argparse.ArgumentParser( 37145ba7a89SDavid Gow description='Helps writing and running KUnit tests.') 37245ba7a89SDavid Gow subparser = parser.add_subparsers(dest='subcommand') 37345ba7a89SDavid Gow 37445ba7a89SDavid Gow # The 'run' command will config, build, exec, and parse in one go. 37545ba7a89SDavid Gow run_parser = subparser.add_parser('run', help='Runs KUnit tests.') 37645ba7a89SDavid Gow add_common_opts(run_parser) 37745ba7a89SDavid Gow add_build_opts(run_parser) 37845ba7a89SDavid Gow add_exec_opts(run_parser) 37945ba7a89SDavid Gow add_parse_opts(run_parser) 38045ba7a89SDavid Gow 38145ba7a89SDavid Gow config_parser = subparser.add_parser('config', 38245ba7a89SDavid Gow help='Ensures that .config contains all of ' 38345ba7a89SDavid Gow 'the options in .kunitconfig') 38445ba7a89SDavid Gow add_common_opts(config_parser) 38545ba7a89SDavid Gow 38645ba7a89SDavid Gow build_parser = subparser.add_parser('build', help='Builds a kernel with KUnit tests') 38745ba7a89SDavid Gow add_common_opts(build_parser) 38845ba7a89SDavid Gow add_build_opts(build_parser) 38945ba7a89SDavid Gow 39045ba7a89SDavid Gow exec_parser = subparser.add_parser('exec', help='Run a kernel with KUnit tests') 39145ba7a89SDavid Gow add_common_opts(exec_parser) 39245ba7a89SDavid Gow add_exec_opts(exec_parser) 39345ba7a89SDavid Gow add_parse_opts(exec_parser) 39445ba7a89SDavid Gow 39545ba7a89SDavid Gow # The 'parse' option is special, as it doesn't need the kernel source 39645ba7a89SDavid Gow # (therefore there is no need for a build_dir, hence no add_common_opts) 39745ba7a89SDavid Gow # and the '--file' argument is not relevant to 'run', so isn't in 39845ba7a89SDavid Gow # add_parse_opts() 39945ba7a89SDavid Gow parse_parser = subparser.add_parser('parse', 40045ba7a89SDavid Gow help='Parses KUnit results from a file, ' 40145ba7a89SDavid Gow 'and parses formatted results.') 40245ba7a89SDavid Gow add_parse_opts(parse_parser) 40345ba7a89SDavid Gow parse_parser.add_argument('file', 40445ba7a89SDavid Gow help='Specifies the file to read results from.', 40545ba7a89SDavid Gow type=str, nargs='?', metavar='input_file') 4060476e69fSGreg Thelen 407d8c23eadSDaniel Latypov cli_args = parser.parse_args(massage_argv(argv)) 4086ebf5866SFelix Guo 4095578d008SBrendan Higgins if get_kernel_root_path(): 4105578d008SBrendan Higgins os.chdir(get_kernel_root_path()) 4115578d008SBrendan Higgins 4126ebf5866SFelix Guo if cli_args.subcommand == 'run': 413e3212513SSeongJae Park if not os.path.exists(cli_args.build_dir): 414e3212513SSeongJae Park os.mkdir(cli_args.build_dir) 41582206a0cSBrendan Higgins 416ff7b437fSBrendan Higgins if not linux: 41787c9c163SBrendan Higgins linux = kunit_kernel.LinuxSourceTree(cli_args.build_dir, 41887c9c163SBrendan Higgins kunitconfig_path=cli_args.kunitconfig, 4199f57cc76SDaniel Latypov kconfig_add=cli_args.kconfig_add, 42087c9c163SBrendan Higgins arch=cli_args.arch, 42187c9c163SBrendan Higgins cross_compile=cli_args.cross_compile, 42287c9c163SBrendan Higgins qemu_config_path=cli_args.qemu_config) 423fcdb0bc0SAndy Shevchenko 424db167981SDaniel Latypov request = KunitRequest(build_dir=cli_args.build_dir, 425db167981SDaniel Latypov make_options=cli_args.make_options, 426db167981SDaniel Latypov jobs=cli_args.jobs, 427db167981SDaniel Latypov alltests=cli_args.alltests, 428db167981SDaniel Latypov raw_output=cli_args.raw_output, 429db167981SDaniel Latypov json=cli_args.json, 430db167981SDaniel Latypov timeout=cli_args.timeout, 431db167981SDaniel Latypov filter_glob=cli_args.filter_glob, 432db167981SDaniel Latypov kernel_args=cli_args.kernel_args, 433db167981SDaniel Latypov run_isolated=cli_args.run_isolated) 4346ebf5866SFelix Guo result = run_tests(linux, request) 4356ebf5866SFelix Guo if result.status != KunitStatus.SUCCESS: 4366ebf5866SFelix Guo sys.exit(1) 43745ba7a89SDavid Gow elif cli_args.subcommand == 'config': 43882206a0cSBrendan Higgins if cli_args.build_dir and ( 43982206a0cSBrendan Higgins not os.path.exists(cli_args.build_dir)): 44045ba7a89SDavid Gow os.mkdir(cli_args.build_dir) 44182206a0cSBrendan Higgins 44245ba7a89SDavid Gow if not linux: 44387c9c163SBrendan Higgins linux = kunit_kernel.LinuxSourceTree(cli_args.build_dir, 44487c9c163SBrendan Higgins kunitconfig_path=cli_args.kunitconfig, 4459f57cc76SDaniel Latypov kconfig_add=cli_args.kconfig_add, 44687c9c163SBrendan Higgins arch=cli_args.arch, 44787c9c163SBrendan Higgins cross_compile=cli_args.cross_compile, 44887c9c163SBrendan Higgins qemu_config_path=cli_args.qemu_config) 449fcdb0bc0SAndy Shevchenko 450db167981SDaniel Latypov request = KunitConfigRequest(build_dir=cli_args.build_dir, 451db167981SDaniel Latypov make_options=cli_args.make_options) 45245ba7a89SDavid Gow result = config_tests(linux, request) 45345ba7a89SDavid Gow kunit_parser.print_with_timestamp(( 45445ba7a89SDavid Gow 'Elapsed time: %.3fs\n') % ( 45545ba7a89SDavid Gow result.elapsed_time)) 45645ba7a89SDavid Gow if result.status != KunitStatus.SUCCESS: 45745ba7a89SDavid Gow sys.exit(1) 45845ba7a89SDavid Gow elif cli_args.subcommand == 'build': 45945ba7a89SDavid Gow if not linux: 46087c9c163SBrendan Higgins linux = kunit_kernel.LinuxSourceTree(cli_args.build_dir, 46187c9c163SBrendan Higgins kunitconfig_path=cli_args.kunitconfig, 4629f57cc76SDaniel Latypov kconfig_add=cli_args.kconfig_add, 46387c9c163SBrendan Higgins arch=cli_args.arch, 46487c9c163SBrendan Higgins cross_compile=cli_args.cross_compile, 46587c9c163SBrendan Higgins qemu_config_path=cli_args.qemu_config) 466fcdb0bc0SAndy Shevchenko 467db167981SDaniel Latypov request = KunitBuildRequest(build_dir=cli_args.build_dir, 468db167981SDaniel Latypov make_options=cli_args.make_options, 469db167981SDaniel Latypov jobs=cli_args.jobs, 470db167981SDaniel Latypov alltests=cli_args.alltests) 4711ee2ba89SDaniel Latypov result = config_and_build_tests(linux, request) 47245ba7a89SDavid Gow kunit_parser.print_with_timestamp(( 47345ba7a89SDavid Gow 'Elapsed time: %.3fs\n') % ( 47445ba7a89SDavid Gow result.elapsed_time)) 47545ba7a89SDavid Gow if result.status != KunitStatus.SUCCESS: 47645ba7a89SDavid Gow sys.exit(1) 47745ba7a89SDavid Gow elif cli_args.subcommand == 'exec': 47845ba7a89SDavid Gow if not linux: 47987c9c163SBrendan Higgins linux = kunit_kernel.LinuxSourceTree(cli_args.build_dir, 48087c9c163SBrendan Higgins kunitconfig_path=cli_args.kunitconfig, 4819f57cc76SDaniel Latypov kconfig_add=cli_args.kconfig_add, 48287c9c163SBrendan Higgins arch=cli_args.arch, 48387c9c163SBrendan Higgins cross_compile=cli_args.cross_compile, 48487c9c163SBrendan Higgins qemu_config_path=cli_args.qemu_config) 485fcdb0bc0SAndy Shevchenko 486db167981SDaniel Latypov exec_request = KunitExecRequest(raw_output=cli_args.raw_output, 487db167981SDaniel Latypov build_dir=cli_args.build_dir, 488db167981SDaniel Latypov json=cli_args.json, 489db167981SDaniel Latypov timeout=cli_args.timeout, 490db167981SDaniel Latypov alltests=cli_args.alltests, 491db167981SDaniel Latypov filter_glob=cli_args.filter_glob, 492db167981SDaniel Latypov kernel_args=cli_args.kernel_args, 493db167981SDaniel Latypov run_isolated=cli_args.run_isolated) 494db167981SDaniel Latypov result = exec_tests(linux, exec_request) 49545ba7a89SDavid Gow kunit_parser.print_with_timestamp(( 4967ef925eaSDaniel Latypov 'Elapsed time: %.3fs\n') % (result.elapsed_time)) 49745ba7a89SDavid Gow if result.status != KunitStatus.SUCCESS: 49845ba7a89SDavid Gow sys.exit(1) 49945ba7a89SDavid Gow elif cli_args.subcommand == 'parse': 50045ba7a89SDavid Gow if cli_args.file == None: 5012ab5d5e6SDaniel Latypov sys.stdin.reconfigure(errors='backslashreplace') # pytype: disable=attribute-error 50245ba7a89SDavid Gow kunit_output = sys.stdin 50345ba7a89SDavid Gow else: 5042ab5d5e6SDaniel Latypov with open(cli_args.file, 'r', errors='backslashreplace') as f: 50545ba7a89SDavid Gow kunit_output = f.read().splitlines() 506ee96d25fSDaniel Latypov # We know nothing about how the result was created! 507ee96d25fSDaniel Latypov metadata = kunit_json.Metadata() 508db167981SDaniel Latypov request = KunitParseRequest(raw_output=cli_args.raw_output, 509db167981SDaniel Latypov json=cli_args.json) 510ee96d25fSDaniel Latypov result, _ = parse_tests(request, metadata, kunit_output) 51145ba7a89SDavid Gow if result.status != KunitStatus.SUCCESS: 51245ba7a89SDavid Gow sys.exit(1) 5136ebf5866SFelix Guo else: 5146ebf5866SFelix Guo parser.print_help() 5156ebf5866SFelix Guo 5166ebf5866SFelix Guoif __name__ == '__main__': 517ff7b437fSBrendan Higgins main(sys.argv[1:]) 518