1c25ce589SFinn Behrens#!/usr/bin/env python3 26ebf5866SFelix Guo# SPDX-License-Identifier: GPL-2.0 36ebf5866SFelix Guo# 46ebf5866SFelix Guo# A thin wrapper on top of the KUnit Kernel 56ebf5866SFelix Guo# 66ebf5866SFelix Guo# Copyright (C) 2019, Google LLC. 76ebf5866SFelix Guo# Author: Felix Guo <felixguoxiuping@gmail.com> 86ebf5866SFelix Guo# Author: Brendan Higgins <brendanhiggins@google.com> 96ebf5866SFelix Guo 106ebf5866SFelix Guoimport argparse 116ebf5866SFelix Guoimport os 12ff9e09a3SDaniel Latypovimport re 13ff9e09a3SDaniel Latypovimport sys 146ebf5866SFelix Guoimport time 156ebf5866SFelix Guo 16df4b0807SSeongJae Parkassert sys.version_info >= (3, 7), "Python version is too old" 17df4b0807SSeongJae Park 18db167981SDaniel Latypovfrom dataclasses import dataclass 196ebf5866SFelix Guofrom enum import Enum, auto 2095dcbc55SDaniel Latypovfrom typing import Iterable, List, Optional, Sequence, Tuple 216ebf5866SFelix Guo 2221a6d178SHeidi Fahimimport kunit_json 236ebf5866SFelix Guoimport kunit_kernel 246ebf5866SFelix Guoimport kunit_parser 25*e756dbebSDaniel Latypovfrom kunit_printer import stdout 266ebf5866SFelix Guo 276ebf5866SFelix Guoclass KunitStatus(Enum): 286ebf5866SFelix Guo SUCCESS = auto() 296ebf5866SFelix Guo CONFIG_FAILURE = auto() 306ebf5866SFelix Guo BUILD_FAILURE = auto() 316ebf5866SFelix Guo TEST_FAILURE = auto() 326ebf5866SFelix Guo 33db167981SDaniel Latypov@dataclass 34db167981SDaniel Latypovclass KunitResult: 35db167981SDaniel Latypov status: KunitStatus 36db167981SDaniel Latypov elapsed_time: float 37db167981SDaniel Latypov 38db167981SDaniel Latypov@dataclass 39db167981SDaniel Latypovclass KunitConfigRequest: 40db167981SDaniel Latypov build_dir: str 41db167981SDaniel Latypov make_options: Optional[List[str]] 42db167981SDaniel Latypov 43db167981SDaniel Latypov@dataclass 44db167981SDaniel Latypovclass KunitBuildRequest(KunitConfigRequest): 45db167981SDaniel Latypov jobs: int 46db167981SDaniel Latypov alltests: bool 47db167981SDaniel Latypov 48db167981SDaniel Latypov@dataclass 49db167981SDaniel Latypovclass KunitParseRequest: 50db167981SDaniel Latypov raw_output: Optional[str] 51db167981SDaniel Latypov json: Optional[str] 52db167981SDaniel Latypov 53db167981SDaniel Latypov@dataclass 54db167981SDaniel Latypovclass KunitExecRequest(KunitParseRequest): 55ee96d25fSDaniel Latypov build_dir: str 56db167981SDaniel Latypov timeout: int 57db167981SDaniel Latypov alltests: bool 58db167981SDaniel Latypov filter_glob: str 59db167981SDaniel Latypov kernel_args: Optional[List[str]] 60db167981SDaniel Latypov run_isolated: Optional[str] 61db167981SDaniel Latypov 62db167981SDaniel Latypov@dataclass 63db167981SDaniel Latypovclass KunitRequest(KunitExecRequest, KunitBuildRequest): 64db167981SDaniel Latypov pass 65db167981SDaniel Latypov 66db167981SDaniel Latypov 6709641f7cSDaniel Latypovdef get_kernel_root_path() -> str: 6809641f7cSDaniel Latypov path = sys.argv[0] if not __file__ else __file__ 6909641f7cSDaniel Latypov parts = os.path.realpath(path).split('tools/testing/kunit') 70be886ba9SHeidi Fahim if len(parts) != 2: 71be886ba9SHeidi Fahim sys.exit(1) 72be886ba9SHeidi Fahim return parts[0] 73be886ba9SHeidi Fahim 7445ba7a89SDavid Gowdef config_tests(linux: kunit_kernel.LinuxSourceTree, 7545ba7a89SDavid Gow request: KunitConfigRequest) -> KunitResult: 76*e756dbebSDaniel Latypov stdout.print_with_timestamp('Configuring KUnit Kernel ...') 7745ba7a89SDavid Gow 786ebf5866SFelix Guo config_start = time.time() 790476e69fSGreg Thelen success = linux.build_reconfig(request.build_dir, request.make_options) 806ebf5866SFelix Guo config_end = time.time() 816ebf5866SFelix Guo if not success: 8245ba7a89SDavid Gow return KunitResult(KunitStatus.CONFIG_FAILURE, 8345ba7a89SDavid Gow config_end - config_start) 8445ba7a89SDavid Gow return KunitResult(KunitStatus.SUCCESS, 8545ba7a89SDavid Gow config_end - config_start) 866ebf5866SFelix Guo 8745ba7a89SDavid Gowdef build_tests(linux: kunit_kernel.LinuxSourceTree, 8845ba7a89SDavid Gow request: KunitBuildRequest) -> KunitResult: 89*e756dbebSDaniel Latypov stdout.print_with_timestamp('Building KUnit Kernel ...') 906ebf5866SFelix Guo 916ebf5866SFelix Guo build_start = time.time() 9287c9c163SBrendan Higgins success = linux.build_kernel(request.alltests, 93021ed9f5SHeidi Fahim request.jobs, 940476e69fSGreg Thelen request.build_dir, 950476e69fSGreg Thelen request.make_options) 966ebf5866SFelix Guo build_end = time.time() 976ebf5866SFelix Guo if not success: 98ee61492aSDavid Gow return KunitResult(KunitStatus.BUILD_FAILURE, 99ee61492aSDavid Gow build_end - build_start) 10045ba7a89SDavid Gow if not success: 10145ba7a89SDavid Gow return KunitResult(KunitStatus.BUILD_FAILURE, 10245ba7a89SDavid Gow build_end - build_start) 10345ba7a89SDavid Gow return KunitResult(KunitStatus.SUCCESS, 10445ba7a89SDavid Gow build_end - build_start) 1056ebf5866SFelix Guo 1061ee2ba89SDaniel Latypovdef config_and_build_tests(linux: kunit_kernel.LinuxSourceTree, 1071ee2ba89SDaniel Latypov request: KunitBuildRequest) -> KunitResult: 1081ee2ba89SDaniel Latypov config_result = config_tests(linux, request) 1091ee2ba89SDaniel Latypov if config_result.status != KunitStatus.SUCCESS: 1101ee2ba89SDaniel Latypov return config_result 1111ee2ba89SDaniel Latypov 1121ee2ba89SDaniel Latypov return build_tests(linux, request) 1131ee2ba89SDaniel Latypov 114ff9e09a3SDaniel Latypovdef _list_tests(linux: kunit_kernel.LinuxSourceTree, request: KunitExecRequest) -> List[str]: 115ff9e09a3SDaniel Latypov args = ['kunit.action=list'] 116ff9e09a3SDaniel Latypov if request.kernel_args: 117ff9e09a3SDaniel Latypov args.extend(request.kernel_args) 118ff9e09a3SDaniel Latypov 119ff9e09a3SDaniel Latypov output = linux.run_kernel(args=args, 120ff9e09a3SDaniel Latypov timeout=None if request.alltests else request.timeout, 121ff9e09a3SDaniel Latypov filter_glob=request.filter_glob, 122ff9e09a3SDaniel Latypov build_dir=request.build_dir) 123ff9e09a3SDaniel Latypov lines = kunit_parser.extract_tap_lines(output) 124ff9e09a3SDaniel Latypov # Hack! Drop the dummy TAP version header that the executor prints out. 125ff9e09a3SDaniel Latypov lines.pop() 126ff9e09a3SDaniel Latypov 127ff9e09a3SDaniel Latypov # Filter out any extraneous non-test output that might have gotten mixed in. 1280453f984SDaniel Latypov return [l for l in lines if re.match(r'^[^\s.]+\.[^\s.]+$', l)] 129ff9e09a3SDaniel Latypov 130ff9e09a3SDaniel Latypovdef _suites_from_test_list(tests: List[str]) -> List[str]: 131ff9e09a3SDaniel Latypov """Extracts all the suites from an ordered list of tests.""" 132ff9e09a3SDaniel Latypov suites = [] # type: List[str] 133ff9e09a3SDaniel Latypov for t in tests: 134ff9e09a3SDaniel Latypov parts = t.split('.', maxsplit=2) 135ff9e09a3SDaniel Latypov if len(parts) != 2: 136ff9e09a3SDaniel Latypov raise ValueError(f'internal KUnit error, test name should be of the form "<suite>.<test>", got "{t}"') 137ff9e09a3SDaniel Latypov suite, case = parts 138ff9e09a3SDaniel Latypov if not suites or suites[-1] != suite: 139ff9e09a3SDaniel Latypov suites.append(suite) 140ff9e09a3SDaniel Latypov return suites 141ff9e09a3SDaniel Latypov 142ff9e09a3SDaniel Latypov 143ff9e09a3SDaniel Latypov 144db167981SDaniel Latypovdef exec_tests(linux: kunit_kernel.LinuxSourceTree, request: KunitExecRequest) -> KunitResult: 145ff9e09a3SDaniel Latypov filter_globs = [request.filter_glob] 146ff9e09a3SDaniel Latypov if request.run_isolated: 147ff9e09a3SDaniel Latypov tests = _list_tests(linux, request) 148ff9e09a3SDaniel Latypov if request.run_isolated == 'test': 149ff9e09a3SDaniel Latypov filter_globs = tests 150ff9e09a3SDaniel Latypov if request.run_isolated == 'suite': 151ff9e09a3SDaniel Latypov filter_globs = _suites_from_test_list(tests) 152ff9e09a3SDaniel Latypov # Apply the test-part of the user's glob, if present. 153ff9e09a3SDaniel Latypov if '.' in request.filter_glob: 154ff9e09a3SDaniel Latypov test_glob = request.filter_glob.split('.', maxsplit=2)[1] 155ff9e09a3SDaniel Latypov filter_globs = [g + '.'+ test_glob for g in filter_globs] 156ff9e09a3SDaniel Latypov 157885210d3SDaniel Latypov metadata = kunit_json.Metadata(arch=linux.arch(), build_dir=request.build_dir, def_config='kunit_defconfig') 158ee96d25fSDaniel Latypov 159d65d07cbSRae Moar test_counts = kunit_parser.TestCounts() 160ff9e09a3SDaniel Latypov exec_time = 0.0 161ff9e09a3SDaniel Latypov for i, filter_glob in enumerate(filter_globs): 162*e756dbebSDaniel Latypov stdout.print_with_timestamp('Starting KUnit Kernel ({}/{})...'.format(i+1, len(filter_globs))) 163ff9e09a3SDaniel Latypov 1646ebf5866SFelix Guo test_start = time.time() 1657ef925eaSDaniel Latypov run_result = linux.run_kernel( 1666cb51a18SDaniel Latypov args=request.kernel_args, 167021ed9f5SHeidi Fahim timeout=None if request.alltests else request.timeout, 168ff9e09a3SDaniel Latypov filter_glob=filter_glob, 1696ec1b81dSSeongJae Park build_dir=request.build_dir) 17045ba7a89SDavid Gow 171ee96d25fSDaniel Latypov _, test_result = parse_tests(request, metadata, run_result) 1725f6aa6d8SDaniel Latypov # run_kernel() doesn't block on the kernel exiting. 1735f6aa6d8SDaniel Latypov # That only happens after we get the last line of output from `run_result`. 1745f6aa6d8SDaniel Latypov # So exec_time here actually contains parsing + execution time, which is fine. 1756ebf5866SFelix Guo test_end = time.time() 176ff9e09a3SDaniel Latypov exec_time += test_end - test_start 177ff9e09a3SDaniel Latypov 17895dcbc55SDaniel Latypov test_counts.add_subtest_counts(test_result.counts) 1796ebf5866SFelix Guo 1807fa7ffcfSDaniel Latypov if len(filter_globs) == 1 and test_counts.crashed > 0: 1817fa7ffcfSDaniel Latypov bd = request.build_dir 1827fa7ffcfSDaniel Latypov print('The kernel seems to have crashed; you can decode the stack traces with:') 1837fa7ffcfSDaniel Latypov print('$ scripts/decode_stacktrace.sh {}/vmlinux {} < {} | tee {}/decoded.log | {} parse'.format( 1847fa7ffcfSDaniel Latypov bd, bd, kunit_kernel.get_outfile_path(bd), bd, sys.argv[0])) 1857fa7ffcfSDaniel Latypov 186d65d07cbSRae Moar kunit_status = _map_to_overall_status(test_counts.get_status()) 18795dcbc55SDaniel Latypov return KunitResult(status=kunit_status, elapsed_time=exec_time) 188d65d07cbSRae Moar 189d65d07cbSRae Moardef _map_to_overall_status(test_status: kunit_parser.TestStatus) -> KunitStatus: 190d65d07cbSRae Moar if test_status in (kunit_parser.TestStatus.SUCCESS, kunit_parser.TestStatus.SKIPPED): 191d65d07cbSRae Moar return KunitStatus.SUCCESS 192d65d07cbSRae Moar return KunitStatus.TEST_FAILURE 1937ef925eaSDaniel Latypov 194ee96d25fSDaniel Latypovdef parse_tests(request: KunitParseRequest, metadata: kunit_json.Metadata, input_data: Iterable[str]) -> Tuple[KunitResult, kunit_parser.Test]: 19545ba7a89SDavid Gow parse_start = time.time() 19645ba7a89SDavid Gow 197e0cc8c05SDaniel Latypov test_result = kunit_parser.Test() 19821a6d178SHeidi Fahim 19945ba7a89SDavid Gow if request.raw_output: 200d65d07cbSRae Moar # Treat unparsed results as one passing test. 201e0cc8c05SDaniel Latypov test_result.status = kunit_parser.TestStatus.SUCCESS 202e0cc8c05SDaniel Latypov test_result.counts.passed = 1 203d65d07cbSRae Moar 2047ef925eaSDaniel Latypov output: Iterable[str] = input_data 2056a499c9cSDaniel Latypov if request.raw_output == 'all': 2066a499c9cSDaniel Latypov pass 2076a499c9cSDaniel Latypov elif request.raw_output == 'kunit': 2086a499c9cSDaniel Latypov output = kunit_parser.extract_tap_lines(output) 2096a499c9cSDaniel Latypov for line in output: 2106a499c9cSDaniel Latypov print(line.rstrip()) 2116a499c9cSDaniel Latypov 21245ba7a89SDavid Gow else: 2137ef925eaSDaniel Latypov test_result = kunit_parser.parse_run_tests(input_data) 21445ba7a89SDavid Gow parse_end = time.time() 21545ba7a89SDavid Gow 21621a6d178SHeidi Fahim if request.json: 21700f75043SDaniel Latypov json_str = kunit_json.get_json_result( 218e0cc8c05SDaniel Latypov test=test_result, 219ee96d25fSDaniel Latypov metadata=metadata) 22021a6d178SHeidi Fahim if request.json == 'stdout': 22100f75043SDaniel Latypov print(json_str) 22200f75043SDaniel Latypov else: 22300f75043SDaniel Latypov with open(request.json, 'w') as f: 22400f75043SDaniel Latypov f.write(json_str) 225*e756dbebSDaniel Latypov stdout.print_with_timestamp("Test results stored in %s" % 22600f75043SDaniel Latypov os.path.abspath(request.json)) 22721a6d178SHeidi Fahim 22845ba7a89SDavid Gow if test_result.status != kunit_parser.TestStatus.SUCCESS: 22995dcbc55SDaniel Latypov return KunitResult(KunitStatus.TEST_FAILURE, parse_end - parse_start), test_result 23045ba7a89SDavid Gow 23195dcbc55SDaniel Latypov return KunitResult(KunitStatus.SUCCESS, parse_end - parse_start), test_result 23245ba7a89SDavid Gow 23345ba7a89SDavid Gowdef run_tests(linux: kunit_kernel.LinuxSourceTree, 23445ba7a89SDavid Gow request: KunitRequest) -> KunitResult: 23545ba7a89SDavid Gow run_start = time.time() 23645ba7a89SDavid Gow 237db167981SDaniel Latypov config_result = config_tests(linux, request) 23845ba7a89SDavid Gow if config_result.status != KunitStatus.SUCCESS: 23945ba7a89SDavid Gow return config_result 24045ba7a89SDavid Gow 241db167981SDaniel Latypov build_result = build_tests(linux, request) 24245ba7a89SDavid Gow if build_result.status != KunitStatus.SUCCESS: 24345ba7a89SDavid Gow return build_result 24445ba7a89SDavid Gow 245db167981SDaniel Latypov exec_result = exec_tests(linux, request) 24645ba7a89SDavid Gow 24745ba7a89SDavid Gow run_end = time.time() 24845ba7a89SDavid Gow 249*e756dbebSDaniel Latypov stdout.print_with_timestamp(( 2506ebf5866SFelix Guo 'Elapsed time: %.3fs total, %.3fs configuring, %.3fs ' + 2516ebf5866SFelix Guo 'building, %.3fs running\n') % ( 25245ba7a89SDavid Gow run_end - run_start, 25345ba7a89SDavid Gow config_result.elapsed_time, 25445ba7a89SDavid Gow build_result.elapsed_time, 25545ba7a89SDavid Gow exec_result.elapsed_time)) 2567ef925eaSDaniel Latypov return exec_result 2576ebf5866SFelix Guo 258d8c23eadSDaniel Latypov# Problem: 259d8c23eadSDaniel Latypov# $ kunit.py run --json 260d8c23eadSDaniel Latypov# works as one would expect and prints the parsed test results as JSON. 261d8c23eadSDaniel Latypov# $ kunit.py run --json suite_name 262d8c23eadSDaniel Latypov# would *not* pass suite_name as the filter_glob and print as json. 263d8c23eadSDaniel Latypov# argparse will consider it to be another way of writing 264d8c23eadSDaniel Latypov# $ kunit.py run --json=suite_name 265d8c23eadSDaniel Latypov# i.e. it would run all tests, and dump the json to a `suite_name` file. 266d8c23eadSDaniel Latypov# So we hackily automatically rewrite --json => --json=stdout 267d8c23eadSDaniel Latypovpseudo_bool_flag_defaults = { 268d8c23eadSDaniel Latypov '--json': 'stdout', 269d8c23eadSDaniel Latypov '--raw_output': 'kunit', 270d8c23eadSDaniel Latypov} 271d8c23eadSDaniel Latypovdef massage_argv(argv: Sequence[str]) -> Sequence[str]: 272d8c23eadSDaniel Latypov def massage_arg(arg: str) -> str: 273d8c23eadSDaniel Latypov if arg not in pseudo_bool_flag_defaults: 274d8c23eadSDaniel Latypov return arg 275d8c23eadSDaniel Latypov return f'{arg}={pseudo_bool_flag_defaults[arg]}' 276d8c23eadSDaniel Latypov return list(map(massage_arg, argv)) 277d8c23eadSDaniel Latypov 278ad659ccbSDavid Gowdef get_default_jobs() -> int: 279ad659ccbSDavid Gow return len(os.sched_getaffinity(0)) 280ad659ccbSDavid Gow 28109641f7cSDaniel Latypovdef add_common_opts(parser) -> None: 28245ba7a89SDavid Gow parser.add_argument('--build_dir', 28345ba7a89SDavid Gow help='As in the make command, it specifies the build ' 28445ba7a89SDavid Gow 'directory.', 285baa33315SDaniel Latypov type=str, default='.kunit', metavar='DIR') 28645ba7a89SDavid Gow parser.add_argument('--make_options', 28745ba7a89SDavid Gow help='X=Y make option, can be repeated.', 288baa33315SDaniel Latypov action='append', metavar='X=Y') 28945ba7a89SDavid Gow parser.add_argument('--alltests', 29045ba7a89SDavid Gow help='Run all KUnit tests through allyesconfig', 2916ebf5866SFelix Guo action='store_true') 292243180f5SDaniel Latypov parser.add_argument('--kunitconfig', 2939854781dSDaniel Latypov help='Path to Kconfig fragment that enables KUnit tests.' 2949854781dSDaniel Latypov ' If given a directory, (e.g. lib/kunit), "/.kunitconfig" ' 2959854781dSDaniel Latypov 'will get automatically appended.', 296baa33315SDaniel Latypov metavar='PATH') 2979f57cc76SDaniel Latypov parser.add_argument('--kconfig_add', 2989f57cc76SDaniel Latypov help='Additional Kconfig options to append to the ' 2999f57cc76SDaniel Latypov '.kunitconfig, e.g. CONFIG_KASAN=y. Can be repeated.', 300baa33315SDaniel Latypov action='append', metavar='CONFIG_X=Y') 3016ebf5866SFelix Guo 30287c9c163SBrendan Higgins parser.add_argument('--arch', 30387c9c163SBrendan Higgins help=('Specifies the architecture to run tests under. ' 30487c9c163SBrendan Higgins 'The architecture specified here must match the ' 30587c9c163SBrendan Higgins 'string passed to the ARCH make param, ' 30687c9c163SBrendan Higgins 'e.g. i386, x86_64, arm, um, etc. Non-UML ' 30787c9c163SBrendan Higgins 'architectures run on QEMU.'), 308baa33315SDaniel Latypov type=str, default='um', metavar='ARCH') 30987c9c163SBrendan Higgins 31087c9c163SBrendan Higgins parser.add_argument('--cross_compile', 31187c9c163SBrendan Higgins help=('Sets make\'s CROSS_COMPILE variable; it should ' 31287c9c163SBrendan Higgins 'be set to a toolchain path prefix (the prefix ' 31387c9c163SBrendan Higgins 'of gcc and other tools in your toolchain, for ' 31487c9c163SBrendan Higgins 'example `sparc64-linux-gnu-` if you have the ' 31587c9c163SBrendan Higgins 'sparc toolchain installed on your system, or ' 31687c9c163SBrendan Higgins '`$HOME/toolchains/microblaze/gcc-9.2.0-nolibc/microblaze-linux/bin/microblaze-linux-` ' 31787c9c163SBrendan Higgins 'if you have downloaded the microblaze toolchain ' 31887c9c163SBrendan Higgins 'from the 0-day website to a directory in your ' 31987c9c163SBrendan Higgins 'home directory called `toolchains`).'), 320baa33315SDaniel Latypov metavar='PREFIX') 32187c9c163SBrendan Higgins 32287c9c163SBrendan Higgins parser.add_argument('--qemu_config', 32387c9c163SBrendan Higgins help=('Takes a path to a path to a file containing ' 32487c9c163SBrendan Higgins 'a QemuArchParams object.'), 325baa33315SDaniel Latypov type=str, metavar='FILE') 32687c9c163SBrendan Higgins 32709641f7cSDaniel Latypovdef add_build_opts(parser) -> None: 32845ba7a89SDavid Gow parser.add_argument('--jobs', 32945ba7a89SDavid Gow help='As in the make command, "Specifies the number of ' 33045ba7a89SDavid Gow 'jobs (commands) to run simultaneously."', 331baa33315SDaniel Latypov type=int, default=get_default_jobs(), metavar='N') 33245ba7a89SDavid Gow 33309641f7cSDaniel Latypovdef add_exec_opts(parser) -> None: 33445ba7a89SDavid Gow parser.add_argument('--timeout', 3356ebf5866SFelix Guo help='maximum number of seconds to allow for all tests ' 3366ebf5866SFelix Guo 'to run. This does not include time taken to build the ' 3376ebf5866SFelix Guo 'tests.', 3386ebf5866SFelix Guo type=int, 3396ebf5866SFelix Guo default=300, 340baa33315SDaniel Latypov metavar='SECONDS') 341d992880bSDaniel Latypov parser.add_argument('filter_glob', 342a127b154SDaniel Latypov help='Filter which KUnit test suites/tests run at ' 343a127b154SDaniel Latypov 'boot-time, e.g. list* or list*.*del_test', 344d992880bSDaniel Latypov type=str, 345d992880bSDaniel Latypov nargs='?', 346d992880bSDaniel Latypov default='', 347d992880bSDaniel Latypov metavar='filter_glob') 3486cb51a18SDaniel Latypov parser.add_argument('--kernel_args', 3496cb51a18SDaniel Latypov help='Kernel command-line parameters. Maybe be repeated', 350baa33315SDaniel Latypov action='append', metavar='') 351ff9e09a3SDaniel Latypov parser.add_argument('--run_isolated', help='If set, boot the kernel for each ' 352ff9e09a3SDaniel Latypov 'individual suite/test. This is can be useful for debugging ' 353ff9e09a3SDaniel Latypov 'a non-hermetic test, one that might pass/fail based on ' 354ff9e09a3SDaniel Latypov 'what ran before it.', 355ff9e09a3SDaniel Latypov type=str, 3560453f984SDaniel Latypov choices=['suite', 'test']) 3576ebf5866SFelix Guo 35809641f7cSDaniel Latypovdef add_parse_opts(parser) -> None: 3596a499c9cSDaniel Latypov parser.add_argument('--raw_output', help='If set don\'t format output from kernel. ' 3606a499c9cSDaniel Latypov 'If set to --raw_output=kunit, filters to just KUnit output.', 361baa33315SDaniel Latypov type=str, nargs='?', const='all', default=None, choices=['all', 'kunit']) 36221a6d178SHeidi Fahim parser.add_argument('--json', 36321a6d178SHeidi Fahim nargs='?', 36421a6d178SHeidi Fahim help='Stores test results in a JSON, and either ' 36521a6d178SHeidi Fahim 'prints to stdout or saves to file if a ' 36621a6d178SHeidi Fahim 'filename is specified', 367baa33315SDaniel Latypov type=str, const='stdout', default=None, metavar='FILE') 368021ed9f5SHeidi Fahim 3698a04930fSDaniel Latypov 3708a04930fSDaniel Latypovdef tree_from_args(cli_args: argparse.Namespace) -> kunit_kernel.LinuxSourceTree: 3718a04930fSDaniel Latypov """Returns a LinuxSourceTree based on the user's arguments.""" 3728a04930fSDaniel Latypov return kunit_kernel.LinuxSourceTree(cli_args.build_dir, 3738a04930fSDaniel Latypov kunitconfig_path=cli_args.kunitconfig, 3748a04930fSDaniel Latypov kconfig_add=cli_args.kconfig_add, 3758a04930fSDaniel Latypov arch=cli_args.arch, 3768a04930fSDaniel Latypov cross_compile=cli_args.cross_compile, 3778a04930fSDaniel Latypov qemu_config_path=cli_args.qemu_config) 3788a04930fSDaniel Latypov 3798a04930fSDaniel Latypov 3808a04930fSDaniel Latypovdef main(argv): 38145ba7a89SDavid Gow parser = argparse.ArgumentParser( 38245ba7a89SDavid Gow description='Helps writing and running KUnit tests.') 38345ba7a89SDavid Gow subparser = parser.add_subparsers(dest='subcommand') 38445ba7a89SDavid Gow 38545ba7a89SDavid Gow # The 'run' command will config, build, exec, and parse in one go. 38645ba7a89SDavid Gow run_parser = subparser.add_parser('run', help='Runs KUnit tests.') 38745ba7a89SDavid Gow add_common_opts(run_parser) 38845ba7a89SDavid Gow add_build_opts(run_parser) 38945ba7a89SDavid Gow add_exec_opts(run_parser) 39045ba7a89SDavid Gow add_parse_opts(run_parser) 39145ba7a89SDavid Gow 39245ba7a89SDavid Gow config_parser = subparser.add_parser('config', 39345ba7a89SDavid Gow help='Ensures that .config contains all of ' 39445ba7a89SDavid Gow 'the options in .kunitconfig') 39545ba7a89SDavid Gow add_common_opts(config_parser) 39645ba7a89SDavid Gow 39745ba7a89SDavid Gow build_parser = subparser.add_parser('build', help='Builds a kernel with KUnit tests') 39845ba7a89SDavid Gow add_common_opts(build_parser) 39945ba7a89SDavid Gow add_build_opts(build_parser) 40045ba7a89SDavid Gow 40145ba7a89SDavid Gow exec_parser = subparser.add_parser('exec', help='Run a kernel with KUnit tests') 40245ba7a89SDavid Gow add_common_opts(exec_parser) 40345ba7a89SDavid Gow add_exec_opts(exec_parser) 40445ba7a89SDavid Gow add_parse_opts(exec_parser) 40545ba7a89SDavid Gow 40645ba7a89SDavid Gow # The 'parse' option is special, as it doesn't need the kernel source 40745ba7a89SDavid Gow # (therefore there is no need for a build_dir, hence no add_common_opts) 40845ba7a89SDavid Gow # and the '--file' argument is not relevant to 'run', so isn't in 40945ba7a89SDavid Gow # add_parse_opts() 41045ba7a89SDavid Gow parse_parser = subparser.add_parser('parse', 41145ba7a89SDavid Gow help='Parses KUnit results from a file, ' 41245ba7a89SDavid Gow 'and parses formatted results.') 41345ba7a89SDavid Gow add_parse_opts(parse_parser) 41445ba7a89SDavid Gow parse_parser.add_argument('file', 41545ba7a89SDavid Gow help='Specifies the file to read results from.', 41645ba7a89SDavid Gow type=str, nargs='?', metavar='input_file') 4170476e69fSGreg Thelen 418d8c23eadSDaniel Latypov cli_args = parser.parse_args(massage_argv(argv)) 4196ebf5866SFelix Guo 4205578d008SBrendan Higgins if get_kernel_root_path(): 4215578d008SBrendan Higgins os.chdir(get_kernel_root_path()) 4225578d008SBrendan Higgins 4236ebf5866SFelix Guo if cli_args.subcommand == 'run': 424e3212513SSeongJae Park if not os.path.exists(cli_args.build_dir): 425e3212513SSeongJae Park os.mkdir(cli_args.build_dir) 42682206a0cSBrendan Higgins 4278a04930fSDaniel Latypov linux = tree_from_args(cli_args) 428db167981SDaniel Latypov request = KunitRequest(build_dir=cli_args.build_dir, 429db167981SDaniel Latypov make_options=cli_args.make_options, 430db167981SDaniel Latypov jobs=cli_args.jobs, 431db167981SDaniel Latypov alltests=cli_args.alltests, 432db167981SDaniel Latypov raw_output=cli_args.raw_output, 433db167981SDaniel Latypov json=cli_args.json, 434db167981SDaniel Latypov timeout=cli_args.timeout, 435db167981SDaniel Latypov filter_glob=cli_args.filter_glob, 436db167981SDaniel Latypov kernel_args=cli_args.kernel_args, 437db167981SDaniel Latypov run_isolated=cli_args.run_isolated) 4386ebf5866SFelix Guo result = run_tests(linux, request) 4396ebf5866SFelix Guo if result.status != KunitStatus.SUCCESS: 4406ebf5866SFelix Guo sys.exit(1) 44145ba7a89SDavid Gow elif cli_args.subcommand == 'config': 44282206a0cSBrendan Higgins if cli_args.build_dir and ( 44382206a0cSBrendan Higgins not os.path.exists(cli_args.build_dir)): 44445ba7a89SDavid Gow os.mkdir(cli_args.build_dir) 44582206a0cSBrendan Higgins 4468a04930fSDaniel Latypov linux = tree_from_args(cli_args) 447db167981SDaniel Latypov request = KunitConfigRequest(build_dir=cli_args.build_dir, 448db167981SDaniel Latypov make_options=cli_args.make_options) 44945ba7a89SDavid Gow result = config_tests(linux, request) 450*e756dbebSDaniel Latypov stdout.print_with_timestamp(( 45145ba7a89SDavid Gow 'Elapsed time: %.3fs\n') % ( 45245ba7a89SDavid Gow result.elapsed_time)) 45345ba7a89SDavid Gow if result.status != KunitStatus.SUCCESS: 45445ba7a89SDavid Gow sys.exit(1) 45545ba7a89SDavid Gow elif cli_args.subcommand == 'build': 4568a04930fSDaniel Latypov linux = tree_from_args(cli_args) 457db167981SDaniel Latypov request = KunitBuildRequest(build_dir=cli_args.build_dir, 458db167981SDaniel Latypov make_options=cli_args.make_options, 459db167981SDaniel Latypov jobs=cli_args.jobs, 460db167981SDaniel Latypov alltests=cli_args.alltests) 4611ee2ba89SDaniel Latypov result = config_and_build_tests(linux, request) 462*e756dbebSDaniel Latypov stdout.print_with_timestamp(( 46345ba7a89SDavid Gow 'Elapsed time: %.3fs\n') % ( 46445ba7a89SDavid Gow result.elapsed_time)) 46545ba7a89SDavid Gow if result.status != KunitStatus.SUCCESS: 46645ba7a89SDavid Gow sys.exit(1) 46745ba7a89SDavid Gow elif cli_args.subcommand == 'exec': 4688a04930fSDaniel Latypov linux = tree_from_args(cli_args) 469db167981SDaniel Latypov exec_request = KunitExecRequest(raw_output=cli_args.raw_output, 470db167981SDaniel Latypov build_dir=cli_args.build_dir, 471db167981SDaniel Latypov json=cli_args.json, 472db167981SDaniel Latypov timeout=cli_args.timeout, 473db167981SDaniel Latypov alltests=cli_args.alltests, 474db167981SDaniel Latypov filter_glob=cli_args.filter_glob, 475db167981SDaniel Latypov kernel_args=cli_args.kernel_args, 476db167981SDaniel Latypov run_isolated=cli_args.run_isolated) 477db167981SDaniel Latypov result = exec_tests(linux, exec_request) 478*e756dbebSDaniel Latypov stdout.print_with_timestamp(( 4797ef925eaSDaniel Latypov 'Elapsed time: %.3fs\n') % (result.elapsed_time)) 48045ba7a89SDavid Gow if result.status != KunitStatus.SUCCESS: 48145ba7a89SDavid Gow sys.exit(1) 48245ba7a89SDavid Gow elif cli_args.subcommand == 'parse': 4830453f984SDaniel Latypov if cli_args.file is None: 4842ab5d5e6SDaniel Latypov sys.stdin.reconfigure(errors='backslashreplace') # pytype: disable=attribute-error 48545ba7a89SDavid Gow kunit_output = sys.stdin 48645ba7a89SDavid Gow else: 4872ab5d5e6SDaniel Latypov with open(cli_args.file, 'r', errors='backslashreplace') as f: 48845ba7a89SDavid Gow kunit_output = f.read().splitlines() 489ee96d25fSDaniel Latypov # We know nothing about how the result was created! 490ee96d25fSDaniel Latypov metadata = kunit_json.Metadata() 491db167981SDaniel Latypov request = KunitParseRequest(raw_output=cli_args.raw_output, 492db167981SDaniel Latypov json=cli_args.json) 493ee96d25fSDaniel Latypov result, _ = parse_tests(request, metadata, kunit_output) 49445ba7a89SDavid Gow if result.status != KunitStatus.SUCCESS: 49545ba7a89SDavid Gow sys.exit(1) 4966ebf5866SFelix Guo else: 4976ebf5866SFelix Guo parser.print_help() 4986ebf5866SFelix Guo 4996ebf5866SFelix Guoif __name__ == '__main__': 500ff7b437fSBrendan Higgins main(sys.argv[1:]) 501