1c25ce589SFinn Behrens#!/usr/bin/env python3 26ebf5866SFelix Guo# SPDX-License-Identifier: GPL-2.0 36ebf5866SFelix Guo# 46ebf5866SFelix Guo# A thin wrapper on top of the KUnit Kernel 56ebf5866SFelix Guo# 66ebf5866SFelix Guo# Copyright (C) 2019, Google LLC. 76ebf5866SFelix Guo# Author: Felix Guo <felixguoxiuping@gmail.com> 86ebf5866SFelix Guo# Author: Brendan Higgins <brendanhiggins@google.com> 96ebf5866SFelix Guo 106ebf5866SFelix Guoimport argparse 116ebf5866SFelix Guoimport os 12ff9e09a3SDaniel Latypovimport re 13*a9333bd3SDaniel Latypovimport shlex 14ff9e09a3SDaniel Latypovimport sys 156ebf5866SFelix Guoimport time 166ebf5866SFelix Guo 17df4b0807SSeongJae Parkassert sys.version_info >= (3, 7), "Python version is too old" 18df4b0807SSeongJae Park 19db167981SDaniel Latypovfrom dataclasses import dataclass 206ebf5866SFelix Guofrom enum import Enum, auto 2195dcbc55SDaniel Latypovfrom typing import Iterable, List, Optional, Sequence, Tuple 226ebf5866SFelix Guo 2321a6d178SHeidi Fahimimport kunit_json 246ebf5866SFelix Guoimport kunit_kernel 256ebf5866SFelix Guoimport kunit_parser 26e756dbebSDaniel Latypovfrom kunit_printer import stdout 276ebf5866SFelix Guo 286ebf5866SFelix Guoclass KunitStatus(Enum): 296ebf5866SFelix Guo SUCCESS = auto() 306ebf5866SFelix Guo CONFIG_FAILURE = auto() 316ebf5866SFelix Guo BUILD_FAILURE = auto() 326ebf5866SFelix Guo TEST_FAILURE = auto() 336ebf5866SFelix Guo 34db167981SDaniel Latypov@dataclass 35db167981SDaniel Latypovclass KunitResult: 36db167981SDaniel Latypov status: KunitStatus 37db167981SDaniel Latypov elapsed_time: float 38db167981SDaniel Latypov 39db167981SDaniel Latypov@dataclass 40db167981SDaniel Latypovclass KunitConfigRequest: 41db167981SDaniel Latypov build_dir: str 42db167981SDaniel Latypov make_options: Optional[List[str]] 43db167981SDaniel Latypov 44db167981SDaniel Latypov@dataclass 45db167981SDaniel Latypovclass KunitBuildRequest(KunitConfigRequest): 46db167981SDaniel Latypov jobs: int 47db167981SDaniel Latypov alltests: bool 48db167981SDaniel Latypov 49db167981SDaniel Latypov@dataclass 50db167981SDaniel Latypovclass KunitParseRequest: 51db167981SDaniel Latypov raw_output: Optional[str] 52db167981SDaniel Latypov json: Optional[str] 53db167981SDaniel Latypov 54db167981SDaniel Latypov@dataclass 55db167981SDaniel Latypovclass KunitExecRequest(KunitParseRequest): 56ee96d25fSDaniel Latypov build_dir: str 57db167981SDaniel Latypov timeout: int 58db167981SDaniel Latypov alltests: bool 59db167981SDaniel Latypov filter_glob: str 60db167981SDaniel Latypov kernel_args: Optional[List[str]] 61db167981SDaniel Latypov run_isolated: Optional[str] 62db167981SDaniel Latypov 63db167981SDaniel Latypov@dataclass 64db167981SDaniel Latypovclass KunitRequest(KunitExecRequest, KunitBuildRequest): 65db167981SDaniel Latypov pass 66db167981SDaniel Latypov 67db167981SDaniel Latypov 6809641f7cSDaniel Latypovdef get_kernel_root_path() -> str: 6909641f7cSDaniel Latypov path = sys.argv[0] if not __file__ else __file__ 7009641f7cSDaniel Latypov parts = os.path.realpath(path).split('tools/testing/kunit') 71be886ba9SHeidi Fahim if len(parts) != 2: 72be886ba9SHeidi Fahim sys.exit(1) 73be886ba9SHeidi Fahim return parts[0] 74be886ba9SHeidi Fahim 7545ba7a89SDavid Gowdef config_tests(linux: kunit_kernel.LinuxSourceTree, 7645ba7a89SDavid Gow request: KunitConfigRequest) -> KunitResult: 77e756dbebSDaniel Latypov stdout.print_with_timestamp('Configuring KUnit Kernel ...') 7845ba7a89SDavid Gow 796ebf5866SFelix Guo config_start = time.time() 800476e69fSGreg Thelen success = linux.build_reconfig(request.build_dir, request.make_options) 816ebf5866SFelix Guo config_end = time.time() 826ebf5866SFelix Guo if not success: 8345ba7a89SDavid Gow return KunitResult(KunitStatus.CONFIG_FAILURE, 8445ba7a89SDavid Gow config_end - config_start) 8545ba7a89SDavid Gow return KunitResult(KunitStatus.SUCCESS, 8645ba7a89SDavid Gow config_end - config_start) 876ebf5866SFelix Guo 8845ba7a89SDavid Gowdef build_tests(linux: kunit_kernel.LinuxSourceTree, 8945ba7a89SDavid Gow request: KunitBuildRequest) -> KunitResult: 90e756dbebSDaniel Latypov stdout.print_with_timestamp('Building KUnit Kernel ...') 916ebf5866SFelix Guo 926ebf5866SFelix Guo build_start = time.time() 9387c9c163SBrendan Higgins success = linux.build_kernel(request.alltests, 94021ed9f5SHeidi Fahim request.jobs, 950476e69fSGreg Thelen request.build_dir, 960476e69fSGreg Thelen request.make_options) 976ebf5866SFelix Guo build_end = time.time() 986ebf5866SFelix Guo if not success: 99ee61492aSDavid Gow return KunitResult(KunitStatus.BUILD_FAILURE, 100ee61492aSDavid Gow build_end - build_start) 10145ba7a89SDavid Gow if not success: 10245ba7a89SDavid Gow return KunitResult(KunitStatus.BUILD_FAILURE, 10345ba7a89SDavid Gow build_end - build_start) 10445ba7a89SDavid Gow return KunitResult(KunitStatus.SUCCESS, 10545ba7a89SDavid Gow build_end - build_start) 1066ebf5866SFelix Guo 1071ee2ba89SDaniel Latypovdef config_and_build_tests(linux: kunit_kernel.LinuxSourceTree, 1081ee2ba89SDaniel Latypov request: KunitBuildRequest) -> KunitResult: 1091ee2ba89SDaniel Latypov config_result = config_tests(linux, request) 1101ee2ba89SDaniel Latypov if config_result.status != KunitStatus.SUCCESS: 1111ee2ba89SDaniel Latypov return config_result 1121ee2ba89SDaniel Latypov 1131ee2ba89SDaniel Latypov return build_tests(linux, request) 1141ee2ba89SDaniel Latypov 115ff9e09a3SDaniel Latypovdef _list_tests(linux: kunit_kernel.LinuxSourceTree, request: KunitExecRequest) -> List[str]: 116ff9e09a3SDaniel Latypov args = ['kunit.action=list'] 117ff9e09a3SDaniel Latypov if request.kernel_args: 118ff9e09a3SDaniel Latypov args.extend(request.kernel_args) 119ff9e09a3SDaniel Latypov 120ff9e09a3SDaniel Latypov output = linux.run_kernel(args=args, 121ff9e09a3SDaniel Latypov timeout=None if request.alltests else request.timeout, 122ff9e09a3SDaniel Latypov filter_glob=request.filter_glob, 123ff9e09a3SDaniel Latypov build_dir=request.build_dir) 124ff9e09a3SDaniel Latypov lines = kunit_parser.extract_tap_lines(output) 125ff9e09a3SDaniel Latypov # Hack! Drop the dummy TAP version header that the executor prints out. 126ff9e09a3SDaniel Latypov lines.pop() 127ff9e09a3SDaniel Latypov 128ff9e09a3SDaniel Latypov # Filter out any extraneous non-test output that might have gotten mixed in. 1290453f984SDaniel Latypov return [l for l in lines if re.match(r'^[^\s.]+\.[^\s.]+$', l)] 130ff9e09a3SDaniel Latypov 131ff9e09a3SDaniel Latypovdef _suites_from_test_list(tests: List[str]) -> List[str]: 132ff9e09a3SDaniel Latypov """Extracts all the suites from an ordered list of tests.""" 133ff9e09a3SDaniel Latypov suites = [] # type: List[str] 134ff9e09a3SDaniel Latypov for t in tests: 135ff9e09a3SDaniel Latypov parts = t.split('.', maxsplit=2) 136ff9e09a3SDaniel Latypov if len(parts) != 2: 137ff9e09a3SDaniel Latypov raise ValueError(f'internal KUnit error, test name should be of the form "<suite>.<test>", got "{t}"') 138ff9e09a3SDaniel Latypov suite, case = parts 139ff9e09a3SDaniel Latypov if not suites or suites[-1] != suite: 140ff9e09a3SDaniel Latypov suites.append(suite) 141ff9e09a3SDaniel Latypov return suites 142ff9e09a3SDaniel Latypov 143ff9e09a3SDaniel Latypov 144ff9e09a3SDaniel Latypov 145db167981SDaniel Latypovdef exec_tests(linux: kunit_kernel.LinuxSourceTree, request: KunitExecRequest) -> KunitResult: 146ff9e09a3SDaniel Latypov filter_globs = [request.filter_glob] 147ff9e09a3SDaniel Latypov if request.run_isolated: 148ff9e09a3SDaniel Latypov tests = _list_tests(linux, request) 149ff9e09a3SDaniel Latypov if request.run_isolated == 'test': 150ff9e09a3SDaniel Latypov filter_globs = tests 151ff9e09a3SDaniel Latypov if request.run_isolated == 'suite': 152ff9e09a3SDaniel Latypov filter_globs = _suites_from_test_list(tests) 153ff9e09a3SDaniel Latypov # Apply the test-part of the user's glob, if present. 154ff9e09a3SDaniel Latypov if '.' in request.filter_glob: 155ff9e09a3SDaniel Latypov test_glob = request.filter_glob.split('.', maxsplit=2)[1] 156ff9e09a3SDaniel Latypov filter_globs = [g + '.'+ test_glob for g in filter_globs] 157ff9e09a3SDaniel Latypov 158885210d3SDaniel Latypov metadata = kunit_json.Metadata(arch=linux.arch(), build_dir=request.build_dir, def_config='kunit_defconfig') 159ee96d25fSDaniel Latypov 160d65d07cbSRae Moar test_counts = kunit_parser.TestCounts() 161ff9e09a3SDaniel Latypov exec_time = 0.0 162ff9e09a3SDaniel Latypov for i, filter_glob in enumerate(filter_globs): 163e756dbebSDaniel Latypov stdout.print_with_timestamp('Starting KUnit Kernel ({}/{})...'.format(i+1, len(filter_globs))) 164ff9e09a3SDaniel Latypov 1656ebf5866SFelix Guo test_start = time.time() 1667ef925eaSDaniel Latypov run_result = linux.run_kernel( 1676cb51a18SDaniel Latypov args=request.kernel_args, 168021ed9f5SHeidi Fahim timeout=None if request.alltests else request.timeout, 169ff9e09a3SDaniel Latypov filter_glob=filter_glob, 1706ec1b81dSSeongJae Park build_dir=request.build_dir) 17145ba7a89SDavid Gow 172ee96d25fSDaniel Latypov _, test_result = parse_tests(request, metadata, run_result) 1735f6aa6d8SDaniel Latypov # run_kernel() doesn't block on the kernel exiting. 1745f6aa6d8SDaniel Latypov # That only happens after we get the last line of output from `run_result`. 1755f6aa6d8SDaniel Latypov # So exec_time here actually contains parsing + execution time, which is fine. 1766ebf5866SFelix Guo test_end = time.time() 177ff9e09a3SDaniel Latypov exec_time += test_end - test_start 178ff9e09a3SDaniel Latypov 17995dcbc55SDaniel Latypov test_counts.add_subtest_counts(test_result.counts) 1806ebf5866SFelix Guo 1817fa7ffcfSDaniel Latypov if len(filter_globs) == 1 and test_counts.crashed > 0: 1827fa7ffcfSDaniel Latypov bd = request.build_dir 1837fa7ffcfSDaniel Latypov print('The kernel seems to have crashed; you can decode the stack traces with:') 1847fa7ffcfSDaniel Latypov print('$ scripts/decode_stacktrace.sh {}/vmlinux {} < {} | tee {}/decoded.log | {} parse'.format( 1857fa7ffcfSDaniel Latypov bd, bd, kunit_kernel.get_outfile_path(bd), bd, sys.argv[0])) 1867fa7ffcfSDaniel Latypov 187d65d07cbSRae Moar kunit_status = _map_to_overall_status(test_counts.get_status()) 18895dcbc55SDaniel Latypov return KunitResult(status=kunit_status, elapsed_time=exec_time) 189d65d07cbSRae Moar 190d65d07cbSRae Moardef _map_to_overall_status(test_status: kunit_parser.TestStatus) -> KunitStatus: 191d65d07cbSRae Moar if test_status in (kunit_parser.TestStatus.SUCCESS, kunit_parser.TestStatus.SKIPPED): 192d65d07cbSRae Moar return KunitStatus.SUCCESS 193d65d07cbSRae Moar return KunitStatus.TEST_FAILURE 1947ef925eaSDaniel Latypov 195ee96d25fSDaniel Latypovdef parse_tests(request: KunitParseRequest, metadata: kunit_json.Metadata, input_data: Iterable[str]) -> Tuple[KunitResult, kunit_parser.Test]: 19645ba7a89SDavid Gow parse_start = time.time() 19745ba7a89SDavid Gow 198e0cc8c05SDaniel Latypov test_result = kunit_parser.Test() 19921a6d178SHeidi Fahim 20045ba7a89SDavid Gow if request.raw_output: 201d65d07cbSRae Moar # Treat unparsed results as one passing test. 202e0cc8c05SDaniel Latypov test_result.status = kunit_parser.TestStatus.SUCCESS 203e0cc8c05SDaniel Latypov test_result.counts.passed = 1 204d65d07cbSRae Moar 2057ef925eaSDaniel Latypov output: Iterable[str] = input_data 2066a499c9cSDaniel Latypov if request.raw_output == 'all': 2076a499c9cSDaniel Latypov pass 2086a499c9cSDaniel Latypov elif request.raw_output == 'kunit': 2096a499c9cSDaniel Latypov output = kunit_parser.extract_tap_lines(output) 2106a499c9cSDaniel Latypov for line in output: 2116a499c9cSDaniel Latypov print(line.rstrip()) 2126a499c9cSDaniel Latypov 21345ba7a89SDavid Gow else: 2147ef925eaSDaniel Latypov test_result = kunit_parser.parse_run_tests(input_data) 21545ba7a89SDavid Gow parse_end = time.time() 21645ba7a89SDavid Gow 21721a6d178SHeidi Fahim if request.json: 21800f75043SDaniel Latypov json_str = kunit_json.get_json_result( 219e0cc8c05SDaniel Latypov test=test_result, 220ee96d25fSDaniel Latypov metadata=metadata) 22121a6d178SHeidi Fahim if request.json == 'stdout': 22200f75043SDaniel Latypov print(json_str) 22300f75043SDaniel Latypov else: 22400f75043SDaniel Latypov with open(request.json, 'w') as f: 22500f75043SDaniel Latypov f.write(json_str) 226e756dbebSDaniel Latypov stdout.print_with_timestamp("Test results stored in %s" % 22700f75043SDaniel Latypov os.path.abspath(request.json)) 22821a6d178SHeidi Fahim 22945ba7a89SDavid Gow if test_result.status != kunit_parser.TestStatus.SUCCESS: 23095dcbc55SDaniel Latypov return KunitResult(KunitStatus.TEST_FAILURE, parse_end - parse_start), test_result 23145ba7a89SDavid Gow 23295dcbc55SDaniel Latypov return KunitResult(KunitStatus.SUCCESS, parse_end - parse_start), test_result 23345ba7a89SDavid Gow 23445ba7a89SDavid Gowdef run_tests(linux: kunit_kernel.LinuxSourceTree, 23545ba7a89SDavid Gow request: KunitRequest) -> KunitResult: 23645ba7a89SDavid Gow run_start = time.time() 23745ba7a89SDavid Gow 238db167981SDaniel Latypov config_result = config_tests(linux, request) 23945ba7a89SDavid Gow if config_result.status != KunitStatus.SUCCESS: 24045ba7a89SDavid Gow return config_result 24145ba7a89SDavid Gow 242db167981SDaniel Latypov build_result = build_tests(linux, request) 24345ba7a89SDavid Gow if build_result.status != KunitStatus.SUCCESS: 24445ba7a89SDavid Gow return build_result 24545ba7a89SDavid Gow 246db167981SDaniel Latypov exec_result = exec_tests(linux, request) 24745ba7a89SDavid Gow 24845ba7a89SDavid Gow run_end = time.time() 24945ba7a89SDavid Gow 250e756dbebSDaniel Latypov stdout.print_with_timestamp(( 2516ebf5866SFelix Guo 'Elapsed time: %.3fs total, %.3fs configuring, %.3fs ' + 2526ebf5866SFelix Guo 'building, %.3fs running\n') % ( 25345ba7a89SDavid Gow run_end - run_start, 25445ba7a89SDavid Gow config_result.elapsed_time, 25545ba7a89SDavid Gow build_result.elapsed_time, 25645ba7a89SDavid Gow exec_result.elapsed_time)) 2577ef925eaSDaniel Latypov return exec_result 2586ebf5866SFelix Guo 259d8c23eadSDaniel Latypov# Problem: 260d8c23eadSDaniel Latypov# $ kunit.py run --json 261d8c23eadSDaniel Latypov# works as one would expect and prints the parsed test results as JSON. 262d8c23eadSDaniel Latypov# $ kunit.py run --json suite_name 263d8c23eadSDaniel Latypov# would *not* pass suite_name as the filter_glob and print as json. 264d8c23eadSDaniel Latypov# argparse will consider it to be another way of writing 265d8c23eadSDaniel Latypov# $ kunit.py run --json=suite_name 266d8c23eadSDaniel Latypov# i.e. it would run all tests, and dump the json to a `suite_name` file. 267d8c23eadSDaniel Latypov# So we hackily automatically rewrite --json => --json=stdout 268d8c23eadSDaniel Latypovpseudo_bool_flag_defaults = { 269d8c23eadSDaniel Latypov '--json': 'stdout', 270d8c23eadSDaniel Latypov '--raw_output': 'kunit', 271d8c23eadSDaniel Latypov} 272d8c23eadSDaniel Latypovdef massage_argv(argv: Sequence[str]) -> Sequence[str]: 273d8c23eadSDaniel Latypov def massage_arg(arg: str) -> str: 274d8c23eadSDaniel Latypov if arg not in pseudo_bool_flag_defaults: 275d8c23eadSDaniel Latypov return arg 276d8c23eadSDaniel Latypov return f'{arg}={pseudo_bool_flag_defaults[arg]}' 277d8c23eadSDaniel Latypov return list(map(massage_arg, argv)) 278d8c23eadSDaniel Latypov 279ad659ccbSDavid Gowdef get_default_jobs() -> int: 280ad659ccbSDavid Gow return len(os.sched_getaffinity(0)) 281ad659ccbSDavid Gow 28209641f7cSDaniel Latypovdef add_common_opts(parser) -> None: 28345ba7a89SDavid Gow parser.add_argument('--build_dir', 28445ba7a89SDavid Gow help='As in the make command, it specifies the build ' 28545ba7a89SDavid Gow 'directory.', 286baa33315SDaniel Latypov type=str, default='.kunit', metavar='DIR') 28745ba7a89SDavid Gow parser.add_argument('--make_options', 28845ba7a89SDavid Gow help='X=Y make option, can be repeated.', 289baa33315SDaniel Latypov action='append', metavar='X=Y') 29045ba7a89SDavid Gow parser.add_argument('--alltests', 29145ba7a89SDavid Gow help='Run all KUnit tests through allyesconfig', 2926ebf5866SFelix Guo action='store_true') 293243180f5SDaniel Latypov parser.add_argument('--kunitconfig', 2949854781dSDaniel Latypov help='Path to Kconfig fragment that enables KUnit tests.' 2959854781dSDaniel Latypov ' If given a directory, (e.g. lib/kunit), "/.kunitconfig" ' 2969854781dSDaniel Latypov 'will get automatically appended.', 297baa33315SDaniel Latypov metavar='PATH') 2989f57cc76SDaniel Latypov parser.add_argument('--kconfig_add', 2999f57cc76SDaniel Latypov help='Additional Kconfig options to append to the ' 3009f57cc76SDaniel Latypov '.kunitconfig, e.g. CONFIG_KASAN=y. Can be repeated.', 301baa33315SDaniel Latypov action='append', metavar='CONFIG_X=Y') 3026ebf5866SFelix Guo 30387c9c163SBrendan Higgins parser.add_argument('--arch', 30487c9c163SBrendan Higgins help=('Specifies the architecture to run tests under. ' 30587c9c163SBrendan Higgins 'The architecture specified here must match the ' 30687c9c163SBrendan Higgins 'string passed to the ARCH make param, ' 30787c9c163SBrendan Higgins 'e.g. i386, x86_64, arm, um, etc. Non-UML ' 30887c9c163SBrendan Higgins 'architectures run on QEMU.'), 309baa33315SDaniel Latypov type=str, default='um', metavar='ARCH') 31087c9c163SBrendan Higgins 31187c9c163SBrendan Higgins parser.add_argument('--cross_compile', 31287c9c163SBrendan Higgins help=('Sets make\'s CROSS_COMPILE variable; it should ' 31387c9c163SBrendan Higgins 'be set to a toolchain path prefix (the prefix ' 31487c9c163SBrendan Higgins 'of gcc and other tools in your toolchain, for ' 31587c9c163SBrendan Higgins 'example `sparc64-linux-gnu-` if you have the ' 31687c9c163SBrendan Higgins 'sparc toolchain installed on your system, or ' 31787c9c163SBrendan Higgins '`$HOME/toolchains/microblaze/gcc-9.2.0-nolibc/microblaze-linux/bin/microblaze-linux-` ' 31887c9c163SBrendan Higgins 'if you have downloaded the microblaze toolchain ' 31987c9c163SBrendan Higgins 'from the 0-day website to a directory in your ' 32087c9c163SBrendan Higgins 'home directory called `toolchains`).'), 321baa33315SDaniel Latypov metavar='PREFIX') 32287c9c163SBrendan Higgins 32387c9c163SBrendan Higgins parser.add_argument('--qemu_config', 32487c9c163SBrendan Higgins help=('Takes a path to a path to a file containing ' 32587c9c163SBrendan Higgins 'a QemuArchParams object.'), 326baa33315SDaniel Latypov type=str, metavar='FILE') 32787c9c163SBrendan Higgins 328*a9333bd3SDaniel Latypov parser.add_argument('--qemu_args', 329*a9333bd3SDaniel Latypov help='Additional QEMU arguments, e.g. "-smp 8"', 330*a9333bd3SDaniel Latypov action='append', metavar='') 331*a9333bd3SDaniel Latypov 33209641f7cSDaniel Latypovdef add_build_opts(parser) -> None: 33345ba7a89SDavid Gow parser.add_argument('--jobs', 33445ba7a89SDavid Gow help='As in the make command, "Specifies the number of ' 33545ba7a89SDavid Gow 'jobs (commands) to run simultaneously."', 336baa33315SDaniel Latypov type=int, default=get_default_jobs(), metavar='N') 33745ba7a89SDavid Gow 33809641f7cSDaniel Latypovdef add_exec_opts(parser) -> None: 33945ba7a89SDavid Gow parser.add_argument('--timeout', 3406ebf5866SFelix Guo help='maximum number of seconds to allow for all tests ' 3416ebf5866SFelix Guo 'to run. This does not include time taken to build the ' 3426ebf5866SFelix Guo 'tests.', 3436ebf5866SFelix Guo type=int, 3446ebf5866SFelix Guo default=300, 345baa33315SDaniel Latypov metavar='SECONDS') 346d992880bSDaniel Latypov parser.add_argument('filter_glob', 347a127b154SDaniel Latypov help='Filter which KUnit test suites/tests run at ' 348a127b154SDaniel Latypov 'boot-time, e.g. list* or list*.*del_test', 349d992880bSDaniel Latypov type=str, 350d992880bSDaniel Latypov nargs='?', 351d992880bSDaniel Latypov default='', 352d992880bSDaniel Latypov metavar='filter_glob') 3536cb51a18SDaniel Latypov parser.add_argument('--kernel_args', 3546cb51a18SDaniel Latypov help='Kernel command-line parameters. Maybe be repeated', 355baa33315SDaniel Latypov action='append', metavar='') 356ff9e09a3SDaniel Latypov parser.add_argument('--run_isolated', help='If set, boot the kernel for each ' 357ff9e09a3SDaniel Latypov 'individual suite/test. This is can be useful for debugging ' 358ff9e09a3SDaniel Latypov 'a non-hermetic test, one that might pass/fail based on ' 359ff9e09a3SDaniel Latypov 'what ran before it.', 360ff9e09a3SDaniel Latypov type=str, 3610453f984SDaniel Latypov choices=['suite', 'test']) 3626ebf5866SFelix Guo 36309641f7cSDaniel Latypovdef add_parse_opts(parser) -> None: 3646a499c9cSDaniel Latypov parser.add_argument('--raw_output', help='If set don\'t format output from kernel. ' 3656a499c9cSDaniel Latypov 'If set to --raw_output=kunit, filters to just KUnit output.', 366baa33315SDaniel Latypov type=str, nargs='?', const='all', default=None, choices=['all', 'kunit']) 36721a6d178SHeidi Fahim parser.add_argument('--json', 36821a6d178SHeidi Fahim nargs='?', 36921a6d178SHeidi Fahim help='Stores test results in a JSON, and either ' 37021a6d178SHeidi Fahim 'prints to stdout or saves to file if a ' 37121a6d178SHeidi Fahim 'filename is specified', 372baa33315SDaniel Latypov type=str, const='stdout', default=None, metavar='FILE') 373021ed9f5SHeidi Fahim 3748a04930fSDaniel Latypov 3758a04930fSDaniel Latypovdef tree_from_args(cli_args: argparse.Namespace) -> kunit_kernel.LinuxSourceTree: 3768a04930fSDaniel Latypov """Returns a LinuxSourceTree based on the user's arguments.""" 377*a9333bd3SDaniel Latypov # Allow users to specify multiple arguments in one string, e.g. '-smp 8' 378*a9333bd3SDaniel Latypov qemu_args: List[str] = [] 379*a9333bd3SDaniel Latypov if cli_args.qemu_args: 380*a9333bd3SDaniel Latypov for arg in cli_args.qemu_args: 381*a9333bd3SDaniel Latypov qemu_args.extend(shlex.split(arg)) 382*a9333bd3SDaniel Latypov 3838a04930fSDaniel Latypov return kunit_kernel.LinuxSourceTree(cli_args.build_dir, 3848a04930fSDaniel Latypov kunitconfig_path=cli_args.kunitconfig, 3858a04930fSDaniel Latypov kconfig_add=cli_args.kconfig_add, 3868a04930fSDaniel Latypov arch=cli_args.arch, 3878a04930fSDaniel Latypov cross_compile=cli_args.cross_compile, 388*a9333bd3SDaniel Latypov qemu_config_path=cli_args.qemu_config, 389*a9333bd3SDaniel Latypov extra_qemu_args=qemu_args) 3908a04930fSDaniel Latypov 3918a04930fSDaniel Latypov 3928a04930fSDaniel Latypovdef main(argv): 39345ba7a89SDavid Gow parser = argparse.ArgumentParser( 39445ba7a89SDavid Gow description='Helps writing and running KUnit tests.') 39545ba7a89SDavid Gow subparser = parser.add_subparsers(dest='subcommand') 39645ba7a89SDavid Gow 39745ba7a89SDavid Gow # The 'run' command will config, build, exec, and parse in one go. 39845ba7a89SDavid Gow run_parser = subparser.add_parser('run', help='Runs KUnit tests.') 39945ba7a89SDavid Gow add_common_opts(run_parser) 40045ba7a89SDavid Gow add_build_opts(run_parser) 40145ba7a89SDavid Gow add_exec_opts(run_parser) 40245ba7a89SDavid Gow add_parse_opts(run_parser) 40345ba7a89SDavid Gow 40445ba7a89SDavid Gow config_parser = subparser.add_parser('config', 40545ba7a89SDavid Gow help='Ensures that .config contains all of ' 40645ba7a89SDavid Gow 'the options in .kunitconfig') 40745ba7a89SDavid Gow add_common_opts(config_parser) 40845ba7a89SDavid Gow 40945ba7a89SDavid Gow build_parser = subparser.add_parser('build', help='Builds a kernel with KUnit tests') 41045ba7a89SDavid Gow add_common_opts(build_parser) 41145ba7a89SDavid Gow add_build_opts(build_parser) 41245ba7a89SDavid Gow 41345ba7a89SDavid Gow exec_parser = subparser.add_parser('exec', help='Run a kernel with KUnit tests') 41445ba7a89SDavid Gow add_common_opts(exec_parser) 41545ba7a89SDavid Gow add_exec_opts(exec_parser) 41645ba7a89SDavid Gow add_parse_opts(exec_parser) 41745ba7a89SDavid Gow 41845ba7a89SDavid Gow # The 'parse' option is special, as it doesn't need the kernel source 41945ba7a89SDavid Gow # (therefore there is no need for a build_dir, hence no add_common_opts) 42045ba7a89SDavid Gow # and the '--file' argument is not relevant to 'run', so isn't in 42145ba7a89SDavid Gow # add_parse_opts() 42245ba7a89SDavid Gow parse_parser = subparser.add_parser('parse', 42345ba7a89SDavid Gow help='Parses KUnit results from a file, ' 42445ba7a89SDavid Gow 'and parses formatted results.') 42545ba7a89SDavid Gow add_parse_opts(parse_parser) 42645ba7a89SDavid Gow parse_parser.add_argument('file', 42745ba7a89SDavid Gow help='Specifies the file to read results from.', 42845ba7a89SDavid Gow type=str, nargs='?', metavar='input_file') 4290476e69fSGreg Thelen 430d8c23eadSDaniel Latypov cli_args = parser.parse_args(massage_argv(argv)) 4316ebf5866SFelix Guo 4325578d008SBrendan Higgins if get_kernel_root_path(): 4335578d008SBrendan Higgins os.chdir(get_kernel_root_path()) 4345578d008SBrendan Higgins 4356ebf5866SFelix Guo if cli_args.subcommand == 'run': 436e3212513SSeongJae Park if not os.path.exists(cli_args.build_dir): 437e3212513SSeongJae Park os.mkdir(cli_args.build_dir) 43882206a0cSBrendan Higgins 4398a04930fSDaniel Latypov linux = tree_from_args(cli_args) 440db167981SDaniel Latypov request = KunitRequest(build_dir=cli_args.build_dir, 441db167981SDaniel Latypov make_options=cli_args.make_options, 442db167981SDaniel Latypov jobs=cli_args.jobs, 443db167981SDaniel Latypov alltests=cli_args.alltests, 444db167981SDaniel Latypov raw_output=cli_args.raw_output, 445db167981SDaniel Latypov json=cli_args.json, 446db167981SDaniel Latypov timeout=cli_args.timeout, 447db167981SDaniel Latypov filter_glob=cli_args.filter_glob, 448db167981SDaniel Latypov kernel_args=cli_args.kernel_args, 449db167981SDaniel Latypov run_isolated=cli_args.run_isolated) 4506ebf5866SFelix Guo result = run_tests(linux, request) 4516ebf5866SFelix Guo if result.status != KunitStatus.SUCCESS: 4526ebf5866SFelix Guo sys.exit(1) 45345ba7a89SDavid Gow elif cli_args.subcommand == 'config': 45482206a0cSBrendan Higgins if cli_args.build_dir and ( 45582206a0cSBrendan Higgins not os.path.exists(cli_args.build_dir)): 45645ba7a89SDavid Gow os.mkdir(cli_args.build_dir) 45782206a0cSBrendan Higgins 4588a04930fSDaniel Latypov linux = tree_from_args(cli_args) 459db167981SDaniel Latypov request = KunitConfigRequest(build_dir=cli_args.build_dir, 460db167981SDaniel Latypov make_options=cli_args.make_options) 46145ba7a89SDavid Gow result = config_tests(linux, request) 462e756dbebSDaniel Latypov stdout.print_with_timestamp(( 46345ba7a89SDavid Gow 'Elapsed time: %.3fs\n') % ( 46445ba7a89SDavid Gow result.elapsed_time)) 46545ba7a89SDavid Gow if result.status != KunitStatus.SUCCESS: 46645ba7a89SDavid Gow sys.exit(1) 46745ba7a89SDavid Gow elif cli_args.subcommand == 'build': 4688a04930fSDaniel Latypov linux = tree_from_args(cli_args) 469db167981SDaniel Latypov request = KunitBuildRequest(build_dir=cli_args.build_dir, 470db167981SDaniel Latypov make_options=cli_args.make_options, 471db167981SDaniel Latypov jobs=cli_args.jobs, 472db167981SDaniel Latypov alltests=cli_args.alltests) 4731ee2ba89SDaniel Latypov result = config_and_build_tests(linux, request) 474e756dbebSDaniel Latypov stdout.print_with_timestamp(( 47545ba7a89SDavid Gow 'Elapsed time: %.3fs\n') % ( 47645ba7a89SDavid Gow result.elapsed_time)) 47745ba7a89SDavid Gow if result.status != KunitStatus.SUCCESS: 47845ba7a89SDavid Gow sys.exit(1) 47945ba7a89SDavid Gow elif cli_args.subcommand == 'exec': 4808a04930fSDaniel Latypov linux = tree_from_args(cli_args) 481db167981SDaniel Latypov exec_request = KunitExecRequest(raw_output=cli_args.raw_output, 482db167981SDaniel Latypov build_dir=cli_args.build_dir, 483db167981SDaniel Latypov json=cli_args.json, 484db167981SDaniel Latypov timeout=cli_args.timeout, 485db167981SDaniel Latypov alltests=cli_args.alltests, 486db167981SDaniel Latypov filter_glob=cli_args.filter_glob, 487db167981SDaniel Latypov kernel_args=cli_args.kernel_args, 488db167981SDaniel Latypov run_isolated=cli_args.run_isolated) 489db167981SDaniel Latypov result = exec_tests(linux, exec_request) 490e756dbebSDaniel Latypov stdout.print_with_timestamp(( 4917ef925eaSDaniel Latypov 'Elapsed time: %.3fs\n') % (result.elapsed_time)) 49245ba7a89SDavid Gow if result.status != KunitStatus.SUCCESS: 49345ba7a89SDavid Gow sys.exit(1) 49445ba7a89SDavid Gow elif cli_args.subcommand == 'parse': 4950453f984SDaniel Latypov if cli_args.file is None: 4962ab5d5e6SDaniel Latypov sys.stdin.reconfigure(errors='backslashreplace') # pytype: disable=attribute-error 49745ba7a89SDavid Gow kunit_output = sys.stdin 49845ba7a89SDavid Gow else: 4992ab5d5e6SDaniel Latypov with open(cli_args.file, 'r', errors='backslashreplace') as f: 50045ba7a89SDavid Gow kunit_output = f.read().splitlines() 501ee96d25fSDaniel Latypov # We know nothing about how the result was created! 502ee96d25fSDaniel Latypov metadata = kunit_json.Metadata() 503db167981SDaniel Latypov request = KunitParseRequest(raw_output=cli_args.raw_output, 504db167981SDaniel Latypov json=cli_args.json) 505ee96d25fSDaniel Latypov result, _ = parse_tests(request, metadata, kunit_output) 50645ba7a89SDavid Gow if result.status != KunitStatus.SUCCESS: 50745ba7a89SDavid Gow sys.exit(1) 5086ebf5866SFelix Guo else: 5096ebf5866SFelix Guo parser.print_help() 5106ebf5866SFelix Guo 5116ebf5866SFelix Guoif __name__ == '__main__': 512ff7b437fSBrendan Higgins main(sys.argv[1:]) 513