1c25ce589SFinn Behrens#!/usr/bin/env python3 26ebf5866SFelix Guo# SPDX-License-Identifier: GPL-2.0 36ebf5866SFelix Guo# 46ebf5866SFelix Guo# A thin wrapper on top of the KUnit Kernel 56ebf5866SFelix Guo# 66ebf5866SFelix Guo# Copyright (C) 2019, Google LLC. 76ebf5866SFelix Guo# Author: Felix Guo <felixguoxiuping@gmail.com> 86ebf5866SFelix Guo# Author: Brendan Higgins <brendanhiggins@google.com> 96ebf5866SFelix Guo 106ebf5866SFelix Guoimport argparse 116ebf5866SFelix Guoimport os 12ff9e09a3SDaniel Latypovimport re 13a9333bd3SDaniel Latypovimport shlex 14ff9e09a3SDaniel Latypovimport sys 156ebf5866SFelix Guoimport time 166ebf5866SFelix Guo 17df4b0807SSeongJae Parkassert sys.version_info >= (3, 7), "Python version is too old" 18df4b0807SSeongJae Park 19db167981SDaniel Latypovfrom dataclasses import dataclass 206ebf5866SFelix Guofrom enum import Enum, auto 2195dcbc55SDaniel Latypovfrom typing import Iterable, List, Optional, Sequence, Tuple 226ebf5866SFelix Guo 2321a6d178SHeidi Fahimimport kunit_json 246ebf5866SFelix Guoimport kunit_kernel 256ebf5866SFelix Guoimport kunit_parser 26e756dbebSDaniel Latypovfrom kunit_printer import stdout 276ebf5866SFelix Guo 286ebf5866SFelix Guoclass KunitStatus(Enum): 296ebf5866SFelix Guo SUCCESS = auto() 306ebf5866SFelix Guo CONFIG_FAILURE = auto() 316ebf5866SFelix Guo BUILD_FAILURE = auto() 326ebf5866SFelix Guo TEST_FAILURE = auto() 336ebf5866SFelix Guo 34db167981SDaniel Latypov@dataclass 35db167981SDaniel Latypovclass KunitResult: 36db167981SDaniel Latypov status: KunitStatus 37db167981SDaniel Latypov elapsed_time: float 38db167981SDaniel Latypov 39db167981SDaniel Latypov@dataclass 40db167981SDaniel Latypovclass KunitConfigRequest: 41db167981SDaniel Latypov build_dir: str 42db167981SDaniel Latypov make_options: Optional[List[str]] 43db167981SDaniel Latypov 44db167981SDaniel Latypov@dataclass 45db167981SDaniel Latypovclass KunitBuildRequest(KunitConfigRequest): 46db167981SDaniel Latypov jobs: int 47db167981SDaniel Latypov 48db167981SDaniel Latypov@dataclass 49db167981SDaniel Latypovclass KunitParseRequest: 50db167981SDaniel Latypov raw_output: Optional[str] 51db167981SDaniel Latypov json: Optional[str] 52db167981SDaniel Latypov 53db167981SDaniel Latypov@dataclass 54db167981SDaniel Latypovclass KunitExecRequest(KunitParseRequest): 55ee96d25fSDaniel Latypov build_dir: str 56db167981SDaniel Latypov timeout: int 57db167981SDaniel Latypov filter_glob: str 58db167981SDaniel Latypov kernel_args: Optional[List[str]] 59db167981SDaniel Latypov run_isolated: Optional[str] 60db167981SDaniel Latypov 61db167981SDaniel Latypov@dataclass 62db167981SDaniel Latypovclass KunitRequest(KunitExecRequest, KunitBuildRequest): 63db167981SDaniel Latypov pass 64db167981SDaniel Latypov 65db167981SDaniel Latypov 6609641f7cSDaniel Latypovdef get_kernel_root_path() -> str: 6709641f7cSDaniel Latypov path = sys.argv[0] if not __file__ else __file__ 6809641f7cSDaniel Latypov parts = os.path.realpath(path).split('tools/testing/kunit') 69be886ba9SHeidi Fahim if len(parts) != 2: 70be886ba9SHeidi Fahim sys.exit(1) 71be886ba9SHeidi Fahim return parts[0] 72be886ba9SHeidi Fahim 7345ba7a89SDavid Gowdef config_tests(linux: kunit_kernel.LinuxSourceTree, 7445ba7a89SDavid Gow request: KunitConfigRequest) -> KunitResult: 75e756dbebSDaniel Latypov stdout.print_with_timestamp('Configuring KUnit Kernel ...') 7645ba7a89SDavid Gow 776ebf5866SFelix Guo config_start = time.time() 780476e69fSGreg Thelen success = linux.build_reconfig(request.build_dir, request.make_options) 796ebf5866SFelix Guo config_end = time.time() 801fdc6f4fSAlexander Pantyukhin status = KunitStatus.SUCCESS if success else KunitStatus.CONFIG_FAILURE 811fdc6f4fSAlexander Pantyukhin return KunitResult(status, config_end - config_start) 826ebf5866SFelix Guo 8345ba7a89SDavid Gowdef build_tests(linux: kunit_kernel.LinuxSourceTree, 8445ba7a89SDavid Gow request: KunitBuildRequest) -> KunitResult: 85e756dbebSDaniel Latypov stdout.print_with_timestamp('Building KUnit Kernel ...') 866ebf5866SFelix Guo 876ebf5866SFelix Guo build_start = time.time() 88980ac3adSDaniel Latypov success = linux.build_kernel(request.jobs, 890476e69fSGreg Thelen request.build_dir, 900476e69fSGreg Thelen request.make_options) 916ebf5866SFelix Guo build_end = time.time() 921fdc6f4fSAlexander Pantyukhin status = KunitStatus.SUCCESS if success else KunitStatus.BUILD_FAILURE 931fdc6f4fSAlexander Pantyukhin return KunitResult(status, build_end - build_start) 946ebf5866SFelix Guo 951ee2ba89SDaniel Latypovdef config_and_build_tests(linux: kunit_kernel.LinuxSourceTree, 961ee2ba89SDaniel Latypov request: KunitBuildRequest) -> KunitResult: 971ee2ba89SDaniel Latypov config_result = config_tests(linux, request) 981ee2ba89SDaniel Latypov if config_result.status != KunitStatus.SUCCESS: 991ee2ba89SDaniel Latypov return config_result 1001ee2ba89SDaniel Latypov 1011ee2ba89SDaniel Latypov return build_tests(linux, request) 1021ee2ba89SDaniel Latypov 103ff9e09a3SDaniel Latypovdef _list_tests(linux: kunit_kernel.LinuxSourceTree, request: KunitExecRequest) -> List[str]: 104ff9e09a3SDaniel Latypov args = ['kunit.action=list'] 105ff9e09a3SDaniel Latypov if request.kernel_args: 106ff9e09a3SDaniel Latypov args.extend(request.kernel_args) 107ff9e09a3SDaniel Latypov 108ff9e09a3SDaniel Latypov output = linux.run_kernel(args=args, 109980ac3adSDaniel Latypov timeout=request.timeout, 110ff9e09a3SDaniel Latypov filter_glob=request.filter_glob, 111ff9e09a3SDaniel Latypov build_dir=request.build_dir) 112ff9e09a3SDaniel Latypov lines = kunit_parser.extract_tap_lines(output) 113ff9e09a3SDaniel Latypov # Hack! Drop the dummy TAP version header that the executor prints out. 114ff9e09a3SDaniel Latypov lines.pop() 115ff9e09a3SDaniel Latypov 116ff9e09a3SDaniel Latypov # Filter out any extraneous non-test output that might have gotten mixed in. 1170453f984SDaniel Latypov return [l for l in lines if re.match(r'^[^\s.]+\.[^\s.]+$', l)] 118ff9e09a3SDaniel Latypov 119ff9e09a3SDaniel Latypovdef _suites_from_test_list(tests: List[str]) -> List[str]: 120ff9e09a3SDaniel Latypov """Extracts all the suites from an ordered list of tests.""" 121ff9e09a3SDaniel Latypov suites = [] # type: List[str] 122ff9e09a3SDaniel Latypov for t in tests: 123ff9e09a3SDaniel Latypov parts = t.split('.', maxsplit=2) 124ff9e09a3SDaniel Latypov if len(parts) != 2: 125ff9e09a3SDaniel Latypov raise ValueError(f'internal KUnit error, test name should be of the form "<suite>.<test>", got "{t}"') 126*126901baSDaniel Latypov suite, _ = parts 127ff9e09a3SDaniel Latypov if not suites or suites[-1] != suite: 128ff9e09a3SDaniel Latypov suites.append(suite) 129ff9e09a3SDaniel Latypov return suites 130ff9e09a3SDaniel Latypov 131ff9e09a3SDaniel Latypov 132ff9e09a3SDaniel Latypov 133db167981SDaniel Latypovdef exec_tests(linux: kunit_kernel.LinuxSourceTree, request: KunitExecRequest) -> KunitResult: 134ff9e09a3SDaniel Latypov filter_globs = [request.filter_glob] 135ff9e09a3SDaniel Latypov if request.run_isolated: 136ff9e09a3SDaniel Latypov tests = _list_tests(linux, request) 137ff9e09a3SDaniel Latypov if request.run_isolated == 'test': 138ff9e09a3SDaniel Latypov filter_globs = tests 1391fdc6f4fSAlexander Pantyukhin elif request.run_isolated == 'suite': 140ff9e09a3SDaniel Latypov filter_globs = _suites_from_test_list(tests) 141ff9e09a3SDaniel Latypov # Apply the test-part of the user's glob, if present. 142ff9e09a3SDaniel Latypov if '.' in request.filter_glob: 143ff9e09a3SDaniel Latypov test_glob = request.filter_glob.split('.', maxsplit=2)[1] 144ff9e09a3SDaniel Latypov filter_globs = [g + '.'+ test_glob for g in filter_globs] 145ff9e09a3SDaniel Latypov 146885210d3SDaniel Latypov metadata = kunit_json.Metadata(arch=linux.arch(), build_dir=request.build_dir, def_config='kunit_defconfig') 147ee96d25fSDaniel Latypov 148d65d07cbSRae Moar test_counts = kunit_parser.TestCounts() 149ff9e09a3SDaniel Latypov exec_time = 0.0 150ff9e09a3SDaniel Latypov for i, filter_glob in enumerate(filter_globs): 151e756dbebSDaniel Latypov stdout.print_with_timestamp('Starting KUnit Kernel ({}/{})...'.format(i+1, len(filter_globs))) 152ff9e09a3SDaniel Latypov 1536ebf5866SFelix Guo test_start = time.time() 1547ef925eaSDaniel Latypov run_result = linux.run_kernel( 1556cb51a18SDaniel Latypov args=request.kernel_args, 156980ac3adSDaniel Latypov timeout=request.timeout, 157ff9e09a3SDaniel Latypov filter_glob=filter_glob, 1586ec1b81dSSeongJae Park build_dir=request.build_dir) 15945ba7a89SDavid Gow 160ee96d25fSDaniel Latypov _, test_result = parse_tests(request, metadata, run_result) 1615f6aa6d8SDaniel Latypov # run_kernel() doesn't block on the kernel exiting. 1625f6aa6d8SDaniel Latypov # That only happens after we get the last line of output from `run_result`. 1635f6aa6d8SDaniel Latypov # So exec_time here actually contains parsing + execution time, which is fine. 1646ebf5866SFelix Guo test_end = time.time() 165ff9e09a3SDaniel Latypov exec_time += test_end - test_start 166ff9e09a3SDaniel Latypov 16795dcbc55SDaniel Latypov test_counts.add_subtest_counts(test_result.counts) 1686ebf5866SFelix Guo 1697fa7ffcfSDaniel Latypov if len(filter_globs) == 1 and test_counts.crashed > 0: 1707fa7ffcfSDaniel Latypov bd = request.build_dir 1717fa7ffcfSDaniel Latypov print('The kernel seems to have crashed; you can decode the stack traces with:') 1727fa7ffcfSDaniel Latypov print('$ scripts/decode_stacktrace.sh {}/vmlinux {} < {} | tee {}/decoded.log | {} parse'.format( 1737fa7ffcfSDaniel Latypov bd, bd, kunit_kernel.get_outfile_path(bd), bd, sys.argv[0])) 1747fa7ffcfSDaniel Latypov 175d65d07cbSRae Moar kunit_status = _map_to_overall_status(test_counts.get_status()) 17695dcbc55SDaniel Latypov return KunitResult(status=kunit_status, elapsed_time=exec_time) 177d65d07cbSRae Moar 178d65d07cbSRae Moardef _map_to_overall_status(test_status: kunit_parser.TestStatus) -> KunitStatus: 179d65d07cbSRae Moar if test_status in (kunit_parser.TestStatus.SUCCESS, kunit_parser.TestStatus.SKIPPED): 180d65d07cbSRae Moar return KunitStatus.SUCCESS 181d65d07cbSRae Moar return KunitStatus.TEST_FAILURE 1827ef925eaSDaniel Latypov 183ee96d25fSDaniel Latypovdef parse_tests(request: KunitParseRequest, metadata: kunit_json.Metadata, input_data: Iterable[str]) -> Tuple[KunitResult, kunit_parser.Test]: 18445ba7a89SDavid Gow parse_start = time.time() 18545ba7a89SDavid Gow 18645ba7a89SDavid Gow if request.raw_output: 187d65d07cbSRae Moar # Treat unparsed results as one passing test. 188309e22efSDaniel Latypov fake_test = kunit_parser.Test() 189309e22efSDaniel Latypov fake_test.status = kunit_parser.TestStatus.SUCCESS 190309e22efSDaniel Latypov fake_test.counts.passed = 1 191d65d07cbSRae Moar 1927ef925eaSDaniel Latypov output: Iterable[str] = input_data 1936a499c9cSDaniel Latypov if request.raw_output == 'all': 1946a499c9cSDaniel Latypov pass 1956a499c9cSDaniel Latypov elif request.raw_output == 'kunit': 196c2bb92bcSDaniel Latypov output = kunit_parser.extract_tap_lines(output) 1976a499c9cSDaniel Latypov for line in output: 1986a499c9cSDaniel Latypov print(line.rstrip()) 199309e22efSDaniel Latypov parse_time = time.time() - parse_start 200309e22efSDaniel Latypov return KunitResult(KunitStatus.SUCCESS, parse_time), fake_test 2016a499c9cSDaniel Latypov 202309e22efSDaniel Latypov 203309e22efSDaniel Latypov # Actually parse the test results. 204309e22efSDaniel Latypov test = kunit_parser.parse_run_tests(input_data) 205309e22efSDaniel Latypov parse_time = time.time() - parse_start 20645ba7a89SDavid Gow 20721a6d178SHeidi Fahim if request.json: 20800f75043SDaniel Latypov json_str = kunit_json.get_json_result( 209309e22efSDaniel Latypov test=test, 210ee96d25fSDaniel Latypov metadata=metadata) 21121a6d178SHeidi Fahim if request.json == 'stdout': 21200f75043SDaniel Latypov print(json_str) 21300f75043SDaniel Latypov else: 21400f75043SDaniel Latypov with open(request.json, 'w') as f: 21500f75043SDaniel Latypov f.write(json_str) 216e756dbebSDaniel Latypov stdout.print_with_timestamp("Test results stored in %s" % 21700f75043SDaniel Latypov os.path.abspath(request.json)) 21821a6d178SHeidi Fahim 219309e22efSDaniel Latypov if test.status != kunit_parser.TestStatus.SUCCESS: 220309e22efSDaniel Latypov return KunitResult(KunitStatus.TEST_FAILURE, parse_time), test 22145ba7a89SDavid Gow 222309e22efSDaniel Latypov return KunitResult(KunitStatus.SUCCESS, parse_time), test 22345ba7a89SDavid Gow 22445ba7a89SDavid Gowdef run_tests(linux: kunit_kernel.LinuxSourceTree, 22545ba7a89SDavid Gow request: KunitRequest) -> KunitResult: 22645ba7a89SDavid Gow run_start = time.time() 22745ba7a89SDavid Gow 228db167981SDaniel Latypov config_result = config_tests(linux, request) 22945ba7a89SDavid Gow if config_result.status != KunitStatus.SUCCESS: 23045ba7a89SDavid Gow return config_result 23145ba7a89SDavid Gow 232db167981SDaniel Latypov build_result = build_tests(linux, request) 23345ba7a89SDavid Gow if build_result.status != KunitStatus.SUCCESS: 23445ba7a89SDavid Gow return build_result 23545ba7a89SDavid Gow 236db167981SDaniel Latypov exec_result = exec_tests(linux, request) 23745ba7a89SDavid Gow 23845ba7a89SDavid Gow run_end = time.time() 23945ba7a89SDavid Gow 240e756dbebSDaniel Latypov stdout.print_with_timestamp(( 2416ebf5866SFelix Guo 'Elapsed time: %.3fs total, %.3fs configuring, %.3fs ' + 2426ebf5866SFelix Guo 'building, %.3fs running\n') % ( 24345ba7a89SDavid Gow run_end - run_start, 24445ba7a89SDavid Gow config_result.elapsed_time, 24545ba7a89SDavid Gow build_result.elapsed_time, 24645ba7a89SDavid Gow exec_result.elapsed_time)) 2477ef925eaSDaniel Latypov return exec_result 2486ebf5866SFelix Guo 249d8c23eadSDaniel Latypov# Problem: 250d8c23eadSDaniel Latypov# $ kunit.py run --json 251d8c23eadSDaniel Latypov# works as one would expect and prints the parsed test results as JSON. 252d8c23eadSDaniel Latypov# $ kunit.py run --json suite_name 253d8c23eadSDaniel Latypov# would *not* pass suite_name as the filter_glob and print as json. 254d8c23eadSDaniel Latypov# argparse will consider it to be another way of writing 255d8c23eadSDaniel Latypov# $ kunit.py run --json=suite_name 256d8c23eadSDaniel Latypov# i.e. it would run all tests, and dump the json to a `suite_name` file. 257d8c23eadSDaniel Latypov# So we hackily automatically rewrite --json => --json=stdout 258d8c23eadSDaniel Latypovpseudo_bool_flag_defaults = { 259d8c23eadSDaniel Latypov '--json': 'stdout', 260d8c23eadSDaniel Latypov '--raw_output': 'kunit', 261d8c23eadSDaniel Latypov} 262d8c23eadSDaniel Latypovdef massage_argv(argv: Sequence[str]) -> Sequence[str]: 263d8c23eadSDaniel Latypov def massage_arg(arg: str) -> str: 264d8c23eadSDaniel Latypov if arg not in pseudo_bool_flag_defaults: 265d8c23eadSDaniel Latypov return arg 266d8c23eadSDaniel Latypov return f'{arg}={pseudo_bool_flag_defaults[arg]}' 267d8c23eadSDaniel Latypov return list(map(massage_arg, argv)) 268d8c23eadSDaniel Latypov 269ad659ccbSDavid Gowdef get_default_jobs() -> int: 270ad659ccbSDavid Gow return len(os.sched_getaffinity(0)) 271ad659ccbSDavid Gow 27209641f7cSDaniel Latypovdef add_common_opts(parser) -> None: 27345ba7a89SDavid Gow parser.add_argument('--build_dir', 27445ba7a89SDavid Gow help='As in the make command, it specifies the build ' 27545ba7a89SDavid Gow 'directory.', 276baa33315SDaniel Latypov type=str, default='.kunit', metavar='DIR') 27745ba7a89SDavid Gow parser.add_argument('--make_options', 27845ba7a89SDavid Gow help='X=Y make option, can be repeated.', 279baa33315SDaniel Latypov action='append', metavar='X=Y') 28045ba7a89SDavid Gow parser.add_argument('--alltests', 281980ac3adSDaniel Latypov help='Run all KUnit tests via tools/testing/kunit/configs/all_tests.config', 2826ebf5866SFelix Guo action='store_true') 283243180f5SDaniel Latypov parser.add_argument('--kunitconfig', 2849854781dSDaniel Latypov help='Path to Kconfig fragment that enables KUnit tests.' 2859854781dSDaniel Latypov ' If given a directory, (e.g. lib/kunit), "/.kunitconfig" ' 28653b46621SDaniel Latypov 'will get automatically appended. If repeated, the files ' 28753b46621SDaniel Latypov 'blindly concatenated, which might not work in all cases.', 28853b46621SDaniel Latypov action='append', metavar='PATHS') 2899f57cc76SDaniel Latypov parser.add_argument('--kconfig_add', 2909f57cc76SDaniel Latypov help='Additional Kconfig options to append to the ' 2919f57cc76SDaniel Latypov '.kunitconfig, e.g. CONFIG_KASAN=y. Can be repeated.', 292baa33315SDaniel Latypov action='append', metavar='CONFIG_X=Y') 2936ebf5866SFelix Guo 29487c9c163SBrendan Higgins parser.add_argument('--arch', 29587c9c163SBrendan Higgins help=('Specifies the architecture to run tests under. ' 29687c9c163SBrendan Higgins 'The architecture specified here must match the ' 29787c9c163SBrendan Higgins 'string passed to the ARCH make param, ' 29887c9c163SBrendan Higgins 'e.g. i386, x86_64, arm, um, etc. Non-UML ' 29987c9c163SBrendan Higgins 'architectures run on QEMU.'), 300baa33315SDaniel Latypov type=str, default='um', metavar='ARCH') 30187c9c163SBrendan Higgins 30287c9c163SBrendan Higgins parser.add_argument('--cross_compile', 30387c9c163SBrendan Higgins help=('Sets make\'s CROSS_COMPILE variable; it should ' 30487c9c163SBrendan Higgins 'be set to a toolchain path prefix (the prefix ' 30587c9c163SBrendan Higgins 'of gcc and other tools in your toolchain, for ' 30687c9c163SBrendan Higgins 'example `sparc64-linux-gnu-` if you have the ' 30787c9c163SBrendan Higgins 'sparc toolchain installed on your system, or ' 30887c9c163SBrendan Higgins '`$HOME/toolchains/microblaze/gcc-9.2.0-nolibc/microblaze-linux/bin/microblaze-linux-` ' 30987c9c163SBrendan Higgins 'if you have downloaded the microblaze toolchain ' 31087c9c163SBrendan Higgins 'from the 0-day website to a directory in your ' 31187c9c163SBrendan Higgins 'home directory called `toolchains`).'), 312baa33315SDaniel Latypov metavar='PREFIX') 31387c9c163SBrendan Higgins 31487c9c163SBrendan Higgins parser.add_argument('--qemu_config', 31587c9c163SBrendan Higgins help=('Takes a path to a path to a file containing ' 31687c9c163SBrendan Higgins 'a QemuArchParams object.'), 317baa33315SDaniel Latypov type=str, metavar='FILE') 31887c9c163SBrendan Higgins 319a9333bd3SDaniel Latypov parser.add_argument('--qemu_args', 320a9333bd3SDaniel Latypov help='Additional QEMU arguments, e.g. "-smp 8"', 321a9333bd3SDaniel Latypov action='append', metavar='') 322a9333bd3SDaniel Latypov 32309641f7cSDaniel Latypovdef add_build_opts(parser) -> None: 32445ba7a89SDavid Gow parser.add_argument('--jobs', 32545ba7a89SDavid Gow help='As in the make command, "Specifies the number of ' 32645ba7a89SDavid Gow 'jobs (commands) to run simultaneously."', 327baa33315SDaniel Latypov type=int, default=get_default_jobs(), metavar='N') 32845ba7a89SDavid Gow 32909641f7cSDaniel Latypovdef add_exec_opts(parser) -> None: 33045ba7a89SDavid Gow parser.add_argument('--timeout', 3316ebf5866SFelix Guo help='maximum number of seconds to allow for all tests ' 3326ebf5866SFelix Guo 'to run. This does not include time taken to build the ' 3336ebf5866SFelix Guo 'tests.', 3346ebf5866SFelix Guo type=int, 3356ebf5866SFelix Guo default=300, 336baa33315SDaniel Latypov metavar='SECONDS') 337d992880bSDaniel Latypov parser.add_argument('filter_glob', 338a127b154SDaniel Latypov help='Filter which KUnit test suites/tests run at ' 339a127b154SDaniel Latypov 'boot-time, e.g. list* or list*.*del_test', 340d992880bSDaniel Latypov type=str, 341d992880bSDaniel Latypov nargs='?', 342d992880bSDaniel Latypov default='', 343d992880bSDaniel Latypov metavar='filter_glob') 3446cb51a18SDaniel Latypov parser.add_argument('--kernel_args', 3456cb51a18SDaniel Latypov help='Kernel command-line parameters. Maybe be repeated', 346baa33315SDaniel Latypov action='append', metavar='') 347ff9e09a3SDaniel Latypov parser.add_argument('--run_isolated', help='If set, boot the kernel for each ' 348ff9e09a3SDaniel Latypov 'individual suite/test. This is can be useful for debugging ' 349ff9e09a3SDaniel Latypov 'a non-hermetic test, one that might pass/fail based on ' 350ff9e09a3SDaniel Latypov 'what ran before it.', 351ff9e09a3SDaniel Latypov type=str, 3520453f984SDaniel Latypov choices=['suite', 'test']) 3536ebf5866SFelix Guo 35409641f7cSDaniel Latypovdef add_parse_opts(parser) -> None: 355309e22efSDaniel Latypov parser.add_argument('--raw_output', help='If set don\'t parse output from kernel. ' 356309e22efSDaniel Latypov 'By default, filters to just KUnit output. Use ' 357309e22efSDaniel Latypov '--raw_output=all to show everything', 358baa33315SDaniel Latypov type=str, nargs='?', const='all', default=None, choices=['all', 'kunit']) 35921a6d178SHeidi Fahim parser.add_argument('--json', 36021a6d178SHeidi Fahim nargs='?', 361309e22efSDaniel Latypov help='Prints parsed test results as JSON to stdout or a file if ' 362309e22efSDaniel Latypov 'a filename is specified. Does nothing if --raw_output is set.', 363baa33315SDaniel Latypov type=str, const='stdout', default=None, metavar='FILE') 364021ed9f5SHeidi Fahim 3658a04930fSDaniel Latypov 3668a04930fSDaniel Latypovdef tree_from_args(cli_args: argparse.Namespace) -> kunit_kernel.LinuxSourceTree: 3678a04930fSDaniel Latypov """Returns a LinuxSourceTree based on the user's arguments.""" 368a9333bd3SDaniel Latypov # Allow users to specify multiple arguments in one string, e.g. '-smp 8' 369a9333bd3SDaniel Latypov qemu_args: List[str] = [] 370a9333bd3SDaniel Latypov if cli_args.qemu_args: 371a9333bd3SDaniel Latypov for arg in cli_args.qemu_args: 372a9333bd3SDaniel Latypov qemu_args.extend(shlex.split(arg)) 373a9333bd3SDaniel Latypov 374980ac3adSDaniel Latypov kunitconfigs = cli_args.kunitconfig if cli_args.kunitconfig else [] 375980ac3adSDaniel Latypov if cli_args.alltests: 376980ac3adSDaniel Latypov # Prepend so user-specified options take prio if we ever allow 377980ac3adSDaniel Latypov # --kunitconfig options to have differing options. 378980ac3adSDaniel Latypov kunitconfigs = [kunit_kernel.ALL_TESTS_CONFIG_PATH] + kunitconfigs 379980ac3adSDaniel Latypov 3808a04930fSDaniel Latypov return kunit_kernel.LinuxSourceTree(cli_args.build_dir, 381980ac3adSDaniel Latypov kunitconfig_paths=kunitconfigs, 3828a04930fSDaniel Latypov kconfig_add=cli_args.kconfig_add, 3838a04930fSDaniel Latypov arch=cli_args.arch, 3848a04930fSDaniel Latypov cross_compile=cli_args.cross_compile, 385a9333bd3SDaniel Latypov qemu_config_path=cli_args.qemu_config, 386a9333bd3SDaniel Latypov extra_qemu_args=qemu_args) 3878a04930fSDaniel Latypov 3888a04930fSDaniel Latypov 3892dc9d6caSAlexander Pantyukhindef run_handler(cli_args): 3902dc9d6caSAlexander Pantyukhin if not os.path.exists(cli_args.build_dir): 3912dc9d6caSAlexander Pantyukhin os.mkdir(cli_args.build_dir) 3922dc9d6caSAlexander Pantyukhin 3932dc9d6caSAlexander Pantyukhin linux = tree_from_args(cli_args) 3942dc9d6caSAlexander Pantyukhin request = KunitRequest(build_dir=cli_args.build_dir, 3952dc9d6caSAlexander Pantyukhin make_options=cli_args.make_options, 3962dc9d6caSAlexander Pantyukhin jobs=cli_args.jobs, 3972dc9d6caSAlexander Pantyukhin raw_output=cli_args.raw_output, 3982dc9d6caSAlexander Pantyukhin json=cli_args.json, 3992dc9d6caSAlexander Pantyukhin timeout=cli_args.timeout, 4002dc9d6caSAlexander Pantyukhin filter_glob=cli_args.filter_glob, 4012dc9d6caSAlexander Pantyukhin kernel_args=cli_args.kernel_args, 4022dc9d6caSAlexander Pantyukhin run_isolated=cli_args.run_isolated) 4032dc9d6caSAlexander Pantyukhin result = run_tests(linux, request) 4042dc9d6caSAlexander Pantyukhin if result.status != KunitStatus.SUCCESS: 4052dc9d6caSAlexander Pantyukhin sys.exit(1) 4062dc9d6caSAlexander Pantyukhin 4072dc9d6caSAlexander Pantyukhin 4082dc9d6caSAlexander Pantyukhindef config_handler(cli_args): 4092dc9d6caSAlexander Pantyukhin if cli_args.build_dir and ( 4102dc9d6caSAlexander Pantyukhin not os.path.exists(cli_args.build_dir)): 4112dc9d6caSAlexander Pantyukhin os.mkdir(cli_args.build_dir) 4122dc9d6caSAlexander Pantyukhin 4132dc9d6caSAlexander Pantyukhin linux = tree_from_args(cli_args) 4142dc9d6caSAlexander Pantyukhin request = KunitConfigRequest(build_dir=cli_args.build_dir, 4152dc9d6caSAlexander Pantyukhin make_options=cli_args.make_options) 4162dc9d6caSAlexander Pantyukhin result = config_tests(linux, request) 4172dc9d6caSAlexander Pantyukhin stdout.print_with_timestamp(( 4182dc9d6caSAlexander Pantyukhin 'Elapsed time: %.3fs\n') % ( 4192dc9d6caSAlexander Pantyukhin result.elapsed_time)) 4202dc9d6caSAlexander Pantyukhin if result.status != KunitStatus.SUCCESS: 4212dc9d6caSAlexander Pantyukhin sys.exit(1) 4222dc9d6caSAlexander Pantyukhin 4232dc9d6caSAlexander Pantyukhin 4242dc9d6caSAlexander Pantyukhindef build_handler(cli_args): 4252dc9d6caSAlexander Pantyukhin linux = tree_from_args(cli_args) 4262dc9d6caSAlexander Pantyukhin request = KunitBuildRequest(build_dir=cli_args.build_dir, 4272dc9d6caSAlexander Pantyukhin make_options=cli_args.make_options, 4282dc9d6caSAlexander Pantyukhin jobs=cli_args.jobs) 4292dc9d6caSAlexander Pantyukhin result = config_and_build_tests(linux, request) 4302dc9d6caSAlexander Pantyukhin stdout.print_with_timestamp(( 4312dc9d6caSAlexander Pantyukhin 'Elapsed time: %.3fs\n') % ( 4322dc9d6caSAlexander Pantyukhin result.elapsed_time)) 4332dc9d6caSAlexander Pantyukhin if result.status != KunitStatus.SUCCESS: 4342dc9d6caSAlexander Pantyukhin sys.exit(1) 4352dc9d6caSAlexander Pantyukhin 4362dc9d6caSAlexander Pantyukhin 4372dc9d6caSAlexander Pantyukhindef exec_handler(cli_args): 4382dc9d6caSAlexander Pantyukhin linux = tree_from_args(cli_args) 4392dc9d6caSAlexander Pantyukhin exec_request = KunitExecRequest(raw_output=cli_args.raw_output, 4402dc9d6caSAlexander Pantyukhin build_dir=cli_args.build_dir, 4412dc9d6caSAlexander Pantyukhin json=cli_args.json, 4422dc9d6caSAlexander Pantyukhin timeout=cli_args.timeout, 4432dc9d6caSAlexander Pantyukhin filter_glob=cli_args.filter_glob, 4442dc9d6caSAlexander Pantyukhin kernel_args=cli_args.kernel_args, 4452dc9d6caSAlexander Pantyukhin run_isolated=cli_args.run_isolated) 4462dc9d6caSAlexander Pantyukhin result = exec_tests(linux, exec_request) 4472dc9d6caSAlexander Pantyukhin stdout.print_with_timestamp(( 4482dc9d6caSAlexander Pantyukhin 'Elapsed time: %.3fs\n') % (result.elapsed_time)) 4492dc9d6caSAlexander Pantyukhin if result.status != KunitStatus.SUCCESS: 4502dc9d6caSAlexander Pantyukhin sys.exit(1) 4512dc9d6caSAlexander Pantyukhin 4522dc9d6caSAlexander Pantyukhin 4532dc9d6caSAlexander Pantyukhindef parse_handler(cli_args): 4542dc9d6caSAlexander Pantyukhin if cli_args.file is None: 4552dc9d6caSAlexander Pantyukhin sys.stdin.reconfigure(errors='backslashreplace') # pytype: disable=attribute-error 4562dc9d6caSAlexander Pantyukhin kunit_output = sys.stdin 4572dc9d6caSAlexander Pantyukhin else: 4582dc9d6caSAlexander Pantyukhin with open(cli_args.file, 'r', errors='backslashreplace') as f: 4592dc9d6caSAlexander Pantyukhin kunit_output = f.read().splitlines() 4602dc9d6caSAlexander Pantyukhin # We know nothing about how the result was created! 4612dc9d6caSAlexander Pantyukhin metadata = kunit_json.Metadata() 4622dc9d6caSAlexander Pantyukhin request = KunitParseRequest(raw_output=cli_args.raw_output, 4632dc9d6caSAlexander Pantyukhin json=cli_args.json) 4642dc9d6caSAlexander Pantyukhin result, _ = parse_tests(request, metadata, kunit_output) 4652dc9d6caSAlexander Pantyukhin if result.status != KunitStatus.SUCCESS: 4662dc9d6caSAlexander Pantyukhin sys.exit(1) 4672dc9d6caSAlexander Pantyukhin 4682dc9d6caSAlexander Pantyukhin 4692dc9d6caSAlexander Pantyukhinsubcommand_handlers_map = { 4702dc9d6caSAlexander Pantyukhin 'run': run_handler, 4712dc9d6caSAlexander Pantyukhin 'config': config_handler, 4722dc9d6caSAlexander Pantyukhin 'build': build_handler, 4732dc9d6caSAlexander Pantyukhin 'exec': exec_handler, 4742dc9d6caSAlexander Pantyukhin 'parse': parse_handler 4752dc9d6caSAlexander Pantyukhin} 4762dc9d6caSAlexander Pantyukhin 4772dc9d6caSAlexander Pantyukhin 4788a04930fSDaniel Latypovdef main(argv): 47945ba7a89SDavid Gow parser = argparse.ArgumentParser( 48045ba7a89SDavid Gow description='Helps writing and running KUnit tests.') 48145ba7a89SDavid Gow subparser = parser.add_subparsers(dest='subcommand') 48245ba7a89SDavid Gow 48345ba7a89SDavid Gow # The 'run' command will config, build, exec, and parse in one go. 48445ba7a89SDavid Gow run_parser = subparser.add_parser('run', help='Runs KUnit tests.') 48545ba7a89SDavid Gow add_common_opts(run_parser) 48645ba7a89SDavid Gow add_build_opts(run_parser) 48745ba7a89SDavid Gow add_exec_opts(run_parser) 48845ba7a89SDavid Gow add_parse_opts(run_parser) 48945ba7a89SDavid Gow 49045ba7a89SDavid Gow config_parser = subparser.add_parser('config', 49145ba7a89SDavid Gow help='Ensures that .config contains all of ' 49245ba7a89SDavid Gow 'the options in .kunitconfig') 49345ba7a89SDavid Gow add_common_opts(config_parser) 49445ba7a89SDavid Gow 49545ba7a89SDavid Gow build_parser = subparser.add_parser('build', help='Builds a kernel with KUnit tests') 49645ba7a89SDavid Gow add_common_opts(build_parser) 49745ba7a89SDavid Gow add_build_opts(build_parser) 49845ba7a89SDavid Gow 49945ba7a89SDavid Gow exec_parser = subparser.add_parser('exec', help='Run a kernel with KUnit tests') 50045ba7a89SDavid Gow add_common_opts(exec_parser) 50145ba7a89SDavid Gow add_exec_opts(exec_parser) 50245ba7a89SDavid Gow add_parse_opts(exec_parser) 50345ba7a89SDavid Gow 50445ba7a89SDavid Gow # The 'parse' option is special, as it doesn't need the kernel source 50545ba7a89SDavid Gow # (therefore there is no need for a build_dir, hence no add_common_opts) 50645ba7a89SDavid Gow # and the '--file' argument is not relevant to 'run', so isn't in 50745ba7a89SDavid Gow # add_parse_opts() 50845ba7a89SDavid Gow parse_parser = subparser.add_parser('parse', 50945ba7a89SDavid Gow help='Parses KUnit results from a file, ' 51045ba7a89SDavid Gow 'and parses formatted results.') 51145ba7a89SDavid Gow add_parse_opts(parse_parser) 51245ba7a89SDavid Gow parse_parser.add_argument('file', 51345ba7a89SDavid Gow help='Specifies the file to read results from.', 51445ba7a89SDavid Gow type=str, nargs='?', metavar='input_file') 5150476e69fSGreg Thelen 516d8c23eadSDaniel Latypov cli_args = parser.parse_args(massage_argv(argv)) 5176ebf5866SFelix Guo 5185578d008SBrendan Higgins if get_kernel_root_path(): 5195578d008SBrendan Higgins os.chdir(get_kernel_root_path()) 5205578d008SBrendan Higgins 5212dc9d6caSAlexander Pantyukhin subcomand_handler = subcommand_handlers_map.get(cli_args.subcommand, None) 52282206a0cSBrendan Higgins 5232dc9d6caSAlexander Pantyukhin if subcomand_handler is None: 5246ebf5866SFelix Guo parser.print_help() 5252dc9d6caSAlexander Pantyukhin return 5262dc9d6caSAlexander Pantyukhin 5272dc9d6caSAlexander Pantyukhin subcomand_handler(cli_args) 5282dc9d6caSAlexander Pantyukhin 5296ebf5866SFelix Guo 5306ebf5866SFelix Guoif __name__ == '__main__': 531ff7b437fSBrendan Higgins main(sys.argv[1:]) 532