1#!/usr/bin/env python3 2# SPDX-License-Identifier: GPL-2.0 3# 4# A thin wrapper on top of the KUnit Kernel 5# 6# Copyright (C) 2019, Google LLC. 7# Author: Felix Guo <felixguoxiuping@gmail.com> 8# Author: Brendan Higgins <brendanhiggins@google.com> 9 10import argparse 11import sys 12import os 13import time 14 15assert sys.version_info >= (3, 7), "Python version is too old" 16 17from collections import namedtuple 18from enum import Enum, auto 19from typing import Iterable, Sequence 20 21import kunit_config 22import kunit_json 23import kunit_kernel 24import kunit_parser 25 26KunitResult = namedtuple('KunitResult', ['status','result','elapsed_time']) 27 28KunitConfigRequest = namedtuple('KunitConfigRequest', 29 ['build_dir', 'make_options']) 30KunitBuildRequest = namedtuple('KunitBuildRequest', 31 ['jobs', 'build_dir', 'alltests', 32 'make_options']) 33KunitExecRequest = namedtuple('KunitExecRequest', 34 ['timeout', 'build_dir', 'alltests', 35 'filter_glob', 'kernel_args']) 36KunitParseRequest = namedtuple('KunitParseRequest', 37 ['raw_output', 'input_data', 'build_dir', 'json']) 38KunitRequest = namedtuple('KunitRequest', ['raw_output','timeout', 'jobs', 39 'build_dir', 'alltests', 'filter_glob', 40 'kernel_args', 'json', 'make_options']) 41 42KernelDirectoryPath = sys.argv[0].split('tools/testing/kunit/')[0] 43 44class KunitStatus(Enum): 45 SUCCESS = auto() 46 CONFIG_FAILURE = auto() 47 BUILD_FAILURE = auto() 48 TEST_FAILURE = auto() 49 50def get_kernel_root_path() -> str: 51 path = sys.argv[0] if not __file__ else __file__ 52 parts = os.path.realpath(path).split('tools/testing/kunit') 53 if len(parts) != 2: 54 sys.exit(1) 55 return parts[0] 56 57def config_tests(linux: kunit_kernel.LinuxSourceTree, 58 request: KunitConfigRequest) -> KunitResult: 59 kunit_parser.print_with_timestamp('Configuring KUnit Kernel ...') 60 61 config_start = time.time() 62 success = linux.build_reconfig(request.build_dir, request.make_options) 63 config_end = time.time() 64 if not success: 65 return KunitResult(KunitStatus.CONFIG_FAILURE, 66 'could not configure kernel', 67 config_end - config_start) 68 return KunitResult(KunitStatus.SUCCESS, 69 'configured kernel successfully', 70 config_end - config_start) 71 72def build_tests(linux: kunit_kernel.LinuxSourceTree, 73 request: KunitBuildRequest) -> KunitResult: 74 kunit_parser.print_with_timestamp('Building KUnit Kernel ...') 75 76 build_start = time.time() 77 success = linux.build_kernel(request.alltests, 78 request.jobs, 79 request.build_dir, 80 request.make_options) 81 build_end = time.time() 82 if not success: 83 return KunitResult(KunitStatus.BUILD_FAILURE, 84 'could not build kernel', 85 build_end - build_start) 86 if not success: 87 return KunitResult(KunitStatus.BUILD_FAILURE, 88 'could not build kernel', 89 build_end - build_start) 90 return KunitResult(KunitStatus.SUCCESS, 91 'built kernel successfully', 92 build_end - build_start) 93 94def exec_tests(linux: kunit_kernel.LinuxSourceTree, 95 request: KunitExecRequest) -> KunitResult: 96 kunit_parser.print_with_timestamp('Starting KUnit Kernel ...') 97 test_start = time.time() 98 result = linux.run_kernel( 99 args=request.kernel_args, 100 timeout=None if request.alltests else request.timeout, 101 filter_glob=request.filter_glob, 102 build_dir=request.build_dir) 103 104 test_end = time.time() 105 106 return KunitResult(KunitStatus.SUCCESS, 107 result, 108 test_end - test_start) 109 110def parse_tests(request: KunitParseRequest) -> KunitResult: 111 parse_start = time.time() 112 113 test_result = kunit_parser.TestResult(kunit_parser.TestStatus.SUCCESS, 114 [], 115 'Tests not Parsed.') 116 117 if request.raw_output: 118 output: Iterable[str] = request.input_data 119 if request.raw_output == 'all': 120 pass 121 elif request.raw_output == 'kunit': 122 output = kunit_parser.extract_tap_lines(output) 123 else: 124 print(f'Unknown --raw_output option "{request.raw_output}"', file=sys.stderr) 125 for line in output: 126 print(line.rstrip()) 127 128 else: 129 test_result = kunit_parser.parse_run_tests(request.input_data) 130 parse_end = time.time() 131 132 if request.json: 133 json_obj = kunit_json.get_json_result( 134 test_result=test_result, 135 def_config='kunit_defconfig', 136 build_dir=request.build_dir, 137 json_path=request.json) 138 if request.json == 'stdout': 139 print(json_obj) 140 141 if test_result.status != kunit_parser.TestStatus.SUCCESS: 142 return KunitResult(KunitStatus.TEST_FAILURE, test_result, 143 parse_end - parse_start) 144 145 return KunitResult(KunitStatus.SUCCESS, test_result, 146 parse_end - parse_start) 147 148def run_tests(linux: kunit_kernel.LinuxSourceTree, 149 request: KunitRequest) -> KunitResult: 150 run_start = time.time() 151 152 config_request = KunitConfigRequest(request.build_dir, 153 request.make_options) 154 config_result = config_tests(linux, config_request) 155 if config_result.status != KunitStatus.SUCCESS: 156 return config_result 157 158 build_request = KunitBuildRequest(request.jobs, request.build_dir, 159 request.alltests, 160 request.make_options) 161 build_result = build_tests(linux, build_request) 162 if build_result.status != KunitStatus.SUCCESS: 163 return build_result 164 165 exec_request = KunitExecRequest(request.timeout, request.build_dir, 166 request.alltests, request.filter_glob, 167 request.kernel_args) 168 exec_result = exec_tests(linux, exec_request) 169 if exec_result.status != KunitStatus.SUCCESS: 170 return exec_result 171 172 parse_request = KunitParseRequest(request.raw_output, 173 exec_result.result, 174 request.build_dir, 175 request.json) 176 parse_result = parse_tests(parse_request) 177 178 run_end = time.time() 179 180 kunit_parser.print_with_timestamp(( 181 'Elapsed time: %.3fs total, %.3fs configuring, %.3fs ' + 182 'building, %.3fs running\n') % ( 183 run_end - run_start, 184 config_result.elapsed_time, 185 build_result.elapsed_time, 186 exec_result.elapsed_time)) 187 return parse_result 188 189# Problem: 190# $ kunit.py run --json 191# works as one would expect and prints the parsed test results as JSON. 192# $ kunit.py run --json suite_name 193# would *not* pass suite_name as the filter_glob and print as json. 194# argparse will consider it to be another way of writing 195# $ kunit.py run --json=suite_name 196# i.e. it would run all tests, and dump the json to a `suite_name` file. 197# So we hackily automatically rewrite --json => --json=stdout 198pseudo_bool_flag_defaults = { 199 '--json': 'stdout', 200 '--raw_output': 'kunit', 201} 202def massage_argv(argv: Sequence[str]) -> Sequence[str]: 203 def massage_arg(arg: str) -> str: 204 if arg not in pseudo_bool_flag_defaults: 205 return arg 206 return f'{arg}={pseudo_bool_flag_defaults[arg]}' 207 return list(map(massage_arg, argv)) 208 209def add_common_opts(parser) -> None: 210 parser.add_argument('--build_dir', 211 help='As in the make command, it specifies the build ' 212 'directory.', 213 type=str, default='.kunit', metavar='build_dir') 214 parser.add_argument('--make_options', 215 help='X=Y make option, can be repeated.', 216 action='append') 217 parser.add_argument('--alltests', 218 help='Run all KUnit tests through allyesconfig', 219 action='store_true') 220 parser.add_argument('--kunitconfig', 221 help='Path to Kconfig fragment that enables KUnit tests.' 222 ' If given a directory, (e.g. lib/kunit), "/.kunitconfig" ' 223 'will get automatically appended.', 224 metavar='kunitconfig') 225 226 parser.add_argument('--arch', 227 help=('Specifies the architecture to run tests under. ' 228 'The architecture specified here must match the ' 229 'string passed to the ARCH make param, ' 230 'e.g. i386, x86_64, arm, um, etc. Non-UML ' 231 'architectures run on QEMU.'), 232 type=str, default='um', metavar='arch') 233 234 parser.add_argument('--cross_compile', 235 help=('Sets make\'s CROSS_COMPILE variable; it should ' 236 'be set to a toolchain path prefix (the prefix ' 237 'of gcc and other tools in your toolchain, for ' 238 'example `sparc64-linux-gnu-` if you have the ' 239 'sparc toolchain installed on your system, or ' 240 '`$HOME/toolchains/microblaze/gcc-9.2.0-nolibc/microblaze-linux/bin/microblaze-linux-` ' 241 'if you have downloaded the microblaze toolchain ' 242 'from the 0-day website to a directory in your ' 243 'home directory called `toolchains`).'), 244 metavar='cross_compile') 245 246 parser.add_argument('--qemu_config', 247 help=('Takes a path to a path to a file containing ' 248 'a QemuArchParams object.'), 249 type=str, metavar='qemu_config') 250 251def add_build_opts(parser) -> None: 252 parser.add_argument('--jobs', 253 help='As in the make command, "Specifies the number of ' 254 'jobs (commands) to run simultaneously."', 255 type=int, default=8, metavar='jobs') 256 257def add_exec_opts(parser) -> None: 258 parser.add_argument('--timeout', 259 help='maximum number of seconds to allow for all tests ' 260 'to run. This does not include time taken to build the ' 261 'tests.', 262 type=int, 263 default=300, 264 metavar='timeout') 265 parser.add_argument('filter_glob', 266 help='maximum number of seconds to allow for all tests ' 267 'to run. This does not include time taken to build the ' 268 'tests.', 269 type=str, 270 nargs='?', 271 default='', 272 metavar='filter_glob') 273 parser.add_argument('--kernel_args', 274 help='Kernel command-line parameters. Maybe be repeated', 275 action='append') 276 277def add_parse_opts(parser) -> None: 278 parser.add_argument('--raw_output', help='If set don\'t format output from kernel. ' 279 'If set to --raw_output=kunit, filters to just KUnit output.', 280 type=str, nargs='?', const='all', default=None) 281 parser.add_argument('--json', 282 nargs='?', 283 help='Stores test results in a JSON, and either ' 284 'prints to stdout or saves to file if a ' 285 'filename is specified', 286 type=str, const='stdout', default=None) 287 288def main(argv, linux=None): 289 parser = argparse.ArgumentParser( 290 description='Helps writing and running KUnit tests.') 291 subparser = parser.add_subparsers(dest='subcommand') 292 293 # The 'run' command will config, build, exec, and parse in one go. 294 run_parser = subparser.add_parser('run', help='Runs KUnit tests.') 295 add_common_opts(run_parser) 296 add_build_opts(run_parser) 297 add_exec_opts(run_parser) 298 add_parse_opts(run_parser) 299 300 config_parser = subparser.add_parser('config', 301 help='Ensures that .config contains all of ' 302 'the options in .kunitconfig') 303 add_common_opts(config_parser) 304 305 build_parser = subparser.add_parser('build', help='Builds a kernel with KUnit tests') 306 add_common_opts(build_parser) 307 add_build_opts(build_parser) 308 309 exec_parser = subparser.add_parser('exec', help='Run a kernel with KUnit tests') 310 add_common_opts(exec_parser) 311 add_exec_opts(exec_parser) 312 add_parse_opts(exec_parser) 313 314 # The 'parse' option is special, as it doesn't need the kernel source 315 # (therefore there is no need for a build_dir, hence no add_common_opts) 316 # and the '--file' argument is not relevant to 'run', so isn't in 317 # add_parse_opts() 318 parse_parser = subparser.add_parser('parse', 319 help='Parses KUnit results from a file, ' 320 'and parses formatted results.') 321 add_parse_opts(parse_parser) 322 parse_parser.add_argument('file', 323 help='Specifies the file to read results from.', 324 type=str, nargs='?', metavar='input_file') 325 326 cli_args = parser.parse_args(massage_argv(argv)) 327 328 if get_kernel_root_path(): 329 os.chdir(get_kernel_root_path()) 330 331 if cli_args.subcommand == 'run': 332 if not os.path.exists(cli_args.build_dir): 333 os.mkdir(cli_args.build_dir) 334 335 if not linux: 336 linux = kunit_kernel.LinuxSourceTree(cli_args.build_dir, 337 kunitconfig_path=cli_args.kunitconfig, 338 arch=cli_args.arch, 339 cross_compile=cli_args.cross_compile, 340 qemu_config_path=cli_args.qemu_config) 341 342 request = KunitRequest(cli_args.raw_output, 343 cli_args.timeout, 344 cli_args.jobs, 345 cli_args.build_dir, 346 cli_args.alltests, 347 cli_args.filter_glob, 348 cli_args.kernel_args, 349 cli_args.json, 350 cli_args.make_options) 351 result = run_tests(linux, request) 352 if result.status != KunitStatus.SUCCESS: 353 sys.exit(1) 354 elif cli_args.subcommand == 'config': 355 if cli_args.build_dir and ( 356 not os.path.exists(cli_args.build_dir)): 357 os.mkdir(cli_args.build_dir) 358 359 if not linux: 360 linux = kunit_kernel.LinuxSourceTree(cli_args.build_dir, 361 kunitconfig_path=cli_args.kunitconfig, 362 arch=cli_args.arch, 363 cross_compile=cli_args.cross_compile, 364 qemu_config_path=cli_args.qemu_config) 365 366 request = KunitConfigRequest(cli_args.build_dir, 367 cli_args.make_options) 368 result = config_tests(linux, request) 369 kunit_parser.print_with_timestamp(( 370 'Elapsed time: %.3fs\n') % ( 371 result.elapsed_time)) 372 if result.status != KunitStatus.SUCCESS: 373 sys.exit(1) 374 elif cli_args.subcommand == 'build': 375 if not linux: 376 linux = kunit_kernel.LinuxSourceTree(cli_args.build_dir, 377 kunitconfig_path=cli_args.kunitconfig, 378 arch=cli_args.arch, 379 cross_compile=cli_args.cross_compile, 380 qemu_config_path=cli_args.qemu_config) 381 382 request = KunitBuildRequest(cli_args.jobs, 383 cli_args.build_dir, 384 cli_args.alltests, 385 cli_args.make_options) 386 result = build_tests(linux, request) 387 kunit_parser.print_with_timestamp(( 388 'Elapsed time: %.3fs\n') % ( 389 result.elapsed_time)) 390 if result.status != KunitStatus.SUCCESS: 391 sys.exit(1) 392 elif cli_args.subcommand == 'exec': 393 if not linux: 394 linux = kunit_kernel.LinuxSourceTree(cli_args.build_dir, 395 kunitconfig_path=cli_args.kunitconfig, 396 arch=cli_args.arch, 397 cross_compile=cli_args.cross_compile, 398 qemu_config_path=cli_args.qemu_config) 399 400 exec_request = KunitExecRequest(cli_args.timeout, 401 cli_args.build_dir, 402 cli_args.alltests, 403 cli_args.filter_glob, 404 cli_args.kernel_args) 405 exec_result = exec_tests(linux, exec_request) 406 parse_request = KunitParseRequest(cli_args.raw_output, 407 exec_result.result, 408 cli_args.build_dir, 409 cli_args.json) 410 result = parse_tests(parse_request) 411 kunit_parser.print_with_timestamp(( 412 'Elapsed time: %.3fs\n') % ( 413 exec_result.elapsed_time)) 414 if result.status != KunitStatus.SUCCESS: 415 sys.exit(1) 416 elif cli_args.subcommand == 'parse': 417 if cli_args.file == None: 418 kunit_output = sys.stdin 419 else: 420 with open(cli_args.file, 'r') as f: 421 kunit_output = f.read().splitlines() 422 request = KunitParseRequest(cli_args.raw_output, 423 kunit_output, 424 None, 425 cli_args.json) 426 result = parse_tests(request) 427 if result.status != KunitStatus.SUCCESS: 428 sys.exit(1) 429 else: 430 parser.print_help() 431 432if __name__ == '__main__': 433 main(sys.argv[1:]) 434