| kunit.py (0aea30a07ec6b50de0fc5f5b2ec34a68ead86b61) | kunit.py (0453f984a7b9458f0e469afb039f2841308b1bef) |
|---|---|
| 1#!/usr/bin/env python3 2# SPDX-License-Identifier: GPL-2.0 3# 4# A thin wrapper on top of the KUnit Kernel 5# 6# Copyright (C) 2019, Google LLC. 7# Author: Felix Guo <felixguoxiuping@gmail.com> 8# Author: Brendan Higgins <brendanhiggins@google.com> --- 33 unchanged lines hidden (view full) --- 42@dataclass 43class KunitBuildRequest(KunitConfigRequest): 44 jobs: int 45 alltests: bool 46 47@dataclass 48class KunitParseRequest: 49 raw_output: Optional[str] | 1#!/usr/bin/env python3 2# SPDX-License-Identifier: GPL-2.0 3# 4# A thin wrapper on top of the KUnit Kernel 5# 6# Copyright (C) 2019, Google LLC. 7# Author: Felix Guo <felixguoxiuping@gmail.com> 8# Author: Brendan Higgins <brendanhiggins@google.com> --- 33 unchanged lines hidden (view full) --- 42@dataclass 43class KunitBuildRequest(KunitConfigRequest): 44 jobs: int 45 alltests: bool 46 47@dataclass 48class KunitParseRequest: 49 raw_output: Optional[str] |
| 50 build_dir: str | |
| 51 json: Optional[str] 52 53@dataclass 54class KunitExecRequest(KunitParseRequest): | 50 json: Optional[str] 51 52@dataclass 53class KunitExecRequest(KunitParseRequest): |
| 54 build_dir: str |
|
| 55 timeout: int 56 alltests: bool 57 filter_glob: str 58 kernel_args: Optional[List[str]] 59 run_isolated: Optional[str] 60 61@dataclass 62class KunitRequest(KunitExecRequest, KunitBuildRequest): 63 pass 64 65 | 55 timeout: int 56 alltests: bool 57 filter_glob: str 58 kernel_args: Optional[List[str]] 59 run_isolated: Optional[str] 60 61@dataclass 62class KunitRequest(KunitExecRequest, KunitBuildRequest): 63 pass 64 65 |
| 66KernelDirectoryPath = sys.argv[0].split('tools/testing/kunit/')[0] 67 | |
| 68def get_kernel_root_path() -> str: 69 path = sys.argv[0] if not __file__ else __file__ 70 parts = os.path.realpath(path).split('tools/testing/kunit') 71 if len(parts) != 2: 72 sys.exit(1) 73 return parts[0] 74 75def config_tests(linux: kunit_kernel.LinuxSourceTree, --- 45 unchanged lines hidden (view full) --- 121 timeout=None if request.alltests else request.timeout, 122 filter_glob=request.filter_glob, 123 build_dir=request.build_dir) 124 lines = kunit_parser.extract_tap_lines(output) 125 # Hack! Drop the dummy TAP version header that the executor prints out. 126 lines.pop() 127 128 # Filter out any extraneous non-test output that might have gotten mixed in. | 66def get_kernel_root_path() -> str: 67 path = sys.argv[0] if not __file__ else __file__ 68 parts = os.path.realpath(path).split('tools/testing/kunit') 69 if len(parts) != 2: 70 sys.exit(1) 71 return parts[0] 72 73def config_tests(linux: kunit_kernel.LinuxSourceTree, --- 45 unchanged lines hidden (view full) --- 119 timeout=None if request.alltests else request.timeout, 120 filter_glob=request.filter_glob, 121 build_dir=request.build_dir) 122 lines = kunit_parser.extract_tap_lines(output) 123 # Hack! Drop the dummy TAP version header that the executor prints out. 124 lines.pop() 125 126 # Filter out any extraneous non-test output that might have gotten mixed in. |
| 129 return [l for l in lines if re.match('^[^\s.]+\.[^\s.]+$', l)] | 127 return [l for l in lines if re.match(r'^[^\s.]+\.[^\s.]+$', l)] |
| 130 131def _suites_from_test_list(tests: List[str]) -> List[str]: 132 """Extracts all the suites from an ordered list of tests.""" 133 suites = [] # type: List[str] 134 for t in tests: 135 parts = t.split('.', maxsplit=2) 136 if len(parts) != 2: 137 raise ValueError(f'internal KUnit error, test name should be of the form "<suite>.<test>", got "{t}"') --- 12 unchanged lines hidden (view full) --- 150 filter_globs = tests 151 if request.run_isolated == 'suite': 152 filter_globs = _suites_from_test_list(tests) 153 # Apply the test-part of the user's glob, if present. 154 if '.' in request.filter_glob: 155 test_glob = request.filter_glob.split('.', maxsplit=2)[1] 156 filter_globs = [g + '.'+ test_glob for g in filter_globs] 157 | 128 129def _suites_from_test_list(tests: List[str]) -> List[str]: 130 """Extracts all the suites from an ordered list of tests.""" 131 suites = [] # type: List[str] 132 for t in tests: 133 parts = t.split('.', maxsplit=2) 134 if len(parts) != 2: 135 raise ValueError(f'internal KUnit error, test name should be of the form "<suite>.<test>", got "{t}"') --- 12 unchanged lines hidden (view full) --- 148 filter_globs = tests 149 if request.run_isolated == 'suite': 150 filter_globs = _suites_from_test_list(tests) 151 # Apply the test-part of the user's glob, if present. 152 if '.' in request.filter_glob: 153 test_glob = request.filter_glob.split('.', maxsplit=2)[1] 154 filter_globs = [g + '.'+ test_glob for g in filter_globs] 155 |
| 156 metadata = kunit_json.Metadata(arch=linux.arch(), build_dir=request.build_dir, def_config='kunit_defconfig') 157 |
|
| 158 test_counts = kunit_parser.TestCounts() 159 exec_time = 0.0 160 for i, filter_glob in enumerate(filter_globs): 161 kunit_parser.print_with_timestamp('Starting KUnit Kernel ({}/{})...'.format(i+1, len(filter_globs))) 162 163 test_start = time.time() 164 run_result = linux.run_kernel( 165 args=request.kernel_args, 166 timeout=None if request.alltests else request.timeout, 167 filter_glob=filter_glob, 168 build_dir=request.build_dir) 169 | 158 test_counts = kunit_parser.TestCounts() 159 exec_time = 0.0 160 for i, filter_glob in enumerate(filter_globs): 161 kunit_parser.print_with_timestamp('Starting KUnit Kernel ({}/{})...'.format(i+1, len(filter_globs))) 162 163 test_start = time.time() 164 run_result = linux.run_kernel( 165 args=request.kernel_args, 166 timeout=None if request.alltests else request.timeout, 167 filter_glob=filter_glob, 168 build_dir=request.build_dir) 169 |
| 170 _, test_result = parse_tests(request, run_result) | 170 _, test_result = parse_tests(request, metadata, run_result) |
| 171 # run_kernel() doesn't block on the kernel exiting. 172 # That only happens after we get the last line of output from `run_result`. 173 # So exec_time here actually contains parsing + execution time, which is fine. 174 test_end = time.time() 175 exec_time += test_end - test_start 176 177 test_counts.add_subtest_counts(test_result.counts) 178 --- 4 unchanged lines hidden (view full) --- 183 bd, bd, kunit_kernel.get_outfile_path(bd), bd, sys.argv[0])) 184 185 kunit_status = _map_to_overall_status(test_counts.get_status()) 186 return KunitResult(status=kunit_status, elapsed_time=exec_time) 187 188def _map_to_overall_status(test_status: kunit_parser.TestStatus) -> KunitStatus: 189 if test_status in (kunit_parser.TestStatus.SUCCESS, kunit_parser.TestStatus.SKIPPED): 190 return KunitStatus.SUCCESS | 171 # run_kernel() doesn't block on the kernel exiting. 172 # That only happens after we get the last line of output from `run_result`. 173 # So exec_time here actually contains parsing + execution time, which is fine. 174 test_end = time.time() 175 exec_time += test_end - test_start 176 177 test_counts.add_subtest_counts(test_result.counts) 178 --- 4 unchanged lines hidden (view full) --- 183 bd, bd, kunit_kernel.get_outfile_path(bd), bd, sys.argv[0])) 184 185 kunit_status = _map_to_overall_status(test_counts.get_status()) 186 return KunitResult(status=kunit_status, elapsed_time=exec_time) 187 188def _map_to_overall_status(test_status: kunit_parser.TestStatus) -> KunitStatus: 189 if test_status in (kunit_parser.TestStatus.SUCCESS, kunit_parser.TestStatus.SKIPPED): 190 return KunitStatus.SUCCESS |
| 191 else: 192 return KunitStatus.TEST_FAILURE | 191 return KunitStatus.TEST_FAILURE |
| 193 | 192 |
| 194def parse_tests(request: KunitParseRequest, input_data: Iterable[str]) -> Tuple[KunitResult, kunit_parser.Test]: | 193def parse_tests(request: KunitParseRequest, metadata: kunit_json.Metadata, input_data: Iterable[str]) -> Tuple[KunitResult, kunit_parser.Test]: |
| 195 parse_start = time.time() 196 197 test_result = kunit_parser.Test() 198 199 if request.raw_output: 200 # Treat unparsed results as one passing test. 201 test_result.status = kunit_parser.TestStatus.SUCCESS 202 test_result.counts.passed = 1 203 204 output: Iterable[str] = input_data 205 if request.raw_output == 'all': 206 pass 207 elif request.raw_output == 'kunit': 208 output = kunit_parser.extract_tap_lines(output) | 194 parse_start = time.time() 195 196 test_result = kunit_parser.Test() 197 198 if request.raw_output: 199 # Treat unparsed results as one passing test. 200 test_result.status = kunit_parser.TestStatus.SUCCESS 201 test_result.counts.passed = 1 202 203 output: Iterable[str] = input_data 204 if request.raw_output == 'all': 205 pass 206 elif request.raw_output == 'kunit': 207 output = kunit_parser.extract_tap_lines(output) |
| 209 else: 210 print(f'Unknown --raw_output option "{request.raw_output}"', file=sys.stderr) | |
| 211 for line in output: 212 print(line.rstrip()) 213 214 else: 215 test_result = kunit_parser.parse_run_tests(input_data) 216 parse_end = time.time() 217 218 if request.json: | 208 for line in output: 209 print(line.rstrip()) 210 211 else: 212 test_result = kunit_parser.parse_run_tests(input_data) 213 parse_end = time.time() 214 215 if request.json: |
| 219 json_obj = kunit_json.get_json_result( | 216 json_str = kunit_json.get_json_result( |
| 220 test=test_result, | 217 test=test_result, |
| 221 def_config='kunit_defconfig', 222 build_dir=request.build_dir, 223 json_path=request.json) | 218 metadata=metadata) |
| 224 if request.json == 'stdout': | 219 if request.json == 'stdout': |
| 225 print(json_obj) | 220 print(json_str) 221 else: 222 with open(request.json, 'w') as f: 223 f.write(json_str) 224 kunit_parser.print_with_timestamp("Test results stored in %s" % 225 os.path.abspath(request.json)) |
| 226 227 if test_result.status != kunit_parser.TestStatus.SUCCESS: 228 return KunitResult(KunitStatus.TEST_FAILURE, parse_end - parse_start), test_result 229 230 return KunitResult(KunitStatus.SUCCESS, parse_end - parse_start), test_result 231 232def run_tests(linux: kunit_kernel.LinuxSourceTree, 233 request: KunitRequest) -> KunitResult: --- 42 unchanged lines hidden (view full) --- 276 277def get_default_jobs() -> int: 278 return len(os.sched_getaffinity(0)) 279 280def add_common_opts(parser) -> None: 281 parser.add_argument('--build_dir', 282 help='As in the make command, it specifies the build ' 283 'directory.', | 226 227 if test_result.status != kunit_parser.TestStatus.SUCCESS: 228 return KunitResult(KunitStatus.TEST_FAILURE, parse_end - parse_start), test_result 229 230 return KunitResult(KunitStatus.SUCCESS, parse_end - parse_start), test_result 231 232def run_tests(linux: kunit_kernel.LinuxSourceTree, 233 request: KunitRequest) -> KunitResult: --- 42 unchanged lines hidden (view full) --- 276 277def get_default_jobs() -> int: 278 return len(os.sched_getaffinity(0)) 279 280def add_common_opts(parser) -> None: 281 parser.add_argument('--build_dir', 282 help='As in the make command, it specifies the build ' 283 'directory.', |
| 284 type=str, default='.kunit', metavar='build_dir') | 284 type=str, default='.kunit', metavar='DIR') |
| 285 parser.add_argument('--make_options', 286 help='X=Y make option, can be repeated.', | 285 parser.add_argument('--make_options', 286 help='X=Y make option, can be repeated.', |
| 287 action='append') | 287 action='append', metavar='X=Y') |
| 288 parser.add_argument('--alltests', 289 help='Run all KUnit tests through allyesconfig', 290 action='store_true') 291 parser.add_argument('--kunitconfig', 292 help='Path to Kconfig fragment that enables KUnit tests.' 293 ' If given a directory, (e.g. lib/kunit), "/.kunitconfig" ' 294 'will get automatically appended.', | 288 parser.add_argument('--alltests', 289 help='Run all KUnit tests through allyesconfig', 290 action='store_true') 291 parser.add_argument('--kunitconfig', 292 help='Path to Kconfig fragment that enables KUnit tests.' 293 ' If given a directory, (e.g. lib/kunit), "/.kunitconfig" ' 294 'will get automatically appended.', |
| 295 metavar='kunitconfig') | 295 metavar='PATH') |
| 296 parser.add_argument('--kconfig_add', 297 help='Additional Kconfig options to append to the ' 298 '.kunitconfig, e.g. CONFIG_KASAN=y. Can be repeated.', | 296 parser.add_argument('--kconfig_add', 297 help='Additional Kconfig options to append to the ' 298 '.kunitconfig, e.g. CONFIG_KASAN=y. Can be repeated.', |
| 299 action='append') | 299 action='append', metavar='CONFIG_X=Y') |
| 300 301 parser.add_argument('--arch', 302 help=('Specifies the architecture to run tests under. ' 303 'The architecture specified here must match the ' 304 'string passed to the ARCH make param, ' 305 'e.g. i386, x86_64, arm, um, etc. Non-UML ' 306 'architectures run on QEMU.'), | 300 301 parser.add_argument('--arch', 302 help=('Specifies the architecture to run tests under. ' 303 'The architecture specified here must match the ' 304 'string passed to the ARCH make param, ' 305 'e.g. i386, x86_64, arm, um, etc. Non-UML ' 306 'architectures run on QEMU.'), |
| 307 type=str, default='um', metavar='arch') | 307 type=str, default='um', metavar='ARCH') |
| 308 309 parser.add_argument('--cross_compile', 310 help=('Sets make\'s CROSS_COMPILE variable; it should ' 311 'be set to a toolchain path prefix (the prefix ' 312 'of gcc and other tools in your toolchain, for ' 313 'example `sparc64-linux-gnu-` if you have the ' 314 'sparc toolchain installed on your system, or ' 315 '`$HOME/toolchains/microblaze/gcc-9.2.0-nolibc/microblaze-linux/bin/microblaze-linux-` ' 316 'if you have downloaded the microblaze toolchain ' 317 'from the 0-day website to a directory in your ' 318 'home directory called `toolchains`).'), | 308 309 parser.add_argument('--cross_compile', 310 help=('Sets make\'s CROSS_COMPILE variable; it should ' 311 'be set to a toolchain path prefix (the prefix ' 312 'of gcc and other tools in your toolchain, for ' 313 'example `sparc64-linux-gnu-` if you have the ' 314 'sparc toolchain installed on your system, or ' 315 '`$HOME/toolchains/microblaze/gcc-9.2.0-nolibc/microblaze-linux/bin/microblaze-linux-` ' 316 'if you have downloaded the microblaze toolchain ' 317 'from the 0-day website to a directory in your ' 318 'home directory called `toolchains`).'), |
| 319 metavar='cross_compile') | 319 metavar='PREFIX') |
| 320 321 parser.add_argument('--qemu_config', 322 help=('Takes a path to a path to a file containing ' 323 'a QemuArchParams object.'), | 320 321 parser.add_argument('--qemu_config', 322 help=('Takes a path to a path to a file containing ' 323 'a QemuArchParams object.'), |
| 324 type=str, metavar='qemu_config') | 324 type=str, metavar='FILE') |
| 325 326def add_build_opts(parser) -> None: 327 parser.add_argument('--jobs', 328 help='As in the make command, "Specifies the number of ' 329 'jobs (commands) to run simultaneously."', | 325 326def add_build_opts(parser) -> None: 327 parser.add_argument('--jobs', 328 help='As in the make command, "Specifies the number of ' 329 'jobs (commands) to run simultaneously."', |
| 330 type=int, default=get_default_jobs(), metavar='jobs') | 330 type=int, default=get_default_jobs(), metavar='N') |
| 331 332def add_exec_opts(parser) -> None: 333 parser.add_argument('--timeout', 334 help='maximum number of seconds to allow for all tests ' 335 'to run. This does not include time taken to build the ' 336 'tests.', 337 type=int, 338 default=300, | 331 332def add_exec_opts(parser) -> None: 333 parser.add_argument('--timeout', 334 help='maximum number of seconds to allow for all tests ' 335 'to run. This does not include time taken to build the ' 336 'tests.', 337 type=int, 338 default=300, |
| 339 metavar='timeout') | 339 metavar='SECONDS') |
| 340 parser.add_argument('filter_glob', 341 help='Filter which KUnit test suites/tests run at ' 342 'boot-time, e.g. list* or list*.*del_test', 343 type=str, 344 nargs='?', 345 default='', 346 metavar='filter_glob') 347 parser.add_argument('--kernel_args', 348 help='Kernel command-line parameters. Maybe be repeated', | 340 parser.add_argument('filter_glob', 341 help='Filter which KUnit test suites/tests run at ' 342 'boot-time, e.g. list* or list*.*del_test', 343 type=str, 344 nargs='?', 345 default='', 346 metavar='filter_glob') 347 parser.add_argument('--kernel_args', 348 help='Kernel command-line parameters. Maybe be repeated', |
| 349 action='append') | 349 action='append', metavar='') |
| 350 parser.add_argument('--run_isolated', help='If set, boot the kernel for each ' 351 'individual suite/test. This is can be useful for debugging ' 352 'a non-hermetic test, one that might pass/fail based on ' 353 'what ran before it.', 354 type=str, | 350 parser.add_argument('--run_isolated', help='If set, boot the kernel for each ' 351 'individual suite/test. This is can be useful for debugging ' 352 'a non-hermetic test, one that might pass/fail based on ' 353 'what ran before it.', 354 type=str, |
| 355 choices=['suite', 'test']), | 355 choices=['suite', 'test']) |
| 356 357def add_parse_opts(parser) -> None: 358 parser.add_argument('--raw_output', help='If set don\'t format output from kernel. ' 359 'If set to --raw_output=kunit, filters to just KUnit output.', | 356 357def add_parse_opts(parser) -> None: 358 parser.add_argument('--raw_output', help='If set don\'t format output from kernel. ' 359 'If set to --raw_output=kunit, filters to just KUnit output.', |
| 360 type=str, nargs='?', const='all', default=None) | 360 type=str, nargs='?', const='all', default=None, choices=['all', 'kunit']) |
| 361 parser.add_argument('--json', 362 nargs='?', 363 help='Stores test results in a JSON, and either ' 364 'prints to stdout or saves to file if a ' 365 'filename is specified', | 361 parser.add_argument('--json', 362 nargs='?', 363 help='Stores test results in a JSON, and either ' 364 'prints to stdout or saves to file if a ' 365 'filename is specified', |
| 366 type=str, const='stdout', default=None) | 366 type=str, const='stdout', default=None, metavar='FILE') |
| 367 368def main(argv, linux=None): 369 parser = argparse.ArgumentParser( 370 description='Helps writing and running KUnit tests.') 371 subparser = parser.add_subparsers(dest='subcommand') 372 373 # The 'run' command will config, build, exec, and parse in one go. 374 run_parser = subparser.add_parser('run', help='Runs KUnit tests.') --- 116 unchanged lines hidden (view full) --- 491 kernel_args=cli_args.kernel_args, 492 run_isolated=cli_args.run_isolated) 493 result = exec_tests(linux, exec_request) 494 kunit_parser.print_with_timestamp(( 495 'Elapsed time: %.3fs\n') % (result.elapsed_time)) 496 if result.status != KunitStatus.SUCCESS: 497 sys.exit(1) 498 elif cli_args.subcommand == 'parse': | 367 368def main(argv, linux=None): 369 parser = argparse.ArgumentParser( 370 description='Helps writing and running KUnit tests.') 371 subparser = parser.add_subparsers(dest='subcommand') 372 373 # The 'run' command will config, build, exec, and parse in one go. 374 run_parser = subparser.add_parser('run', help='Runs KUnit tests.') --- 116 unchanged lines hidden (view full) --- 491 kernel_args=cli_args.kernel_args, 492 run_isolated=cli_args.run_isolated) 493 result = exec_tests(linux, exec_request) 494 kunit_parser.print_with_timestamp(( 495 'Elapsed time: %.3fs\n') % (result.elapsed_time)) 496 if result.status != KunitStatus.SUCCESS: 497 sys.exit(1) 498 elif cli_args.subcommand == 'parse': |
| 499 if cli_args.file == None: | 499 if cli_args.file is None: |
| 500 sys.stdin.reconfigure(errors='backslashreplace') # pytype: disable=attribute-error 501 kunit_output = sys.stdin 502 else: 503 with open(cli_args.file, 'r', errors='backslashreplace') as f: 504 kunit_output = f.read().splitlines() | 500 sys.stdin.reconfigure(errors='backslashreplace') # pytype: disable=attribute-error 501 kunit_output = sys.stdin 502 else: 503 with open(cli_args.file, 'r', errors='backslashreplace') as f: 504 kunit_output = f.read().splitlines() |
| 505 # We know nothing about how the result was created! 506 metadata = kunit_json.Metadata() |
|
| 505 request = KunitParseRequest(raw_output=cli_args.raw_output, | 507 request = KunitParseRequest(raw_output=cli_args.raw_output, |
| 506 build_dir='', | |
| 507 json=cli_args.json) | 508 json=cli_args.json) |
| 508 result, _ = parse_tests(request, kunit_output) | 509 result, _ = parse_tests(request, metadata, kunit_output) |
| 509 if result.status != KunitStatus.SUCCESS: 510 sys.exit(1) 511 else: 512 parser.print_help() 513 514if __name__ == '__main__': 515 main(sys.argv[1:]) | 510 if result.status != KunitStatus.SUCCESS: 511 sys.exit(1) 512 else: 513 parser.print_help() 514 515if __name__ == '__main__': 516 main(sys.argv[1:]) |