| kunit.py (60dd57c7479418e2bc902143eb46a2fdcfeecbbb) | kunit.py (2ab5d5e67f7ab2d2ecf67b8855ac65691f4e4b4d) |
|---|---|
| 1#!/usr/bin/env python3 2# SPDX-License-Identifier: GPL-2.0 3# 4# A thin wrapper on top of the KUnit Kernel 5# 6# Copyright (C) 2019, Google LLC. 7# Author: Felix Guo <felixguoxiuping@gmail.com> 8# Author: Brendan Higgins <brendanhiggins@google.com> 9 10import argparse | 1#!/usr/bin/env python3 2# SPDX-License-Identifier: GPL-2.0 3# 4# A thin wrapper on top of the KUnit Kernel 5# 6# Copyright (C) 2019, Google LLC. 7# Author: Felix Guo <felixguoxiuping@gmail.com> 8# Author: Brendan Higgins <brendanhiggins@google.com> 9 10import argparse |
| 11import sys | |
| 12import os | 11import os |
| 12import re 13import sys |
|
| 13import time 14 15assert sys.version_info >= (3, 7), "Python version is too old" 16 17from collections import namedtuple 18from enum import Enum, auto | 14import time 15 16assert sys.version_info >= (3, 7), "Python version is too old" 17 18from collections import namedtuple 19from enum import Enum, auto |
| 19from typing import Iterable, Sequence | 20from typing import Iterable, Sequence, List |
| 20 | 21 |
| 21import kunit_config | |
| 22import kunit_json 23import kunit_kernel 24import kunit_parser 25 26KunitResult = namedtuple('KunitResult', ['status','result','elapsed_time']) 27 28KunitConfigRequest = namedtuple('KunitConfigRequest', 29 ['build_dir', 'make_options']) 30KunitBuildRequest = namedtuple('KunitBuildRequest', 31 ['jobs', 'build_dir', 'alltests', 32 'make_options']) 33KunitExecRequest = namedtuple('KunitExecRequest', | 22import kunit_json 23import kunit_kernel 24import kunit_parser 25 26KunitResult = namedtuple('KunitResult', ['status','result','elapsed_time']) 27 28KunitConfigRequest = namedtuple('KunitConfigRequest', 29 ['build_dir', 'make_options']) 30KunitBuildRequest = namedtuple('KunitBuildRequest', 31 ['jobs', 'build_dir', 'alltests', 32 'make_options']) 33KunitExecRequest = namedtuple('KunitExecRequest', |
| 34 ['timeout', 'build_dir', 'alltests', 35 'filter_glob', 'kernel_args']) | 34 ['timeout', 'build_dir', 'alltests', 35 'filter_glob', 'kernel_args', 'run_isolated']) |
| 36KunitParseRequest = namedtuple('KunitParseRequest', | 36KunitParseRequest = namedtuple('KunitParseRequest', |
| 37 ['raw_output', 'input_data', 'build_dir', 'json']) | 37 ['raw_output', 'build_dir', 'json']) |
| 38KunitRequest = namedtuple('KunitRequest', ['raw_output','timeout', 'jobs', 39 'build_dir', 'alltests', 'filter_glob', | 38KunitRequest = namedtuple('KunitRequest', ['raw_output','timeout', 'jobs', 39 'build_dir', 'alltests', 'filter_glob', |
| 40 'kernel_args', 'json', 'make_options']) | 40 'kernel_args', 'run_isolated', 'json', 'make_options']) |
| 41 42KernelDirectoryPath = sys.argv[0].split('tools/testing/kunit/')[0] 43 44class KunitStatus(Enum): 45 SUCCESS = auto() 46 CONFIG_FAILURE = auto() 47 BUILD_FAILURE = auto() 48 TEST_FAILURE = auto() --- 37 unchanged lines hidden (view full) --- 86 if not success: 87 return KunitResult(KunitStatus.BUILD_FAILURE, 88 'could not build kernel', 89 build_end - build_start) 90 return KunitResult(KunitStatus.SUCCESS, 91 'built kernel successfully', 92 build_end - build_start) 93 | 41 42KernelDirectoryPath = sys.argv[0].split('tools/testing/kunit/')[0] 43 44class KunitStatus(Enum): 45 SUCCESS = auto() 46 CONFIG_FAILURE = auto() 47 BUILD_FAILURE = auto() 48 TEST_FAILURE = auto() --- 37 unchanged lines hidden (view full) --- 86 if not success: 87 return KunitResult(KunitStatus.BUILD_FAILURE, 88 'could not build kernel', 89 build_end - build_start) 90 return KunitResult(KunitStatus.SUCCESS, 91 'built kernel successfully', 92 build_end - build_start) 93 |
| 94def exec_tests(linux: kunit_kernel.LinuxSourceTree, 95 request: KunitExecRequest) -> KunitResult: 96 kunit_parser.print_with_timestamp('Starting KUnit Kernel ...') 97 test_start = time.time() 98 result = linux.run_kernel( 99 args=request.kernel_args, 100 timeout=None if request.alltests else request.timeout, 101 filter_glob=request.filter_glob, 102 build_dir=request.build_dir) | 94def _list_tests(linux: kunit_kernel.LinuxSourceTree, request: KunitExecRequest) -> List[str]: 95 args = ['kunit.action=list'] 96 if request.kernel_args: 97 args.extend(request.kernel_args) |
| 103 | 98 |
| 104 test_end = time.time() | 99 output = linux.run_kernel(args=args, 100 timeout=None if request.alltests else request.timeout, 101 filter_glob=request.filter_glob, 102 build_dir=request.build_dir) 103 lines = kunit_parser.extract_tap_lines(output) 104 # Hack! Drop the dummy TAP version header that the executor prints out. 105 lines.pop() |
| 105 | 106 |
| 106 return KunitResult(KunitStatus.SUCCESS, 107 result, 108 test_end - test_start) | 107 # Filter out any extraneous non-test output that might have gotten mixed in. 108 return [l for l in lines if re.match('^[^\s.]+\.[^\s.]+$', l)] |
| 109 | 109 |
| 110def parse_tests(request: KunitParseRequest) -> KunitResult: | 110def _suites_from_test_list(tests: List[str]) -> List[str]: 111 """Extracts all the suites from an ordered list of tests.""" 112 suites = [] # type: List[str] 113 for t in tests: 114 parts = t.split('.', maxsplit=2) 115 if len(parts) != 2: 116 raise ValueError(f'internal KUnit error, test name should be of the form "<suite>.<test>", got "{t}"') 117 suite, case = parts 118 if not suites or suites[-1] != suite: 119 suites.append(suite) 120 return suites 121 122 123 124def exec_tests(linux: kunit_kernel.LinuxSourceTree, request: KunitExecRequest, 125 parse_request: KunitParseRequest) -> KunitResult: 126 filter_globs = [request.filter_glob] 127 if request.run_isolated: 128 tests = _list_tests(linux, request) 129 if request.run_isolated == 'test': 130 filter_globs = tests 131 if request.run_isolated == 'suite': 132 filter_globs = _suites_from_test_list(tests) 133 # Apply the test-part of the user's glob, if present. 134 if '.' in request.filter_glob: 135 test_glob = request.filter_glob.split('.', maxsplit=2)[1] 136 filter_globs = [g + '.'+ test_glob for g in filter_globs] 137 138 test_counts = kunit_parser.TestCounts() 139 exec_time = 0.0 140 for i, filter_glob in enumerate(filter_globs): 141 kunit_parser.print_with_timestamp('Starting KUnit Kernel ({}/{})...'.format(i+1, len(filter_globs))) 142 143 test_start = time.time() 144 run_result = linux.run_kernel( 145 args=request.kernel_args, 146 timeout=None if request.alltests else request.timeout, 147 filter_glob=filter_glob, 148 build_dir=request.build_dir) 149 150 result = parse_tests(parse_request, run_result) 151 # run_kernel() doesn't block on the kernel exiting. 152 # That only happens after we get the last line of output from `run_result`. 153 # So exec_time here actually contains parsing + execution time, which is fine. 154 test_end = time.time() 155 exec_time += test_end - test_start 156 157 test_counts.add_subtest_counts(result.result.test.counts) 158 159 kunit_status = _map_to_overall_status(test_counts.get_status()) 160 return KunitResult(status=kunit_status, result=result.result, elapsed_time=exec_time) 161 162def _map_to_overall_status(test_status: kunit_parser.TestStatus) -> KunitStatus: 163 if test_status in (kunit_parser.TestStatus.SUCCESS, kunit_parser.TestStatus.SKIPPED): 164 return KunitStatus.SUCCESS 165 else: 166 return KunitStatus.TEST_FAILURE 167 168def parse_tests(request: KunitParseRequest, input_data: Iterable[str]) -> KunitResult: |
| 111 parse_start = time.time() 112 113 test_result = kunit_parser.TestResult(kunit_parser.TestStatus.SUCCESS, | 169 parse_start = time.time() 170 171 test_result = kunit_parser.TestResult(kunit_parser.TestStatus.SUCCESS, |
| 114 [], | 172 kunit_parser.Test(), |
| 115 'Tests not Parsed.') 116 117 if request.raw_output: | 173 'Tests not Parsed.') 174 175 if request.raw_output: |
| 118 output: Iterable[str] = request.input_data | 176 # Treat unparsed results as one passing test. 177 test_result.test.status = kunit_parser.TestStatus.SUCCESS 178 test_result.test.counts.passed = 1 179 180 output: Iterable[str] = input_data |
| 119 if request.raw_output == 'all': 120 pass 121 elif request.raw_output == 'kunit': 122 output = kunit_parser.extract_tap_lines(output) 123 else: 124 print(f'Unknown --raw_output option "{request.raw_output}"', file=sys.stderr) 125 for line in output: 126 print(line.rstrip()) 127 128 else: | 181 if request.raw_output == 'all': 182 pass 183 elif request.raw_output == 'kunit': 184 output = kunit_parser.extract_tap_lines(output) 185 else: 186 print(f'Unknown --raw_output option "{request.raw_output}"', file=sys.stderr) 187 for line in output: 188 print(line.rstrip()) 189 190 else: |
| 129 test_result = kunit_parser.parse_run_tests(request.input_data) | 191 test_result = kunit_parser.parse_run_tests(input_data) |
| 130 parse_end = time.time() 131 132 if request.json: 133 json_obj = kunit_json.get_json_result( 134 test_result=test_result, 135 def_config='kunit_defconfig', 136 build_dir=request.build_dir, 137 json_path=request.json) --- 21 unchanged lines hidden (view full) --- 159 request.alltests, 160 request.make_options) 161 build_result = build_tests(linux, build_request) 162 if build_result.status != KunitStatus.SUCCESS: 163 return build_result 164 165 exec_request = KunitExecRequest(request.timeout, request.build_dir, 166 request.alltests, request.filter_glob, | 192 parse_end = time.time() 193 194 if request.json: 195 json_obj = kunit_json.get_json_result( 196 test_result=test_result, 197 def_config='kunit_defconfig', 198 build_dir=request.build_dir, 199 json_path=request.json) --- 21 unchanged lines hidden (view full) --- 221 request.alltests, 222 request.make_options) 223 build_result = build_tests(linux, build_request) 224 if build_result.status != KunitStatus.SUCCESS: 225 return build_result 226 227 exec_request = KunitExecRequest(request.timeout, request.build_dir, 228 request.alltests, request.filter_glob, |
| 167 request.kernel_args) 168 exec_result = exec_tests(linux, exec_request) 169 if exec_result.status != KunitStatus.SUCCESS: 170 return exec_result 171 | 229 request.kernel_args, request.run_isolated) |
| 172 parse_request = KunitParseRequest(request.raw_output, | 230 parse_request = KunitParseRequest(request.raw_output, |
| 173 exec_result.result, | |
| 174 request.build_dir, 175 request.json) | 231 request.build_dir, 232 request.json) |
| 176 parse_result = parse_tests(parse_request) | |
| 177 | 233 |
| 234 exec_result = exec_tests(linux, exec_request, parse_request) 235 |
|
| 178 run_end = time.time() 179 180 kunit_parser.print_with_timestamp(( 181 'Elapsed time: %.3fs total, %.3fs configuring, %.3fs ' + 182 'building, %.3fs running\n') % ( 183 run_end - run_start, 184 config_result.elapsed_time, 185 build_result.elapsed_time, 186 exec_result.elapsed_time)) | 236 run_end = time.time() 237 238 kunit_parser.print_with_timestamp(( 239 'Elapsed time: %.3fs total, %.3fs configuring, %.3fs ' + 240 'building, %.3fs running\n') % ( 241 run_end - run_start, 242 config_result.elapsed_time, 243 build_result.elapsed_time, 244 exec_result.elapsed_time)) |
| 187 return parse_result | 245 return exec_result |
| 188 189# Problem: 190# $ kunit.py run --json 191# works as one would expect and prints the parsed test results as JSON. 192# $ kunit.py run --json suite_name 193# would *not* pass suite_name as the filter_glob and print as json. 194# argparse will consider it to be another way of writing 195# $ kunit.py run --json=suite_name --- 62 unchanged lines hidden (view full) --- 258 parser.add_argument('--timeout', 259 help='maximum number of seconds to allow for all tests ' 260 'to run. This does not include time taken to build the ' 261 'tests.', 262 type=int, 263 default=300, 264 metavar='timeout') 265 parser.add_argument('filter_glob', | 246 247# Problem: 248# $ kunit.py run --json 249# works as one would expect and prints the parsed test results as JSON. 250# $ kunit.py run --json suite_name 251# would *not* pass suite_name as the filter_glob and print as json. 252# argparse will consider it to be another way of writing 253# $ kunit.py run --json=suite_name --- 62 unchanged lines hidden (view full) --- 316 parser.add_argument('--timeout', 317 help='maximum number of seconds to allow for all tests ' 318 'to run. This does not include time taken to build the ' 319 'tests.', 320 type=int, 321 default=300, 322 metavar='timeout') 323 parser.add_argument('filter_glob', |
| 266 help='maximum number of seconds to allow for all tests ' 267 'to run. This does not include time taken to build the ' 268 'tests.', | 324 help='Filter which KUnit test suites/tests run at ' 325 'boot-time, e.g. list* or list*.*del_test', |
| 269 type=str, 270 nargs='?', 271 default='', 272 metavar='filter_glob') 273 parser.add_argument('--kernel_args', 274 help='Kernel command-line parameters. Maybe be repeated', 275 action='append') | 326 type=str, 327 nargs='?', 328 default='', 329 metavar='filter_glob') 330 parser.add_argument('--kernel_args', 331 help='Kernel command-line parameters. Maybe be repeated', 332 action='append') |
| 333 parser.add_argument('--run_isolated', help='If set, boot the kernel for each ' 334 'individual suite/test. This is can be useful for debugging ' 335 'a non-hermetic test, one that might pass/fail based on ' 336 'what ran before it.', 337 type=str, 338 choices=['suite', 'test']), |
|
| 276 277def add_parse_opts(parser) -> None: 278 parser.add_argument('--raw_output', help='If set don\'t format output from kernel. ' 279 'If set to --raw_output=kunit, filters to just KUnit output.', 280 type=str, nargs='?', const='all', default=None) 281 parser.add_argument('--json', 282 nargs='?', 283 help='Stores test results in a JSON, and either ' --- 57 unchanged lines hidden (view full) --- 341 342 request = KunitRequest(cli_args.raw_output, 343 cli_args.timeout, 344 cli_args.jobs, 345 cli_args.build_dir, 346 cli_args.alltests, 347 cli_args.filter_glob, 348 cli_args.kernel_args, | 339 340def add_parse_opts(parser) -> None: 341 parser.add_argument('--raw_output', help='If set don\'t format output from kernel. ' 342 'If set to --raw_output=kunit, filters to just KUnit output.', 343 type=str, nargs='?', const='all', default=None) 344 parser.add_argument('--json', 345 nargs='?', 346 help='Stores test results in a JSON, and either ' --- 57 unchanged lines hidden (view full) --- 404 405 request = KunitRequest(cli_args.raw_output, 406 cli_args.timeout, 407 cli_args.jobs, 408 cli_args.build_dir, 409 cli_args.alltests, 410 cli_args.filter_glob, 411 cli_args.kernel_args, |
| 412 cli_args.run_isolated, |
|
| 349 cli_args.json, 350 cli_args.make_options) 351 result = run_tests(linux, request) 352 if result.status != KunitStatus.SUCCESS: 353 sys.exit(1) 354 elif cli_args.subcommand == 'config': 355 if cli_args.build_dir and ( 356 not os.path.exists(cli_args.build_dir)): --- 39 unchanged lines hidden (view full) --- 396 arch=cli_args.arch, 397 cross_compile=cli_args.cross_compile, 398 qemu_config_path=cli_args.qemu_config) 399 400 exec_request = KunitExecRequest(cli_args.timeout, 401 cli_args.build_dir, 402 cli_args.alltests, 403 cli_args.filter_glob, | 413 cli_args.json, 414 cli_args.make_options) 415 result = run_tests(linux, request) 416 if result.status != KunitStatus.SUCCESS: 417 sys.exit(1) 418 elif cli_args.subcommand == 'config': 419 if cli_args.build_dir and ( 420 not os.path.exists(cli_args.build_dir)): --- 39 unchanged lines hidden (view full) --- 460 arch=cli_args.arch, 461 cross_compile=cli_args.cross_compile, 462 qemu_config_path=cli_args.qemu_config) 463 464 exec_request = KunitExecRequest(cli_args.timeout, 465 cli_args.build_dir, 466 cli_args.alltests, 467 cli_args.filter_glob, |
| 404 cli_args.kernel_args) 405 exec_result = exec_tests(linux, exec_request) | 468 cli_args.kernel_args, 469 cli_args.run_isolated) |
| 406 parse_request = KunitParseRequest(cli_args.raw_output, | 470 parse_request = KunitParseRequest(cli_args.raw_output, |
| 407 exec_result.result, | |
| 408 cli_args.build_dir, 409 cli_args.json) | 471 cli_args.build_dir, 472 cli_args.json) |
| 410 result = parse_tests(parse_request) | 473 result = exec_tests(linux, exec_request, parse_request) |
| 411 kunit_parser.print_with_timestamp(( | 474 kunit_parser.print_with_timestamp(( |
| 412 'Elapsed time: %.3fs\n') % ( 413 exec_result.elapsed_time)) | 475 'Elapsed time: %.3fs\n') % (result.elapsed_time)) |
| 414 if result.status != KunitStatus.SUCCESS: 415 sys.exit(1) 416 elif cli_args.subcommand == 'parse': 417 if cli_args.file == None: | 476 if result.status != KunitStatus.SUCCESS: 477 sys.exit(1) 478 elif cli_args.subcommand == 'parse': 479 if cli_args.file == None: |
| 480 sys.stdin.reconfigure(errors='backslashreplace') # pytype: disable=attribute-error |
|
| 418 kunit_output = sys.stdin 419 else: | 481 kunit_output = sys.stdin 482 else: |
| 420 with open(cli_args.file, 'r') as f: | 483 with open(cli_args.file, 'r', errors='backslashreplace') as f: |
| 421 kunit_output = f.read().splitlines() 422 request = KunitParseRequest(cli_args.raw_output, | 484 kunit_output = f.read().splitlines() 485 request = KunitParseRequest(cli_args.raw_output, |
| 423 kunit_output, | |
| 424 None, 425 cli_args.json) | 486 None, 487 cli_args.json) |
| 426 result = parse_tests(request) | 488 result = parse_tests(request, kunit_output) |
| 427 if result.status != KunitStatus.SUCCESS: 428 sys.exit(1) 429 else: 430 parser.print_help() 431 432if __name__ == '__main__': 433 main(sys.argv[1:]) | 489 if result.status != KunitStatus.SUCCESS: 490 sys.exit(1) 491 else: 492 parser.print_help() 493 494if __name__ == '__main__': 495 main(sys.argv[1:]) |