xref: /linux/tools/testing/kunit/kunit.py (revision 126901ba3499880c9ed033633817cf7493120fda)
1c25ce589SFinn Behrens#!/usr/bin/env python3
26ebf5866SFelix Guo# SPDX-License-Identifier: GPL-2.0
36ebf5866SFelix Guo#
46ebf5866SFelix Guo# A thin wrapper on top of the KUnit Kernel
56ebf5866SFelix Guo#
66ebf5866SFelix Guo# Copyright (C) 2019, Google LLC.
76ebf5866SFelix Guo# Author: Felix Guo <felixguoxiuping@gmail.com>
86ebf5866SFelix Guo# Author: Brendan Higgins <brendanhiggins@google.com>
96ebf5866SFelix Guo
106ebf5866SFelix Guoimport argparse
116ebf5866SFelix Guoimport os
12ff9e09a3SDaniel Latypovimport re
13a9333bd3SDaniel Latypovimport shlex
14ff9e09a3SDaniel Latypovimport sys
156ebf5866SFelix Guoimport time
166ebf5866SFelix Guo
17df4b0807SSeongJae Parkassert sys.version_info >= (3, 7), "Python version is too old"
18df4b0807SSeongJae Park
19db167981SDaniel Latypovfrom dataclasses import dataclass
206ebf5866SFelix Guofrom enum import Enum, auto
2195dcbc55SDaniel Latypovfrom typing import Iterable, List, Optional, Sequence, Tuple
226ebf5866SFelix Guo
2321a6d178SHeidi Fahimimport kunit_json
246ebf5866SFelix Guoimport kunit_kernel
256ebf5866SFelix Guoimport kunit_parser
26e756dbebSDaniel Latypovfrom kunit_printer import stdout
276ebf5866SFelix Guo
286ebf5866SFelix Guoclass KunitStatus(Enum):
296ebf5866SFelix Guo	SUCCESS = auto()
306ebf5866SFelix Guo	CONFIG_FAILURE = auto()
316ebf5866SFelix Guo	BUILD_FAILURE = auto()
326ebf5866SFelix Guo	TEST_FAILURE = auto()
336ebf5866SFelix Guo
34db167981SDaniel Latypov@dataclass
35db167981SDaniel Latypovclass KunitResult:
36db167981SDaniel Latypov	status: KunitStatus
37db167981SDaniel Latypov	elapsed_time: float
38db167981SDaniel Latypov
39db167981SDaniel Latypov@dataclass
40db167981SDaniel Latypovclass KunitConfigRequest:
41db167981SDaniel Latypov	build_dir: str
42db167981SDaniel Latypov	make_options: Optional[List[str]]
43db167981SDaniel Latypov
44db167981SDaniel Latypov@dataclass
45db167981SDaniel Latypovclass KunitBuildRequest(KunitConfigRequest):
46db167981SDaniel Latypov	jobs: int
47db167981SDaniel Latypov
48db167981SDaniel Latypov@dataclass
49db167981SDaniel Latypovclass KunitParseRequest:
50db167981SDaniel Latypov	raw_output: Optional[str]
51db167981SDaniel Latypov	json: Optional[str]
52db167981SDaniel Latypov
53db167981SDaniel Latypov@dataclass
54db167981SDaniel Latypovclass KunitExecRequest(KunitParseRequest):
55ee96d25fSDaniel Latypov	build_dir: str
56db167981SDaniel Latypov	timeout: int
57db167981SDaniel Latypov	filter_glob: str
58db167981SDaniel Latypov	kernel_args: Optional[List[str]]
59db167981SDaniel Latypov	run_isolated: Optional[str]
60db167981SDaniel Latypov
61db167981SDaniel Latypov@dataclass
62db167981SDaniel Latypovclass KunitRequest(KunitExecRequest, KunitBuildRequest):
63db167981SDaniel Latypov	pass
64db167981SDaniel Latypov
65db167981SDaniel Latypov
6609641f7cSDaniel Latypovdef get_kernel_root_path() -> str:
6709641f7cSDaniel Latypov	path = sys.argv[0] if not __file__ else __file__
6809641f7cSDaniel Latypov	parts = os.path.realpath(path).split('tools/testing/kunit')
69be886ba9SHeidi Fahim	if len(parts) != 2:
70be886ba9SHeidi Fahim		sys.exit(1)
71be886ba9SHeidi Fahim	return parts[0]
72be886ba9SHeidi Fahim
7345ba7a89SDavid Gowdef config_tests(linux: kunit_kernel.LinuxSourceTree,
7445ba7a89SDavid Gow		 request: KunitConfigRequest) -> KunitResult:
75e756dbebSDaniel Latypov	stdout.print_with_timestamp('Configuring KUnit Kernel ...')
7645ba7a89SDavid Gow
776ebf5866SFelix Guo	config_start = time.time()
780476e69fSGreg Thelen	success = linux.build_reconfig(request.build_dir, request.make_options)
796ebf5866SFelix Guo	config_end = time.time()
801fdc6f4fSAlexander Pantyukhin	status = KunitStatus.SUCCESS if success else KunitStatus.CONFIG_FAILURE
811fdc6f4fSAlexander Pantyukhin	return KunitResult(status, config_end - config_start)
826ebf5866SFelix Guo
8345ba7a89SDavid Gowdef build_tests(linux: kunit_kernel.LinuxSourceTree,
8445ba7a89SDavid Gow		request: KunitBuildRequest) -> KunitResult:
85e756dbebSDaniel Latypov	stdout.print_with_timestamp('Building KUnit Kernel ...')
866ebf5866SFelix Guo
876ebf5866SFelix Guo	build_start = time.time()
88980ac3adSDaniel Latypov	success = linux.build_kernel(request.jobs,
890476e69fSGreg Thelen				     request.build_dir,
900476e69fSGreg Thelen				     request.make_options)
916ebf5866SFelix Guo	build_end = time.time()
921fdc6f4fSAlexander Pantyukhin	status = KunitStatus.SUCCESS if success else KunitStatus.BUILD_FAILURE
931fdc6f4fSAlexander Pantyukhin	return KunitResult(status, build_end - build_start)
946ebf5866SFelix Guo
951ee2ba89SDaniel Latypovdef config_and_build_tests(linux: kunit_kernel.LinuxSourceTree,
961ee2ba89SDaniel Latypov			   request: KunitBuildRequest) -> KunitResult:
971ee2ba89SDaniel Latypov	config_result = config_tests(linux, request)
981ee2ba89SDaniel Latypov	if config_result.status != KunitStatus.SUCCESS:
991ee2ba89SDaniel Latypov		return config_result
1001ee2ba89SDaniel Latypov
1011ee2ba89SDaniel Latypov	return build_tests(linux, request)
1021ee2ba89SDaniel Latypov
103ff9e09a3SDaniel Latypovdef _list_tests(linux: kunit_kernel.LinuxSourceTree, request: KunitExecRequest) -> List[str]:
104ff9e09a3SDaniel Latypov	args = ['kunit.action=list']
105ff9e09a3SDaniel Latypov	if request.kernel_args:
106ff9e09a3SDaniel Latypov		args.extend(request.kernel_args)
107ff9e09a3SDaniel Latypov
108ff9e09a3SDaniel Latypov	output = linux.run_kernel(args=args,
109980ac3adSDaniel Latypov			   timeout=request.timeout,
110ff9e09a3SDaniel Latypov			   filter_glob=request.filter_glob,
111ff9e09a3SDaniel Latypov			   build_dir=request.build_dir)
112ff9e09a3SDaniel Latypov	lines = kunit_parser.extract_tap_lines(output)
113ff9e09a3SDaniel Latypov	# Hack! Drop the dummy TAP version header that the executor prints out.
114ff9e09a3SDaniel Latypov	lines.pop()
115ff9e09a3SDaniel Latypov
116ff9e09a3SDaniel Latypov	# Filter out any extraneous non-test output that might have gotten mixed in.
1170453f984SDaniel Latypov	return [l for l in lines if re.match(r'^[^\s.]+\.[^\s.]+$', l)]
118ff9e09a3SDaniel Latypov
119ff9e09a3SDaniel Latypovdef _suites_from_test_list(tests: List[str]) -> List[str]:
120ff9e09a3SDaniel Latypov	"""Extracts all the suites from an ordered list of tests."""
121ff9e09a3SDaniel Latypov	suites = []  # type: List[str]
122ff9e09a3SDaniel Latypov	for t in tests:
123ff9e09a3SDaniel Latypov		parts = t.split('.', maxsplit=2)
124ff9e09a3SDaniel Latypov		if len(parts) != 2:
125ff9e09a3SDaniel Latypov			raise ValueError(f'internal KUnit error, test name should be of the form "<suite>.<test>", got "{t}"')
126*126901baSDaniel Latypov		suite, _ = parts
127ff9e09a3SDaniel Latypov		if not suites or suites[-1] != suite:
128ff9e09a3SDaniel Latypov			suites.append(suite)
129ff9e09a3SDaniel Latypov	return suites
130ff9e09a3SDaniel Latypov
131ff9e09a3SDaniel Latypov
132ff9e09a3SDaniel Latypov
133db167981SDaniel Latypovdef exec_tests(linux: kunit_kernel.LinuxSourceTree, request: KunitExecRequest) -> KunitResult:
134ff9e09a3SDaniel Latypov	filter_globs = [request.filter_glob]
135ff9e09a3SDaniel Latypov	if request.run_isolated:
136ff9e09a3SDaniel Latypov		tests = _list_tests(linux, request)
137ff9e09a3SDaniel Latypov		if request.run_isolated == 'test':
138ff9e09a3SDaniel Latypov			filter_globs = tests
1391fdc6f4fSAlexander Pantyukhin		elif request.run_isolated == 'suite':
140ff9e09a3SDaniel Latypov			filter_globs = _suites_from_test_list(tests)
141ff9e09a3SDaniel Latypov			# Apply the test-part of the user's glob, if present.
142ff9e09a3SDaniel Latypov			if '.' in request.filter_glob:
143ff9e09a3SDaniel Latypov				test_glob = request.filter_glob.split('.', maxsplit=2)[1]
144ff9e09a3SDaniel Latypov				filter_globs = [g + '.'+ test_glob for g in filter_globs]
145ff9e09a3SDaniel Latypov
146885210d3SDaniel Latypov	metadata = kunit_json.Metadata(arch=linux.arch(), build_dir=request.build_dir, def_config='kunit_defconfig')
147ee96d25fSDaniel Latypov
148d65d07cbSRae Moar	test_counts = kunit_parser.TestCounts()
149ff9e09a3SDaniel Latypov	exec_time = 0.0
150ff9e09a3SDaniel Latypov	for i, filter_glob in enumerate(filter_globs):
151e756dbebSDaniel Latypov		stdout.print_with_timestamp('Starting KUnit Kernel ({}/{})...'.format(i+1, len(filter_globs)))
152ff9e09a3SDaniel Latypov
1536ebf5866SFelix Guo		test_start = time.time()
1547ef925eaSDaniel Latypov		run_result = linux.run_kernel(
1556cb51a18SDaniel Latypov			args=request.kernel_args,
156980ac3adSDaniel Latypov			timeout=request.timeout,
157ff9e09a3SDaniel Latypov			filter_glob=filter_glob,
1586ec1b81dSSeongJae Park			build_dir=request.build_dir)
15945ba7a89SDavid Gow
160ee96d25fSDaniel Latypov		_, test_result = parse_tests(request, metadata, run_result)
1615f6aa6d8SDaniel Latypov		# run_kernel() doesn't block on the kernel exiting.
1625f6aa6d8SDaniel Latypov		# That only happens after we get the last line of output from `run_result`.
1635f6aa6d8SDaniel Latypov		# So exec_time here actually contains parsing + execution time, which is fine.
1646ebf5866SFelix Guo		test_end = time.time()
165ff9e09a3SDaniel Latypov		exec_time += test_end - test_start
166ff9e09a3SDaniel Latypov
16795dcbc55SDaniel Latypov		test_counts.add_subtest_counts(test_result.counts)
1686ebf5866SFelix Guo
1697fa7ffcfSDaniel Latypov	if len(filter_globs) == 1 and test_counts.crashed > 0:
1707fa7ffcfSDaniel Latypov		bd = request.build_dir
1717fa7ffcfSDaniel Latypov		print('The kernel seems to have crashed; you can decode the stack traces with:')
1727fa7ffcfSDaniel Latypov		print('$ scripts/decode_stacktrace.sh {}/vmlinux {} < {} | tee {}/decoded.log | {} parse'.format(
1737fa7ffcfSDaniel Latypov				bd, bd, kunit_kernel.get_outfile_path(bd), bd, sys.argv[0]))
1747fa7ffcfSDaniel Latypov
175d65d07cbSRae Moar	kunit_status = _map_to_overall_status(test_counts.get_status())
17695dcbc55SDaniel Latypov	return KunitResult(status=kunit_status, elapsed_time=exec_time)
177d65d07cbSRae Moar
178d65d07cbSRae Moardef _map_to_overall_status(test_status: kunit_parser.TestStatus) -> KunitStatus:
179d65d07cbSRae Moar	if test_status in (kunit_parser.TestStatus.SUCCESS, kunit_parser.TestStatus.SKIPPED):
180d65d07cbSRae Moar		return KunitStatus.SUCCESS
181d65d07cbSRae Moar	return KunitStatus.TEST_FAILURE
1827ef925eaSDaniel Latypov
183ee96d25fSDaniel Latypovdef parse_tests(request: KunitParseRequest, metadata: kunit_json.Metadata, input_data: Iterable[str]) -> Tuple[KunitResult, kunit_parser.Test]:
18445ba7a89SDavid Gow	parse_start = time.time()
18545ba7a89SDavid Gow
18645ba7a89SDavid Gow	if request.raw_output:
187d65d07cbSRae Moar		# Treat unparsed results as one passing test.
188309e22efSDaniel Latypov		fake_test = kunit_parser.Test()
189309e22efSDaniel Latypov		fake_test.status = kunit_parser.TestStatus.SUCCESS
190309e22efSDaniel Latypov		fake_test.counts.passed = 1
191d65d07cbSRae Moar
1927ef925eaSDaniel Latypov		output: Iterable[str] = input_data
1936a499c9cSDaniel Latypov		if request.raw_output == 'all':
1946a499c9cSDaniel Latypov			pass
1956a499c9cSDaniel Latypov		elif request.raw_output == 'kunit':
196c2bb92bcSDaniel Latypov			output = kunit_parser.extract_tap_lines(output)
1976a499c9cSDaniel Latypov		for line in output:
1986a499c9cSDaniel Latypov			print(line.rstrip())
199309e22efSDaniel Latypov		parse_time = time.time() - parse_start
200309e22efSDaniel Latypov		return KunitResult(KunitStatus.SUCCESS, parse_time), fake_test
2016a499c9cSDaniel Latypov
202309e22efSDaniel Latypov
203309e22efSDaniel Latypov	# Actually parse the test results.
204309e22efSDaniel Latypov	test = kunit_parser.parse_run_tests(input_data)
205309e22efSDaniel Latypov	parse_time = time.time() - parse_start
20645ba7a89SDavid Gow
20721a6d178SHeidi Fahim	if request.json:
20800f75043SDaniel Latypov		json_str = kunit_json.get_json_result(
209309e22efSDaniel Latypov					test=test,
210ee96d25fSDaniel Latypov					metadata=metadata)
21121a6d178SHeidi Fahim		if request.json == 'stdout':
21200f75043SDaniel Latypov			print(json_str)
21300f75043SDaniel Latypov		else:
21400f75043SDaniel Latypov			with open(request.json, 'w') as f:
21500f75043SDaniel Latypov				f.write(json_str)
216e756dbebSDaniel Latypov			stdout.print_with_timestamp("Test results stored in %s" %
21700f75043SDaniel Latypov				os.path.abspath(request.json))
21821a6d178SHeidi Fahim
219309e22efSDaniel Latypov	if test.status != kunit_parser.TestStatus.SUCCESS:
220309e22efSDaniel Latypov		return KunitResult(KunitStatus.TEST_FAILURE, parse_time), test
22145ba7a89SDavid Gow
222309e22efSDaniel Latypov	return KunitResult(KunitStatus.SUCCESS, parse_time), test
22345ba7a89SDavid Gow
22445ba7a89SDavid Gowdef run_tests(linux: kunit_kernel.LinuxSourceTree,
22545ba7a89SDavid Gow	      request: KunitRequest) -> KunitResult:
22645ba7a89SDavid Gow	run_start = time.time()
22745ba7a89SDavid Gow
228db167981SDaniel Latypov	config_result = config_tests(linux, request)
22945ba7a89SDavid Gow	if config_result.status != KunitStatus.SUCCESS:
23045ba7a89SDavid Gow		return config_result
23145ba7a89SDavid Gow
232db167981SDaniel Latypov	build_result = build_tests(linux, request)
23345ba7a89SDavid Gow	if build_result.status != KunitStatus.SUCCESS:
23445ba7a89SDavid Gow		return build_result
23545ba7a89SDavid Gow
236db167981SDaniel Latypov	exec_result = exec_tests(linux, request)
23745ba7a89SDavid Gow
23845ba7a89SDavid Gow	run_end = time.time()
23945ba7a89SDavid Gow
240e756dbebSDaniel Latypov	stdout.print_with_timestamp((
2416ebf5866SFelix Guo		'Elapsed time: %.3fs total, %.3fs configuring, %.3fs ' +
2426ebf5866SFelix Guo		'building, %.3fs running\n') % (
24345ba7a89SDavid Gow				run_end - run_start,
24445ba7a89SDavid Gow				config_result.elapsed_time,
24545ba7a89SDavid Gow				build_result.elapsed_time,
24645ba7a89SDavid Gow				exec_result.elapsed_time))
2477ef925eaSDaniel Latypov	return exec_result
2486ebf5866SFelix Guo
249d8c23eadSDaniel Latypov# Problem:
250d8c23eadSDaniel Latypov# $ kunit.py run --json
251d8c23eadSDaniel Latypov# works as one would expect and prints the parsed test results as JSON.
252d8c23eadSDaniel Latypov# $ kunit.py run --json suite_name
253d8c23eadSDaniel Latypov# would *not* pass suite_name as the filter_glob and print as json.
254d8c23eadSDaniel Latypov# argparse will consider it to be another way of writing
255d8c23eadSDaniel Latypov# $ kunit.py run --json=suite_name
256d8c23eadSDaniel Latypov# i.e. it would run all tests, and dump the json to a `suite_name` file.
257d8c23eadSDaniel Latypov# So we hackily automatically rewrite --json => --json=stdout
258d8c23eadSDaniel Latypovpseudo_bool_flag_defaults = {
259d8c23eadSDaniel Latypov		'--json': 'stdout',
260d8c23eadSDaniel Latypov		'--raw_output': 'kunit',
261d8c23eadSDaniel Latypov}
262d8c23eadSDaniel Latypovdef massage_argv(argv: Sequence[str]) -> Sequence[str]:
263d8c23eadSDaniel Latypov	def massage_arg(arg: str) -> str:
264d8c23eadSDaniel Latypov		if arg not in pseudo_bool_flag_defaults:
265d8c23eadSDaniel Latypov			return arg
266d8c23eadSDaniel Latypov		return  f'{arg}={pseudo_bool_flag_defaults[arg]}'
267d8c23eadSDaniel Latypov	return list(map(massage_arg, argv))
268d8c23eadSDaniel Latypov
269ad659ccbSDavid Gowdef get_default_jobs() -> int:
270ad659ccbSDavid Gow	return len(os.sched_getaffinity(0))
271ad659ccbSDavid Gow
27209641f7cSDaniel Latypovdef add_common_opts(parser) -> None:
27345ba7a89SDavid Gow	parser.add_argument('--build_dir',
27445ba7a89SDavid Gow			    help='As in the make command, it specifies the build '
27545ba7a89SDavid Gow			    'directory.',
276baa33315SDaniel Latypov			    type=str, default='.kunit', metavar='DIR')
27745ba7a89SDavid Gow	parser.add_argument('--make_options',
27845ba7a89SDavid Gow			    help='X=Y make option, can be repeated.',
279baa33315SDaniel Latypov			    action='append', metavar='X=Y')
28045ba7a89SDavid Gow	parser.add_argument('--alltests',
281980ac3adSDaniel Latypov			    help='Run all KUnit tests via tools/testing/kunit/configs/all_tests.config',
2826ebf5866SFelix Guo			    action='store_true')
283243180f5SDaniel Latypov	parser.add_argument('--kunitconfig',
2849854781dSDaniel Latypov			     help='Path to Kconfig fragment that enables KUnit tests.'
2859854781dSDaniel Latypov			     ' If given a directory, (e.g. lib/kunit), "/.kunitconfig" '
28653b46621SDaniel Latypov			     'will get  automatically appended. If repeated, the files '
28753b46621SDaniel Latypov			     'blindly concatenated, which might not work in all cases.',
28853b46621SDaniel Latypov			     action='append', metavar='PATHS')
2899f57cc76SDaniel Latypov	parser.add_argument('--kconfig_add',
2909f57cc76SDaniel Latypov			     help='Additional Kconfig options to append to the '
2919f57cc76SDaniel Latypov			     '.kunitconfig, e.g. CONFIG_KASAN=y. Can be repeated.',
292baa33315SDaniel Latypov			    action='append', metavar='CONFIG_X=Y')
2936ebf5866SFelix Guo
29487c9c163SBrendan Higgins	parser.add_argument('--arch',
29587c9c163SBrendan Higgins			    help=('Specifies the architecture to run tests under. '
29687c9c163SBrendan Higgins				  'The architecture specified here must match the '
29787c9c163SBrendan Higgins				  'string passed to the ARCH make param, '
29887c9c163SBrendan Higgins				  'e.g. i386, x86_64, arm, um, etc. Non-UML '
29987c9c163SBrendan Higgins				  'architectures run on QEMU.'),
300baa33315SDaniel Latypov			    type=str, default='um', metavar='ARCH')
30187c9c163SBrendan Higgins
30287c9c163SBrendan Higgins	parser.add_argument('--cross_compile',
30387c9c163SBrendan Higgins			    help=('Sets make\'s CROSS_COMPILE variable; it should '
30487c9c163SBrendan Higgins				  'be set to a toolchain path prefix (the prefix '
30587c9c163SBrendan Higgins				  'of gcc and other tools in your toolchain, for '
30687c9c163SBrendan Higgins				  'example `sparc64-linux-gnu-` if you have the '
30787c9c163SBrendan Higgins				  'sparc toolchain installed on your system, or '
30887c9c163SBrendan Higgins				  '`$HOME/toolchains/microblaze/gcc-9.2.0-nolibc/microblaze-linux/bin/microblaze-linux-` '
30987c9c163SBrendan Higgins				  'if you have downloaded the microblaze toolchain '
31087c9c163SBrendan Higgins				  'from the 0-day website to a directory in your '
31187c9c163SBrendan Higgins				  'home directory called `toolchains`).'),
312baa33315SDaniel Latypov			    metavar='PREFIX')
31387c9c163SBrendan Higgins
31487c9c163SBrendan Higgins	parser.add_argument('--qemu_config',
31587c9c163SBrendan Higgins			    help=('Takes a path to a path to a file containing '
31687c9c163SBrendan Higgins				  'a QemuArchParams object.'),
317baa33315SDaniel Latypov			    type=str, metavar='FILE')
31887c9c163SBrendan Higgins
319a9333bd3SDaniel Latypov	parser.add_argument('--qemu_args',
320a9333bd3SDaniel Latypov			    help='Additional QEMU arguments, e.g. "-smp 8"',
321a9333bd3SDaniel Latypov			    action='append', metavar='')
322a9333bd3SDaniel Latypov
32309641f7cSDaniel Latypovdef add_build_opts(parser) -> None:
32445ba7a89SDavid Gow	parser.add_argument('--jobs',
32545ba7a89SDavid Gow			    help='As in the make command, "Specifies  the number of '
32645ba7a89SDavid Gow			    'jobs (commands) to run simultaneously."',
327baa33315SDaniel Latypov			    type=int, default=get_default_jobs(), metavar='N')
32845ba7a89SDavid Gow
32909641f7cSDaniel Latypovdef add_exec_opts(parser) -> None:
33045ba7a89SDavid Gow	parser.add_argument('--timeout',
3316ebf5866SFelix Guo			    help='maximum number of seconds to allow for all tests '
3326ebf5866SFelix Guo			    'to run. This does not include time taken to build the '
3336ebf5866SFelix Guo			    'tests.',
3346ebf5866SFelix Guo			    type=int,
3356ebf5866SFelix Guo			    default=300,
336baa33315SDaniel Latypov			    metavar='SECONDS')
337d992880bSDaniel Latypov	parser.add_argument('filter_glob',
338a127b154SDaniel Latypov			    help='Filter which KUnit test suites/tests run at '
339a127b154SDaniel Latypov			    'boot-time, e.g. list* or list*.*del_test',
340d992880bSDaniel Latypov			    type=str,
341d992880bSDaniel Latypov			    nargs='?',
342d992880bSDaniel Latypov			    default='',
343d992880bSDaniel Latypov			    metavar='filter_glob')
3446cb51a18SDaniel Latypov	parser.add_argument('--kernel_args',
3456cb51a18SDaniel Latypov			    help='Kernel command-line parameters. Maybe be repeated',
346baa33315SDaniel Latypov			     action='append', metavar='')
347ff9e09a3SDaniel Latypov	parser.add_argument('--run_isolated', help='If set, boot the kernel for each '
348ff9e09a3SDaniel Latypov			    'individual suite/test. This is can be useful for debugging '
349ff9e09a3SDaniel Latypov			    'a non-hermetic test, one that might pass/fail based on '
350ff9e09a3SDaniel Latypov			    'what ran before it.',
351ff9e09a3SDaniel Latypov			    type=str,
3520453f984SDaniel Latypov			    choices=['suite', 'test'])
3536ebf5866SFelix Guo
35409641f7cSDaniel Latypovdef add_parse_opts(parser) -> None:
355309e22efSDaniel Latypov	parser.add_argument('--raw_output', help='If set don\'t parse output from kernel. '
356309e22efSDaniel Latypov			    'By default, filters to just KUnit output. Use '
357309e22efSDaniel Latypov			    '--raw_output=all to show everything',
358baa33315SDaniel Latypov			     type=str, nargs='?', const='all', default=None, choices=['all', 'kunit'])
35921a6d178SHeidi Fahim	parser.add_argument('--json',
36021a6d178SHeidi Fahim			    nargs='?',
361309e22efSDaniel Latypov			    help='Prints parsed test results as JSON to stdout or a file if '
362309e22efSDaniel Latypov			    'a filename is specified. Does nothing if --raw_output is set.',
363baa33315SDaniel Latypov			    type=str, const='stdout', default=None, metavar='FILE')
364021ed9f5SHeidi Fahim
3658a04930fSDaniel Latypov
3668a04930fSDaniel Latypovdef tree_from_args(cli_args: argparse.Namespace) -> kunit_kernel.LinuxSourceTree:
3678a04930fSDaniel Latypov	"""Returns a LinuxSourceTree based on the user's arguments."""
368a9333bd3SDaniel Latypov	# Allow users to specify multiple arguments in one string, e.g. '-smp 8'
369a9333bd3SDaniel Latypov	qemu_args: List[str] = []
370a9333bd3SDaniel Latypov	if cli_args.qemu_args:
371a9333bd3SDaniel Latypov		for arg in cli_args.qemu_args:
372a9333bd3SDaniel Latypov			qemu_args.extend(shlex.split(arg))
373a9333bd3SDaniel Latypov
374980ac3adSDaniel Latypov	kunitconfigs = cli_args.kunitconfig if cli_args.kunitconfig else []
375980ac3adSDaniel Latypov	if cli_args.alltests:
376980ac3adSDaniel Latypov		# Prepend so user-specified options take prio if we ever allow
377980ac3adSDaniel Latypov		# --kunitconfig options to have differing options.
378980ac3adSDaniel Latypov		kunitconfigs = [kunit_kernel.ALL_TESTS_CONFIG_PATH] + kunitconfigs
379980ac3adSDaniel Latypov
3808a04930fSDaniel Latypov	return kunit_kernel.LinuxSourceTree(cli_args.build_dir,
381980ac3adSDaniel Latypov			kunitconfig_paths=kunitconfigs,
3828a04930fSDaniel Latypov			kconfig_add=cli_args.kconfig_add,
3838a04930fSDaniel Latypov			arch=cli_args.arch,
3848a04930fSDaniel Latypov			cross_compile=cli_args.cross_compile,
385a9333bd3SDaniel Latypov			qemu_config_path=cli_args.qemu_config,
386a9333bd3SDaniel Latypov			extra_qemu_args=qemu_args)
3878a04930fSDaniel Latypov
3888a04930fSDaniel Latypov
3892dc9d6caSAlexander Pantyukhindef run_handler(cli_args):
3902dc9d6caSAlexander Pantyukhin	if not os.path.exists(cli_args.build_dir):
3912dc9d6caSAlexander Pantyukhin		os.mkdir(cli_args.build_dir)
3922dc9d6caSAlexander Pantyukhin
3932dc9d6caSAlexander Pantyukhin	linux = tree_from_args(cli_args)
3942dc9d6caSAlexander Pantyukhin	request = KunitRequest(build_dir=cli_args.build_dir,
3952dc9d6caSAlexander Pantyukhin					make_options=cli_args.make_options,
3962dc9d6caSAlexander Pantyukhin					jobs=cli_args.jobs,
3972dc9d6caSAlexander Pantyukhin					raw_output=cli_args.raw_output,
3982dc9d6caSAlexander Pantyukhin					json=cli_args.json,
3992dc9d6caSAlexander Pantyukhin					timeout=cli_args.timeout,
4002dc9d6caSAlexander Pantyukhin					filter_glob=cli_args.filter_glob,
4012dc9d6caSAlexander Pantyukhin					kernel_args=cli_args.kernel_args,
4022dc9d6caSAlexander Pantyukhin					run_isolated=cli_args.run_isolated)
4032dc9d6caSAlexander Pantyukhin	result = run_tests(linux, request)
4042dc9d6caSAlexander Pantyukhin	if result.status != KunitStatus.SUCCESS:
4052dc9d6caSAlexander Pantyukhin		sys.exit(1)
4062dc9d6caSAlexander Pantyukhin
4072dc9d6caSAlexander Pantyukhin
4082dc9d6caSAlexander Pantyukhindef config_handler(cli_args):
4092dc9d6caSAlexander Pantyukhin	if cli_args.build_dir and (
4102dc9d6caSAlexander Pantyukhin			not os.path.exists(cli_args.build_dir)):
4112dc9d6caSAlexander Pantyukhin		os.mkdir(cli_args.build_dir)
4122dc9d6caSAlexander Pantyukhin
4132dc9d6caSAlexander Pantyukhin	linux = tree_from_args(cli_args)
4142dc9d6caSAlexander Pantyukhin	request = KunitConfigRequest(build_dir=cli_args.build_dir,
4152dc9d6caSAlexander Pantyukhin						make_options=cli_args.make_options)
4162dc9d6caSAlexander Pantyukhin	result = config_tests(linux, request)
4172dc9d6caSAlexander Pantyukhin	stdout.print_with_timestamp((
4182dc9d6caSAlexander Pantyukhin		'Elapsed time: %.3fs\n') % (
4192dc9d6caSAlexander Pantyukhin			result.elapsed_time))
4202dc9d6caSAlexander Pantyukhin	if result.status != KunitStatus.SUCCESS:
4212dc9d6caSAlexander Pantyukhin		sys.exit(1)
4222dc9d6caSAlexander Pantyukhin
4232dc9d6caSAlexander Pantyukhin
4242dc9d6caSAlexander Pantyukhindef build_handler(cli_args):
4252dc9d6caSAlexander Pantyukhin	linux = tree_from_args(cli_args)
4262dc9d6caSAlexander Pantyukhin	request = KunitBuildRequest(build_dir=cli_args.build_dir,
4272dc9d6caSAlexander Pantyukhin					make_options=cli_args.make_options,
4282dc9d6caSAlexander Pantyukhin					jobs=cli_args.jobs)
4292dc9d6caSAlexander Pantyukhin	result = config_and_build_tests(linux, request)
4302dc9d6caSAlexander Pantyukhin	stdout.print_with_timestamp((
4312dc9d6caSAlexander Pantyukhin		'Elapsed time: %.3fs\n') % (
4322dc9d6caSAlexander Pantyukhin			result.elapsed_time))
4332dc9d6caSAlexander Pantyukhin	if result.status != KunitStatus.SUCCESS:
4342dc9d6caSAlexander Pantyukhin		sys.exit(1)
4352dc9d6caSAlexander Pantyukhin
4362dc9d6caSAlexander Pantyukhin
4372dc9d6caSAlexander Pantyukhindef exec_handler(cli_args):
4382dc9d6caSAlexander Pantyukhin	linux = tree_from_args(cli_args)
4392dc9d6caSAlexander Pantyukhin	exec_request = KunitExecRequest(raw_output=cli_args.raw_output,
4402dc9d6caSAlexander Pantyukhin					build_dir=cli_args.build_dir,
4412dc9d6caSAlexander Pantyukhin					json=cli_args.json,
4422dc9d6caSAlexander Pantyukhin					timeout=cli_args.timeout,
4432dc9d6caSAlexander Pantyukhin					filter_glob=cli_args.filter_glob,
4442dc9d6caSAlexander Pantyukhin					kernel_args=cli_args.kernel_args,
4452dc9d6caSAlexander Pantyukhin					run_isolated=cli_args.run_isolated)
4462dc9d6caSAlexander Pantyukhin	result = exec_tests(linux, exec_request)
4472dc9d6caSAlexander Pantyukhin	stdout.print_with_timestamp((
4482dc9d6caSAlexander Pantyukhin		'Elapsed time: %.3fs\n') % (result.elapsed_time))
4492dc9d6caSAlexander Pantyukhin	if result.status != KunitStatus.SUCCESS:
4502dc9d6caSAlexander Pantyukhin		sys.exit(1)
4512dc9d6caSAlexander Pantyukhin
4522dc9d6caSAlexander Pantyukhin
4532dc9d6caSAlexander Pantyukhindef parse_handler(cli_args):
4542dc9d6caSAlexander Pantyukhin	if cli_args.file is None:
4552dc9d6caSAlexander Pantyukhin		sys.stdin.reconfigure(errors='backslashreplace')  # pytype: disable=attribute-error
4562dc9d6caSAlexander Pantyukhin		kunit_output = sys.stdin
4572dc9d6caSAlexander Pantyukhin	else:
4582dc9d6caSAlexander Pantyukhin		with open(cli_args.file, 'r', errors='backslashreplace') as f:
4592dc9d6caSAlexander Pantyukhin			kunit_output = f.read().splitlines()
4602dc9d6caSAlexander Pantyukhin	# We know nothing about how the result was created!
4612dc9d6caSAlexander Pantyukhin	metadata = kunit_json.Metadata()
4622dc9d6caSAlexander Pantyukhin	request = KunitParseRequest(raw_output=cli_args.raw_output,
4632dc9d6caSAlexander Pantyukhin					json=cli_args.json)
4642dc9d6caSAlexander Pantyukhin	result, _ = parse_tests(request, metadata, kunit_output)
4652dc9d6caSAlexander Pantyukhin	if result.status != KunitStatus.SUCCESS:
4662dc9d6caSAlexander Pantyukhin		sys.exit(1)
4672dc9d6caSAlexander Pantyukhin
4682dc9d6caSAlexander Pantyukhin
4692dc9d6caSAlexander Pantyukhinsubcommand_handlers_map = {
4702dc9d6caSAlexander Pantyukhin	'run': run_handler,
4712dc9d6caSAlexander Pantyukhin	'config': config_handler,
4722dc9d6caSAlexander Pantyukhin	'build': build_handler,
4732dc9d6caSAlexander Pantyukhin	'exec': exec_handler,
4742dc9d6caSAlexander Pantyukhin	'parse': parse_handler
4752dc9d6caSAlexander Pantyukhin}
4762dc9d6caSAlexander Pantyukhin
4772dc9d6caSAlexander Pantyukhin
4788a04930fSDaniel Latypovdef main(argv):
47945ba7a89SDavid Gow	parser = argparse.ArgumentParser(
48045ba7a89SDavid Gow			description='Helps writing and running KUnit tests.')
48145ba7a89SDavid Gow	subparser = parser.add_subparsers(dest='subcommand')
48245ba7a89SDavid Gow
48345ba7a89SDavid Gow	# The 'run' command will config, build, exec, and parse in one go.
48445ba7a89SDavid Gow	run_parser = subparser.add_parser('run', help='Runs KUnit tests.')
48545ba7a89SDavid Gow	add_common_opts(run_parser)
48645ba7a89SDavid Gow	add_build_opts(run_parser)
48745ba7a89SDavid Gow	add_exec_opts(run_parser)
48845ba7a89SDavid Gow	add_parse_opts(run_parser)
48945ba7a89SDavid Gow
49045ba7a89SDavid Gow	config_parser = subparser.add_parser('config',
49145ba7a89SDavid Gow						help='Ensures that .config contains all of '
49245ba7a89SDavid Gow						'the options in .kunitconfig')
49345ba7a89SDavid Gow	add_common_opts(config_parser)
49445ba7a89SDavid Gow
49545ba7a89SDavid Gow	build_parser = subparser.add_parser('build', help='Builds a kernel with KUnit tests')
49645ba7a89SDavid Gow	add_common_opts(build_parser)
49745ba7a89SDavid Gow	add_build_opts(build_parser)
49845ba7a89SDavid Gow
49945ba7a89SDavid Gow	exec_parser = subparser.add_parser('exec', help='Run a kernel with KUnit tests')
50045ba7a89SDavid Gow	add_common_opts(exec_parser)
50145ba7a89SDavid Gow	add_exec_opts(exec_parser)
50245ba7a89SDavid Gow	add_parse_opts(exec_parser)
50345ba7a89SDavid Gow
50445ba7a89SDavid Gow	# The 'parse' option is special, as it doesn't need the kernel source
50545ba7a89SDavid Gow	# (therefore there is no need for a build_dir, hence no add_common_opts)
50645ba7a89SDavid Gow	# and the '--file' argument is not relevant to 'run', so isn't in
50745ba7a89SDavid Gow	# add_parse_opts()
50845ba7a89SDavid Gow	parse_parser = subparser.add_parser('parse',
50945ba7a89SDavid Gow					    help='Parses KUnit results from a file, '
51045ba7a89SDavid Gow					    'and parses formatted results.')
51145ba7a89SDavid Gow	add_parse_opts(parse_parser)
51245ba7a89SDavid Gow	parse_parser.add_argument('file',
51345ba7a89SDavid Gow				  help='Specifies the file to read results from.',
51445ba7a89SDavid Gow				  type=str, nargs='?', metavar='input_file')
5150476e69fSGreg Thelen
516d8c23eadSDaniel Latypov	cli_args = parser.parse_args(massage_argv(argv))
5176ebf5866SFelix Guo
5185578d008SBrendan Higgins	if get_kernel_root_path():
5195578d008SBrendan Higgins		os.chdir(get_kernel_root_path())
5205578d008SBrendan Higgins
5212dc9d6caSAlexander Pantyukhin	subcomand_handler = subcommand_handlers_map.get(cli_args.subcommand, None)
52282206a0cSBrendan Higgins
5232dc9d6caSAlexander Pantyukhin	if subcomand_handler is None:
5246ebf5866SFelix Guo		parser.print_help()
5252dc9d6caSAlexander Pantyukhin		return
5262dc9d6caSAlexander Pantyukhin
5272dc9d6caSAlexander Pantyukhin	subcomand_handler(cli_args)
5282dc9d6caSAlexander Pantyukhin
5296ebf5866SFelix Guo
5306ebf5866SFelix Guoif __name__ == '__main__':
531ff7b437fSBrendan Higgins	main(sys.argv[1:])
532