Lines Matching +full:test +full:- +full:trailing
1 # SPDX-License-Identifier: GPL-2.0
3 # Parses KTAP test results from a kernel dmesg log and incrementally prints
4 # results with reader-friendly format. Stores and returns test results in a
5 # Test object.
22 class Test:
24 A class to represent a test parsed from KTAP results. All KTAP
25 results within a test log are stored in a main Test object as
29 status : TestStatus - status of the test
30 name : str - name of the test
31 expected_count : int - expected number of subtests (0 if single
32 test case and None if unknown expected number of subtests)
33 subtests : List[Test] - list of subtests
34 log : List[str] - log of KTAP lines that correspond to the test
35 counts : TestCounts - counts of the test statuses and errors of
36 subtests or of the test itself if the test is a single
37 test case.
39 def __init__(self) -> None:
40 """Creates Test object with default attributes."""
44 self.subtests = [] # type: List[Test]
49 def __str__(self) -> str:
50 """Returns string representation of a Test class object."""
51 return (f'Test({self.status}, {self.name}, {self.expected_count}, '
54 def __repr__(self) -> str:
55 """Returns string representation of a Test class object."""
58 def add_error(self, printer: Printer, error_message: str) -> None:
59 """Records an error that occurred while parsing this test."""
61 printer.print_with_timestamp(printer.red('[ERROR]') + f' Test: {self.name}: {error_message}')
63 def ok_status(self) -> bool:
68 """An enumeration class to represent the status of a test."""
79 Tracks the counts of statuses of all test cases and any errors within
80 a Test.
88 def __str__(self) -> str:
96 def total(self) -> int:
97 """Returns the total number of test cases within a test
98 object, where a test case is a test with no subtests.
103 def add_subtest_counts(self, counts: TestCounts) -> None:
107 parent test.
110 counts - a different TestCounts object whose counts
119 def get_status(self) -> TestStatus:
120 """Returns the aggregated status of a Test using test
136 def add_status(self, status: TestStatus) -> None:
165 def _get_next(self) -> None:
176 def peek(self) -> str:
182 def pop(self) -> str:
192 def __bool__(self) -> bool:
198 def __iter__(self) -> Iterator[str]:
205 def line_number(self) -> int:
212 KTAP_START = re.compile(r'\s*KTAP version ([0-9]+)$')
213 TAP_START = re.compile(r'\s*TAP version ([0-9]+)$')
215 'Kernel panic - not syncing: VFS:|reboot: System halted)')
218 def extract_tap_lines(kernel_output: Iterable[str]) -> LineStream:
221 -> Iterator[Tuple[int, str]]:
226 line = line.rstrip() # remove trailing \n
255 version_type: str, test: Test, printer: Printer) -> None:
257 Adds error to test object if version number is too high or too
261 version_num - The inputted version number from the parsed KTAP or TAP
263 accepted_version - List of accepted KTAP or TAP versions
264 version_type - 'KTAP' or 'TAP' depending on the type of
266 test - Test object for current test being parsed
267 printer - Printer object to output error
270 test.add_error(printer, f'{version_type} version lower than expected!')
272 test.add_error(printer, f'{version_type} version higher than expected!')
274 def parse_ktap_header(lines: LineStream, test: Test, printer: Printer) -> bool:
280 - 'KTAP version [version number]'
281 - 'TAP version [version number]'
284 lines - LineStream of KTAP output to parse
285 test - Test object for current test being parsed
286 printer - Printer object to output results
295 check_version(version_num, KTAP_VERSIONS, 'KTAP', test, printer)
298 check_version(version_num, TAP_VERSIONS, 'TAP', test, printer)
306 def parse_test_header(lines: LineStream, test: Test) -> bool:
308 Parses test header and stores test name in test object.
309 Returns False if fails to parse test header line.
312 - '# Subtest: [test name]'
315 lines - LineStream of KTAP output to parse
316 test - Test object for current test being parsed
319 True if successfully parsed test header line
324 test.name = match.group(1)
328 TEST_PLAN = re.compile(r'^\s*1\.\.([0-9]+)')
330 def parse_test_plan(lines: LineStream, test: Test) -> bool:
332 Parses test plan line and stores the expected number of subtests in
333 test object. Reports an error if expected count is 0.
334 Returns False and sets expected_count to None if there is no valid test
338 - '1..[number of subtests]'
341 lines - LineStream of KTAP output to parse
342 test - Test object for current test being parsed
345 True if successfully parsed test plan line
349 test.expected_count = None
352 test.expected_count = expected_count
356 TEST_RESULT = re.compile(r'^\s*(ok|not ok) ([0-9]+) ?(:?- )?([^#]*)( # .*)?$')
358 TEST_RESULT_SKIP = re.compile(r'^\s*(ok|not ok) ([0-9]+) ?(:?- )?(.*) # SKIP ?(.*)$')
360 def peek_test_name_match(lines: LineStream, test: Test) -> bool:
362 Matches current line with the format of a test result line and checks
363 if the name matches the name of the current test.
367 - '[ok|not ok] [test number] [-] [test name] [optional skip
371 lines - LineStream of KTAP output to parse
372 test - Test object for current test being parsed
375 True if matched a test result line and the name matching the
376 expected test name
385 return name == test.name
387 def parse_test_result(lines: LineStream, test: Test,
388 expected_num: int, printer: Printer) -> bool:
390 Parses test result line and stores the status and name in the test
391 object. Reports an error if the test number does not match expected
392 test number.
393 Returns False if fails to parse test result line.
399 - '[ok|not ok] [test number] [-] [test name] [optional skip
403 lines - LineStream of KTAP output to parse
404 test - Test object for current test being parsed
405 expected_num - expected test number for current test
406 printer - Printer object to output results
409 True if successfully parsed a test result line.
415 # Check if line matches test result line format
420 # Set name of test object
422 test.name = skip_match.group(4)
424 test.name = match.group(4)
426 # Check test num
429 test.add_error(printer, f'Expected test number {expected_num} but found {num}')
431 # Set status of test object
434 test.status = TestStatus.SKIPPED
435 test.skip_reason = skip_match.group(5) or ''
437 test.status = TestStatus.SUCCESS
439 test.status = TestStatus.FAILURE
442 def parse_diagnostic(lines: LineStream) -> List[str]:
444 Parse lines that do not match the format of a test result line or
445 test header line and returns them in list.
448 - '# Subtest: [test name]'
449 - '[ok|not ok] [test number] [-] [test name] [optional skip
451 - 'KTAP version [version number]'
454 lines - LineStream of KTAP output to parse
471 def format_test_divider(message: str, len_message: int) -> str:
479 message - message to be centered in divider line
480 len_message - length of the message to be printed such that
489 difference = len(DIVIDER) - len_message - 2 # 2 spaces added
493 len_2 = difference - len_1
496 def print_test_header(test: Test, printer: Printer) -> None:
498 Prints test header with test name and optionally the expected number
505 test - Test object representing current test being printed
506 printer - Printer object to output results
508 message = test.name
510 # Add a leading space before the subtest counts only if a test name
513 if test.expected_count:
514 if test.expected_count == 1:
517 message += f'({test.expected_count} subtests)'
520 def print_log(log: Iterable[str], printer: Printer) -> None:
521 """Prints all strings in saved log for test in yellow."""
526 def format_test_result(test: Test, printer: Printer) -> str:
528 Returns string with formatted test result with colored status and test
535 test - Test object representing current test being printed
536 printer - Printer object to output results
539 String containing formatted test result
541 if test.status == TestStatus.SUCCESS:
542 return printer.green('[PASSED] ') + test.name
543 if test.status == TestStatus.SKIPPED:
544 skip_message = printer.yellow('[SKIPPED] ') + test.name
545 if test.skip_reason != '':
546 skip_message += printer.yellow(' (' + test.skip_reason + ')')
548 if test.status == TestStatus.NO_TESTS:
549 return printer.yellow('[NO TESTS RUN] ') + test.name
550 if test.status == TestStatus.TEST_CRASHED:
551 print_log(test.log, printer)
552 return printer.red('[CRASHED] ') + test.name
553 print_log(test.log, printer)
554 return printer.red('[FAILED] ') + test.name
556 def print_test_result(test: Test, printer: Printer) -> None:
558 Prints result line with status of test.
564 test - Test object representing current test being printed
565 printer - Printer object
567 printer.print_with_timestamp(format_test_result(test, printer))
569 def print_test_footer(test: Test, printer: Printer) -> None:
571 Prints test footer with status of test.
577 test - Test object representing current test being printed
578 printer - Printer object to output results
580 message = format_test_result(test, printer)
582 len(message) - printer.color_len()))
584 def print_test(test: Test, failed_only: bool, printer: Printer) -> None:
586 Prints Test object to given printer. For a child test, the result line is
587 printed. For a parent test, the test header, all child test results, and
588 the test footer are all printed. If failed_only is true, only failed/crashed
592 test - Test object to print
593 failed_only - True if only failed/crashed tests should be printed.
594 printer - Printer object to output results
596 if test.name == "main":
598 for subtest in test.subtests:
601 elif test.subtests != []:
602 if not failed_only or not test.ok_status():
603 print_test_header(test, printer)
604 for subtest in test.subtests:
606 print_test_footer(test, printer)
608 if not failed_only or not test.ok_status():
609 print_test_result(test, printer)
611 def _summarize_failed_tests(test: Test) -> str:
612 """Tries to summarize all the failing subtests in `test`."""
614 def failed_names(test: Test, parent_name: str) -> List[str]:
615 # Note: we use 'main' internally for the top-level test.
617 full_name = test.name
619 full_name = parent_name + '.' + test.name
621 if not test.subtests: # this is a leaf node
625 # Don't summarize it down "the top-level test failed", though.
626 failed_subtests = [sub for sub in test.subtests if not sub.ok_status()]
627 if parent_name and len(failed_subtests) == len(test.subtests):
635 failures = failed_names(test, '')
643 def print_summary_line(test: Test, printer: Printer) -> None:
645 Prints summary line of test object. Color of line is dependent on
646 status of test. Color is green if test passes, yellow if test is
647 skipped, and red if the test fails or crashes. Summary line contains
648 counts of the statuses of the tests subtests or the test itself if it
655 test - Test object representing current test being printed
656 printer - Printer object to output results
658 if test.status == TestStatus.SUCCESS:
660 elif test.status in (TestStatus.SKIPPED, TestStatus.NO_TESTS):
664 printer.print_with_timestamp(color(f'Testing complete. {test.counts}'))
666 # Summarize failures that might have gone off-screen since we had a lot
668 if test.ok_status() or test.counts.total() < 100:
670 summarized = _summarize_failed_tests(test)
677 def bubble_up_test_results(test: Test) -> None:
679 If the test has subtests, add the test counts of the subtests to the
680 test and check if any of the tests crashed and if so set the test
681 status to crashed. Otherwise if the test has no subtests add the
682 status of the test to the test counts.
685 test - Test object for current test being parsed
687 subtests = test.subtests
688 counts = test.counts
689 status = test.status
694 elif test.counts.get_status() == TestStatus.TEST_CRASHED:
695 test.status = TestStatus.TEST_CRASHED
697 if status == TestStatus.FAILURE and test.counts.get_status() == TestStatus.SUCCESS:
700 def parse_test(lines: LineStream, expected_num: int, log: List[str], is_subtest: bool, printer: Printer) -> Test:
702 Finds next test to parse in LineStream, creates new Test object,
703 parses any subtests of the test, populates Test object with all
704 information (status, name) about the test and the Test objects for
705 any subtests, and then returns the Test object. The method accepts
708 Accepted test formats:
710 - Main KTAP/TAP header
718 - Subtest header (must include either the KTAP version line or
744 - Test result line
748 ok 1 - test
751 lines - LineStream of KTAP output to parse
752 expected_num - expected test number for test to be parsed
753 log - list of strings containing any preceding diagnostic lines
754 corresponding to the current test
755 is_subtest - boolean indicating whether test is a subtest
756 printer - Printer object to output results
759 Test object populated with characteristics and any subtests
761 test = Test()
762 test.log.extend(log)
766 test.log.extend(err_log)
769 # If parsing the main/top-level test, parse KTAP version line and
770 # test plan
771 test.name = "main"
772 parse_ktap_header(lines, test, printer)
773 test.log.extend(parse_diagnostic(lines))
774 parse_test_plan(lines, test)
777 # If not the main test, attempt to parse a test header containing
779 ktap_line = parse_ktap_header(lines, test, printer)
780 subtest_line = parse_test_header(lines, test)
781 test.log.extend(parse_diagnostic(lines))
782 parse_test_plan(lines, test)
785 print_test_header(test, printer)
787 expected_count = test.expected_count
793 # if expected number of tests is unknown break when test
797 sub_test = Test()
798 if not lines or (peek_test_name_match(lines, test) and
801 # If parser reaches end of test before
804 test.add_error(printer, 'missing expected subtest!')
806 test.counts.add_status(
810 test.log.extend(sub_log)
816 test.subtests = subtests
818 # If not main test, look for test result line
819 test.log.extend(parse_diagnostic(lines))
820 if test.name != "" and not peek_test_name_match(lines, test):
821 test.add_error(printer, 'missing subtest result line!')
823 print_log(test.log, printer)
824 test.status = TestStatus.NO_TESTS
825 test.add_error(printer, 'No more test results!')
827 parse_test_result(lines, test, expected_num, printer)
829 # Check for there being no subtests within parent test
831 # Don't override a bad status if this test had one reported.
832 # Assumption: no subtests means CRASHED is from Test.__init__()
833 if test.status in (TestStatus.TEST_CRASHED, TestStatus.SUCCESS):
834 print_log(test.log, printer)
835 test.status = TestStatus.NO_TESTS
836 test.add_error(printer, '0 tests run!')
838 # Add statuses to TestCounts attribute in Test object
839 bubble_up_test_results(test)
841 # If test has subtests and is not the main test object, print
843 print_test_footer(test, printer)
845 print_test_result(test, printer)
846 return test
848 def parse_run_tests(kernel_output: Iterable[str], printer: Printer) -> Test:
850 Using kernel output, extract KTAP lines, parse the lines for test
851 results and print condensed test results and summary line.
854 kernel_output - Iterable object contains lines of kernel output
855 printer - Printer object to output results
858 Test - the main test object with all subtests.
862 test = Test()
864 test.name = '<missing>'
865 test.add_error(printer, 'Could not find any KTAP output. Did any KUnit tests run?\n' +
866 'Try running with the --raw_output=all option to see any log messages.')
867 test.status = TestStatus.FAILURE_TO_PARSE_TESTS
869 test = parse_test(lines, 0, [], False, printer)
870 if test.status != TestStatus.NO_TESTS:
871 test.status = test.counts.get_status()
873 return test