1#!/usr/bin/env @PYTHON_SHEBANG@ 2 3# 4# This file and its contents are supplied under the terms of the 5# Common Development and Distribution License ("CDDL"), version 1.0. 6# You may only use this file in accordance with the terms of version 7# 1.0 of the CDDL. 8# 9# A full copy of the text of the CDDL should have accompanied this 10# source. A copy of the CDDL is also available via the Internet at 11# http://www.illumos.org/license/CDDL. 12# 13 14# 15# Copyright (c) 2017 by Delphix. All rights reserved. 16# Copyright (c) 2018 by Lawrence Livermore National Security, LLC. 17# 18# This script must remain compatible with Python 2.6+ and Python 3.4+. 19# 20 21import os 22import re 23import sys 24 25# 26# This script parses the stdout of zfstest, which has this format: 27# 28# Test: /path/to/testa (run as root) [00:00] [PASS] 29# Test: /path/to/testb (run as jkennedy) [00:00] [PASS] 30# Test: /path/to/testc (run as root) [00:00] [FAIL] 31# [...many more results...] 32# 33# Results Summary 34# FAIL 22 35# SKIP 32 36# PASS 1156 37# 38# Running Time: 02:50:31 39# Percent passed: 95.5% 40# Log directory: /var/tmp/test_results/20180615T205926 41# 42 43# 44# Common generic reasons for a test or test group to be skipped. 45# 46# Some test cases are known to fail in ways which are not harmful or dangerous. 47# In these cases simply mark the test as a known failure until it can be 48# updated and the issue resolved. Note that it's preferable to open a unique 49# issue on the GitHub issue tracker for each test case failure. 50# 51known_reason = 'Known issue' 52 53# 54# Some tests require that a test user be able to execute the zfs utilities. 55# This may not be possible when testing in-tree due to the default permissions 56# on the user's home directory. When testing this can be resolved by granting 57# group read access. 58# 59# chmod 0750 $HOME 60# 61exec_reason = 'Test user execute permissions required for utilities' 62 63# 64# Some tests require a minimum python version of 3.5 and will be skipped when 65# the default system version is too old. There may also be tests which require 66# additional python modules be installed, for example python-cffi is required 67# by the pyzfs tests. 68# 69python_reason = 'Python v3.5 or newer required' 70python_deps_reason = 'Python modules missing: python-cffi' 71 72# 73# Some tests require the O_TMPFILE flag which was first introduced in the 74# 3.11 kernel. 75# 76tmpfile_reason = 'Kernel O_TMPFILE support required' 77 78# 79# Some tests require that the NFS client and server utilities be installed. 80# 81share_reason = 'NFS client and server utilities required' 82 83# 84# Some tests require that the lsattr utility support the project id feature. 85# 86project_id_reason = 'lsattr with set/show project ID required' 87 88# 89# Some tests require that the kernel support user namespaces. 90# 91user_ns_reason = 'Kernel user namespace support required' 92 93# 94# Some rewind tests can fail since nothing guarantees that old MOS blocks 95# are not overwritten. Snapshots protect datasets and data files but not 96# the MOS. Reasonable efforts are made in the test case to increase the 97# odds that some txgs will have their MOS data left untouched, but it is 98# never a sure thing. 99# 100rewind_reason = 'Arbitrary pool rewind is not guaranteed' 101 102# 103# Some tests may by structured in a way that relies on exact knowledge 104# of how much free space in available in a pool. These tests cannot be 105# made completely reliable because the internal details of how free space 106# is managed are not exposed to user space. 107# 108enospc_reason = 'Exact free space reporting is not guaranteed' 109 110# 111# Some tests require a minimum version of the fio benchmark utility. 112# Older distributions such as CentOS 6.x only provide fio-2.0.13. 113# 114fio_reason = 'Fio v2.3 or newer required' 115 116# 117# Some tests require that the DISKS provided support the discard operation. 118# Normally this is not an issue because loop back devices are used for DISKS 119# and they support discard (TRIM/UNMAP). 120# 121trim_reason = 'DISKS must support discard (TRIM/UNMAP)' 122 123# 124# Some tests are not applicable to a platform or need to be updated to operate 125# in the manor required by the platform. Any tests which are skipped for this 126# reason will be suppressed in the final analysis output. 127# 128na_reason = "Not applicable" 129 130summary = { 131 'total': float(0), 132 'passed': float(0), 133 'logfile': "Could not determine logfile location." 134} 135 136# 137# These tests are known to fail, thus we use this list to prevent these 138# failures from failing the job as a whole; only unexpected failures 139# bubble up to cause this script to exit with a non-zero exit status. 140# 141# Format: { 'test-name': ['expected result', 'issue-number | reason'] } 142# 143# For each known failure it is recommended to link to a GitHub issue by 144# setting the reason to the issue number. Alternately, one of the generic 145# reasons listed above can be used. 146# 147known = { 148 'casenorm/mixed_none_lookup_ci': ['FAIL', '7633'], 149 'casenorm/mixed_formd_lookup_ci': ['FAIL', '7633'], 150 'cli_root/zfs_unshare/zfs_unshare_002_pos': ['SKIP', na_reason], 151 'cli_root/zfs_unshare/zfs_unshare_006_pos': ['SKIP', na_reason], 152 'cli_user/misc/zfs_share_001_neg': ['SKIP', na_reason], 153 'cli_user/misc/zfs_unshare_001_neg': ['SKIP', na_reason], 154 'privilege/setup': ['SKIP', na_reason], 155 'refreserv/refreserv_004_pos': ['FAIL', known_reason], 156 'rootpool/setup': ['SKIP', na_reason], 157 'rsend/rsend_008_pos': ['SKIP', '6066'], 158 'vdev_zaps/vdev_zaps_007_pos': ['FAIL', known_reason], 159} 160 161if sys.platform.startswith('freebsd'): 162 known.update({ 163 'cli_root/zpool_wait/zpool_wait_trim_basic': ['SKIP', trim_reason], 164 'cli_root/zpool_wait/zpool_wait_trim_cancel': ['SKIP', trim_reason], 165 'cli_root/zpool_wait/zpool_wait_trim_flag': ['SKIP', trim_reason], 166 'link_count/link_count_001': ['SKIP', na_reason], 167 }) 168elif sys.platform.startswith('linux'): 169 known.update({ 170 'casenorm/mixed_formd_lookup': ['FAIL', '7633'], 171 'casenorm/mixed_formd_delete': ['FAIL', '7633'], 172 'casenorm/sensitive_formd_lookup': ['FAIL', '7633'], 173 'casenorm/sensitive_formd_delete': ['FAIL', '7633'], 174 'removal/removal_with_zdb': ['SKIP', known_reason], 175 }) 176 177 178# 179# These tests may occasionally fail or be skipped. We want there failures 180# to be reported but only unexpected failures should bubble up to cause 181# this script to exit with a non-zero exit status. 182# 183# Format: { 'test-name': ['expected result', 'issue-number | reason'] } 184# 185# For each known failure it is recommended to link to a GitHub issue by 186# setting the reason to the issue number. Alternately, one of the generic 187# reasons listed above can be used. 188# 189maybe = { 190 'chattr/setup': ['SKIP', exec_reason], 191 'cli_root/zdb/zdb_006_pos': ['FAIL', known_reason], 192 'cli_root/zfs_get/zfs_get_004_pos': ['FAIL', known_reason], 193 'cli_root/zfs_get/zfs_get_009_pos': ['SKIP', '5479'], 194 'cli_root/zfs_share/setup': ['SKIP', share_reason], 195 'cli_root/zfs_snapshot/zfs_snapshot_002_neg': ['FAIL', known_reason], 196 'cli_root/zfs_unshare/setup': ['SKIP', share_reason], 197 'cli_root/zpool_add/zpool_add_004_pos': ['FAIL', known_reason], 198 'cli_root/zpool_destroy/zpool_destroy_001_pos': ['SKIP', '6145'], 199 'cli_root/zpool_import/import_rewind_device_replaced': 200 ['FAIL', rewind_reason], 201 'cli_root/zpool_import/import_rewind_config_changed': 202 ['FAIL', rewind_reason], 203 'cli_root/zpool_import/zpool_import_missing_003_pos': ['SKIP', '6839'], 204 'cli_root/zpool_trim/setup': ['SKIP', trim_reason], 205 'cli_root/zpool_upgrade/zpool_upgrade_004_pos': ['FAIL', '6141'], 206 'delegate/setup': ['SKIP', exec_reason], 207 'history/history_004_pos': ['FAIL', '7026'], 208 'history/history_005_neg': ['FAIL', '6680'], 209 'history/history_006_neg': ['FAIL', '5657'], 210 'history/history_008_pos': ['FAIL', known_reason], 211 'history/history_010_pos': ['SKIP', exec_reason], 212 'io/mmap': ['SKIP', fio_reason], 213 'largest_pool/largest_pool_001_pos': ['FAIL', known_reason], 214 'mmp/mmp_on_uberblocks': ['FAIL', known_reason], 215 'pyzfs/pyzfs_unittest': ['SKIP', python_deps_reason], 216 'no_space/enospc_002_pos': ['FAIL', enospc_reason], 217 'projectquota/setup': ['SKIP', exec_reason], 218 'redundancy/redundancy_004_neg': ['FAIL', '7290'], 219 'redundancy/redundancy_draid_spare3': ['SKIP', known_reason], 220 'reservation/reservation_008_pos': ['FAIL', '7741'], 221 'reservation/reservation_018_pos': ['FAIL', '5642'], 222 'rsend/rsend_019_pos': ['FAIL', '6086'], 223 'rsend/rsend_020_pos': ['FAIL', '6446'], 224 'rsend/rsend_021_pos': ['FAIL', '6446'], 225 'rsend/rsend_024_pos': ['FAIL', '5665'], 226 'rsend/send-c_volume': ['FAIL', '6087'], 227 'rsend/send_partial_dataset': ['FAIL', known_reason], 228 'snapshot/clone_001_pos': ['FAIL', known_reason], 229 'snapshot/snapshot_009_pos': ['FAIL', '7961'], 230 'snapshot/snapshot_010_pos': ['FAIL', '7961'], 231 'snapused/snapused_004_pos': ['FAIL', '5513'], 232 'tmpfile/setup': ['SKIP', tmpfile_reason], 233 'threadsappend/threadsappend_001_pos': ['FAIL', '6136'], 234 'trim/setup': ['SKIP', trim_reason], 235 'upgrade/upgrade_projectquota_001_pos': ['SKIP', project_id_reason], 236 'user_namespace/setup': ['SKIP', user_ns_reason], 237 'userquota/setup': ['SKIP', exec_reason], 238 'vdev_zaps/vdev_zaps_004_pos': ['FAIL', '6935'], 239 'zvol/zvol_ENOSPC/zvol_ENOSPC_001_pos': ['FAIL', '5848'], 240 'pam/setup': ['SKIP', "pamtester might be not available"], 241} 242 243if sys.platform.startswith('freebsd'): 244 maybe.update({ 245 'cli_root/zfs_copies/zfs_copies_002_pos': ['FAIL', known_reason], 246 'cli_root/zfs_inherit/zfs_inherit_001_neg': ['FAIL', known_reason], 247 'cli_root/zfs_share/zfs_share_011_pos': ['FAIL', known_reason], 248 'cli_root/zfs_share/zfs_share_concurrent_shares': 249 ['FAIL', known_reason], 250 'cli_root/zpool_import/zpool_import_012_pos': ['FAIL', known_reason], 251 'delegate/zfs_allow_003_pos': ['FAIL', known_reason], 252 'removal/removal_condense_export': ['FAIL', known_reason], 253 'removal/removal_with_export': ['FAIL', known_reason], 254 'resilver/resilver_restart_001': ['FAIL', known_reason], 255 'zvol/zvol_misc/zvol_misc_volmode': ['FAIL', known_reason], 256 }) 257elif sys.platform.startswith('linux'): 258 maybe.update({ 259 'alloc_class/alloc_class_009_pos': ['FAIL', known_reason], 260 'alloc_class/alloc_class_010_pos': ['FAIL', known_reason], 261 'alloc_class/alloc_class_011_neg': ['FAIL', known_reason], 262 'cli_root/zfs_rename/zfs_rename_002_pos': ['FAIL', known_reason], 263 'cli_root/zpool_expand/zpool_expand_001_pos': ['FAIL', known_reason], 264 'cli_root/zpool_expand/zpool_expand_005_pos': ['FAIL', known_reason], 265 'cli_root/zpool_reopen/zpool_reopen_003_pos': ['FAIL', known_reason], 266 'limits/filesystem_limit': ['SKIP', known_reason], 267 'limits/snapshot_limit': ['SKIP', known_reason], 268 'mmp/mmp_exported_import': ['FAIL', known_reason], 269 'mmp/mmp_inactive_import': ['FAIL', known_reason], 270 'refreserv/refreserv_raidz': ['FAIL', known_reason], 271 'rsend/rsend_007_pos': ['FAIL', known_reason], 272 'rsend/rsend_010_pos': ['FAIL', known_reason], 273 'rsend/rsend_011_pos': ['FAIL', known_reason], 274 'snapshot/rollback_003_pos': ['FAIL', known_reason], 275 }) 276 277 278def usage(s): 279 print(s) 280 sys.exit(1) 281 282 283def process_results(pathname): 284 try: 285 f = open(pathname) 286 except IOError as e: 287 print('Error opening file: %s' % e) 288 sys.exit(1) 289 290 prefix = '/zfs-tests/tests/functional/' 291 pattern = \ 292 r'^Test(?:\s+\(\S+\))?:' + \ 293 r'\s*\S*%s(\S+)\s*\(run as (\S+)\)\s*\[(\S+)\]\s*\[(\S+)\]' \ 294 % prefix 295 pattern_log = r'^\s*Log directory:\s*(\S*)' 296 297 d = {} 298 for line in f.readlines(): 299 m = re.match(pattern, line) 300 if m and len(m.groups()) == 4: 301 summary['total'] += 1 302 if m.group(4) == "PASS": 303 summary['passed'] += 1 304 d[m.group(1)] = m.group(4) 305 continue 306 307 m = re.match(pattern_log, line) 308 if m: 309 summary['logfile'] = m.group(1) 310 311 return d 312 313 314if __name__ == "__main__": 315 if len(sys.argv) != 2: 316 usage('usage: %s <pathname>' % sys.argv[0]) 317 results = process_results(sys.argv[1]) 318 319 if summary['total'] == 0: 320 print("\n\nNo test results were found.") 321 print("Log directory: %s" % summary['logfile']) 322 sys.exit(0) 323 324 expected = [] 325 unexpected = [] 326 327 for test in list(results.keys()): 328 if results[test] == "PASS": 329 continue 330 331 setup = test.replace(os.path.basename(test), "setup") 332 if results[test] == "SKIP" and test != setup: 333 if setup in known and known[setup][0] == "SKIP": 334 continue 335 if setup in maybe and maybe[setup][0] == "SKIP": 336 continue 337 338 if ((test not in known or results[test] not in known[test][0]) and 339 (test not in maybe or results[test] not in maybe[test][0])): 340 unexpected.append(test) 341 else: 342 expected.append(test) 343 344 print("\nTests with results other than PASS that are expected:") 345 for test in sorted(expected): 346 issue_url = 'https://github.com/openzfs/zfs/issues/' 347 348 # Include the reason why the result is expected, given the following: 349 # 1. Suppress test results which set the "Not applicable" reason. 350 # 2. Numerical reasons are assumed to be GitHub issue numbers. 351 # 3. When an entire test group is skipped only report the setup reason. 352 if test in known: 353 if known[test][1] == na_reason: 354 continue 355 elif known[test][1].isdigit(): 356 expect = issue_url + known[test][1] 357 else: 358 expect = known[test][1] 359 elif test in maybe: 360 if maybe[test][1].isdigit(): 361 expect = issue_url + maybe[test][1] 362 else: 363 expect = maybe[test][1] 364 elif setup in known and known[setup][0] == "SKIP" and setup != test: 365 continue 366 elif setup in maybe and maybe[setup][0] == "SKIP" and setup != test: 367 continue 368 else: 369 expect = "UNKNOWN REASON" 370 print(" %s %s (%s)" % (results[test], test, expect)) 371 372 print("\nTests with result of PASS that are unexpected:") 373 for test in sorted(known.keys()): 374 # We probably should not be silently ignoring the case 375 # where "test" is not in "results". 376 if test not in results or results[test] != "PASS": 377 continue 378 print(" %s %s (expected %s)" % (results[test], test, 379 known[test][0])) 380 381 print("\nTests with results other than PASS that are unexpected:") 382 for test in sorted(unexpected): 383 expect = "PASS" if test not in known else known[test][0] 384 print(" %s %s (expected %s)" % (results[test], test, expect)) 385 386 if len(unexpected) == 0: 387 sys.exit(0) 388 else: 389 sys.exit(1) 390