1# SPDX-License-Identifier: CDDL-1.0 2# 3# This file and its contents are supplied under the terms of the 4# Common Development and Distribution License ("CDDL"), version 1.0. 5# You may only use this file in accordance with the terms of version 6# 1.0 of the CDDL. 7# 8# A full copy of the text of the CDDL should have accompanied this 9# source. A copy of the CDDL is also available via the Internet at 10# https://opensource.org/license/CDDL-1.0. 11# 12 13# 14# Copyright (c) 2009, Sun Microsystems Inc. All rights reserved. 15# Copyright (c) 2012, 2020, Delphix. All rights reserved. 16# Copyright (c) 2017, Tim Chase. All rights reserved. 17# Copyright (c) 2017, Nexenta Systems Inc. All rights reserved. 18# Copyright (c) 2017, Lawrence Livermore National Security LLC. 19# Copyright (c) 2017, Datto Inc. All rights reserved. 20# Copyright (c) 2017, Open-E Inc. All rights reserved. 21# Copyright (c) 2021, The FreeBSD Foundation. 22# Copyright (c) 2025, Klara, Inc. 23# Copyright (c) 2026, TrueNAS. 24# Use is subject to license terms. 25# 26 27. ${STF_SUITE}/include/tunables.cfg 28 29. ${STF_TOOLS}/include/logapi.shlib 30. ${STF_SUITE}/include/math.shlib 31. ${STF_SUITE}/include/blkdev.shlib 32 33 34# On AlmaLinux 9 we will see $PWD = '.' instead of the full path. This causes 35# some tests to fail. Fix it up here. 36if [ "$PWD" = "." ] ; then 37 PWD="$(readlink -f $PWD)" 38fi 39 40# 41# Apply constrained path when available. This is required since the 42# PATH may have been modified by sudo's secure_path behavior. 43# 44if [ -n "$STF_PATH" ]; then 45 export PATH="$STF_PATH" 46fi 47 48# 49# Generic dot version comparison function 50# 51# Returns success when version $1 is greater than or equal to $2. 52# 53function compare_version_gte 54{ 55 [ "$(printf "$1\n$2" | sort -V | tail -n1)" = "$1" ] 56} 57 58# Helper function used by linux_version() and freebsd_version() 59# $1, if provided, should be a MAJOR, MAJOR.MINOR or MAJOR.MINOR.PATCH 60# version number 61function kernel_version 62{ 63 typeset ver="$1" 64 65 [ -z "$ver" ] && case "$UNAME" in 66 Linux) 67 # Linux version numbers are X.Y.Z followed by optional 68 # vendor/distro specific stuff 69 # RHEL7: 3.10.0-1160.108.1.el7.x86_64 70 # Fedora 37: 6.5.12-100.fc37.x86_64 71 # Debian 12.6: 6.1.0-22-amd64 72 ver=$(uname -r | grep -Eo "^[0-9]+\.[0-9]+\.[0-9]+") 73 ;; 74 FreeBSD) 75 # FreeBSD version numbers are X.Y-BRANCH-pZ. Depending on 76 # branch, -pZ may not be present, but this is typically only 77 # on pre-release or true .0 releases, so can be assumed 0 78 # if not present. 79 # eg: 80 # 13.2-RELEASE-p4 81 # 14.1-RELEASE 82 # 15.0-CURRENT 83 ver=$(uname -r | \ 84 grep -Eo "[0-9]+\.[0-9]+(-[A-Z0-9]+-p[0-9]+)?" | \ 85 sed -E "s/-[^-]+-p/./") 86 ;; 87 *) 88 # Unknown system 89 log_fail "Don't know how to get kernel version for '$UNAME'" 90 ;; 91 esac 92 93 typeset version major minor _ 94 IFS='.' read -r version major minor _ <<<"$ver" 95 96 [ -z "$version" ] && version=0 97 [ -z "$major" ] && major=0 98 [ -z "$minor" ] && minor=0 99 100 echo $((version * 100000 + major * 1000 + minor)) 101} 102 103# Linux kernel version comparison function 104# 105# $1 Linux version ("4.10", "2.6.32") or blank for installed Linux version 106# 107# Used for comparison: if [ $(linux_version) -ge $(linux_version "2.6.32") ] 108function linux_version { 109 kernel_version "$1" 110} 111 112# FreeBSD version comparison function 113# 114# $1 FreeBSD version ("13.2", "14.0") or blank for installed FreeBSD version 115# 116# Used for comparison: if [ $(freebsd_version) -ge $(freebsd_version "13.2") ] 117function freebsd_version { 118 kernel_version "$1" 119} 120 121# Determine if this is a Linux test system 122# 123# Return 0 if platform Linux, 1 if otherwise 124 125function is_linux 126{ 127 [ "$UNAME" = "Linux" ] 128} 129 130# Determine if this is an illumos test system 131# 132# Return 0 if platform illumos, 1 if otherwise 133function is_illumos 134{ 135 [ "$UNAME" = "illumos" ] 136} 137 138# Determine if this is a FreeBSD test system 139# 140# Return 0 if platform FreeBSD, 1 if otherwise 141 142function is_freebsd 143{ 144 [ "$UNAME" = "FreeBSD" ] 145} 146 147# Determine if this is a 32-bit system 148# 149# Return 0 if platform is 32-bit, 1 if otherwise 150 151function is_32bit 152{ 153 [ $(getconf LONG_BIT) = "32" ] 154} 155 156# Determine if kmemleak is enabled 157# 158# Return 0 if kmemleak is enabled, 1 if otherwise 159 160function is_kmemleak 161{ 162 is_linux && [ -e /sys/kernel/debug/kmemleak ] 163} 164 165# Determine whether a dataset is mounted 166# 167# $1 dataset name 168# $2 filesystem type; optional - defaulted to zfs 169# 170# Return 0 if dataset is mounted; 1 if unmounted; 2 on error 171 172function ismounted 173{ 174 typeset fstype=$2 175 [[ -z $fstype ]] && fstype=zfs 176 typeset out dir name 177 178 case $fstype in 179 zfs) 180 if [[ "$1" == "/"* ]] ; then 181 ! zfs mount | awk -v fs="$1" '$2 == fs {exit 1}' 182 else 183 ! zfs mount | awk -v ds="$1" '$1 == ds {exit 1}' 184 fi 185 ;; 186 ufs|nfs) 187 if is_freebsd; then 188 mount -pt $fstype | while read dev dir _t _flags; do 189 [[ "$1" == "$dev" || "$1" == "$dir" ]] && return 0 190 done 191 else 192 out=$(df -F $fstype $1 2>/dev/null) || return 193 194 dir=${out%%\(*} 195 dir=${dir%% *} 196 name=${out##*\(} 197 name=${name%%\)*} 198 name=${name%% *} 199 200 [[ "$1" == "$dir" || "$1" == "$name" ]] && return 0 201 fi 202 ;; 203 ext*) 204 df -t $fstype $1 > /dev/null 2>&1 205 ;; 206 zvol) 207 if [[ -L "$ZVOL_DEVDIR/$1" ]]; then 208 link=$(readlink -f $ZVOL_DEVDIR/$1) 209 [[ -n "$link" ]] && \ 210 mount | grep -q "^$link" && \ 211 return 0 212 fi 213 ;; 214 *) 215 false 216 ;; 217 esac 218} 219 220# Return 0 if a dataset is mounted; 1 otherwise 221# 222# $1 dataset name 223# $2 filesystem type; optional - defaulted to zfs 224 225function mounted 226{ 227 ismounted $1 $2 228} 229 230# Return 0 if a dataset is unmounted; 1 otherwise 231# 232# $1 dataset name 233# $2 filesystem type; optional - defaulted to zfs 234 235function unmounted 236{ 237 ! ismounted $1 $2 238} 239 240function default_setup 241{ 242 default_setup_noexit "$@" 243 244 log_pass 245} 246 247function default_setup_no_mountpoint 248{ 249 default_setup_noexit "$1" "$2" "$3" "yes" 250 251 log_pass 252} 253 254# 255# Given a list of disks, setup storage pools and datasets. 256# 257function default_setup_noexit 258{ 259 typeset disklist=$1 260 typeset container=$2 261 typeset volume=$3 262 typeset no_mountpoint=$4 263 log_note begin default_setup_noexit 264 265 if [[ "/${TESTPOOL}" -ef / ]]; then 266 log_fail "TESTPOOL is not set" 267 fi 268 269 if is_global_zone; then 270 if poolexists $TESTPOOL ; then 271 destroy_pool $TESTPOOL 272 fi 273 [[ -d /$TESTPOOL ]] && rm -rf /$TESTPOOL 274 log_must zpool create -f $TESTPOOL $disklist 275 else 276 reexport_pool 277 fi 278 279 rm -rf $TESTDIR || log_unresolved Could not remove $TESTDIR 280 mkdir -p $TESTDIR || log_unresolved Could not create $TESTDIR 281 282 log_must zfs create $TESTPOOL/$TESTFS 283 if [[ -z $no_mountpoint ]]; then 284 log_must zfs set mountpoint=$TESTDIR $TESTPOOL/$TESTFS 285 fi 286 287 if [[ -n $container ]]; then 288 rm -rf $TESTDIR1 || \ 289 log_unresolved Could not remove $TESTDIR1 290 mkdir -p $TESTDIR1 || \ 291 log_unresolved Could not create $TESTDIR1 292 293 log_must zfs create $TESTPOOL/$TESTCTR 294 log_must zfs set canmount=off $TESTPOOL/$TESTCTR 295 log_must zfs create $TESTPOOL/$TESTCTR/$TESTFS1 296 if [[ -z $no_mountpoint ]]; then 297 log_must zfs set mountpoint=$TESTDIR1 \ 298 $TESTPOOL/$TESTCTR/$TESTFS1 299 fi 300 fi 301 302 if [[ -n $volume ]]; then 303 if is_global_zone ; then 304 log_must zfs create -V $VOLSIZE $TESTPOOL/$TESTVOL 305 block_device_wait 306 else 307 log_must zfs create $TESTPOOL/$TESTVOL 308 fi 309 fi 310} 311 312# 313# Given a list of disks, setup a storage pool, file system and 314# a container. 315# 316function default_container_setup 317{ 318 typeset disklist=$1 319 320 default_setup "$disklist" "true" 321} 322 323# 324# Given a list of disks, setup a storage pool,file system 325# and a volume. 326# 327function default_volume_setup 328{ 329 typeset disklist=$1 330 331 default_setup "$disklist" "" "true" 332} 333 334# 335# Given a list of disks, setup a storage pool,file system, 336# a container and a volume. 337# 338function default_container_volume_setup 339{ 340 typeset disklist=$1 341 342 default_setup "$disklist" "true" "true" 343} 344 345# 346# Create a snapshot on a filesystem or volume. Defaultly create a snapshot on 347# filesystem 348# 349# $1 Existing filesystem or volume name. Default, $TESTPOOL/$TESTFS 350# $2 snapshot name. Default, $TESTSNAP 351# 352function create_snapshot 353{ 354 typeset fs_vol=${1:-$TESTPOOL/$TESTFS} 355 typeset snap=${2:-$TESTSNAP} 356 357 [[ -z $fs_vol ]] && log_fail "Filesystem or volume's name is undefined." 358 [[ -z $snap ]] && log_fail "Snapshot's name is undefined." 359 360 if snapexists $fs_vol@$snap; then 361 log_fail "$fs_vol@$snap already exists." 362 fi 363 datasetexists $fs_vol || \ 364 log_fail "$fs_vol must exist." 365 366 log_must zfs snapshot $fs_vol@$snap 367} 368 369# 370# Create a clone from a snapshot, default clone name is $TESTCLONE. 371# 372# $1 Existing snapshot, $TESTPOOL/$TESTFS@$TESTSNAP is default. 373# $2 Clone name, $TESTPOOL/$TESTCLONE is default. 374# 375function create_clone # snapshot clone 376{ 377 typeset snap=${1:-$TESTPOOL/$TESTFS@$TESTSNAP} 378 typeset clone=${2:-$TESTPOOL/$TESTCLONE} 379 380 [[ -z $snap ]] && \ 381 log_fail "Snapshot name is undefined." 382 [[ -z $clone ]] && \ 383 log_fail "Clone name is undefined." 384 385 log_must zfs clone $snap $clone 386} 387 388# 389# Create a bookmark of the given snapshot. Defaultly create a bookmark on 390# filesystem. 391# 392# $1 Existing filesystem or volume name. Default, $TESTFS 393# $2 Existing snapshot name. Default, $TESTSNAP 394# $3 bookmark name. Default, $TESTBKMARK 395# 396function create_bookmark 397{ 398 typeset fs_vol=${1:-$TESTFS} 399 typeset snap=${2:-$TESTSNAP} 400 typeset bkmark=${3:-$TESTBKMARK} 401 402 [[ -z $fs_vol ]] && log_fail "Filesystem or volume's name is undefined." 403 [[ -z $snap ]] && log_fail "Snapshot's name is undefined." 404 [[ -z $bkmark ]] && log_fail "Bookmark's name is undefined." 405 406 if bkmarkexists $fs_vol#$bkmark; then 407 log_fail "$fs_vol#$bkmark already exists." 408 fi 409 datasetexists $fs_vol || \ 410 log_fail "$fs_vol must exist." 411 snapexists $fs_vol@$snap || \ 412 log_fail "$fs_vol@$snap must exist." 413 414 log_must zfs bookmark $fs_vol@$snap $fs_vol#$bkmark 415} 416 417# 418# Create a temporary clone result of an interrupted resumable 'zfs receive' 419# $1 Destination filesystem name. Must not exist, will be created as the result 420# of this function along with its %recv temporary clone 421# $2 Source filesystem name. Must not exist, will be created and destroyed 422# 423function create_recv_clone 424{ 425 typeset recvfs="$1" 426 typeset sendfs="${2:-$TESTPOOL/create_recv_clone}" 427 typeset snap="$sendfs@snap1" 428 typeset incr="$sendfs@snap2" 429 typeset mountpoint="$TESTDIR/create_recv_clone" 430 typeset sendfile="$TESTDIR/create_recv_clone.zsnap" 431 432 [[ -z $recvfs ]] && log_fail "Recv filesystem's name is undefined." 433 434 datasetexists $recvfs && log_fail "Recv filesystem must not exist." 435 datasetexists $sendfs && log_fail "Send filesystem must not exist." 436 437 log_must zfs create -o compression=off -o mountpoint="$mountpoint" $sendfs 438 log_must zfs snapshot $snap 439 log_must eval "zfs send $snap | zfs recv -u $recvfs" 440 log_must mkfile 1m "$mountpoint/data" 441 log_must zfs snapshot $incr 442 log_must eval "zfs send -i $snap $incr | dd bs=10K count=1 \ 443 iflag=fullblock > $sendfile" 444 log_mustnot eval "zfs recv -su $recvfs < $sendfile" 445 destroy_dataset "$sendfs" "-r" 446 log_must rm -f "$sendfile" 447 448 if [[ $(get_prop 'inconsistent' "$recvfs/%recv") -ne 1 ]]; then 449 log_fail "Error creating temporary $recvfs/%recv clone" 450 fi 451} 452 453function default_mirror_setup 454{ 455 default_mirror_setup_noexit $1 $2 $3 456 457 log_pass 458} 459 460# 461# Given a pair of disks, set up a storage pool and dataset for the mirror 462# @parameters: $1 the primary side of the mirror 463# $2 the secondary side of the mirror 464# @uses: ZPOOL ZFS TESTPOOL TESTFS 465function default_mirror_setup_noexit 466{ 467 readonly func="default_mirror_setup_noexit" 468 typeset primary=$1 469 typeset secondary=$2 470 471 if [[ "/${TESTPOOL}" -ef / ]]; then 472 log_fail "TESTPOOL is not set" 473 fi 474 475 [[ -z $primary ]] && \ 476 log_fail "$func: No parameters passed" 477 [[ -z $secondary ]] && \ 478 log_fail "$func: No secondary partition passed" 479 [[ -d /$TESTPOOL ]] && rm -rf /$TESTPOOL 480 log_must zpool create -f $TESTPOOL mirror $@ 481 log_must zfs create $TESTPOOL/$TESTFS 482 log_must zfs set mountpoint=$TESTDIR $TESTPOOL/$TESTFS 483} 484 485# 486# Destroy the configured testpool mirrors. 487# the mirrors are of the form ${TESTPOOL}{number} 488# @uses: ZPOOL ZFS TESTPOOL 489function destroy_mirrors 490{ 491 default_cleanup_noexit 492 493 log_pass 494} 495 496function default_raidz_setup 497{ 498 default_raidz_setup_noexit "$*" 499 500 log_pass 501} 502 503# 504# Given a minimum of two disks, set up a storage pool and dataset for the raid-z 505# $1 the list of disks 506# 507function default_raidz_setup_noexit 508{ 509 typeset disklist="$*" 510 disks=(${disklist[*]}) 511 512 if [[ "/${TESTPOOL}" -ef / ]]; then 513 log_fail "TESTPOOL is not set" 514 fi 515 516 if [[ ${#disks[*]} -lt 2 ]]; then 517 log_fail "A raid-z requires a minimum of two disks." 518 fi 519 520 [[ -d /$TESTPOOL ]] && rm -rf /$TESTPOOL 521 log_must zpool create -f $TESTPOOL raidz $disklist 522 log_must zfs create $TESTPOOL/$TESTFS 523 log_must zfs set mountpoint=$TESTDIR $TESTPOOL/$TESTFS 524} 525 526# 527# Common function used to cleanup storage pools and datasets. 528# 529# Invoked at the start of the test suite to ensure the system 530# is in a known state, and also at the end of each set of 531# sub-tests to ensure errors from one set of tests doesn't 532# impact the execution of the next set. 533 534function default_cleanup 535{ 536 default_cleanup_noexit 537 538 log_pass 539} 540 541# 542# Utility function used to list all available pool names. 543# 544# NOTE: $KEEP is a variable containing pool names, separated by a newline 545# character, that must be excluded from the returned list. 546# 547function get_all_pools 548{ 549 zpool list -H -o name | grep -Fvx "$KEEP" | grep -v "$NO_POOLS" 550} 551 552function default_cleanup_noexit 553{ 554 typeset pool="" 555 # 556 # Destroying the pool will also destroy any 557 # filesystems it contains. 558 # 559 if is_global_zone; then 560 zfs unmount -a > /dev/null 2>&1 561 ALL_POOLS=$(get_all_pools) 562 # Here, we loop through the pools we're allowed to 563 # destroy, only destroying them if it's safe to do 564 # so. 565 while [ -n "${ALL_POOLS}" ] 566 do 567 for pool in ${ALL_POOLS} 568 do 569 if safe_to_destroy_pool $pool ; 570 then 571 destroy_pool $pool 572 fi 573 done 574 ALL_POOLS=$(get_all_pools) 575 done 576 577 zfs mount -a 578 else 579 typeset fs="" 580 for fs in $(zfs list -H -o name \ 581 | grep "^$ZONE_POOL/$ZONE_CTR[01234]/"); do 582 destroy_dataset "$fs" "-Rf" 583 done 584 585 # Need cleanup here to avoid garbage dir left. 586 for fs in $(zfs list -H -o name); do 587 [[ $fs == /$ZONE_POOL ]] && continue 588 [[ -d $fs ]] && log_must rm -rf $fs/* 589 done 590 591 # 592 # Reset the $ZONE_POOL/$ZONE_CTR[01234] file systems property to 593 # the default value 594 # 595 for fs in $(zfs list -H -o name); do 596 if [[ $fs == $ZONE_POOL/$ZONE_CTR[01234] ]]; then 597 log_must zfs set reservation=none $fs 598 log_must zfs set recordsize=128K $fs 599 log_must zfs set mountpoint=/$fs $fs 600 typeset enc=$(get_prop encryption $fs) 601 if [ -z "$enc" ] || [ "$enc" = "off" ]; then 602 log_must zfs set checksum=on $fs 603 fi 604 log_must zfs set compression=off $fs 605 log_must zfs set atime=on $fs 606 log_must zfs set devices=off $fs 607 log_must zfs set exec=on $fs 608 log_must zfs set setuid=on $fs 609 log_must zfs set readonly=off $fs 610 log_must zfs set snapdir=hidden $fs 611 log_must zfs set aclmode=groupmask $fs 612 log_must zfs set aclinherit=secure $fs 613 fi 614 done 615 fi 616 617 [[ -d $TESTDIR ]] && \ 618 log_must rm -rf $TESTDIR 619 620 disk1=${DISKS%% *} 621 if is_mpath_device $disk1; then 622 delete_partitions 623 fi 624 625 rm -f $TEST_BASE_DIR/{err,out} 626} 627 628 629# 630# Common function used to cleanup storage pools, file systems 631# and containers. 632# 633function default_container_cleanup 634{ 635 if ! is_global_zone; then 636 reexport_pool 637 fi 638 639 ismounted $TESTPOOL/$TESTCTR/$TESTFS1 && 640 log_must zfs unmount $TESTPOOL/$TESTCTR/$TESTFS1 641 642 destroy_dataset "$TESTPOOL/$TESTCTR/$TESTFS1" "-R" 643 destroy_dataset "$TESTPOOL/$TESTCTR" "-Rf" 644 645 [[ -e $TESTDIR1 ]] && \ 646 log_must rm -rf $TESTDIR1 647 648 default_cleanup 649} 650 651# 652# Common function used to cleanup snapshot of file system or volume. Default to 653# delete the file system's snapshot 654# 655# $1 snapshot name 656# 657function destroy_snapshot 658{ 659 typeset snap=${1:-$TESTPOOL/$TESTFS@$TESTSNAP} 660 661 if ! snapexists $snap; then 662 log_fail "'$snap' does not exist." 663 fi 664 665 # 666 # For the sake of the value which come from 'get_prop' is not equal 667 # to the really mountpoint when the snapshot is unmounted. So, firstly 668 # check and make sure this snapshot's been mounted in current system. 669 # 670 typeset mtpt="" 671 if ismounted $snap; then 672 mtpt=$(get_prop mountpoint $snap) 673 fi 674 675 destroy_dataset "$snap" 676 [[ $mtpt != "" && -d $mtpt ]] && \ 677 log_must rm -rf $mtpt 678} 679 680# 681# Common function used to cleanup clone. 682# 683# $1 clone name 684# 685function destroy_clone 686{ 687 typeset clone=${1:-$TESTPOOL/$TESTCLONE} 688 689 if ! datasetexists $clone; then 690 log_fail "'$clone' does not existed." 691 fi 692 693 # With the same reason in destroy_snapshot 694 typeset mtpt="" 695 if ismounted $clone; then 696 mtpt=$(get_prop mountpoint $clone) 697 fi 698 699 destroy_dataset "$clone" 700 [[ $mtpt != "" && -d $mtpt ]] && \ 701 log_must rm -rf $mtpt 702} 703 704# 705# Common function used to cleanup bookmark of file system or volume. Default 706# to delete the file system's bookmark. 707# 708# $1 bookmark name 709# 710function destroy_bookmark 711{ 712 typeset bkmark=${1:-$TESTPOOL/$TESTFS#$TESTBKMARK} 713 714 if ! bkmarkexists $bkmark; then 715 log_fail "'$bkmarkp' does not existed." 716 fi 717 718 destroy_dataset "$bkmark" 719} 720 721# Return 0 if a snapshot exists; $? otherwise 722# 723# $1 - snapshot name 724 725function snapexists 726{ 727 zfs list -H -t snapshot "$1" > /dev/null 2>&1 728} 729 730# 731# Return 0 if a bookmark exists; $? otherwise 732# 733# $1 - bookmark name 734# 735function bkmarkexists 736{ 737 zfs list -H -t bookmark "$1" > /dev/null 2>&1 738} 739 740# 741# Return 0 if a hold exists; $? otherwise 742# 743# $1 - hold tag 744# $2 - snapshot name 745# 746function holdexists 747{ 748 ! zfs holds "$2" | awk -v t="$1" '$2 ~ t { exit 1 }' 749} 750 751# 752# Set a property to a certain value on a dataset. 753# Sets a property of the dataset to the value as passed in. 754# @param: 755# $1 dataset who's property is being set 756# $2 property to set 757# $3 value to set property to 758# @return: 759# 0 if the property could be set. 760# non-zero otherwise. 761# @use: ZFS 762# 763function dataset_setprop 764{ 765 typeset fn=dataset_setprop 766 767 if (($# < 3)); then 768 log_note "$fn: Insufficient parameters (need 3, had $#)" 769 return 1 770 fi 771 typeset output= 772 output=$(zfs set $2=$3 $1 2>&1) 773 typeset rv=$? 774 if ((rv != 0)); then 775 log_note "Setting property on $1 failed." 776 log_note "property $2=$3" 777 log_note "Return Code: $rv" 778 log_note "Output: $output" 779 return $rv 780 fi 781 return 0 782} 783 784# 785# Check a numeric assertion 786# @parameter: $@ the assertion to check 787# @output: big loud notice if assertion failed 788# @use: log_fail 789# 790function assert 791{ 792 (($@)) || log_fail "$@" 793} 794 795# 796# Function to format partition size of a disk 797# Given a disk cxtxdx reduces all partitions 798# to 0 size 799# 800function zero_partitions #<whole_disk_name> 801{ 802 typeset diskname=$1 803 typeset i 804 805 if is_freebsd; then 806 gpart destroy -F $diskname 807 elif is_linux; then 808 DSK=$DEV_DSKDIR/$diskname 809 DSK=$(echo $DSK | sed -e "s|//|/|g") 810 log_must parted $DSK -s -- mklabel gpt 811 blockdev --rereadpt $DSK 2>/dev/null 812 block_device_wait 813 else 814 for i in 0 1 3 4 5 6 7 815 do 816 log_must set_partition $i "" 0mb $diskname 817 done 818 fi 819 820 return 0 821} 822 823# 824# Given a slice, size and disk, this function 825# formats the slice to the specified size. 826# Size should be specified with units as per 827# the `format` command requirements eg. 100mb 3gb 828# 829# NOTE: This entire interface is problematic for the Linux parted utility 830# which requires the end of the partition to be specified. It would be 831# best to retire this interface and replace it with something more flexible. 832# At the moment a best effort is made. 833# 834# arguments: <slice_num> <slice_start> <size_plus_units> <whole_disk_name> 835function set_partition 836{ 837 typeset -i slicenum=$1 838 typeset start=$2 839 typeset size=$3 840 typeset disk=${4#$DEV_DSKDIR/} 841 disk=${disk#$DEV_RDSKDIR/} 842 843 case "$UNAME" in 844 Linux) 845 if [[ -z $size || -z $disk ]]; then 846 log_fail "The size or disk name is unspecified." 847 fi 848 disk=$DEV_DSKDIR/$disk 849 typeset size_mb=${size%%[mMgG]} 850 851 size_mb=${size_mb%%[mMgG][bB]} 852 if [[ ${size:1:1} == 'g' ]]; then 853 ((size_mb = size_mb * 1024)) 854 fi 855 856 # Create GPT partition table when setting slice 0 or 857 # when the device doesn't already contain a GPT label. 858 parted $disk -s -- print 1 >/dev/null 859 typeset ret_val=$? 860 if [[ $slicenum -eq 0 || $ret_val -ne 0 ]]; then 861 if ! parted $disk -s -- mklabel gpt; then 862 log_note "Failed to create GPT partition table on $disk" 863 return 1 864 fi 865 fi 866 867 # When no start is given align on the first cylinder. 868 if [[ -z "$start" ]]; then 869 start=1 870 fi 871 872 # Determine the cylinder size for the device and using 873 # that calculate the end offset in cylinders. 874 typeset -i cly_size_kb=0 875 cly_size_kb=$(parted -m $disk -s -- unit cyl print | 876 awk -F '[:k.]' 'NR == 3 {print $4}') 877 ((end = (size_mb * 1024 / cly_size_kb) + start)) 878 879 parted $disk -s -- \ 880 mkpart part$slicenum ${start}cyl ${end}cyl 881 typeset ret_val=$? 882 if [[ $ret_val -ne 0 ]]; then 883 log_note "Failed to create partition $slicenum on $disk" 884 return 1 885 fi 886 887 blockdev --rereadpt $disk 2>/dev/null 888 block_device_wait $disk 889 ;; 890 FreeBSD) 891 if [[ -z $size || -z $disk ]]; then 892 log_fail "The size or disk name is unspecified." 893 fi 894 disk=$DEV_DSKDIR/$disk 895 896 if [[ $slicenum -eq 0 ]] || ! gpart show $disk >/dev/null 2>&1; then 897 gpart destroy -F $disk >/dev/null 2>&1 898 if ! gpart create -s GPT $disk; then 899 log_note "Failed to create GPT partition table on $disk" 900 return 1 901 fi 902 fi 903 904 typeset index=$((slicenum + 1)) 905 906 if [[ -n $start ]]; then 907 start="-b $start" 908 fi 909 gpart add -t freebsd-zfs $start -s $size -i $index $disk 910 if [[ $ret_val -ne 0 ]]; then 911 log_note "Failed to create partition $slicenum on $disk" 912 return 1 913 fi 914 915 block_device_wait $disk 916 ;; 917 *) 918 if [[ -z $slicenum || -z $size || -z $disk ]]; then 919 log_fail "The slice, size or disk name is unspecified." 920 fi 921 922 typeset format_file="$TEST_BASE_DIR"/format_in.$$ 923 924 echo "partition" >$format_file 925 echo "$slicenum" >> $format_file 926 echo "" >> $format_file 927 echo "" >> $format_file 928 echo "$start" >> $format_file 929 echo "$size" >> $format_file 930 echo "label" >> $format_file 931 echo "" >> $format_file 932 echo "q" >> $format_file 933 echo "q" >> $format_file 934 935 format -e -s -d $disk -f $format_file 936 typeset ret_val=$? 937 rm -f $format_file 938 ;; 939 esac 940 941 if [[ $ret_val -ne 0 ]]; then 942 log_note "Unable to format $disk slice $slicenum to $size" 943 return 1 944 fi 945 return 0 946} 947 948# 949# Delete all partitions on all disks - this is specifically for the use of multipath 950# devices which currently can only be used in the test suite as raw/un-partitioned 951# devices (ie a zpool cannot be created on a whole mpath device that has partitions) 952# 953function delete_partitions 954{ 955 typeset disk 956 957 if [[ -z $DISKSARRAY ]]; then 958 DISKSARRAY=$DISKS 959 fi 960 961 if is_linux; then 962 typeset -i part 963 for disk in $DISKSARRAY; do 964 for (( part = 1; part < MAX_PARTITIONS; part++ )); do 965 typeset partition=${disk}${SLICE_PREFIX}${part} 966 parted $DEV_DSKDIR/$disk -s rm $part > /dev/null 2>&1 967 if lsblk | grep -qF ${partition}; then 968 log_fail "Partition ${partition} not deleted" 969 else 970 log_note "Partition ${partition} deleted" 971 fi 972 done 973 done 974 elif is_freebsd; then 975 for disk in $DISKSARRAY; do 976 if gpart destroy -F $disk; then 977 log_note "Partitions for ${disk} deleted" 978 else 979 log_fail "Partitions for ${disk} not deleted" 980 fi 981 done 982 fi 983} 984 985# 986# Get the end cyl of the given slice 987# 988function get_endslice #<disk> <slice> 989{ 990 typeset disk=$1 991 typeset slice=$2 992 if [[ -z $disk || -z $slice ]] ; then 993 log_fail "The disk name or slice number is unspecified." 994 fi 995 996 case "$UNAME" in 997 Linux) 998 endcyl=$(parted -s $DEV_DSKDIR/$disk -- unit cyl print | \ 999 awk "/part${slice}/"' {sub(/cyl/, "", $3); print $3}') 1000 ((endcyl = (endcyl + 1))) 1001 ;; 1002 FreeBSD) 1003 disk=${disk#/dev/zvol/} 1004 disk=${disk%p*} 1005 slice=$((slice + 1)) 1006 endcyl=$(gpart show $disk | \ 1007 awk -v slice=$slice '$3 == slice { print $1 + $2 }') 1008 ;; 1009 *) 1010 disk=${disk#/dev/dsk/} 1011 disk=${disk#/dev/rdsk/} 1012 disk=${disk%s*} 1013 1014 typeset -i ratio=0 1015 ratio=$(prtvtoc /dev/rdsk/${disk}s2 | \ 1016 awk '/sectors\/cylinder/ {print $2}') 1017 1018 if ((ratio == 0)); then 1019 return 1020 fi 1021 1022 typeset -i endcyl=$(prtvtoc -h /dev/rdsk/${disk}s2 | 1023 awk -v token="$slice" '$1 == token {print $6}') 1024 1025 ((endcyl = (endcyl + 1) / ratio)) 1026 ;; 1027 esac 1028 1029 echo $endcyl 1030} 1031 1032 1033# 1034# Given a size,disk and total slice number, this function formats the 1035# disk slices from 0 to the total slice number with the same specified 1036# size. 1037# 1038function partition_disk #<slice_size> <whole_disk_name> <total_slices> 1039{ 1040 typeset -i i=0 1041 typeset slice_size=$1 1042 typeset disk_name=$2 1043 typeset total_slices=$3 1044 typeset cyl 1045 1046 zero_partitions $disk_name 1047 while ((i < $total_slices)); do 1048 if ! is_linux; then 1049 if ((i == 2)); then 1050 ((i = i + 1)) 1051 continue 1052 fi 1053 fi 1054 log_must set_partition $i "$cyl" $slice_size $disk_name 1055 cyl=$(get_endslice $disk_name $i) 1056 ((i = i+1)) 1057 done 1058} 1059 1060# 1061# This function continues to write to a filenum number of files into dirnum 1062# number of directories until either file_write returns an error or the 1063# maximum number of files per directory have been written. 1064# 1065# Usage: 1066# fill_fs [destdir] [dirnum] [filenum] [bytes] [num_writes] [data] 1067# 1068# Return value: 0 on success 1069# non 0 on error 1070# 1071# Where : 1072# destdir: is the directory where everything is to be created under 1073# dirnum: the maximum number of subdirectories to use, -1 no limit 1074# filenum: the maximum number of files per subdirectory 1075# bytes: number of bytes to write 1076# num_writes: number of types to write out bytes 1077# data: the data that will be written 1078# 1079# E.g. 1080# fill_fs /testdir 20 25 1024 256 0 1081# 1082# Note: bytes * num_writes equals the size of the testfile 1083# 1084function fill_fs # destdir dirnum filenum bytes num_writes data 1085{ 1086 typeset destdir=${1:-$TESTDIR} 1087 typeset -i dirnum=${2:-50} 1088 typeset -i filenum=${3:-50} 1089 typeset -i bytes=${4:-8192} 1090 typeset -i num_writes=${5:-10240} 1091 typeset data=${6:-"R"} 1092 1093 mkdir -p $destdir/{1..$dirnum} 1094 for f in $destdir/{1..$dirnum}/$TESTFILE{1..$filenum}; do 1095 file_write -o create -f $f -b $bytes -c $num_writes -d $data \ 1096 || return 1097 done 1098} 1099 1100# Get the specified dataset property in parsable format or fail 1101function get_prop # property dataset 1102{ 1103 typeset prop=$1 1104 typeset dataset=$2 1105 1106 zfs get -Hpo value "$prop" "$dataset" || log_fail "zfs get $prop $dataset" 1107} 1108 1109# Get the specified pool property in parsable format or fail 1110function get_pool_prop # property pool 1111{ 1112 typeset prop=$1 1113 typeset pool=$2 1114 1115 zpool get -Hpo value "$prop" "$pool" || log_fail "zpool get $prop $pool" 1116} 1117 1118# Get the specified vdev property in parsable format or fail 1119function get_vdev_prop 1120{ 1121 typeset prop="$1" 1122 typeset pool="$2" 1123 typeset vdev="$3" 1124 1125 zpool get -Hpo value "$prop" "$pool" "$vdev" || log_fail "zpool get $prop $pool $vdev" 1126} 1127 1128# Return 0 if a pool exists; $? otherwise 1129# 1130# $1 - pool name 1131 1132function poolexists 1133{ 1134 typeset pool=$1 1135 1136 if [[ -z $pool ]]; then 1137 log_note "No pool name given." 1138 return 1 1139 fi 1140 1141 zpool get name "$pool" > /dev/null 2>&1 1142} 1143 1144# Return 0 if all the specified datasets exist; $? otherwise 1145# 1146# $1-n dataset name 1147function datasetexists 1148{ 1149 if (($# == 0)); then 1150 log_note "No dataset name given." 1151 return 1 1152 fi 1153 1154 zfs get name "$@" > /dev/null 2>&1 1155} 1156 1157# return 0 if none of the specified datasets exists, otherwise return 1. 1158# 1159# $1-n dataset name 1160function datasetnonexists 1161{ 1162 if (($# == 0)); then 1163 log_note "No dataset name given." 1164 return 1 1165 fi 1166 1167 while (($# > 0)); do 1168 zfs list -H -t filesystem,snapshot,volume $1 > /dev/null 2>&1 \ 1169 && return 1 1170 shift 1171 done 1172 1173 return 0 1174} 1175 1176# Check if the specified dataset property has the expected value or fail 1177function dataset_has_prop # property expected_value dataset 1178{ 1179 typeset prop=$1 1180 typeset expected=$2 1181 typeset dataset=$3 1182 1183 typeset value="" 1184 1185 value="$(get_prop "$prop" "$dataset")" 1186 [[ "$value" == "$expected" ]] || { 1187 log_note "dataset $dataset: property $prop == $value (!= $expected)" 1188 return 1 1189 } 1190} 1191 1192# FreeBSD breaks exports(5) at whitespace and doesn't process escapes 1193# Solaris just breaks 1194# 1195# cf. https://github.com/openzfs/zfs/pull/13165#issuecomment-1059845807 1196# 1197# Linux can have spaces (which are \OOO-escaped), 1198# but can't have backslashes because they're parsed recursively 1199function shares_can_have_whitespace 1200{ 1201 is_linux 1202} 1203 1204function is_shared_freebsd 1205{ 1206 typeset fs=$1 1207 1208 pgrep -q mountd && showmount -E | grep -qx "$fs" 1209} 1210 1211function is_shared_illumos 1212{ 1213 typeset fs=$1 1214 typeset mtpt 1215 1216 for mtpt in `share | awk '{print $2}'` ; do 1217 if [[ $mtpt == $fs ]] ; then 1218 return 0 1219 fi 1220 done 1221 1222 typeset stat=$(svcs -H -o STA nfs/server:default) 1223 if [[ $stat != "ON" ]]; then 1224 log_note "Current nfs/server status: $stat" 1225 fi 1226 1227 return 1 1228} 1229 1230function is_shared_linux 1231{ 1232 typeset fs=$1 1233 ! exportfs -s | awk -v fs="${fs//\\/\\\\}" '/^\// && $1 == fs {exit 1}' 1234} 1235 1236# 1237# Given a mountpoint, or a dataset name, determine if it is shared via NFS. 1238# 1239# Returns 0 if shared, 1 otherwise. 1240# 1241function is_shared 1242{ 1243 typeset fs=$1 1244 typeset mtpt 1245 1246 if [[ $fs != "/"* ]] ; then 1247 if datasetnonexists "$fs" ; then 1248 return 1 1249 else 1250 mtpt=$(get_prop mountpoint "$fs") 1251 case "$mtpt" in 1252 none|legacy|-) return 1 1253 ;; 1254 *) fs=$mtpt 1255 ;; 1256 esac 1257 fi 1258 fi 1259 1260 case "$UNAME" in 1261 FreeBSD) is_shared_freebsd "$fs" ;; 1262 Linux) is_shared_linux "$fs" ;; 1263 *) is_shared_illumos "$fs" ;; 1264 esac 1265} 1266 1267function is_exported_illumos 1268{ 1269 typeset fs=$1 1270 typeset mtpt _ 1271 1272 while read -r mtpt _; do 1273 [ "$mtpt" = "$fs" ] && return 1274 done < /etc/dfs/sharetab 1275 1276 return 1 1277} 1278 1279function is_exported_freebsd 1280{ 1281 typeset fs=$1 1282 typeset mtpt _ 1283 1284 while read -r mtpt _; do 1285 [ "$mtpt" = "$fs" ] && return 1286 done < /etc/zfs/exports 1287 1288 return 1 1289} 1290 1291function is_exported_linux 1292{ 1293 typeset fs=$1 1294 typeset mtpt _ 1295 1296 while read -r mtpt _; do 1297 [ "$(printf "$mtpt")" = "$fs" ] && return 1298 done < /etc/exports.d/zfs.exports 1299 1300 return 1 1301} 1302 1303# 1304# Given a mountpoint, or a dataset name, determine if it is exported via 1305# the os-specific NFS exports file. 1306# 1307# Returns 0 if exported, 1 otherwise. 1308# 1309function is_exported 1310{ 1311 typeset fs=$1 1312 typeset mtpt 1313 1314 if [[ $fs != "/"* ]] ; then 1315 if datasetnonexists "$fs" ; then 1316 return 1 1317 else 1318 mtpt=$(get_prop mountpoint "$fs") 1319 case $mtpt in 1320 none|legacy|-) return 1 1321 ;; 1322 *) fs=$mtpt 1323 ;; 1324 esac 1325 fi 1326 fi 1327 1328 case "$UNAME" in 1329 FreeBSD) is_exported_freebsd "$fs" ;; 1330 Linux) is_exported_linux "$fs" ;; 1331 *) is_exported_illumos "$fs" ;; 1332 esac 1333} 1334 1335# 1336# Given a dataset name determine if it is shared via SMB. 1337# 1338# Returns 0 if shared, 1 otherwise. 1339# 1340function is_shared_smb 1341{ 1342 typeset fs=$1 1343 1344 datasetexists "$fs" || return 1345 1346 if is_linux; then 1347 net usershare list | grep -xFq "${fs//[-\/]/_}" 1348 else 1349 log_note "SMB on $UNAME currently unsupported by the test framework" 1350 return 1 1351 fi 1352} 1353 1354# 1355# Given a mountpoint, determine if it is not shared via NFS. 1356# 1357# Returns 0 if not shared, 1 otherwise. 1358# 1359function not_shared 1360{ 1361 ! is_shared $1 1362} 1363 1364# 1365# Given a dataset determine if it is not shared via SMB. 1366# 1367# Returns 0 if not shared, 1 otherwise. 1368# 1369function not_shared_smb 1370{ 1371 ! is_shared_smb $1 1372} 1373 1374# 1375# Helper function to unshare a mountpoint. 1376# 1377function unshare_fs #fs 1378{ 1379 typeset fs=$1 1380 1381 if is_shared $fs || is_shared_smb $fs; then 1382 log_must zfs unshare $fs 1383 fi 1384} 1385 1386# 1387# Helper function to share a NFS mountpoint. 1388# 1389function share_nfs #fs 1390{ 1391 typeset fs=$1 1392 1393 is_shared "$fs" && return 1394 1395 case "$UNAME" in 1396 Linux) 1397 log_must exportfs "*:$fs" 1398 ;; 1399 FreeBSD) 1400 typeset mountd 1401 read -r mountd < /var/run/mountd.pid 1402 log_must eval "printf '%s\t\n' \"$fs\" >> /etc/zfs/exports" 1403 log_must kill -s HUP "$mountd" 1404 ;; 1405 *) 1406 log_must share -F nfs "$fs" 1407 ;; 1408 esac 1409 1410 return 0 1411} 1412 1413# 1414# Helper function to unshare a NFS mountpoint. 1415# 1416function unshare_nfs #fs 1417{ 1418 typeset fs=$1 1419 1420 ! is_shared "$fs" && return 1421 1422 case "$UNAME" in 1423 Linux) 1424 log_must exportfs -u "*:$fs" 1425 ;; 1426 FreeBSD) 1427 typeset mountd 1428 read -r mountd < /var/run/mountd.pid 1429 awk -v fs="${fs//\\/\\\\}" '$1 != fs' /etc/zfs/exports > /etc/zfs/exports.$$ 1430 log_must mv /etc/zfs/exports.$$ /etc/zfs/exports 1431 log_must kill -s HUP "$mountd" 1432 ;; 1433 *) 1434 log_must unshare -F nfs $fs 1435 ;; 1436 esac 1437 1438 return 0 1439} 1440 1441# 1442# Helper function to show NFS shares. 1443# 1444function showshares_nfs 1445{ 1446 case "$UNAME" in 1447 Linux) 1448 exportfs -v 1449 ;; 1450 FreeBSD) 1451 showmount 1452 ;; 1453 *) 1454 share -F nfs 1455 ;; 1456 esac 1457} 1458 1459function check_nfs 1460{ 1461 case "$UNAME" in 1462 Linux) 1463 exportfs -s 1464 ;; 1465 FreeBSD) 1466 showmount -e 1467 ;; 1468 *) 1469 log_unsupported "Unknown platform" 1470 ;; 1471 esac || log_unsupported "The NFS utilities are not installed" 1472} 1473 1474# 1475# Check NFS server status and trigger it online. 1476# 1477function setup_nfs_server 1478{ 1479 # Cannot share directory in non-global zone. 1480 # 1481 if ! is_global_zone; then 1482 log_note "Cannot trigger NFS server by sharing in LZ." 1483 return 1484 fi 1485 1486 if is_linux; then 1487 # 1488 # Re-synchronize /var/lib/nfs/etab with /etc/exports and 1489 # /etc/exports.d./* to provide a clean test environment. 1490 # 1491 log_must exportfs -r 1492 1493 log_note "NFS server must be started prior to running ZTS." 1494 return 1495 elif is_freebsd; then 1496 log_must kill -s HUP $(</var/run/mountd.pid) 1497 1498 log_note "NFS server must be started prior to running ZTS." 1499 return 1500 fi 1501 1502 typeset nfs_fmri="svc:/network/nfs/server:default" 1503 if [[ $(svcs -Ho STA $nfs_fmri) != "ON" ]]; then 1504 # 1505 # Only really sharing operation can enable NFS server 1506 # to online permanently. 1507 # 1508 typeset dummy=/tmp/dummy 1509 1510 if [[ -d $dummy ]]; then 1511 log_must rm -rf $dummy 1512 fi 1513 1514 log_must mkdir $dummy 1515 log_must share $dummy 1516 1517 # 1518 # Waiting for fmri's status to be the final status. 1519 # Otherwise, in transition, an asterisk (*) is appended for 1520 # instances, unshare will reverse status to 'DIS' again. 1521 # 1522 # Waiting for 1's at least. 1523 # 1524 log_must sleep 1 1525 timeout=10 1526 while [[ timeout -ne 0 && $(svcs -Ho STA $nfs_fmri) == *'*' ]] 1527 do 1528 log_must sleep 1 1529 1530 ((timeout -= 1)) 1531 done 1532 1533 log_must unshare $dummy 1534 log_must rm -rf $dummy 1535 fi 1536 1537 log_note "Current NFS status: '$(svcs -Ho STA,FMRI $nfs_fmri)'" 1538} 1539 1540# 1541# To verify whether calling process is in global zone 1542# 1543# Return 0 if in global zone, 1 in non-global zone 1544# 1545function is_global_zone 1546{ 1547 if is_linux || is_freebsd; then 1548 return 0 1549 else 1550 typeset cur_zone=$(zonename 2>/dev/null) 1551 [ $cur_zone = "global" ] 1552 fi 1553} 1554 1555# 1556# Verify whether test is permitted to run from 1557# global zone, local zone, or both 1558# 1559# $1 zone limit, could be "global", "local", or "both"(no limit) 1560# 1561# Return 0 if permitted, otherwise exit with log_unsupported 1562# 1563function verify_runnable # zone limit 1564{ 1565 typeset limit=$1 1566 1567 [[ -z $limit ]] && return 0 1568 1569 if is_global_zone ; then 1570 case $limit in 1571 global|both) 1572 ;; 1573 local) log_unsupported "Test is unable to run from "\ 1574 "global zone." 1575 ;; 1576 *) log_note "Warning: unknown limit $limit - " \ 1577 "use both." 1578 ;; 1579 esac 1580 else 1581 case $limit in 1582 local|both) 1583 ;; 1584 global) log_unsupported "Test is unable to run from "\ 1585 "local zone." 1586 ;; 1587 *) log_note "Warning: unknown limit $limit - " \ 1588 "use both." 1589 ;; 1590 esac 1591 1592 reexport_pool 1593 fi 1594 1595 return 0 1596} 1597 1598# Return 0 if create successfully or the pool exists; $? otherwise 1599# Note: In local zones, this function should return 0 silently. 1600# 1601# $1 - pool name 1602# $2-n - [keyword] devs_list 1603 1604function create_pool #pool devs_list 1605{ 1606 typeset pool=${1%%/*} 1607 1608 shift 1609 1610 if [[ -z $pool ]]; then 1611 log_note "Missing pool name." 1612 return 1 1613 fi 1614 1615 if poolexists $pool ; then 1616 destroy_pool $pool 1617 fi 1618 1619 if is_global_zone ; then 1620 [[ -d /$pool ]] && rm -rf /$pool 1621 log_must zpool create -f $pool $@ 1622 fi 1623 1624 return 0 1625} 1626 1627# Return 0 if destroy successfully or the pool exists; $? otherwise 1628# Note: In local zones, this function should return 0 silently. 1629# 1630# $1 - pool name 1631# Destroy pool with the given parameters. 1632 1633function destroy_pool #pool 1634{ 1635 typeset pool=${1%%/*} 1636 typeset mtpt 1637 1638 if [[ -z $pool ]]; then 1639 log_note "No pool name given." 1640 return 1 1641 fi 1642 1643 if is_global_zone ; then 1644 if poolexists "$pool" ; then 1645 mtpt=$(get_prop mountpoint "$pool") 1646 1647 # At times, syseventd/udev activity can cause attempts 1648 # to destroy a pool to fail with EBUSY. We retry a few 1649 # times allowing failures before requiring the destroy 1650 # to succeed. 1651 log_must_busy zpool destroy -f $pool 1652 1653 [[ -d $mtpt ]] && \ 1654 log_must rm -rf $mtpt 1655 else 1656 log_note "Pool does not exist. ($pool)" 1657 return 1 1658 fi 1659 fi 1660 1661 return 0 1662} 1663 1664# Return 0 if created successfully; $? otherwise 1665# 1666# $1 - dataset name 1667# $2-n - dataset options 1668 1669function create_dataset #dataset dataset_options 1670{ 1671 typeset dataset=$1 1672 1673 shift 1674 1675 if [[ -z $dataset ]]; then 1676 log_note "Missing dataset name." 1677 return 1 1678 fi 1679 1680 if datasetexists $dataset ; then 1681 destroy_dataset $dataset 1682 fi 1683 1684 log_must zfs create $@ $dataset 1685 1686 return 0 1687} 1688 1689# Return 0 if destroy successfully or the dataset exists; $? otherwise 1690# Note: In local zones, this function should return 0 silently. 1691# 1692# $1 - dataset name 1693# $2 - custom arguments for zfs destroy 1694# Destroy dataset with the given parameters. 1695 1696function destroy_dataset # dataset [args] 1697{ 1698 typeset dataset=$1 1699 typeset mtpt 1700 typeset args=${2:-""} 1701 1702 if [[ -z $dataset ]]; then 1703 log_note "No dataset name given." 1704 return 1 1705 fi 1706 1707 if is_global_zone ; then 1708 if datasetexists "$dataset" ; then 1709 mtpt=$(get_prop mountpoint "$dataset") 1710 log_must_busy zfs destroy $args $dataset 1711 1712 [ -d $mtpt ] && log_must rm -rf $mtpt 1713 else 1714 log_note "Dataset does not exist. ($dataset)" 1715 return 1 1716 fi 1717 fi 1718 1719 return 0 1720} 1721 1722# 1723# Reexport TESTPOOL & TESTPOOL(1-4) 1724# 1725function reexport_pool 1726{ 1727 typeset -i cntctr=5 1728 typeset -i i=0 1729 1730 while ((i < cntctr)); do 1731 if ((i == 0)); then 1732 TESTPOOL=$ZONE_POOL/$ZONE_CTR$i 1733 if ! ismounted $TESTPOOL; then 1734 log_must zfs mount $TESTPOOL 1735 fi 1736 else 1737 eval TESTPOOL$i=$ZONE_POOL/$ZONE_CTR$i 1738 if eval ! ismounted \$TESTPOOL$i; then 1739 log_must eval zfs mount \$TESTPOOL$i 1740 fi 1741 fi 1742 ((i += 1)) 1743 done 1744} 1745 1746# 1747# Verify a given disk or pool state 1748# 1749# Return 0 is pool/disk matches expected state, 1 otherwise 1750# 1751function check_state # pool disk state{online,offline,degraded} 1752{ 1753 typeset pool=$1 1754 typeset disk=${2#$DEV_DSKDIR/} 1755 typeset state=$3 1756 1757 [[ -z $pool ]] || [[ -z $state ]] \ 1758 && log_fail "Arguments invalid or missing" 1759 1760 if [[ -z $disk ]]; then 1761 #check pool state only 1762 zpool get -H -o value health $pool | grep -qi "$state" 1763 else 1764 zpool status -v $pool | grep "$disk" | grep -qi "$state" 1765 fi 1766} 1767 1768# 1769# Get the mountpoint of snapshot 1770# For the snapshot use <mp_filesystem>/.zfs/snapshot/<snap> 1771# as its mountpoint 1772# 1773function snapshot_mountpoint 1774{ 1775 typeset dataset=${1:-$TESTPOOL/$TESTFS@$TESTSNAP} 1776 1777 if [[ $dataset != *@* ]]; then 1778 log_fail "Error name of snapshot '$dataset'." 1779 fi 1780 1781 typeset fs=${dataset%@*} 1782 typeset snap=${dataset#*@} 1783 1784 if [[ -z $fs || -z $snap ]]; then 1785 log_fail "Error name of snapshot '$dataset'." 1786 fi 1787 1788 echo $(get_prop mountpoint $fs)/.zfs/snapshot/$snap 1789} 1790 1791# 1792# Given a device and 'ashift' value verify it's correctly set on every label 1793# 1794function verify_ashift # device ashift 1795{ 1796 typeset device="$1" 1797 typeset ashift="$2" 1798 1799 zdb -e -lll $device | awk -v ashift=$ashift ' 1800 /ashift: / { 1801 if (ashift != $2) 1802 exit 1; 1803 else 1804 count++; 1805 } 1806 END { 1807 exit (count != 4); 1808 }' 1809} 1810 1811# 1812# Given a pool and file system, this function will verify the file system 1813# using the zdb internal tool. Note that the pool is exported and imported 1814# to ensure it has consistent state. 1815# 1816function verify_filesys # pool filesystem dir 1817{ 1818 typeset pool="$1" 1819 typeset filesys="$2" 1820 typeset zdbout="/tmp/zdbout.$$" 1821 1822 shift 1823 shift 1824 typeset dirs=$@ 1825 typeset search_path="" 1826 1827 log_note "Calling zdb to verify filesystem '$filesys'" 1828 zfs unmount -a > /dev/null 2>&1 1829 log_must zpool export $pool 1830 1831 if [[ -n $dirs ]] ; then 1832 for dir in $dirs ; do 1833 search_path="$search_path -d $dir" 1834 done 1835 fi 1836 1837 log_must zpool import $search_path $pool 1838 1839 if ! zdb -cudi $filesys > $zdbout 2>&1; then 1840 log_note "Output: zdb -cudi $filesys" 1841 cat $zdbout 1842 rm -f $zdbout 1843 log_fail "zdb detected errors with: '$filesys'" 1844 fi 1845 1846 log_must zfs mount -a 1847 log_must rm -rf $zdbout 1848} 1849 1850# 1851# Given a pool issue a scrub and verify that no checksum errors are reported. 1852# 1853function verify_pool 1854{ 1855 typeset pool=${1:-$TESTPOOL} 1856 1857 log_must zpool scrub $pool 1858 log_must wait_scrubbed $pool 1859 1860 typeset -i cksum=$(zpool status $pool | awk ' 1861 !NF { isvdev = 0 } 1862 isvdev { errors += $NF } 1863 /CKSUM$/ { isvdev = 1 } 1864 END { print errors } 1865 ') 1866 if [[ $cksum != 0 ]]; then 1867 log_must zpool status -v 1868 log_fail "Unexpected CKSUM errors found on $pool ($cksum)" 1869 fi 1870} 1871 1872# 1873# Given a pool, and this function list all disks in the pool 1874# 1875function get_disklist # pool 1876{ 1877 echo $(zpool iostat -v $1 | awk '(NR > 4) {print $1}' | \ 1878 grep -vEe '^-----' -e "^(mirror|raidz[1-3]|draid[1-3]|spare|log|cache|special|dedup)|\-[0-9]$") 1879} 1880 1881# 1882# Given a pool, and this function list all disks in the pool with their full 1883# path (like "/dev/sda" instead of "sda"). 1884# 1885function get_disklist_fullpath # pool 1886{ 1887 get_disklist "-P $1" 1888} 1889 1890 1891 1892# /** 1893# This function kills a given list of processes after a time period. We use 1894# this in the stress tests instead of STF_TIMEOUT so that we can have processes 1895# run for a fixed amount of time, yet still pass. Tests that hit STF_TIMEOUT 1896# would be listed as FAIL, which we don't want : we're happy with stress tests 1897# running for a certain amount of time, then finishing. 1898# 1899# @param $1 the time in seconds after which we should terminate these processes 1900# @param $2..$n the processes we wish to terminate. 1901# */ 1902function stress_timeout 1903{ 1904 typeset -i TIMEOUT=$1 1905 shift 1906 typeset cpids="$@" 1907 1908 log_note "Waiting for child processes($cpids). " \ 1909 "It could last dozens of minutes, please be patient ..." 1910 log_must sleep $TIMEOUT 1911 1912 log_note "Killing child processes after ${TIMEOUT} stress timeout." 1913 typeset pid 1914 for pid in $cpids; do 1915 ps -p $pid > /dev/null 2>&1 && 1916 log_must kill -USR1 $pid 1917 done 1918} 1919 1920# 1921# Verify a given hotspare disk is inuse or avail 1922# 1923# Return 0 is pool/disk matches expected state, 1 otherwise 1924# 1925function check_hotspare_state # pool disk state{inuse,avail} 1926{ 1927 typeset pool=$1 1928 typeset disk=${2#$DEV_DSKDIR/} 1929 typeset state=$3 1930 1931 cur_state=$(get_device_state $pool $disk "spares") 1932 1933 [ $state = $cur_state ] 1934} 1935 1936# 1937# Wait until a hotspare transitions to a given state or times out. 1938# 1939# Return 0 when pool/disk matches expected state, 1 on timeout. 1940# 1941function wait_hotspare_state # pool disk state timeout 1942{ 1943 typeset pool=$1 1944 typeset disk=${2#*$DEV_DSKDIR/} 1945 typeset state=$3 1946 typeset timeout=${4:-60} 1947 typeset -i i=0 1948 1949 while [[ $i -lt $timeout ]]; do 1950 if check_hotspare_state $pool $disk $state; then 1951 return 0 1952 fi 1953 1954 i=$((i+1)) 1955 sleep 1 1956 done 1957 1958 return 1 1959} 1960 1961# 1962# Verify a given vdev disk is inuse or avail 1963# 1964# Return 0 is pool/disk matches expected state, 1 otherwise 1965# 1966function check_vdev_state # pool disk state{online,offline,unavail,removed} 1967{ 1968 typeset pool=$1 1969 typeset disk=${2#*$DEV_DSKDIR/} 1970 typeset state=$3 1971 1972 cur_state=$(get_device_state $pool $disk) 1973 1974 [ $state = $cur_state ] 1975} 1976 1977# 1978# Wait until a vdev transitions to a given state or times out. 1979# 1980# Return 0 when pool/disk matches expected state, 1 on timeout. 1981# 1982function wait_vdev_state # pool disk state timeout 1983{ 1984 typeset pool=$1 1985 typeset disk=${2#*$DEV_DSKDIR/} 1986 typeset state=$3 1987 typeset timeout=${4:-60} 1988 typeset -i i=0 1989 1990 while [[ $i -lt $timeout ]]; do 1991 if check_vdev_state $pool $disk $state; then 1992 return 0 1993 fi 1994 1995 i=$((i+1)) 1996 sleep 1 1997 done 1998 1999 return 1 2000} 2001 2002# 2003# Wait for vdev 'sit_out' property to be cleared. 2004# 2005# $1 pool name 2006# $2 vdev name 2007# $3 timeout 2008# 2009function wait_sit_out #pool vdev timeout 2010{ 2011 typeset pool=${1:-$TESTPOOL} 2012 typeset vdev="$2" 2013 typeset timeout=${3:-300} 2014 for (( timer = 0; timer < $timeout; timer++ )); do 2015 if [ "$(get_vdev_prop sit_out "$pool" "$vdev")" = "off" ]; then 2016 return 0 2017 fi 2018 sleep 1; 2019 done 2020 2021 return 1 2022} 2023 2024# 2025# Check the output of 'zpool status -v <pool>', 2026# and to see if the counts of <device> contain the <regex> specified. 2027# 2028# Return 0 is contain, 1 otherwise 2029# 2030function check_pool_device # pool device regex <verbose> 2031{ 2032 typeset pool=$1 2033 typeset device=$2 2034 typeset regex=$3 2035 typeset verbose=${4:-false} 2036 2037 scan=$(zpool status -v "$pool" 2>/dev/null | grep $device) 2038 if [[ $verbose == true ]]; then 2039 log_note $scan 2040 fi 2041 echo $scan | grep -qi "$regex" 2042} 2043 2044# 2045# Check the output of 'zpool status -v <pool>', 2046# and to see if the content of <token> contain the <keyword> specified. 2047# 2048# Return 0 is contain, 1 otherwise 2049# 2050function check_pool_status # pool token keyword <verbose> 2051{ 2052 typeset pool=$1 2053 typeset token=$2 2054 typeset keyword=$3 2055 typeset verbose=${4:-false} 2056 2057 scan=$(zpool status -v "$pool" 2>/dev/null | awk -v token="$token:" '$1==token') 2058 if [[ $verbose == true ]]; then 2059 log_note $scan 2060 fi 2061 echo $scan | grep -qi "$keyword" 2062} 2063 2064# 2065# The following functions are instance of check_pool_status() 2066# is_pool_resilvering - to check if the pool resilver is in progress 2067# is_pool_resilvered - to check if the pool resilver is completed 2068# is_pool_scrubbing - to check if the pool scrub is in progress 2069# is_pool_scrubbed - to check if the pool scrub is completed 2070# is_pool_scrub_stopped - to check if the pool scrub is stopped 2071# is_pool_scrub_paused - to check if the pool scrub has paused 2072# pause_scrub - start and pause a scrub without racing completion 2073# is_pool_removing - to check if the pool removing is a vdev 2074# is_pool_removed - to check if the pool remove is completed 2075# is_pool_discarding - to check if the pool checkpoint is being discarded 2076# is_pool_replacing - to check if the pool is performing a replacement 2077# 2078function is_pool_resilvering #pool <verbose> 2079{ 2080 check_pool_status "$1" "scan" \ 2081 "resilver[ ()0-9A-Za-z:_-]* in progress since" $2 2082} 2083 2084function is_pool_resilvered #pool <verbose> 2085{ 2086 check_pool_status "$1" "scan" "resilvered " $2 2087} 2088 2089function is_pool_scrubbing #pool <verbose> 2090{ 2091 check_pool_status "$1" "scan" "scrub in progress since " $2 2092} 2093 2094function is_pool_error_scrubbing #pool <verbose> 2095{ 2096 check_pool_status "$1" "scrub" "error scrub in progress since " $2 2097 return $? 2098} 2099 2100function is_pool_scrubbed #pool <verbose> 2101{ 2102 check_pool_status "$1" "scan" "scrub repaired" $2 2103} 2104 2105function is_pool_scrub_stopped #pool <verbose> 2106{ 2107 check_pool_status "$1" "scan" "scrub canceled" $2 2108} 2109 2110function is_pool_error_scrub_stopped #pool <verbose> 2111{ 2112 check_pool_status "$1" "scrub" "error scrub canceled on " $2 2113 return $? 2114} 2115 2116function is_pool_scrub_paused #pool <verbose> 2117{ 2118 check_pool_status "$1" "scan" "scrub paused since " $2 2119} 2120 2121function is_pool_error_scrub_paused #pool <verbose> 2122{ 2123 check_pool_status "$1" "scrub" "error scrub paused since " $2 2124 return $? 2125} 2126 2127# 2128# Start a scrub and pause it without racing completion. Pins 2129# SCAN_SUSPEND_PROGRESS until the scrub is paused, then clears it so a 2130# later resume/wait can finish. Extra args are passed to `zpool scrub` 2131# (e.g. -t). Callers should still reset SCAN_SUSPEND_PROGRESS in cleanup. 2132# 2133function pause_scrub # pool [scrub-args...] 2134{ 2135 typeset pool=$1 2136 shift 2137 2138 log_must set_tunable32 SCAN_SUSPEND_PROGRESS 1 2139 log_must zpool scrub "$@" $pool 2140 log_must zpool scrub -p $pool 2141 log_must is_pool_scrub_paused $pool true 2142 log_must set_tunable32 SCAN_SUSPEND_PROGRESS 0 2143} 2144 2145function is_pool_removing #pool 2146{ 2147 check_pool_status "$1" "remove" "in progress since " 2148} 2149 2150function is_pool_removed #pool 2151{ 2152 check_pool_status "$1" "remove" "completed on" 2153} 2154 2155function is_pool_discarding #pool 2156{ 2157 check_pool_status "$1" "checkpoint" "discarding" 2158} 2159function is_pool_replacing #pool 2160{ 2161 zpool status "$1" | grep -qE 'replacing-[0-9]+' 2162} 2163 2164function wait_for_degraded 2165{ 2166 typeset pool=$1 2167 typeset timeout=${2:-30} 2168 typeset t0=$SECONDS 2169 2170 while :; do 2171 [[ $(get_pool_prop health $pool) == "DEGRADED" ]] && break 2172 log_note "$pool is not yet degraded." 2173 sleep 1 2174 if ((SECONDS - t0 > $timeout)); then 2175 log_note "$pool not degraded after $timeout seconds." 2176 return 1 2177 fi 2178 done 2179 2180 return 0 2181} 2182 2183# 2184# Use create_pool()/destroy_pool() to clean up the information in 2185# in the given disk to avoid slice overlapping. 2186# 2187function cleanup_devices #vdevs 2188{ 2189 typeset pool="foopool$$" 2190 2191 for vdev in $@; do 2192 zero_partitions $vdev 2193 done 2194 2195 poolexists $pool && destroy_pool $pool 2196 create_pool $pool $@ 2197 destroy_pool $pool 2198 2199 return 0 2200} 2201 2202#/** 2203# A function to find and locate free disks on a system or from given 2204# disks as the parameter. It works by locating disks that are in use 2205# as swap devices and dump devices, and also disks listed in /etc/vfstab 2206# 2207# $@ given disks to find which are free, default is all disks in 2208# the test system 2209# 2210# @return a string containing the list of available disks 2211#*/ 2212function find_disks 2213{ 2214 # Trust provided list, no attempt is made to locate unused devices. 2215 if is_linux || is_freebsd; then 2216 echo "$@" 2217 return 2218 fi 2219 2220 2221 sfi=/tmp/swaplist.$$ 2222 dmpi=/tmp/dumpdev.$$ 2223 max_finddisksnum=${MAX_FINDDISKSNUM:-6} 2224 2225 swap -l > $sfi 2226 dumpadm > $dmpi 2>/dev/null 2227 2228 disks=${@:-$(echo "" | format -e 2>/dev/null | awk ' 2229BEGIN { FS="."; } 2230 2231/^Specify disk/{ 2232 searchdisks=0; 2233} 2234 2235{ 2236 if (searchdisks && $2 !~ "^$"){ 2237 split($2,arr," "); 2238 print arr[1]; 2239 } 2240} 2241 2242/^AVAILABLE DISK SELECTIONS:/{ 2243 searchdisks=1; 2244} 2245')} 2246 2247 unused="" 2248 for disk in $disks; do 2249 # Check for mounted 2250 grep -q "${disk}[sp]" /etc/mnttab && continue 2251 # Check for swap 2252 grep -q "${disk}[sp]" $sfi && continue 2253 # check for dump device 2254 grep -q "${disk}[sp]" $dmpi && continue 2255 # check to see if this disk hasn't been explicitly excluded 2256 # by a user-set environment variable 2257 echo "${ZFS_HOST_DEVICES_IGNORE}" | grep -q "${disk}" && continue 2258 unused_candidates="$unused_candidates $disk" 2259 done 2260 rm $sfi $dmpi 2261 2262# now just check to see if those disks do actually exist 2263# by looking for a device pointing to the first slice in 2264# each case. limit the number to max_finddisksnum 2265 count=0 2266 for disk in $unused_candidates; do 2267 if is_disk_device $DEV_DSKDIR/${disk}s0 && \ 2268 [ $count -lt $max_finddisksnum ]; then 2269 unused="$unused $disk" 2270 # do not impose limit if $@ is provided 2271 [[ -z $@ ]] && ((count = count + 1)) 2272 fi 2273 done 2274 2275# finally, return our disk list 2276 echo $unused 2277} 2278 2279function add_user_freebsd #<group_name> <user_name> <basedir> 2280{ 2281 typeset group=$1 2282 typeset user=$2 2283 typeset basedir=$3 2284 2285 # Check to see if the user exists. 2286 if id $user > /dev/null 2>&1; then 2287 return 0 2288 fi 2289 2290 # Assign 1000 as the base uid 2291 typeset -i uid=1000 2292 while true; do 2293 pw useradd -u $uid -g $group -d $basedir/$user -m -n $user 2294 case $? in 2295 0) break ;; 2296 # The uid is not unique 2297 65) ((uid += 1)) ;; 2298 *) return 1 ;; 2299 esac 2300 if [[ $uid == 65000 ]]; then 2301 log_fail "No user id available under 65000 for $user" 2302 fi 2303 done 2304 2305 # Silence MOTD 2306 touch $basedir/$user/.hushlogin 2307 2308 return 0 2309} 2310 2311# 2312# Delete the specified user. 2313# 2314# $1 login name 2315# 2316function del_user_freebsd #<logname> 2317{ 2318 typeset user=$1 2319 2320 if id $user > /dev/null 2>&1; then 2321 log_must pw userdel $user 2322 fi 2323 2324 return 0 2325} 2326 2327# 2328# Select valid gid and create specified group. 2329# 2330# $1 group name 2331# 2332function add_group_freebsd #<group_name> 2333{ 2334 typeset group=$1 2335 2336 # See if the group already exists. 2337 if pw groupshow $group >/dev/null 2>&1; then 2338 return 0 2339 fi 2340 2341 # Assign 1000 as the base gid 2342 typeset -i gid=1000 2343 while true; do 2344 pw groupadd -g $gid -n $group > /dev/null 2>&1 2345 case $? in 2346 0) return 0 ;; 2347 # The gid is not unique 2348 65) ((gid += 1)) ;; 2349 *) return 1 ;; 2350 esac 2351 if [[ $gid == 65000 ]]; then 2352 log_fail "No user id available under 65000 for $group" 2353 fi 2354 done 2355} 2356 2357# 2358# Delete the specified group. 2359# 2360# $1 group name 2361# 2362function del_group_freebsd #<group_name> 2363{ 2364 typeset group=$1 2365 2366 pw groupdel -n $group > /dev/null 2>&1 2367 case $? in 2368 # Group does not exist, or was deleted successfully. 2369 0|6|65) return 0 ;; 2370 # Name already exists as a group name 2371 9) log_must pw groupdel $group ;; 2372 *) return 1 ;; 2373 esac 2374 2375 return 0 2376} 2377 2378function add_user_illumos #<group_name> <user_name> <basedir> 2379{ 2380 typeset group=$1 2381 typeset user=$2 2382 typeset basedir=$3 2383 2384 log_must useradd -g $group -d $basedir/$user -m $user 2385 2386 return 0 2387} 2388 2389function del_user_illumos #<user_name> 2390{ 2391 typeset user=$1 2392 2393 if id $user > /dev/null 2>&1; then 2394 log_must_retry "currently used" 6 userdel $user 2395 fi 2396 2397 return 0 2398} 2399 2400function add_group_illumos #<group_name> 2401{ 2402 typeset group=$1 2403 2404 typeset -i gid=100 2405 while true; do 2406 groupadd -g $gid $group > /dev/null 2>&1 2407 case $? in 2408 0) return 0 ;; 2409 # The gid is not unique 2410 4) ((gid += 1)) ;; 2411 *) return 1 ;; 2412 esac 2413 done 2414} 2415 2416function del_group_illumos #<group_name> 2417{ 2418 typeset group=$1 2419 2420 groupmod -n $grp $grp > /dev/null 2>&1 2421 case $? in 2422 # Group does not exist. 2423 6) return 0 ;; 2424 # Name already exists as a group name 2425 9) log_must groupdel $grp ;; 2426 *) return 1 ;; 2427 esac 2428} 2429 2430function add_user_linux #<group_name> <user_name> <basedir> 2431{ 2432 typeset group=$1 2433 typeset user=$2 2434 typeset basedir=$3 2435 2436 log_must useradd -g $group -d $basedir/$user -m $user 2437 2438 # Add new users to the same group and the command line utils. 2439 # This allows them to be run out of the original users home 2440 # directory as long as it permissioned to be group readable. 2441 cmd_group=$(stat --format="%G" $(command -v zfs)) 2442 log_must usermod -a -G $cmd_group $user 2443 2444 return 0 2445} 2446 2447function del_user_linux #<user_name> 2448{ 2449 typeset user=$1 2450 2451 if id $user > /dev/null 2>&1; then 2452 log_must_retry "currently used" 6 userdel $user 2453 fi 2454} 2455 2456function add_group_linux #<group_name> 2457{ 2458 typeset group=$1 2459 2460 # Assign 100 as the base gid, a larger value is selected for 2461 # Linux because for many distributions 1000 and under are reserved. 2462 while true; do 2463 groupadd $group > /dev/null 2>&1 2464 case $? in 2465 0) return 0 ;; 2466 *) return 1 ;; 2467 esac 2468 done 2469} 2470 2471function del_group_linux #<group_name> 2472{ 2473 typeset group=$1 2474 2475 getent group $group > /dev/null 2>&1 2476 case $? in 2477 # Group does not exist. 2478 2) return 0 ;; 2479 # Name already exists as a group name 2480 0) log_must groupdel $group ;; 2481 *) return 1 ;; 2482 esac 2483 2484 return 0 2485} 2486 2487# 2488# Add specified user to specified group 2489# 2490# $1 group name 2491# $2 user name 2492# $3 base of the homedir (optional) 2493# 2494function add_user #<group_name> <user_name> <basedir> 2495{ 2496 typeset group=$1 2497 typeset user=$2 2498 typeset basedir=${3:-"$TEST_BASE_DIR"} 2499 2500 if ((${#group} == 0 || ${#user} == 0)); then 2501 log_fail "group name or user name are not defined." 2502 fi 2503 2504 case "$UNAME" in 2505 FreeBSD) 2506 add_user_freebsd "$group" "$user" "$basedir" 2507 ;; 2508 Linux) 2509 add_user_linux "$group" "$user" "$basedir" 2510 ;; 2511 *) 2512 add_user_illumos "$group" "$user" "$basedir" 2513 ;; 2514 esac 2515 2516 return 0 2517} 2518 2519# 2520# Delete the specified user. 2521# 2522# $1 login name 2523# $2 base of the homedir (optional) 2524# 2525function del_user #<logname> <basedir> 2526{ 2527 typeset user=$1 2528 typeset basedir=${2:-"$TEST_BASE_DIR"} 2529 2530 if ((${#user} == 0)); then 2531 log_fail "login name is necessary." 2532 fi 2533 2534 case "$UNAME" in 2535 FreeBSD) 2536 del_user_freebsd "$user" 2537 ;; 2538 Linux) 2539 del_user_linux "$user" 2540 ;; 2541 *) 2542 del_user_illumos "$user" 2543 ;; 2544 esac 2545 2546 [[ -d $basedir/$user ]] && rm -fr $basedir/$user 2547 2548 return 0 2549} 2550 2551# 2552# Select valid gid and create specified group. 2553# 2554# $1 group name 2555# 2556function add_group #<group_name> 2557{ 2558 typeset group=$1 2559 2560 if ((${#group} == 0)); then 2561 log_fail "group name is necessary." 2562 fi 2563 2564 case "$UNAME" in 2565 FreeBSD) 2566 add_group_freebsd "$group" 2567 ;; 2568 Linux) 2569 add_group_linux "$group" 2570 ;; 2571 *) 2572 add_group_illumos "$group" 2573 ;; 2574 esac 2575 2576 return 0 2577} 2578 2579# 2580# Delete the specified group. 2581# 2582# $1 group name 2583# 2584function del_group #<group_name> 2585{ 2586 typeset group=$1 2587 2588 if ((${#group} == 0)); then 2589 log_fail "group name is necessary." 2590 fi 2591 2592 case "$UNAME" in 2593 FreeBSD) 2594 del_group_freebsd "$group" 2595 ;; 2596 Linux) 2597 del_group_linux "$group" 2598 ;; 2599 *) 2600 del_group_illumos "$group" 2601 ;; 2602 esac 2603 2604 return 0 2605} 2606 2607# 2608# This function will return true if it's safe to destroy the pool passed 2609# as argument 1. It checks for pools based on zvols and files, and also 2610# files contained in a pool that may have a different mountpoint. 2611# 2612function safe_to_destroy_pool { # $1 the pool name 2613 2614 typeset pool="" 2615 typeset DONT_DESTROY="" 2616 2617 # We check that by deleting the $1 pool, we're not 2618 # going to pull the rug out from other pools. Do this 2619 # by looking at all other pools, ensuring that they 2620 # aren't built from files or zvols contained in this pool. 2621 2622 for pool in $(zpool list -H -o name) 2623 do 2624 ALTMOUNTPOOL="" 2625 2626 # this is a list of the top-level directories in each of the 2627 # files that make up the path to the files the pool is based on 2628 FILEPOOL=$(zpool status -v $pool | awk -v pool="/$1/" '$0 ~ pool {print $1}') 2629 2630 # this is a list of the zvols that make up the pool 2631 ZVOLPOOL=$(zpool status -v $pool | awk -v zvols="$ZVOL_DEVDIR/$1$" '$0 ~ zvols {print $1}') 2632 2633 # also want to determine if it's a file-based pool using an 2634 # alternate mountpoint... 2635 POOL_FILE_DIRS=$(zpool status -v $pool | \ 2636 awk '/\// {print $1}' | \ 2637 awk -F/ '!/dev/ {print $2}') 2638 2639 for pooldir in $POOL_FILE_DIRS 2640 do 2641 OUTPUT=$(zfs list -H -r -o mountpoint $1 | \ 2642 awk -v pd="${pooldir}$" '$0 ~ pd {print $1}') 2643 2644 ALTMOUNTPOOL="${ALTMOUNTPOOL}${OUTPUT}" 2645 done 2646 2647 2648 if [ ! -z "$ZVOLPOOL" ] 2649 then 2650 DONT_DESTROY="true" 2651 log_note "Pool $pool is built from $ZVOLPOOL on $1" 2652 fi 2653 2654 if [ ! -z "$FILEPOOL" ] 2655 then 2656 DONT_DESTROY="true" 2657 log_note "Pool $pool is built from $FILEPOOL on $1" 2658 fi 2659 2660 if [ ! -z "$ALTMOUNTPOOL" ] 2661 then 2662 DONT_DESTROY="true" 2663 log_note "Pool $pool is built from $ALTMOUNTPOOL on $1" 2664 fi 2665 done 2666 2667 if [ -z "${DONT_DESTROY}" ] 2668 then 2669 return 0 2670 else 2671 log_note "Warning: it is not safe to destroy $1!" 2672 return 1 2673 fi 2674} 2675 2676# 2677# Verify zfs operation with -p option work as expected 2678# $1 operation, value could be create, clone or rename 2679# $2 dataset type, value could be fs or vol 2680# $3 dataset name 2681# $4 new dataset name 2682# 2683function verify_opt_p_ops 2684{ 2685 typeset ops=$1 2686 typeset datatype=$2 2687 typeset dataset=$3 2688 typeset newdataset=$4 2689 typeset popt=$5 2690 2691 if [[ $datatype != "fs" && $datatype != "vol" ]]; then 2692 log_fail "$datatype is not supported." 2693 fi 2694 2695 if [[ -z "$popt" ]]; then 2696 popt=-p 2697 fi 2698 2699 # check parameters accordingly 2700 case $ops in 2701 create) 2702 newdataset=$dataset 2703 dataset="" 2704 if [[ $datatype == "vol" ]]; then 2705 ops="create -V $VOLSIZE" 2706 fi 2707 ;; 2708 clone) 2709 if [[ -z $newdataset ]]; then 2710 log_fail "newdataset should not be empty" \ 2711 "when ops is $ops." 2712 fi 2713 log_must datasetexists $dataset 2714 log_must snapexists $dataset 2715 ;; 2716 rename) 2717 if [[ -z $newdataset ]]; then 2718 log_fail "newdataset should not be empty" \ 2719 "when ops is $ops." 2720 fi 2721 log_must datasetexists $dataset 2722 ;; 2723 *) 2724 log_fail "$ops is not supported." 2725 ;; 2726 esac 2727 2728 # make sure the upper level filesystem does not exist 2729 destroy_dataset "${newdataset%/*}" "-rRf" 2730 2731 # without -p option, operation will fail 2732 log_mustnot zfs $ops $dataset $newdataset 2733 log_mustnot datasetexists $newdataset ${newdataset%/*} 2734 2735 # with -p option, operation should succeed 2736 log_must zfs $ops $popt $dataset $newdataset 2737 block_device_wait 2738 2739 if ! datasetexists $newdataset ; then 2740 log_fail "-p option does not work for $ops" 2741 fi 2742 2743 # when $ops is create or clone, redo the operation still return zero 2744 if [[ $ops != "rename" ]]; then 2745 log_must zfs $ops $popt $dataset $newdataset 2746 fi 2747 2748 return 0 2749} 2750 2751# 2752# Get configuration of pool 2753# $1 pool name 2754# $2 config name 2755# 2756function get_config 2757{ 2758 typeset pool=$1 2759 typeset config=$2 2760 2761 if ! poolexists "$pool" ; then 2762 return 1 2763 fi 2764 if [ "$(get_pool_prop cachefile "$pool")" = "none" ]; then 2765 zdb -e $pool 2766 else 2767 zdb -C $pool 2768 fi | awk -F: -v cfg="$config:" '$0 ~ cfg {sub(/^'\''/, $2); sub(/'\''$/, $2); print $2}' 2769} 2770 2771# 2772# Privated function. Random select one of items from arguments. 2773# 2774# $1 count 2775# $2-n string 2776# 2777function _random_get 2778{ 2779 typeset cnt=$1 2780 shift 2781 2782 typeset str="$@" 2783 typeset -i ind 2784 ((ind = RANDOM % cnt + 1)) 2785 2786 echo "$str" | cut -f $ind -d ' ' 2787} 2788 2789# 2790# Random select one of item from arguments which include NONE string 2791# 2792function random_get_with_non 2793{ 2794 typeset -i cnt=$# 2795 ((cnt =+ 1)) 2796 2797 _random_get "$cnt" "$@" 2798} 2799 2800# 2801# Random select one of item from arguments which doesn't include NONE string 2802# 2803function random_get 2804{ 2805 _random_get "$#" "$@" 2806} 2807 2808# 2809# The function will generate a dataset name with specific length 2810# $1, the length of the name 2811# $2, the base string to construct the name 2812# 2813function gen_dataset_name 2814{ 2815 typeset -i len=$1 2816 typeset basestr="$2" 2817 typeset -i baselen=${#basestr} 2818 typeset -i iter=0 2819 typeset l_name="" 2820 2821 if ((len % baselen == 0)); then 2822 ((iter = len / baselen)) 2823 else 2824 ((iter = len / baselen + 1)) 2825 fi 2826 while ((iter > 0)); do 2827 l_name="${l_name}$basestr" 2828 2829 ((iter -= 1)) 2830 done 2831 2832 echo $l_name 2833} 2834 2835# 2836# Get cksum tuple of dataset 2837# $1 dataset name 2838# 2839# sample zdb output: 2840# Dataset data/test [ZPL], ID 355, cr_txg 2413856, 31.0K, 7 objects, rootbp 2841# DVA[0]=<0:803046400:200> DVA[1]=<0:81199000:200> [L0 DMU objset] fletcher4 2842# lzjb LE contiguous unique double size=800L/200P birth=2413856L/2413856P 2843# fill=7 cksum=11ce125712:643a9c18ee2:125e25238fca0:254a3f74b59744 2844function datasetcksum 2845{ 2846 typeset cksum 2847 sync 2848 sync_all_pools 2849 zdb -vvv $1 | awk -F= -v ds="^Dataset $1 "'\\[' '$0 ~ ds && /cksum/ {print $7}' 2850} 2851 2852# 2853# Get the given disk/slice state from the specific field of the pool 2854# 2855function get_device_state #pool disk field("", "spares","logs") 2856{ 2857 typeset pool=$1 2858 typeset disk=${2#$DEV_DSKDIR/} 2859 typeset field=${3:-$pool} 2860 2861 zpool status -v "$pool" 2>/dev/null | \ 2862 awk -v device=$disk -v pool=$pool -v field=$field \ 2863 'BEGIN {startconfig=0; startfield=0; } 2864 /config:/ {startconfig=1} 2865 (startconfig==1) && ($1==field) {startfield=1; next;} 2866 (startfield==1) && ($1==device) {print $2; exit;} 2867 (startfield==1) && 2868 ($1==field || $1 ~ "^spares$" || $1 ~ "^logs$") {startfield=0}' 2869} 2870 2871# 2872# get the root filesystem name if it's zfsroot system. 2873# 2874# return: root filesystem name 2875function get_rootfs 2876{ 2877 typeset rootfs="" 2878 2879 if is_freebsd; then 2880 rootfs=$(mount -p | awk '$2 == "/" && $3 == "zfs" {print $1}') 2881 elif ! is_linux; then 2882 rootfs=$(awk '$2 == "/" && $3 == "zfs" {print $1}' \ 2883 /etc/mnttab) 2884 fi 2885 if [[ -z "$rootfs" ]]; then 2886 log_fail "Can not get rootfs" 2887 fi 2888 if datasetexists $rootfs; then 2889 echo $rootfs 2890 else 2891 log_fail "This is not a zfsroot system." 2892 fi 2893} 2894 2895# 2896# get the rootfs's pool name 2897# return: 2898# rootpool name 2899# 2900function get_rootpool 2901{ 2902 typeset rootfs=$(get_rootfs) 2903 echo ${rootfs%%/*} 2904} 2905 2906# 2907# To verify if the require numbers of disks is given 2908# 2909function verify_disk_count 2910{ 2911 typeset -i min=${2:-1} 2912 2913 typeset -i count=$(echo "$1" | wc -w) 2914 2915 if ((count < min)); then 2916 log_untested "A minimum of $min disks is required to run." \ 2917 " You specified $count disk(s)" 2918 fi 2919} 2920 2921function ds_is_volume 2922{ 2923 typeset type=$(get_prop type $1) 2924 [ $type = "volume" ] 2925} 2926 2927function ds_is_filesystem 2928{ 2929 typeset type=$(get_prop type $1) 2930 [ $type = "filesystem" ] 2931} 2932 2933# 2934# Check if Trusted Extensions are installed and enabled 2935# 2936function is_te_enabled 2937{ 2938 svcs -H -o state labeld 2>/dev/null | grep -q "enabled" 2939} 2940 2941# Return the number of CPUs (cross-platform) 2942function get_num_cpus 2943{ 2944 if is_linux ; then 2945 grep -c '^processor' /proc/cpuinfo 2946 elif is_freebsd; then 2947 sysctl -n kern.smp.cpus 2948 else 2949 psrinfo | wc -l 2950 fi 2951} 2952 2953# Utility function to determine if a system has multiple cpus. 2954function is_mp 2955{ 2956 [[ $(get_num_cpus) -gt 1 ]] 2957} 2958 2959function get_cpu_freq 2960{ 2961 if is_linux; then 2962 lscpu | awk '/CPU( max)? MHz/ { print $NF }' 2963 elif is_freebsd; then 2964 sysctl -n hw.clockrate 2965 else 2966 psrinfo -v 0 | awk '/processor operates at/ {print $6}' 2967 fi 2968} 2969 2970# Run the given command as the user provided. 2971function user_run 2972{ 2973 typeset user=$1 2974 shift 2975 2976 log_note "user: $user" 2977 log_note "cmd: $*" 2978 2979 if ! sudo -Eu $user test -x $PATH ; then 2980 log_note "-------------------------------------------------" 2981 log_note "Warning: $user doesn't have permissions on $PATH" 2982 log_note "" 2983 log_note "This usually happens when you're running ZTS locally" 2984 log_note "from inside the ZFS source dir, and are attempting to" 2985 log_note "run a test that calls user_run. The ephemeral user" 2986 log_note "($user) that ZTS is creating does not have permission" 2987 log_note "to traverse to $PATH, or the binaries in $PATH are" 2988 log_note "not the right permissions." 2989 log_note "" 2990 log_note "To get around this, copy your ZFS source directory" 2991 log_note "to a world-accessible location (like /tmp), and " 2992 log_note "change the permissions on your ZFS source dir " 2993 log_note "to allow access." 2994 log_note "" 2995 log_note "Also, verify that /dev/zfs is RW for others:" 2996 log_note "" 2997 log_note " sudo chmod o+rw /dev/zfs" 2998 log_note "-------------------------------------------------" 2999 fi 3000 3001 typeset out=$TEST_BASE_DIR/out 3002 typeset err=$TEST_BASE_DIR/err 3003 3004 sudo -Eu $user \ 3005 env PATH="$PATH" ZTS_LOG_SUPPRESS_TIMESTAMP=1 \ 3006 ksh <<<"$*" >$out 2>$err 3007 typeset res=$? 3008 log_note "out: $(<$out)" 3009 log_note "err: $(<$err)" 3010 return $res 3011} 3012 3013# 3014# Check if the pool contains the specified vdevs 3015# 3016# $1 pool 3017# $2..n <vdev> ... 3018# 3019# Return 0 if the vdevs are contained in the pool, 1 if any of the specified 3020# vdevs is not in the pool, and 2 if pool name is missing. 3021# 3022function vdevs_in_pool 3023{ 3024 typeset pool=$1 3025 typeset vdev 3026 3027 if [[ -z $pool ]]; then 3028 log_note "Missing pool name." 3029 return 2 3030 fi 3031 3032 shift 3033 3034 # We could use 'zpool list' to only get the vdevs of the pool but we 3035 # can't reference a mirror/raidz vdev using its ID (i.e mirror-0), 3036 # therefore we use the 'zpool status' output. 3037 typeset tmpfile=$(mktemp) 3038 zpool status -v "$pool" | grep -A 1000 "config:" >$tmpfile 3039 for vdev in "$@"; do 3040 grep -wq ${vdev##*/} $tmpfile || return 1 3041 done 3042 3043 rm -f $tmpfile 3044 return 0 3045} 3046 3047function get_max 3048{ 3049 typeset -l i max=$1 3050 shift 3051 3052 for i in "$@"; do 3053 max=$((max > i ? max : i)) 3054 done 3055 3056 echo $max 3057} 3058 3059# Write data that can be compressed into a directory 3060function write_compressible 3061{ 3062 typeset dir=$1 3063 typeset megs=$2 3064 typeset nfiles=${3:-1} 3065 typeset bs=${4:-1024k} 3066 typeset fname=${5:-file} 3067 3068 [[ -d $dir ]] || log_fail "No directory: $dir" 3069 3070 # Under Linux fio is not currently used since its behavior can 3071 # differ significantly across versions. This includes missing 3072 # command line options and cases where the --buffer_compress_* 3073 # options fail to behave as expected. 3074 if is_linux; then 3075 typeset file_bytes=$(to_bytes $megs) 3076 typeset bs_bytes=4096 3077 typeset blocks=$(($file_bytes / $bs_bytes)) 3078 3079 for (( i = 0; i < $nfiles; i++ )); do 3080 truncate -s $file_bytes $dir/$fname.$i 3081 3082 # Write every third block to get 66% compression. 3083 for (( j = 0; j < $blocks; j += 3 )); do 3084 dd if=/dev/urandom of=$dir/$fname.$i \ 3085 seek=$j bs=$bs_bytes count=1 \ 3086 conv=notrunc >/dev/null 2>&1 3087 done 3088 done 3089 else 3090 command -v fio > /dev/null || log_unsupported "fio missing" 3091 log_must eval fio \ 3092 --name=job \ 3093 --fallocate=0 \ 3094 --minimal \ 3095 --randrepeat=0 \ 3096 --buffer_compress_percentage=66 \ 3097 --buffer_compress_chunk=4096 \ 3098 --directory="$dir" \ 3099 --numjobs="$nfiles" \ 3100 --nrfiles="$nfiles" \ 3101 --rw=write \ 3102 --bs="$bs" \ 3103 --filesize="$megs" \ 3104 "--filename_format='$fname.\$jobnum' >/dev/null" 3105 fi 3106} 3107 3108function get_objnum 3109{ 3110 typeset pathname=$1 3111 typeset objnum 3112 3113 [[ -e $pathname ]] || log_fail "No such file or directory: $pathname" 3114 if is_freebsd; then 3115 objnum=$(stat -f "%i" $pathname) 3116 else 3117 objnum=$(stat -c %i $pathname) 3118 fi 3119 echo $objnum 3120} 3121 3122# 3123# Sync data to the pool 3124# 3125# $1 pool name 3126# $2 boolean to force uberblock (and config including zpool cache file) update 3127# 3128function sync_pool #pool <force> 3129{ 3130 typeset pool=${1:-$TESTPOOL} 3131 typeset force=${2:-false} 3132 3133 if [[ $force == true ]]; then 3134 log_must zpool sync -f $pool 3135 else 3136 log_must zpool sync $pool 3137 fi 3138 3139 return 0 3140} 3141 3142# 3143# Sync all pools 3144# 3145# $1 boolean to force uberblock (and config including zpool cache file) update 3146# 3147function sync_all_pools #<force> 3148{ 3149 typeset force=${1:-false} 3150 3151 if [[ $force == true ]]; then 3152 log_must zpool sync -f 3153 else 3154 log_must zpool sync 3155 fi 3156 3157 return 0 3158} 3159 3160# 3161# Wait for zpool 'freeing' property drops to zero. 3162# 3163# $1 pool name 3164# 3165function wait_freeing #pool 3166{ 3167 typeset pool=${1:-$TESTPOOL} 3168 while true; do 3169 [[ "0" == "$(zpool list -Ho freeing $pool)" ]] && break 3170 log_must sleep 1 3171 done 3172} 3173 3174# 3175# Wait for every device replace operation to complete 3176# 3177# $1 pool name 3178# $2 timeout 3179# 3180function wait_replacing #pool timeout 3181{ 3182 typeset timeout=${2:-300} 3183 typeset pool=${1:-$TESTPOOL} 3184 for (( timer = 0; timer < $timeout; timer++ )); do 3185 is_pool_replacing $pool || break; 3186 sleep 1; 3187 done 3188} 3189 3190# Wait for a pool to be scrubbed 3191# 3192# $1 pool name 3193# $2 timeout 3194# 3195function wait_scrubbed #pool timeout 3196{ 3197 typeset timeout=${2:-300} 3198 typeset pool=${1:-$TESTPOOL} 3199 for (( timer = 0; timer < $timeout; timer++ )); do 3200 is_pool_scrubbed $pool && break; 3201 sleep 1; 3202 done 3203} 3204 3205# Wait for a pool to be resilvered 3206# 3207# $1 pool name 3208# $2 timeout 3209# 3210function wait_resilvered #pool timeout 3211{ 3212 typeset timeout=${2:-300} 3213 typeset pool=${1:-$TESTPOOL} 3214 for (( timer = 0; timer < $timeout; timer++ )); do 3215 is_pool_resilvered $pool && break; 3216 sleep 1; 3217 done 3218} 3219 3220# Wait for a raidz expansion to stop reflowing 3221# 3222# The raidz expansion tests pause a reflow by setting the tunable 3223# RAIDZ_EXPAND_MAX_REFLOW_BYTES, then inspect the pool while it is 3224# stopped. A pause offset at or past the end of the reflow lets the 3225# expansion run to completion instead, and it may already have completed 3226# before this is called, so both states end the wait. 3227# 3228# $1 pool name 3229# $2 timeout 3230# 3231function wait_raidz_expand_paused #pool timeout 3232{ 3233 typeset pool=${1:-$TESTPOOL} 3234 typeset -i timeout=${2:-300} 3235 typeset -i t0=$SECONDS 3236 typeset status 3237 typeset oldcopied='' 3238 typeset newcopied='' 3239 3240 while :; do 3241 status=$(zpool status $pool) 3242 3243 # Nothing is left to wait for once it is no longer running. 3244 echo "$status" | grep -q 'expansion of .* in progress' || return 3245 3246 # It has paused once the amount copied stops changing. 3247 newcopied=$(echo "$status" | grep ' copied at ' | \ 3248 awk '{print $1}') 3249 [[ -n $newcopied && $newcopied == "$oldcopied" ]] && return 3250 oldcopied=$newcopied 3251 3252 if ((SECONDS - t0 > timeout)); then 3253 log_fail "expansion of $pool neither paused nor" \ 3254 "completed in $timeout seconds" 3255 fi 3256 sleep 1 3257 done 3258} 3259 3260# Backup the zed.rc in our test directory so that we can edit it for our test. 3261# 3262# Returns: Backup file name. You will need to pass this to zed_rc_restore(). 3263function zed_rc_backup 3264{ 3265 zedrc_backup="$(mktemp)" 3266 cp $ZEDLET_DIR/zed.rc $zedrc_backup 3267 echo $zedrc_backup 3268} 3269 3270function zed_rc_restore 3271{ 3272 mv $1 $ZEDLET_DIR/zed.rc 3273} 3274 3275# 3276# Setup custom environment for the ZED. 3277# 3278# $@ Optional list of zedlets to run under zed. 3279function zed_setup 3280{ 3281 if ! is_linux; then 3282 log_unsupported "No zed on $UNAME" 3283 fi 3284 3285 if [[ ! -d $ZEDLET_DIR ]]; then 3286 log_must mkdir $ZEDLET_DIR 3287 fi 3288 3289 if [[ ! -e $VDEVID_CONF ]]; then 3290 log_must touch $VDEVID_CONF 3291 fi 3292 3293 if [[ -e $VDEVID_CONF_ETC ]]; then 3294 log_fail "Must not have $VDEVID_CONF_ETC file present on system" 3295 fi 3296 EXTRA_ZEDLETS=$@ 3297 3298 # Create a symlink for /etc/zfs/vdev_id.conf file. 3299 log_must ln -s $VDEVID_CONF $VDEVID_CONF_ETC 3300 3301 # Setup minimal ZED configuration. Individual test cases should 3302 # add additional ZEDLETs as needed for their specific test. 3303 log_must cp ${ZEDLET_ETC_DIR}/zed.rc $ZEDLET_DIR 3304 log_must cp ${ZEDLET_ETC_DIR}/zed-functions.sh $ZEDLET_DIR 3305 3306 # Scripts must only be user writable. 3307 if [[ -n "$EXTRA_ZEDLETS" ]] ; then 3308 saved_umask=$(umask) 3309 log_must umask 0022 3310 for i in $EXTRA_ZEDLETS ; do 3311 log_must cp ${ZEDLET_LIBEXEC_DIR}/$i $ZEDLET_DIR 3312 done 3313 log_must umask $saved_umask 3314 fi 3315 3316 # Customize the zed.rc file to enable the full debug log. 3317 log_must sed -i '/\#ZED_DEBUG_LOG=.*/d' $ZEDLET_DIR/zed.rc 3318 echo "ZED_DEBUG_LOG=$ZED_DEBUG_LOG" >>$ZEDLET_DIR/zed.rc 3319 3320} 3321 3322# 3323# Cleanup custom ZED environment. 3324# 3325# $@ Optional list of zedlets to remove from our test zed.d directory. 3326function zed_cleanup 3327{ 3328 if ! is_linux; then 3329 return 3330 fi 3331 3332 for extra_zedlet; do 3333 log_must rm -f ${ZEDLET_DIR}/$extra_zedlet 3334 done 3335 log_must rm -fd ${ZEDLET_DIR}/zed.rc ${ZEDLET_DIR}/zed-functions.sh ${ZEDLET_DIR}/all-syslog.sh ${ZEDLET_DIR}/all-debug.sh ${ZEDLET_DIR}/state \ 3336 $ZED_LOG $ZED_DEBUG_LOG $VDEVID_CONF_ETC $VDEVID_CONF \ 3337 $ZEDLET_DIR 3338} 3339 3340# 3341# Check if ZED is currently running; if so, returns PIDs 3342# 3343function zed_check 3344{ 3345 if ! is_linux; then 3346 return 3347 fi 3348 zedpids="$(pgrep -x zed)" 3349 zedpids2="$(pgrep -x lt-zed)" 3350 echo ${zedpids} ${zedpids2} 3351} 3352 3353# 3354# Check if ZED is currently running, if not start ZED. 3355# 3356function zed_start 3357{ 3358 if ! is_linux; then 3359 return 3360 fi 3361 3362 # ZEDLET_DIR=$TEST_BASE_DIR/zed 3363 if [[ ! -d $ZEDLET_DIR ]]; then 3364 log_must mkdir $ZEDLET_DIR 3365 fi 3366 3367 # Verify the ZED is not already running. 3368 zedpids=$(zed_check) 3369 if [ -n "$zedpids" ]; then 3370 # We never, ever, really want it to just keep going if zed 3371 # is already running - usually this implies our test cases 3372 # will break very strangely because whatever we wanted to 3373 # configure zed for won't be listening to our changes in the 3374 # tmpdir 3375 log_fail "ZED already running - ${zedpids}" 3376 else 3377 log_note "Starting ZED" 3378 # run ZED in the background and redirect foreground logging 3379 # output to $ZED_LOG. 3380 log_must truncate -s 0 $ZED_DEBUG_LOG 3381 log_must eval "zed -vF -d $ZEDLET_DIR -P $PATH" \ 3382 "-s $ZEDLET_DIR/state -j 1 2>$ZED_LOG &" 3383 fi 3384 3385 return 0 3386} 3387 3388# 3389# Kill ZED process 3390# 3391function zed_stop 3392{ 3393 if ! is_linux; then 3394 return "" 3395 fi 3396 3397 log_note "Stopping ZED" 3398 while true; do 3399 zedpids=$(zed_check) 3400 [ ! -n "$zedpids" ] && break 3401 3402 log_must kill $zedpids 3403 sleep 1 3404 done 3405 return 0 3406} 3407 3408# 3409# Drain all zevents 3410# 3411function zed_events_drain 3412{ 3413 while [ $(zpool events -H | wc -l) -ne 0 ]; do 3414 sleep 1 3415 zpool events -c >/dev/null 3416 done 3417} 3418 3419# Set a variable in zed.rc to something, un-commenting it in the process. 3420# 3421# $1 variable 3422# $2 value 3423function zed_rc_set 3424{ 3425 var="$1" 3426 val="$2" 3427 # Remove the line 3428 cmd="'/$var/d'" 3429 eval sed -i $cmd $ZEDLET_DIR/zed.rc 3430 3431 # Add it at the end 3432 echo "$var=$val" >> $ZEDLET_DIR/zed.rc 3433} 3434 3435 3436# 3437# Check is provided device is being active used as a swap device. 3438# 3439function is_swap_inuse 3440{ 3441 typeset device=$1 3442 3443 if [[ -z $device ]] ; then 3444 log_note "No device specified." 3445 return 1 3446 fi 3447 3448 case "$UNAME" in 3449 Linux) 3450 swapon -s | grep -wq $(readlink -f $device) 3451 ;; 3452 FreeBSD) 3453 swapctl -l | grep -wq $device 3454 ;; 3455 *) 3456 swap -l | grep -wq $device 3457 ;; 3458 esac 3459} 3460 3461# 3462# Setup a swap device using the provided device. 3463# 3464function swap_setup 3465{ 3466 typeset swapdev=$1 3467 3468 case "$UNAME" in 3469 Linux) 3470 log_must eval "mkswap $swapdev > /dev/null 2>&1" 3471 log_must swapon $swapdev 3472 ;; 3473 FreeBSD) 3474 log_must swapctl -a $swapdev 3475 ;; 3476 *) 3477 log_must swap -a $swapdev 3478 ;; 3479 esac 3480 3481 return 0 3482} 3483 3484# 3485# Cleanup a swap device on the provided device. 3486# 3487function swap_cleanup 3488{ 3489 typeset swapdev=$1 3490 3491 if is_swap_inuse $swapdev; then 3492 if is_linux; then 3493 log_must swapoff $swapdev 3494 elif is_freebsd; then 3495 log_must swapoff $swapdev 3496 else 3497 log_must swap -d $swapdev 3498 fi 3499 fi 3500 3501 return 0 3502} 3503 3504# 3505# Set a global system tunable (64-bit value) 3506# 3507# $1 tunable name (use a NAME defined in tunables.cfg) 3508# $2 tunable values 3509# 3510function set_tunable64 3511{ 3512 set_tunable_impl "$1" "$2" Z 3513} 3514 3515# 3516# Set a global system tunable (32-bit value) 3517# 3518# $1 tunable name (use a NAME defined in tunables.cfg) 3519# $2 tunable values 3520# 3521function set_tunable32 3522{ 3523 set_tunable_impl "$1" "$2" W 3524} 3525 3526function set_tunable_impl 3527{ 3528 typeset name="$1" 3529 typeset value="$2" 3530 typeset mdb_cmd="$3" 3531 3532 eval "typeset tunable=\$$name" 3533 case "$tunable" in 3534 UNSUPPORTED) 3535 log_unsupported "Tunable '$name' is unsupported on $UNAME" 3536 ;; 3537 "") 3538 log_fail "Tunable '$name' must be added to tunables.cfg" 3539 ;; 3540 *) 3541 ;; 3542 esac 3543 3544 [[ -z "$value" ]] && return 1 3545 [[ -z "$mdb_cmd" ]] && return 1 3546 3547 case "$UNAME" in 3548 Linux) 3549 typeset zfs_tunables="/sys/module/zfs/parameters" 3550 echo "$value" >"$zfs_tunables/$tunable" 3551 ;; 3552 FreeBSD) 3553 sysctl vfs.zfs.$tunable=$value 3554 ;; 3555 SunOS) 3556 echo "${tunable}/${mdb_cmd}0t${value}" | mdb -kw 3557 ;; 3558 esac 3559} 3560 3561function save_tunable 3562{ 3563 if tunable_exists $1 ; then 3564 [[ ! -d $TEST_BASE_DIR ]] && return 1 3565 [[ -e $TEST_BASE_DIR/tunable-$1 ]] && return 2 3566 echo "$(get_tunable """$1""")" > "$TEST_BASE_DIR"/tunable-"$1" 3567 fi 3568} 3569 3570function restore_tunable 3571{ 3572 if tunable_exists $1 ; then 3573 [[ ! -e $TEST_BASE_DIR/tunable-$1 ]] && return 1 3574 val="$(cat $TEST_BASE_DIR/tunable-"""$1""")" 3575 set_tunable64 "$1" "$val" 3576 rm $TEST_BASE_DIR/tunable-$1 3577 fi 3578} 3579 3580# 3581# Get a global system tunable 3582# 3583# $1 tunable name (use a NAME defined in tunables.cfg) 3584# 3585function get_tunable 3586{ 3587 get_tunable_impl "$1" 3588} 3589 3590function get_tunable_impl 3591{ 3592 typeset name="$1" 3593 typeset module="${2:-zfs}" 3594 typeset check_only="$3" 3595 3596 eval "typeset tunable=\$$name" 3597 case "$tunable" in 3598 UNSUPPORTED) 3599 if [ -z "$check_only" ] ; then 3600 log_unsupported "Tunable '$name' is unsupported on $UNAME" 3601 else 3602 return 1 3603 fi 3604 ;; 3605 "") 3606 if [ -z "$check_only" ] ; then 3607 log_fail "Tunable '$name' must be added to tunables.cfg" 3608 else 3609 return 1 3610 fi 3611 ;; 3612 *) 3613 ;; 3614 esac 3615 3616 case "$UNAME" in 3617 Linux) 3618 typeset zfs_tunables="/sys/module/$module/parameters" 3619 cat $zfs_tunables/$tunable 3620 ;; 3621 FreeBSD) 3622 sysctl -n vfs.zfs.$tunable 3623 ;; 3624 SunOS) 3625 [[ "$module" -eq "zfs" ]] || return 1 3626 ;; 3627 esac 3628} 3629 3630# Does a tunable exist? 3631# 3632# $1: Tunable name 3633function tunable_exists 3634{ 3635 get_tunable_impl $1 "zfs" 1 3636} 3637 3638# 3639# Compute xxh128sum for given file or stdin if no file given. 3640# Note: file path must not contain spaces 3641# 3642function xxh128digest 3643{ 3644 xxh128sum $1 | awk '{print $1}' 3645} 3646 3647# 3648# Compare the xxhash128 digest of two files. 3649# 3650function cmp_xxh128 { 3651 typeset file1=$1 3652 typeset file2=$2 3653 3654 typeset sum1=$(xxh128digest $file1) 3655 typeset sum2=$(xxh128digest $file2) 3656 test "$sum1" = "$sum2" 3657} 3658 3659function new_fs #<args> 3660{ 3661 case "$UNAME" in 3662 FreeBSD) 3663 newfs "$@" 3664 ;; 3665 *) 3666 echo y | newfs -v "$@" 3667 ;; 3668 esac 3669} 3670 3671function stat_size #<path> 3672{ 3673 typeset path=$1 3674 3675 case "$UNAME" in 3676 FreeBSD) 3677 stat -f %z "$path" 3678 ;; 3679 *) 3680 stat -c %s "$path" 3681 ;; 3682 esac 3683} 3684 3685function stat_blksz #<path> 3686{ 3687 typeset path=$1 3688 3689 case "$UNAME" in 3690 FreeBSD) 3691 stat -f %k "$path" 3692 ;; 3693 *) 3694 stat -c %o "$path" 3695 ;; 3696 esac 3697} 3698 3699function stat_blocks #<path> 3700{ 3701 typeset path=$1 3702 3703 case "$UNAME" in 3704 FreeBSD) 3705 stat -f %b "$path" 3706 ;; 3707 *) 3708 stat -c %b "$path" 3709 ;; 3710 esac 3711} 3712 3713function stat_mtime #<path> 3714{ 3715 typeset path=$1 3716 3717 case "$UNAME" in 3718 FreeBSD) 3719 stat -f %m "$path" 3720 ;; 3721 *) 3722 stat -c %Y "$path" 3723 ;; 3724 esac 3725} 3726 3727function stat_ctime #<path> 3728{ 3729 typeset path=$1 3730 3731 case "$UNAME" in 3732 FreeBSD) 3733 stat -f %c "$path" 3734 ;; 3735 *) 3736 stat -c %Z "$path" 3737 ;; 3738 esac 3739} 3740 3741function stat_crtime #<path> 3742{ 3743 typeset path=$1 3744 3745 case "$UNAME" in 3746 FreeBSD) 3747 stat -f %B "$path" 3748 ;; 3749 *) 3750 stat -c %W "$path" 3751 ;; 3752 esac 3753} 3754 3755function stat_generation #<path> 3756{ 3757 typeset path=$1 3758 3759 case "$UNAME" in 3760 Linux) 3761 getversion "${path}" 3762 ;; 3763 *) 3764 stat -f %v "${path}" 3765 ;; 3766 esac 3767} 3768 3769# Run a command as if it was being run in a TTY. 3770# 3771# Usage: 3772# 3773# faketty command 3774# 3775function faketty 3776{ 3777 if is_freebsd; then 3778 script -q /dev/null env "$@" 3779 else 3780 script --return --quiet -c "$*" /dev/null 3781 fi 3782} 3783 3784# 3785# Produce a random permutation of the integers in a given range (inclusive). 3786# 3787function range_shuffle # begin end 3788{ 3789 typeset -i begin=$1 3790 typeset -i end=$2 3791 3792 seq ${begin} ${end} | sort -R 3793} 3794 3795# 3796# Cross-platform xattr helpers 3797# 3798 3799function get_xattr # name path 3800{ 3801 typeset name=$1 3802 typeset path=$2 3803 3804 case "$UNAME" in 3805 FreeBSD) 3806 getextattr -qq user "${name}" "${path}" 3807 ;; 3808 *) 3809 attr -qg "${name}" "${path}" 3810 ;; 3811 esac 3812} 3813 3814function set_xattr # name value path 3815{ 3816 typeset name=$1 3817 typeset value=$2 3818 typeset path=$3 3819 3820 case "$UNAME" in 3821 FreeBSD) 3822 setextattr user "${name}" "${value}" "${path}" 3823 ;; 3824 *) 3825 attr -qs "${name}" -V "${value}" "${path}" 3826 ;; 3827 esac 3828} 3829 3830function set_xattr_stdin # name value 3831{ 3832 typeset name=$1 3833 typeset path=$2 3834 3835 case "$UNAME" in 3836 FreeBSD) 3837 setextattr -i user "${name}" "${path}" 3838 ;; 3839 *) 3840 attr -qs "${name}" "${path}" 3841 ;; 3842 esac 3843} 3844 3845function rm_xattr # name path 3846{ 3847 typeset name=$1 3848 typeset path=$2 3849 3850 case "$UNAME" in 3851 FreeBSD) 3852 rmextattr -q user "${name}" "${path}" 3853 ;; 3854 *) 3855 attr -qr "${name}" "${path}" 3856 ;; 3857 esac 3858} 3859 3860function ls_xattr # path 3861{ 3862 typeset path=$1 3863 3864 case "$UNAME" in 3865 FreeBSD) 3866 lsextattr -qq user "${path}" 3867 ;; 3868 *) 3869 attr -ql "${path}" 3870 ;; 3871 esac 3872} 3873 3874function punch_hole # offset length file 3875{ 3876 typeset offset=$1 3877 typeset length=$2 3878 typeset file=$3 3879 3880 case "$UNAME" in 3881 FreeBSD) 3882 truncate -d -o $offset -l $length "$file" 3883 ;; 3884 Linux) 3885 fallocate --punch-hole --offset $offset --length $length "$file" 3886 ;; 3887 *) 3888 false 3889 ;; 3890 esac 3891} 3892 3893function zero_range # offset length file 3894{ 3895 typeset offset=$1 3896 typeset length=$2 3897 typeset file=$3 3898 3899 case "$UNAME" in 3900 Linux) 3901 fallocate --zero-range --offset $offset --length $length "$file" 3902 ;; 3903 *) 3904 false 3905 ;; 3906 esac 3907} 3908 3909# 3910# Wait for the specified arcstat to reach non-zero quiescence. 3911# If echo is 1 echo the value after reaching quiescence, otherwise 3912# if echo is 0 print the arcstat we are waiting on. 3913# 3914function arcstat_quiescence # stat echo 3915{ 3916 typeset stat=$1 3917 typeset echo=$2 3918 typeset do_once=true 3919 3920 if [[ $echo -eq 0 ]]; then 3921 echo "Waiting for arcstat $1 quiescence." 3922 fi 3923 3924 while $do_once || [ $stat1 -ne $stat2 ] || [ $stat2 -eq 0 ]; do 3925 typeset stat1=$(kstat arcstats.$stat) 3926 sleep 0.5 3927 typeset stat2=$(kstat arcstats.$stat) 3928 do_once=false 3929 done 3930 3931 if [[ $echo -eq 1 ]]; then 3932 echo $stat2 3933 fi 3934} 3935 3936function arcstat_quiescence_noecho # stat 3937{ 3938 typeset stat=$1 3939 arcstat_quiescence $stat 0 3940} 3941 3942function arcstat_quiescence_echo # stat 3943{ 3944 typeset stat=$1 3945 arcstat_quiescence $stat 1 3946} 3947 3948# 3949# Given an array of pids, wait until all processes 3950# have completed and check their return status. 3951# 3952function wait_for_children #children 3953{ 3954 rv=0 3955 children=("$@") 3956 for child in "${children[@]}" 3957 do 3958 child_exit=0 3959 wait ${child} || child_exit=$? 3960 if [ $child_exit -ne 0 ]; then 3961 echo "child ${child} failed with ${child_exit}" 3962 rv=1 3963 fi 3964 done 3965 return $rv 3966} 3967 3968# 3969# Compare two directory trees recursively in a manner similar to diff(1), but 3970# using rsync. If there are any discrepancies, a summary of the differences are 3971# output and a non-zero error is returned. 3972# 3973# If you're comparing a directory after a ZIL replay, you should set 3974# LIBTEST_DIFF_ZIL_REPLAY=1 or use replay_directory_diff which will cause 3975# directory_diff to ignore mtime changes (the ZIL replay won't fix up mtime 3976# information). 3977# 3978function directory_diff # dir_a dir_b 3979{ 3980 dir_a="$1" 3981 dir_b="$2" 3982 zil_replay="${LIBTEST_DIFF_ZIL_REPLAY:-0}" 3983 3984 # If one of the directories doesn't exist, return 2. This is to match the 3985 # semantics of diff. 3986 if ! [ -d "$dir_a" -a -d "$dir_b" ]; then 3987 return 2 3988 fi 3989 3990 # Run rsync with --dry-run --itemize-changes to get something akin to diff 3991 # output, but rsync is far more thorough in detecting differences (diff 3992 # doesn't compare file metadata, and cannot handle special files). 3993 # 3994 # Also make sure to filter out non-user.* xattrs when comparing. On 3995 # SELinux-enabled systems the copied tree will probably have different 3996 # SELinux labels. 3997 args=("-nicaAHX" '--filter=-x! user.*' "--delete") 3998 3999 # NOTE: Quite a few rsync builds do not support --crtimes which would be 4000 # necessary to verify that creation times are being maintained properly. 4001 # Unfortunately because of this we cannot use it unconditionally but we can 4002 # check if this rsync build supports it and use it then. This check is 4003 # based on the same check in the rsync test suite (testsuite/crtimes.test). 4004 # 4005 # We check ctimes even with zil_replay=1 because the ZIL does store 4006 # creation times and we should make sure they match (if the creation times 4007 # do not match there is a "c" entry in one of the columns). 4008 if rsync --version | grep -q "[, ] crtimes"; then 4009 args+=("--crtimes") 4010 fi 4011 4012 # If we are testing a ZIL replay, we need to ignore timestamp changes. 4013 # Unfortunately --no-times doesn't do what we want -- it will still tell 4014 # you if the timestamps don't match but rsync will set the timestamps to 4015 # the current time (leading to an itemised change entry). It's simpler to 4016 # just filter out those lines. 4017 if [ "$zil_replay" -eq 0 ]; then 4018 filter=("cat") 4019 else 4020 # Different rsync versions have different numbers of columns. So just 4021 # require that aside from the first two, all other columns must be 4022 # blank (literal ".") or a timestamp field ("[tT]"). 4023 filter=("grep" "-v" '^\..[.Tt]\+ ') 4024 fi 4025 4026 diff="$(rsync "${args[@]}" "$dir_a/" "$dir_b/" | "${filter[@]}")" 4027 rv=0 4028 if [ -n "$diff" ]; then 4029 echo "$diff" 4030 rv=1 4031 fi 4032 return $rv 4033} 4034 4035# 4036# Compare two directory trees recursively, without checking whether the mtimes 4037# match (creation times will be checked if the available rsync binary supports 4038# it). This is necessary for ZIL replay checks (because the ZIL does not 4039# contain mtimes and thus after a ZIL replay, mtimes won't match). 4040# 4041# This is shorthand for LIBTEST_DIFF_ZIL_REPLAY=1 directory_diff <...>. 4042# 4043function replay_directory_diff # dir_a dir_b 4044{ 4045 LIBTEST_DIFF_ZIL_REPLAY=1 directory_diff "$@" 4046} 4047 4048# 4049# Put coredumps into $1/core.{basename} 4050# 4051# Output must be saved and passed to pop_coredump_pattern on cleanup 4052# 4053function push_coredump_pattern # dir 4054{ 4055 ulimit -c unlimited 4056 case "$UNAME" in 4057 Linux) 4058 cat /proc/sys/kernel/core_pattern /proc/sys/kernel/core_uses_pid 4059 echo "$1/core.%e" >/proc/sys/kernel/core_pattern && 4060 echo 0 >/proc/sys/kernel/core_uses_pid 4061 ;; 4062 FreeBSD) 4063 sysctl -n kern.corefile 4064 sysctl kern.corefile="$1/core.%N" >/dev/null 4065 ;; 4066 *) 4067 # Nothing to output – set only for this shell 4068 coreadm -p "$1/core.%f" 4069 ;; 4070 esac 4071} 4072 4073# 4074# Put coredumps back into the default location 4075# 4076function pop_coredump_pattern 4077{ 4078 [ -s "$1" ] || return 0 4079 case "$UNAME" in 4080 Linux) 4081 typeset pat pid 4082 { read -r pat; read -r pid; } < "$1" 4083 echo "$pat" >/proc/sys/kernel/core_pattern && 4084 echo "$pid" >/proc/sys/kernel/core_uses_pid 4085 ;; 4086 FreeBSD) 4087 sysctl kern.corefile="$(<"$1")" >/dev/null 4088 ;; 4089 esac 4090} 4091 4092# 4093# get_same_blocks dataset1 path/to/file1 dataset2 path/to/file2 [key] 4094# 4095# Returns a space-separated list of the indexes (starting at 0) of the L0 4096# blocks that are shared between both files (by first DVA and checksum). 4097# 4098function get_same_blocks # dataset1 file1 dataset2 file2 [key] 4099{ 4100 typeset ds1=$1 4101 typeset file1=$2 4102 typeset ds2=$3 4103 typeset file2=$4 4104 4105 typeset key=$5 4106 typeset keyarg= 4107 if [ ${#key} -gt 0 ]; then 4108 keyarg="--key=$key" 4109 fi 4110 4111 # this is usually called as $(get_same_blocks ...), and so expected 4112 # to put its result on stdout, and usually the caller is not watching 4113 # for failure. this makes things a little tricky to fail properly if 4114 # zdb fails or crashes, as we end up returning an empty string, which 4115 # is a valid return (no blocks the same) 4116 # 4117 # to get around this, we check zdb's return and echo a dummy value 4118 # before returning failure. this will not match whatever the caller 4119 # is checking for. if they do call it with log_must, then they get 4120 # a failure as expected. 4121 4122 typeset zdbout1=$(mktemp) 4123 typeset zdbout2=$(mktemp) 4124 typeset awkout1=$(mktemp) 4125 typeset awkout2=$(mktemp) 4126 4127 zdb $keyarg -vvvvv $ds1 -O $file1 > $zdbout1 4128 [[ $? -ne 0 ]] && echo "zdb $ds1 failed" && return 1 4129 4130 zdb $keyarg -vvvvv $ds2 -O $file2 > $zdbout2 4131 [[ $? -ne 0 ]] && echo "zdb $ds2 failed" && return 1 4132 4133 awk '/ L0 / { print l++ " " $3 " " $7 }' < $zdbout1 > $awkout1 4134 awk '/ L0 / { print l++ " " $3 " " $7 }' < $zdbout2 > $awkout2 4135 4136 echo $(sort -n $awkout1 $awkout2 | uniq -d | cut -f1 -d' ') 4137 4138 rm -f $zdbout1 $zdbout2 $awkout1 $awkout2 4139} 4140 4141. ${STF_SUITE}/include/kstat.shlib 4142