1# vim: filetype=sh 2# 3# CDDL HEADER START 4# 5# The contents of this file are subject to the terms of the 6# Common Development and Distribution License (the "License"). 7# You may not use this file except in compliance with the License. 8# 9# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 10# or http://www.opensolaris.org/os/licensing. 11# See the License for the specific language governing permissions 12# and limitations under the License. 13# 14# When distributing Covered Code, include this CDDL HEADER in each 15# file and include the License file at usr/src/OPENSOLARIS.LICENSE. 16# If applicable, add the following below this CDDL HEADER, with the 17# fields enclosed by brackets "[]" replaced with your own identifying 18# information: Portions Copyright [yyyy] [name of copyright owner] 19# 20# CDDL HEADER END 21# 22 23# 24# Copyright 2009 Sun Microsystems, Inc. All rights reserved. 25# Use is subject to license terms. 26# 27# ident "@(#)redundancy.kshlib 1.8 09/01/12 SMI" 28# 29 30. ${STF_SUITE}/include/libtest.kshlib 31 32function cleanup 33{ 34 # Log the status of the pool to assist failures. 35 poolexists $TESTPOOL && $ZPOOL status -v $TESTPOOL 36 destroy_pool $TESTPOOL 37 typeset dir 38 for dir in $TESTDIR $BASEDIR; do 39 if [[ -d $dir ]]; then 40 log_must $RM -rf $dir 41 fi 42 done 43} 44 45# 46# Record the directories construction and checksum all the files which reside 47# within the specified pool 48# 49# $1 The specified pool 50# $2 The file which save the record. 51# 52function record_data 53{ 54 typeset pool=$1 55 typeset recordfile=$2 56 57 [[ -z $pool ]] && log_fail "No specified pool." 58 [[ -f $recordfile ]] && log_must $RM -f $recordfile 59 60 typeset mntpnt 61 mntpnt=$(get_prop mountpoint $pool) 62 log_must eval "$DU -a $mntpnt > $recordfile 2>&1" 63 # 64 # When the data was damaged, checksum is failing and return 1 65 # So, will not use log_must 66 # 67 $FIND $mntpnt -type f -exec $CKSUM {} + >> $recordfile 2>&1 68} 69 70# 71# Create test pool and fill with files and directories. 72# 73# $1 pool name 74# $2 pool type 75# $3 virtual devices number 76# 77function setup_test_env 78{ 79 typeset pool=$1 80 typeset keyword=$2 81 typeset -i vdev_cnt=$3 82 typeset vdevs 83 84 typeset -i i=0 85 while (( i < vdev_cnt )); do 86 vdevs="$vdevs $BASEDIR/vdev$i" 87 ((i += 1)) 88 done 89 90 log_must $MKDIR -p $BASEDIR 91 destroy_pool $pool 92 log_must create_vdevs $vdevs 93 94 $ECHO $vdevs | tr ' ' '\n' > $BASEDIR/vdevs 95 log_must $ZPOOL create -m $TESTDIR $pool $keyword $vdevs 96 97 typeset file=$TESTDIR/file 98 log_must $FILE_WRITE -o create -f $file -b $BLOCKSZ -c $NUM_WRITES 99 force_sync_path $BASEDIR 100 record_data $TESTPOOL $PRE_RECORD_FILE 101} 102 103# 104# Check pool data is valid 105# 106# $1 pool 107# 108function is_data_valid 109{ 110 typeset pool=$1 111 112 record_data $pool $PST_RECORD_FILE 113 if ! $DIFF $PRE_RECORD_FILE $PST_RECORD_FILE > /dev/null 2>&1; then 114 return 1 115 fi 116 117 return 0 118} 119 120# 121# Get the specified count devices name 122# 123# $1 pool name 124# $2 devices count 125# 126function get_vdevs #pool cnt 127{ 128 typeset pool=$1 129 typeset -i cnt=$2 130 131 head -$cnt $BASEDIR/vdevs | tr '\n' ' ' 132} 133 134# 135# Synchronize all the data in pool 136# 137# $1 pool name 138# 139function sync_pool #pool 140{ 141 typeset pool=$1 142 143 force_sync_path $BASEDIR 144 145 # If the OS has detected corruption on the pool, it will have 146 # automatically initiated a scrub. In that case, our "zpool scrub" 147 # command will fail. So we ignore its exit status and just check that 148 # the pool is scrubbing or has been scrubbed. 149 $ZPOOL scrub $pool >/dev/null 2>&1 150 is_pool_scrubbing $pool || is_pool_scrubbed $pool || \ 151 log_fail "$ZPOOL scrub $pool failed." 152 log_note "$pool: $ZPOOL scrub issued." 153} 154 155# 156# Create and replace the same name virtual device files 157# 158# $1 pool name 159# $2-n virtual device files 160# 161function replace_missing_devs 162{ 163 typeset pool=$1 164 shift 165 166 typeset vdev 167 for vdev in $@; do 168 [ ! -f $vdev ] && log_must create_vdevs $vdev 169 log_must $ZPOOL replace -f $pool $vdev $vdev 170 wait_for 20 1 is_pool_resilvered $pool 171 done 172} 173 174# 175# Damage the labels of the specified devices. Returns 0 if all such devices 176# are UNAVAIL, 1 otherwise. 177# 178function damage_dev_labels # pool <vdev> [vdev ...] 179{ 180 typeset pool=$1 181 typeset -i ret=0 182 shift 183 184 for vdev in $*; do 185 check_state $pool $vdev UNAVAIL && continue 186 log_must create_vdevs $vdev 187 ret=1 188 done 189 [ $ret -eq 0 ] && return $ret 190 sync_pool $pool 191 return $ret 192} 193 194# 195# Damage the pool's virtual device files. 196# 197# $1 pool name 198# $2 Failing devices count 199# $3 damage vdevs method, if not null, we keep the label for the vdevs 200# 201function damage_devs 202{ 203 typeset pool=$1 204 typeset -i cnt=$2 205 typeset label="$3" 206 typeset vdevs 207 typeset -i bs_count 208 209 vdevs=$(get_vdevs $pool $cnt) 210 log_note "Damaging pool $pool devices: $vdevs" 211 if [[ -n $label ]]; then 212 typeset -i i=0 213 log_mustnot pool_has_errors $pool 214 while [ $i -lt $cnt ]; do 215 corrupt_file $TESTPOOL $TESTDIR/file $i 216 (( i += 1 )) 217 done 218 sync_pool $pool 219 wait_for 20 1 is_pool_scrubbed $pool 220 221 log_must pool_has_errors $pool 222 else 223 # The pool can be syncing, thus fixing its labels. So we 224 # have to keep trying until all the devices go offline. 225 wait_for 20 1 damage_dev_labels $pool $vdevs 226 fi 227 228 log_note "Pool $pool vdevs $vdevs damage completed." 229} 230 231# 232# Clear errors in the pool caused by data corruptions 233# 234# $1 pool name 235# 236function clear_errors 237{ 238 typeset pool=$1 239 240 log_must $ZPOOL clear $pool 241 # The pool may need to resilver (issued async by 'zpool clear'), 242 # give it a chance to do so. 243 wait_for 30 1 is_pool_healthy $pool 244 245 if ! is_data_valid $pool ; then 246 $ZPOOL status -x $pool 247 log_note "Data should be valid in $pool." 248 return 1 249 fi 250 251 return 0 252} 253 254# 255# Remove the specified pool's virtual device files 256# 257# $1 Pool name 258# $2 Missing devices count 259# 260function remove_devs 261{ 262 typeset pool=$1 263 typeset -i cnt=$2 264 typeset vdevs 265 266 vdevs=$(get_vdevs $pool $cnt) 267 log_note "Removing pool $pool vdevs: $vdevs" 268 log_must $RM -f $vdevs 269 270 sync_pool $pool 271 for vdev in $vdevs; do 272 wait_for 20 1 check_state $pool $vdev UNAVAIL 273 done 274} 275 276# 277# Recover the bad or missing device files in the pool 278# 279# $1 Pool name 280# $2 Missing devices count 281# 282function recover_bad_missing_devs 283{ 284 typeset pool=$1 285 typeset -i cnt=$2 286 typeset vdevs 287 288 vdevs=$(get_vdevs $pool $cnt) 289 log_note "Replacing missing pool $pool vdevs: $vdevs" 290 replace_missing_devs $pool $vdevs 291 292 if ! is_pool_healthy $pool ; then 293 log_note "$pool should be healthy." 294 return 1 295 fi 296 if ! is_data_valid $pool ; then 297 log_note "Data should be valid in $pool." 298 return 1 299 fi 300 301 return 0 302} 303