1# 2# CDDL HEADER START 3# 4# The contents of this file are subject to the terms of the 5# Common Development and Distribution License (the "License"). 6# You may not use this file except in compliance with the License. 7# 8# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9# or http://www.opensolaris.org/os/licensing. 10# See the License for the specific language governing permissions 11# and limitations under the License. 12# 13# When distributing Covered Code, include this CDDL HEADER in each 14# file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15# If applicable, add the following below this CDDL HEADER, with the 16# fields enclosed by brackets "[]" replaced with your own identifying 17# information: Portions Copyright [yyyy] [name of copyright owner] 18# 19# CDDL HEADER END 20# 21 22# 23# Copyright 2009 Sun Microsystems, Inc. All rights reserved. 24# Use is subject to license terms. 25# 26 27# 28# Copyright (c) 2013 by Delphix. All rights reserved. 29# 30 31. $STF_SUITE/include/libtest.shlib 32. $STF_SUITE/tests/functional/redundancy/redundancy.cfg 33 34function cleanup 35{ 36 if poolexists $TESTPOOL; then 37 destroy_pool $TESTPOOL 38 fi 39 typeset dir 40 for dir in $TESTDIR $BASEDIR; do 41 if [[ -d $dir ]]; then 42 log_must $RM -rf $dir 43 fi 44 done 45} 46 47# 48# Get random number between min and max number. 49# 50# $1 Minimal value 51# $2 Maximal value 52# 53function random 54{ 55 typeset -i min=$1 56 typeset -i max=$2 57 typeset -i value 58 59 while true; do 60 ((value = RANDOM % (max + 1))) 61 if ((value >= min)); then 62 break 63 fi 64 done 65 66 $ECHO $value 67} 68 69# 70# Record the directories construction and checksum all the files which reside 71# within the specified pool 72# 73# $1 The specified pool 74# $2 The file which save the record. 75# 76function record_data 77{ 78 typeset pool=$1 79 typeset recordfile=$2 80 81 [[ -z $pool ]] && log_fail "No specified pool." 82 [[ -f $recordfile ]] && log_must $RM -f $recordfile 83 84 typeset mntpnt 85 mntpnt=$(get_prop mountpoint $pool) 86 log_must eval "$DU -a $mntpnt > $recordfile 2>&1" 87 # 88 # When the data was damaged, checksum is failing and return 1 89 # So, will not use log_must 90 # 91 $FIND $mntpnt -type f -exec $CKSUM {} + >> $recordfile 2>&1 92} 93 94# 95# Create test pool and fill with files and directories. 96# 97# $1 pool name 98# $2 pool type 99# $3 virtual devices number 100# 101function setup_test_env 102{ 103 typeset pool=$1 104 typeset keyword=$2 105 typeset -i vdev_cnt=$3 106 typeset vdevs 107 108 typeset -i i=0 109 while (( i < vdev_cnt )); do 110 vdevs="$vdevs $BASEDIR/vdev$i" 111 ((i += 1)) 112 done 113 114 if [[ ! -d $BASEDIR ]]; then 115 log_must $MKDIR $BASEDIR 116 fi 117 118 if poolexists $pool ; then 119 destroy_pool $pool 120 fi 121 122 log_must $MKFILE $DEV_SIZE $vdevs 123 124 log_must $ZPOOL create -m $TESTDIR $pool $keyword $vdevs 125 126 log_note "Filling up the filesystem ..." 127 typeset -i ret=0 128 typeset -i i=0 129 typeset file=$TESTDIR/file 130 while $TRUE ; do 131 $FILE_WRITE -o create -f $file.$i \ 132 -b $BLOCKSZ -c $NUM_WRITES 133 ret=$? 134 (( $ret != 0 )) && break 135 (( i = i + 1 )) 136 done 137 (($ret != 28 )) && log_note "$FILE_WRITE return value($ret) is unexpected." 138 139 record_data $TESTPOOL $PRE_RECORD_FILE 140} 141 142# 143# Check pool status is healthy 144# 145# $1 pool 146# 147function is_healthy 148{ 149 typeset pool=$1 150 151 typeset healthy_output="pool '$pool' is healthy" 152 typeset real_output=$($ZPOOL status -x $pool) 153 154 if [[ "$real_output" == "$healthy_output" ]]; then 155 return 0 156 else 157 typeset -i ret 158 $ZPOOL status -x $pool | $GREP "state:" | \ 159 $GREP "FAULTED" >/dev/null 2>&1 160 ret=$? 161 (( $ret == 0 )) && return 1 162 typeset l_scan 163 typeset errnum 164 l_scan=$($ZPOOL status -x $pool | $GREP "scan:") 165 l_scan=${l_scan##*"with"} 166 errnum=$($ECHO $l_scan | $AWK '{print $1}') 167 168 return $errnum 169 fi 170} 171 172# 173# Check pool data is valid 174# 175# $1 pool 176# 177function is_data_valid 178{ 179 typeset pool=$1 180 181 record_data $pool $PST_RECORD_FILE 182 if ! $DIFF $PRE_RECORD_FILE $PST_RECORD_FILE > /dev/null 2>&1; then 183 return 1 184 fi 185 186 return 0 187} 188 189# 190# Get the specified count devices name 191# 192# $1 pool name 193# $2 devices count 194# 195function get_vdevs #pool cnt 196{ 197 typeset pool=$1 198 typeset -i cnt=$2 199 200 typeset all_devs=$($ZPOOL iostat -v $pool | $AWK '{print $1}'| \ 201 $EGREP -v "^pool$|^capacity$|^mirror$|^raidz1$|^raidz2$|---" | \ 202 $EGREP -v "/old$|^$pool$") 203 typeset -i i=0 204 typeset vdevs 205 while ((i < cnt)); do 206 typeset dev=$($ECHO $all_devs | $AWK '{print $1}') 207 eval all_devs=\${all_devs##*$dev} 208 209 vdevs="$dev $vdevs" 210 ((i += 1)) 211 done 212 213 $ECHO "$vdevs" 214} 215 216# 217# Synchronize all the data in pool 218# 219# $1 pool name 220# 221function sync_pool #pool 222{ 223 typeset pool=$1 224 225 log_must $SYNC 226 log_must $SLEEP 2 227 # Flush all the pool data. 228 typeset -i ret 229 $ZPOOL scrub $pool >/dev/null 2>&1 230 ret=$? 231 (( $ret != 0 )) && \ 232 log_fail "$ZPOOL scrub $pool failed." 233 234 while ! is_pool_scrubbed $pool; do 235 if is_pool_resilvered $pool ; then 236 log_fail "$pool should not be resilver completed." 237 fi 238 log_must $SLEEP 2 239 done 240} 241 242# 243# Create and replace the same name virtual device files 244# 245# $1 pool name 246# $2-n virtual device files 247# 248function replace_missing_devs 249{ 250 typeset pool=$1 251 shift 252 253 typeset vdev 254 for vdev in $@; do 255 log_must $MKFILE $DEV_SIZE $vdev 256 log_must $ZPOOL replace -f $pool $vdev $vdev 257 while true; do 258 if ! is_pool_resilvered $pool ; then 259 log_must $SLEEP 2 260 else 261 break 262 fi 263 done 264 done 265} 266 267# 268# Damage the pool's virtual device files. 269# 270# $1 pool name 271# $2 Failing devices count 272# $3 damage vdevs method, if not null, we keep 273# the label for the vdevs 274# 275function damage_devs 276{ 277 typeset pool=$1 278 typeset -i cnt=$2 279 typeset label="$3" 280 typeset vdevs 281 typeset -i bs_count 282 283 vdevs=$(get_vdevs $pool $cnt) 284 if [[ -n $label ]]; then 285 typeset dev 286 for dev in $vdevs; do 287 bs_count=$($LS -l $dev | $AWK '{print $5}') 288 (( bs_count = bs_count/1024 - 512 )) 289 $DD if=/dev/zero of=$dev seek=512 bs=1024 \ 290 count=$bs_count conv=notrunc >/dev/null 2>&1 291 done 292 else 293 log_must $MKFILE $DEV_SIZE $vdevs 294 fi 295 296 sync_pool $pool 297} 298 299# 300# Clear errors in the pool caused by data corruptions 301# 302# $1 pool name 303# 304function clear_errors 305{ 306 typeset pool=$1 307 308 log_must $ZPOOL clear $pool 309 310 if ! is_healthy $pool ; then 311 log_note "$pool should be healthy." 312 return 1 313 fi 314 if ! is_data_valid $pool ; then 315 log_note "Data should be valid in $pool." 316 return 1 317 fi 318 319 return 0 320} 321 322# 323# Remove the specified pool's virtual device files 324# 325# $1 Pool name 326# $2 Missing devices count 327# 328function remove_devs 329{ 330 typeset pool=$1 331 typeset -i cnt=$2 332 typeset vdevs 333 334 vdevs=$(get_vdevs $pool $cnt) 335 log_must $RM -f $vdevs 336 337 sync_pool $pool 338} 339 340# 341# Recover the bad or missing device files in the pool 342# 343# $1 Pool name 344# $2 Missing devices count 345# 346function recover_bad_missing_devs 347{ 348 typeset pool=$1 349 typeset -i cnt=$2 350 typeset vdevs 351 352 vdevs=$(get_vdevs $pool $cnt) 353 replace_missing_devs $pool $vdevs 354 355 if ! is_healthy $pool ; then 356 log_note "$pool should be healthy." 357 return 1 358 fi 359 if ! is_data_valid $pool ; then 360 log_note "Data should be valid in $pool." 361 return 1 362 fi 363 364 return 0 365} 366