1# 2# CDDL HEADER START 3# 4# The contents of this file are subject to the terms of the 5# Common Development and Distribution License (the "License"). 6# You may not use this file except in compliance with the License. 7# 8# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9# or http://www.opensolaris.org/os/licensing. 10# See the License for the specific language governing permissions 11# and limitations under the License. 12# 13# When distributing Covered Code, include this CDDL HEADER in each 14# file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15# If applicable, add the following below this CDDL HEADER, with the 16# fields enclosed by brackets "[]" replaced with your own identifying 17# information: Portions Copyright [yyyy] [name of copyright owner] 18# 19# CDDL HEADER END 20# 21 22# 23# Copyright 2009 Sun Microsystems, Inc. All rights reserved. 24# Use is subject to license terms. 25# 26 27# 28# Copyright (c) 2013, 2016 by Delphix. All rights reserved. 29# 30 31. $STF_SUITE/include/libtest.shlib 32. $STF_SUITE/tests/functional/redundancy/redundancy.cfg 33 34function cleanup 35{ 36 if poolexists $TESTPOOL; then 37 destroy_pool $TESTPOOL 38 fi 39 typeset dir 40 for dir in $TESTDIR $BASEDIR; do 41 if [[ -d $dir ]]; then 42 log_must rm -rf $dir 43 fi 44 done 45} 46 47# 48# Get random number between min and max number. 49# 50# $1 Minimal value 51# $2 Maximal value 52# 53function random 54{ 55 typeset -i min=$1 56 typeset -i max=$2 57 typeset -i value 58 59 while true; do 60 ((value = RANDOM % (max + 1))) 61 if ((value >= min)); then 62 break 63 fi 64 done 65 66 echo $value 67} 68 69# 70# Record the directories construction and checksum all the files which reside 71# within the specified pool 72# 73# $1 The specified pool 74# $2 The file which save the record. 75# 76function record_data 77{ 78 typeset pool=$1 79 typeset recordfile=$2 80 81 [[ -z $pool ]] && log_fail "No specified pool." 82 [[ -f $recordfile ]] && log_must rm -f $recordfile 83 84 typeset mntpnt 85 mntpnt=$(get_prop mountpoint $pool) 86 log_must eval "du -a $mntpnt > $recordfile 2>&1" 87 # 88 # When the data was damaged, checksum is failing and return 1 89 # So, will not use log_must 90 # 91 find $mntpnt -type f -exec cksum {} + >> $recordfile 2>&1 92} 93 94# 95# Create test pool and fill with files and directories. 96# 97# $1 pool name 98# $2 pool type 99# $3 virtual devices number 100# 101function setup_test_env 102{ 103 typeset pool=$1 104 typeset keyword=$2 105 typeset -i vdev_cnt=$3 106 typeset vdevs 107 108 typeset -i i=0 109 while (( i < vdev_cnt )); do 110 vdevs="$vdevs $BASEDIR/vdev$i" 111 ((i += 1)) 112 done 113 114 if [[ ! -d $BASEDIR ]]; then 115 log_must mkdir $BASEDIR 116 fi 117 118 if poolexists $pool ; then 119 destroy_pool $pool 120 fi 121 122 log_must mkfile $MINVDEVSIZE $vdevs 123 124 log_must zpool create -m $TESTDIR $pool $keyword $vdevs 125 126 log_note "Filling up the filesystem ..." 127 typeset -i ret=0 128 typeset -i i=0 129 typeset file=$TESTDIR/file 130 while true ; do 131 file_write -o create -f $file.$i \ 132 -b $BLOCKSZ -c $NUM_WRITES 133 ret=$? 134 (( $ret != 0 )) && break 135 (( i = i + 1 )) 136 done 137 (($ret != 28 )) && log_note "file_write return value($ret) is unexpected." 138 139 record_data $TESTPOOL $PRE_RECORD_FILE 140} 141 142# 143# Check pool status is healthy 144# 145# $1 pool 146# 147function is_healthy 148{ 149 typeset pool=$1 150 151 typeset healthy_output="pool '$pool' is healthy" 152 typeset real_output=$(zpool status -x $pool) 153 154 if [[ "$real_output" == "$healthy_output" ]]; then 155 return 0 156 else 157 typeset -i ret 158 zpool status -x $pool | grep "state:" | \ 159 grep "FAULTED" >/dev/null 2>&1 160 ret=$? 161 (( $ret == 0 )) && return 1 162 typeset l_scan 163 typeset errnum 164 l_scan=$(zpool status -x $pool | grep "scan:") 165 l_scan=${l_scan##*"with"} 166 errnum=$(echo $l_scan | awk '{print $1}') 167 168 return $errnum 169 fi 170} 171 172# 173# Check pool data is valid 174# 175# $1 pool 176# 177function is_data_valid 178{ 179 typeset pool=$1 180 181 record_data $pool $PST_RECORD_FILE 182 if ! diff $PRE_RECORD_FILE $PST_RECORD_FILE > /dev/null 2>&1; then 183 return 1 184 fi 185 186 return 0 187} 188 189# 190# Get the specified count devices name 191# 192# $1 pool name 193# $2 devices count 194# 195function get_vdevs #pool cnt 196{ 197 typeset pool=$1 198 typeset -i cnt=$2 199 200 typeset all_devs=$(zpool iostat -v $pool | awk '{print $1}'| \ 201 egrep -v "^pool$|^capacity$|^mirror$|^raidz1$|^raidz2$|---" | \ 202 egrep -v "/old$|^$pool$") 203 typeset -i i=0 204 typeset vdevs 205 while ((i < cnt)); do 206 typeset dev=$(echo $all_devs | awk '{print $1}') 207 eval all_devs=\${all_devs##*$dev} 208 209 vdevs="$dev $vdevs" 210 ((i += 1)) 211 done 212 213 echo "$vdevs" 214} 215 216# 217# Synchronize all the data in pool 218# 219# $1 pool name 220# 221function sync_pool #pool 222{ 223 typeset pool=$1 224 225 log_must sync 226 log_must sleep 2 227 # Flush all the pool data. 228 typeset -i ret 229 zpool scrub $pool >/dev/null 2>&1 230 ret=$? 231 (( $ret != 0 )) && \ 232 log_fail "zpool scrub $pool failed." 233 234 while ! is_pool_scrubbed $pool; do 235 if is_pool_resilvered $pool ; then 236 log_fail "$pool should not be resilver completed." 237 fi 238 log_must sleep 2 239 done 240} 241 242# 243# Create and replace the same name virtual device files 244# 245# $1 pool name 246# $2-n virtual device files 247# 248function replace_missing_devs 249{ 250 typeset pool=$1 251 shift 252 253 typeset vdev 254 for vdev in $@; do 255 log_must mkfile $MINVDEVSIZE $vdev 256 log_must zpool replace -f $pool $vdev $vdev 257 while true; do 258 if ! is_pool_resilvered $pool ; then 259 log_must sleep 2 260 else 261 break 262 fi 263 done 264 done 265} 266 267# 268# Damage the pool's virtual device files. 269# 270# $1 pool name 271# $2 Failing devices count 272# $3 damage vdevs method, if not null, we keep 273# the label for the vdevs 274# 275function damage_devs 276{ 277 typeset pool=$1 278 typeset -i cnt=$2 279 typeset label="$3" 280 typeset vdevs 281 typeset -i bs_count=$((64 * 1024)) 282 283 vdevs=$(get_vdevs $pool $cnt) 284 typeset dev 285 if [[ -n $label ]]; then 286 for dev in $vdevs; do 287 dd if=/dev/zero of=$dev seek=512 bs=1024 \ 288 count=$bs_count conv=notrunc >/dev/null 2>&1 289 done 290 else 291 for dev in $vdevs; do 292 dd if=/dev/zero of=$dev bs=1024 count=$bs_count \ 293 conv=notrunc >/dev/null 2>&1 294 done 295 fi 296 297 sync_pool $pool 298} 299 300# 301# Clear errors in the pool caused by data corruptions 302# 303# $1 pool name 304# 305function clear_errors 306{ 307 typeset pool=$1 308 309 log_must zpool clear $pool 310 311 if ! is_healthy $pool ; then 312 log_note "$pool should be healthy." 313 return 1 314 fi 315 if ! is_data_valid $pool ; then 316 log_note "Data should be valid in $pool." 317 return 1 318 fi 319 320 return 0 321} 322 323# 324# Remove the specified pool's virtual device files 325# 326# $1 Pool name 327# $2 Missing devices count 328# 329function remove_devs 330{ 331 typeset pool=$1 332 typeset -i cnt=$2 333 typeset vdevs 334 335 vdevs=$(get_vdevs $pool $cnt) 336 log_must rm -f $vdevs 337 338 sync_pool $pool 339} 340 341# 342# Recover the bad or missing device files in the pool 343# 344# $1 Pool name 345# $2 Missing devices count 346# 347function recover_bad_missing_devs 348{ 349 typeset pool=$1 350 typeset -i cnt=$2 351 typeset vdevs 352 353 vdevs=$(get_vdevs $pool $cnt) 354 replace_missing_devs $pool $vdevs 355 356 if ! is_healthy $pool ; then 357 log_note "$pool should be healthy." 358 return 1 359 fi 360 if ! is_data_valid $pool ; then 361 log_note "Data should be valid in $pool." 362 return 1 363 fi 364 365 return 0 366} 367