1# 2# CDDL HEADER START 3# 4# The contents of this file are subject to the terms of the 5# Common Development and Distribution License (the "License"). 6# You may not use this file except in compliance with the License. 7# 8# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9# or http://www.opensolaris.org/os/licensing. 10# See the License for the specific language governing permissions 11# and limitations under the License. 12# 13# When distributing Covered Code, include this CDDL HEADER in each 14# file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15# If applicable, add the following below this CDDL HEADER, with the 16# fields enclosed by brackets "[]" replaced with your own identifying 17# information: Portions Copyright [yyyy] [name of copyright owner] 18# 19# CDDL HEADER END 20# 21 22# 23# Copyright (c) 2017 by Lawrence Livermore National Security, LLC. 24# Use is subject to license terms. 25# Copyright 2019 Joyent, Inc. 26# 27 28. $STF_SUITE/include/libtest.shlib 29. $STF_SUITE/tests/functional/mmp/mmp.cfg 30 31 32function check_pool_import # pool opts token keyword 33{ 34 typeset pool=${1:-$MMP_POOL} 35 typeset opts=$2 36 typeset token=$3 37 typeset keyword=$4 38 39 zpool import $opts 2>&1 | \ 40 nawk -v token="$token:" '($1==token) {print $0}' | \ 41 grep -i "$keyword" > /dev/null 2>&1 42 43 return $? 44} 45 46function is_pool_imported # pool opts 47{ 48 typeset pool=${1:-$MMP_POOL} 49 typeset opts=$2 50 51 check_pool_import "$pool" "$opts" "status" \ 52 "The pool is currently imported" 53 return $? 54} 55 56function wait_pool_imported # pool opts 57{ 58 typeset pool=${1:-$MMP_POOL} 59 typeset opts=$2 60 61 while is_pool_imported "$pool" "$opts"; do 62 log_must sleep 5 63 done 64 65 return 0 66} 67 68function try_pool_import # pool opts message 69{ 70 typeset pool=${1:-$MMP_POOL} 71 typeset opts=$2 72 typeset msg=$3 73 74 zpool import $opts $pool 2>&1 | grep -i "$msg" 75 76 return $? 77} 78 79function chr2ascii 80{ 81 case "$1" in 82 0) asc="30";; 83 1) asc="31";; 84 2) asc="32";; 85 3) asc="33";; 86 4) asc="34";; 87 5) asc="35";; 88 6) asc="36";; 89 7) asc="37";; 90 8) asc="38";; 91 9) asc="39";; 92 a) asc="61";; 93 b) asc="62";; 94 c) asc="63";; 95 d) asc="64";; 96 e) asc="65";; 97 f) asc="66";; 98 esac 99} 100 101function mmp_set_hostid 102{ 103 typeset hostid=$1 104 105 case "$(uname)" in 106 Linux) 107 a=${hostid:6:2} 108 b=${hostid:4:2} 109 c=${hostid:2:2} 110 d=${hostid:0:2} 111 112 printf "\\x$a\\x$b\\x$c\\x$d" >$HOSTID_FILE 113 114 if [ $(hostid) != "$hostid" ]; then 115 return 1 116 fi 117 ;; 118 SunOS) 119 # 120 # Given a hostid in hex, we have to convert to decimal, then 121 # save the ascii string representation in the kernel. The 122 # 'hostid' command will get the decimal SI_HW_SERIAL value via 123 # sysinfo, then print that as an 8 digit hex number. 124 # 125 typeset dec=$(mdb -e "$hostid=E" | sed -e 's/ *//g') 126 typeset len=$(echo $dec | awk '{print length($0)}') 127 if [[ $len -lt 0 || $len -gt 10 ]]; then 128 return 129 fi 130 typeset pos=0 131 while [[ $pos -lt $len ]]; do 132 chr2ascii ${dec:$pos:1} 133 echo "hw_serial+${pos}/v $asc" | mdb -kw >/dev/null 2>&1 134 pos=$(($pos + 1)) 135 done 136 echo "hw_serial+${pos}/v 0" | mdb -kw >/dev/null 2>&1 137 ;; 138 esac 139 140 return 0 141} 142 143function mmp_clear_hostid 144{ 145 case "$(uname)" in 146 Linux) rm -f $HOSTID_FILE;; 147 SunOS) mmp_set_hostid "00000000";; 148 esac 149} 150 151function mmp_pool_create_simple # pool dir 152{ 153 typeset pool=${1:-$MMP_POOL} 154 typeset dir=${2:-$MMP_DIR} 155 156 log_must mkdir -p $dir 157 log_must rm -f $dir/* 158 log_must truncate -s $MINVDEVSIZE $dir/vdev1 $dir/vdev2 159 160 log_must mmp_set_hostid $HOSTID1 161 log_must zpool create -f -o cachefile=$MMP_CACHE $pool \ 162 mirror $dir/vdev1 $dir/vdev2 163 log_must zpool set multihost=on $pool 164} 165 166function mmp_pool_create # pool dir 167{ 168 typeset pool=${1:-$MMP_POOL} 169 typeset dir=${2:-$MMP_DIR} 170 typeset opts="-VVVVV -T120 -M -k0 -f $dir -E -p $pool" 171 172 mmp_pool_create_simple $pool $dir 173 174 log_must mv $MMP_CACHE ${MMP_CACHE}.stale 175 log_must zpool export $pool 176 log_must mmp_set_hostid $HOSTID2 177 178 log_note "Starting ztest in the background as hostid $HOSTID1" 179 log_must eval "ZFS_HOSTID=$HOSTID1 /usr/bin/ztest $opts >$MMP_ZTEST_LOG 2>&1 &" 180 181 while ! is_pool_imported "$pool" "-d $dir"; do 182 log_must pgrep ztest 183 log_must sleep 5 184 done 185} 186 187function mmp_pool_destroy # pool dir 188{ 189 typeset pool=${1:-$MMP_POOL} 190 typeset dir=${2:-$MMP_DIR} 191 192 ZTESTPID=$(pgrep ztest) 193 if [ -n "$ZTESTPID" ]; then 194 log_must kill $ZTESTPID 195 wait $ZTESTPID 196 fi 197 198 if poolexists $pool; then 199 destroy_pool $pool 200 fi 201 202 log_must rm -f $dir/* 203 mmp_clear_hostid 204} 205 206function mmp_pool_set_hostid # pool hostid 207{ 208 typeset pool=$1 209 typeset hostid=$2 210 211 log_must mmp_set_hostid $hostid 212 log_must zpool export $pool 213 log_must zpool import $pool 214 215 return 0 216} 217# Return the number of seconds the activity check portion of the import process 218# will take. Does not include the time to find devices and assemble a config. 219# Note that the activity check may be skipped, e.g. if the pool and host 220# hostid's match, but this will return non-zero because mmp_* are populated. 221function seconds_mmp_waits_for_activity 222{ 223 typeset pool=$1 224 typeset devpath=$2 225 226 typeset seconds=0 227 typeset devices=${#DISK[@]} 228 typeset import_intervals=$(get_tunable zfs_multihost_import_intervals) 229 typeset import_interval=$(get_tunable zfs_multihost_interval) 230 typeset tmpfile=$(mktemp) 231 typeset mmp_fail 232 typeset mmp_write 233 typeset mmp_delay 234 235 log_must zdb -e -p $devpath $pool >$tmpfile 2>/dev/null 236 mmp_fail=$(awk '/mmp_fail/ {print $NF}' $tmpfile) 237 mmp_write=$(awk '/mmp_write/ {print $NF}' $tmpfile) 238 mmp_delay=$(awk '/mmp_delay/ {print $NF}' $tmpfile) 239 if [ -f $tmpfile ]; then 240 rm $tmpfile 241 fi 242 243 # In order of preference: 244 if [ -n $mmp_fail -a -n $mmp_write ]; then 245 seconds=$((2*mmp_fail*mmp_write/1000)) 246 elif [ -n $mmp_delay ]; then 247 # MMP V0: Based on mmp_delay from the best Uberblock 248 seconds=$((import_intervals*devices*mmp_delay/1000000000)) 249 else 250 # Non-MMP aware: Based on zfs_multihost_interval and import_intervals 251 seconds=$((import_intervals*import_interval/1000)) 252 fi 253 254 echo $seconds 255} 256 257function import_no_activity_check # pool opts 258{ 259 typeset pool=$1 260 typeset opts=$2 261 262 typeset max_duration=$((MMP_TEST_DURATION_DEFAULT-1)) 263 264 SECONDS=0 265 zpool import $opts $pool 266 typeset rc=$? 267 268 if [[ $SECONDS -gt $max_duration ]]; then 269 log_fail "ERROR: import_no_activity_check unexpected activity \ 270check (${SECONDS}s gt $max_duration)" 271 fi 272 273 return $rc 274} 275 276function import_activity_check # pool opts act_test_duration 277{ 278 typeset pool=$1 279 typeset opts=$2 280 typeset min_duration=${3:-$MMP_TEST_DURATION_DEFAULT} 281 282 SECONDS=0 283 zpool import $opts $pool 284 typeset rc=$? 285 286 if [[ $SECONDS -le $min_duration ]]; then 287 log_fail "ERROR: import_activity_check expected activity check \ 288(${SECONDS}s le min_duration $min_duration)" 289 fi 290 291 return $rc 292} 293 294function clear_mmp_history 295{ 296 log_must set_tunable64 zfs_multihost_history $MMP_HISTORY_OFF 297 log_must set_tunable64 zfs_multihost_history $MMP_HISTORY 298} 299 300function count_skipped_mmp_writes # pool duration 301{ 302 typeset pool=$1 303 typeset -i duration=$2 304 typeset hist_path="/proc/spl/kstat/zfs/$pool/multihost" 305 306 sleep $duration 307 awk 'BEGIN {count=0}; $NF == "-" {count++}; END {print count};' "$hist_path" 308} 309 310function count_mmp_writes # pool duration 311{ 312 typeset pool=$1 313 typeset -i duration=$2 314 typeset hist_path="/proc/spl/kstat/zfs/$pool/multihost" 315 316 log_must sleep $duration 317 awk 'BEGIN {count=0}; $NF != "-" {count++}; END {print count};' "$hist_path" 318} 319 320function summarize_uberblock_mmp # device 321{ 322 typeset device=$1 323 324 zdb -luuuu $device | awk ' 325 BEGIN {write_fail_present=0; write_fail_missing=0; uber_invalid=0;} 326 /Uberblock\[[0-9][0-9]*\]/ {delay=-99; write=-99; fail=-99; total++; if (/invalid/) {uber_invalid++};}; 327 /mmp_fail/ {fail=$3}; 328 /mmp_seq/ {seq=$3}; 329 /mmp_write/ {write=$3}; 330 /mmp_delay/ {delay=$3; if (delay==0) {delay_zero++};}; 331 /mmp_valid/ && delay>0 && write>0 && fail>0 {write_fail_present++}; 332 /mmp_valid/ && delay>0 && (write<=0 || fail<=0) {write_fail_missing++}; 333 /mmp_valid/ && delay>0 && write<=0 {write_missing++}; 334 /mmp_valid/ && delay>0 && fail<=0 {fail_missing++}; 335 /mmp_valid/ && delay>0 && seq>0 {seq_nonzero++}; 336 END { 337 print "total_uberblocks " total; 338 print "delay_zero " delay_zero; 339 print "write_fail_present " write_fail_present; 340 print "write_fail_missing " write_fail_missing; 341 print "write_missing " write_missing; 342 print "fail_missing " fail_missing; 343 print "seq_nonzero " seq_nonzero; 344 print "uberblock_invalid " uber_invalid; 345 }' 346} 347 348function count_mmp_write_fail_present # device 349{ 350 typeset device=$1 351 352 summarize_uberblock_mmp $device | awk '/write_fail_present/ {print $NF}' 353} 354 355function count_mmp_write_fail_missing # device 356{ 357 typeset device=$1 358 359 summarize_uberblock_mmp $device | awk '/write_fail_missing/ {print $NF}' 360} 361 362function verify_mmp_write_fail_present # device 363{ 364 typeset device=$1 365 366 count=$(count_mmp_write_fail_present $device) 367 log_note "present count: $count" 368 if [ $count -eq 0 ]; then 369 summarize_uberblock_mmp $device 370 log_note "----- snip -----" 371 zdb -luuuu $device 372 log_note "----- snip -----" 373 log_fail "No Uberblocks contain valid mmp_write and fail values" 374 fi 375 376 count=$(count_mmp_write_fail_missing $device) 377 log_note "missing count: $count" 378 if [ $count -gt 0 ]; then 379 summarize_uberblock_mmp $device 380 log_note "----- snip -----" 381 zdb -luuuu $device 382 log_note "----- snip -----" 383 log_fail "Uberblocks missing mmp_write or mmp_fail" 384 fi 385} 386