1# 2# CDDL HEADER START 3# 4# The contents of this file are subject to the terms of the 5# Common Development and Distribution License (the "License"). 6# You may not use this file except in compliance with the License. 7# 8# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9# or http://www.opensolaris.org/os/licensing. 10# See the License for the specific language governing permissions 11# and limitations under the License. 12# 13# When distributing Covered Code, include this CDDL HEADER in each 14# file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15# If applicable, add the following below this CDDL HEADER, with the 16# fields enclosed by brackets "[]" replaced with your own identifying 17# information: Portions Copyright [yyyy] [name of copyright owner] 18# 19# CDDL HEADER END 20# 21 22# 23# Copyright (c) 2017 by Lawrence Livermore National Security, LLC. 24# Use is subject to license terms. 25# Copyright 2019 Joyent, Inc. 26# Copyright 2024 MNX Cloud, Inc. 27# 28 29. $STF_SUITE/include/libtest.shlib 30. $STF_SUITE/tests/functional/mmp/mmp.cfg 31 32 33function check_pool_import # pool opts token keyword 34{ 35 typeset pool=${1:-$MMP_POOL} 36 typeset opts=$2 37 typeset token=$3 38 typeset keyword=$4 39 40 zpool import $opts 2>&1 | \ 41 nawk -v token="$token:" '($1==token) {print $0}' | \ 42 grep -i "$keyword" > /dev/null 2>&1 43 44 return $? 45} 46 47function is_pool_imported # pool opts 48{ 49 typeset pool=${1:-$MMP_POOL} 50 typeset opts=$2 51 52 check_pool_import "$pool" "$opts" "status" \ 53 "The pool is currently imported" 54 return $? 55} 56 57function wait_pool_imported # pool opts 58{ 59 typeset pool=${1:-$MMP_POOL} 60 typeset opts=$2 61 62 while is_pool_imported "$pool" "$opts"; do 63 log_must sleep 5 64 done 65 66 return 0 67} 68 69function try_pool_import # pool opts message 70{ 71 typeset pool=${1:-$MMP_POOL} 72 typeset opts=$2 73 typeset msg=$3 74 75 zpool import $opts $pool 2>&1 | grep -i "$msg" 76 77 return $? 78} 79 80function chr2ascii 81{ 82 case "$1" in 83 0) asc="30";; 84 1) asc="31";; 85 2) asc="32";; 86 3) asc="33";; 87 4) asc="34";; 88 5) asc="35";; 89 6) asc="36";; 90 7) asc="37";; 91 8) asc="38";; 92 9) asc="39";; 93 a) asc="61";; 94 b) asc="62";; 95 c) asc="63";; 96 d) asc="64";; 97 e) asc="65";; 98 f) asc="66";; 99 esac 100} 101 102function mmp_set_hostid 103{ 104 typeset hostid=$1 105 106 case "$(uname)" in 107 Linux) 108 a=${hostid:6:2} 109 b=${hostid:4:2} 110 c=${hostid:2:2} 111 d=${hostid:0:2} 112 113 printf "\\x$a\\x$b\\x$c\\x$d" >$HOSTID_FILE 114 115 if [ $(hostid) != "$hostid" ]; then 116 return 1 117 fi 118 ;; 119 SunOS) 120 # 121 # Given a hostid in hex, we have to convert to decimal, then 122 # save the ascii string representation in the kernel. The 123 # 'hostid' command will get the decimal SI_HW_SERIAL value via 124 # sysinfo, then print that as an 8 digit hex number. 125 # 126 typeset dec=$(mdb -e "$hostid=E" | sed -e 's/ *//g') 127 typeset len=$(echo $dec | awk '{print length($0)}') 128 if [[ $len -lt 0 || $len -gt 10 ]]; then 129 return 130 fi 131 typeset pos=0 132 while [[ $pos -lt $len ]]; do 133 chr2ascii ${dec:$pos:1} 134 echo "hw_serial+${pos}/v $asc" | mdb -kw >/dev/null 2>&1 135 pos=$(($pos + 1)) 136 done 137 echo "hw_serial+${pos}/v 0" | mdb -kw >/dev/null 2>&1 138 ;; 139 esac 140 141 return 0 142} 143 144function mmp_clear_hostid 145{ 146 case "$(uname)" in 147 Linux) rm -f $HOSTID_FILE;; 148 SunOS) mmp_set_hostid "00000000";; 149 esac 150} 151 152function mmp_pool_create_simple # pool dir 153{ 154 typeset pool=${1:-$MMP_POOL} 155 typeset dir=${2:-$MMP_DIR} 156 157 log_must mkdir -p $dir 158 log_must rm -f $dir/* 159 log_must truncate -s $MINVDEVSIZE $dir/vdev1 $dir/vdev2 160 161 log_must mmp_set_hostid $HOSTID1 162 log_must zpool create -f -o cachefile=$MMP_CACHE $pool \ 163 mirror $dir/vdev1 $dir/vdev2 164 log_must zpool set multihost=on $pool 165} 166 167function mmp_pool_create # pool dir 168{ 169 typeset pool=${1:-$MMP_POOL} 170 typeset dir=${2:-$MMP_DIR} 171 typeset opts="-VVVVV -T120 -M -k0 -f $dir -E -p $pool" 172 173 mmp_pool_create_simple $pool $dir 174 175 log_must mv $MMP_CACHE ${MMP_CACHE}.stale 176 log_must zpool export $pool 177 log_must mmp_set_hostid $HOSTID2 178 179 log_note "Starting ztest in the background as hostid $HOSTID1" 180 log_must eval "ZFS_HOSTID=$HOSTID1 /usr/bin/ztest $opts >$MMP_ZTEST_LOG 2>&1 &" 181 182 while ! is_pool_imported "$pool" "-d $dir"; do 183 log_must pgrep ztest 184 log_must sleep 5 185 done 186} 187 188function mmp_pool_destroy # pool dir 189{ 190 typeset pool=${1:-$MMP_POOL} 191 typeset dir=${2:-$MMP_DIR} 192 193 ZTESTPID=$(pgrep ztest) 194 if [ -n "$ZTESTPID" ]; then 195 log_must kill $ZTESTPID 196 wait $ZTESTPID 197 fi 198 199 if poolexists $pool; then 200 destroy_pool $pool 201 fi 202 203 if [[ -d $dir ]]; then 204 log_must rm -f $dir/* 205 log_must rmdir $dir 206 fi 207 mmp_clear_hostid 208} 209 210function mmp_pool_set_hostid # pool hostid 211{ 212 typeset pool=$1 213 typeset hostid=$2 214 215 log_must mmp_set_hostid $hostid 216 log_must zpool export $pool 217 log_must zpool import $pool 218 219 return 0 220} 221# Return the number of seconds the activity check portion of the import process 222# will take. Does not include the time to find devices and assemble a config. 223# Note that the activity check may be skipped, e.g. if the pool and host 224# hostid's match, but this will return non-zero because mmp_* are populated. 225function seconds_mmp_waits_for_activity 226{ 227 typeset pool=$1 228 typeset devpath=$2 229 230 typeset seconds=0 231 typeset devices=${#DISK[@]} 232 typeset import_intervals=$(get_tunable zfs_multihost_import_intervals) 233 typeset import_interval=$(get_tunable zfs_multihost_interval) 234 typeset tmpfile=$(mktemp) 235 typeset mmp_fail 236 typeset mmp_write 237 typeset mmp_delay 238 239 log_must zdb -e -p $devpath $pool >$tmpfile 2>/dev/null 240 mmp_fail=$(awk '/mmp_fail/ {print $NF}' $tmpfile) 241 mmp_write=$(awk '/mmp_write/ {print $NF}' $tmpfile) 242 mmp_delay=$(awk '/mmp_delay/ {print $NF}' $tmpfile) 243 if [ -f $tmpfile ]; then 244 rm $tmpfile 245 fi 246 247 # In order of preference: 248 if [ -n $mmp_fail -a -n $mmp_write ]; then 249 seconds=$((2*mmp_fail*mmp_write/1000)) 250 elif [ -n $mmp_delay ]; then 251 # MMP V0: Based on mmp_delay from the best Uberblock 252 seconds=$((import_intervals*devices*mmp_delay/1000000000)) 253 else 254 # Non-MMP aware: Based on zfs_multihost_interval and import_intervals 255 seconds=$((import_intervals*import_interval/1000)) 256 fi 257 258 echo $seconds 259} 260 261function import_no_activity_check # pool opts 262{ 263 typeset pool=$1 264 typeset opts=$2 265 266 typeset max_duration=$((MMP_TEST_DURATION_DEFAULT-1)) 267 268 SECONDS=0 269 zpool import $opts $pool 270 typeset rc=$? 271 272 if [[ $SECONDS -gt $max_duration ]]; then 273 log_fail "ERROR: import_no_activity_check unexpected activity \ 274check (${SECONDS}s gt $max_duration)" 275 fi 276 277 return $rc 278} 279 280function import_activity_check # pool opts act_test_duration 281{ 282 typeset pool=$1 283 typeset opts=$2 284 typeset min_duration=${3:-$MMP_TEST_DURATION_DEFAULT} 285 286 SECONDS=0 287 zpool import $opts $pool 288 typeset rc=$? 289 290 if [[ $SECONDS -le $min_duration ]]; then 291 log_fail "ERROR: import_activity_check expected activity check \ 292(${SECONDS}s le min_duration $min_duration)" 293 fi 294 295 return $rc 296} 297 298function clear_mmp_history 299{ 300 log_must set_tunable64 zfs_multihost_history $MMP_HISTORY_OFF 301 log_must set_tunable64 zfs_multihost_history $MMP_HISTORY 302} 303 304function count_skipped_mmp_writes # pool duration 305{ 306 typeset pool=$1 307 typeset -i duration=$2 308 typeset hist_path="/proc/spl/kstat/zfs/$pool/multihost" 309 310 sleep $duration 311 awk 'BEGIN {count=0}; $NF == "-" {count++}; END {print count};' "$hist_path" 312} 313 314function count_mmp_writes # pool duration 315{ 316 typeset pool=$1 317 typeset -i duration=$2 318 typeset hist_path="/proc/spl/kstat/zfs/$pool/multihost" 319 320 log_must sleep $duration 321 awk 'BEGIN {count=0}; $NF != "-" {count++}; END {print count};' "$hist_path" 322} 323 324function summarize_uberblock_mmp # device 325{ 326 typeset device=$1 327 328 zdb -luuuu $device | awk ' 329 BEGIN {write_fail_present=0; write_fail_missing=0; uber_invalid=0;} 330 /Uberblock\[[0-9][0-9]*\]/ {delay=-99; write=-99; fail=-99; total++; if (/invalid/) {uber_invalid++};}; 331 /mmp_fail/ {fail=$3}; 332 /mmp_seq/ {seq=$3}; 333 /mmp_write/ {write=$3}; 334 /mmp_delay/ {delay=$3; if (delay==0) {delay_zero++};}; 335 /mmp_valid/ && delay>0 && write>0 && fail>0 {write_fail_present++}; 336 /mmp_valid/ && delay>0 && (write<=0 || fail<=0) {write_fail_missing++}; 337 /mmp_valid/ && delay>0 && write<=0 {write_missing++}; 338 /mmp_valid/ && delay>0 && fail<=0 {fail_missing++}; 339 /mmp_valid/ && delay>0 && seq>0 {seq_nonzero++}; 340 END { 341 print "total_uberblocks " total; 342 print "delay_zero " delay_zero; 343 print "write_fail_present " write_fail_present; 344 print "write_fail_missing " write_fail_missing; 345 print "write_missing " write_missing; 346 print "fail_missing " fail_missing; 347 print "seq_nonzero " seq_nonzero; 348 print "uberblock_invalid " uber_invalid; 349 }' 350} 351 352function count_mmp_write_fail_present # device 353{ 354 typeset device=$1 355 356 summarize_uberblock_mmp $device | awk '/write_fail_present/ {print $NF}' 357} 358 359function count_mmp_write_fail_missing # device 360{ 361 typeset device=$1 362 363 summarize_uberblock_mmp $device | awk '/write_fail_missing/ {print $NF}' 364} 365 366function verify_mmp_write_fail_present # device 367{ 368 typeset device=$1 369 370 count=$(count_mmp_write_fail_present $device) 371 log_note "present count: $count" 372 if [ $count -eq 0 ]; then 373 summarize_uberblock_mmp $device 374 log_note "----- snip -----" 375 zdb -luuuu $device 376 log_note "----- snip -----" 377 log_fail "No Uberblocks contain valid mmp_write and fail values" 378 fi 379 380 count=$(count_mmp_write_fail_missing $device) 381 log_note "missing count: $count" 382 if [ $count -gt 0 ]; then 383 summarize_uberblock_mmp $device 384 log_note "----- snip -----" 385 zdb -luuuu $device 386 log_note "----- snip -----" 387 log_fail "Uberblocks missing mmp_write or mmp_fail" 388 fi 389} 390