1# SPDX-License-Identifier: CDDL-1.0 2# 3# CDDL HEADER START 4# 5# The contents of this file are subject to the terms of the 6# Common Development and Distribution License (the "License"). 7# You may not use this file except in compliance with the License. 8# 9# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 10# or https://opensource.org/licenses/CDDL-1.0. 11# See the License for the specific language governing permissions 12# and limitations under the License. 13# 14# When distributing Covered Code, include this CDDL HEADER in each 15# file and include the License file at usr/src/OPENSOLARIS.LICENSE. 16# If applicable, add the following below this CDDL HEADER, with the 17# fields enclosed by brackets "[]" replaced with your own identifying 18# information: Portions Copyright [yyyy] [name of copyright owner] 19# 20# CDDL HEADER END 21# 22 23# 24# Copyright (c) 2017 by Lawrence Livermore National Security, LLC. 25# Use is subject to license terms. 26# 27 28. $STF_SUITE/include/libtest.shlib 29. $STF_SUITE/tests/functional/mmp/mmp.cfg 30 31 32function check_pool_import # pool opts token keyword 33{ 34 typeset pool=${1:-$MMP_POOL} 35 typeset opts=$2 36 typeset token=$3 37 typeset keyword=$4 38 39 zpool import $opts 2>&1 | \ 40 awk -v token="$token:" '($1==token) {print}' | \ 41 grep -iq "$keyword" 42} 43 44function is_pool_imported # pool opts 45{ 46 typeset pool=${1:-$MMP_POOL} 47 typeset opts=$2 48 49 check_pool_import "$pool" "$opts" "status" \ 50 "The pool is currently imported" 51} 52 53function wait_pool_imported # pool opts 54{ 55 typeset pool=${1:-$MMP_POOL} 56 typeset opts=$2 57 58 while is_pool_imported "$pool" "$opts"; do 59 log_must sleep 5 60 done 61} 62 63function try_pool_import # pool opts message 64{ 65 typeset pool=${1:-$MMP_POOL} 66 typeset opts=$2 67 typeset msg=$3 68 69 zpool import $opts $pool 2>&1 | grep -i "$msg" 70} 71 72function mmp_set_hostid 73{ 74 typeset hostid=$1 75 76 zgenhostid $1 77 78 [ $(hostid) = "$hostid" ] 79} 80 81function mmp_clear_hostid 82{ 83 rm -f $HOSTID_FILE 84} 85 86function mmp_pool_create_simple # pool dir 87{ 88 typeset pool=${1:-$MMP_POOL} 89 typeset dir=${2:-$MMP_DIR} 90 91 log_must mkdir -p $dir 92 log_must rm -f $dir/* 93 log_must truncate -s $MINVDEVSIZE $dir/vdev1 $dir/vdev2 94 95 log_must mmp_clear_hostid 96 log_must mmp_set_hostid $HOSTID1 97 log_must zpool create -f -o cachefile=$MMP_CACHE $pool \ 98 mirror $dir/vdev1 $dir/vdev2 99 log_must zpool set multihost=on $pool 100} 101 102function mmp_pool_create # pool dir 103{ 104 typeset pool=${1:-$MMP_POOL} 105 typeset dir=${2:-$MMP_DIR} 106 typeset opts="-VVVVV -T120 -M -k0 -f $dir -E -p $pool" 107 108 mmp_pool_create_simple $pool $dir 109 110 log_must mv $MMP_CACHE ${MMP_CACHE}.stale 111 log_must zpool export $pool 112 log_must mmp_clear_hostid 113 log_must mmp_set_hostid $HOSTID2 114 115 log_note "Starting ztest in the background as hostid $HOSTID1" 116 log_must eval "ZFS_HOSTID=$HOSTID1 ztest $opts >$MMP_ZTEST_LOG 2>&1 &" 117 118 while ! is_pool_imported "$pool" "-d $dir"; do 119 log_must pgrep ztest 120 log_must sleep 5 121 done 122} 123 124function mmp_pool_destroy # pool dir 125{ 126 typeset pool=${1:-$MMP_POOL} 127 typeset dir=${2:-$MMP_DIR} 128 129 ZTESTPID=$(pgrep ztest) 130 if [ -n "$ZTESTPID" ]; then 131 log_must kill $ZTESTPID 132 wait $ZTESTPID 133 fi 134 135 if poolexists $pool; then 136 destroy_pool $pool 137 fi 138 139 log_must rm -f $dir/* 140 mmp_clear_hostid 141} 142 143function mmp_pool_set_hostid # pool hostid 144{ 145 typeset pool=$1 146 typeset hostid=$2 147 148 log_must mmp_clear_hostid 149 log_must mmp_set_hostid $hostid 150 log_must zpool export $pool 151 log_must zpool import $pool 152 153 return 0 154} 155 156function import_no_activity_check # pool opts 157{ 158 typeset pool=$1 159 typeset opts=$2 160 161 typeset max_duration=$((MMP_TEST_DURATION_DEFAULT-1)) 162 163 SECONDS=0 164 zpool import $opts $pool 165 typeset rc=$? 166 167 if [[ $SECONDS -gt $max_duration ]]; then 168 log_fail "ERROR: import_no_activity_check unexpected activity \ 169check (${SECONDS}s gt $max_duration)" 170 fi 171 172 return $rc 173} 174 175function import_activity_check # pool opts act_test_duration 176{ 177 typeset pool=$1 178 typeset opts=$2 179 typeset min_duration=${3:-$MMP_TEST_DURATION_DEFAULT} 180 181 SECONDS=0 182 zpool import $opts $pool 183 typeset rc=$? 184 185 if [[ $SECONDS -le $min_duration ]]; then 186 log_fail "ERROR: import_activity_check expected activity check \ 187(${SECONDS}s le min_duration $min_duration)" 188 fi 189 190 return $rc 191} 192 193function clear_mmp_history 194{ 195 log_must set_tunable64 MULTIHOST_HISTORY $MMP_HISTORY_OFF 196 log_must set_tunable64 MULTIHOST_HISTORY $MMP_HISTORY 197} 198 199function count_skipped_mmp_writes # pool duration 200{ 201 typeset pool=$1 202 typeset -i duration=$2 203 204 sleep $duration 205 kstat_pool $pool multihost | \ 206 awk 'BEGIN {count=0}; $NF == "-" {count++}; END {print count};' 207} 208 209function count_mmp_writes # pool duration 210{ 211 typeset pool=$1 212 typeset -i duration=$2 213 214 sleep $duration 215 kstat_pool $pool multihost | \ 216 awk 'BEGIN {count=0}; $NF != "-" {count++}; END {print count};' 217} 218 219function summarize_uberblock_mmp # device 220{ 221 typeset device=$1 222 223 zdb -luuuu $device | awk ' 224 BEGIN {write_fail_present=0; write_fail_missing=0; uber_invalid=0;} 225 /Uberblock\[[0-9][0-9]*\]/ {delay=-99; write=-99; fail=-99; total++; if (/invalid/) {uber_invalid++};}; 226 /mmp_fail/ {fail=$3}; 227 /mmp_seq/ {seq=$3}; 228 /mmp_write/ {write=$3}; 229 /mmp_delay/ {delay=$3; if (delay==0) {delay_zero++};}; 230 /mmp_valid/ && delay>0 && write>0 && fail>0 {write_fail_present++}; 231 /mmp_valid/ && delay>0 && (write<=0 || fail<=0) {write_fail_missing++}; 232 /mmp_valid/ && delay>0 && write<=0 {write_missing++}; 233 /mmp_valid/ && delay>0 && fail<=0 {fail_missing++}; 234 /mmp_valid/ && delay>0 && seq>0 {seq_nonzero++}; 235 END { 236 print "total_uberblocks " total; 237 print "delay_zero " delay_zero; 238 print "write_fail_present " write_fail_present; 239 print "write_fail_missing " write_fail_missing; 240 print "write_missing " write_missing; 241 print "fail_missing " fail_missing; 242 print "seq_nonzero " seq_nonzero; 243 print "uberblock_invalid " uber_invalid; 244 }' 245} 246 247function count_mmp_write_fail_present # device 248{ 249 typeset device=$1 250 251 summarize_uberblock_mmp $device | awk '/write_fail_present/ {print $NF}' 252} 253 254function count_mmp_write_fail_missing # device 255{ 256 typeset device=$1 257 258 summarize_uberblock_mmp $device | awk '/write_fail_missing/ {print $NF}' 259} 260 261function verify_mmp_write_fail_present # device 262{ 263 typeset device=$1 264 265 count=$(count_mmp_write_fail_present $device) 266 log_note "present count: $count" 267 if [ $count -eq 0 ]; then 268 summarize_uberblock_mmp $device 269 log_note "----- snip -----" 270 zdb -luuuu $device 271 log_note "----- snip -----" 272 log_fail "No Uberblocks contain valid mmp_write and fail values" 273 fi 274 275 count=$(count_mmp_write_fail_missing $device) 276 log_note "missing count: $count" 277 if [ $count -gt 0 ]; then 278 summarize_uberblock_mmp $device 279 log_note "----- snip -----" 280 zdb -luuuu $device 281 log_note "----- snip -----" 282 log_fail "Uberblocks missing mmp_write or mmp_fail" 283 fi 284} 285