1#!/bin/ksh 2# 3# This file and its contents are supplied under the terms of the 4# Common Development and Distribution License ("CDDL"), version 1.0. 5# You may only use this file in accordance with the terms of version 6# 1.0 of the CDDL. 7# 8# A full copy of the text of the CDDL should have accompanied this 9# source. A copy of the CDDL is also available via the Internet at 10# http://www.illumos.org/license/CDDL. 11# 12 13# 14# Copyright (c) 2018 by Delphix. All rights reserved. 15# 16 17typeset -a disk_array=($(find_disks $DISKS)) 18 19typeset -r DISK1=${disk_array[0]} 20typeset -r DISK2=${disk_array[1]} 21typeset -r DISK3=${disk_array[2]} 22 23# 24# When the condition it is waiting for becomes true, 'zpool wait' should return 25# promptly. We want to enforce this, but any check will be racey because it will 26# take some small but indeterminate amount of time for the waiting thread to be 27# woken up and for the process to exit. 28# 29# To deal with this, we provide a grace period after the condition becomes true 30# during which 'zpool wait' can exit. If it hasn't exited by the time the grace 31# period expires we assume something is wrong and fail the test. While there is 32# no value that can really be correct, the idea is we choose something large 33# enough that it shouldn't cause issues in practice. 34# 35typeset -r WAIT_EXIT_GRACE=2.0 36 37function add_io_delay # pool 38{ 39 for disk in $(get_disklist $1); do 40 log_must zinject -d $disk -D20:1 $1 41 done 42} 43 44function remove_io_delay 45{ 46 log_must zinject -c all 47} 48 49function proc_exists # pid 50{ 51 ps -p $1 >/dev/null 52} 53 54function proc_must_exist # pid 55{ 56 proc_exists $1 || log_fail "zpool process exited too soon" 57} 58 59function proc_must_not_exist # pid 60{ 61 proc_exists $1 && log_fail "zpool process took too long to exit" 62} 63 64function get_time 65{ 66 date +'%H:%M:%S' 67} 68 69function kill_if_running 70{ 71 typeset pid=$1 72 [[ $pid ]] && proc_exists $pid && log_must kill -s TERM $pid 73} 74 75# Log a command and then start it running in the background 76function log_bkgrnd 77{ 78 log_note "$(get_time) Starting cmd in background '$@'" 79 "$@" & 80} 81 82# Check that a background process has completed and exited with a status of 0 83function bkgrnd_proc_succeeded 84{ 85 typeset pid=$1 86 87 log_must sleep $WAIT_EXIT_GRACE 88 89 proc_must_not_exist $pid 90 wait $pid || log_fail "zpool process exited with status $?" 91 log_note "$(get_time) wait completed successfully" 92} 93 94# 95# Check that 'zpool wait' returns reasonably promptly after the condition 96# waited for becomes true, and not before. 97# 98function check_while_waiting 99{ 100 # The pid of the waiting process 101 typeset wait_proc_pid=$1 102 # A check that should be true while the activity is in progress 103 typeset activity_check=$2 104 105 log_note "$(get_time) waiting for process $wait_proc_pid using" \ 106 "activity check '$activity_check'" 107 while proc_exists $wait_proc_pid && eval "$activity_check"; do 108 log_must sleep .5 109 done 110 111 # 112 # If the activity being waited on is still in progress, then zpool wait 113 # exited too soon. 114 # 115 log_mustnot eval "$activity_check" 116 117 bkgrnd_proc_succeeded $wait_proc_pid 118} 119 120# Whether any vdev in the given pool is initializing 121function is_vdev_initializing # pool 122{ 123 zpool status -i "$1" | grep -q 'initialized, started' 124} 125