1#!/bin/ksh -p 2# SPDX-License-Identifier: CDDL-1.0 3# 4# CDDL HEADER START 5# 6# The contents of this file are subject to the terms of the 7# Common Development and Distribution License (the "License"). 8# You may not use this file except in compliance with the License. 9# 10# You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 11# or http://www.opensolaris.org/os/licensing. 12# See the License for the specific language governing permissions 13# and limitations under the License. 14# 15# When distributing Covered Code, include this CDDL HEADER in each 16# file and include the License file at usr/src/OPENSOLARIS.LICENSE. 17# If applicable, add the following below this CDDL HEADER, with the 18# fields enclosed by brackets "[]" replaced with your own identifying 19# information: Portions Copyright [yyyy] [name of copyright owner] 20# 21# CDDL HEADER END 22# 23 24# 25# Copyright (c) 2021 by vStack. All rights reserved. 26# 27 28. $STF_SUITE/include/libtest.shlib 29 30# 31# DESCRIPTION: 32# Check device replacement during raidz expansion using expansion pausing. 33# 34# STRATEGY: 35# 1. Create block device files for the test raidz pool 36# 2. For each parity value [1..3] 37# - create raidz pool with minimum block device files required 38# - create couple of datasets with different recordsize and fill it 39# - set raidz expand maximum reflow bytes 40# - attach new device to the pool 41# - wait for reflow bytes to reach the maximum 42# - offline and zero vdevs allowed by parity 43# - wait some time and start offlined vdevs replacement 44# - wait replacement completion and verify pool status 45# - loop thru vdevs replacing with the max reflow bytes increasing 46# - verify pool 47# - set reflow bytes to max value to complete the expansion 48 49typeset -r devs=10 50typeset -r dev_size_mb=128 51 52typeset -a disks 53 54embedded_slog_min_ms=$(get_tunable EMBEDDED_SLOG_MIN_MS) 55original_scrub_after_expand=$(get_tunable SCRUB_AFTER_EXPAND) 56 57function cleanup 58{ 59 poolexists "$TESTPOOL" && zpool status -v "$TESTPOOL" 60 poolexists "$TESTPOOL" && log_must_busy zpool destroy "$TESTPOOL" 61 62 for i in {0..$devs}; do 63 log_must rm -f "$TEST_BASE_DIR/dev-$i" 64 done 65 66 log_must set_tunable32 EMBEDDED_SLOG_MIN_MS $embedded_slog_min_ms 67 log_must set_tunable64 RAIDZ_EXPAND_MAX_REFLOW_BYTES 0 68 log_must set_tunable32 SCRUB_AFTER_EXPAND $original_scrub_after_expand 69} 70 71function wait_expand_paused 72{ 73 oldcopied='0' 74 newcopied='1' 75 while [[ $oldcopied != $newcopied ]]; do 76 oldcopied=$newcopied 77 sleep 1 78 newcopied=$(zpool status $TESTPOOL | \ 79 grep 'copied out of' | \ 80 awk '{print $1}') 81 done 82} 83 84log_onexit cleanup 85 86function test_replace # <pool> <devices> <parity> 87{ 88 pool=${1} 89 devices=${2} 90 nparity=${3} 91 device_count=0 92 93 log_must echo "devices=$devices" 94 95 for dev in ${devices}; do 96 device_count=$((device_count+1)) 97 done 98 99 index=$((RANDOM%(device_count-nparity))) 100 for (( j=1; j<=$nparity; j=j+1 )); do 101 log_must zpool offline $pool ${disks[$((index+j))]} 102 log_must dd if=/dev/zero of=${disks[$((index+j))]} \ 103 bs=1024k count=$dev_size_mb conv=notrunc 104 done 105 106 for (( j=1; j<=$nparity; j=j+1 )); do 107 log_must zpool replace $pool ${disks[$((index+j))]} 108 done 109 110 log_must zpool wait -t replace $pool 111 log_must check_pool_status $pool "scan" "with 0 errors" 112 113 log_must zpool clear $pool 114 log_must zpool scrub -w $pool 115 116 log_must zpool status -v 117 log_must check_pool_status $pool "scan" "with 0 errors" 118} 119 120log_must set_tunable32 EMBEDDED_SLOG_MIN_MS 99999 121 122# Disk files which will be used by pool 123for i in {0..$(($devs))}; do 124 device=$TEST_BASE_DIR/dev-$i 125 log_must truncate -s ${dev_size_mb}M $device 126 disks[${#disks[*]}+1]=$device 127done 128 129nparity=$((RANDOM%(3) + 1)) 130raid=raidz$nparity 131pool=$TESTPOOL 132opts="-o cachefile=none" 133devices="" 134 135log_must set_tunable32 SCRUB_AFTER_EXPAND 0 136 137log_must zpool create -f $opts $pool $raid ${disks[1..$(($nparity+1))]} 138devices="${disks[1..$(($nparity+1))]}" 139 140log_must zfs create -o recordsize=8k $pool/fs 141log_must fill_fs /$pool/fs 1 128 102400 1 R 142 143log_must zfs create -o recordsize=128k $pool/fs2 144log_must fill_fs /$pool/fs2 1 128 102400 1 R 145 146for disk in ${disks[$(($nparity+2))..$devs]}; do 147 # Set pause to some random value near halfway point 148 reflow_size=$(get_pool_prop allocated $pool) 149 pause=$((((RANDOM << 15) + RANDOM) % reflow_size / 2)) 150 log_must set_tunable64 RAIDZ_EXPAND_MAX_REFLOW_BYTES $pause 151 152 log_must zpool attach $pool ${raid}-0 $disk 153 devices="$devices $disk" 154 155 wait_expand_paused 156 157 for (( i=0; i<2; i++ )); do 158 test_replace $pool "$devices" $nparity 159 160 # Increase pause by about 25% 161 pause=$((pause + (((RANDOM << 15) + RANDOM) % \ 162 reflow_size) / 4)) 163 log_must set_tunable64 RAIDZ_EXPAND_MAX_REFLOW_BYTES $pause 164 165 wait_expand_paused 166 done 167 168 # Set pause past largest possible value for this pool 169 pause=$((devs*dev_size_mb*1024*1024)) 170 log_must set_tunable64 RAIDZ_EXPAND_MAX_REFLOW_BYTES $pause 171 172 log_must zpool wait -t raidz_expand $pool 173done 174 175log_must zpool destroy "$pool" 176 177log_pass "raidz expansion test succeeded." 178 179