1# SPDX-License-Identifier: CDDL-1.0 2# 3# CDDL HEADER START 4# 5# This file and its contents are supplied under the terms of the 6# Common Development and Distribution License ("CDDL"), version 1.0. 7# You may only use this file in accordance with the terms of version 8# 1.0 of the CDDL. 9# 10# A full copy of the text of the CDDL should have accompanied this 11# source. A copy of the CDDL is also available via the Internet at 12# http://www.illumos.org/license/CDDL. 13# 14# CDDL HEADER END 15# 16 17# 18# Copyright (c) 2021 by Lawrence Livermore National Security, LLC. 19# 20 21. $STF_SUITE/include/libtest.shlib 22. $STF_SUITE/tests/functional/direct/dio.cfg 23 24function dio_cleanup 25{ 26 if poolexists $TESTPOOL1; then 27 destroy_pool $TESTPOOL1 28 fi 29 30 rm -f $DIO_VDEVS 31} 32 33# 34# Generate an IO workload using fio and then verify the resulting data. 35# 36function dio_and_verify # mode file-size block-size directory ioengine extra-args 37{ 38 typeset mode=$1 39 typeset size=$2 40 typeset bs=$3 41 typeset mntpnt=$4 42 typeset ioengine=$5 43 typeset extra_args=$6 44 45 # Invoke an fio workload via Direct I/O and verify with Direct I/O. 46 log_must fio --directory=$mntpnt --name=direct-$mode \ 47 --rw=$mode --size=$size --bs=$bs --direct=1 --numjobs=1 \ 48 --verify=sha1 --ioengine=$ioengine --fallocate=none \ 49 --group_reporting --minimal --do_verify=1 $extra_args 50 51 # Now just read back the file without Direct I/O into the ARC as an 52 # additional verfication step. 53 log_must fio --directory=$mntpnt --name=direct-$mode \ 54 --rw=read --size=$size --bs=$bs --direct=0 --numjobs=1 \ 55 --ioengine=$ioengine --group_reporting --minimal 56 57 log_must rm -f "$mntpnt/direct-*" 58} 59 60# 61# Get zpool status -d checksum verify failures 62# 63function get_zpool_status_chksum_verify_failures # pool_name vdev_type 64{ 65 typeset pool=$1 66 typeset vdev_type=$2 67 68 if [[ "$vdev_type" == "stripe" ]]; then 69 val=$(zpool status -dp $pool | \ 70 awk '{s+=$6} END {print s}' ) 71 elif [[ "$vdev_type" == "mirror" || "$vdev_type" == "raidz" || 72 "$vdev_type" == "draid" ]]; then 73 val=$(zpool status -dp $pool | \ 74 awk -v d="$vdev_type" '$0 ~ d {print $6}' ) 75 else 76 log_fail "Unsupported VDEV type in \ 77 get_zpool_status_chksum_verify_failures(): $vdev_type" 78 fi 79 echo "$val" 80} 81 82# 83# Get ZED dio_verify events 84# 85function get_zed_dio_verify_events # pool 86{ 87 typeset pool=$1 88 typeset op=$2 89 90 val=$(zpool events $pool | grep -c "dio_verify_${op}") 91 92 echo "$val" 93} 94 95# 96# Checking for checksum verify write failures with: 97# zpool status -d 98# zpool events 99# After getting that counts will clear the out the ZPool errors and events 100# 101function check_dio_chksum_verify_failures # pool vdev_type op expect_errors 102{ 103 typeset pool=$1 104 typeset vdev_type=$2 105 typeset expect_errors=$3 106 typeset op=$4 107 typeset note_str="expecting none" 108 109 if [[ $expect_errors -ne 0 ]]; then 110 note_str="expecting some" 111 fi 112 113 log_note "Checking for Direct I/O write checksum verify errors \ 114 $note_str on ZPool: $pool with $vdev_type" 115 116 status_failures=$(get_zpool_status_chksum_verify_failures $pool $vdev_type) 117 zed_dio_verify_events=$(get_zed_dio_verify_events $pool $op) 118 119 if [[ $expect_errors -ne 0 ]]; then 120 if [[ $status_failures -eq 0 || 121 $zed_dio_verify_events -eq 0 ]]; then 122 zpool status -dp $pool 123 zpool events $pool 124 log_fail "Checksum verifies in zpool status -d \ 125 $status_failures. ZED dio_verify events \ 126 $zed_dio_verify_events. Neither should be 0." 127 fi 128 else 129 if [[ $status_failures -ne 0 || 130 $zed_dio_verify_events -ne 0 ]]; then 131 zpool status -dp $pool 132 zpool events $pool 133 log_fail "Checksum verifies in zpool status -d \ 134 $status_failures. ZED dio_verify events \ 135 $zed_dio_verify_events. Both should be zero." 136 fi 137 fi 138 139 log_must zpool clear $pool 140 log_must zpool events -c 141 142} 143 144# 145# Evict any buffered blocks by overwritting them using an O_DIRECT request. 146# 147function evict_blocks 148{ 149 typeset pool=$1 150 typeset file=$2 151 typeset size=$3 152 153 log_must stride_dd -i /dev/urandom -o $file -b $size -c 1 -D 154} 155 156# 157# Perform FIO Direct I/O writes to a file with the given arguments. 158# Then verify thae minimum expected number of blocks were written as 159# Direct I/O. 160# 161function verify_dio_write_count #pool bs size mnpnt 162{ 163 typeset pool=$1 164 typeset bs=$2 165 typeset size=$3 166 typeset mntpnt=$4 167 typeset dio_wr_expected=$(((size / bs) -1)) 168 169 log_note "Checking for $dio_wr_expected Direct I/O writes" 170 171 prev_dio_wr=$(kstat_pool $pool iostats.direct_write_count) 172 dio_and_verify write $size $bs $mntpnt "sync" 173 curr_dio_wr=$(kstat_pool $pool iostats.direct_write_count) 174 dio_wr_actual=$((curr_dio_wr - prev_dio_wr)) 175 176 if [[ $dio_wr_actual -lt $dio_wr_expected ]]; then 177 kstat_pool -g $pool iostats 178 log_fail "Direct writes $dio_wr_actual of $dio_wr_expected" 179 fi 180} 181 182# 183# Perform a stride_dd write command to the file with the given arguments. 184# Then verify the minimum expected number of blocks were written as either 185# buffered IO (by the ARC), or Direct I/O to the application (dd). 186# 187function check_write # pool file bs count seek flags buf_wr dio_wr 188{ 189 typeset pool=$1 190 typeset file=$2 191 typeset bs=$3 192 typeset count=$4 193 typeset seek=$5 194 typeset flags=$6 195 typeset buf_wr_expect=$7 196 typeset dio_wr_expect=$8 197 198 log_note "Checking $count * $bs write(s) at offset $seek, $flags" 199 200 prev_buf_wr=$(kstat_pool $pool iostats.arc_write_count) 201 prev_dio_wr=$(kstat_pool $pool iostats.direct_write_count) 202 203 log_must stride_dd -i /dev/urandom -o $file -b $bs -c $count \ 204 -k $seek $flags 205 206 curr_buf_wr=$(kstat_pool $pool iostats.arc_write_count) 207 buf_wr_actual=$((curr_buf_wr - prev_buf_wr)) 208 209 curr_dio_wr=$(kstat_pool $pool iostats.direct_write_count) 210 dio_wr_actual=$((curr_dio_wr - prev_dio_wr)) 211 212 if [[ $buf_wr_actual -lt $buf_wr_expect ]]; then 213 kstat_pool -g $pool iostats 214 log_fail "Buffered writes $buf_wr_actual of $buf_wr_expect" 215 fi 216 217 if [[ $dio_wr_actual -lt $dio_wr_expect ]]; then 218 kstat_pool -g $pool iostats 219 log_fail "Direct writes $dio_wr_actual of $dio_wr_expect" 220 fi 221} 222 223# 224# Perform a stride_dd read command to the file with the given arguments. 225# Then verify the minimum expected number of blocks were read as either 226# buffered IO (by the ARC), or Direct I/O to the application (dd). 227# 228function check_read # pool file bs count skip flags buf_rd dio_rd 229{ 230 typeset pool=$1 231 typeset file=$2 232 typeset bs=$3 233 typeset count=$4 234 typeset skip=$5 235 typeset flags=$6 236 typeset buf_rd_expect=$7 237 typeset dio_rd_expect=$8 238 239 log_note "Checking $count * $bs read(s) at offset $skip, $flags" 240 241 prev_buf_rd=$(kstat_pool $pool iostats.arc_read_count) 242 prev_dio_rd=$(kstat_pool $pool iostats.direct_read_count) 243 244 log_must stride_dd -i $file -o /dev/null -b $bs -c $count \ 245 -p $skip $flags 246 247 curr_buf_rd=$(kstat_pool $pool iostats.arc_read_count) 248 buf_rd_actual=$((curr_buf_rd - prev_buf_rd)) 249 250 curr_dio_rd=$(kstat_pool $pool iostats.direct_read_count) 251 dio_rd_actual=$((curr_dio_rd - prev_dio_rd)) 252 253 if [[ $buf_rd_actual -lt $buf_rd_expect ]]; then 254 kstat_pool -g $pool iostats 255 log_fail "Buffered reads $buf_rd_actual of $buf_rd_expect" 256 fi 257 258 if [[ $dio_rd_actual -lt $dio_rd_expect ]]; then 259 kstat_pool -g $pool iostats 260 log_fail "Direct reads $dio_rd_actual of $dio_rd_expect" 261 fi 262} 263 264function get_file_size 265{ 266 typeset filename="$1" 267 268 if is_linux; then 269 filesize=$(stat -c %s $filename) 270 else 271 filesize=$(stat -s $filename | awk '{print $8}' | grep -o '[0-9]\+') 272 fi 273 274 echo $filesize 275} 276 277function do_truncate_reduce 278{ 279 typeset filename=$1 280 typeset size=$2 281 282 filesize=$(get_file_size $filename) 283 eval "echo original filesize: $filesize" 284 if is_linux; then 285 truncate $filename -s $((filesize - size)) 286 else 287 truncate -s -$size $filename 288 fi 289 filesize=$(get_file_size $filename) 290 eval "echo new filesize after truncate: $filesize" 291} 292