1# SPDX-License-Identifier: CDDL-1.0 2# 3# This file and its contents are supplied under the terms of the 4# Common Development and Distribution License ("CDDL"), version 1.0. 5# You may only use this file in accordance with the terms of version 6# 1.0 of the CDDL. 7# 8# A full copy of the text of the CDDL should have accompanied this 9# source. A copy of the CDDL is also available via the Internet at 10# http://www.illumos.org/license/CDDL. 11# 12 13# 14# Copyright (c) 2017, 2018 by Delphix. All rights reserved. 15# 16 17. $STF_SUITE/include/libtest.shlib 18. $STF_SUITE/tests/functional/removal/removal.kshlib 19 20# 21# In general all the tests related to the pool checkpoint can 22# be divided into two categories. TESTS that verify features 23# provided by the checkpoint (e.g. checkpoint_rewind) and tests 24# that stress-test the checkpoint (e.g. checkpoint_big_rewind). 25# 26# For the first group we don't really care about the size of 27# the pool or the individual file sizes within the filesystems. 28# This is why these tests run directly on pools that use a 29# "real disk vdev" (meaning not a file based one). These tests 30# use the $TESTPOOL pool that is created on top of $TESTDISK. 31# This pool is referred to as the "test pool" and thus all 32# the tests of this group use the testpool-related functions of 33# this file (not the nested_pools ones). 34# 35# For the second group we generally try to bring the pool to its 36# limits by increasing fragmentation, filling all allocatable 37# space, attempting to use vdevs that the checkpoint spacemap 38# cannot represent, etc. For these tests we need to control 39# almost all parameters of the pool and the vdevs that back it 40# so we create them based on file-based vdevs that we carefully 41# create within the $TESTPOOL pool. So most of these tests, in 42# order to create this nested pool sctructure, generally start 43# like this: 44# 1] We create the test pool ($TESTPOOL). 45# 2] We create a filesystem and we populate it with files of 46# some predetermined size. 47# 3] We use those files as vdevs for the pool that the test 48# will use ($NESTEDPOOL). 49# 4] Go on and let the test run and operate on $NESTEDPOOL. 50# 51 52# 53# These disks are used to back $TESTPOOL 54# 55TESTDISK="$(echo $DISKS | cut -d' ' -f1)" 56EXTRATESTDISK="$(echo $DISKS | cut -d' ' -f2)" 57 58FS0=$TESTPOOL/$TESTFS 59FS1=$TESTPOOL/$TESTFS1 60FS2=$TESTPOOL/$TESTFS2 61 62FS0FILE=/$FS0/$TESTFILE0 63FS1FILE=/$FS1/$TESTFILE1 64FS2FILE=/$FS2/$TESTFILE2 65 66# 67# The following are created within $TESTPOOL and 68# will be used to back $NESTEDPOOL 69# 70DISKFS=$TESTPOOL/disks 71FILEDISKDIR=/$DISKFS 72FILEDISK1=/$DISKFS/dsk1 73FILEDISK2=/$DISKFS/dsk2 74FILEDISKS="$FILEDISK1 $FILEDISK2" 75 76# 77# $NESTEDPOOL related variables 78# 79NESTEDPOOL=nestedpool 80NESTEDFS0=$NESTEDPOOL/$TESTFS 81NESTEDFS1=$NESTEDPOOL/$TESTFS1 82NESTEDFS2=$NESTEDPOOL/$TESTFS2 83NESTEDFS0FILE=/$NESTEDFS0/$TESTFILE0 84NESTEDFS1FILE=/$NESTEDFS1/$TESTFILE1 85NESTEDFS2FILE=/$NESTEDFS2/$TESTFILE2 86 87# 88# In the tests that stress-test the pool (second category 89# mentioned above), there exist some that need to bring 90# fragmentation at high percentages in a relatively short 91# period of time. In order to do that we set the following 92# parameters: 93# 94# * We use two disks of 1G each, to create a pool of size 2G. 95# The point is that 2G is not small nor large, and we also 96# want to have 2 disks to introduce indirect vdevs on our 97# setup. 98# * We enable compression and set the record size of all 99# filesystems to 8K. The point of compression is to 100# ensure that we are not filling up the whole pool (that's 101# what checkpoint_capacity is for), and the specific 102# record size is set to match the block size of randwritecomp 103# which is used to increase fragmentation by writing on 104# files. 105# * We always have 2 big files present of 512M each, which 106# should account for 40%~50% capacity by the end of each 107# test with fragmentation around 50~60%. 108# * At each file we attempt to do enough random writes to 109# touch every offset twice on average. 110# 111# Note that the amount of random writes per files are based 112# on the following calculation: 113# 114# ((512M / 8K) * 3) * 2 = ~400000 115# 116# Given that the file is 512M and one write is 8K, we would 117# need (512M / 8K) writes to go through the whole file. 118# Assuming though that each write has a compression ratio of 119# 3, then we want 3 times that to cover the same amount of 120# space. Finally, we multiply that by 2 since our goal is to 121# touch each offset twice on average. 122# 123# Examples of those tests are checkpoint_big_rewind and 124# checkpoint_discard_busy. 125# 126FILEDISKSIZE=1g 127DISKSIZE=1g 128BIGFILESIZE=512M 129RANDOMWRITES=400000 130 131 132# 133# Assumes create_test_pool has been called beforehand. 134# 135function setup_nested_pool 136{ 137 log_must zfs create $DISKFS 138 139 log_must truncate -s $DISKSIZE $FILEDISK1 140 log_must truncate -s $DISKSIZE $FILEDISK2 141 142 log_must zpool create -O primarycache=metadata -O sync=disabled \ 143 $NESTEDPOOL $FILEDISKS 144} 145 146function setup_test_pool 147{ 148 log_must zpool create -O sync=disabled $TESTPOOL "$TESTDISK" 149} 150 151function setup_nested_pools 152{ 153 setup_test_pool 154 setup_nested_pool 155} 156 157function cleanup_nested_pool 158{ 159 if poolexists $NESTEDPOOL; then 160 log_must zpool destroy $NESTEDPOOL 161 fi 162 163 log_must rm -f $FILEDISKS 164} 165 166function cleanup_test_pool 167{ 168 if poolexists $TESTPOOL; then 169 log_must zpool destroy $TESTPOOL 170 fi 171 172 # 173 # We always clear the labels of all disks 174 # between tests so imports from zpool or 175 # or zdb do not get confused with leftover 176 # data from old pools. 177 # 178 for disk in $DISKS; do 179 zpool labelclear -f $disk 180 done 181} 182 183function cleanup_nested_pools 184{ 185 cleanup_nested_pool 186 cleanup_test_pool 187} 188 189# 190# Remove and re-add each vdev to ensure that data is 191# moved between disks and indirect mappings are created 192# 193function introduce_indirection 194{ 195 for disk in ${FILEDISKS[@]}; do 196 log_must zpool remove $NESTEDPOOL $disk 197 log_must wait_for_removal $NESTEDPOOL 198 log_mustnot vdevs_in_pool $NESTEDPOOL $disk 199 log_must zpool add $NESTEDPOOL $disk 200 done 201} 202 203FILECONTENTS0="Can't wait to be checkpointed!" 204FILECONTENTS1="Can't wait to be checkpointed too!" 205NEWFILECONTENTS0="I survived after the checkpoint!" 206NEWFILECONTENTS2="I was born after the checkpoint!" 207 208function populate_test_pool 209{ 210 log_must zfs create -o compression=lz4 -o recordsize=8k $FS0 211 log_must zfs create -o compression=lz4 -o recordsize=8k $FS1 212 213 echo $FILECONTENTS0 > $FS0FILE 214 echo $FILECONTENTS1 > $FS1FILE 215} 216 217function populate_nested_pool 218{ 219 log_must zfs create -o compression=lz4 -o recordsize=8k $NESTEDFS0 220 log_must zfs create -o compression=lz4 -o recordsize=8k $NESTEDFS1 221 222 echo $FILECONTENTS0 > $NESTEDFS0FILE 223 echo $FILECONTENTS1 > $NESTEDFS1FILE 224} 225 226function test_verify_pre_checkpoint_state 227{ 228 log_must zfs list $FS0 229 log_must zfs list $FS1 230 log_must [ "$(<$FS0FILE)" = "$FILECONTENTS0" ] 231 log_must [ "$(<$FS1FILE)" = "$FILECONTENTS1" ] 232 233 # 234 # If we've opened the checkpointed state of the 235 # pool as read-only without rewinding on-disk we 236 # can't really use zdb on it. 237 # 238 if [[ "$1" != "ro-check" ]] ; then 239 log_must zdb $TESTPOOL 240 fi 241 242 # 243 # Ensure post-checkpoint state is not present 244 # 245 log_mustnot zfs list $FS2 246 log_mustnot [ "$(<$FS0FILE)" = "$NEWFILECONTENTS0" ] 247} 248 249function nested_verify_pre_checkpoint_state 250{ 251 log_must zfs list $NESTEDFS0 252 log_must zfs list $NESTEDFS1 253 log_must [ "$(<$NESTEDFS0FILE)" = "$FILECONTENTS0" ] 254 log_must [ "$(<$NESTEDFS1FILE)" = "$FILECONTENTS1" ] 255 256 # 257 # If we've opened the checkpointed state of the 258 # pool as read-only without rewinding on-disk we 259 # can't really use zdb on it. 260 # 261 if [[ "$1" != "ro-check" ]] ; then 262 log_must zdb $NESTEDPOOL 263 fi 264 265 # 266 # Ensure post-checkpoint state is not present 267 # 268 log_mustnot zfs list $NESTEDFS2 269 log_mustnot [ "$(<$NESTEDFS0FILE)" = "$NEWFILECONTENTS0" ] 270} 271 272function test_change_state_after_checkpoint 273{ 274 log_must zfs destroy $FS1 275 log_must zfs create -o compression=lz4 -o recordsize=8k $FS2 276 277 echo $NEWFILECONTENTS0 > $FS0FILE 278 echo $NEWFILECONTENTS2 > $FS2FILE 279} 280 281function nested_change_state_after_checkpoint 282{ 283 log_must zfs destroy $NESTEDFS1 284 log_must zfs create -o compression=lz4 -o recordsize=8k $NESTEDFS2 285 286 echo $NEWFILECONTENTS0 > $NESTEDFS0FILE 287 echo $NEWFILECONTENTS2 > $NESTEDFS2FILE 288} 289 290function test_verify_post_checkpoint_state 291{ 292 log_must zfs list $FS0 293 log_must zfs list $FS2 294 log_must [ "$(<$FS0FILE)" = "$NEWFILECONTENTS0" ] 295 log_must [ "$(<$FS2FILE)" = "$NEWFILECONTENTS2" ] 296 297 log_must zdb $TESTPOOL 298 299 # 300 # Ensure pre-checkpointed state that was removed post-checkpoint 301 # is not present 302 # 303 log_mustnot zfs list $FS1 304 log_mustnot [ "$(<$FS0FILE)" = "$FILECONTENTS0" ] 305} 306 307function fragment_before_checkpoint 308{ 309 populate_nested_pool 310 log_must mkfile -n $BIGFILESIZE $NESTEDFS0FILE 311 log_must mkfile -n $BIGFILESIZE $NESTEDFS1FILE 312 log_must randwritecomp $NESTEDFS0FILE $RANDOMWRITES 313 log_must randwritecomp $NESTEDFS1FILE $RANDOMWRITES 314 315 # 316 # Display fragmentation on test log 317 # 318 log_must zpool list -v 319} 320 321function fragment_after_checkpoint_and_verify 322{ 323 log_must zfs destroy $NESTEDFS1 324 log_must zfs create -o compression=lz4 -o recordsize=8k $NESTEDFS2 325 log_must mkfile -n $BIGFILESIZE $NESTEDFS2FILE 326 log_must randwritecomp $NESTEDFS0FILE $RANDOMWRITES 327 log_must randwritecomp $NESTEDFS2FILE $RANDOMWRITES 328 329 # 330 # Display fragmentation on test log 331 # 332 log_must zpool list -v 333 334 # 335 # Typically we would just run zdb at this point and things 336 # would be fine. Unfortunately, if there is still any 337 # background I/O in the pool the zdb command can fail with 338 # checksum errors temporarily. 339 # 340 # Export the pool when running zdb so the pool is idle and 341 # the verification results are consistent. 342 # 343 log_must zpool export $NESTEDPOOL 344 log_must zdb -e -p $FILEDISKDIR $NESTEDPOOL 345 log_must zdb -e -p $FILEDISKDIR -kc $NESTEDPOOL 346 log_must zpool import -d $FILEDISKDIR $NESTEDPOOL 347} 348 349function wait_discard_finish 350{ 351 typeset pool="$1" 352 353 typeset status 354 status=$(zpool status $pool | grep "checkpoint:") 355 while [ "" != "$status" ]; do 356 sleep 5 357 status=$(zpool status $pool | grep "checkpoint:") 358 done 359} 360 361function test_wait_discard_finish 362{ 363 wait_discard_finish $TESTPOOL 364} 365 366function nested_wait_discard_finish 367{ 368 wait_discard_finish $NESTEDPOOL 369} 370 371# 372# Creating the setup for the second group of tests mentioned in 373# block comment of this file can take some time as we are doing 374# random writes to raise capacity and fragmentation before taking 375# the checkpoint. Thus we create this setup once and save the 376# disks of the nested pool in a temporary directory where we can 377# reuse it for each test that requires that setup. 378# 379SAVEDPOOLDIR="$TEST_BASE_DIR/ckpoint_saved_pool" 380 381function test_group_premake_nested_pools 382{ 383 setup_nested_pools 384 385 # 386 # Populate and fragment the pool. 387 # 388 fragment_before_checkpoint 389 390 # 391 # Export and save the pool for other tests. 392 # 393 log_must zpool export $NESTEDPOOL 394 log_must mkdir $SAVEDPOOLDIR 395 log_must cp $FILEDISKS $SAVEDPOOLDIR 396 397 # 398 # Reimport pool to be destroyed by 399 # cleanup_nested_pools function 400 # 401 log_must zpool import -d $FILEDISKDIR $NESTEDPOOL 402} 403 404function test_group_destroy_saved_pool 405{ 406 log_must rm -rf $SAVEDPOOLDIR 407} 408 409# 410# Recreate nested pool setup from saved pool. 411# 412function setup_nested_pool_state 413{ 414 setup_test_pool 415 416 log_must zfs create $DISKFS 417 log_must cp $SAVEDPOOLDIR/* $FILEDISKDIR 418 419 log_must zpool import -d $FILEDISKDIR $NESTEDPOOL 420} 421