common/os/dkioc_free_util.c

/*
 * This file and its contents are supplied under the terms of the
 * Common Development and Distribution License ("CDDL"), version 1.0.
 * You may only use this file in accordance with the terms of version
 * 1.0 of the CDDL.
 *
 * A full copy of the text of the CDDL should have accompanied this
 * source.  A copy of the CDDL is also available via the Internet at
 * http://www.illumos.org/license/CDDL.
 */

/*
 * Copyright 2021 Tintri by DDN, Inc. All rights reserved.
 * Copyright 2020 Joyent, Inc.
 */

/* needed when building libzpool */
#ifndef	_KERNEL
#include <sys/zfs_context.h>
#endif

#include <sys/sunddi.h>
#include <sys/dkio.h>
#include <sys/dkioc_free_util.h>
#include <sys/sysmacros.h>
#include <sys/file.h>
#include <sys/sdt.h>

static int adjust_exts(dkioc_free_list_t *, const dkioc_free_info_t *,
    uint64_t len_blk);
static int split_extent(dkioc_free_list_t *, const dkioc_free_info_t *,
    uint64_t, dfl_iter_fn_t, void *, int);
static int process_range(dkioc_free_list_t *, uint64_t, uint64_t,
    dfl_iter_fn_t, void *, int);

/*
 * Copy-in convenience function for variable-length dkioc_free_list_t
 * structures. The pointer to be copied from is in `arg' (may be a pointer
 * to userspace). A new buffer is allocated and a pointer to it is placed
 * in `out'. `ddi_flags' indicates whether the pointer is from user-
 * or kernelspace (FKIOCTL) and `kmflags' are the flags passed to
 * kmem_zalloc when allocating the new structure.
 * Returns 0 on success, or an errno on failure.
 */
int
dfl_copyin(void *arg, dkioc_free_list_t **out, int ddi_flags, int kmflags)
{
	dkioc_free_list_t *dfl;

	if (ddi_flags & FKIOCTL) {
		dkioc_free_list_t *dfl_in = arg;

		if (dfl_in->dfl_num_exts == 0 ||
		    dfl_in->dfl_num_exts > DFL_COPYIN_MAX_EXTS)
			return (SET_ERROR(EINVAL));
		dfl = kmem_alloc(DFL_SZ(dfl_in->dfl_num_exts), kmflags);
		if (dfl == NULL)
			return (SET_ERROR(ENOMEM));
		bcopy(dfl_in, dfl, DFL_SZ(dfl_in->dfl_num_exts));
	} else {
		uint64_t num_exts;

		if (ddi_copyin(((uint8_t *)arg) + offsetof(dkioc_free_list_t,
		    dfl_num_exts), &num_exts, sizeof (num_exts),
		    ddi_flags) != 0)
			return (SET_ERROR(EFAULT));
		if (num_exts == 0 || num_exts > DFL_COPYIN_MAX_EXTS)
			return (SET_ERROR(EINVAL));
		dfl = kmem_alloc(DFL_SZ(num_exts), kmflags);
		if (dfl == NULL)
			return (SET_ERROR(ENOMEM));
		if (ddi_copyin(arg, dfl, DFL_SZ(num_exts), ddi_flags) != 0 ||
		    dfl->dfl_num_exts != num_exts) {
			kmem_free(dfl, DFL_SZ(num_exts));
			return (SET_ERROR(EFAULT));
		}
	}

	*out = dfl;
	return (0);
}

/* Frees a variable-length dkioc_free_list_t structure. */
void
dfl_free(dkioc_free_list_t *dfl)
{
	kmem_free(dfl, DFL_SZ(dfl->dfl_num_exts));
}

/*
 * Convenience function to resize and segment the array of extents in
 * a DKIOCFREE request as required by a driver.
 *
 * Some devices that implement DKIOCFREE (e.g. vioblk) have limits
 * on either the number of extents that can be submitted in a single request,
 * or the total number of blocks that can be submitted in a single request.
 * In addition, devices may have alignment requirements on the starting
 * address stricter than the device block size.
 *
 * Since there is currently no mechanism for callers of DKIOCFREE to discover
 * such restrictions, instead of rejecting any requests that do not conform to
 * some undiscoverable (to the caller) set of requirements, a driver can use
 * dfl_iter() to adjust and resegment the extents from a DKIOCFREE call as
 * required to conform to its requirements.
 *
 * The original request is passed as 'dfl' and the alignment requirements
 * are passed in 'dfi'. Additionally the maximum offset of the device allowed
 * in bytes) is passed as max_off -- this allows a driver with
 * multiple instances of different sizes but similar requirements (e.g.
 * a partitioned blkdev device) to not construct a separate dkioc_free_info_t
 * struct for each device.
 *
 * dfl_iter() will call 'func' with a dkioc_free_list_t and the value of
 * arg passed to it as needed. If the extents in the dkioc_free_list_t passed
 * to dfl_iter() meet all the requirements in 'dfi', the dkioc_free_list_t will
 * be passed on to 'func' unmodified. If any of the extents passed to dfl_iter()
 * do not meet the requirements, dfl_iter() will allocate new dkioc_free_list_t
 * instances and populate them with the adjusted extents that do conform to the
 * requirements in 'dfi'. dfl_iter() will also free the dkioc_free_list_t
 * passed to it when this occurs. The net result is that 'func' can always
 * assume it will be called with a dkioc_free_list_t with extents that
 * comply with the requirements in 'dfi'. 'func' is also responsible for
 * freeing the dkioc_free_list_t passed to it (likely via a completion
 * callback).
 *
 * Combined with the behavior described above, dfl_iter() can be viewed as
 * consuming the dkioc_free_list_t passed to it. Either it will pass it along
 * to 'func' (and let 'func' handle freeing it), or it will free it and
 * allocate one or more new dkioc_free_list_ts to pass to 'func' (while still
 * letting 'func' handle freeing the new instances). This way neither the
 * dfl_iter() caller nor nor the driver need to worry about treating
 * conforming and non-conforming requests differently.
 *
 * Unfortunately, the DKIOCFREE ioctl provides no method for communicating
 * any notion of partial completion -- either it returns success (0) or
 * an error. It's not clear if such a notion would even be possible while
 * supporting multiple types of devices (NVMe, SCSI, etc.) with the same
 * interface. As such, there's little benefit to providing more detailed error
 * semantics beyond what DKIOCFREE can handle.
 *
 * Due to this, a somewhat simplistic approach is taken to error handling. The
 * original list of extents is first checked to make sure they all appear
 * valid -- that is they do not start or extend beyond the end of the device.
 * Any request that contains such extents is always rejected in it's entirety.
 * It is possible after applying any needed adjustments to the original list
 * of extents that the result is not acceptable to the driver. For example,
 * a device with a 512 byte block size that tries to free the range 513-1023
 * (bytes) would not be able to be processed. Such extents will be silently
 * ignored. If the original request consists of nothing but such requests,
 * dfl_iter() will never call 'func' and will merely return 0.
 */
int
dfl_iter(dkioc_free_list_t *dfl, const dkioc_free_info_t *dfi, uint64_t max_off,
    dfl_iter_fn_t func, void *arg, int kmflag)
{
	dkioc_free_list_ext_t *ext;
	uint64_t n_bytes, n_segs, start_idx, i;
	uint_t bsize = 1U << dfi->dfi_bshift;
	int r = 0;
	boolean_t need_copy = B_FALSE;

	/*
	 * Make sure the block size derived from dfi_bshift is at least 512
	 * (1U << DEV_BSHIFT) bytes and less than 2^30. The lower bound is
	 * to prevent any problems with other parts of the system that might
	 * assume a minimum block size of 512, and the upper bound is just
	 * to prevent overflow when creating the block size from dfi_bshift
	 * (though it seems unlikely we'll have _block_ sizes near a GiB
	 * any time soon).
	 */
	if (dfi->dfi_bshift < DEV_BSHIFT || dfi->dfi_bshift > 30) {
		r = SET_ERROR(EINVAL);
		goto done;
	}

	/* Max bytes must be a multiple of the block size */
	if (!IS_P2ALIGNED(dfi->dfi_max_bytes, bsize)) {
		r = SET_ERROR(EINVAL);
		goto done;
	}

	/* Start offset alignment must also be a multiple of the block size */
	if (dfi->dfi_align == 0 || !IS_P2ALIGNED(dfi->dfi_align, bsize)) {
		r = SET_ERROR(EINVAL);
		goto done;
	}

	/* Max bytes in an extent must be a multiple of the block size */
	if (!IS_P2ALIGNED(dfi->dfi_max_ext_bytes, bsize)) {
		r = SET_ERROR(EINVAL);
		goto done;
	}

	/*
	 * It makes no sense to allow a single extent to be larger than the
	 * total allowed for an entire request.
	 */
	if (dfi->dfi_max_ext_bytes > 0 &&
	    dfi->dfi_max_ext_bytes > dfi->dfi_max_bytes) {
		r = SET_ERROR(EINVAL);
		goto done;
	}

	/*
	 * The first pass, align everything as needed and make sure all the
	 * extents look valid.
	 */
	if ((r = adjust_exts(dfl, dfi, max_off)) != 0) {
		goto done;
	}

	/*
	 * Go through and split things up as needed. The general idea is to
	 * split along the original extent boundaries when needed. We only
	 * split an extent from the original request into multiple extents
	 * if the original extent is by itself too big for the device to
	 * process in a single request.
	 */
	start_idx = 0;
	n_bytes = n_segs = 0;
	ext = dfl->dfl_exts;
	for (i = 0; i < dfl->dfl_num_exts; i++, ext++) {
		uint64_t start = dfl->dfl_offset + ext->dfle_start;
		uint64_t len = ext->dfle_length;

		if (len == 0) {
			/*
			 * If we encounter a zero length extent, we're going
			 * to create a new copy of dfl no matter what --
			 * the size of dfl is determined by dfl_num_exts so
			 * we cannot do things like shift the contents and
			 * reduce dfl_num_exts to get a contiguous array
			 * of non-zero length extents.
			 */
			need_copy = B_TRUE;
			continue;
		}

		if (dfi->dfi_max_ext_bytes > 0 &&
		    len > dfi->dfi_max_ext_bytes) {
			/*
			 * An extent that's too large. Dispatch what we've
			 * accumulated, and then split this extent into
			 * smaller ones the device can accept.
			 */
			if ((r = process_range(dfl, start_idx, i - start_idx,
			    func, arg, kmflag)) != 0) {
				goto done;
			}

			if ((r = split_extent(dfl, dfi, i, func, arg,
			    kmflag)) != 0) {
				goto done;
			}
			start_idx = i + 1;
			n_segs = 0;
			n_bytes = 0;
			continue;
		}

		if (dfi->dfi_max_bytes > 0 &&
		    n_bytes + len > dfi->dfi_max_bytes) {
			/*
			 * This extent would put us over the limit for total
			 * bytes that can be trimmed in one request.
			 * Dispatch what we've accumulated. Then deal
			 * with this extent.
			 */
			if ((r = process_range(dfl, start_idx, i - start_idx,
			    func, arg, kmflag)) != 0) {
				goto done;
			}

			if (len < dfi->dfi_max_bytes) {
				/*
				 * After dispatching what we've accumulated,
				 * this extent can fit in a new request
				 * Just add it to the accumulated list of
				 * extents and move on.
				 */
				start_idx = i;
				n_segs = 1;
				n_bytes = len;
				continue;
			}

			/*
			 * Even after starting a new request, this extent
			 * is too big. Split it until it fits.
			 */
			if ((r = split_extent(dfl, dfi, i, func, arg,
			    kmflag)) != 0) {
				goto done;
			}

			start_idx = i + 1;
			n_segs = 0;
			n_bytes = 0;
			continue;
		}

		if (dfi->dfi_max_ext > 0 && n_segs + 1 > dfi->dfi_max_ext) {
			/*
			 * This extent will put us over the limit on the number
			 * of extents the device can accept. Dispatch what
			 * we've accumulated so far.
			 */
			if ((r = process_range(dfl, start_idx, i - start_idx,
			    func, arg, kmflag)) != 0) {
				goto done;
			}

			start_idx = i;
			n_segs = 1;
			n_bytes = len;
			continue;
		}

		n_segs++;
		n_bytes += len;
	}

	/*
	 * If a copy wasn't required, and we haven't processed a subset of
	 * the extents already, we can just use the original request.
	 */
	if (!need_copy && start_idx == 0) {
		return (func(dfl, arg, kmflag));
	}

	r = process_range(dfl, start_idx, i - start_idx, func, arg, kmflag);

done:
	dfl_free(dfl);
	return (r);
}

/*
 * Adjust the start and length of each extent in dfl so that it conforms to
 * the requirements in dfi. It also verifies that no extent extends beyond
 * the end of the device (given by len_blk).
 *
 * Returns 0 on success, or an error value.
 */
static int
adjust_exts(dkioc_free_list_t *dfl, const dkioc_free_info_t *dfi,
    uint64_t max_off)
{
	dkioc_free_list_ext_t *exts = dfl->dfl_exts;
	/*
	 * These must be uint64_t to prevent the P2 macros from truncating
	 * the result.
	 */
	const uint64_t align = dfi->dfi_align;
	const uint64_t bsize = (uint64_t)1 << dfi->dfi_bshift;

	for (uint64_t i = 0; i < dfl->dfl_num_exts; i++, exts++) {
		/*
		 * Since there are no known requirements on the value of
		 * dfl_offset, it's possible (though odd) to have a scenario
		 * where dfl_offset == 1, and dfle_start == 511 (resulting
		 * in an actual start offset of 512). As such, we always
		 * apply the offset and find the resulting starting offset
		 * and length (in bytes) first, then apply any rounding
		 * and alignment.
		 */
		uint64_t start = exts->dfle_start + dfl->dfl_offset;
		uint64_t end = start + exts->dfle_length;

		/*
		 * Make sure after applying dfl->dfl_offset and any alignment
		 * adjustments that the results don't overflow.
		 */
		if (start < dfl->dfl_offset || start > (UINT64_MAX - bsize)) {
			return (SET_ERROR(EOVERFLOW));
		}

		if (end < start) {
			return (SET_ERROR(EOVERFLOW));
		}

		/*
		 * Make sure we don't extend past the end of the device
		 */
		if (end > max_off) {
			return (SET_ERROR(EINVAL));
		}

		start = P2ROUNDUP(start, align);
		end = P2ALIGN(end, bsize);

		/*
		 * Remove the offset so that when it's later applied again,
		 * the correct start value is obtained.
		 */
		exts->dfle_start = start - dfl->dfl_offset;

		/*
		 * If the original length was less than the block size
		 * of the device, we can end up with end < start. If that
		 * happens we just set the length to zero.
		 */
		exts->dfle_length = (end < start) ? 0 : end - start;
	}

	return (0);
}

/*
 * Take a subset of extents from dfl (starting at start_idx, with n entries)
 * and create a new dkioc_free_list_t, passing that to func.
 */
static int
process_range(dkioc_free_list_t *dfl, uint64_t start_idx, uint64_t n,
    dfl_iter_fn_t func, void *arg, int kmflag)
{
	dkioc_free_list_t *new_dfl = NULL;
	dkioc_free_list_ext_t *new_exts = NULL;
	dkioc_free_list_ext_t *exts = dfl->dfl_exts + start_idx;
	size_t actual_n = n;
	int r = 0;

	if (n == 0) {
		return (0);
	}

	/*
	 * Ignore any zero length extents. No known devices attach any
	 * semantic meaning to such extents, and are likely just a result of
	 * narrowing the range of the extent to fit the device alignment
	 * requirements. It is possible the original caller submitted a
	 * zero length extent, but we ignore those as well. Since we can't
	 * communicate partial results back to the caller anyway, it's
	 * unclear whether reporting that one of potentially many exents was
	 * too small (without being able to identify which one) to the caller
	 * of the DKIOCFREE request would be useful.
	 */
	for (uint64_t i = 0; i < n; i++) {
		if (exts[i].dfle_length == 0 && --actual_n == 0) {
			return (0);
		}
	}

	new_dfl = kmem_zalloc(DFL_SZ(actual_n), kmflag);
	if (new_dfl == NULL) {
		return (SET_ERROR(ENOMEM));
	}

	new_dfl->dfl_flags = dfl->dfl_flags;
	new_dfl->dfl_num_exts = actual_n;
	new_dfl->dfl_offset = dfl->dfl_offset;
	new_exts = new_dfl->dfl_exts;

	for (uint64_t i = 0; i < n; i++) {
		if (exts[i].dfle_length == 0) {
			continue;
		}

		*new_exts++ = exts[i];
	}

	return (func(new_dfl, arg, kmflag));
}

/*
 * If dfi_max_ext_bytes is set, use as the max segment length,
 * otherwise use dfi_max_bytes if set, otherwise fallback to UINT64_MAX
 */
#define	MAX_SEGLEN(dfi) \
	(((dfi)->dfi_max_ext_bytes > 0) ? (dfi)->dfi_max_ext_bytes :	\
	((dfi)->dfi_max_bytes > 0) ? (dfi)->dfi_max_bytes : UINT64_MAX)

/*
 * Split the extent at idx into multiple lists (calling func for each one).
 */
static int
split_extent(dkioc_free_list_t *dfl, const dkioc_free_info_t *dfi, uint64_t idx,
    dfl_iter_fn_t func, void *arg, int kmflag)
{
	ASSERT3U(idx, <, dfl->dfl_num_exts);

	const uint64_t		maxlen = MAX_SEGLEN(dfi);
	dkioc_free_list_ext_t	*ext = dfl->dfl_exts + idx;
	uint64_t		remain = ext->dfle_length;
	int			r;

	/*
	 * Break the extent into as many single requests as needed. While it
	 * would be possible in some circumstances to combine the final chunk
	 * of the extent (after splitting) with the remaining extents in the
	 * original request, it's not clear there's much benefit from the
	 * added complexity. Such behavior could be added in the future if
	 * it's determined to be worthwhile.
	 */
	while (remain > 0) {
		uint64_t start = dfl->dfl_offset + ext->dfle_start;
		uint64_t len = remain;

		/*
		 * If we know we have at least one more segment left after
		 * the current iteration of this loop, split it so that
		 * the next segment starts on an aligned boundary.
		 */
		if (len > maxlen) {
			uint64_t end = P2ALIGN(start + maxlen, dfi->dfi_align);
			len = end - start;
		}

		ext->dfle_length = len;

		if ((r = process_range(dfl, idx, 1, func, arg, kmflag)) != 0) {
			return (r);
		}

		ext->dfle_start += len;
		remain -= len;
	}

	return (0);
}