xref: /freebsd/sys/contrib/openzfs/module/zfs/zvol.c (revision be181ee2a28aa2b4b0e76684bce9f673ef668874)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or https://opensource.org/licenses/CDDL-1.0.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 /*
22  * Copyright (C) 2008-2010 Lawrence Livermore National Security, LLC.
23  * Produced at Lawrence Livermore National Laboratory (cf, DISCLAIMER).
24  * Rewritten for Linux by Brian Behlendorf <behlendorf1@llnl.gov>.
25  * LLNL-CODE-403049.
26  *
27  * ZFS volume emulation driver.
28  *
29  * Makes a DMU object look like a volume of arbitrary size, up to 2^64 bytes.
30  * Volumes are accessed through the symbolic links named:
31  *
32  * /dev/<pool_name>/<dataset_name>
33  *
34  * Volumes are persistent through reboot and module load.  No user command
35  * needs to be run before opening and using a device.
36  *
37  * Copyright 2014 Nexenta Systems, Inc.  All rights reserved.
38  * Copyright (c) 2016 Actifio, Inc. All rights reserved.
39  * Copyright (c) 2012, 2019 by Delphix. All rights reserved.
40  */
41 
42 /*
43  * Note on locking of zvol state structures.
44  *
45  * These structures are used to maintain internal state used to emulate block
46  * devices on top of zvols. In particular, management of device minor number
47  * operations - create, remove, rename, and set_snapdev - involves access to
48  * these structures. The zvol_state_lock is primarily used to protect the
49  * zvol_state_list. The zv->zv_state_lock is used to protect the contents
50  * of the zvol_state_t structures, as well as to make sure that when the
51  * time comes to remove the structure from the list, it is not in use, and
52  * therefore, it can be taken off zvol_state_list and freed.
53  *
54  * The zv_suspend_lock was introduced to allow for suspending I/O to a zvol,
55  * e.g. for the duration of receive and rollback operations. This lock can be
56  * held for significant periods of time. Given that it is undesirable to hold
57  * mutexes for long periods of time, the following lock ordering applies:
58  * - take zvol_state_lock if necessary, to protect zvol_state_list
59  * - take zv_suspend_lock if necessary, by the code path in question
60  * - take zv_state_lock to protect zvol_state_t
61  *
62  * The minor operations are issued to spa->spa_zvol_taskq queues, that are
63  * single-threaded (to preserve order of minor operations), and are executed
64  * through the zvol_task_cb that dispatches the specific operations. Therefore,
65  * these operations are serialized per pool. Consequently, we can be certain
66  * that for a given zvol, there is only one operation at a time in progress.
67  * That is why one can be sure that first, zvol_state_t for a given zvol is
68  * allocated and placed on zvol_state_list, and then other minor operations
69  * for this zvol are going to proceed in the order of issue.
70  *
71  */
72 
73 #include <sys/dataset_kstats.h>
74 #include <sys/dbuf.h>
75 #include <sys/dmu_traverse.h>
76 #include <sys/dsl_dataset.h>
77 #include <sys/dsl_prop.h>
78 #include <sys/dsl_dir.h>
79 #include <sys/zap.h>
80 #include <sys/zfeature.h>
81 #include <sys/zil_impl.h>
82 #include <sys/dmu_tx.h>
83 #include <sys/zio.h>
84 #include <sys/zfs_rlock.h>
85 #include <sys/spa_impl.h>
86 #include <sys/zvol.h>
87 #include <sys/zvol_impl.h>
88 
89 unsigned int zvol_inhibit_dev = 0;
90 unsigned int zvol_volmode = ZFS_VOLMODE_GEOM;
91 
92 struct hlist_head *zvol_htable;
93 static list_t zvol_state_list;
94 krwlock_t zvol_state_lock;
95 
96 typedef enum {
97 	ZVOL_ASYNC_REMOVE_MINORS,
98 	ZVOL_ASYNC_RENAME_MINORS,
99 	ZVOL_ASYNC_SET_SNAPDEV,
100 	ZVOL_ASYNC_SET_VOLMODE,
101 	ZVOL_ASYNC_MAX
102 } zvol_async_op_t;
103 
104 typedef struct {
105 	zvol_async_op_t op;
106 	char name1[MAXNAMELEN];
107 	char name2[MAXNAMELEN];
108 	uint64_t value;
109 } zvol_task_t;
110 
111 uint64_t
112 zvol_name_hash(const char *name)
113 {
114 	int i;
115 	uint64_t crc = -1ULL;
116 	const uint8_t *p = (const uint8_t *)name;
117 	ASSERT(zfs_crc64_table[128] == ZFS_CRC64_POLY);
118 	for (i = 0; i < MAXNAMELEN - 1 && *p; i++, p++) {
119 		crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ (*p)) & 0xFF];
120 	}
121 	return (crc);
122 }
123 
124 /*
125  * Find a zvol_state_t given the name and hash generated by zvol_name_hash.
126  * If found, return with zv_suspend_lock and zv_state_lock taken, otherwise,
127  * return (NULL) without the taking locks. The zv_suspend_lock is always taken
128  * before zv_state_lock. The mode argument indicates the mode (including none)
129  * for zv_suspend_lock to be taken.
130  */
131 zvol_state_t *
132 zvol_find_by_name_hash(const char *name, uint64_t hash, int mode)
133 {
134 	zvol_state_t *zv;
135 	struct hlist_node *p = NULL;
136 
137 	rw_enter(&zvol_state_lock, RW_READER);
138 	hlist_for_each(p, ZVOL_HT_HEAD(hash)) {
139 		zv = hlist_entry(p, zvol_state_t, zv_hlink);
140 		mutex_enter(&zv->zv_state_lock);
141 		if (zv->zv_hash == hash &&
142 		    strncmp(zv->zv_name, name, MAXNAMELEN) == 0) {
143 			/*
144 			 * this is the right zvol, take the locks in the
145 			 * right order
146 			 */
147 			if (mode != RW_NONE &&
148 			    !rw_tryenter(&zv->zv_suspend_lock, mode)) {
149 				mutex_exit(&zv->zv_state_lock);
150 				rw_enter(&zv->zv_suspend_lock, mode);
151 				mutex_enter(&zv->zv_state_lock);
152 				/*
153 				 * zvol cannot be renamed as we continue
154 				 * to hold zvol_state_lock
155 				 */
156 				ASSERT(zv->zv_hash == hash &&
157 				    strncmp(zv->zv_name, name, MAXNAMELEN)
158 				    == 0);
159 			}
160 			rw_exit(&zvol_state_lock);
161 			return (zv);
162 		}
163 		mutex_exit(&zv->zv_state_lock);
164 	}
165 	rw_exit(&zvol_state_lock);
166 
167 	return (NULL);
168 }
169 
170 /*
171  * Find a zvol_state_t given the name.
172  * If found, return with zv_suspend_lock and zv_state_lock taken, otherwise,
173  * return (NULL) without the taking locks. The zv_suspend_lock is always taken
174  * before zv_state_lock. The mode argument indicates the mode (including none)
175  * for zv_suspend_lock to be taken.
176  */
177 static zvol_state_t *
178 zvol_find_by_name(const char *name, int mode)
179 {
180 	return (zvol_find_by_name_hash(name, zvol_name_hash(name), mode));
181 }
182 
183 /*
184  * ZFS_IOC_CREATE callback handles dmu zvol and zap object creation.
185  */
186 void
187 zvol_create_cb(objset_t *os, void *arg, cred_t *cr, dmu_tx_t *tx)
188 {
189 	zfs_creat_t *zct = arg;
190 	nvlist_t *nvprops = zct->zct_props;
191 	int error;
192 	uint64_t volblocksize, volsize;
193 
194 	VERIFY(nvlist_lookup_uint64(nvprops,
195 	    zfs_prop_to_name(ZFS_PROP_VOLSIZE), &volsize) == 0);
196 	if (nvlist_lookup_uint64(nvprops,
197 	    zfs_prop_to_name(ZFS_PROP_VOLBLOCKSIZE), &volblocksize) != 0)
198 		volblocksize = zfs_prop_default_numeric(ZFS_PROP_VOLBLOCKSIZE);
199 
200 	/*
201 	 * These properties must be removed from the list so the generic
202 	 * property setting step won't apply to them.
203 	 */
204 	VERIFY(nvlist_remove_all(nvprops,
205 	    zfs_prop_to_name(ZFS_PROP_VOLSIZE)) == 0);
206 	(void) nvlist_remove_all(nvprops,
207 	    zfs_prop_to_name(ZFS_PROP_VOLBLOCKSIZE));
208 
209 	error = dmu_object_claim(os, ZVOL_OBJ, DMU_OT_ZVOL, volblocksize,
210 	    DMU_OT_NONE, 0, tx);
211 	ASSERT(error == 0);
212 
213 	error = zap_create_claim(os, ZVOL_ZAP_OBJ, DMU_OT_ZVOL_PROP,
214 	    DMU_OT_NONE, 0, tx);
215 	ASSERT(error == 0);
216 
217 	error = zap_update(os, ZVOL_ZAP_OBJ, "size", 8, 1, &volsize, tx);
218 	ASSERT(error == 0);
219 }
220 
221 /*
222  * ZFS_IOC_OBJSET_STATS entry point.
223  */
224 int
225 zvol_get_stats(objset_t *os, nvlist_t *nv)
226 {
227 	int error;
228 	dmu_object_info_t *doi;
229 	uint64_t val;
230 
231 	error = zap_lookup(os, ZVOL_ZAP_OBJ, "size", 8, 1, &val);
232 	if (error)
233 		return (SET_ERROR(error));
234 
235 	dsl_prop_nvlist_add_uint64(nv, ZFS_PROP_VOLSIZE, val);
236 	doi = kmem_alloc(sizeof (dmu_object_info_t), KM_SLEEP);
237 	error = dmu_object_info(os, ZVOL_OBJ, doi);
238 
239 	if (error == 0) {
240 		dsl_prop_nvlist_add_uint64(nv, ZFS_PROP_VOLBLOCKSIZE,
241 		    doi->doi_data_block_size);
242 	}
243 
244 	kmem_free(doi, sizeof (dmu_object_info_t));
245 
246 	return (SET_ERROR(error));
247 }
248 
249 /*
250  * Sanity check volume size.
251  */
252 int
253 zvol_check_volsize(uint64_t volsize, uint64_t blocksize)
254 {
255 	if (volsize == 0)
256 		return (SET_ERROR(EINVAL));
257 
258 	if (volsize % blocksize != 0)
259 		return (SET_ERROR(EINVAL));
260 
261 #ifdef _ILP32
262 	if (volsize - 1 > SPEC_MAXOFFSET_T)
263 		return (SET_ERROR(EOVERFLOW));
264 #endif
265 	return (0);
266 }
267 
268 /*
269  * Ensure the zap is flushed then inform the VFS of the capacity change.
270  */
271 static int
272 zvol_update_volsize(uint64_t volsize, objset_t *os)
273 {
274 	dmu_tx_t *tx;
275 	int error;
276 	uint64_t txg;
277 
278 	tx = dmu_tx_create(os);
279 	dmu_tx_hold_zap(tx, ZVOL_ZAP_OBJ, TRUE, NULL);
280 	dmu_tx_mark_netfree(tx);
281 	error = dmu_tx_assign(tx, TXG_WAIT);
282 	if (error) {
283 		dmu_tx_abort(tx);
284 		return (SET_ERROR(error));
285 	}
286 	txg = dmu_tx_get_txg(tx);
287 
288 	error = zap_update(os, ZVOL_ZAP_OBJ, "size", 8, 1,
289 	    &volsize, tx);
290 	dmu_tx_commit(tx);
291 
292 	txg_wait_synced(dmu_objset_pool(os), txg);
293 
294 	if (error == 0)
295 		error = dmu_free_long_range(os,
296 		    ZVOL_OBJ, volsize, DMU_OBJECT_END);
297 
298 	return (error);
299 }
300 
301 /*
302  * Set ZFS_PROP_VOLSIZE set entry point.  Note that modifying the volume
303  * size will result in a udev "change" event being generated.
304  */
305 int
306 zvol_set_volsize(const char *name, uint64_t volsize)
307 {
308 	objset_t *os = NULL;
309 	uint64_t readonly;
310 	int error;
311 	boolean_t owned = B_FALSE;
312 
313 	error = dsl_prop_get_integer(name,
314 	    zfs_prop_to_name(ZFS_PROP_READONLY), &readonly, NULL);
315 	if (error != 0)
316 		return (SET_ERROR(error));
317 	if (readonly)
318 		return (SET_ERROR(EROFS));
319 
320 	zvol_state_t *zv = zvol_find_by_name(name, RW_READER);
321 
322 	ASSERT(zv == NULL || (MUTEX_HELD(&zv->zv_state_lock) &&
323 	    RW_READ_HELD(&zv->zv_suspend_lock)));
324 
325 	if (zv == NULL || zv->zv_objset == NULL) {
326 		if (zv != NULL)
327 			rw_exit(&zv->zv_suspend_lock);
328 		if ((error = dmu_objset_own(name, DMU_OST_ZVOL, B_FALSE, B_TRUE,
329 		    FTAG, &os)) != 0) {
330 			if (zv != NULL)
331 				mutex_exit(&zv->zv_state_lock);
332 			return (SET_ERROR(error));
333 		}
334 		owned = B_TRUE;
335 		if (zv != NULL)
336 			zv->zv_objset = os;
337 	} else {
338 		os = zv->zv_objset;
339 	}
340 
341 	dmu_object_info_t *doi = kmem_alloc(sizeof (*doi), KM_SLEEP);
342 
343 	if ((error = dmu_object_info(os, ZVOL_OBJ, doi)) ||
344 	    (error = zvol_check_volsize(volsize, doi->doi_data_block_size)))
345 		goto out;
346 
347 	error = zvol_update_volsize(volsize, os);
348 	if (error == 0 && zv != NULL) {
349 		zv->zv_volsize = volsize;
350 		zv->zv_changed = 1;
351 	}
352 out:
353 	kmem_free(doi, sizeof (dmu_object_info_t));
354 
355 	if (owned) {
356 		dmu_objset_disown(os, B_TRUE, FTAG);
357 		if (zv != NULL)
358 			zv->zv_objset = NULL;
359 	} else {
360 		rw_exit(&zv->zv_suspend_lock);
361 	}
362 
363 	if (zv != NULL)
364 		mutex_exit(&zv->zv_state_lock);
365 
366 	if (error == 0 && zv != NULL)
367 		zvol_os_update_volsize(zv, volsize);
368 
369 	return (SET_ERROR(error));
370 }
371 
372 /*
373  * Sanity check volume block size.
374  */
375 int
376 zvol_check_volblocksize(const char *name, uint64_t volblocksize)
377 {
378 	/* Record sizes above 128k need the feature to be enabled */
379 	if (volblocksize > SPA_OLD_MAXBLOCKSIZE) {
380 		spa_t *spa;
381 		int error;
382 
383 		if ((error = spa_open(name, &spa, FTAG)) != 0)
384 			return (error);
385 
386 		if (!spa_feature_is_enabled(spa, SPA_FEATURE_LARGE_BLOCKS)) {
387 			spa_close(spa, FTAG);
388 			return (SET_ERROR(ENOTSUP));
389 		}
390 
391 		/*
392 		 * We don't allow setting the property above 1MB,
393 		 * unless the tunable has been changed.
394 		 */
395 		if (volblocksize > zfs_max_recordsize)
396 			return (SET_ERROR(EDOM));
397 
398 		spa_close(spa, FTAG);
399 	}
400 
401 	if (volblocksize < SPA_MINBLOCKSIZE ||
402 	    volblocksize > SPA_MAXBLOCKSIZE ||
403 	    !ISP2(volblocksize))
404 		return (SET_ERROR(EDOM));
405 
406 	return (0);
407 }
408 
409 /*
410  * Replay a TX_TRUNCATE ZIL transaction if asked.  TX_TRUNCATE is how we
411  * implement DKIOCFREE/free-long-range.
412  */
413 static int
414 zvol_replay_truncate(void *arg1, void *arg2, boolean_t byteswap)
415 {
416 	zvol_state_t *zv = arg1;
417 	lr_truncate_t *lr = arg2;
418 	uint64_t offset, length;
419 
420 	if (byteswap)
421 		byteswap_uint64_array(lr, sizeof (*lr));
422 
423 	offset = lr->lr_offset;
424 	length = lr->lr_length;
425 
426 	dmu_tx_t *tx = dmu_tx_create(zv->zv_objset);
427 	dmu_tx_mark_netfree(tx);
428 	int error = dmu_tx_assign(tx, TXG_WAIT);
429 	if (error != 0) {
430 		dmu_tx_abort(tx);
431 	} else {
432 		(void) zil_replaying(zv->zv_zilog, tx);
433 		dmu_tx_commit(tx);
434 		error = dmu_free_long_range(zv->zv_objset, ZVOL_OBJ, offset,
435 		    length);
436 	}
437 
438 	return (error);
439 }
440 
441 /*
442  * Replay a TX_WRITE ZIL transaction that didn't get committed
443  * after a system failure
444  */
445 static int
446 zvol_replay_write(void *arg1, void *arg2, boolean_t byteswap)
447 {
448 	zvol_state_t *zv = arg1;
449 	lr_write_t *lr = arg2;
450 	objset_t *os = zv->zv_objset;
451 	char *data = (char *)(lr + 1);  /* data follows lr_write_t */
452 	uint64_t offset, length;
453 	dmu_tx_t *tx;
454 	int error;
455 
456 	if (byteswap)
457 		byteswap_uint64_array(lr, sizeof (*lr));
458 
459 	offset = lr->lr_offset;
460 	length = lr->lr_length;
461 
462 	/* If it's a dmu_sync() block, write the whole block */
463 	if (lr->lr_common.lrc_reclen == sizeof (lr_write_t)) {
464 		uint64_t blocksize = BP_GET_LSIZE(&lr->lr_blkptr);
465 		if (length < blocksize) {
466 			offset -= offset % blocksize;
467 			length = blocksize;
468 		}
469 	}
470 
471 	tx = dmu_tx_create(os);
472 	dmu_tx_hold_write(tx, ZVOL_OBJ, offset, length);
473 	error = dmu_tx_assign(tx, TXG_WAIT);
474 	if (error) {
475 		dmu_tx_abort(tx);
476 	} else {
477 		dmu_write(os, ZVOL_OBJ, offset, length, data, tx);
478 		(void) zil_replaying(zv->zv_zilog, tx);
479 		dmu_tx_commit(tx);
480 	}
481 
482 	return (error);
483 }
484 
485 static int
486 zvol_replay_err(void *arg1, void *arg2, boolean_t byteswap)
487 {
488 	(void) arg1, (void) arg2, (void) byteswap;
489 	return (SET_ERROR(ENOTSUP));
490 }
491 
492 /*
493  * Callback vectors for replaying records.
494  * Only TX_WRITE and TX_TRUNCATE are needed for zvol.
495  */
496 zil_replay_func_t *const zvol_replay_vector[TX_MAX_TYPE] = {
497 	zvol_replay_err,	/* no such transaction type */
498 	zvol_replay_err,	/* TX_CREATE */
499 	zvol_replay_err,	/* TX_MKDIR */
500 	zvol_replay_err,	/* TX_MKXATTR */
501 	zvol_replay_err,	/* TX_SYMLINK */
502 	zvol_replay_err,	/* TX_REMOVE */
503 	zvol_replay_err,	/* TX_RMDIR */
504 	zvol_replay_err,	/* TX_LINK */
505 	zvol_replay_err,	/* TX_RENAME */
506 	zvol_replay_write,	/* TX_WRITE */
507 	zvol_replay_truncate,	/* TX_TRUNCATE */
508 	zvol_replay_err,	/* TX_SETATTR */
509 	zvol_replay_err,	/* TX_ACL */
510 	zvol_replay_err,	/* TX_CREATE_ATTR */
511 	zvol_replay_err,	/* TX_CREATE_ACL_ATTR */
512 	zvol_replay_err,	/* TX_MKDIR_ACL */
513 	zvol_replay_err,	/* TX_MKDIR_ATTR */
514 	zvol_replay_err,	/* TX_MKDIR_ACL_ATTR */
515 	zvol_replay_err,	/* TX_WRITE2 */
516 	zvol_replay_err,	/* TX_SETSAXATTR */
517 };
518 
519 /*
520  * zvol_log_write() handles synchronous writes using TX_WRITE ZIL transactions.
521  *
522  * We store data in the log buffers if it's small enough.
523  * Otherwise we will later flush the data out via dmu_sync().
524  */
525 static const ssize_t zvol_immediate_write_sz = 32768;
526 
527 void
528 zvol_log_write(zvol_state_t *zv, dmu_tx_t *tx, uint64_t offset,
529     uint64_t size, int sync)
530 {
531 	uint32_t blocksize = zv->zv_volblocksize;
532 	zilog_t *zilog = zv->zv_zilog;
533 	itx_wr_state_t write_state;
534 	uint64_t sz = size;
535 
536 	if (zil_replaying(zilog, tx))
537 		return;
538 
539 	if (zilog->zl_logbias == ZFS_LOGBIAS_THROUGHPUT)
540 		write_state = WR_INDIRECT;
541 	else if (!spa_has_slogs(zilog->zl_spa) &&
542 	    size >= blocksize && blocksize > zvol_immediate_write_sz)
543 		write_state = WR_INDIRECT;
544 	else if (sync)
545 		write_state = WR_COPIED;
546 	else
547 		write_state = WR_NEED_COPY;
548 
549 	while (size) {
550 		itx_t *itx;
551 		lr_write_t *lr;
552 		itx_wr_state_t wr_state = write_state;
553 		ssize_t len = size;
554 
555 		if (wr_state == WR_COPIED && size > zil_max_copied_data(zilog))
556 			wr_state = WR_NEED_COPY;
557 		else if (wr_state == WR_INDIRECT)
558 			len = MIN(blocksize - P2PHASE(offset, blocksize), size);
559 
560 		itx = zil_itx_create(TX_WRITE, sizeof (*lr) +
561 		    (wr_state == WR_COPIED ? len : 0));
562 		lr = (lr_write_t *)&itx->itx_lr;
563 		if (wr_state == WR_COPIED && dmu_read_by_dnode(zv->zv_dn,
564 		    offset, len, lr+1, DMU_READ_NO_PREFETCH) != 0) {
565 			zil_itx_destroy(itx);
566 			itx = zil_itx_create(TX_WRITE, sizeof (*lr));
567 			lr = (lr_write_t *)&itx->itx_lr;
568 			wr_state = WR_NEED_COPY;
569 		}
570 
571 		itx->itx_wr_state = wr_state;
572 		lr->lr_foid = ZVOL_OBJ;
573 		lr->lr_offset = offset;
574 		lr->lr_length = len;
575 		lr->lr_blkoff = 0;
576 		BP_ZERO(&lr->lr_blkptr);
577 
578 		itx->itx_private = zv;
579 		itx->itx_sync = sync;
580 
581 		(void) zil_itx_assign(zilog, itx, tx);
582 
583 		offset += len;
584 		size -= len;
585 	}
586 
587 	if (write_state == WR_COPIED || write_state == WR_NEED_COPY) {
588 		dsl_pool_wrlog_count(zilog->zl_dmu_pool, sz, tx->tx_txg);
589 	}
590 }
591 
592 /*
593  * Log a DKIOCFREE/free-long-range to the ZIL with TX_TRUNCATE.
594  */
595 void
596 zvol_log_truncate(zvol_state_t *zv, dmu_tx_t *tx, uint64_t off, uint64_t len,
597     boolean_t sync)
598 {
599 	itx_t *itx;
600 	lr_truncate_t *lr;
601 	zilog_t *zilog = zv->zv_zilog;
602 
603 	if (zil_replaying(zilog, tx))
604 		return;
605 
606 	itx = zil_itx_create(TX_TRUNCATE, sizeof (*lr));
607 	lr = (lr_truncate_t *)&itx->itx_lr;
608 	lr->lr_foid = ZVOL_OBJ;
609 	lr->lr_offset = off;
610 	lr->lr_length = len;
611 
612 	itx->itx_sync = sync;
613 	zil_itx_assign(zilog, itx, tx);
614 }
615 
616 
617 static void
618 zvol_get_done(zgd_t *zgd, int error)
619 {
620 	(void) error;
621 	if (zgd->zgd_db)
622 		dmu_buf_rele(zgd->zgd_db, zgd);
623 
624 	zfs_rangelock_exit(zgd->zgd_lr);
625 
626 	kmem_free(zgd, sizeof (zgd_t));
627 }
628 
629 /*
630  * Get data to generate a TX_WRITE intent log record.
631  */
632 int
633 zvol_get_data(void *arg, uint64_t arg2, lr_write_t *lr, char *buf,
634     struct lwb *lwb, zio_t *zio)
635 {
636 	zvol_state_t *zv = arg;
637 	uint64_t offset = lr->lr_offset;
638 	uint64_t size = lr->lr_length;
639 	dmu_buf_t *db;
640 	zgd_t *zgd;
641 	int error;
642 
643 	ASSERT3P(lwb, !=, NULL);
644 	ASSERT3P(zio, !=, NULL);
645 	ASSERT3U(size, !=, 0);
646 
647 	zgd = (zgd_t *)kmem_zalloc(sizeof (zgd_t), KM_SLEEP);
648 	zgd->zgd_lwb = lwb;
649 
650 	/*
651 	 * Write records come in two flavors: immediate and indirect.
652 	 * For small writes it's cheaper to store the data with the
653 	 * log record (immediate); for large writes it's cheaper to
654 	 * sync the data and get a pointer to it (indirect) so that
655 	 * we don't have to write the data twice.
656 	 */
657 	if (buf != NULL) { /* immediate write */
658 		zgd->zgd_lr = zfs_rangelock_enter(&zv->zv_rangelock, offset,
659 		    size, RL_READER);
660 		error = dmu_read_by_dnode(zv->zv_dn, offset, size, buf,
661 		    DMU_READ_NO_PREFETCH);
662 	} else { /* indirect write */
663 		/*
664 		 * Have to lock the whole block to ensure when it's written out
665 		 * and its checksum is being calculated that no one can change
666 		 * the data. Contrarily to zfs_get_data we need not re-check
667 		 * blocksize after we get the lock because it cannot be changed.
668 		 */
669 		size = zv->zv_volblocksize;
670 		offset = P2ALIGN_TYPED(offset, size, uint64_t);
671 		zgd->zgd_lr = zfs_rangelock_enter(&zv->zv_rangelock, offset,
672 		    size, RL_READER);
673 		error = dmu_buf_hold_by_dnode(zv->zv_dn, offset, zgd, &db,
674 		    DMU_READ_NO_PREFETCH);
675 		if (error == 0) {
676 			blkptr_t *bp = &lr->lr_blkptr;
677 
678 			zgd->zgd_db = db;
679 			zgd->zgd_bp = bp;
680 
681 			ASSERT(db != NULL);
682 			ASSERT(db->db_offset == offset);
683 			ASSERT(db->db_size == size);
684 
685 			error = dmu_sync(zio, lr->lr_common.lrc_txg,
686 			    zvol_get_done, zgd);
687 
688 			if (error == 0)
689 				return (0);
690 		}
691 	}
692 
693 	zvol_get_done(zgd, error);
694 
695 	return (SET_ERROR(error));
696 }
697 
698 /*
699  * The zvol_state_t's are inserted into zvol_state_list and zvol_htable.
700  */
701 
702 void
703 zvol_insert(zvol_state_t *zv)
704 {
705 	ASSERT(RW_WRITE_HELD(&zvol_state_lock));
706 	list_insert_head(&zvol_state_list, zv);
707 	hlist_add_head(&zv->zv_hlink, ZVOL_HT_HEAD(zv->zv_hash));
708 }
709 
710 /*
711  * Simply remove the zvol from to list of zvols.
712  */
713 static void
714 zvol_remove(zvol_state_t *zv)
715 {
716 	ASSERT(RW_WRITE_HELD(&zvol_state_lock));
717 	list_remove(&zvol_state_list, zv);
718 	hlist_del(&zv->zv_hlink);
719 }
720 
721 /*
722  * Setup zv after we just own the zv->objset
723  */
724 static int
725 zvol_setup_zv(zvol_state_t *zv)
726 {
727 	uint64_t volsize;
728 	int error;
729 	uint64_t ro;
730 	objset_t *os = zv->zv_objset;
731 
732 	ASSERT(MUTEX_HELD(&zv->zv_state_lock));
733 	ASSERT(RW_LOCK_HELD(&zv->zv_suspend_lock));
734 
735 	zv->zv_zilog = NULL;
736 	zv->zv_flags &= ~ZVOL_WRITTEN_TO;
737 
738 	error = dsl_prop_get_integer(zv->zv_name, "readonly", &ro, NULL);
739 	if (error)
740 		return (SET_ERROR(error));
741 
742 	error = zap_lookup(os, ZVOL_ZAP_OBJ, "size", 8, 1, &volsize);
743 	if (error)
744 		return (SET_ERROR(error));
745 
746 	error = dnode_hold(os, ZVOL_OBJ, zv, &zv->zv_dn);
747 	if (error)
748 		return (SET_ERROR(error));
749 
750 	zvol_os_set_capacity(zv, volsize >> 9);
751 	zv->zv_volsize = volsize;
752 
753 	if (ro || dmu_objset_is_snapshot(os) ||
754 	    !spa_writeable(dmu_objset_spa(os))) {
755 		zvol_os_set_disk_ro(zv, 1);
756 		zv->zv_flags |= ZVOL_RDONLY;
757 	} else {
758 		zvol_os_set_disk_ro(zv, 0);
759 		zv->zv_flags &= ~ZVOL_RDONLY;
760 	}
761 	return (0);
762 }
763 
764 /*
765  * Shutdown every zv_objset related stuff except zv_objset itself.
766  * The is the reverse of zvol_setup_zv.
767  */
768 static void
769 zvol_shutdown_zv(zvol_state_t *zv)
770 {
771 	ASSERT(MUTEX_HELD(&zv->zv_state_lock) &&
772 	    RW_LOCK_HELD(&zv->zv_suspend_lock));
773 
774 	if (zv->zv_flags & ZVOL_WRITTEN_TO) {
775 		ASSERT(zv->zv_zilog != NULL);
776 		zil_close(zv->zv_zilog);
777 	}
778 
779 	zv->zv_zilog = NULL;
780 
781 	dnode_rele(zv->zv_dn, zv);
782 	zv->zv_dn = NULL;
783 
784 	/*
785 	 * Evict cached data. We must write out any dirty data before
786 	 * disowning the dataset.
787 	 */
788 	if (zv->zv_flags & ZVOL_WRITTEN_TO)
789 		txg_wait_synced(dmu_objset_pool(zv->zv_objset), 0);
790 	(void) dmu_objset_evict_dbufs(zv->zv_objset);
791 }
792 
793 /*
794  * return the proper tag for rollback and recv
795  */
796 void *
797 zvol_tag(zvol_state_t *zv)
798 {
799 	ASSERT(RW_WRITE_HELD(&zv->zv_suspend_lock));
800 	return (zv->zv_open_count > 0 ? zv : NULL);
801 }
802 
803 /*
804  * Suspend the zvol for recv and rollback.
805  */
806 zvol_state_t *
807 zvol_suspend(const char *name)
808 {
809 	zvol_state_t *zv;
810 
811 	zv = zvol_find_by_name(name, RW_WRITER);
812 
813 	if (zv == NULL)
814 		return (NULL);
815 
816 	/* block all I/O, release in zvol_resume. */
817 	ASSERT(MUTEX_HELD(&zv->zv_state_lock));
818 	ASSERT(RW_WRITE_HELD(&zv->zv_suspend_lock));
819 
820 	atomic_inc(&zv->zv_suspend_ref);
821 
822 	if (zv->zv_open_count > 0)
823 		zvol_shutdown_zv(zv);
824 
825 	/*
826 	 * do not hold zv_state_lock across suspend/resume to
827 	 * avoid locking up zvol lookups
828 	 */
829 	mutex_exit(&zv->zv_state_lock);
830 
831 	/* zv_suspend_lock is released in zvol_resume() */
832 	return (zv);
833 }
834 
835 int
836 zvol_resume(zvol_state_t *zv)
837 {
838 	int error = 0;
839 
840 	ASSERT(RW_WRITE_HELD(&zv->zv_suspend_lock));
841 
842 	mutex_enter(&zv->zv_state_lock);
843 
844 	if (zv->zv_open_count > 0) {
845 		VERIFY0(dmu_objset_hold(zv->zv_name, zv, &zv->zv_objset));
846 		VERIFY3P(zv->zv_objset->os_dsl_dataset->ds_owner, ==, zv);
847 		VERIFY(dsl_dataset_long_held(zv->zv_objset->os_dsl_dataset));
848 		dmu_objset_rele(zv->zv_objset, zv);
849 
850 		error = zvol_setup_zv(zv);
851 	}
852 
853 	mutex_exit(&zv->zv_state_lock);
854 
855 	rw_exit(&zv->zv_suspend_lock);
856 	/*
857 	 * We need this because we don't hold zvol_state_lock while releasing
858 	 * zv_suspend_lock. zvol_remove_minors_impl thus cannot check
859 	 * zv_suspend_lock to determine it is safe to free because rwlock is
860 	 * not inherent atomic.
861 	 */
862 	atomic_dec(&zv->zv_suspend_ref);
863 
864 	return (SET_ERROR(error));
865 }
866 
867 int
868 zvol_first_open(zvol_state_t *zv, boolean_t readonly)
869 {
870 	objset_t *os;
871 	int error;
872 
873 	ASSERT(RW_READ_HELD(&zv->zv_suspend_lock));
874 	ASSERT(MUTEX_HELD(&zv->zv_state_lock));
875 	ASSERT(mutex_owned(&spa_namespace_lock));
876 
877 	boolean_t ro = (readonly || (strchr(zv->zv_name, '@') != NULL));
878 	error = dmu_objset_own(zv->zv_name, DMU_OST_ZVOL, ro, B_TRUE, zv, &os);
879 	if (error)
880 		return (SET_ERROR(error));
881 
882 	zv->zv_objset = os;
883 
884 	error = zvol_setup_zv(zv);
885 	if (error) {
886 		dmu_objset_disown(os, 1, zv);
887 		zv->zv_objset = NULL;
888 	}
889 
890 	return (error);
891 }
892 
893 void
894 zvol_last_close(zvol_state_t *zv)
895 {
896 	ASSERT(RW_READ_HELD(&zv->zv_suspend_lock));
897 	ASSERT(MUTEX_HELD(&zv->zv_state_lock));
898 
899 	zvol_shutdown_zv(zv);
900 
901 	dmu_objset_disown(zv->zv_objset, 1, zv);
902 	zv->zv_objset = NULL;
903 }
904 
905 typedef struct minors_job {
906 	list_t *list;
907 	list_node_t link;
908 	/* input */
909 	char *name;
910 	/* output */
911 	int error;
912 } minors_job_t;
913 
914 /*
915  * Prefetch zvol dnodes for the minors_job
916  */
917 static void
918 zvol_prefetch_minors_impl(void *arg)
919 {
920 	minors_job_t *job = arg;
921 	char *dsname = job->name;
922 	objset_t *os = NULL;
923 
924 	job->error = dmu_objset_own(dsname, DMU_OST_ZVOL, B_TRUE, B_TRUE,
925 	    FTAG, &os);
926 	if (job->error == 0) {
927 		dmu_prefetch(os, ZVOL_OBJ, 0, 0, 0, ZIO_PRIORITY_SYNC_READ);
928 		dmu_objset_disown(os, B_TRUE, FTAG);
929 	}
930 }
931 
932 /*
933  * Mask errors to continue dmu_objset_find() traversal
934  */
935 static int
936 zvol_create_snap_minor_cb(const char *dsname, void *arg)
937 {
938 	minors_job_t *j = arg;
939 	list_t *minors_list = j->list;
940 	const char *name = j->name;
941 
942 	ASSERT0(MUTEX_HELD(&spa_namespace_lock));
943 
944 	/* skip the designated dataset */
945 	if (name && strcmp(dsname, name) == 0)
946 		return (0);
947 
948 	/* at this point, the dsname should name a snapshot */
949 	if (strchr(dsname, '@') == 0) {
950 		dprintf("zvol_create_snap_minor_cb(): "
951 		    "%s is not a snapshot name\n", dsname);
952 	} else {
953 		minors_job_t *job;
954 		char *n = kmem_strdup(dsname);
955 		if (n == NULL)
956 			return (0);
957 
958 		job = kmem_alloc(sizeof (minors_job_t), KM_SLEEP);
959 		job->name = n;
960 		job->list = minors_list;
961 		job->error = 0;
962 		list_insert_tail(minors_list, job);
963 		/* don't care if dispatch fails, because job->error is 0 */
964 		taskq_dispatch(system_taskq, zvol_prefetch_minors_impl, job,
965 		    TQ_SLEEP);
966 	}
967 
968 	return (0);
969 }
970 
971 /*
972  * If spa_keystore_load_wkey() is called for an encrypted zvol,
973  * we need to look for any clones also using the key. This function
974  * is "best effort" - so we just skip over it if there are failures.
975  */
976 static void
977 zvol_add_clones(const char *dsname, list_t *minors_list)
978 {
979 	/* Also check if it has clones */
980 	dsl_dir_t *dd = NULL;
981 	dsl_pool_t *dp = NULL;
982 
983 	if (dsl_pool_hold(dsname, FTAG, &dp) != 0)
984 		return;
985 
986 	if (!spa_feature_is_enabled(dp->dp_spa,
987 	    SPA_FEATURE_ENCRYPTION))
988 		goto out;
989 
990 	if (dsl_dir_hold(dp, dsname, FTAG, &dd, NULL) != 0)
991 		goto out;
992 
993 	if (dsl_dir_phys(dd)->dd_clones == 0)
994 		goto out;
995 
996 	zap_cursor_t *zc = kmem_alloc(sizeof (zap_cursor_t), KM_SLEEP);
997 	zap_attribute_t *za = kmem_alloc(sizeof (zap_attribute_t), KM_SLEEP);
998 	objset_t *mos = dd->dd_pool->dp_meta_objset;
999 
1000 	for (zap_cursor_init(zc, mos, dsl_dir_phys(dd)->dd_clones);
1001 	    zap_cursor_retrieve(zc, za) == 0;
1002 	    zap_cursor_advance(zc)) {
1003 		dsl_dataset_t *clone;
1004 		minors_job_t *job;
1005 
1006 		if (dsl_dataset_hold_obj(dd->dd_pool,
1007 		    za->za_first_integer, FTAG, &clone) == 0) {
1008 
1009 			char name[ZFS_MAX_DATASET_NAME_LEN];
1010 			dsl_dataset_name(clone, name);
1011 
1012 			char *n = kmem_strdup(name);
1013 			job = kmem_alloc(sizeof (minors_job_t), KM_SLEEP);
1014 			job->name = n;
1015 			job->list = minors_list;
1016 			job->error = 0;
1017 			list_insert_tail(minors_list, job);
1018 
1019 			dsl_dataset_rele(clone, FTAG);
1020 		}
1021 	}
1022 	zap_cursor_fini(zc);
1023 	kmem_free(za, sizeof (zap_attribute_t));
1024 	kmem_free(zc, sizeof (zap_cursor_t));
1025 
1026 out:
1027 	if (dd != NULL)
1028 		dsl_dir_rele(dd, FTAG);
1029 	if (dp != NULL)
1030 		dsl_pool_rele(dp, FTAG);
1031 }
1032 
1033 /*
1034  * Mask errors to continue dmu_objset_find() traversal
1035  */
1036 static int
1037 zvol_create_minors_cb(const char *dsname, void *arg)
1038 {
1039 	uint64_t snapdev;
1040 	int error;
1041 	list_t *minors_list = arg;
1042 
1043 	ASSERT0(MUTEX_HELD(&spa_namespace_lock));
1044 
1045 	error = dsl_prop_get_integer(dsname, "snapdev", &snapdev, NULL);
1046 	if (error)
1047 		return (0);
1048 
1049 	/*
1050 	 * Given the name and the 'snapdev' property, create device minor nodes
1051 	 * with the linkages to zvols/snapshots as needed.
1052 	 * If the name represents a zvol, create a minor node for the zvol, then
1053 	 * check if its snapshots are 'visible', and if so, iterate over the
1054 	 * snapshots and create device minor nodes for those.
1055 	 */
1056 	if (strchr(dsname, '@') == 0) {
1057 		minors_job_t *job;
1058 		char *n = kmem_strdup(dsname);
1059 		if (n == NULL)
1060 			return (0);
1061 
1062 		job = kmem_alloc(sizeof (minors_job_t), KM_SLEEP);
1063 		job->name = n;
1064 		job->list = minors_list;
1065 		job->error = 0;
1066 		list_insert_tail(minors_list, job);
1067 		/* don't care if dispatch fails, because job->error is 0 */
1068 		taskq_dispatch(system_taskq, zvol_prefetch_minors_impl, job,
1069 		    TQ_SLEEP);
1070 
1071 		zvol_add_clones(dsname, minors_list);
1072 
1073 		if (snapdev == ZFS_SNAPDEV_VISIBLE) {
1074 			/*
1075 			 * traverse snapshots only, do not traverse children,
1076 			 * and skip the 'dsname'
1077 			 */
1078 			error = dmu_objset_find(dsname,
1079 			    zvol_create_snap_minor_cb, (void *)job,
1080 			    DS_FIND_SNAPSHOTS);
1081 		}
1082 	} else {
1083 		dprintf("zvol_create_minors_cb(): %s is not a zvol name\n",
1084 		    dsname);
1085 	}
1086 
1087 	return (0);
1088 }
1089 
1090 /*
1091  * Create minors for the specified dataset, including children and snapshots.
1092  * Pay attention to the 'snapdev' property and iterate over the snapshots
1093  * only if they are 'visible'. This approach allows one to assure that the
1094  * snapshot metadata is read from disk only if it is needed.
1095  *
1096  * The name can represent a dataset to be recursively scanned for zvols and
1097  * their snapshots, or a single zvol snapshot. If the name represents a
1098  * dataset, the scan is performed in two nested stages:
1099  * - scan the dataset for zvols, and
1100  * - for each zvol, create a minor node, then check if the zvol's snapshots
1101  *   are 'visible', and only then iterate over the snapshots if needed
1102  *
1103  * If the name represents a snapshot, a check is performed if the snapshot is
1104  * 'visible' (which also verifies that the parent is a zvol), and if so,
1105  * a minor node for that snapshot is created.
1106  */
1107 void
1108 zvol_create_minors_recursive(const char *name)
1109 {
1110 	list_t minors_list;
1111 	minors_job_t *job;
1112 
1113 	if (zvol_inhibit_dev)
1114 		return;
1115 
1116 	/*
1117 	 * This is the list for prefetch jobs. Whenever we found a match
1118 	 * during dmu_objset_find, we insert a minors_job to the list and do
1119 	 * taskq_dispatch to parallel prefetch zvol dnodes. Note we don't need
1120 	 * any lock because all list operation is done on the current thread.
1121 	 *
1122 	 * We will use this list to do zvol_os_create_minor after prefetch
1123 	 * so we don't have to traverse using dmu_objset_find again.
1124 	 */
1125 	list_create(&minors_list, sizeof (minors_job_t),
1126 	    offsetof(minors_job_t, link));
1127 
1128 
1129 	if (strchr(name, '@') != NULL) {
1130 		uint64_t snapdev;
1131 
1132 		int error = dsl_prop_get_integer(name, "snapdev",
1133 		    &snapdev, NULL);
1134 
1135 		if (error == 0 && snapdev == ZFS_SNAPDEV_VISIBLE)
1136 			(void) zvol_os_create_minor(name);
1137 	} else {
1138 		fstrans_cookie_t cookie = spl_fstrans_mark();
1139 		(void) dmu_objset_find(name, zvol_create_minors_cb,
1140 		    &minors_list, DS_FIND_CHILDREN);
1141 		spl_fstrans_unmark(cookie);
1142 	}
1143 
1144 	taskq_wait_outstanding(system_taskq, 0);
1145 
1146 	/*
1147 	 * Prefetch is completed, we can do zvol_os_create_minor
1148 	 * sequentially.
1149 	 */
1150 	while ((job = list_head(&minors_list)) != NULL) {
1151 		list_remove(&minors_list, job);
1152 		if (!job->error)
1153 			(void) zvol_os_create_minor(job->name);
1154 		kmem_strfree(job->name);
1155 		kmem_free(job, sizeof (minors_job_t));
1156 	}
1157 
1158 	list_destroy(&minors_list);
1159 }
1160 
1161 void
1162 zvol_create_minor(const char *name)
1163 {
1164 	/*
1165 	 * Note: the dsl_pool_config_lock must not be held.
1166 	 * Minor node creation needs to obtain the zvol_state_lock.
1167 	 * zvol_open() obtains the zvol_state_lock and then the dsl pool
1168 	 * config lock.  Therefore, we can't have the config lock now if
1169 	 * we are going to wait for the zvol_state_lock, because it
1170 	 * would be a lock order inversion which could lead to deadlock.
1171 	 */
1172 
1173 	if (zvol_inhibit_dev)
1174 		return;
1175 
1176 	if (strchr(name, '@') != NULL) {
1177 		uint64_t snapdev;
1178 
1179 		int error = dsl_prop_get_integer(name,
1180 		    "snapdev", &snapdev, NULL);
1181 
1182 		if (error == 0 && snapdev == ZFS_SNAPDEV_VISIBLE)
1183 			(void) zvol_os_create_minor(name);
1184 	} else {
1185 		(void) zvol_os_create_minor(name);
1186 	}
1187 }
1188 
1189 /*
1190  * Remove minors for specified dataset including children and snapshots.
1191  */
1192 
1193 static void
1194 zvol_free_task(void *arg)
1195 {
1196 	zvol_os_free(arg);
1197 }
1198 
1199 void
1200 zvol_remove_minors_impl(const char *name)
1201 {
1202 	zvol_state_t *zv, *zv_next;
1203 	int namelen = ((name) ? strlen(name) : 0);
1204 	taskqid_t t;
1205 	list_t free_list;
1206 
1207 	if (zvol_inhibit_dev)
1208 		return;
1209 
1210 	list_create(&free_list, sizeof (zvol_state_t),
1211 	    offsetof(zvol_state_t, zv_next));
1212 
1213 	rw_enter(&zvol_state_lock, RW_WRITER);
1214 
1215 	for (zv = list_head(&zvol_state_list); zv != NULL; zv = zv_next) {
1216 		zv_next = list_next(&zvol_state_list, zv);
1217 
1218 		mutex_enter(&zv->zv_state_lock);
1219 		if (name == NULL || strcmp(zv->zv_name, name) == 0 ||
1220 		    (strncmp(zv->zv_name, name, namelen) == 0 &&
1221 		    (zv->zv_name[namelen] == '/' ||
1222 		    zv->zv_name[namelen] == '@'))) {
1223 			/*
1224 			 * By holding zv_state_lock here, we guarantee that no
1225 			 * one is currently using this zv
1226 			 */
1227 
1228 			/* If in use, leave alone */
1229 			if (zv->zv_open_count > 0 ||
1230 			    atomic_read(&zv->zv_suspend_ref)) {
1231 				mutex_exit(&zv->zv_state_lock);
1232 				continue;
1233 			}
1234 
1235 			zvol_remove(zv);
1236 
1237 			/*
1238 			 * Cleared while holding zvol_state_lock as a writer
1239 			 * which will prevent zvol_open() from opening it.
1240 			 */
1241 			zvol_os_clear_private(zv);
1242 
1243 			/* Drop zv_state_lock before zvol_free() */
1244 			mutex_exit(&zv->zv_state_lock);
1245 
1246 			/* Try parallel zv_free, if failed do it in place */
1247 			t = taskq_dispatch(system_taskq, zvol_free_task, zv,
1248 			    TQ_SLEEP);
1249 			if (t == TASKQID_INVALID)
1250 				list_insert_head(&free_list, zv);
1251 		} else {
1252 			mutex_exit(&zv->zv_state_lock);
1253 		}
1254 	}
1255 	rw_exit(&zvol_state_lock);
1256 
1257 	/* Drop zvol_state_lock before calling zvol_free() */
1258 	while ((zv = list_head(&free_list)) != NULL) {
1259 		list_remove(&free_list, zv);
1260 		zvol_os_free(zv);
1261 	}
1262 }
1263 
1264 /* Remove minor for this specific volume only */
1265 static void
1266 zvol_remove_minor_impl(const char *name)
1267 {
1268 	zvol_state_t *zv = NULL, *zv_next;
1269 
1270 	if (zvol_inhibit_dev)
1271 		return;
1272 
1273 	rw_enter(&zvol_state_lock, RW_WRITER);
1274 
1275 	for (zv = list_head(&zvol_state_list); zv != NULL; zv = zv_next) {
1276 		zv_next = list_next(&zvol_state_list, zv);
1277 
1278 		mutex_enter(&zv->zv_state_lock);
1279 		if (strcmp(zv->zv_name, name) == 0) {
1280 			/*
1281 			 * By holding zv_state_lock here, we guarantee that no
1282 			 * one is currently using this zv
1283 			 */
1284 
1285 			/* If in use, leave alone */
1286 			if (zv->zv_open_count > 0 ||
1287 			    atomic_read(&zv->zv_suspend_ref)) {
1288 				mutex_exit(&zv->zv_state_lock);
1289 				continue;
1290 			}
1291 			zvol_remove(zv);
1292 
1293 			zvol_os_clear_private(zv);
1294 			mutex_exit(&zv->zv_state_lock);
1295 			break;
1296 		} else {
1297 			mutex_exit(&zv->zv_state_lock);
1298 		}
1299 	}
1300 
1301 	/* Drop zvol_state_lock before calling zvol_free() */
1302 	rw_exit(&zvol_state_lock);
1303 
1304 	if (zv != NULL)
1305 		zvol_os_free(zv);
1306 }
1307 
1308 /*
1309  * Rename minors for specified dataset including children and snapshots.
1310  */
1311 static void
1312 zvol_rename_minors_impl(const char *oldname, const char *newname)
1313 {
1314 	zvol_state_t *zv, *zv_next;
1315 	int oldnamelen;
1316 
1317 	if (zvol_inhibit_dev)
1318 		return;
1319 
1320 	oldnamelen = strlen(oldname);
1321 
1322 	rw_enter(&zvol_state_lock, RW_READER);
1323 
1324 	for (zv = list_head(&zvol_state_list); zv != NULL; zv = zv_next) {
1325 		zv_next = list_next(&zvol_state_list, zv);
1326 
1327 		mutex_enter(&zv->zv_state_lock);
1328 
1329 		if (strcmp(zv->zv_name, oldname) == 0) {
1330 			zvol_os_rename_minor(zv, newname);
1331 		} else if (strncmp(zv->zv_name, oldname, oldnamelen) == 0 &&
1332 		    (zv->zv_name[oldnamelen] == '/' ||
1333 		    zv->zv_name[oldnamelen] == '@')) {
1334 			char *name = kmem_asprintf("%s%c%s", newname,
1335 			    zv->zv_name[oldnamelen],
1336 			    zv->zv_name + oldnamelen + 1);
1337 			zvol_os_rename_minor(zv, name);
1338 			kmem_strfree(name);
1339 		}
1340 
1341 		mutex_exit(&zv->zv_state_lock);
1342 	}
1343 
1344 	rw_exit(&zvol_state_lock);
1345 }
1346 
1347 typedef struct zvol_snapdev_cb_arg {
1348 	uint64_t snapdev;
1349 } zvol_snapdev_cb_arg_t;
1350 
1351 static int
1352 zvol_set_snapdev_cb(const char *dsname, void *param)
1353 {
1354 	zvol_snapdev_cb_arg_t *arg = param;
1355 
1356 	if (strchr(dsname, '@') == NULL)
1357 		return (0);
1358 
1359 	switch (arg->snapdev) {
1360 		case ZFS_SNAPDEV_VISIBLE:
1361 			(void) zvol_os_create_minor(dsname);
1362 			break;
1363 		case ZFS_SNAPDEV_HIDDEN:
1364 			(void) zvol_remove_minor_impl(dsname);
1365 			break;
1366 	}
1367 
1368 	return (0);
1369 }
1370 
1371 static void
1372 zvol_set_snapdev_impl(char *name, uint64_t snapdev)
1373 {
1374 	zvol_snapdev_cb_arg_t arg = {snapdev};
1375 	fstrans_cookie_t cookie = spl_fstrans_mark();
1376 	/*
1377 	 * The zvol_set_snapdev_sync() sets snapdev appropriately
1378 	 * in the dataset hierarchy. Here, we only scan snapshots.
1379 	 */
1380 	dmu_objset_find(name, zvol_set_snapdev_cb, &arg, DS_FIND_SNAPSHOTS);
1381 	spl_fstrans_unmark(cookie);
1382 }
1383 
1384 static void
1385 zvol_set_volmode_impl(char *name, uint64_t volmode)
1386 {
1387 	fstrans_cookie_t cookie;
1388 	uint64_t old_volmode;
1389 	zvol_state_t *zv;
1390 
1391 	if (strchr(name, '@') != NULL)
1392 		return;
1393 
1394 	/*
1395 	 * It's unfortunate we need to remove minors before we create new ones:
1396 	 * this is necessary because our backing gendisk (zvol_state->zv_disk)
1397 	 * could be different when we set, for instance, volmode from "geom"
1398 	 * to "dev" (or vice versa).
1399 	 */
1400 	zv = zvol_find_by_name(name, RW_NONE);
1401 	if (zv == NULL && volmode == ZFS_VOLMODE_NONE)
1402 			return;
1403 	if (zv != NULL) {
1404 		old_volmode = zv->zv_volmode;
1405 		mutex_exit(&zv->zv_state_lock);
1406 		if (old_volmode == volmode)
1407 			return;
1408 		zvol_wait_close(zv);
1409 	}
1410 	cookie = spl_fstrans_mark();
1411 	switch (volmode) {
1412 		case ZFS_VOLMODE_NONE:
1413 			(void) zvol_remove_minor_impl(name);
1414 			break;
1415 		case ZFS_VOLMODE_GEOM:
1416 		case ZFS_VOLMODE_DEV:
1417 			(void) zvol_remove_minor_impl(name);
1418 			(void) zvol_os_create_minor(name);
1419 			break;
1420 		case ZFS_VOLMODE_DEFAULT:
1421 			(void) zvol_remove_minor_impl(name);
1422 			if (zvol_volmode == ZFS_VOLMODE_NONE)
1423 				break;
1424 			else /* if zvol_volmode is invalid defaults to "geom" */
1425 				(void) zvol_os_create_minor(name);
1426 			break;
1427 	}
1428 	spl_fstrans_unmark(cookie);
1429 }
1430 
1431 static zvol_task_t *
1432 zvol_task_alloc(zvol_async_op_t op, const char *name1, const char *name2,
1433     uint64_t value)
1434 {
1435 	zvol_task_t *task;
1436 
1437 	/* Never allow tasks on hidden names. */
1438 	if (name1[0] == '$')
1439 		return (NULL);
1440 
1441 	task = kmem_zalloc(sizeof (zvol_task_t), KM_SLEEP);
1442 	task->op = op;
1443 	task->value = value;
1444 
1445 	strlcpy(task->name1, name1, MAXNAMELEN);
1446 	if (name2 != NULL)
1447 		strlcpy(task->name2, name2, MAXNAMELEN);
1448 
1449 	return (task);
1450 }
1451 
1452 static void
1453 zvol_task_free(zvol_task_t *task)
1454 {
1455 	kmem_free(task, sizeof (zvol_task_t));
1456 }
1457 
1458 /*
1459  * The worker thread function performed asynchronously.
1460  */
1461 static void
1462 zvol_task_cb(void *arg)
1463 {
1464 	zvol_task_t *task = arg;
1465 
1466 	switch (task->op) {
1467 	case ZVOL_ASYNC_REMOVE_MINORS:
1468 		zvol_remove_minors_impl(task->name1);
1469 		break;
1470 	case ZVOL_ASYNC_RENAME_MINORS:
1471 		zvol_rename_minors_impl(task->name1, task->name2);
1472 		break;
1473 	case ZVOL_ASYNC_SET_SNAPDEV:
1474 		zvol_set_snapdev_impl(task->name1, task->value);
1475 		break;
1476 	case ZVOL_ASYNC_SET_VOLMODE:
1477 		zvol_set_volmode_impl(task->name1, task->value);
1478 		break;
1479 	default:
1480 		VERIFY(0);
1481 		break;
1482 	}
1483 
1484 	zvol_task_free(task);
1485 }
1486 
1487 typedef struct zvol_set_prop_int_arg {
1488 	const char *zsda_name;
1489 	uint64_t zsda_value;
1490 	zprop_source_t zsda_source;
1491 	dmu_tx_t *zsda_tx;
1492 } zvol_set_prop_int_arg_t;
1493 
1494 /*
1495  * Sanity check the dataset for safe use by the sync task.  No additional
1496  * conditions are imposed.
1497  */
1498 static int
1499 zvol_set_snapdev_check(void *arg, dmu_tx_t *tx)
1500 {
1501 	zvol_set_prop_int_arg_t *zsda = arg;
1502 	dsl_pool_t *dp = dmu_tx_pool(tx);
1503 	dsl_dir_t *dd;
1504 	int error;
1505 
1506 	error = dsl_dir_hold(dp, zsda->zsda_name, FTAG, &dd, NULL);
1507 	if (error != 0)
1508 		return (error);
1509 
1510 	dsl_dir_rele(dd, FTAG);
1511 
1512 	return (error);
1513 }
1514 
1515 static int
1516 zvol_set_snapdev_sync_cb(dsl_pool_t *dp, dsl_dataset_t *ds, void *arg)
1517 {
1518 	(void) arg;
1519 	char dsname[MAXNAMELEN];
1520 	zvol_task_t *task;
1521 	uint64_t snapdev;
1522 
1523 	dsl_dataset_name(ds, dsname);
1524 	if (dsl_prop_get_int_ds(ds, "snapdev", &snapdev) != 0)
1525 		return (0);
1526 	task = zvol_task_alloc(ZVOL_ASYNC_SET_SNAPDEV, dsname, NULL, snapdev);
1527 	if (task == NULL)
1528 		return (0);
1529 
1530 	(void) taskq_dispatch(dp->dp_spa->spa_zvol_taskq, zvol_task_cb,
1531 	    task, TQ_SLEEP);
1532 	return (0);
1533 }
1534 
1535 /*
1536  * Traverse all child datasets and apply snapdev appropriately.
1537  * We call dsl_prop_set_sync_impl() here to set the value only on the toplevel
1538  * dataset and read the effective "snapdev" on every child in the callback
1539  * function: this is because the value is not guaranteed to be the same in the
1540  * whole dataset hierarchy.
1541  */
1542 static void
1543 zvol_set_snapdev_sync(void *arg, dmu_tx_t *tx)
1544 {
1545 	zvol_set_prop_int_arg_t *zsda = arg;
1546 	dsl_pool_t *dp = dmu_tx_pool(tx);
1547 	dsl_dir_t *dd;
1548 	dsl_dataset_t *ds;
1549 	int error;
1550 
1551 	VERIFY0(dsl_dir_hold(dp, zsda->zsda_name, FTAG, &dd, NULL));
1552 	zsda->zsda_tx = tx;
1553 
1554 	error = dsl_dataset_hold(dp, zsda->zsda_name, FTAG, &ds);
1555 	if (error == 0) {
1556 		dsl_prop_set_sync_impl(ds, zfs_prop_to_name(ZFS_PROP_SNAPDEV),
1557 		    zsda->zsda_source, sizeof (zsda->zsda_value), 1,
1558 		    &zsda->zsda_value, zsda->zsda_tx);
1559 		dsl_dataset_rele(ds, FTAG);
1560 	}
1561 	dmu_objset_find_dp(dp, dd->dd_object, zvol_set_snapdev_sync_cb,
1562 	    zsda, DS_FIND_CHILDREN);
1563 
1564 	dsl_dir_rele(dd, FTAG);
1565 }
1566 
1567 int
1568 zvol_set_snapdev(const char *ddname, zprop_source_t source, uint64_t snapdev)
1569 {
1570 	zvol_set_prop_int_arg_t zsda;
1571 
1572 	zsda.zsda_name = ddname;
1573 	zsda.zsda_source = source;
1574 	zsda.zsda_value = snapdev;
1575 
1576 	return (dsl_sync_task(ddname, zvol_set_snapdev_check,
1577 	    zvol_set_snapdev_sync, &zsda, 0, ZFS_SPACE_CHECK_NONE));
1578 }
1579 
1580 /*
1581  * Sanity check the dataset for safe use by the sync task.  No additional
1582  * conditions are imposed.
1583  */
1584 static int
1585 zvol_set_volmode_check(void *arg, dmu_tx_t *tx)
1586 {
1587 	zvol_set_prop_int_arg_t *zsda = arg;
1588 	dsl_pool_t *dp = dmu_tx_pool(tx);
1589 	dsl_dir_t *dd;
1590 	int error;
1591 
1592 	error = dsl_dir_hold(dp, zsda->zsda_name, FTAG, &dd, NULL);
1593 	if (error != 0)
1594 		return (error);
1595 
1596 	dsl_dir_rele(dd, FTAG);
1597 
1598 	return (error);
1599 }
1600 
1601 static int
1602 zvol_set_volmode_sync_cb(dsl_pool_t *dp, dsl_dataset_t *ds, void *arg)
1603 {
1604 	(void) arg;
1605 	char dsname[MAXNAMELEN];
1606 	zvol_task_t *task;
1607 	uint64_t volmode;
1608 
1609 	dsl_dataset_name(ds, dsname);
1610 	if (dsl_prop_get_int_ds(ds, "volmode", &volmode) != 0)
1611 		return (0);
1612 	task = zvol_task_alloc(ZVOL_ASYNC_SET_VOLMODE, dsname, NULL, volmode);
1613 	if (task == NULL)
1614 		return (0);
1615 
1616 	(void) taskq_dispatch(dp->dp_spa->spa_zvol_taskq, zvol_task_cb,
1617 	    task, TQ_SLEEP);
1618 	return (0);
1619 }
1620 
1621 /*
1622  * Traverse all child datasets and apply volmode appropriately.
1623  * We call dsl_prop_set_sync_impl() here to set the value only on the toplevel
1624  * dataset and read the effective "volmode" on every child in the callback
1625  * function: this is because the value is not guaranteed to be the same in the
1626  * whole dataset hierarchy.
1627  */
1628 static void
1629 zvol_set_volmode_sync(void *arg, dmu_tx_t *tx)
1630 {
1631 	zvol_set_prop_int_arg_t *zsda = arg;
1632 	dsl_pool_t *dp = dmu_tx_pool(tx);
1633 	dsl_dir_t *dd;
1634 	dsl_dataset_t *ds;
1635 	int error;
1636 
1637 	VERIFY0(dsl_dir_hold(dp, zsda->zsda_name, FTAG, &dd, NULL));
1638 	zsda->zsda_tx = tx;
1639 
1640 	error = dsl_dataset_hold(dp, zsda->zsda_name, FTAG, &ds);
1641 	if (error == 0) {
1642 		dsl_prop_set_sync_impl(ds, zfs_prop_to_name(ZFS_PROP_VOLMODE),
1643 		    zsda->zsda_source, sizeof (zsda->zsda_value), 1,
1644 		    &zsda->zsda_value, zsda->zsda_tx);
1645 		dsl_dataset_rele(ds, FTAG);
1646 	}
1647 
1648 	dmu_objset_find_dp(dp, dd->dd_object, zvol_set_volmode_sync_cb,
1649 	    zsda, DS_FIND_CHILDREN);
1650 
1651 	dsl_dir_rele(dd, FTAG);
1652 }
1653 
1654 int
1655 zvol_set_volmode(const char *ddname, zprop_source_t source, uint64_t volmode)
1656 {
1657 	zvol_set_prop_int_arg_t zsda;
1658 
1659 	zsda.zsda_name = ddname;
1660 	zsda.zsda_source = source;
1661 	zsda.zsda_value = volmode;
1662 
1663 	return (dsl_sync_task(ddname, zvol_set_volmode_check,
1664 	    zvol_set_volmode_sync, &zsda, 0, ZFS_SPACE_CHECK_NONE));
1665 }
1666 
1667 void
1668 zvol_remove_minors(spa_t *spa, const char *name, boolean_t async)
1669 {
1670 	zvol_task_t *task;
1671 	taskqid_t id;
1672 
1673 	task = zvol_task_alloc(ZVOL_ASYNC_REMOVE_MINORS, name, NULL, ~0ULL);
1674 	if (task == NULL)
1675 		return;
1676 
1677 	id = taskq_dispatch(spa->spa_zvol_taskq, zvol_task_cb, task, TQ_SLEEP);
1678 	if ((async == B_FALSE) && (id != TASKQID_INVALID))
1679 		taskq_wait_id(spa->spa_zvol_taskq, id);
1680 }
1681 
1682 void
1683 zvol_rename_minors(spa_t *spa, const char *name1, const char *name2,
1684     boolean_t async)
1685 {
1686 	zvol_task_t *task;
1687 	taskqid_t id;
1688 
1689 	task = zvol_task_alloc(ZVOL_ASYNC_RENAME_MINORS, name1, name2, ~0ULL);
1690 	if (task == NULL)
1691 		return;
1692 
1693 	id = taskq_dispatch(spa->spa_zvol_taskq, zvol_task_cb, task, TQ_SLEEP);
1694 	if ((async == B_FALSE) && (id != TASKQID_INVALID))
1695 		taskq_wait_id(spa->spa_zvol_taskq, id);
1696 }
1697 
1698 boolean_t
1699 zvol_is_zvol(const char *name)
1700 {
1701 
1702 	return (zvol_os_is_zvol(name));
1703 }
1704 
1705 int
1706 zvol_init_impl(void)
1707 {
1708 	int i;
1709 
1710 	list_create(&zvol_state_list, sizeof (zvol_state_t),
1711 	    offsetof(zvol_state_t, zv_next));
1712 	rw_init(&zvol_state_lock, NULL, RW_DEFAULT, NULL);
1713 
1714 	zvol_htable = kmem_alloc(ZVOL_HT_SIZE * sizeof (struct hlist_head),
1715 	    KM_SLEEP);
1716 	for (i = 0; i < ZVOL_HT_SIZE; i++)
1717 		INIT_HLIST_HEAD(&zvol_htable[i]);
1718 
1719 	return (0);
1720 }
1721 
1722 void
1723 zvol_fini_impl(void)
1724 {
1725 	zvol_remove_minors_impl(NULL);
1726 
1727 	/*
1728 	 * The call to "zvol_remove_minors_impl" may dispatch entries to
1729 	 * the system_taskq, but it doesn't wait for those entries to
1730 	 * complete before it returns. Thus, we must wait for all of the
1731 	 * removals to finish, before we can continue.
1732 	 */
1733 	taskq_wait_outstanding(system_taskq, 0);
1734 
1735 	kmem_free(zvol_htable, ZVOL_HT_SIZE * sizeof (struct hlist_head));
1736 	list_destroy(&zvol_state_list);
1737 	rw_destroy(&zvol_state_lock);
1738 }
1739