1 // SPDX-License-Identifier: CDDL-1.0
2 /*
3 * This file and its contents are supplied under the terms of the
4 * Common Development and Distribution License ("CDDL"), version 1.0.
5 * You may only use this file in accordance with the terms of version
6 * 1.0 of the CDDL.
7 *
8 * A full copy of the text of the CDDL should have accompanied this
9 * source. A copy of the CDDL is also available via the Internet at
10 * https://opensource.org/license/CDDL-1.0.
11 */
12 /*
13 * Copyright (C) 2008-2010 Lawrence Livermore National Security, LLC.
14 * Produced at Lawrence Livermore National Laboratory (cf, DISCLAIMER).
15 * Rewritten for Linux by Brian Behlendorf <behlendorf1@llnl.gov>.
16 * LLNL-CODE-403049.
17 *
18 * ZFS volume emulation driver.
19 *
20 * Makes a DMU object look like a volume of arbitrary size, up to 2^64 bytes.
21 * Volumes are accessed through the symbolic links named:
22 *
23 * /dev/<pool_name>/<dataset_name>
24 *
25 * Volumes are persistent through reboot and module load. No user command
26 * needs to be run before opening and using a device.
27 *
28 * Copyright 2014 Nexenta Systems, Inc. All rights reserved.
29 * Copyright (c) 2016 Actifio, Inc. All rights reserved.
30 * Copyright (c) 2012, 2019 by Delphix. All rights reserved.
31 * Copyright (c) 2024, 2025, Klara, Inc.
32 */
33
34 /*
35 * Note on locking of zvol state structures.
36 *
37 * zvol_state_t represents the connection between a single dataset
38 * (DMU_OST_ZVOL) and the device "minor" (some OS-specific representation of a
39 * "disk" or "device" or "volume", eg, a /dev/zdXX node, a GEOM object, etc).
40 *
41 * The global zvol_state_lock is used to protect access to zvol_state_list and
42 * zvol_htable, which are the primary way to obtain a zvol_state_t from a name.
43 * It should not be used for anything not name-relateds, and you should avoid
44 * sleeping or waiting while its held. See zvol_find_by_name(), zvol_insert(),
45 * zvol_remove().
46 *
47 * The zv_state_lock is used to protect the contents of the associated
48 * zvol_state_t. Most of the zvol_state_t is dedicated to control and
49 * configuration; almost none of it is needed for data operations (that is,
50 * read, write, flush) so this lock is rarely taken during general IO. It
51 * should be released quickly; you should avoid sleeping or waiting while its
52 * held.
53 *
54 * zv_suspend_lock is used to suspend IO/data operations to a zvol. The read
55 * half should held for the duration of an IO operation. The write half should
56 * be taken when something to wait for IO to complete and the block further IO,
57 * eg for the duration of receive and rollback operations. This lock can be
58 * held for long periods of time.
59 *
60 * Thus, the following lock ordering appies.
61 * - take zvol_state_lock if necessary, to protect zvol_state_list
62 * - take zv_suspend_lock if necessary, by the code path in question
63 * - take zv_state_lock to protect zvol_state_t
64 *
65 * The minor operations are issued to spa->spa_zvol_taskq queues, that are
66 * single-threaded (to preserve order of minor operations), and are executed
67 * through the zvol_task_cb that dispatches the specific operations. Therefore,
68 * these operations are serialized per pool. Consequently, we can be certain
69 * that for a given zvol, there is only one operation at a time in progress.
70 * That is why one can be sure that first, zvol_state_t for a given zvol is
71 * allocated and placed on zvol_state_list, and then other minor operations for
72 * this zvol are going to proceed in the order of issue.
73 */
74
75 #include <sys/dataset_kstats.h>
76 #include <sys/dbuf.h>
77 #include <sys/dmu_traverse.h>
78 #include <sys/dsl_dataset.h>
79 #include <sys/dsl_prop.h>
80 #include <sys/dsl_dir.h>
81 #include <sys/zap.h>
82 #include <sys/zfeature.h>
83 #include <sys/zil_impl.h>
84 #include <sys/dmu_tx.h>
85 #include <sys/zio.h>
86 #include <sys/zfs_rlock.h>
87 #include <sys/spa_impl.h>
88 #include <sys/zvol.h>
89 #include <sys/zvol_impl.h>
90
91 unsigned int zvol_inhibit_dev = 0;
92 unsigned int zvol_prefetch_bytes = (128 * 1024);
93 unsigned int zvol_volmode = ZFS_VOLMODE_GEOM;
94 unsigned int zvol_threads = 0;
95 unsigned int zvol_num_taskqs = 0;
96 unsigned int zvol_request_sync = 0;
97
98 struct hlist_head *zvol_htable;
99 static list_t zvol_state_list;
100 krwlock_t zvol_state_lock;
101 extern int zfs_bclone_strict_properties;
102 extern int zfs_bclone_wait_dirty;
103 zv_taskq_t zvol_taskqs;
104
105 typedef enum {
106 ZVOL_ASYNC_CREATE_MINORS,
107 ZVOL_ASYNC_REMOVE_MINORS,
108 ZVOL_ASYNC_RENAME_MINORS,
109 ZVOL_ASYNC_SET_SNAPDEV,
110 ZVOL_ASYNC_SET_VOLMODE,
111 ZVOL_ASYNC_MAX
112 } zvol_async_op_t;
113
114 typedef struct {
115 zvol_async_op_t zt_op;
116 char zt_name1[MAXNAMELEN];
117 char zt_name2[MAXNAMELEN];
118 uint64_t zt_value;
119 uint32_t zt_total;
120 uint32_t zt_done;
121 int32_t zt_status;
122 int zt_error;
123 } zvol_task_t;
124
125 zv_request_task_t *
zv_request_task_create(zv_request_t zvr)126 zv_request_task_create(zv_request_t zvr)
127 {
128 zv_request_task_t *task;
129 task = kmem_alloc(sizeof (zv_request_task_t), KM_SLEEP);
130 taskq_init_ent(&task->ent);
131 task->zvr = zvr;
132 return (task);
133 }
134
135 void
zv_request_task_free(zv_request_task_t * task)136 zv_request_task_free(zv_request_task_t *task)
137 {
138 kmem_free(task, sizeof (*task));
139 }
140
141 uint64_t
zvol_name_hash(const char * name)142 zvol_name_hash(const char *name)
143 {
144 uint64_t crc = -1ULL;
145 ASSERT(zfs_crc64_table[128] == ZFS_CRC64_POLY);
146 for (const uint8_t *p = (const uint8_t *)name; *p != 0; p++)
147 crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ (*p)) & 0xFF];
148 return (crc);
149 }
150
151 /*
152 * Find a zvol_state_t given the name and hash generated by zvol_name_hash.
153 * If found, return with zv_suspend_lock and zv_state_lock taken, otherwise,
154 * return (NULL) without the taking locks. The zv_suspend_lock is always taken
155 * before zv_state_lock. The mode argument indicates the mode (including none)
156 * for zv_suspend_lock to be taken.
157 */
158 zvol_state_t *
zvol_find_by_name_hash(const char * name,uint64_t hash,int mode)159 zvol_find_by_name_hash(const char *name, uint64_t hash, int mode)
160 {
161 zvol_state_t *zv;
162 struct hlist_node *p = NULL;
163
164 rw_enter(&zvol_state_lock, RW_READER);
165 hlist_for_each(p, ZVOL_HT_HEAD(hash)) {
166 zv = hlist_entry(p, zvol_state_t, zv_hlink);
167 mutex_enter(&zv->zv_state_lock);
168 if (zv->zv_hash == hash && strcmp(zv->zv_name, name) == 0) {
169 /*
170 * this is the right zvol, take the locks in the
171 * right order
172 */
173 if (mode != RW_NONE &&
174 !rw_tryenter(&zv->zv_suspend_lock, mode)) {
175 mutex_exit(&zv->zv_state_lock);
176 rw_enter(&zv->zv_suspend_lock, mode);
177 mutex_enter(&zv->zv_state_lock);
178 /*
179 * zvol cannot be renamed as we continue
180 * to hold zvol_state_lock
181 */
182 ASSERT(zv->zv_hash == hash &&
183 strcmp(zv->zv_name, name) == 0);
184 }
185 rw_exit(&zvol_state_lock);
186 return (zv);
187 }
188 mutex_exit(&zv->zv_state_lock);
189 }
190 rw_exit(&zvol_state_lock);
191
192 return (NULL);
193 }
194
195 /*
196 * Find a zvol_state_t given the name.
197 * If found, return with zv_suspend_lock and zv_state_lock taken, otherwise,
198 * return (NULL) without the taking locks. The zv_suspend_lock is always taken
199 * before zv_state_lock. The mode argument indicates the mode (including none)
200 * for zv_suspend_lock to be taken.
201 */
202 static zvol_state_t *
zvol_find_by_name(const char * name,int mode)203 zvol_find_by_name(const char *name, int mode)
204 {
205 return (zvol_find_by_name_hash(name, zvol_name_hash(name), mode));
206 }
207
208 /*
209 * ZFS_IOC_CREATE callback handles dmu zvol and zap object creation.
210 */
211 void
zvol_create_cb(objset_t * os,void * arg,cred_t * cr,dmu_tx_t * tx)212 zvol_create_cb(objset_t *os, void *arg, cred_t *cr, dmu_tx_t *tx)
213 {
214 zfs_creat_t *zct = arg;
215 nvlist_t *nvprops = zct->zct_props;
216 int error;
217 uint64_t volblocksize, volsize;
218
219 VERIFY0(nvlist_lookup_uint64(nvprops,
220 zfs_prop_to_name(ZFS_PROP_VOLSIZE), &volsize));
221 if (nvlist_lookup_uint64(nvprops,
222 zfs_prop_to_name(ZFS_PROP_VOLBLOCKSIZE), &volblocksize) != 0)
223 volblocksize = zfs_prop_default_numeric(ZFS_PROP_VOLBLOCKSIZE);
224
225 /*
226 * These properties must be removed from the list so the generic
227 * property setting step won't apply to them.
228 */
229 VERIFY0(nvlist_remove_all(nvprops, zfs_prop_to_name(ZFS_PROP_VOLSIZE)));
230 (void) nvlist_remove_all(nvprops,
231 zfs_prop_to_name(ZFS_PROP_VOLBLOCKSIZE));
232
233 error = dmu_object_claim(os, ZVOL_OBJ, DMU_OT_ZVOL, volblocksize,
234 DMU_OT_NONE, 0, tx);
235 ASSERT0(error);
236
237 error = zap_create_claim(os, ZVOL_ZAP_OBJ, DMU_OT_ZVOL_PROP,
238 DMU_OT_NONE, 0, tx);
239 ASSERT0(error);
240
241 error = zap_update(os, ZVOL_ZAP_OBJ, "size", 8, 1, &volsize, tx);
242 ASSERT0(error);
243 }
244
245 /*
246 * ZFS_IOC_OBJSET_STATS entry point.
247 */
248 int
zvol_get_stats(objset_t * os,nvlist_t * nv)249 zvol_get_stats(objset_t *os, nvlist_t *nv)
250 {
251 int error;
252 dmu_object_info_t *doi;
253 uint64_t val;
254
255 error = zap_lookup(os, ZVOL_ZAP_OBJ, "size", 8, 1, &val);
256 if (error)
257 return (error);
258
259 dsl_prop_nvlist_add_uint64(nv, ZFS_PROP_VOLSIZE, val);
260 doi = kmem_alloc(sizeof (dmu_object_info_t), KM_SLEEP);
261 error = dmu_object_info(os, ZVOL_OBJ, doi);
262
263 if (error == 0) {
264 dsl_prop_nvlist_add_uint64(nv, ZFS_PROP_VOLBLOCKSIZE,
265 doi->doi_data_block_size);
266 }
267
268 kmem_free(doi, sizeof (dmu_object_info_t));
269
270 return (error);
271 }
272
273 /*
274 * Sanity check volume size.
275 */
276 int
zvol_check_volsize(uint64_t volsize,uint64_t blocksize)277 zvol_check_volsize(uint64_t volsize, uint64_t blocksize)
278 {
279 if (volsize == 0)
280 return (SET_ERROR(EINVAL));
281
282 if (volsize % blocksize != 0)
283 return (SET_ERROR(EINVAL));
284
285 #ifdef _ILP32
286 if (volsize - 1 > SPEC_MAXOFFSET_T)
287 return (SET_ERROR(EOVERFLOW));
288 #endif
289 return (0);
290 }
291
292 /*
293 * Ensure the zap is flushed then inform the VFS of the capacity change.
294 */
295 static int
zvol_update_volsize(uint64_t volsize,objset_t * os)296 zvol_update_volsize(uint64_t volsize, objset_t *os)
297 {
298 dmu_tx_t *tx;
299 int error;
300 uint64_t txg;
301
302 tx = dmu_tx_create(os);
303 dmu_tx_hold_zap(tx, ZVOL_ZAP_OBJ, TRUE, NULL);
304 dmu_tx_mark_netfree(tx);
305 error = dmu_tx_assign(tx, DMU_TX_WAIT);
306 if (error) {
307 dmu_tx_abort(tx);
308 return (error);
309 }
310 txg = dmu_tx_get_txg(tx);
311
312 error = zap_update(os, ZVOL_ZAP_OBJ, "size", 8, 1,
313 &volsize, tx);
314 dmu_tx_commit(tx);
315
316 txg_wait_synced(dmu_objset_pool(os), txg);
317
318 if (error == 0)
319 error = dmu_free_long_range(os,
320 ZVOL_OBJ, volsize, DMU_OBJECT_END);
321
322 return (error);
323 }
324
325 /*
326 * Set ZFS_PROP_VOLSIZE set entry point. Note that modifying the volume
327 * size will result in a udev "change" event being generated.
328 */
329 int
zvol_set_volsize(const char * name,uint64_t volsize)330 zvol_set_volsize(const char *name, uint64_t volsize)
331 {
332 objset_t *os = NULL;
333 uint64_t readonly;
334 int error;
335 boolean_t owned = B_FALSE;
336
337 error = dsl_prop_get_integer(name,
338 zfs_prop_to_name(ZFS_PROP_READONLY), &readonly, NULL);
339 if (error != 0)
340 return (error);
341 if (readonly)
342 return (SET_ERROR(EROFS));
343
344 zvol_state_t *zv = zvol_find_by_name(name, RW_READER);
345
346 ASSERT(zv == NULL || (MUTEX_HELD(&zv->zv_state_lock) &&
347 RW_READ_HELD(&zv->zv_suspend_lock)));
348
349 if (zv == NULL || zv->zv_objset == NULL) {
350 if (zv != NULL)
351 rw_exit(&zv->zv_suspend_lock);
352 if ((error = dmu_objset_own(name, DMU_OST_ZVOL, B_FALSE, B_TRUE,
353 FTAG, &os)) != 0) {
354 if (zv != NULL)
355 mutex_exit(&zv->zv_state_lock);
356 return (error);
357 }
358 owned = B_TRUE;
359 if (zv != NULL)
360 zv->zv_objset = os;
361 } else {
362 os = zv->zv_objset;
363 }
364
365 dmu_object_info_t *doi = kmem_alloc(sizeof (*doi), KM_SLEEP);
366
367 if ((error = dmu_object_info(os, ZVOL_OBJ, doi)) ||
368 (error = zvol_check_volsize(volsize, doi->doi_data_block_size)))
369 goto out;
370
371 error = zvol_update_volsize(volsize, os);
372 if (error == 0 && zv != NULL) {
373 zv->zv_volsize = volsize;
374 zv->zv_changed = 1;
375 }
376 out:
377 kmem_free(doi, sizeof (dmu_object_info_t));
378
379 if (owned) {
380 dmu_objset_disown(os, B_TRUE, FTAG);
381 if (zv != NULL)
382 zv->zv_objset = NULL;
383 } else {
384 rw_exit(&zv->zv_suspend_lock);
385 }
386
387 if (zv != NULL)
388 mutex_exit(&zv->zv_state_lock);
389
390 if (error == 0 && zv != NULL)
391 zvol_os_update_volsize(zv, volsize);
392
393 return (error);
394 }
395
396 /*
397 * Update volthreading.
398 */
399 int
zvol_set_volthreading(const char * name,boolean_t value)400 zvol_set_volthreading(const char *name, boolean_t value)
401 {
402 zvol_state_t *zv = zvol_find_by_name(name, RW_NONE);
403 if (zv == NULL)
404 return (-1);
405 zv->zv_threading = value;
406 mutex_exit(&zv->zv_state_lock);
407 return (0);
408 }
409
410 /*
411 * Update zvol ro property.
412 */
413 int
zvol_set_ro(const char * name,boolean_t value)414 zvol_set_ro(const char *name, boolean_t value)
415 {
416 zvol_state_t *zv = zvol_find_by_name(name, RW_NONE);
417 if (zv == NULL)
418 return (-1);
419 if (value) {
420 zvol_os_set_disk_ro(zv, 1);
421 zv->zv_flags |= ZVOL_RDONLY;
422 } else {
423 zvol_os_set_disk_ro(zv, 0);
424 zv->zv_flags &= ~ZVOL_RDONLY;
425 }
426 mutex_exit(&zv->zv_state_lock);
427 return (0);
428 }
429
430 /*
431 * Sanity check volume block size.
432 */
433 int
zvol_check_volblocksize(const char * name,uint64_t volblocksize)434 zvol_check_volblocksize(const char *name, uint64_t volblocksize)
435 {
436 /* Record sizes above 128k need the feature to be enabled */
437 if (volblocksize > SPA_OLD_MAXBLOCKSIZE) {
438 spa_t *spa;
439 int error;
440
441 if ((error = spa_open(name, &spa, FTAG)) != 0)
442 return (error);
443
444 if (!spa_feature_is_enabled(spa, SPA_FEATURE_LARGE_BLOCKS)) {
445 spa_close(spa, FTAG);
446 return (SET_ERROR(ENOTSUP));
447 }
448
449 /*
450 * We don't allow setting the property above 1MB,
451 * unless the tunable has been changed.
452 */
453 if (volblocksize > zfs_max_recordsize) {
454 spa_close(spa, FTAG);
455 return (SET_ERROR(EDOM));
456 }
457
458 spa_close(spa, FTAG);
459 }
460
461 if (volblocksize < SPA_MINBLOCKSIZE ||
462 volblocksize > SPA_MAXBLOCKSIZE ||
463 !ISP2(volblocksize))
464 return (SET_ERROR(EDOM));
465
466 return (0);
467 }
468
469 /*
470 * Replay a TX_TRUNCATE ZIL transaction if asked. TX_TRUNCATE is how we
471 * implement DKIOCFREE/free-long-range.
472 */
473 static int
zvol_replay_truncate(void * arg1,void * arg2,boolean_t byteswap)474 zvol_replay_truncate(void *arg1, void *arg2, boolean_t byteswap)
475 {
476 zvol_state_t *zv = arg1;
477 lr_truncate_t *lr = arg2;
478 uint64_t offset, length;
479
480 ASSERT3U(lr->lr_common.lrc_reclen, >=, sizeof (*lr));
481
482 if (byteswap)
483 byteswap_uint64_array(lr, sizeof (*lr));
484
485 offset = lr->lr_offset;
486 length = lr->lr_length;
487
488 dmu_tx_t *tx = dmu_tx_create(zv->zv_objset);
489 dmu_tx_mark_netfree(tx);
490 int error = dmu_tx_assign(tx, DMU_TX_WAIT);
491 if (error != 0) {
492 dmu_tx_abort(tx);
493 } else {
494 (void) zil_replaying(zv->zv_zilog, tx);
495 dmu_tx_commit(tx);
496 error = dmu_free_long_range(zv->zv_objset, ZVOL_OBJ, offset,
497 length);
498 }
499
500 return (error);
501 }
502
503 /*
504 * Replay a TX_WRITE ZIL transaction that didn't get committed
505 * after a system failure
506 */
507 static int
zvol_replay_write(void * arg1,void * arg2,boolean_t byteswap)508 zvol_replay_write(void *arg1, void *arg2, boolean_t byteswap)
509 {
510 zvol_state_t *zv = arg1;
511 lr_write_t *lr = arg2;
512 objset_t *os = zv->zv_objset;
513 char *data = (char *)(lr + 1); /* data follows lr_write_t */
514 uint64_t offset, length;
515 dmu_tx_t *tx;
516 int error;
517
518 ASSERT3U(lr->lr_common.lrc_reclen, >=, sizeof (*lr));
519
520 if (byteswap)
521 byteswap_uint64_array(lr, sizeof (*lr));
522
523 offset = lr->lr_offset;
524 length = lr->lr_length;
525
526 /* If it's a dmu_sync() block, write the whole block */
527 if (lr->lr_common.lrc_reclen == sizeof (lr_write_t)) {
528 uint64_t blocksize = BP_GET_LSIZE(&lr->lr_blkptr);
529 if (length < blocksize) {
530 offset -= offset % blocksize;
531 length = blocksize;
532 }
533 }
534
535 tx = dmu_tx_create(os);
536 dmu_tx_hold_write(tx, ZVOL_OBJ, offset, length);
537 error = dmu_tx_assign(tx, DMU_TX_WAIT);
538 if (error) {
539 dmu_tx_abort(tx);
540 } else {
541 dmu_write(os, ZVOL_OBJ, offset, length, data, tx,
542 DMU_READ_PREFETCH);
543 (void) zil_replaying(zv->zv_zilog, tx);
544 dmu_tx_commit(tx);
545 }
546
547 return (error);
548 }
549
550 /*
551 * Replay a TX_CLONE_RANGE ZIL transaction that didn't get committed
552 * after a system failure
553 */
554 static int
zvol_replay_clone_range(void * arg1,void * arg2,boolean_t byteswap)555 zvol_replay_clone_range(void *arg1, void *arg2, boolean_t byteswap)
556 {
557 zvol_state_t *zv = arg1;
558 lr_clone_range_t *lr = arg2;
559 objset_t *os = zv->zv_objset;
560 dmu_tx_t *tx;
561 int error;
562 uint64_t blksz;
563 uint64_t off;
564 uint64_t len;
565
566 ASSERT3U(lr->lr_common.lrc_reclen, >=, sizeof (*lr));
567 ASSERT3U(lr->lr_common.lrc_reclen, >=, offsetof(lr_clone_range_t,
568 lr_bps[lr->lr_nbps]));
569
570 if (byteswap)
571 byteswap_uint64_array(lr, sizeof (*lr));
572
573 ASSERT(spa_feature_is_enabled(dmu_objset_spa(os),
574 SPA_FEATURE_BLOCK_CLONING));
575
576 off = lr->lr_offset;
577 len = lr->lr_length;
578 blksz = lr->lr_blksz;
579
580 if ((off % blksz) != 0) {
581 return (SET_ERROR(EINVAL));
582 }
583
584 error = dnode_hold(os, ZVOL_OBJ, zv, &zv->zv_dn);
585 if (error != 0 || !zv->zv_dn)
586 return (error);
587 tx = dmu_tx_create(os);
588 dmu_tx_hold_clone_by_dnode(tx, zv->zv_dn, off, len, blksz);
589 error = dmu_tx_assign(tx, DMU_TX_WAIT);
590 if (error != 0) {
591 dmu_tx_abort(tx);
592 goto out;
593 }
594 error = dmu_brt_clone(zv->zv_objset, ZVOL_OBJ, off, len,
595 tx, lr->lr_bps, lr->lr_nbps);
596 if (error != 0) {
597 dmu_tx_commit(tx);
598 goto out;
599 }
600
601 /*
602 * zil_replaying() not only check if we are replaying ZIL, but also
603 * updates the ZIL header to record replay progress.
604 */
605 VERIFY(zil_replaying(zv->zv_zilog, tx));
606 dmu_tx_commit(tx);
607
608 out:
609 dnode_rele(zv->zv_dn, zv);
610 zv->zv_dn = NULL;
611 return (error);
612 }
613
614 int
zvol_clone_range(zvol_state_t * zv_src,uint64_t inoff,zvol_state_t * zv_dst,uint64_t outoff,uint64_t len)615 zvol_clone_range(zvol_state_t *zv_src, uint64_t inoff, zvol_state_t *zv_dst,
616 uint64_t outoff, uint64_t len)
617 {
618 zilog_t *zilog_dst;
619 zfs_locked_range_t *inlr, *outlr;
620 objset_t *inos, *outos;
621 dmu_tx_t *tx;
622 blkptr_t *bps;
623 size_t maxblocks;
624 int error = 0;
625
626 rw_enter(&zv_dst->zv_suspend_lock, RW_READER);
627 if (zv_dst->zv_zilog == NULL) {
628 rw_exit(&zv_dst->zv_suspend_lock);
629 rw_enter(&zv_dst->zv_suspend_lock, RW_WRITER);
630 if (zv_dst->zv_zilog == NULL) {
631 zv_dst->zv_zilog = zil_open(zv_dst->zv_objset,
632 zvol_get_data, &zv_dst->zv_kstat.dk_zil_sums);
633 zv_dst->zv_flags |= ZVOL_WRITTEN_TO;
634 VERIFY0((zv_dst->zv_zilog->zl_header->zh_flags &
635 ZIL_REPLAY_NEEDED));
636 }
637 rw_downgrade(&zv_dst->zv_suspend_lock);
638 }
639 if (zv_src != zv_dst)
640 rw_enter(&zv_src->zv_suspend_lock, RW_READER);
641
642 inos = zv_src->zv_objset;
643 outos = zv_dst->zv_objset;
644
645 /*
646 * Sanity checks
647 */
648 if (!spa_feature_is_enabled(dmu_objset_spa(outos),
649 SPA_FEATURE_BLOCK_CLONING)) {
650 error = SET_ERROR(EOPNOTSUPP);
651 goto out;
652 }
653 if (dmu_objset_spa(inos) != dmu_objset_spa(outos)) {
654 error = SET_ERROR(EXDEV);
655 goto out;
656 }
657
658 /*
659 * Block cloning from an unencrypted dataset into an encrypted
660 * dataset and vice versa is not supported.
661 */
662 if (inos->os_encrypted != outos->os_encrypted) {
663 error = SET_ERROR(EXDEV);
664 goto out;
665 }
666
667 /*
668 * Cloning across encrypted datasets is possible only if they
669 * share the same master key.
670 */
671 if (inos != outos && inos->os_encrypted &&
672 !dmu_objset_crypto_key_equal(inos, outos)) {
673 error = SET_ERROR(EXDEV);
674 goto out;
675 }
676
677 /*
678 * Cloning between datasets with different properties is possible,
679 * but it may cause confusions when copying data between them and
680 * expecting new properties to apply.
681 */
682 if (zfs_bclone_strict_properties && inos != outos &&
683 !dmu_objset_is_snapshot(inos) &&
684 (inos->os_checksum != outos->os_checksum ||
685 inos->os_compress != outos->os_compress ||
686 inos->os_copies != outos->os_copies ||
687 inos->os_dedup_checksum != outos->os_dedup_checksum)) {
688 error = SET_ERROR(EXDEV);
689 goto out;
690 }
691
692 if (zv_src->zv_volblocksize != zv_dst->zv_volblocksize) {
693 error = SET_ERROR(EINVAL);
694 goto out;
695 }
696
697 /*
698 * Cloning between datasets with different special_small_blocks would
699 * bypass storage tier migration that would occur with a regular copy.
700 */
701 if (zfs_bclone_strict_properties && inos != outos &&
702 !dmu_objset_is_snapshot(inos) &&
703 spa_has_special(dmu_objset_spa(inos))) {
704 uint64_t in_smallblk = inos->os_zpl_special_smallblock;
705 uint64_t out_smallblk = outos->os_zpl_special_smallblock;
706 if (in_smallblk != out_smallblk) {
707 uint64_t min_smallblk = MIN(in_smallblk, out_smallblk);
708 uint64_t max_smallblk = MAX(in_smallblk, out_smallblk);
709 if (min_smallblk < zv_src->zv_volblocksize &&
710 (inos->os_compress != ZIO_COMPRESS_OFF ||
711 max_smallblk >= zv_src->zv_volblocksize)) {
712 error = SET_ERROR(EXDEV);
713 goto out;
714 }
715 }
716 }
717
718 if (inoff >= zv_src->zv_volsize || outoff >= zv_dst->zv_volsize) {
719 goto out;
720 }
721
722 /*
723 * Do not read beyond boundary
724 */
725 if (len > zv_src->zv_volsize - inoff)
726 len = zv_src->zv_volsize - inoff;
727 if (len > zv_dst->zv_volsize - outoff)
728 len = zv_dst->zv_volsize - outoff;
729 if (len == 0)
730 goto out;
731
732 /*
733 * Callers might not be able to detect properly that we are read-only,
734 * so check it explicitly here.
735 */
736 if (zv_dst->zv_flags & ZVOL_RDONLY) {
737 error = SET_ERROR(EROFS);
738 goto out;
739 }
740
741 /*
742 * No overlapping if we are cloning within the same file
743 */
744 if (zv_src == zv_dst) {
745 if (inoff < outoff + len && outoff < inoff + len) {
746 error = SET_ERROR(EINVAL);
747 goto out;
748 }
749 }
750
751 /*
752 * Offsets and length must be at block boundaries
753 */
754 if ((inoff % zv_src->zv_volblocksize) != 0 ||
755 (outoff % zv_dst->zv_volblocksize) != 0) {
756 error = SET_ERROR(EINVAL);
757 goto out;
758 }
759
760 /*
761 * Length must be multiple of block size
762 */
763 if ((len % zv_src->zv_volblocksize) != 0) {
764 error = SET_ERROR(EINVAL);
765 goto out;
766 }
767
768 zilog_dst = zv_dst->zv_zilog;
769 maxblocks = zil_max_log_data(zilog_dst, sizeof (lr_clone_range_t)) /
770 sizeof (bps[0]);
771 bps = vmem_alloc(sizeof (bps[0]) * maxblocks, KM_SLEEP);
772 /*
773 * Maintain predictable lock order.
774 */
775 if (zv_src < zv_dst || (zv_src == zv_dst && inoff < outoff)) {
776 inlr = zfs_rangelock_enter(&zv_src->zv_rangelock, inoff, len,
777 RL_READER);
778 outlr = zfs_rangelock_enter(&zv_dst->zv_rangelock, outoff, len,
779 RL_WRITER);
780 } else {
781 outlr = zfs_rangelock_enter(&zv_dst->zv_rangelock, outoff, len,
782 RL_WRITER);
783 inlr = zfs_rangelock_enter(&zv_src->zv_rangelock, inoff, len,
784 RL_READER);
785 }
786
787 while (len > 0) {
788 uint64_t size, last_synced_txg;
789 size_t nbps = maxblocks;
790 size = MIN(zv_src->zv_volblocksize * maxblocks, len);
791 last_synced_txg = spa_last_synced_txg(
792 dmu_objset_spa(zv_src->zv_objset));
793 error = dmu_read_l0_bps(zv_src->zv_objset, ZVOL_OBJ, inoff,
794 size, bps, &nbps);
795 if (error != 0) {
796 /*
797 * If we are trying to clone a block that was created
798 * in the current transaction group, the error will be
799 * EAGAIN here. Based on zfs_bclone_wait_dirty either
800 * return a shortened range to the caller so it can
801 * fallback, or wait for the next TXG and check again.
802 */
803 if (error == EAGAIN && zfs_bclone_wait_dirty) {
804 txg_wait_synced(dmu_objset_pool
805 (zv_src->zv_objset), last_synced_txg + 1);
806 continue;
807 }
808 break;
809 }
810
811 tx = dmu_tx_create(zv_dst->zv_objset);
812 dmu_tx_hold_clone_by_dnode(tx, zv_dst->zv_dn, outoff, size,
813 zv_src->zv_volblocksize);
814 error = dmu_tx_assign(tx, DMU_TX_WAIT);
815 if (error != 0) {
816 dmu_tx_abort(tx);
817 break;
818 }
819 error = dmu_brt_clone(zv_dst->zv_objset, ZVOL_OBJ, outoff, size,
820 tx, bps, nbps);
821 if (error != 0) {
822 dmu_tx_commit(tx);
823 break;
824 }
825 zvol_log_clone_range(zilog_dst, tx, TX_CLONE_RANGE, outoff,
826 size, zv_src->zv_volblocksize, bps, nbps);
827 dmu_tx_commit(tx);
828 inoff += size;
829 outoff += size;
830 len -= size;
831 }
832 vmem_free(bps, sizeof (bps[0]) * maxblocks);
833 zfs_rangelock_exit(outlr);
834 zfs_rangelock_exit(inlr);
835 if (error == 0 && zv_dst->zv_objset->os_sync == ZFS_SYNC_ALWAYS) {
836 error = zil_commit(zilog_dst, ZVOL_OBJ);
837 }
838 out:
839 if (zv_src != zv_dst)
840 rw_exit(&zv_src->zv_suspend_lock);
841 rw_exit(&zv_dst->zv_suspend_lock);
842 return (error);
843 }
844
845 /*
846 * Handles TX_CLONE_RANGE transactions.
847 */
848 void
zvol_log_clone_range(zilog_t * zilog,dmu_tx_t * tx,int txtype,uint64_t off,uint64_t len,uint64_t blksz,const blkptr_t * bps,size_t nbps)849 zvol_log_clone_range(zilog_t *zilog, dmu_tx_t *tx, int txtype, uint64_t off,
850 uint64_t len, uint64_t blksz, const blkptr_t *bps, size_t nbps)
851 {
852 itx_t *itx;
853 lr_clone_range_t *lr;
854 uint64_t partlen, max_log_data;
855 size_t partnbps;
856
857 if (zil_replaying(zilog, tx))
858 return;
859
860 max_log_data = zil_max_log_data(zilog, sizeof (lr_clone_range_t));
861
862 while (nbps > 0) {
863 partnbps = MIN(nbps, max_log_data / sizeof (bps[0]));
864 partlen = partnbps * blksz;
865 ASSERT3U(partlen, <, len + blksz);
866 partlen = MIN(partlen, len);
867
868 itx = zil_itx_create(txtype,
869 sizeof (*lr) + sizeof (bps[0]) * partnbps);
870 lr = (lr_clone_range_t *)&itx->itx_lr;
871 lr->lr_foid = ZVOL_OBJ;
872 lr->lr_offset = off;
873 lr->lr_length = partlen;
874 lr->lr_blksz = blksz;
875 lr->lr_nbps = partnbps;
876 memcpy(lr->lr_bps, bps, sizeof (bps[0]) * partnbps);
877
878 zil_itx_assign(zilog, itx, tx);
879
880 bps += partnbps;
881 ASSERT3U(nbps, >=, partnbps);
882 nbps -= partnbps;
883 off += partlen;
884 ASSERT3U(len, >=, partlen);
885 len -= partlen;
886 }
887 }
888
889 static int
zvol_replay_err(void * arg1,void * arg2,boolean_t byteswap)890 zvol_replay_err(void *arg1, void *arg2, boolean_t byteswap)
891 {
892 (void) arg1, (void) arg2, (void) byteswap;
893 return (SET_ERROR(ENOTSUP));
894 }
895
896 /*
897 * Callback vectors for replaying records.
898 * Only TX_WRITE and TX_TRUNCATE are needed for zvol.
899 */
900 zil_replay_func_t *const zvol_replay_vector[TX_MAX_TYPE] = {
901 zvol_replay_err, /* no such transaction type */
902 zvol_replay_err, /* TX_CREATE */
903 zvol_replay_err, /* TX_MKDIR */
904 zvol_replay_err, /* TX_MKXATTR */
905 zvol_replay_err, /* TX_SYMLINK */
906 zvol_replay_err, /* TX_REMOVE */
907 zvol_replay_err, /* TX_RMDIR */
908 zvol_replay_err, /* TX_LINK */
909 zvol_replay_err, /* TX_RENAME */
910 zvol_replay_write, /* TX_WRITE */
911 zvol_replay_truncate, /* TX_TRUNCATE */
912 zvol_replay_err, /* TX_SETATTR */
913 zvol_replay_err, /* TX_ACL_V0 */
914 zvol_replay_err, /* TX_ACL */
915 zvol_replay_err, /* TX_CREATE_ACL */
916 zvol_replay_err, /* TX_CREATE_ATTR */
917 zvol_replay_err, /* TX_CREATE_ACL_ATTR */
918 zvol_replay_err, /* TX_MKDIR_ACL */
919 zvol_replay_err, /* TX_MKDIR_ATTR */
920 zvol_replay_err, /* TX_MKDIR_ACL_ATTR */
921 zvol_replay_err, /* TX_WRITE2 */
922 zvol_replay_err, /* TX_SETSAXATTR */
923 zvol_replay_err, /* TX_RENAME_EXCHANGE */
924 zvol_replay_err, /* TX_RENAME_WHITEOUT */
925 zvol_replay_clone_range, /* TX_CLONE_RANGE */
926 };
927
928 /*
929 * zvol_log_write() handles TX_WRITE transactions.
930 */
931 void
zvol_log_write(zvol_state_t * zv,dmu_tx_t * tx,uint64_t offset,uint64_t size,boolean_t commit)932 zvol_log_write(zvol_state_t *zv, dmu_tx_t *tx, uint64_t offset,
933 uint64_t size, boolean_t commit)
934 {
935 uint32_t blocksize = zv->zv_volblocksize;
936 zilog_t *zilog = zv->zv_zilog;
937 itx_wr_state_t write_state;
938 uint64_t log_size = 0;
939
940 if (zil_replaying(zilog, tx))
941 return;
942
943 write_state = zil_write_state(zilog, size, blocksize, B_FALSE, commit);
944
945 while (size) {
946 itx_t *itx;
947 lr_write_t *lr;
948 itx_wr_state_t wr_state = write_state;
949 ssize_t len = size;
950
951 if (wr_state == WR_COPIED && size > zil_max_copied_data(zilog))
952 wr_state = WR_NEED_COPY;
953 else if (wr_state == WR_INDIRECT)
954 len = MIN(blocksize - P2PHASE(offset, blocksize), size);
955
956 itx = zil_itx_create(TX_WRITE, sizeof (*lr) +
957 (wr_state == WR_COPIED ? len : 0));
958 lr = (lr_write_t *)&itx->itx_lr;
959 if (wr_state == WR_COPIED &&
960 dmu_read_by_dnode(zv->zv_dn, offset, len, lr + 1,
961 DMU_READ_NO_PREFETCH | DMU_KEEP_CACHING) != 0) {
962 zil_itx_destroy(itx, 0);
963 itx = zil_itx_create(TX_WRITE, sizeof (*lr));
964 lr = (lr_write_t *)&itx->itx_lr;
965 wr_state = WR_NEED_COPY;
966 }
967
968 log_size += itx->itx_size;
969 if (wr_state == WR_NEED_COPY)
970 log_size += len;
971
972 itx->itx_wr_state = wr_state;
973 lr->lr_foid = ZVOL_OBJ;
974 lr->lr_offset = offset;
975 lr->lr_length = len;
976 lr->lr_blkoff = 0;
977 BP_ZERO(&lr->lr_blkptr);
978
979 itx->itx_private = zv;
980
981 zil_itx_assign(zilog, itx, tx);
982
983 offset += len;
984 size -= len;
985 }
986
987 dsl_pool_wrlog_count(zilog->zl_dmu_pool, log_size, tx->tx_txg);
988 }
989
990 /*
991 * Log a DKIOCFREE/free-long-range to the ZIL with TX_TRUNCATE.
992 */
993 void
zvol_log_truncate(zvol_state_t * zv,dmu_tx_t * tx,uint64_t off,uint64_t len)994 zvol_log_truncate(zvol_state_t *zv, dmu_tx_t *tx, uint64_t off, uint64_t len)
995 {
996 itx_t *itx;
997 lr_truncate_t *lr;
998 zilog_t *zilog = zv->zv_zilog;
999
1000 if (zil_replaying(zilog, tx))
1001 return;
1002
1003 itx = zil_itx_create(TX_TRUNCATE, sizeof (*lr));
1004 lr = (lr_truncate_t *)&itx->itx_lr;
1005 lr->lr_foid = ZVOL_OBJ;
1006 lr->lr_offset = off;
1007 lr->lr_length = len;
1008
1009 zil_itx_assign(zilog, itx, tx);
1010 }
1011
1012
1013 static void
zvol_get_done(zgd_t * zgd,int error)1014 zvol_get_done(zgd_t *zgd, int error)
1015 {
1016 (void) error;
1017 if (zgd->zgd_db)
1018 dmu_buf_rele(zgd->zgd_db, zgd);
1019
1020 zfs_rangelock_exit(zgd->zgd_lr);
1021
1022 kmem_free(zgd, sizeof (zgd_t));
1023 }
1024
1025 /*
1026 * Get data to generate a TX_WRITE intent log record.
1027 */
1028 int
zvol_get_data(void * arg,uint64_t arg2,lr_write_t * lr,char * buf,struct lwb * lwb,zio_t * zio)1029 zvol_get_data(void *arg, uint64_t arg2, lr_write_t *lr, char *buf,
1030 struct lwb *lwb, zio_t *zio)
1031 {
1032 zvol_state_t *zv = arg;
1033 uint64_t offset = lr->lr_offset;
1034 uint64_t size = lr->lr_length;
1035 dmu_buf_t *db;
1036 zgd_t *zgd;
1037 int error;
1038
1039 ASSERT3P(lwb, !=, NULL);
1040 ASSERT3U(size, !=, 0);
1041
1042 zgd = kmem_zalloc(sizeof (zgd_t), KM_SLEEP);
1043 zgd->zgd_lwb = lwb;
1044
1045 /*
1046 * Write records come in two flavors: immediate and indirect.
1047 * For small writes it's cheaper to store the data with the
1048 * log record (immediate); for large writes it's cheaper to
1049 * sync the data and get a pointer to it (indirect) so that
1050 * we don't have to write the data twice.
1051 */
1052 if (buf != NULL) { /* immediate write */
1053 zgd->zgd_lr = zfs_rangelock_enter(&zv->zv_rangelock, offset,
1054 size, RL_READER);
1055 error = dmu_read_by_dnode(zv->zv_dn, offset, size, buf,
1056 DMU_READ_NO_PREFETCH | DMU_KEEP_CACHING);
1057 } else { /* indirect write */
1058 ASSERT3P(zio, !=, NULL);
1059 /*
1060 * Have to lock the whole block to ensure when it's written out
1061 * and its checksum is being calculated that no one can change
1062 * the data. Contrarily to zfs_get_data we need not re-check
1063 * blocksize after we get the lock because it cannot be changed.
1064 */
1065 size = zv->zv_volblocksize;
1066 offset = P2ALIGN_TYPED(offset, size, uint64_t);
1067 zgd->zgd_lr = zfs_rangelock_enter(&zv->zv_rangelock, offset,
1068 size, RL_READER);
1069 error = dmu_buf_hold_noread_by_dnode(zv->zv_dn, offset, zgd,
1070 &db);
1071 if (error == 0) {
1072 blkptr_t *bp = &lr->lr_blkptr;
1073
1074 zgd->zgd_db = db;
1075 zgd->zgd_bp = bp;
1076
1077 ASSERT(db != NULL);
1078 ASSERT(db->db_offset == offset);
1079 ASSERT(db->db_size == size);
1080
1081 error = dmu_sync(zio, lr->lr_common.lrc_txg,
1082 zvol_get_done, zgd);
1083
1084 if (error == 0)
1085 return (0);
1086 }
1087 }
1088
1089 zvol_get_done(zgd, error);
1090
1091 return (error);
1092 }
1093
1094 /*
1095 * The zvol_state_t's are inserted into zvol_state_list and zvol_htable.
1096 */
1097
1098 void
zvol_insert(zvol_state_t * zv)1099 zvol_insert(zvol_state_t *zv)
1100 {
1101 ASSERT(RW_WRITE_HELD(&zvol_state_lock));
1102 list_insert_head(&zvol_state_list, zv);
1103 hlist_add_head(&zv->zv_hlink, ZVOL_HT_HEAD(zv->zv_hash));
1104 }
1105
1106 /*
1107 * Simply remove the zvol from to list of zvols.
1108 */
1109 static void
zvol_remove(zvol_state_t * zv)1110 zvol_remove(zvol_state_t *zv)
1111 {
1112 ASSERT(RW_WRITE_HELD(&zvol_state_lock));
1113 list_remove(&zvol_state_list, zv);
1114 hlist_del(&zv->zv_hlink);
1115 }
1116
1117 /*
1118 * Setup zv after we just own the zv->objset
1119 */
1120 static int
zvol_setup_zv(zvol_state_t * zv)1121 zvol_setup_zv(zvol_state_t *zv)
1122 {
1123 uint64_t volsize;
1124 int error;
1125 uint64_t ro;
1126 objset_t *os = zv->zv_objset;
1127
1128 ASSERT(MUTEX_HELD(&zv->zv_state_lock));
1129 ASSERT(RW_LOCK_HELD(&zv->zv_suspend_lock));
1130
1131 zv->zv_zilog = NULL;
1132 zv->zv_flags &= ~ZVOL_WRITTEN_TO;
1133
1134 error = dsl_prop_get_integer(zv->zv_name, "readonly", &ro, NULL);
1135 if (error)
1136 return (error);
1137
1138 error = zap_lookup(os, ZVOL_ZAP_OBJ, "size", 8, 1, &volsize);
1139 if (error)
1140 return (error);
1141
1142 error = dnode_hold(os, ZVOL_OBJ, zv, &zv->zv_dn);
1143 if (error)
1144 return (error);
1145
1146 zvol_os_set_capacity(zv, volsize >> 9);
1147 zv->zv_volsize = volsize;
1148
1149 if (ro || dmu_objset_is_snapshot(os) ||
1150 !spa_writeable(dmu_objset_spa(os))) {
1151 zvol_os_set_disk_ro(zv, 1);
1152 zv->zv_flags |= ZVOL_RDONLY;
1153 } else {
1154 zvol_os_set_disk_ro(zv, 0);
1155 zv->zv_flags &= ~ZVOL_RDONLY;
1156 }
1157 return (0);
1158 }
1159
1160 /*
1161 * Shutdown every zv_objset related stuff except zv_objset itself.
1162 * The is the reverse of zvol_setup_zv.
1163 */
1164 static void
zvol_shutdown_zv(zvol_state_t * zv)1165 zvol_shutdown_zv(zvol_state_t *zv)
1166 {
1167 ASSERT(MUTEX_HELD(&zv->zv_state_lock) &&
1168 RW_LOCK_HELD(&zv->zv_suspend_lock));
1169
1170 if (zv->zv_flags & ZVOL_WRITTEN_TO) {
1171 ASSERT(zv->zv_zilog != NULL);
1172 zil_close(zv->zv_zilog);
1173 }
1174
1175 zv->zv_zilog = NULL;
1176
1177 dnode_rele(zv->zv_dn, zv);
1178 zv->zv_dn = NULL;
1179
1180 /*
1181 * Evict cached data. We must write out any dirty data before
1182 * disowning the dataset.
1183 */
1184 if (zv->zv_flags & ZVOL_WRITTEN_TO)
1185 txg_wait_synced(dmu_objset_pool(zv->zv_objset), 0);
1186 dmu_objset_evict_dbufs(zv->zv_objset);
1187 }
1188
1189 /*
1190 * return the proper tag for rollback and recv
1191 */
1192 void *
zvol_tag(zvol_state_t * zv)1193 zvol_tag(zvol_state_t *zv)
1194 {
1195 ASSERT(RW_WRITE_HELD(&zv->zv_suspend_lock));
1196 return (zv->zv_open_count > 0 ? zv : NULL);
1197 }
1198
1199 /*
1200 * Suspend the zvol for recv and rollback.
1201 */
1202 int
zvol_suspend(const char * name,zvol_state_t ** zvp)1203 zvol_suspend(const char *name, zvol_state_t **zvp)
1204 {
1205 zvol_state_t *zv;
1206
1207 zv = zvol_find_by_name(name, RW_WRITER);
1208
1209 if (zv == NULL)
1210 return (SET_ERROR(ENOENT));
1211
1212 /* block all I/O, release in zvol_resume. */
1213 ASSERT(MUTEX_HELD(&zv->zv_state_lock));
1214 ASSERT(RW_WRITE_HELD(&zv->zv_suspend_lock));
1215
1216 /*
1217 * If it's being removed, unlock and return error. It doesn't make any
1218 * sense to try to suspend a zvol being removed, but being here also
1219 * means that zvol_remove_minors_impl() is about to call zvol_remove()
1220 * and then destroy the zvol_state_t, so returning a pointer to it for
1221 * the caller to mess with would be a disaster anyway.
1222 */
1223 if (zv->zv_flags & ZVOL_REMOVING) {
1224 mutex_exit(&zv->zv_state_lock);
1225 rw_exit(&zv->zv_suspend_lock);
1226 /* NB: Returning EIO here to match zfsvfs_teardown() */
1227 return (SET_ERROR(EIO));
1228 }
1229
1230 atomic_inc(&zv->zv_suspend_ref);
1231
1232 if (zv->zv_open_count > 0)
1233 zvol_shutdown_zv(zv);
1234
1235 /*
1236 * do not hold zv_state_lock across suspend/resume to
1237 * avoid locking up zvol lookups
1238 */
1239 mutex_exit(&zv->zv_state_lock);
1240
1241 /* zv_suspend_lock is released in zvol_resume() */
1242 *zvp = zv;
1243 return (0);
1244 }
1245
1246 int
zvol_resume(zvol_state_t * zv)1247 zvol_resume(zvol_state_t *zv)
1248 {
1249 int error = 0;
1250
1251 ASSERT(RW_WRITE_HELD(&zv->zv_suspend_lock));
1252
1253 mutex_enter(&zv->zv_state_lock);
1254
1255 if (zv->zv_open_count > 0) {
1256 VERIFY0(dmu_objset_hold(zv->zv_name, zv, &zv->zv_objset));
1257 VERIFY3P(zv->zv_objset->os_dsl_dataset->ds_owner, ==, zv);
1258 VERIFY(dsl_dataset_long_held(zv->zv_objset->os_dsl_dataset));
1259 dmu_objset_rele(zv->zv_objset, zv);
1260
1261 error = zvol_setup_zv(zv);
1262 }
1263
1264 mutex_exit(&zv->zv_state_lock);
1265
1266 rw_exit(&zv->zv_suspend_lock);
1267 /*
1268 * We need this because we don't hold zvol_state_lock while releasing
1269 * zv_suspend_lock. zvol_remove_minors_impl thus cannot check
1270 * zv_suspend_lock to determine it is safe to free because rwlock is
1271 * not inherent atomic.
1272 */
1273 atomic_dec(&zv->zv_suspend_ref);
1274
1275 if (zv->zv_flags & ZVOL_REMOVING)
1276 cv_broadcast(&zv->zv_removing_cv);
1277
1278 return (error);
1279 }
1280
1281 int
zvol_first_open(zvol_state_t * zv,boolean_t readonly)1282 zvol_first_open(zvol_state_t *zv, boolean_t readonly)
1283 {
1284 objset_t *os;
1285 int error;
1286
1287 ASSERT(RW_READ_HELD(&zv->zv_suspend_lock));
1288 ASSERT(MUTEX_HELD(&zv->zv_state_lock));
1289 ASSERT(spa_namespace_held());
1290
1291 boolean_t ro = (readonly || (strchr(zv->zv_name, '@') != NULL));
1292 error = dmu_objset_own(zv->zv_name, DMU_OST_ZVOL, ro, B_TRUE, zv, &os);
1293 if (error)
1294 return (error);
1295
1296 zv->zv_objset = os;
1297
1298 error = zvol_setup_zv(zv);
1299 if (error) {
1300 dmu_objset_disown(os, 1, zv);
1301 zv->zv_objset = NULL;
1302 } else {
1303 /*
1304 * Take a hold on the spa so that spa_export_common() will
1305 * return EBUSY while the zvol block device is open, just
1306 * as it does for mounted datasets.
1307 */
1308 spa_open_ref(dmu_objset_spa(os), zv);
1309 }
1310
1311 return (error);
1312 }
1313
1314 void
zvol_last_close(zvol_state_t * zv)1315 zvol_last_close(zvol_state_t *zv)
1316 {
1317 ASSERT(RW_READ_HELD(&zv->zv_suspend_lock));
1318 ASSERT(MUTEX_HELD(&zv->zv_state_lock));
1319
1320 if (zv->zv_flags & ZVOL_REMOVING)
1321 cv_broadcast(&zv->zv_removing_cv);
1322
1323 spa_t *spa = dmu_objset_spa(zv->zv_objset);
1324
1325 zvol_shutdown_zv(zv);
1326
1327 dmu_objset_disown(zv->zv_objset, 1, zv);
1328 zv->zv_objset = NULL;
1329
1330 spa_close(spa, zv);
1331 }
1332
1333 typedef struct minors_job {
1334 list_t *list;
1335 list_node_t link;
1336 /* input */
1337 char *name;
1338 /* output */
1339 int error;
1340 } minors_job_t;
1341
1342 /*
1343 * Prefetch zvol dnodes for the minors_job
1344 */
1345 static void
zvol_prefetch_minors_impl(void * arg)1346 zvol_prefetch_minors_impl(void *arg)
1347 {
1348 minors_job_t *job = arg;
1349 char *dsname = job->name;
1350 objset_t *os = NULL;
1351
1352 job->error = dmu_objset_own(dsname, DMU_OST_ZVOL, B_TRUE, B_TRUE,
1353 FTAG, &os);
1354 if (job->error == 0) {
1355 dmu_prefetch_dnode(os, ZVOL_OBJ, ZIO_PRIORITY_SYNC_READ);
1356 dmu_objset_disown(os, B_TRUE, FTAG);
1357 }
1358 }
1359
1360 /*
1361 * Mask errors to continue dmu_objset_find() traversal
1362 */
1363 static int
zvol_create_snap_minor_cb(const char * dsname,void * arg)1364 zvol_create_snap_minor_cb(const char *dsname, void *arg)
1365 {
1366 minors_job_t *j = arg;
1367 list_t *minors_list = j->list;
1368 const char *name = j->name;
1369
1370 ASSERT0(spa_namespace_held());
1371
1372 /* skip the designated dataset */
1373 if (name && strcmp(dsname, name) == 0)
1374 return (0);
1375
1376 /* at this point, the dsname should name a snapshot */
1377 if (strchr(dsname, '@') == 0) {
1378 dprintf("zvol_create_snap_minor_cb(): "
1379 "%s is not a snapshot name\n", dsname);
1380 } else {
1381 minors_job_t *job;
1382 char *n = kmem_strdup(dsname);
1383 if (n == NULL)
1384 return (0);
1385
1386 job = kmem_alloc(sizeof (minors_job_t), KM_SLEEP);
1387 job->name = n;
1388 job->list = minors_list;
1389 job->error = 0;
1390 list_insert_tail(minors_list, job);
1391 /* don't care if dispatch fails, because job->error is 0 */
1392 taskq_dispatch(system_taskq, zvol_prefetch_minors_impl, job,
1393 TQ_SLEEP);
1394 }
1395
1396 return (0);
1397 }
1398
1399 /*
1400 * If spa_keystore_load_wkey() is called for an encrypted zvol,
1401 * we need to look for any clones also using the key. This function
1402 * is "best effort" - so we just skip over it if there are failures.
1403 */
1404 static void
zvol_add_clones(const char * dsname,list_t * minors_list)1405 zvol_add_clones(const char *dsname, list_t *minors_list)
1406 {
1407 /* Also check if it has clones */
1408 dsl_dir_t *dd = NULL;
1409 dsl_pool_t *dp = NULL;
1410
1411 if (dsl_pool_hold(dsname, FTAG, &dp) != 0)
1412 return;
1413
1414 if (!spa_feature_is_enabled(dp->dp_spa,
1415 SPA_FEATURE_ENCRYPTION))
1416 goto out;
1417
1418 if (dsl_dir_hold(dp, dsname, FTAG, &dd, NULL) != 0)
1419 goto out;
1420
1421 if (dsl_dir_phys(dd)->dd_clones == 0)
1422 goto out;
1423
1424 zap_cursor_t *zc = kmem_alloc(sizeof (zap_cursor_t), KM_SLEEP);
1425 zap_attribute_t *za = zap_attribute_alloc();
1426 objset_t *mos = dd->dd_pool->dp_meta_objset;
1427
1428 for (zap_cursor_init(zc, mos, dsl_dir_phys(dd)->dd_clones);
1429 zap_cursor_retrieve(zc, za) == 0;
1430 zap_cursor_advance(zc)) {
1431 dsl_dataset_t *clone;
1432 minors_job_t *job;
1433
1434 if (dsl_dataset_hold_obj(dd->dd_pool,
1435 za->za_first_integer, FTAG, &clone) == 0) {
1436
1437 char name[ZFS_MAX_DATASET_NAME_LEN];
1438 dsl_dataset_name(clone, name);
1439
1440 char *n = kmem_strdup(name);
1441 job = kmem_alloc(sizeof (minors_job_t), KM_SLEEP);
1442 job->name = n;
1443 job->list = minors_list;
1444 job->error = 0;
1445 list_insert_tail(minors_list, job);
1446
1447 dsl_dataset_rele(clone, FTAG);
1448 }
1449 }
1450 zap_cursor_fini(zc);
1451 zap_attribute_free(za);
1452 kmem_free(zc, sizeof (zap_cursor_t));
1453
1454 out:
1455 if (dd != NULL)
1456 dsl_dir_rele(dd, FTAG);
1457 dsl_pool_rele(dp, FTAG);
1458 }
1459
1460 /*
1461 * Mask errors to continue dmu_objset_find() traversal
1462 */
1463 static int
zvol_create_minors_cb(const char * dsname,void * arg)1464 zvol_create_minors_cb(const char *dsname, void *arg)
1465 {
1466 uint64_t snapdev;
1467 int error;
1468 list_t *minors_list = arg;
1469
1470 ASSERT0(spa_namespace_held());
1471
1472 error = dsl_prop_get_integer(dsname, "snapdev", &snapdev, NULL);
1473 if (error)
1474 return (0);
1475
1476 /*
1477 * Given the name and the 'snapdev' property, create device minor nodes
1478 * with the linkages to zvols/snapshots as needed.
1479 * If the name represents a zvol, create a minor node for the zvol, then
1480 * check if its snapshots are 'visible', and if so, iterate over the
1481 * snapshots and create device minor nodes for those.
1482 */
1483 if (strchr(dsname, '@') == 0) {
1484 minors_job_t *job;
1485 char *n = kmem_strdup(dsname);
1486 if (n == NULL)
1487 return (0);
1488
1489 job = kmem_alloc(sizeof (minors_job_t), KM_SLEEP);
1490 job->name = n;
1491 job->list = minors_list;
1492 job->error = 0;
1493 list_insert_tail(minors_list, job);
1494 /* don't care if dispatch fails, because job->error is 0 */
1495 taskq_dispatch(system_taskq, zvol_prefetch_minors_impl, job,
1496 TQ_SLEEP);
1497
1498 zvol_add_clones(dsname, minors_list);
1499
1500 if (snapdev == ZFS_SNAPDEV_VISIBLE) {
1501 /*
1502 * traverse snapshots only, do not traverse children,
1503 * and skip the 'dsname'
1504 */
1505 (void) dmu_objset_find(dsname,
1506 zvol_create_snap_minor_cb, (void *)job,
1507 DS_FIND_SNAPSHOTS);
1508 }
1509 } else {
1510 dprintf("zvol_create_minors_cb(): %s is not a zvol name\n",
1511 dsname);
1512 }
1513
1514 return (0);
1515 }
1516
1517 static void
zvol_task_update_status(zvol_task_t * task,uint64_t total,uint64_t done,int error)1518 zvol_task_update_status(zvol_task_t *task, uint64_t total, uint64_t done,
1519 int error)
1520 {
1521
1522 task->zt_total += total;
1523 task->zt_done += done;
1524 if (task->zt_total != task->zt_done) {
1525 task->zt_status = -1;
1526 if (error)
1527 task->zt_error = error;
1528 }
1529 }
1530
1531 static void
zvol_task_report_status(zvol_task_t * task)1532 zvol_task_report_status(zvol_task_t *task)
1533 {
1534 #ifdef ZFS_DEBUG
1535 static const char *const msg[] = {
1536 "create",
1537 "remove",
1538 "rename",
1539 "set snapdev",
1540 "set volmode",
1541 "unknown",
1542 };
1543
1544 if (task->zt_status == 0)
1545 return;
1546
1547 zvol_async_op_t op = MIN(task->zt_op, ZVOL_ASYNC_MAX);
1548 if (task->zt_error) {
1549 dprintf("The %s minors zvol task was not ok, last error %d\n",
1550 msg[op], task->zt_error);
1551 } else {
1552 dprintf("The %s minors zvol task was not ok\n", msg[op]);
1553 }
1554 #else
1555 (void) task;
1556 #endif
1557 }
1558
1559 /*
1560 * Create minors for the specified dataset, including children and snapshots.
1561 * Pay attention to the 'snapdev' property and iterate over the snapshots
1562 * only if they are 'visible'. This approach allows one to assure that the
1563 * snapshot metadata is read from disk only if it is needed.
1564 *
1565 * The name can represent a dataset to be recursively scanned for zvols and
1566 * their snapshots, or a single zvol snapshot. If the name represents a
1567 * dataset, the scan is performed in two nested stages:
1568 * - scan the dataset for zvols, and
1569 * - for each zvol, create a minor node, then check if the zvol's snapshots
1570 * are 'visible', and only then iterate over the snapshots if needed
1571 *
1572 * If the name represents a snapshot, a check is performed if the snapshot is
1573 * 'visible' (which also verifies that the parent is a zvol), and if so,
1574 * a minor node for that snapshot is created.
1575 */
1576 static void
zvol_create_minors_impl(zvol_task_t * task)1577 zvol_create_minors_impl(zvol_task_t *task)
1578 {
1579 const char *name = task->zt_name1;
1580 list_t minors_list;
1581 minors_job_t *job;
1582 uint64_t snapdev;
1583 int total = 0, done = 0, last_error, error;
1584
1585 /*
1586 * Note: the dsl_pool_config_lock must not be held.
1587 * Minor node creation needs to obtain the zvol_state_lock.
1588 * zvol_open() obtains the zvol_state_lock and then the dsl pool
1589 * config lock. Therefore, we can't have the config lock now if
1590 * we are going to wait for the zvol_state_lock, because it
1591 * would be a lock order inversion which could lead to deadlock.
1592 */
1593
1594 if (zvol_inhibit_dev) {
1595 return;
1596 }
1597
1598 /*
1599 * This is the list for prefetch jobs. Whenever we found a match
1600 * during dmu_objset_find, we insert a minors_job to the list and do
1601 * taskq_dispatch to parallel prefetch zvol dnodes. Note we don't need
1602 * any lock because all list operation is done on the current thread.
1603 *
1604 * We will use this list to do zvol_os_create_minor after prefetch
1605 * so we don't have to traverse using dmu_objset_find again.
1606 */
1607 list_create(&minors_list, sizeof (minors_job_t),
1608 offsetof(minors_job_t, link));
1609
1610
1611 if (strchr(name, '@') != NULL) {
1612 error = dsl_prop_get_integer(name, "snapdev", &snapdev, NULL);
1613 if (error == 0 && snapdev == ZFS_SNAPDEV_VISIBLE) {
1614 error = zvol_os_create_minor(name);
1615 if (error == 0) {
1616 done++;
1617 } else {
1618 last_error = error;
1619 }
1620 total++;
1621 }
1622 } else {
1623 fstrans_cookie_t cookie = spl_fstrans_mark();
1624 (void) dmu_objset_find(name, zvol_create_minors_cb,
1625 &minors_list, DS_FIND_CHILDREN);
1626 spl_fstrans_unmark(cookie);
1627 }
1628
1629 taskq_wait_outstanding(system_taskq, 0);
1630
1631 /*
1632 * Prefetch is completed, we can do zvol_os_create_minor
1633 * sequentially.
1634 */
1635 while ((job = list_remove_head(&minors_list)) != NULL) {
1636 if (!job->error) {
1637 error = zvol_os_create_minor(job->name);
1638 if (error == 0) {
1639 done++;
1640 } else {
1641 last_error = error;
1642 }
1643 } else if (job->error == EINVAL) {
1644 /*
1645 * The objset, with the name requested by current job
1646 * exist, but have the type different from zvol.
1647 * Just ignore this sort of errors.
1648 */
1649 done++;
1650 } else {
1651 last_error = job->error;
1652 }
1653 total++;
1654 kmem_strfree(job->name);
1655 kmem_free(job, sizeof (minors_job_t));
1656 }
1657
1658 list_destroy(&minors_list);
1659 zvol_task_update_status(task, total, done, last_error);
1660 }
1661
1662 /*
1663 * Remove minors for specified dataset and, optionally, its children and
1664 * snapshots.
1665 */
1666 static void
zvol_remove_minors_impl(zvol_task_t * task)1667 zvol_remove_minors_impl(zvol_task_t *task)
1668 {
1669 zvol_state_t *zv, *zv_next;
1670 const char *name = task ? task->zt_name1 : NULL;
1671 int namelen = ((name) ? strlen(name) : 0);
1672 boolean_t children = task ? !!task->zt_value : B_TRUE;
1673
1674 if (zvol_inhibit_dev)
1675 return;
1676
1677 /*
1678 * We collect up zvols that we want to remove on a separate list, so
1679 * that we don't have to hold zvol_state_lock for the whole time.
1680 *
1681 * We can't remove them from the global lists until we're completely
1682 * done with them, because that would make them appear to ZFS-side ops
1683 * that they don't exist, and the name might be reused, which can't be
1684 * good.
1685 */
1686 list_t remove_list;
1687 list_create(&remove_list, sizeof (zvol_state_t),
1688 offsetof(zvol_state_t, zv_remove_node));
1689
1690 rw_enter(&zvol_state_lock, RW_READER);
1691
1692 for (zv = list_head(&zvol_state_list); zv != NULL; zv = zv_next) {
1693 zv_next = list_next(&zvol_state_list, zv);
1694
1695 mutex_enter(&zv->zv_state_lock);
1696 if (zv->zv_flags & ZVOL_REMOVING) {
1697 /* Another thread is handling shutdown, skip it. */
1698 mutex_exit(&zv->zv_state_lock);
1699 continue;
1700 }
1701
1702 /*
1703 * This zvol should be removed if:
1704 * - no name was offered (ie removing all at shutdown); or
1705 * - name matches exactly; or
1706 * - we were asked to remove children, and
1707 * - the start of the name matches, and
1708 * - there is a '/' immediately after the matched name; or
1709 * - there is a '@' immediately after the matched name
1710 */
1711 if (name == NULL || strcmp(zv->zv_name, name) == 0 ||
1712 (children && strncmp(zv->zv_name, name, namelen) == 0 &&
1713 (zv->zv_name[namelen] == '/' ||
1714 zv->zv_name[namelen] == '@'))) {
1715
1716 /*
1717 * Matched, so mark it removal. We want to take the
1718 * write half of the suspend lock to make sure that
1719 * the zvol is not suspended, and give any data ops
1720 * chance to finish.
1721 */
1722 mutex_exit(&zv->zv_state_lock);
1723 rw_enter(&zv->zv_suspend_lock, RW_WRITER);
1724 mutex_enter(&zv->zv_state_lock);
1725
1726 if (zv->zv_flags & ZVOL_REMOVING) {
1727 /* Another thread has taken it, let them. */
1728 mutex_exit(&zv->zv_state_lock);
1729 rw_exit(&zv->zv_suspend_lock);
1730 continue;
1731 }
1732
1733 /*
1734 * Mark it and unlock. New entries will see the flag
1735 * and return ENXIO.
1736 */
1737 zv->zv_flags |= ZVOL_REMOVING;
1738 mutex_exit(&zv->zv_state_lock);
1739 rw_exit(&zv->zv_suspend_lock);
1740
1741 /* Put it on the list for the next stage. */
1742 list_insert_head(&remove_list, zv);
1743 } else
1744 mutex_exit(&zv->zv_state_lock);
1745 }
1746
1747 rw_exit(&zvol_state_lock);
1748
1749 /* Didn't match any, nothing to do! */
1750 if (list_is_empty(&remove_list)) {
1751 if (task)
1752 task->zt_error = SET_ERROR(ENOENT);
1753 return;
1754 }
1755
1756 /* Actually shut them all down. */
1757 for (zv = list_head(&remove_list); zv != NULL; zv = zv_next) {
1758 zv_next = list_next(&remove_list, zv);
1759
1760 mutex_enter(&zv->zv_state_lock);
1761
1762 /*
1763 * Still open or suspended, just wait. This can happen if, for
1764 * example, we managed to acquire zv_state_lock in the moments
1765 * where zvol_open() or zvol_release() are trading locks to
1766 * call zvol_first_open() or zvol_last_close().
1767 */
1768 while (zv->zv_open_count > 0 ||
1769 atomic_read(&zv->zv_suspend_ref))
1770 cv_wait(&zv->zv_removing_cv, &zv->zv_state_lock);
1771
1772 /*
1773 * No users, shut down the OS side. This may not remove the
1774 * minor from view immediately, depending on the kernel
1775 * specifics, but it will ensure that it is unusable and that
1776 * this zvol_state_t can never again be reached from an OS-side
1777 * operation.
1778 */
1779 zvol_os_remove_minor(zv);
1780 mutex_exit(&zv->zv_state_lock);
1781
1782 /* Remove it from the name lookup lists */
1783 rw_enter(&zvol_state_lock, RW_WRITER);
1784 zvol_remove(zv);
1785 rw_exit(&zvol_state_lock);
1786 }
1787
1788 /*
1789 * Our own references on remove_list is the last one, free them and
1790 * we're done.
1791 */
1792 while ((zv = list_remove_head(&remove_list)) != NULL)
1793 zvol_os_free(zv);
1794
1795 list_destroy(&remove_list);
1796 }
1797
1798 /* Remove minor for this specific volume only */
1799 static int
zvol_remove_minor_impl(const char * name)1800 zvol_remove_minor_impl(const char *name)
1801 {
1802 if (zvol_inhibit_dev)
1803 return (0);
1804
1805 zvol_task_t task;
1806 memset(&task, 0, sizeof (zvol_task_t));
1807 strlcpy(task.zt_name1, name, sizeof (task.zt_name1));
1808 task.zt_value = B_FALSE;
1809
1810 zvol_remove_minors_impl(&task);
1811
1812 return (task.zt_error);
1813 }
1814
1815 /*
1816 * Rename minors for specified dataset including children and snapshots.
1817 */
1818 static void
zvol_rename_minors_impl(zvol_task_t * task)1819 zvol_rename_minors_impl(zvol_task_t *task)
1820 {
1821 zvol_state_t *zv, *zv_next;
1822 const char *oldname = task->zt_name1;
1823 const char *newname = task->zt_name2;
1824 int total = 0, done = 0, last_error, error, oldnamelen;
1825
1826 if (zvol_inhibit_dev)
1827 return;
1828
1829 last_error = 0;
1830 oldnamelen = strlen(oldname);
1831
1832 rw_enter(&zvol_state_lock, RW_WRITER);
1833
1834 for (zv = list_head(&zvol_state_list); zv != NULL; zv = zv_next) {
1835 zv_next = list_next(&zvol_state_list, zv);
1836
1837 mutex_enter(&zv->zv_state_lock);
1838
1839 if (strcmp(zv->zv_name, oldname) == 0) {
1840 error = zvol_os_rename_minor(zv, newname);
1841 } else if (strncmp(zv->zv_name, oldname, oldnamelen) == 0 &&
1842 (zv->zv_name[oldnamelen] == '/' ||
1843 zv->zv_name[oldnamelen] == '@')) {
1844 char *name = kmem_asprintf("%s%c%s", newname,
1845 zv->zv_name[oldnamelen],
1846 zv->zv_name + oldnamelen + 1);
1847 error = zvol_os_rename_minor(zv, name);
1848 kmem_strfree(name);
1849 } else {
1850 error = 0;
1851 }
1852 if (error) {
1853 last_error = error;
1854 } else {
1855 done++;
1856 }
1857 total++;
1858 mutex_exit(&zv->zv_state_lock);
1859 }
1860
1861 rw_exit(&zvol_state_lock);
1862 zvol_task_update_status(task, total, done, last_error);
1863 }
1864
1865 typedef struct zvol_snapdev_cb_arg {
1866 zvol_task_t *task;
1867 uint64_t snapdev;
1868 } zvol_snapdev_cb_arg_t;
1869
1870 static int
zvol_set_snapdev_cb(const char * dsname,void * param)1871 zvol_set_snapdev_cb(const char *dsname, void *param)
1872 {
1873 zvol_snapdev_cb_arg_t *arg = param;
1874 int error = 0;
1875
1876 if (strchr(dsname, '@') == NULL)
1877 return (0);
1878
1879 switch (arg->snapdev) {
1880 case ZFS_SNAPDEV_VISIBLE:
1881 error = zvol_os_create_minor(dsname);
1882 break;
1883 case ZFS_SNAPDEV_HIDDEN:
1884 error = zvol_remove_minor_impl(dsname);
1885 break;
1886 }
1887
1888 zvol_task_update_status(arg->task, 1, error == 0, error);
1889 return (0);
1890 }
1891
1892 static void
zvol_set_snapdev_impl(zvol_task_t * task)1893 zvol_set_snapdev_impl(zvol_task_t *task)
1894 {
1895 const char *name = task->zt_name1;
1896 uint64_t snapdev = task->zt_value;
1897
1898 zvol_snapdev_cb_arg_t arg = {task, snapdev};
1899 fstrans_cookie_t cookie = spl_fstrans_mark();
1900 /*
1901 * The zvol_set_snapdev_sync() sets snapdev appropriately
1902 * in the dataset hierarchy. Here, we only scan snapshots.
1903 */
1904 dmu_objset_find(name, zvol_set_snapdev_cb, &arg, DS_FIND_SNAPSHOTS);
1905 spl_fstrans_unmark(cookie);
1906 }
1907
1908 static void
zvol_set_volmode_impl(zvol_task_t * task)1909 zvol_set_volmode_impl(zvol_task_t *task)
1910 {
1911 const char *name = task->zt_name1;
1912 uint64_t volmode = task->zt_value;
1913 fstrans_cookie_t cookie;
1914 uint64_t old_volmode;
1915 zvol_state_t *zv;
1916 int error;
1917
1918 if (strchr(name, '@') != NULL)
1919 return;
1920
1921 /*
1922 * It's unfortunate we need to remove minors before we create new ones:
1923 * this is necessary because our backing gendisk (zvol_state->zv_disk)
1924 * could be different when we set, for instance, volmode from "geom"
1925 * to "dev" (or vice versa).
1926 */
1927 zv = zvol_find_by_name(name, RW_NONE);
1928 if (zv == NULL && volmode == ZFS_VOLMODE_NONE)
1929 return;
1930 if (zv != NULL) {
1931 old_volmode = zv->zv_volmode;
1932 mutex_exit(&zv->zv_state_lock);
1933 if (old_volmode == volmode)
1934 return;
1935 zvol_wait_close(zv);
1936 }
1937 cookie = spl_fstrans_mark();
1938 switch (volmode) {
1939 case ZFS_VOLMODE_NONE:
1940 error = zvol_remove_minor_impl(name);
1941 break;
1942 case ZFS_VOLMODE_GEOM:
1943 case ZFS_VOLMODE_DEV:
1944 error = zvol_remove_minor_impl(name);
1945 /*
1946 * The remove minor function call above, might be not
1947 * needed, if volmode was switched from 'none' value.
1948 * Ignore error in this case.
1949 */
1950 if (error == ENOENT)
1951 error = 0;
1952 else if (error)
1953 break;
1954 error = zvol_os_create_minor(name);
1955 break;
1956 case ZFS_VOLMODE_DEFAULT:
1957 error = zvol_remove_minor_impl(name);
1958 if (zvol_volmode == ZFS_VOLMODE_NONE)
1959 break;
1960 else /* if zvol_volmode is invalid defaults to "geom" */
1961 error = zvol_os_create_minor(name);
1962 break;
1963 }
1964 zvol_task_update_status(task, 1, error == 0, error);
1965 spl_fstrans_unmark(cookie);
1966 }
1967
1968 /*
1969 * The worker thread function performed asynchronously.
1970 */
1971 static void
zvol_task_cb(void * arg)1972 zvol_task_cb(void *arg)
1973 {
1974 zvol_task_t *task = arg;
1975
1976 switch (task->zt_op) {
1977 case ZVOL_ASYNC_CREATE_MINORS:
1978 zvol_create_minors_impl(task);
1979 break;
1980 case ZVOL_ASYNC_REMOVE_MINORS:
1981 zvol_remove_minors_impl(task);
1982 break;
1983 case ZVOL_ASYNC_RENAME_MINORS:
1984 zvol_rename_minors_impl(task);
1985 break;
1986 case ZVOL_ASYNC_SET_SNAPDEV:
1987 zvol_set_snapdev_impl(task);
1988 break;
1989 case ZVOL_ASYNC_SET_VOLMODE:
1990 zvol_set_volmode_impl(task);
1991 break;
1992 default:
1993 VERIFY(0);
1994 break;
1995 }
1996
1997 zvol_task_report_status(task);
1998 kmem_free(task, sizeof (zvol_task_t));
1999 }
2000
2001 typedef struct zvol_set_prop_int_arg {
2002 const char *zsda_name;
2003 uint64_t zsda_value;
2004 zprop_source_t zsda_source;
2005 zfs_prop_t zsda_prop;
2006 taskqid_t zsda_taskqid;
2007 boolean_t zsda_dispatched;
2008 kmutex_t zsda_lock;
2009 kcondvar_t zsda_cv;
2010 } zvol_set_prop_int_arg_t;
2011
2012 /*
2013 * Sanity check the dataset for safe use by the sync task. No additional
2014 * conditions are imposed.
2015 */
2016 static int
zvol_set_common_check(void * arg,dmu_tx_t * tx)2017 zvol_set_common_check(void *arg, dmu_tx_t *tx)
2018 {
2019 zvol_set_prop_int_arg_t *zsda = arg;
2020 dsl_pool_t *dp = dmu_tx_pool(tx);
2021 dsl_dir_t *dd;
2022 int error;
2023
2024 error = dsl_dir_hold(dp, zsda->zsda_name, FTAG, &dd, NULL);
2025 if (error != 0)
2026 return (error);
2027
2028 dsl_dir_rele(dd, FTAG);
2029
2030 return (error);
2031 }
2032
2033 static int
zvol_set_common_sync_cb(dsl_pool_t * dp,dsl_dataset_t * ds,void * arg)2034 zvol_set_common_sync_cb(dsl_pool_t *dp, dsl_dataset_t *ds, void *arg)
2035 {
2036 zvol_set_prop_int_arg_t *zsda = arg;
2037 char dsname[ZFS_MAX_DATASET_NAME_LEN];
2038 zvol_task_t *task;
2039 uint64_t prop;
2040 taskqid_t id;
2041
2042 const char *prop_name = zfs_prop_to_name(zsda->zsda_prop);
2043 dsl_dataset_name(ds, dsname);
2044
2045 if (dsl_prop_get_int_ds(ds, prop_name, &prop) != 0)
2046 return (0);
2047
2048 task = kmem_zalloc(sizeof (zvol_task_t), KM_SLEEP);
2049 if (zsda->zsda_prop == ZFS_PROP_VOLMODE) {
2050 task->zt_op = ZVOL_ASYNC_SET_VOLMODE;
2051 } else if (zsda->zsda_prop == ZFS_PROP_SNAPDEV) {
2052 task->zt_op = ZVOL_ASYNC_SET_SNAPDEV;
2053 } else {
2054 kmem_free(task, sizeof (zvol_task_t));
2055 return (0);
2056 }
2057 task->zt_value = prop;
2058 strlcpy(task->zt_name1, dsname, sizeof (task->zt_name1));
2059 id = taskq_dispatch(dp->dp_spa->spa_zvol_taskq, zvol_task_cb, task,
2060 TQ_SLEEP);
2061 mutex_enter(&zsda->zsda_lock);
2062 if (id != TASKQID_INVALID && id > zsda->zsda_taskqid)
2063 zsda->zsda_taskqid = id;
2064 mutex_exit(&zsda->zsda_lock);
2065 return (0);
2066 }
2067
2068 /*
2069 * Traverse all child datasets and apply the property appropriately.
2070 * We call dsl_prop_set_sync_impl() here to set the value only on the toplevel
2071 * dataset and read the effective "property" on every child in the callback
2072 * function: this is because the value is not guaranteed to be the same in the
2073 * whole dataset hierarchy.
2074 */
2075 static void
zvol_set_common_sync(void * arg,dmu_tx_t * tx)2076 zvol_set_common_sync(void *arg, dmu_tx_t *tx)
2077 {
2078 zvol_set_prop_int_arg_t *zsda = arg;
2079 dsl_pool_t *dp = dmu_tx_pool(tx);
2080 dsl_dir_t *dd;
2081 dsl_dataset_t *ds;
2082 int error;
2083
2084 VERIFY0(dsl_dir_hold(dp, zsda->zsda_name, FTAG, &dd, NULL));
2085
2086 error = dsl_dataset_hold(dp, zsda->zsda_name, FTAG, &ds);
2087 if (error == 0) {
2088 dsl_prop_set_sync_impl(ds, zfs_prop_to_name(zsda->zsda_prop),
2089 zsda->zsda_source, sizeof (zsda->zsda_value), 1,
2090 &zsda->zsda_value, tx);
2091 dsl_dataset_rele(ds, FTAG);
2092 }
2093
2094 dmu_objset_find_dp(dp, dd->dd_object, zvol_set_common_sync_cb,
2095 zsda, DS_FIND_CHILDREN);
2096
2097 mutex_enter(&zsda->zsda_lock);
2098 zsda->zsda_dispatched = TRUE;
2099 cv_broadcast(&zsda->zsda_cv);
2100 mutex_exit(&zsda->zsda_lock);
2101
2102 dsl_dir_rele(dd, FTAG);
2103 }
2104
2105 int
zvol_set_common(const char * ddname,zfs_prop_t prop,zprop_source_t source,uint64_t val)2106 zvol_set_common(const char *ddname, zfs_prop_t prop, zprop_source_t source,
2107 uint64_t val)
2108 {
2109 zvol_set_prop_int_arg_t zsda;
2110 spa_t *spa;
2111 int error;
2112
2113 zsda.zsda_name = ddname;
2114 zsda.zsda_source = source;
2115 zsda.zsda_value = val;
2116 zsda.zsda_prop = prop;
2117 zsda.zsda_taskqid = TASKQID_INVALID;
2118 zsda.zsda_dispatched = FALSE;
2119 mutex_init(&zsda.zsda_lock, NULL, MUTEX_DEFAULT, NULL);
2120 cv_init(&zsda.zsda_cv, NULL, CV_DEFAULT, NULL);
2121
2122 error = spa_open(ddname, &spa, FTAG);
2123 if (error != 0)
2124 goto out;
2125 error = dsl_sync_task(ddname, zvol_set_common_check,
2126 zvol_set_common_sync, &zsda, 0, ZFS_SPACE_CHECK_NONE);
2127 if (error == 0) {
2128 mutex_enter(&zsda.zsda_lock);
2129 while (!zsda.zsda_dispatched)
2130 cv_wait(&zsda.zsda_cv, &zsda.zsda_lock);
2131 mutex_exit(&zsda.zsda_lock);
2132
2133 if (zsda.zsda_taskqid != TASKQID_INVALID)
2134 taskq_wait_outstanding(spa->spa_zvol_taskq,
2135 zsda.zsda_taskqid);
2136 }
2137 spa_close(spa, FTAG);
2138 out:
2139 cv_destroy(&zsda.zsda_cv);
2140 mutex_destroy(&zsda.zsda_lock);
2141 return (error);
2142 }
2143
2144 void
zvol_create_minors(const char * name)2145 zvol_create_minors(const char *name)
2146 {
2147 spa_t *spa;
2148 zvol_task_t *task;
2149 taskqid_t id;
2150
2151 if (spa_open(name, &spa, FTAG) != 0)
2152 return;
2153
2154 task = kmem_zalloc(sizeof (zvol_task_t), KM_SLEEP);
2155 task->zt_op = ZVOL_ASYNC_CREATE_MINORS;
2156 strlcpy(task->zt_name1, name, sizeof (task->zt_name1));
2157 id = taskq_dispatch(spa->spa_zvol_taskq, zvol_task_cb, task, TQ_SLEEP);
2158 if (id != TASKQID_INVALID)
2159 taskq_wait_id(spa->spa_zvol_taskq, id);
2160
2161 spa_close(spa, FTAG);
2162 }
2163
2164 void
zvol_remove_minors(spa_t * spa,const char * name,boolean_t async)2165 zvol_remove_minors(spa_t *spa, const char *name, boolean_t async)
2166 {
2167 zvol_task_t *task;
2168 taskqid_t id;
2169
2170 task = kmem_zalloc(sizeof (zvol_task_t), KM_SLEEP);
2171 task->zt_op = ZVOL_ASYNC_REMOVE_MINORS;
2172 strlcpy(task->zt_name1, name, sizeof (task->zt_name1));
2173 task->zt_value = B_TRUE;
2174 id = taskq_dispatch(spa->spa_zvol_taskq, zvol_task_cb, task, TQ_SLEEP);
2175 if ((async == B_FALSE) && (id != TASKQID_INVALID))
2176 taskq_wait_id(spa->spa_zvol_taskq, id);
2177 }
2178
2179 void
zvol_rename_minors(spa_t * spa,const char * name1,const char * name2,boolean_t async)2180 zvol_rename_minors(spa_t *spa, const char *name1, const char *name2,
2181 boolean_t async)
2182 {
2183 zvol_task_t *task;
2184 taskqid_t id;
2185
2186 task = kmem_zalloc(sizeof (zvol_task_t), KM_SLEEP);
2187 task->zt_op = ZVOL_ASYNC_RENAME_MINORS;
2188 strlcpy(task->zt_name1, name1, sizeof (task->zt_name1));
2189 strlcpy(task->zt_name2, name2, sizeof (task->zt_name2));
2190 id = taskq_dispatch(spa->spa_zvol_taskq, zvol_task_cb, task, TQ_SLEEP);
2191 if ((async == B_FALSE) && (id != TASKQID_INVALID))
2192 taskq_wait_id(spa->spa_zvol_taskq, id);
2193 }
2194
2195 boolean_t
zvol_is_zvol(const char * name)2196 zvol_is_zvol(const char *name)
2197 {
2198
2199 return (zvol_os_is_zvol(name));
2200 }
2201
2202 int
zvol_init_impl(void)2203 zvol_init_impl(void)
2204 {
2205 int i;
2206
2207 /*
2208 * zvol_threads is the module param the user passes in.
2209 *
2210 * zvol_actual_threads is what we use internally, since the user can
2211 * pass zvol_thread = 0 to mean "use all the CPUs" (the default).
2212 */
2213 static unsigned int zvol_actual_threads;
2214
2215 if (zvol_threads == 0) {
2216 /*
2217 * See dde9380a1 for why 32 was chosen here. This should
2218 * probably be refined to be some multiple of the number
2219 * of CPUs.
2220 */
2221 zvol_actual_threads = MAX(max_ncpus, 32);
2222 } else {
2223 zvol_actual_threads = MIN(MAX(zvol_threads, 1), 1024);
2224 }
2225
2226 /*
2227 * Use at least 32 zvol_threads but for many core system,
2228 * prefer 6 threads per taskq, but no more taskqs
2229 * than threads in them on large systems.
2230 *
2231 * taskq total
2232 * cpus taskqs threads threads
2233 * ------- ------- ------- -------
2234 * 1 1 32 32
2235 * 2 1 32 32
2236 * 4 1 32 32
2237 * 8 2 16 32
2238 * 16 3 11 33
2239 * 32 5 7 35
2240 * 64 8 8 64
2241 * 128 11 12 132
2242 * 256 16 16 256
2243 */
2244 zv_taskq_t *ztqs = &zvol_taskqs;
2245 int num_tqs = MIN(max_ncpus, zvol_num_taskqs);
2246 if (num_tqs == 0) {
2247 num_tqs = 1 + max_ncpus / 6;
2248 while (num_tqs * num_tqs > zvol_actual_threads)
2249 num_tqs--;
2250 }
2251
2252 int per_tq_thread = zvol_actual_threads / num_tqs;
2253 if (per_tq_thread * num_tqs < zvol_actual_threads)
2254 per_tq_thread++;
2255
2256 ztqs->tqs_cnt = num_tqs;
2257 ztqs->tqs_taskq = kmem_alloc(num_tqs * sizeof (taskq_t *), KM_SLEEP);
2258
2259 for (uint_t i = 0; i < num_tqs; i++) {
2260 char name[32];
2261 (void) snprintf(name, sizeof (name), "%s_tq-%u",
2262 ZVOL_DRIVER, i);
2263 ztqs->tqs_taskq[i] = taskq_create(name, per_tq_thread,
2264 maxclsyspri, per_tq_thread, INT_MAX,
2265 TASKQ_PREPOPULATE | TASKQ_DYNAMIC);
2266 if (ztqs->tqs_taskq[i] == NULL) {
2267 for (int j = i - 1; j >= 0; j--)
2268 taskq_destroy(ztqs->tqs_taskq[j]);
2269 kmem_free(ztqs->tqs_taskq, ztqs->tqs_cnt *
2270 sizeof (taskq_t *));
2271 ztqs->tqs_taskq = NULL;
2272 return (SET_ERROR(ENOMEM));
2273 }
2274 }
2275
2276 list_create(&zvol_state_list, sizeof (zvol_state_t),
2277 offsetof(zvol_state_t, zv_next));
2278 rw_init(&zvol_state_lock, NULL, RW_DEFAULT, NULL);
2279
2280 zvol_htable = kmem_alloc(ZVOL_HT_SIZE * sizeof (struct hlist_head),
2281 KM_SLEEP);
2282 for (i = 0; i < ZVOL_HT_SIZE; i++)
2283 INIT_HLIST_HEAD(&zvol_htable[i]);
2284
2285 return (0);
2286 }
2287
2288 void
zvol_fini_impl(void)2289 zvol_fini_impl(void)
2290 {
2291 zv_taskq_t *ztqs = &zvol_taskqs;
2292
2293 zvol_remove_minors_impl(NULL);
2294
2295 kmem_free(zvol_htable, ZVOL_HT_SIZE * sizeof (struct hlist_head));
2296 list_destroy(&zvol_state_list);
2297 rw_destroy(&zvol_state_lock);
2298
2299 if (ztqs->tqs_taskq == NULL) {
2300 ASSERT0(ztqs->tqs_cnt);
2301 } else {
2302 for (uint_t i = 0; i < ztqs->tqs_cnt; i++) {
2303 ASSERT3P(ztqs->tqs_taskq[i], !=, NULL);
2304 taskq_destroy(ztqs->tqs_taskq[i]);
2305 }
2306 kmem_free(ztqs->tqs_taskq, ztqs->tqs_cnt *
2307 sizeof (taskq_t *));
2308 ztqs->tqs_taskq = NULL;
2309 }
2310 }
2311
2312 ZFS_MODULE_PARAM(zfs_vol, zvol_, inhibit_dev, UINT, ZMOD_RW,
2313 "Do not create zvol device nodes");
2314 ZFS_MODULE_PARAM(zfs_vol, zvol_, prefetch_bytes, UINT, ZMOD_RW,
2315 "Prefetch N bytes at zvol start+end");
2316 ZFS_MODULE_PARAM(zfs_vol, zvol_vol, mode, UINT, ZMOD_RW,
2317 "Default volmode property value");
2318 ZFS_MODULE_PARAM(zfs_vol, zvol_, threads, UINT, ZMOD_RW,
2319 "Number of threads for I/O requests. Set to 0 to use all active CPUs");
2320 ZFS_MODULE_PARAM(zfs_vol, zvol_, num_taskqs, UINT, ZMOD_RW,
2321 "Number of zvol taskqs");
2322 ZFS_MODULE_PARAM(zfs_vol, zvol_, request_sync, UINT, ZMOD_RW,
2323 "Synchronously handle bio requests");
2324