1 // SPDX-License-Identifier: CDDL-1.0 2 /* 3 * CDDL HEADER START 4 * 5 * The contents of this file are subject to the terms of the 6 * Common Development and Distribution License (the "License"). 7 * You may not use this file except in compliance with the License. 8 * 9 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 10 * or https://opensource.org/licenses/CDDL-1.0. 11 * See the License for the specific language governing permissions 12 * and limitations under the License. 13 * 14 * When distributing Covered Code, include this CDDL HEADER in each 15 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 16 * If applicable, add the following below this CDDL HEADER, with the 17 * fields enclosed by brackets "[]" replaced with your own identifying 18 * information: Portions Copyright [yyyy] [name of copyright owner] 19 * 20 * CDDL HEADER END 21 */ 22 /* 23 * Copyright (C) 2008-2010 Lawrence Livermore National Security, LLC. 24 * Produced at Lawrence Livermore National Laboratory (cf, DISCLAIMER). 25 * Rewritten for Linux by Brian Behlendorf <behlendorf1@llnl.gov>. 26 * LLNL-CODE-403049. 27 * 28 * ZFS volume emulation driver. 29 * 30 * Makes a DMU object look like a volume of arbitrary size, up to 2^64 bytes. 31 * Volumes are accessed through the symbolic links named: 32 * 33 * /dev/<pool_name>/<dataset_name> 34 * 35 * Volumes are persistent through reboot and module load. No user command 36 * needs to be run before opening and using a device. 37 * 38 * Copyright 2014 Nexenta Systems, Inc. All rights reserved. 39 * Copyright (c) 2016 Actifio, Inc. All rights reserved. 40 * Copyright (c) 2012, 2019 by Delphix. All rights reserved. 41 * Copyright (c) 2024, 2025, Klara, Inc. 42 */ 43 44 /* 45 * Note on locking of zvol state structures. 46 * 47 * zvol_state_t represents the connection between a single dataset 48 * (DMU_OST_ZVOL) and the device "minor" (some OS-specific representation of a 49 * "disk" or "device" or "volume", eg, a /dev/zdXX node, a GEOM object, etc). 50 * 51 * The global zvol_state_lock is used to protect access to zvol_state_list and 52 * zvol_htable, which are the primary way to obtain a zvol_state_t from a name. 53 * It should not be used for anything not name-relateds, and you should avoid 54 * sleeping or waiting while its held. See zvol_find_by_name(), zvol_insert(), 55 * zvol_remove(). 56 * 57 * The zv_state_lock is used to protect the contents of the associated 58 * zvol_state_t. Most of the zvol_state_t is dedicated to control and 59 * configuration; almost none of it is needed for data operations (that is, 60 * read, write, flush) so this lock is rarely taken during general IO. It 61 * should be released quickly; you should avoid sleeping or waiting while its 62 * held. 63 * 64 * zv_suspend_lock is used to suspend IO/data operations to a zvol. The read 65 * half should held for the duration of an IO operation. The write half should 66 * be taken when something to wait for IO to complete and the block further IO, 67 * eg for the duration of receive and rollback operations. This lock can be 68 * held for long periods of time. 69 * 70 * Thus, the following lock ordering appies. 71 * - take zvol_state_lock if necessary, to protect zvol_state_list 72 * - take zv_suspend_lock if necessary, by the code path in question 73 * - take zv_state_lock to protect zvol_state_t 74 * 75 * The minor operations are issued to spa->spa_zvol_taskq queues, that are 76 * single-threaded (to preserve order of minor operations), and are executed 77 * through the zvol_task_cb that dispatches the specific operations. Therefore, 78 * these operations are serialized per pool. Consequently, we can be certain 79 * that for a given zvol, there is only one operation at a time in progress. 80 * That is why one can be sure that first, zvol_state_t for a given zvol is 81 * allocated and placed on zvol_state_list, and then other minor operations for 82 * this zvol are going to proceed in the order of issue. 83 */ 84 85 #include <sys/dataset_kstats.h> 86 #include <sys/dbuf.h> 87 #include <sys/dmu_traverse.h> 88 #include <sys/dsl_dataset.h> 89 #include <sys/dsl_prop.h> 90 #include <sys/dsl_dir.h> 91 #include <sys/zap.h> 92 #include <sys/zfeature.h> 93 #include <sys/zil_impl.h> 94 #include <sys/dmu_tx.h> 95 #include <sys/zio.h> 96 #include <sys/zfs_rlock.h> 97 #include <sys/spa_impl.h> 98 #include <sys/zvol.h> 99 #include <sys/zvol_impl.h> 100 101 unsigned int zvol_inhibit_dev = 0; 102 unsigned int zvol_prefetch_bytes = (128 * 1024); 103 unsigned int zvol_volmode = ZFS_VOLMODE_GEOM; 104 unsigned int zvol_threads = 0; 105 unsigned int zvol_num_taskqs = 0; 106 unsigned int zvol_request_sync = 0; 107 108 struct hlist_head *zvol_htable; 109 static list_t zvol_state_list; 110 krwlock_t zvol_state_lock; 111 extern int zfs_bclone_wait_dirty; 112 zv_taskq_t zvol_taskqs; 113 114 typedef enum { 115 ZVOL_ASYNC_CREATE_MINORS, 116 ZVOL_ASYNC_REMOVE_MINORS, 117 ZVOL_ASYNC_RENAME_MINORS, 118 ZVOL_ASYNC_SET_SNAPDEV, 119 ZVOL_ASYNC_SET_VOLMODE, 120 ZVOL_ASYNC_MAX 121 } zvol_async_op_t; 122 123 typedef struct { 124 zvol_async_op_t zt_op; 125 char zt_name1[MAXNAMELEN]; 126 char zt_name2[MAXNAMELEN]; 127 uint64_t zt_value; 128 uint32_t zt_total; 129 uint32_t zt_done; 130 int32_t zt_status; 131 int zt_error; 132 } zvol_task_t; 133 134 zv_request_task_t * 135 zv_request_task_create(zv_request_t zvr) 136 { 137 zv_request_task_t *task; 138 task = kmem_alloc(sizeof (zv_request_task_t), KM_SLEEP); 139 taskq_init_ent(&task->ent); 140 task->zvr = zvr; 141 return (task); 142 } 143 144 void 145 zv_request_task_free(zv_request_task_t *task) 146 { 147 kmem_free(task, sizeof (*task)); 148 } 149 150 uint64_t 151 zvol_name_hash(const char *name) 152 { 153 uint64_t crc = -1ULL; 154 ASSERT(zfs_crc64_table[128] == ZFS_CRC64_POLY); 155 for (const uint8_t *p = (const uint8_t *)name; *p != 0; p++) 156 crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ (*p)) & 0xFF]; 157 return (crc); 158 } 159 160 /* 161 * Find a zvol_state_t given the name and hash generated by zvol_name_hash. 162 * If found, return with zv_suspend_lock and zv_state_lock taken, otherwise, 163 * return (NULL) without the taking locks. The zv_suspend_lock is always taken 164 * before zv_state_lock. The mode argument indicates the mode (including none) 165 * for zv_suspend_lock to be taken. 166 */ 167 zvol_state_t * 168 zvol_find_by_name_hash(const char *name, uint64_t hash, int mode) 169 { 170 zvol_state_t *zv; 171 struct hlist_node *p = NULL; 172 173 rw_enter(&zvol_state_lock, RW_READER); 174 hlist_for_each(p, ZVOL_HT_HEAD(hash)) { 175 zv = hlist_entry(p, zvol_state_t, zv_hlink); 176 mutex_enter(&zv->zv_state_lock); 177 if (zv->zv_hash == hash && strcmp(zv->zv_name, name) == 0) { 178 /* 179 * this is the right zvol, take the locks in the 180 * right order 181 */ 182 if (mode != RW_NONE && 183 !rw_tryenter(&zv->zv_suspend_lock, mode)) { 184 mutex_exit(&zv->zv_state_lock); 185 rw_enter(&zv->zv_suspend_lock, mode); 186 mutex_enter(&zv->zv_state_lock); 187 /* 188 * zvol cannot be renamed as we continue 189 * to hold zvol_state_lock 190 */ 191 ASSERT(zv->zv_hash == hash && 192 strcmp(zv->zv_name, name) == 0); 193 } 194 rw_exit(&zvol_state_lock); 195 return (zv); 196 } 197 mutex_exit(&zv->zv_state_lock); 198 } 199 rw_exit(&zvol_state_lock); 200 201 return (NULL); 202 } 203 204 /* 205 * Find a zvol_state_t given the name. 206 * If found, return with zv_suspend_lock and zv_state_lock taken, otherwise, 207 * return (NULL) without the taking locks. The zv_suspend_lock is always taken 208 * before zv_state_lock. The mode argument indicates the mode (including none) 209 * for zv_suspend_lock to be taken. 210 */ 211 static zvol_state_t * 212 zvol_find_by_name(const char *name, int mode) 213 { 214 return (zvol_find_by_name_hash(name, zvol_name_hash(name), mode)); 215 } 216 217 /* 218 * ZFS_IOC_CREATE callback handles dmu zvol and zap object creation. 219 */ 220 void 221 zvol_create_cb(objset_t *os, void *arg, cred_t *cr, dmu_tx_t *tx) 222 { 223 zfs_creat_t *zct = arg; 224 nvlist_t *nvprops = zct->zct_props; 225 int error; 226 uint64_t volblocksize, volsize; 227 228 VERIFY0(nvlist_lookup_uint64(nvprops, 229 zfs_prop_to_name(ZFS_PROP_VOLSIZE), &volsize)); 230 if (nvlist_lookup_uint64(nvprops, 231 zfs_prop_to_name(ZFS_PROP_VOLBLOCKSIZE), &volblocksize) != 0) 232 volblocksize = zfs_prop_default_numeric(ZFS_PROP_VOLBLOCKSIZE); 233 234 /* 235 * These properties must be removed from the list so the generic 236 * property setting step won't apply to them. 237 */ 238 VERIFY0(nvlist_remove_all(nvprops, zfs_prop_to_name(ZFS_PROP_VOLSIZE))); 239 (void) nvlist_remove_all(nvprops, 240 zfs_prop_to_name(ZFS_PROP_VOLBLOCKSIZE)); 241 242 error = dmu_object_claim(os, ZVOL_OBJ, DMU_OT_ZVOL, volblocksize, 243 DMU_OT_NONE, 0, tx); 244 ASSERT0(error); 245 246 error = zap_create_claim(os, ZVOL_ZAP_OBJ, DMU_OT_ZVOL_PROP, 247 DMU_OT_NONE, 0, tx); 248 ASSERT0(error); 249 250 error = zap_update(os, ZVOL_ZAP_OBJ, "size", 8, 1, &volsize, tx); 251 ASSERT0(error); 252 } 253 254 /* 255 * ZFS_IOC_OBJSET_STATS entry point. 256 */ 257 int 258 zvol_get_stats(objset_t *os, nvlist_t *nv) 259 { 260 int error; 261 dmu_object_info_t *doi; 262 uint64_t val; 263 264 error = zap_lookup(os, ZVOL_ZAP_OBJ, "size", 8, 1, &val); 265 if (error) 266 return (error); 267 268 dsl_prop_nvlist_add_uint64(nv, ZFS_PROP_VOLSIZE, val); 269 doi = kmem_alloc(sizeof (dmu_object_info_t), KM_SLEEP); 270 error = dmu_object_info(os, ZVOL_OBJ, doi); 271 272 if (error == 0) { 273 dsl_prop_nvlist_add_uint64(nv, ZFS_PROP_VOLBLOCKSIZE, 274 doi->doi_data_block_size); 275 } 276 277 kmem_free(doi, sizeof (dmu_object_info_t)); 278 279 return (error); 280 } 281 282 /* 283 * Sanity check volume size. 284 */ 285 int 286 zvol_check_volsize(uint64_t volsize, uint64_t blocksize) 287 { 288 if (volsize == 0) 289 return (SET_ERROR(EINVAL)); 290 291 if (volsize % blocksize != 0) 292 return (SET_ERROR(EINVAL)); 293 294 #ifdef _ILP32 295 if (volsize - 1 > SPEC_MAXOFFSET_T) 296 return (SET_ERROR(EOVERFLOW)); 297 #endif 298 return (0); 299 } 300 301 /* 302 * Ensure the zap is flushed then inform the VFS of the capacity change. 303 */ 304 static int 305 zvol_update_volsize(uint64_t volsize, objset_t *os) 306 { 307 dmu_tx_t *tx; 308 int error; 309 uint64_t txg; 310 311 tx = dmu_tx_create(os); 312 dmu_tx_hold_zap(tx, ZVOL_ZAP_OBJ, TRUE, NULL); 313 dmu_tx_mark_netfree(tx); 314 error = dmu_tx_assign(tx, DMU_TX_WAIT); 315 if (error) { 316 dmu_tx_abort(tx); 317 return (error); 318 } 319 txg = dmu_tx_get_txg(tx); 320 321 error = zap_update(os, ZVOL_ZAP_OBJ, "size", 8, 1, 322 &volsize, tx); 323 dmu_tx_commit(tx); 324 325 txg_wait_synced(dmu_objset_pool(os), txg); 326 327 if (error == 0) 328 error = dmu_free_long_range(os, 329 ZVOL_OBJ, volsize, DMU_OBJECT_END); 330 331 return (error); 332 } 333 334 /* 335 * Set ZFS_PROP_VOLSIZE set entry point. Note that modifying the volume 336 * size will result in a udev "change" event being generated. 337 */ 338 int 339 zvol_set_volsize(const char *name, uint64_t volsize) 340 { 341 objset_t *os = NULL; 342 uint64_t readonly; 343 int error; 344 boolean_t owned = B_FALSE; 345 346 error = dsl_prop_get_integer(name, 347 zfs_prop_to_name(ZFS_PROP_READONLY), &readonly, NULL); 348 if (error != 0) 349 return (error); 350 if (readonly) 351 return (SET_ERROR(EROFS)); 352 353 zvol_state_t *zv = zvol_find_by_name(name, RW_READER); 354 355 ASSERT(zv == NULL || (MUTEX_HELD(&zv->zv_state_lock) && 356 RW_READ_HELD(&zv->zv_suspend_lock))); 357 358 if (zv == NULL || zv->zv_objset == NULL) { 359 if (zv != NULL) 360 rw_exit(&zv->zv_suspend_lock); 361 if ((error = dmu_objset_own(name, DMU_OST_ZVOL, B_FALSE, B_TRUE, 362 FTAG, &os)) != 0) { 363 if (zv != NULL) 364 mutex_exit(&zv->zv_state_lock); 365 return (error); 366 } 367 owned = B_TRUE; 368 if (zv != NULL) 369 zv->zv_objset = os; 370 } else { 371 os = zv->zv_objset; 372 } 373 374 dmu_object_info_t *doi = kmem_alloc(sizeof (*doi), KM_SLEEP); 375 376 if ((error = dmu_object_info(os, ZVOL_OBJ, doi)) || 377 (error = zvol_check_volsize(volsize, doi->doi_data_block_size))) 378 goto out; 379 380 error = zvol_update_volsize(volsize, os); 381 if (error == 0 && zv != NULL) { 382 zv->zv_volsize = volsize; 383 zv->zv_changed = 1; 384 } 385 out: 386 kmem_free(doi, sizeof (dmu_object_info_t)); 387 388 if (owned) { 389 dmu_objset_disown(os, B_TRUE, FTAG); 390 if (zv != NULL) 391 zv->zv_objset = NULL; 392 } else { 393 rw_exit(&zv->zv_suspend_lock); 394 } 395 396 if (zv != NULL) 397 mutex_exit(&zv->zv_state_lock); 398 399 if (error == 0 && zv != NULL) 400 zvol_os_update_volsize(zv, volsize); 401 402 return (error); 403 } 404 405 /* 406 * Update volthreading. 407 */ 408 int 409 zvol_set_volthreading(const char *name, boolean_t value) 410 { 411 zvol_state_t *zv = zvol_find_by_name(name, RW_NONE); 412 if (zv == NULL) 413 return (SET_ERROR(ENOENT)); 414 zv->zv_threading = value; 415 mutex_exit(&zv->zv_state_lock); 416 return (0); 417 } 418 419 /* 420 * Update zvol ro property. 421 */ 422 int 423 zvol_set_ro(const char *name, boolean_t value) 424 { 425 zvol_state_t *zv = zvol_find_by_name(name, RW_NONE); 426 if (zv == NULL) 427 return (-1); 428 if (value) { 429 zvol_os_set_disk_ro(zv, 1); 430 zv->zv_flags |= ZVOL_RDONLY; 431 } else { 432 zvol_os_set_disk_ro(zv, 0); 433 zv->zv_flags &= ~ZVOL_RDONLY; 434 } 435 mutex_exit(&zv->zv_state_lock); 436 return (0); 437 } 438 439 /* 440 * Sanity check volume block size. 441 */ 442 int 443 zvol_check_volblocksize(const char *name, uint64_t volblocksize) 444 { 445 /* Record sizes above 128k need the feature to be enabled */ 446 if (volblocksize > SPA_OLD_MAXBLOCKSIZE) { 447 spa_t *spa; 448 int error; 449 450 if ((error = spa_open(name, &spa, FTAG)) != 0) 451 return (error); 452 453 if (!spa_feature_is_enabled(spa, SPA_FEATURE_LARGE_BLOCKS)) { 454 spa_close(spa, FTAG); 455 return (SET_ERROR(ENOTSUP)); 456 } 457 458 /* 459 * We don't allow setting the property above 1MB, 460 * unless the tunable has been changed. 461 */ 462 if (volblocksize > zfs_max_recordsize) { 463 spa_close(spa, FTAG); 464 return (SET_ERROR(EDOM)); 465 } 466 467 spa_close(spa, FTAG); 468 } 469 470 if (volblocksize < SPA_MINBLOCKSIZE || 471 volblocksize > SPA_MAXBLOCKSIZE || 472 !ISP2(volblocksize)) 473 return (SET_ERROR(EDOM)); 474 475 return (0); 476 } 477 478 /* 479 * Replay a TX_TRUNCATE ZIL transaction if asked. TX_TRUNCATE is how we 480 * implement DKIOCFREE/free-long-range. 481 */ 482 static int 483 zvol_replay_truncate(void *arg1, void *arg2, boolean_t byteswap) 484 { 485 zvol_state_t *zv = arg1; 486 lr_truncate_t *lr = arg2; 487 uint64_t offset, length; 488 489 ASSERT3U(lr->lr_common.lrc_reclen, >=, sizeof (*lr)); 490 491 if (byteswap) 492 byteswap_uint64_array(lr, sizeof (*lr)); 493 494 offset = lr->lr_offset; 495 length = lr->lr_length; 496 497 dmu_tx_t *tx = dmu_tx_create(zv->zv_objset); 498 dmu_tx_mark_netfree(tx); 499 int error = dmu_tx_assign(tx, DMU_TX_WAIT); 500 if (error != 0) { 501 dmu_tx_abort(tx); 502 } else { 503 (void) zil_replaying(zv->zv_zilog, tx); 504 dmu_tx_commit(tx); 505 error = dmu_free_long_range(zv->zv_objset, ZVOL_OBJ, offset, 506 length); 507 } 508 509 return (error); 510 } 511 512 /* 513 * Replay a TX_WRITE ZIL transaction that didn't get committed 514 * after a system failure 515 */ 516 static int 517 zvol_replay_write(void *arg1, void *arg2, boolean_t byteswap) 518 { 519 zvol_state_t *zv = arg1; 520 lr_write_t *lr = arg2; 521 objset_t *os = zv->zv_objset; 522 char *data = (char *)(lr + 1); /* data follows lr_write_t */ 523 uint64_t offset, length; 524 dmu_tx_t *tx; 525 int error; 526 527 ASSERT3U(lr->lr_common.lrc_reclen, >=, sizeof (*lr)); 528 529 if (byteswap) 530 byteswap_uint64_array(lr, sizeof (*lr)); 531 532 offset = lr->lr_offset; 533 length = lr->lr_length; 534 535 /* If it's a dmu_sync() block, write the whole block */ 536 if (lr->lr_common.lrc_reclen == sizeof (lr_write_t)) { 537 uint64_t blocksize = BP_GET_LSIZE(&lr->lr_blkptr); 538 if (length < blocksize) { 539 offset -= offset % blocksize; 540 length = blocksize; 541 } 542 } 543 544 tx = dmu_tx_create(os); 545 dmu_tx_hold_write(tx, ZVOL_OBJ, offset, length); 546 error = dmu_tx_assign(tx, DMU_TX_WAIT); 547 if (error) { 548 dmu_tx_abort(tx); 549 } else { 550 dmu_write(os, ZVOL_OBJ, offset, length, data, tx); 551 (void) zil_replaying(zv->zv_zilog, tx); 552 dmu_tx_commit(tx); 553 } 554 555 return (error); 556 } 557 558 /* 559 * Replay a TX_CLONE_RANGE ZIL transaction that didn't get committed 560 * after a system failure 561 */ 562 static int 563 zvol_replay_clone_range(void *arg1, void *arg2, boolean_t byteswap) 564 { 565 zvol_state_t *zv = arg1; 566 lr_clone_range_t *lr = arg2; 567 objset_t *os = zv->zv_objset; 568 dmu_tx_t *tx; 569 int error; 570 uint64_t blksz; 571 uint64_t off; 572 uint64_t len; 573 574 ASSERT3U(lr->lr_common.lrc_reclen, >=, sizeof (*lr)); 575 ASSERT3U(lr->lr_common.lrc_reclen, >=, offsetof(lr_clone_range_t, 576 lr_bps[lr->lr_nbps])); 577 578 if (byteswap) 579 byteswap_uint64_array(lr, sizeof (*lr)); 580 581 ASSERT(spa_feature_is_enabled(dmu_objset_spa(os), 582 SPA_FEATURE_BLOCK_CLONING)); 583 584 off = lr->lr_offset; 585 len = lr->lr_length; 586 blksz = lr->lr_blksz; 587 588 if ((off % blksz) != 0) { 589 return (SET_ERROR(EINVAL)); 590 } 591 592 error = dnode_hold(os, ZVOL_OBJ, zv, &zv->zv_dn); 593 if (error != 0 || !zv->zv_dn) 594 return (error); 595 tx = dmu_tx_create(os); 596 dmu_tx_hold_clone_by_dnode(tx, zv->zv_dn, off, len, blksz); 597 error = dmu_tx_assign(tx, DMU_TX_WAIT); 598 if (error != 0) { 599 dmu_tx_abort(tx); 600 goto out; 601 } 602 error = dmu_brt_clone(zv->zv_objset, ZVOL_OBJ, off, len, 603 tx, lr->lr_bps, lr->lr_nbps); 604 if (error != 0) { 605 dmu_tx_commit(tx); 606 goto out; 607 } 608 609 /* 610 * zil_replaying() not only check if we are replaying ZIL, but also 611 * updates the ZIL header to record replay progress. 612 */ 613 VERIFY(zil_replaying(zv->zv_zilog, tx)); 614 dmu_tx_commit(tx); 615 616 out: 617 dnode_rele(zv->zv_dn, zv); 618 zv->zv_dn = NULL; 619 return (error); 620 } 621 622 int 623 zvol_clone_range(zvol_state_t *zv_src, uint64_t inoff, zvol_state_t *zv_dst, 624 uint64_t outoff, uint64_t len) 625 { 626 zilog_t *zilog_dst; 627 zfs_locked_range_t *inlr, *outlr; 628 objset_t *inos, *outos; 629 dmu_tx_t *tx; 630 blkptr_t *bps; 631 size_t maxblocks; 632 int error = 0; 633 634 rw_enter(&zv_dst->zv_suspend_lock, RW_READER); 635 if (zv_dst->zv_zilog == NULL) { 636 rw_exit(&zv_dst->zv_suspend_lock); 637 rw_enter(&zv_dst->zv_suspend_lock, RW_WRITER); 638 if (zv_dst->zv_zilog == NULL) { 639 zv_dst->zv_zilog = zil_open(zv_dst->zv_objset, 640 zvol_get_data, &zv_dst->zv_kstat.dk_zil_sums); 641 zv_dst->zv_flags |= ZVOL_WRITTEN_TO; 642 VERIFY0((zv_dst->zv_zilog->zl_header->zh_flags & 643 ZIL_REPLAY_NEEDED)); 644 } 645 rw_downgrade(&zv_dst->zv_suspend_lock); 646 } 647 if (zv_src != zv_dst) 648 rw_enter(&zv_src->zv_suspend_lock, RW_READER); 649 650 inos = zv_src->zv_objset; 651 outos = zv_dst->zv_objset; 652 653 /* 654 * Sanity checks 655 */ 656 if (!spa_feature_is_enabled(dmu_objset_spa(outos), 657 SPA_FEATURE_BLOCK_CLONING)) { 658 error = SET_ERROR(EOPNOTSUPP); 659 goto out; 660 } 661 if (dmu_objset_spa(inos) != dmu_objset_spa(outos)) { 662 error = SET_ERROR(EXDEV); 663 goto out; 664 } 665 if (inos->os_encrypted != outos->os_encrypted) { 666 error = SET_ERROR(EXDEV); 667 goto out; 668 } 669 if (zv_src->zv_volblocksize != zv_dst->zv_volblocksize) { 670 error = SET_ERROR(EINVAL); 671 goto out; 672 } 673 if (inoff >= zv_src->zv_volsize || outoff >= zv_dst->zv_volsize) { 674 goto out; 675 } 676 677 /* 678 * Do not read beyond boundary 679 */ 680 if (len > zv_src->zv_volsize - inoff) 681 len = zv_src->zv_volsize - inoff; 682 if (len > zv_dst->zv_volsize - outoff) 683 len = zv_dst->zv_volsize - outoff; 684 if (len == 0) 685 goto out; 686 687 /* 688 * No overlapping if we are cloning within the same file 689 */ 690 if (zv_src == zv_dst) { 691 if (inoff < outoff + len && outoff < inoff + len) { 692 error = SET_ERROR(EINVAL); 693 goto out; 694 } 695 } 696 697 /* 698 * Offsets and length must be at block boundaries 699 */ 700 if ((inoff % zv_src->zv_volblocksize) != 0 || 701 (outoff % zv_dst->zv_volblocksize) != 0) { 702 error = SET_ERROR(EINVAL); 703 goto out; 704 } 705 706 /* 707 * Length must be multiple of block size 708 */ 709 if ((len % zv_src->zv_volblocksize) != 0) { 710 error = SET_ERROR(EINVAL); 711 goto out; 712 } 713 714 zilog_dst = zv_dst->zv_zilog; 715 maxblocks = zil_max_log_data(zilog_dst, sizeof (lr_clone_range_t)) / 716 sizeof (bps[0]); 717 bps = vmem_alloc(sizeof (bps[0]) * maxblocks, KM_SLEEP); 718 /* 719 * Maintain predictable lock order. 720 */ 721 if (zv_src < zv_dst || (zv_src == zv_dst && inoff < outoff)) { 722 inlr = zfs_rangelock_enter(&zv_src->zv_rangelock, inoff, len, 723 RL_READER); 724 outlr = zfs_rangelock_enter(&zv_dst->zv_rangelock, outoff, len, 725 RL_WRITER); 726 } else { 727 outlr = zfs_rangelock_enter(&zv_dst->zv_rangelock, outoff, len, 728 RL_WRITER); 729 inlr = zfs_rangelock_enter(&zv_src->zv_rangelock, inoff, len, 730 RL_READER); 731 } 732 733 while (len > 0) { 734 uint64_t size, last_synced_txg; 735 size_t nbps = maxblocks; 736 size = MIN(zv_src->zv_volblocksize * maxblocks, len); 737 last_synced_txg = spa_last_synced_txg( 738 dmu_objset_spa(zv_src->zv_objset)); 739 error = dmu_read_l0_bps(zv_src->zv_objset, ZVOL_OBJ, inoff, 740 size, bps, &nbps); 741 if (error != 0) { 742 /* 743 * If we are trying to clone a block that was created 744 * in the current transaction group, the error will be 745 * EAGAIN here. Based on zfs_bclone_wait_dirty either 746 * return a shortened range to the caller so it can 747 * fallback, or wait for the next TXG and check again. 748 */ 749 if (error == EAGAIN && zfs_bclone_wait_dirty) { 750 txg_wait_synced(dmu_objset_pool 751 (zv_src->zv_objset), last_synced_txg + 1); 752 continue; 753 } 754 break; 755 } 756 757 tx = dmu_tx_create(zv_dst->zv_objset); 758 dmu_tx_hold_clone_by_dnode(tx, zv_dst->zv_dn, outoff, size, 759 zv_src->zv_volblocksize); 760 error = dmu_tx_assign(tx, DMU_TX_WAIT); 761 if (error != 0) { 762 dmu_tx_abort(tx); 763 break; 764 } 765 error = dmu_brt_clone(zv_dst->zv_objset, ZVOL_OBJ, outoff, size, 766 tx, bps, nbps); 767 if (error != 0) { 768 dmu_tx_commit(tx); 769 break; 770 } 771 zvol_log_clone_range(zilog_dst, tx, TX_CLONE_RANGE, outoff, 772 size, zv_src->zv_volblocksize, bps, nbps); 773 dmu_tx_commit(tx); 774 inoff += size; 775 outoff += size; 776 len -= size; 777 } 778 vmem_free(bps, sizeof (bps[0]) * maxblocks); 779 zfs_rangelock_exit(outlr); 780 zfs_rangelock_exit(inlr); 781 if (error == 0 && zv_dst->zv_objset->os_sync == ZFS_SYNC_ALWAYS) { 782 error = zil_commit(zilog_dst, ZVOL_OBJ); 783 } 784 out: 785 if (zv_src != zv_dst) 786 rw_exit(&zv_src->zv_suspend_lock); 787 rw_exit(&zv_dst->zv_suspend_lock); 788 return (error); 789 } 790 791 /* 792 * Handles TX_CLONE_RANGE transactions. 793 */ 794 void 795 zvol_log_clone_range(zilog_t *zilog, dmu_tx_t *tx, int txtype, uint64_t off, 796 uint64_t len, uint64_t blksz, const blkptr_t *bps, size_t nbps) 797 { 798 itx_t *itx; 799 lr_clone_range_t *lr; 800 uint64_t partlen, max_log_data; 801 size_t partnbps; 802 803 if (zil_replaying(zilog, tx)) 804 return; 805 806 max_log_data = zil_max_log_data(zilog, sizeof (lr_clone_range_t)); 807 808 while (nbps > 0) { 809 partnbps = MIN(nbps, max_log_data / sizeof (bps[0])); 810 partlen = partnbps * blksz; 811 ASSERT3U(partlen, <, len + blksz); 812 partlen = MIN(partlen, len); 813 814 itx = zil_itx_create(txtype, 815 sizeof (*lr) + sizeof (bps[0]) * partnbps); 816 lr = (lr_clone_range_t *)&itx->itx_lr; 817 lr->lr_foid = ZVOL_OBJ; 818 lr->lr_offset = off; 819 lr->lr_length = partlen; 820 lr->lr_blksz = blksz; 821 lr->lr_nbps = partnbps; 822 memcpy(lr->lr_bps, bps, sizeof (bps[0]) * partnbps); 823 824 zil_itx_assign(zilog, itx, tx); 825 826 bps += partnbps; 827 ASSERT3U(nbps, >=, partnbps); 828 nbps -= partnbps; 829 off += partlen; 830 ASSERT3U(len, >=, partlen); 831 len -= partlen; 832 } 833 } 834 835 static int 836 zvol_replay_err(void *arg1, void *arg2, boolean_t byteswap) 837 { 838 (void) arg1, (void) arg2, (void) byteswap; 839 return (SET_ERROR(ENOTSUP)); 840 } 841 842 /* 843 * Callback vectors for replaying records. 844 * Only TX_WRITE and TX_TRUNCATE are needed for zvol. 845 */ 846 zil_replay_func_t *const zvol_replay_vector[TX_MAX_TYPE] = { 847 zvol_replay_err, /* no such transaction type */ 848 zvol_replay_err, /* TX_CREATE */ 849 zvol_replay_err, /* TX_MKDIR */ 850 zvol_replay_err, /* TX_MKXATTR */ 851 zvol_replay_err, /* TX_SYMLINK */ 852 zvol_replay_err, /* TX_REMOVE */ 853 zvol_replay_err, /* TX_RMDIR */ 854 zvol_replay_err, /* TX_LINK */ 855 zvol_replay_err, /* TX_RENAME */ 856 zvol_replay_write, /* TX_WRITE */ 857 zvol_replay_truncate, /* TX_TRUNCATE */ 858 zvol_replay_err, /* TX_SETATTR */ 859 zvol_replay_err, /* TX_ACL_V0 */ 860 zvol_replay_err, /* TX_ACL */ 861 zvol_replay_err, /* TX_CREATE_ACL */ 862 zvol_replay_err, /* TX_CREATE_ATTR */ 863 zvol_replay_err, /* TX_CREATE_ACL_ATTR */ 864 zvol_replay_err, /* TX_MKDIR_ACL */ 865 zvol_replay_err, /* TX_MKDIR_ATTR */ 866 zvol_replay_err, /* TX_MKDIR_ACL_ATTR */ 867 zvol_replay_err, /* TX_WRITE2 */ 868 zvol_replay_err, /* TX_SETSAXATTR */ 869 zvol_replay_err, /* TX_RENAME_EXCHANGE */ 870 zvol_replay_err, /* TX_RENAME_WHITEOUT */ 871 zvol_replay_clone_range, /* TX_CLONE_RANGE */ 872 }; 873 874 /* 875 * zvol_log_write() handles TX_WRITE transactions. 876 */ 877 void 878 zvol_log_write(zvol_state_t *zv, dmu_tx_t *tx, uint64_t offset, 879 uint64_t size, boolean_t commit) 880 { 881 uint32_t blocksize = zv->zv_volblocksize; 882 zilog_t *zilog = zv->zv_zilog; 883 itx_wr_state_t write_state; 884 uint64_t log_size = 0; 885 886 if (zil_replaying(zilog, tx)) 887 return; 888 889 write_state = zil_write_state(zilog, size, blocksize, B_FALSE, commit); 890 891 while (size) { 892 itx_t *itx; 893 lr_write_t *lr; 894 itx_wr_state_t wr_state = write_state; 895 ssize_t len = size; 896 897 if (wr_state == WR_COPIED && size > zil_max_copied_data(zilog)) 898 wr_state = WR_NEED_COPY; 899 else if (wr_state == WR_INDIRECT) 900 len = MIN(blocksize - P2PHASE(offset, blocksize), size); 901 902 itx = zil_itx_create(TX_WRITE, sizeof (*lr) + 903 (wr_state == WR_COPIED ? len : 0)); 904 lr = (lr_write_t *)&itx->itx_lr; 905 if (wr_state == WR_COPIED && 906 dmu_read_by_dnode(zv->zv_dn, offset, len, lr + 1, 907 DMU_READ_NO_PREFETCH | DMU_KEEP_CACHING) != 0) { 908 zil_itx_destroy(itx, 0); 909 itx = zil_itx_create(TX_WRITE, sizeof (*lr)); 910 lr = (lr_write_t *)&itx->itx_lr; 911 wr_state = WR_NEED_COPY; 912 } 913 914 log_size += itx->itx_size; 915 if (wr_state == WR_NEED_COPY) 916 log_size += len; 917 918 itx->itx_wr_state = wr_state; 919 lr->lr_foid = ZVOL_OBJ; 920 lr->lr_offset = offset; 921 lr->lr_length = len; 922 lr->lr_blkoff = 0; 923 BP_ZERO(&lr->lr_blkptr); 924 925 itx->itx_private = zv; 926 927 zil_itx_assign(zilog, itx, tx); 928 929 offset += len; 930 size -= len; 931 } 932 933 dsl_pool_wrlog_count(zilog->zl_dmu_pool, log_size, tx->tx_txg); 934 } 935 936 /* 937 * Log a DKIOCFREE/free-long-range to the ZIL with TX_TRUNCATE. 938 */ 939 void 940 zvol_log_truncate(zvol_state_t *zv, dmu_tx_t *tx, uint64_t off, uint64_t len) 941 { 942 itx_t *itx; 943 lr_truncate_t *lr; 944 zilog_t *zilog = zv->zv_zilog; 945 946 if (zil_replaying(zilog, tx)) 947 return; 948 949 itx = zil_itx_create(TX_TRUNCATE, sizeof (*lr)); 950 lr = (lr_truncate_t *)&itx->itx_lr; 951 lr->lr_foid = ZVOL_OBJ; 952 lr->lr_offset = off; 953 lr->lr_length = len; 954 955 zil_itx_assign(zilog, itx, tx); 956 } 957 958 959 static void 960 zvol_get_done(zgd_t *zgd, int error) 961 { 962 (void) error; 963 if (zgd->zgd_db) 964 dmu_buf_rele(zgd->zgd_db, zgd); 965 966 zfs_rangelock_exit(zgd->zgd_lr); 967 968 kmem_free(zgd, sizeof (zgd_t)); 969 } 970 971 /* 972 * Get data to generate a TX_WRITE intent log record. 973 */ 974 int 975 zvol_get_data(void *arg, uint64_t arg2, lr_write_t *lr, char *buf, 976 struct lwb *lwb, zio_t *zio) 977 { 978 zvol_state_t *zv = arg; 979 uint64_t offset = lr->lr_offset; 980 uint64_t size = lr->lr_length; 981 dmu_buf_t *db; 982 zgd_t *zgd; 983 int error; 984 985 ASSERT3P(lwb, !=, NULL); 986 ASSERT3U(size, !=, 0); 987 988 zgd = kmem_zalloc(sizeof (zgd_t), KM_SLEEP); 989 zgd->zgd_lwb = lwb; 990 991 /* 992 * Write records come in two flavors: immediate and indirect. 993 * For small writes it's cheaper to store the data with the 994 * log record (immediate); for large writes it's cheaper to 995 * sync the data and get a pointer to it (indirect) so that 996 * we don't have to write the data twice. 997 */ 998 if (buf != NULL) { /* immediate write */ 999 zgd->zgd_lr = zfs_rangelock_enter(&zv->zv_rangelock, offset, 1000 size, RL_READER); 1001 error = dmu_read_by_dnode(zv->zv_dn, offset, size, buf, 1002 DMU_READ_NO_PREFETCH | DMU_KEEP_CACHING); 1003 } else { /* indirect write */ 1004 ASSERT3P(zio, !=, NULL); 1005 /* 1006 * Have to lock the whole block to ensure when it's written out 1007 * and its checksum is being calculated that no one can change 1008 * the data. Contrarily to zfs_get_data we need not re-check 1009 * blocksize after we get the lock because it cannot be changed. 1010 */ 1011 size = zv->zv_volblocksize; 1012 offset = P2ALIGN_TYPED(offset, size, uint64_t); 1013 zgd->zgd_lr = zfs_rangelock_enter(&zv->zv_rangelock, offset, 1014 size, RL_READER); 1015 error = dmu_buf_hold_noread_by_dnode(zv->zv_dn, offset, zgd, 1016 &db); 1017 if (error == 0) { 1018 blkptr_t *bp = &lr->lr_blkptr; 1019 1020 zgd->zgd_db = db; 1021 zgd->zgd_bp = bp; 1022 1023 ASSERT(db != NULL); 1024 ASSERT(db->db_offset == offset); 1025 ASSERT(db->db_size == size); 1026 1027 error = dmu_sync(zio, lr->lr_common.lrc_txg, 1028 zvol_get_done, zgd); 1029 1030 if (error == 0) 1031 return (0); 1032 } 1033 } 1034 1035 zvol_get_done(zgd, error); 1036 1037 return (error); 1038 } 1039 1040 /* 1041 * The zvol_state_t's are inserted into zvol_state_list and zvol_htable. 1042 */ 1043 1044 void 1045 zvol_insert(zvol_state_t *zv) 1046 { 1047 ASSERT(RW_WRITE_HELD(&zvol_state_lock)); 1048 list_insert_head(&zvol_state_list, zv); 1049 hlist_add_head(&zv->zv_hlink, ZVOL_HT_HEAD(zv->zv_hash)); 1050 } 1051 1052 /* 1053 * Simply remove the zvol from to list of zvols. 1054 */ 1055 static void 1056 zvol_remove(zvol_state_t *zv) 1057 { 1058 ASSERT(RW_WRITE_HELD(&zvol_state_lock)); 1059 list_remove(&zvol_state_list, zv); 1060 hlist_del(&zv->zv_hlink); 1061 } 1062 1063 /* 1064 * Setup zv after we just own the zv->objset 1065 */ 1066 static int 1067 zvol_setup_zv(zvol_state_t *zv) 1068 { 1069 uint64_t volsize; 1070 int error; 1071 uint64_t ro; 1072 objset_t *os = zv->zv_objset; 1073 1074 ASSERT(MUTEX_HELD(&zv->zv_state_lock)); 1075 ASSERT(RW_LOCK_HELD(&zv->zv_suspend_lock)); 1076 1077 zv->zv_zilog = NULL; 1078 zv->zv_flags &= ~ZVOL_WRITTEN_TO; 1079 1080 error = dsl_prop_get_integer(zv->zv_name, "readonly", &ro, NULL); 1081 if (error) 1082 return (error); 1083 1084 error = zap_lookup(os, ZVOL_ZAP_OBJ, "size", 8, 1, &volsize); 1085 if (error) 1086 return (error); 1087 1088 error = dnode_hold(os, ZVOL_OBJ, zv, &zv->zv_dn); 1089 if (error) 1090 return (error); 1091 1092 zvol_os_set_capacity(zv, volsize >> 9); 1093 zv->zv_volsize = volsize; 1094 1095 if (ro || dmu_objset_is_snapshot(os) || 1096 !spa_writeable(dmu_objset_spa(os))) { 1097 zvol_os_set_disk_ro(zv, 1); 1098 zv->zv_flags |= ZVOL_RDONLY; 1099 } else { 1100 zvol_os_set_disk_ro(zv, 0); 1101 zv->zv_flags &= ~ZVOL_RDONLY; 1102 } 1103 return (0); 1104 } 1105 1106 /* 1107 * Shutdown every zv_objset related stuff except zv_objset itself. 1108 * The is the reverse of zvol_setup_zv. 1109 */ 1110 static void 1111 zvol_shutdown_zv(zvol_state_t *zv) 1112 { 1113 ASSERT(MUTEX_HELD(&zv->zv_state_lock) && 1114 RW_LOCK_HELD(&zv->zv_suspend_lock)); 1115 1116 if (zv->zv_flags & ZVOL_WRITTEN_TO) { 1117 ASSERT(zv->zv_zilog != NULL); 1118 zil_close(zv->zv_zilog); 1119 } 1120 1121 zv->zv_zilog = NULL; 1122 1123 dnode_rele(zv->zv_dn, zv); 1124 zv->zv_dn = NULL; 1125 1126 /* 1127 * Evict cached data. We must write out any dirty data before 1128 * disowning the dataset. 1129 */ 1130 if (zv->zv_flags & ZVOL_WRITTEN_TO) 1131 txg_wait_synced(dmu_objset_pool(zv->zv_objset), 0); 1132 dmu_objset_evict_dbufs(zv->zv_objset); 1133 } 1134 1135 /* 1136 * return the proper tag for rollback and recv 1137 */ 1138 void * 1139 zvol_tag(zvol_state_t *zv) 1140 { 1141 ASSERT(RW_WRITE_HELD(&zv->zv_suspend_lock)); 1142 return (zv->zv_open_count > 0 ? zv : NULL); 1143 } 1144 1145 /* 1146 * Suspend the zvol for recv and rollback. 1147 */ 1148 int 1149 zvol_suspend(const char *name, zvol_state_t **zvp) 1150 { 1151 zvol_state_t *zv; 1152 1153 zv = zvol_find_by_name(name, RW_WRITER); 1154 1155 if (zv == NULL) 1156 return (SET_ERROR(ENOENT)); 1157 1158 /* block all I/O, release in zvol_resume. */ 1159 ASSERT(MUTEX_HELD(&zv->zv_state_lock)); 1160 ASSERT(RW_WRITE_HELD(&zv->zv_suspend_lock)); 1161 1162 /* 1163 * If it's being removed, unlock and return error. It doesn't make any 1164 * sense to try to suspend a zvol being removed, but being here also 1165 * means that zvol_remove_minors_impl() is about to call zvol_remove() 1166 * and then destroy the zvol_state_t, so returning a pointer to it for 1167 * the caller to mess with would be a disaster anyway. 1168 */ 1169 if (zv->zv_flags & ZVOL_REMOVING) { 1170 mutex_exit(&zv->zv_state_lock); 1171 rw_exit(&zv->zv_suspend_lock); 1172 /* NB: Returning EIO here to match zfsvfs_teardown() */ 1173 return (SET_ERROR(EIO)); 1174 } 1175 1176 atomic_inc(&zv->zv_suspend_ref); 1177 1178 if (zv->zv_open_count > 0) 1179 zvol_shutdown_zv(zv); 1180 1181 /* 1182 * do not hold zv_state_lock across suspend/resume to 1183 * avoid locking up zvol lookups 1184 */ 1185 mutex_exit(&zv->zv_state_lock); 1186 1187 /* zv_suspend_lock is released in zvol_resume() */ 1188 *zvp = zv; 1189 return (0); 1190 } 1191 1192 int 1193 zvol_resume(zvol_state_t *zv) 1194 { 1195 int error = 0; 1196 1197 ASSERT(RW_WRITE_HELD(&zv->zv_suspend_lock)); 1198 1199 mutex_enter(&zv->zv_state_lock); 1200 1201 if (zv->zv_open_count > 0) { 1202 VERIFY0(dmu_objset_hold(zv->zv_name, zv, &zv->zv_objset)); 1203 VERIFY3P(zv->zv_objset->os_dsl_dataset->ds_owner, ==, zv); 1204 VERIFY(dsl_dataset_long_held(zv->zv_objset->os_dsl_dataset)); 1205 dmu_objset_rele(zv->zv_objset, zv); 1206 1207 error = zvol_setup_zv(zv); 1208 } 1209 1210 mutex_exit(&zv->zv_state_lock); 1211 1212 rw_exit(&zv->zv_suspend_lock); 1213 /* 1214 * We need this because we don't hold zvol_state_lock while releasing 1215 * zv_suspend_lock. zvol_remove_minors_impl thus cannot check 1216 * zv_suspend_lock to determine it is safe to free because rwlock is 1217 * not inherent atomic. 1218 */ 1219 atomic_dec(&zv->zv_suspend_ref); 1220 1221 if (zv->zv_flags & ZVOL_REMOVING) 1222 cv_broadcast(&zv->zv_removing_cv); 1223 1224 return (error); 1225 } 1226 1227 int 1228 zvol_first_open(zvol_state_t *zv, boolean_t readonly) 1229 { 1230 objset_t *os; 1231 int error; 1232 1233 ASSERT(RW_READ_HELD(&zv->zv_suspend_lock)); 1234 ASSERT(MUTEX_HELD(&zv->zv_state_lock)); 1235 ASSERT(mutex_owned(&spa_namespace_lock)); 1236 1237 boolean_t ro = (readonly || (strchr(zv->zv_name, '@') != NULL)); 1238 error = dmu_objset_own(zv->zv_name, DMU_OST_ZVOL, ro, B_TRUE, zv, &os); 1239 if (error) 1240 return (error); 1241 1242 zv->zv_objset = os; 1243 1244 error = zvol_setup_zv(zv); 1245 if (error) { 1246 dmu_objset_disown(os, 1, zv); 1247 zv->zv_objset = NULL; 1248 } 1249 1250 return (error); 1251 } 1252 1253 void 1254 zvol_last_close(zvol_state_t *zv) 1255 { 1256 ASSERT(RW_READ_HELD(&zv->zv_suspend_lock)); 1257 ASSERT(MUTEX_HELD(&zv->zv_state_lock)); 1258 1259 if (zv->zv_flags & ZVOL_REMOVING) 1260 cv_broadcast(&zv->zv_removing_cv); 1261 1262 zvol_shutdown_zv(zv); 1263 1264 dmu_objset_disown(zv->zv_objset, 1, zv); 1265 zv->zv_objset = NULL; 1266 } 1267 1268 typedef struct minors_job { 1269 list_t *list; 1270 list_node_t link; 1271 /* input */ 1272 char *name; 1273 /* output */ 1274 int error; 1275 } minors_job_t; 1276 1277 /* 1278 * Prefetch zvol dnodes for the minors_job 1279 */ 1280 static void 1281 zvol_prefetch_minors_impl(void *arg) 1282 { 1283 minors_job_t *job = arg; 1284 char *dsname = job->name; 1285 objset_t *os = NULL; 1286 1287 job->error = dmu_objset_own(dsname, DMU_OST_ZVOL, B_TRUE, B_TRUE, 1288 FTAG, &os); 1289 if (job->error == 0) { 1290 dmu_prefetch_dnode(os, ZVOL_OBJ, ZIO_PRIORITY_SYNC_READ); 1291 dmu_objset_disown(os, B_TRUE, FTAG); 1292 } 1293 } 1294 1295 /* 1296 * Mask errors to continue dmu_objset_find() traversal 1297 */ 1298 static int 1299 zvol_create_snap_minor_cb(const char *dsname, void *arg) 1300 { 1301 minors_job_t *j = arg; 1302 list_t *minors_list = j->list; 1303 const char *name = j->name; 1304 1305 ASSERT0(MUTEX_HELD(&spa_namespace_lock)); 1306 1307 /* skip the designated dataset */ 1308 if (name && strcmp(dsname, name) == 0) 1309 return (0); 1310 1311 /* at this point, the dsname should name a snapshot */ 1312 if (strchr(dsname, '@') == 0) { 1313 dprintf("zvol_create_snap_minor_cb(): " 1314 "%s is not a snapshot name\n", dsname); 1315 } else { 1316 minors_job_t *job; 1317 char *n = kmem_strdup(dsname); 1318 if (n == NULL) 1319 return (0); 1320 1321 job = kmem_alloc(sizeof (minors_job_t), KM_SLEEP); 1322 job->name = n; 1323 job->list = minors_list; 1324 job->error = 0; 1325 list_insert_tail(minors_list, job); 1326 /* don't care if dispatch fails, because job->error is 0 */ 1327 taskq_dispatch(system_taskq, zvol_prefetch_minors_impl, job, 1328 TQ_SLEEP); 1329 } 1330 1331 return (0); 1332 } 1333 1334 /* 1335 * If spa_keystore_load_wkey() is called for an encrypted zvol, 1336 * we need to look for any clones also using the key. This function 1337 * is "best effort" - so we just skip over it if there are failures. 1338 */ 1339 static void 1340 zvol_add_clones(const char *dsname, list_t *minors_list) 1341 { 1342 /* Also check if it has clones */ 1343 dsl_dir_t *dd = NULL; 1344 dsl_pool_t *dp = NULL; 1345 1346 if (dsl_pool_hold(dsname, FTAG, &dp) != 0) 1347 return; 1348 1349 if (!spa_feature_is_enabled(dp->dp_spa, 1350 SPA_FEATURE_ENCRYPTION)) 1351 goto out; 1352 1353 if (dsl_dir_hold(dp, dsname, FTAG, &dd, NULL) != 0) 1354 goto out; 1355 1356 if (dsl_dir_phys(dd)->dd_clones == 0) 1357 goto out; 1358 1359 zap_cursor_t *zc = kmem_alloc(sizeof (zap_cursor_t), KM_SLEEP); 1360 zap_attribute_t *za = zap_attribute_alloc(); 1361 objset_t *mos = dd->dd_pool->dp_meta_objset; 1362 1363 for (zap_cursor_init(zc, mos, dsl_dir_phys(dd)->dd_clones); 1364 zap_cursor_retrieve(zc, za) == 0; 1365 zap_cursor_advance(zc)) { 1366 dsl_dataset_t *clone; 1367 minors_job_t *job; 1368 1369 if (dsl_dataset_hold_obj(dd->dd_pool, 1370 za->za_first_integer, FTAG, &clone) == 0) { 1371 1372 char name[ZFS_MAX_DATASET_NAME_LEN]; 1373 dsl_dataset_name(clone, name); 1374 1375 char *n = kmem_strdup(name); 1376 job = kmem_alloc(sizeof (minors_job_t), KM_SLEEP); 1377 job->name = n; 1378 job->list = minors_list; 1379 job->error = 0; 1380 list_insert_tail(minors_list, job); 1381 1382 dsl_dataset_rele(clone, FTAG); 1383 } 1384 } 1385 zap_cursor_fini(zc); 1386 zap_attribute_free(za); 1387 kmem_free(zc, sizeof (zap_cursor_t)); 1388 1389 out: 1390 if (dd != NULL) 1391 dsl_dir_rele(dd, FTAG); 1392 dsl_pool_rele(dp, FTAG); 1393 } 1394 1395 /* 1396 * Mask errors to continue dmu_objset_find() traversal 1397 */ 1398 static int 1399 zvol_create_minors_cb(const char *dsname, void *arg) 1400 { 1401 uint64_t snapdev; 1402 int error; 1403 list_t *minors_list = arg; 1404 1405 ASSERT0(MUTEX_HELD(&spa_namespace_lock)); 1406 1407 error = dsl_prop_get_integer(dsname, "snapdev", &snapdev, NULL); 1408 if (error) 1409 return (0); 1410 1411 /* 1412 * Given the name and the 'snapdev' property, create device minor nodes 1413 * with the linkages to zvols/snapshots as needed. 1414 * If the name represents a zvol, create a minor node for the zvol, then 1415 * check if its snapshots are 'visible', and if so, iterate over the 1416 * snapshots and create device minor nodes for those. 1417 */ 1418 if (strchr(dsname, '@') == 0) { 1419 minors_job_t *job; 1420 char *n = kmem_strdup(dsname); 1421 if (n == NULL) 1422 return (0); 1423 1424 job = kmem_alloc(sizeof (minors_job_t), KM_SLEEP); 1425 job->name = n; 1426 job->list = minors_list; 1427 job->error = 0; 1428 list_insert_tail(minors_list, job); 1429 /* don't care if dispatch fails, because job->error is 0 */ 1430 taskq_dispatch(system_taskq, zvol_prefetch_minors_impl, job, 1431 TQ_SLEEP); 1432 1433 zvol_add_clones(dsname, minors_list); 1434 1435 if (snapdev == ZFS_SNAPDEV_VISIBLE) { 1436 /* 1437 * traverse snapshots only, do not traverse children, 1438 * and skip the 'dsname' 1439 */ 1440 (void) dmu_objset_find(dsname, 1441 zvol_create_snap_minor_cb, (void *)job, 1442 DS_FIND_SNAPSHOTS); 1443 } 1444 } else { 1445 dprintf("zvol_create_minors_cb(): %s is not a zvol name\n", 1446 dsname); 1447 } 1448 1449 return (0); 1450 } 1451 1452 static void 1453 zvol_task_update_status(zvol_task_t *task, uint64_t total, uint64_t done, 1454 int error) 1455 { 1456 1457 task->zt_total += total; 1458 task->zt_done += done; 1459 if (task->zt_total != task->zt_done) { 1460 task->zt_status = -1; 1461 if (error) 1462 task->zt_error = error; 1463 } 1464 } 1465 1466 static void 1467 zvol_task_report_status(zvol_task_t *task) 1468 { 1469 #ifdef ZFS_DEBUG 1470 static const char *const msg[] = { 1471 "create", 1472 "remove", 1473 "rename", 1474 "set snapdev", 1475 "set volmode", 1476 "unknown", 1477 }; 1478 1479 if (task->zt_status == 0) 1480 return; 1481 1482 zvol_async_op_t op = MIN(task->zt_op, ZVOL_ASYNC_MAX); 1483 if (task->zt_error) { 1484 dprintf("The %s minors zvol task was not ok, last error %d\n", 1485 msg[op], task->zt_error); 1486 } else { 1487 dprintf("The %s minors zvol task was not ok\n", msg[op]); 1488 } 1489 #else 1490 (void) task; 1491 #endif 1492 } 1493 1494 /* 1495 * Create minors for the specified dataset, including children and snapshots. 1496 * Pay attention to the 'snapdev' property and iterate over the snapshots 1497 * only if they are 'visible'. This approach allows one to assure that the 1498 * snapshot metadata is read from disk only if it is needed. 1499 * 1500 * The name can represent a dataset to be recursively scanned for zvols and 1501 * their snapshots, or a single zvol snapshot. If the name represents a 1502 * dataset, the scan is performed in two nested stages: 1503 * - scan the dataset for zvols, and 1504 * - for each zvol, create a minor node, then check if the zvol's snapshots 1505 * are 'visible', and only then iterate over the snapshots if needed 1506 * 1507 * If the name represents a snapshot, a check is performed if the snapshot is 1508 * 'visible' (which also verifies that the parent is a zvol), and if so, 1509 * a minor node for that snapshot is created. 1510 */ 1511 static void 1512 zvol_create_minors_impl(zvol_task_t *task) 1513 { 1514 const char *name = task->zt_name1; 1515 list_t minors_list; 1516 minors_job_t *job; 1517 uint64_t snapdev; 1518 int total = 0, done = 0, last_error, error; 1519 1520 /* 1521 * Note: the dsl_pool_config_lock must not be held. 1522 * Minor node creation needs to obtain the zvol_state_lock. 1523 * zvol_open() obtains the zvol_state_lock and then the dsl pool 1524 * config lock. Therefore, we can't have the config lock now if 1525 * we are going to wait for the zvol_state_lock, because it 1526 * would be a lock order inversion which could lead to deadlock. 1527 */ 1528 1529 if (zvol_inhibit_dev) { 1530 return; 1531 } 1532 1533 /* 1534 * This is the list for prefetch jobs. Whenever we found a match 1535 * during dmu_objset_find, we insert a minors_job to the list and do 1536 * taskq_dispatch to parallel prefetch zvol dnodes. Note we don't need 1537 * any lock because all list operation is done on the current thread. 1538 * 1539 * We will use this list to do zvol_os_create_minor after prefetch 1540 * so we don't have to traverse using dmu_objset_find again. 1541 */ 1542 list_create(&minors_list, sizeof (minors_job_t), 1543 offsetof(minors_job_t, link)); 1544 1545 1546 if (strchr(name, '@') != NULL) { 1547 error = dsl_prop_get_integer(name, "snapdev", &snapdev, NULL); 1548 if (error == 0 && snapdev == ZFS_SNAPDEV_VISIBLE) { 1549 error = zvol_os_create_minor(name); 1550 if (error == 0) { 1551 done++; 1552 } else { 1553 last_error = error; 1554 } 1555 total++; 1556 } 1557 } else { 1558 fstrans_cookie_t cookie = spl_fstrans_mark(); 1559 (void) dmu_objset_find(name, zvol_create_minors_cb, 1560 &minors_list, DS_FIND_CHILDREN); 1561 spl_fstrans_unmark(cookie); 1562 } 1563 1564 taskq_wait_outstanding(system_taskq, 0); 1565 1566 /* 1567 * Prefetch is completed, we can do zvol_os_create_minor 1568 * sequentially. 1569 */ 1570 while ((job = list_remove_head(&minors_list)) != NULL) { 1571 if (!job->error) { 1572 error = zvol_os_create_minor(job->name); 1573 if (error == 0) { 1574 done++; 1575 } else { 1576 last_error = error; 1577 } 1578 } else if (job->error == EINVAL) { 1579 /* 1580 * The objset, with the name requested by current job 1581 * exist, but have the type different from zvol. 1582 * Just ignore this sort of errors. 1583 */ 1584 done++; 1585 } else { 1586 last_error = job->error; 1587 } 1588 total++; 1589 kmem_strfree(job->name); 1590 kmem_free(job, sizeof (minors_job_t)); 1591 } 1592 1593 list_destroy(&minors_list); 1594 zvol_task_update_status(task, total, done, last_error); 1595 } 1596 1597 /* 1598 * Remove minors for specified dataset and, optionally, its children and 1599 * snapshots. 1600 */ 1601 static void 1602 zvol_remove_minors_impl(zvol_task_t *task) 1603 { 1604 zvol_state_t *zv, *zv_next; 1605 const char *name = task ? task->zt_name1 : NULL; 1606 int namelen = ((name) ? strlen(name) : 0); 1607 boolean_t children = task ? !!task->zt_value : B_TRUE; 1608 1609 if (zvol_inhibit_dev) 1610 return; 1611 1612 /* 1613 * We collect up zvols that we want to remove on a separate list, so 1614 * that we don't have to hold zvol_state_lock for the whole time. 1615 * 1616 * We can't remove them from the global lists until we're completely 1617 * done with them, because that would make them appear to ZFS-side ops 1618 * that they don't exist, and the name might be reused, which can't be 1619 * good. 1620 */ 1621 list_t remove_list; 1622 list_create(&remove_list, sizeof (zvol_state_t), 1623 offsetof(zvol_state_t, zv_remove_node)); 1624 1625 rw_enter(&zvol_state_lock, RW_READER); 1626 1627 for (zv = list_head(&zvol_state_list); zv != NULL; zv = zv_next) { 1628 zv_next = list_next(&zvol_state_list, zv); 1629 1630 mutex_enter(&zv->zv_state_lock); 1631 if (zv->zv_flags & ZVOL_REMOVING) { 1632 /* Another thread is handling shutdown, skip it. */ 1633 mutex_exit(&zv->zv_state_lock); 1634 continue; 1635 } 1636 1637 /* 1638 * This zvol should be removed if: 1639 * - no name was offered (ie removing all at shutdown); or 1640 * - name matches exactly; or 1641 * - we were asked to remove children, and 1642 * - the start of the name matches, and 1643 * - there is a '/' immediately after the matched name; or 1644 * - there is a '@' immediately after the matched name 1645 */ 1646 if (name == NULL || strcmp(zv->zv_name, name) == 0 || 1647 (children && strncmp(zv->zv_name, name, namelen) == 0 && 1648 (zv->zv_name[namelen] == '/' || 1649 zv->zv_name[namelen] == '@'))) { 1650 1651 /* 1652 * Matched, so mark it removal. We want to take the 1653 * write half of the suspend lock to make sure that 1654 * the zvol is not suspended, and give any data ops 1655 * chance to finish. 1656 */ 1657 mutex_exit(&zv->zv_state_lock); 1658 rw_enter(&zv->zv_suspend_lock, RW_WRITER); 1659 mutex_enter(&zv->zv_state_lock); 1660 1661 if (zv->zv_flags & ZVOL_REMOVING) { 1662 /* Another thread has taken it, let them. */ 1663 mutex_exit(&zv->zv_state_lock); 1664 rw_exit(&zv->zv_suspend_lock); 1665 continue; 1666 } 1667 1668 /* 1669 * Mark it and unlock. New entries will see the flag 1670 * and return ENXIO. 1671 */ 1672 zv->zv_flags |= ZVOL_REMOVING; 1673 mutex_exit(&zv->zv_state_lock); 1674 rw_exit(&zv->zv_suspend_lock); 1675 1676 /* Put it on the list for the next stage. */ 1677 list_insert_head(&remove_list, zv); 1678 } else 1679 mutex_exit(&zv->zv_state_lock); 1680 } 1681 1682 rw_exit(&zvol_state_lock); 1683 1684 /* Didn't match any, nothing to do! */ 1685 if (list_is_empty(&remove_list)) { 1686 if (task) 1687 task->zt_error = SET_ERROR(ENOENT); 1688 return; 1689 } 1690 1691 /* Actually shut them all down. */ 1692 for (zv = list_head(&remove_list); zv != NULL; zv = zv_next) { 1693 zv_next = list_next(&remove_list, zv); 1694 1695 mutex_enter(&zv->zv_state_lock); 1696 1697 /* 1698 * Still open or suspended, just wait. This can happen if, for 1699 * example, we managed to acquire zv_state_lock in the moments 1700 * where zvol_open() or zvol_release() are trading locks to 1701 * call zvol_first_open() or zvol_last_close(). 1702 */ 1703 while (zv->zv_open_count > 0 || 1704 atomic_read(&zv->zv_suspend_ref)) 1705 cv_wait(&zv->zv_removing_cv, &zv->zv_state_lock); 1706 1707 /* 1708 * No users, shut down the OS side. This may not remove the 1709 * minor from view immediately, depending on the kernel 1710 * specifics, but it will ensure that it is unusable and that 1711 * this zvol_state_t can never again be reached from an OS-side 1712 * operation. 1713 */ 1714 zvol_os_remove_minor(zv); 1715 mutex_exit(&zv->zv_state_lock); 1716 1717 /* Remove it from the name lookup lists */ 1718 rw_enter(&zvol_state_lock, RW_WRITER); 1719 zvol_remove(zv); 1720 rw_exit(&zvol_state_lock); 1721 } 1722 1723 /* 1724 * Our own references on remove_list is the last one, free them and 1725 * we're done. 1726 */ 1727 while ((zv = list_remove_head(&remove_list)) != NULL) 1728 zvol_os_free(zv); 1729 1730 list_destroy(&remove_list); 1731 } 1732 1733 /* Remove minor for this specific volume only */ 1734 static int 1735 zvol_remove_minor_impl(const char *name) 1736 { 1737 if (zvol_inhibit_dev) 1738 return (0); 1739 1740 zvol_task_t task; 1741 memset(&task, 0, sizeof (zvol_task_t)); 1742 strlcpy(task.zt_name1, name, sizeof (task.zt_name1)); 1743 task.zt_value = B_FALSE; 1744 1745 zvol_remove_minors_impl(&task); 1746 1747 return (task.zt_error); 1748 } 1749 1750 /* 1751 * Rename minors for specified dataset including children and snapshots. 1752 */ 1753 static void 1754 zvol_rename_minors_impl(zvol_task_t *task) 1755 { 1756 zvol_state_t *zv, *zv_next; 1757 const char *oldname = task->zt_name1; 1758 const char *newname = task->zt_name2; 1759 int total = 0, done = 0, last_error, error, oldnamelen; 1760 1761 if (zvol_inhibit_dev) 1762 return; 1763 1764 oldnamelen = strlen(oldname); 1765 1766 rw_enter(&zvol_state_lock, RW_READER); 1767 1768 for (zv = list_head(&zvol_state_list); zv != NULL; zv = zv_next) { 1769 zv_next = list_next(&zvol_state_list, zv); 1770 1771 mutex_enter(&zv->zv_state_lock); 1772 1773 if (strcmp(zv->zv_name, oldname) == 0) { 1774 error = zvol_os_rename_minor(zv, newname); 1775 } else if (strncmp(zv->zv_name, oldname, oldnamelen) == 0 && 1776 (zv->zv_name[oldnamelen] == '/' || 1777 zv->zv_name[oldnamelen] == '@')) { 1778 char *name = kmem_asprintf("%s%c%s", newname, 1779 zv->zv_name[oldnamelen], 1780 zv->zv_name + oldnamelen + 1); 1781 error = zvol_os_rename_minor(zv, name); 1782 kmem_strfree(name); 1783 } 1784 if (error) { 1785 last_error = error; 1786 } else { 1787 done++; 1788 } 1789 total++; 1790 mutex_exit(&zv->zv_state_lock); 1791 } 1792 1793 rw_exit(&zvol_state_lock); 1794 zvol_task_update_status(task, total, done, last_error); 1795 } 1796 1797 typedef struct zvol_snapdev_cb_arg { 1798 zvol_task_t *task; 1799 uint64_t snapdev; 1800 } zvol_snapdev_cb_arg_t; 1801 1802 static int 1803 zvol_set_snapdev_cb(const char *dsname, void *param) 1804 { 1805 zvol_snapdev_cb_arg_t *arg = param; 1806 int error = 0; 1807 1808 if (strchr(dsname, '@') == NULL) 1809 return (0); 1810 1811 switch (arg->snapdev) { 1812 case ZFS_SNAPDEV_VISIBLE: 1813 error = zvol_os_create_minor(dsname); 1814 break; 1815 case ZFS_SNAPDEV_HIDDEN: 1816 error = zvol_remove_minor_impl(dsname); 1817 break; 1818 } 1819 1820 zvol_task_update_status(arg->task, 1, error == 0, error); 1821 return (0); 1822 } 1823 1824 static void 1825 zvol_set_snapdev_impl(zvol_task_t *task) 1826 { 1827 const char *name = task->zt_name1; 1828 uint64_t snapdev = task->zt_value; 1829 1830 zvol_snapdev_cb_arg_t arg = {task, snapdev}; 1831 fstrans_cookie_t cookie = spl_fstrans_mark(); 1832 /* 1833 * The zvol_set_snapdev_sync() sets snapdev appropriately 1834 * in the dataset hierarchy. Here, we only scan snapshots. 1835 */ 1836 dmu_objset_find(name, zvol_set_snapdev_cb, &arg, DS_FIND_SNAPSHOTS); 1837 spl_fstrans_unmark(cookie); 1838 } 1839 1840 static void 1841 zvol_set_volmode_impl(zvol_task_t *task) 1842 { 1843 const char *name = task->zt_name1; 1844 uint64_t volmode = task->zt_value; 1845 fstrans_cookie_t cookie; 1846 uint64_t old_volmode; 1847 zvol_state_t *zv; 1848 int error; 1849 1850 if (strchr(name, '@') != NULL) 1851 return; 1852 1853 /* 1854 * It's unfortunate we need to remove minors before we create new ones: 1855 * this is necessary because our backing gendisk (zvol_state->zv_disk) 1856 * could be different when we set, for instance, volmode from "geom" 1857 * to "dev" (or vice versa). 1858 */ 1859 zv = zvol_find_by_name(name, RW_NONE); 1860 if (zv == NULL && volmode == ZFS_VOLMODE_NONE) 1861 return; 1862 if (zv != NULL) { 1863 old_volmode = zv->zv_volmode; 1864 mutex_exit(&zv->zv_state_lock); 1865 if (old_volmode == volmode) 1866 return; 1867 zvol_wait_close(zv); 1868 } 1869 cookie = spl_fstrans_mark(); 1870 switch (volmode) { 1871 case ZFS_VOLMODE_NONE: 1872 error = zvol_remove_minor_impl(name); 1873 break; 1874 case ZFS_VOLMODE_GEOM: 1875 case ZFS_VOLMODE_DEV: 1876 error = zvol_remove_minor_impl(name); 1877 /* 1878 * The remove minor function call above, might be not 1879 * needed, if volmode was switched from 'none' value. 1880 * Ignore error in this case. 1881 */ 1882 if (error == ENOENT) 1883 error = 0; 1884 else if (error) 1885 break; 1886 error = zvol_os_create_minor(name); 1887 break; 1888 case ZFS_VOLMODE_DEFAULT: 1889 error = zvol_remove_minor_impl(name); 1890 if (zvol_volmode == ZFS_VOLMODE_NONE) 1891 break; 1892 else /* if zvol_volmode is invalid defaults to "geom" */ 1893 error = zvol_os_create_minor(name); 1894 break; 1895 } 1896 zvol_task_update_status(task, 1, error == 0, error); 1897 spl_fstrans_unmark(cookie); 1898 } 1899 1900 /* 1901 * The worker thread function performed asynchronously. 1902 */ 1903 static void 1904 zvol_task_cb(void *arg) 1905 { 1906 zvol_task_t *task = arg; 1907 1908 switch (task->zt_op) { 1909 case ZVOL_ASYNC_CREATE_MINORS: 1910 zvol_create_minors_impl(task); 1911 break; 1912 case ZVOL_ASYNC_REMOVE_MINORS: 1913 zvol_remove_minors_impl(task); 1914 break; 1915 case ZVOL_ASYNC_RENAME_MINORS: 1916 zvol_rename_minors_impl(task); 1917 break; 1918 case ZVOL_ASYNC_SET_SNAPDEV: 1919 zvol_set_snapdev_impl(task); 1920 break; 1921 case ZVOL_ASYNC_SET_VOLMODE: 1922 zvol_set_volmode_impl(task); 1923 break; 1924 default: 1925 VERIFY(0); 1926 break; 1927 } 1928 1929 zvol_task_report_status(task); 1930 kmem_free(task, sizeof (zvol_task_t)); 1931 } 1932 1933 typedef struct zvol_set_prop_int_arg { 1934 const char *zsda_name; 1935 uint64_t zsda_value; 1936 zprop_source_t zsda_source; 1937 zfs_prop_t zsda_prop; 1938 } zvol_set_prop_int_arg_t; 1939 1940 /* 1941 * Sanity check the dataset for safe use by the sync task. No additional 1942 * conditions are imposed. 1943 */ 1944 static int 1945 zvol_set_common_check(void *arg, dmu_tx_t *tx) 1946 { 1947 zvol_set_prop_int_arg_t *zsda = arg; 1948 dsl_pool_t *dp = dmu_tx_pool(tx); 1949 dsl_dir_t *dd; 1950 int error; 1951 1952 error = dsl_dir_hold(dp, zsda->zsda_name, FTAG, &dd, NULL); 1953 if (error != 0) 1954 return (error); 1955 1956 dsl_dir_rele(dd, FTAG); 1957 1958 return (error); 1959 } 1960 1961 static int 1962 zvol_set_common_sync_cb(dsl_pool_t *dp, dsl_dataset_t *ds, void *arg) 1963 { 1964 zvol_set_prop_int_arg_t *zsda = arg; 1965 char dsname[ZFS_MAX_DATASET_NAME_LEN]; 1966 zvol_task_t *task; 1967 uint64_t prop; 1968 1969 const char *prop_name = zfs_prop_to_name(zsda->zsda_prop); 1970 dsl_dataset_name(ds, dsname); 1971 1972 if (dsl_prop_get_int_ds(ds, prop_name, &prop) != 0) 1973 return (0); 1974 1975 task = kmem_zalloc(sizeof (zvol_task_t), KM_SLEEP); 1976 if (zsda->zsda_prop == ZFS_PROP_VOLMODE) { 1977 task->zt_op = ZVOL_ASYNC_SET_VOLMODE; 1978 } else if (zsda->zsda_prop == ZFS_PROP_SNAPDEV) { 1979 task->zt_op = ZVOL_ASYNC_SET_SNAPDEV; 1980 } else { 1981 kmem_free(task, sizeof (zvol_task_t)); 1982 return (0); 1983 } 1984 task->zt_value = prop; 1985 strlcpy(task->zt_name1, dsname, sizeof (task->zt_name1)); 1986 (void) taskq_dispatch(dp->dp_spa->spa_zvol_taskq, zvol_task_cb, 1987 task, TQ_SLEEP); 1988 return (0); 1989 } 1990 1991 /* 1992 * Traverse all child datasets and apply the property appropriately. 1993 * We call dsl_prop_set_sync_impl() here to set the value only on the toplevel 1994 * dataset and read the effective "property" on every child in the callback 1995 * function: this is because the value is not guaranteed to be the same in the 1996 * whole dataset hierarchy. 1997 */ 1998 static void 1999 zvol_set_common_sync(void *arg, dmu_tx_t *tx) 2000 { 2001 zvol_set_prop_int_arg_t *zsda = arg; 2002 dsl_pool_t *dp = dmu_tx_pool(tx); 2003 dsl_dir_t *dd; 2004 dsl_dataset_t *ds; 2005 int error; 2006 2007 VERIFY0(dsl_dir_hold(dp, zsda->zsda_name, FTAG, &dd, NULL)); 2008 2009 error = dsl_dataset_hold(dp, zsda->zsda_name, FTAG, &ds); 2010 if (error == 0) { 2011 dsl_prop_set_sync_impl(ds, zfs_prop_to_name(zsda->zsda_prop), 2012 zsda->zsda_source, sizeof (zsda->zsda_value), 1, 2013 &zsda->zsda_value, tx); 2014 dsl_dataset_rele(ds, FTAG); 2015 } 2016 2017 dmu_objset_find_dp(dp, dd->dd_object, zvol_set_common_sync_cb, 2018 zsda, DS_FIND_CHILDREN); 2019 2020 dsl_dir_rele(dd, FTAG); 2021 } 2022 2023 int 2024 zvol_set_common(const char *ddname, zfs_prop_t prop, zprop_source_t source, 2025 uint64_t val) 2026 { 2027 zvol_set_prop_int_arg_t zsda; 2028 2029 zsda.zsda_name = ddname; 2030 zsda.zsda_source = source; 2031 zsda.zsda_value = val; 2032 zsda.zsda_prop = prop; 2033 2034 return (dsl_sync_task(ddname, zvol_set_common_check, 2035 zvol_set_common_sync, &zsda, 0, ZFS_SPACE_CHECK_NONE)); 2036 } 2037 2038 void 2039 zvol_create_minors(const char *name) 2040 { 2041 spa_t *spa; 2042 zvol_task_t *task; 2043 taskqid_t id; 2044 2045 if (spa_open(name, &spa, FTAG) != 0) 2046 return; 2047 2048 task = kmem_zalloc(sizeof (zvol_task_t), KM_SLEEP); 2049 task->zt_op = ZVOL_ASYNC_CREATE_MINORS; 2050 strlcpy(task->zt_name1, name, sizeof (task->zt_name1)); 2051 id = taskq_dispatch(spa->spa_zvol_taskq, zvol_task_cb, task, TQ_SLEEP); 2052 if (id != TASKQID_INVALID) 2053 taskq_wait_id(spa->spa_zvol_taskq, id); 2054 2055 spa_close(spa, FTAG); 2056 } 2057 2058 void 2059 zvol_remove_minors(spa_t *spa, const char *name, boolean_t async) 2060 { 2061 zvol_task_t *task; 2062 taskqid_t id; 2063 2064 task = kmem_zalloc(sizeof (zvol_task_t), KM_SLEEP); 2065 task->zt_op = ZVOL_ASYNC_REMOVE_MINORS; 2066 strlcpy(task->zt_name1, name, sizeof (task->zt_name1)); 2067 task->zt_value = B_TRUE; 2068 id = taskq_dispatch(spa->spa_zvol_taskq, zvol_task_cb, task, TQ_SLEEP); 2069 if ((async == B_FALSE) && (id != TASKQID_INVALID)) 2070 taskq_wait_id(spa->spa_zvol_taskq, id); 2071 } 2072 2073 void 2074 zvol_rename_minors(spa_t *spa, const char *name1, const char *name2, 2075 boolean_t async) 2076 { 2077 zvol_task_t *task; 2078 taskqid_t id; 2079 2080 task = kmem_zalloc(sizeof (zvol_task_t), KM_SLEEP); 2081 task->zt_op = ZVOL_ASYNC_RENAME_MINORS; 2082 strlcpy(task->zt_name1, name1, sizeof (task->zt_name1)); 2083 strlcpy(task->zt_name2, name2, sizeof (task->zt_name2)); 2084 id = taskq_dispatch(spa->spa_zvol_taskq, zvol_task_cb, task, TQ_SLEEP); 2085 if ((async == B_FALSE) && (id != TASKQID_INVALID)) 2086 taskq_wait_id(spa->spa_zvol_taskq, id); 2087 } 2088 2089 boolean_t 2090 zvol_is_zvol(const char *name) 2091 { 2092 2093 return (zvol_os_is_zvol(name)); 2094 } 2095 2096 int 2097 zvol_init_impl(void) 2098 { 2099 int i; 2100 2101 /* 2102 * zvol_threads is the module param the user passes in. 2103 * 2104 * zvol_actual_threads is what we use internally, since the user can 2105 * pass zvol_thread = 0 to mean "use all the CPUs" (the default). 2106 */ 2107 static unsigned int zvol_actual_threads; 2108 2109 if (zvol_threads == 0) { 2110 /* 2111 * See dde9380a1 for why 32 was chosen here. This should 2112 * probably be refined to be some multiple of the number 2113 * of CPUs. 2114 */ 2115 zvol_actual_threads = MAX(max_ncpus, 32); 2116 } else { 2117 zvol_actual_threads = MIN(MAX(zvol_threads, 1), 1024); 2118 } 2119 2120 /* 2121 * Use at least 32 zvol_threads but for many core system, 2122 * prefer 6 threads per taskq, but no more taskqs 2123 * than threads in them on large systems. 2124 * 2125 * taskq total 2126 * cpus taskqs threads threads 2127 * ------- ------- ------- ------- 2128 * 1 1 32 32 2129 * 2 1 32 32 2130 * 4 1 32 32 2131 * 8 2 16 32 2132 * 16 3 11 33 2133 * 32 5 7 35 2134 * 64 8 8 64 2135 * 128 11 12 132 2136 * 256 16 16 256 2137 */ 2138 zv_taskq_t *ztqs = &zvol_taskqs; 2139 int num_tqs = MIN(max_ncpus, zvol_num_taskqs); 2140 if (num_tqs == 0) { 2141 num_tqs = 1 + max_ncpus / 6; 2142 while (num_tqs * num_tqs > zvol_actual_threads) 2143 num_tqs--; 2144 } 2145 2146 int per_tq_thread = zvol_actual_threads / num_tqs; 2147 if (per_tq_thread * num_tqs < zvol_actual_threads) 2148 per_tq_thread++; 2149 2150 ztqs->tqs_cnt = num_tqs; 2151 ztqs->tqs_taskq = kmem_alloc(num_tqs * sizeof (taskq_t *), KM_SLEEP); 2152 2153 for (uint_t i = 0; i < num_tqs; i++) { 2154 char name[32]; 2155 (void) snprintf(name, sizeof (name), "%s_tq-%u", 2156 ZVOL_DRIVER, i); 2157 ztqs->tqs_taskq[i] = taskq_create(name, per_tq_thread, 2158 maxclsyspri, per_tq_thread, INT_MAX, 2159 TASKQ_PREPOPULATE | TASKQ_DYNAMIC); 2160 if (ztqs->tqs_taskq[i] == NULL) { 2161 for (int j = i - 1; j >= 0; j--) 2162 taskq_destroy(ztqs->tqs_taskq[j]); 2163 kmem_free(ztqs->tqs_taskq, ztqs->tqs_cnt * 2164 sizeof (taskq_t *)); 2165 ztqs->tqs_taskq = NULL; 2166 return (SET_ERROR(ENOMEM)); 2167 } 2168 } 2169 2170 list_create(&zvol_state_list, sizeof (zvol_state_t), 2171 offsetof(zvol_state_t, zv_next)); 2172 rw_init(&zvol_state_lock, NULL, RW_DEFAULT, NULL); 2173 2174 zvol_htable = kmem_alloc(ZVOL_HT_SIZE * sizeof (struct hlist_head), 2175 KM_SLEEP); 2176 for (i = 0; i < ZVOL_HT_SIZE; i++) 2177 INIT_HLIST_HEAD(&zvol_htable[i]); 2178 2179 return (0); 2180 } 2181 2182 void 2183 zvol_fini_impl(void) 2184 { 2185 zv_taskq_t *ztqs = &zvol_taskqs; 2186 2187 zvol_remove_minors_impl(NULL); 2188 2189 kmem_free(zvol_htable, ZVOL_HT_SIZE * sizeof (struct hlist_head)); 2190 list_destroy(&zvol_state_list); 2191 rw_destroy(&zvol_state_lock); 2192 2193 if (ztqs->tqs_taskq == NULL) { 2194 ASSERT0(ztqs->tqs_cnt); 2195 } else { 2196 for (uint_t i = 0; i < ztqs->tqs_cnt; i++) { 2197 ASSERT3P(ztqs->tqs_taskq[i], !=, NULL); 2198 taskq_destroy(ztqs->tqs_taskq[i]); 2199 } 2200 kmem_free(ztqs->tqs_taskq, ztqs->tqs_cnt * 2201 sizeof (taskq_t *)); 2202 ztqs->tqs_taskq = NULL; 2203 } 2204 } 2205 2206 ZFS_MODULE_PARAM(zfs_vol, zvol_, inhibit_dev, UINT, ZMOD_RW, 2207 "Do not create zvol device nodes"); 2208 ZFS_MODULE_PARAM(zfs_vol, zvol_, prefetch_bytes, UINT, ZMOD_RW, 2209 "Prefetch N bytes at zvol start+end"); 2210 ZFS_MODULE_PARAM(zfs_vol, zvol_vol, mode, UINT, ZMOD_RW, 2211 "Default volmode property value"); 2212 ZFS_MODULE_PARAM(zfs_vol, zvol_, threads, UINT, ZMOD_RW, 2213 "Number of threads for I/O requests. Set to 0 to use all active CPUs"); 2214 ZFS_MODULE_PARAM(zfs_vol, zvol_, num_taskqs, UINT, ZMOD_RW, 2215 "Number of zvol taskqs"); 2216 ZFS_MODULE_PARAM(zfs_vol, zvol_, request_sync, UINT, ZMOD_RW, 2217 "Synchronously handle bio requests"); 2218