1 // SPDX-License-Identifier: CDDL-1.0 2 /* 3 * CDDL HEADER START 4 * 5 * The contents of this file are subject to the terms of the 6 * Common Development and Distribution License (the "License"). 7 * You may not use this file except in compliance with the License. 8 * 9 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 10 * or https://opensource.org/licenses/CDDL-1.0. 11 * See the License for the specific language governing permissions 12 * and limitations under the License. 13 * 14 * When distributing Covered Code, include this CDDL HEADER in each 15 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 16 * If applicable, add the following below this CDDL HEADER, with the 17 * fields enclosed by brackets "[]" replaced with your own identifying 18 * information: Portions Copyright [yyyy] [name of copyright owner] 19 * 20 * CDDL HEADER END 21 */ 22 /* 23 * Copyright (C) 2008-2010 Lawrence Livermore National Security, LLC. 24 * Produced at Lawrence Livermore National Laboratory (cf, DISCLAIMER). 25 * Rewritten for Linux by Brian Behlendorf <behlendorf1@llnl.gov>. 26 * LLNL-CODE-403049. 27 * 28 * ZFS volume emulation driver. 29 * 30 * Makes a DMU object look like a volume of arbitrary size, up to 2^64 bytes. 31 * Volumes are accessed through the symbolic links named: 32 * 33 * /dev/<pool_name>/<dataset_name> 34 * 35 * Volumes are persistent through reboot and module load. No user command 36 * needs to be run before opening and using a device. 37 * 38 * Copyright 2014 Nexenta Systems, Inc. All rights reserved. 39 * Copyright (c) 2016 Actifio, Inc. All rights reserved. 40 * Copyright (c) 2012, 2019 by Delphix. All rights reserved. 41 * Copyright (c) 2024, Klara, Inc. 42 */ 43 44 /* 45 * Note on locking of zvol state structures. 46 * 47 * These structures are used to maintain internal state used to emulate block 48 * devices on top of zvols. In particular, management of device minor number 49 * operations - create, remove, rename, and set_snapdev - involves access to 50 * these structures. The zvol_state_lock is primarily used to protect the 51 * zvol_state_list. The zv->zv_state_lock is used to protect the contents 52 * of the zvol_state_t structures, as well as to make sure that when the 53 * time comes to remove the structure from the list, it is not in use, and 54 * therefore, it can be taken off zvol_state_list and freed. 55 * 56 * The zv_suspend_lock was introduced to allow for suspending I/O to a zvol, 57 * e.g. for the duration of receive and rollback operations. This lock can be 58 * held for significant periods of time. Given that it is undesirable to hold 59 * mutexes for long periods of time, the following lock ordering applies: 60 * - take zvol_state_lock if necessary, to protect zvol_state_list 61 * - take zv_suspend_lock if necessary, by the code path in question 62 * - take zv_state_lock to protect zvol_state_t 63 * 64 * The minor operations are issued to spa->spa_zvol_taskq queues, that are 65 * single-threaded (to preserve order of minor operations), and are executed 66 * through the zvol_task_cb that dispatches the specific operations. Therefore, 67 * these operations are serialized per pool. Consequently, we can be certain 68 * that for a given zvol, there is only one operation at a time in progress. 69 * That is why one can be sure that first, zvol_state_t for a given zvol is 70 * allocated and placed on zvol_state_list, and then other minor operations 71 * for this zvol are going to proceed in the order of issue. 72 * 73 */ 74 75 #include <sys/dataset_kstats.h> 76 #include <sys/dbuf.h> 77 #include <sys/dmu_traverse.h> 78 #include <sys/dsl_dataset.h> 79 #include <sys/dsl_prop.h> 80 #include <sys/dsl_dir.h> 81 #include <sys/zap.h> 82 #include <sys/zfeature.h> 83 #include <sys/zil_impl.h> 84 #include <sys/dmu_tx.h> 85 #include <sys/zio.h> 86 #include <sys/zfs_rlock.h> 87 #include <sys/spa_impl.h> 88 #include <sys/zvol.h> 89 #include <sys/zvol_impl.h> 90 91 unsigned int zvol_inhibit_dev = 0; 92 unsigned int zvol_prefetch_bytes = (128 * 1024); 93 unsigned int zvol_volmode = ZFS_VOLMODE_GEOM; 94 unsigned int zvol_threads = 0; 95 unsigned int zvol_num_taskqs = 0; 96 unsigned int zvol_request_sync = 0; 97 98 struct hlist_head *zvol_htable; 99 static list_t zvol_state_list; 100 krwlock_t zvol_state_lock; 101 extern int zfs_bclone_wait_dirty; 102 zv_taskq_t zvol_taskqs; 103 104 typedef enum { 105 ZVOL_ASYNC_CREATE_MINORS, 106 ZVOL_ASYNC_REMOVE_MINORS, 107 ZVOL_ASYNC_RENAME_MINORS, 108 ZVOL_ASYNC_SET_SNAPDEV, 109 ZVOL_ASYNC_SET_VOLMODE, 110 ZVOL_ASYNC_MAX 111 } zvol_async_op_t; 112 113 typedef struct { 114 zvol_async_op_t zt_op; 115 char zt_name1[MAXNAMELEN]; 116 char zt_name2[MAXNAMELEN]; 117 uint64_t zt_value; 118 uint32_t zt_total; 119 uint32_t zt_done; 120 int32_t zt_status; 121 int zt_error; 122 } zvol_task_t; 123 124 zv_request_task_t * 125 zv_request_task_create(zv_request_t zvr) 126 { 127 zv_request_task_t *task; 128 task = kmem_alloc(sizeof (zv_request_task_t), KM_SLEEP); 129 taskq_init_ent(&task->ent); 130 task->zvr = zvr; 131 return (task); 132 } 133 134 void 135 zv_request_task_free(zv_request_task_t *task) 136 { 137 kmem_free(task, sizeof (*task)); 138 } 139 140 uint64_t 141 zvol_name_hash(const char *name) 142 { 143 uint64_t crc = -1ULL; 144 ASSERT(zfs_crc64_table[128] == ZFS_CRC64_POLY); 145 for (const uint8_t *p = (const uint8_t *)name; *p != 0; p++) 146 crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ (*p)) & 0xFF]; 147 return (crc); 148 } 149 150 /* 151 * Find a zvol_state_t given the name and hash generated by zvol_name_hash. 152 * If found, return with zv_suspend_lock and zv_state_lock taken, otherwise, 153 * return (NULL) without the taking locks. The zv_suspend_lock is always taken 154 * before zv_state_lock. The mode argument indicates the mode (including none) 155 * for zv_suspend_lock to be taken. 156 */ 157 zvol_state_t * 158 zvol_find_by_name_hash(const char *name, uint64_t hash, int mode) 159 { 160 zvol_state_t *zv; 161 struct hlist_node *p = NULL; 162 163 rw_enter(&zvol_state_lock, RW_READER); 164 hlist_for_each(p, ZVOL_HT_HEAD(hash)) { 165 zv = hlist_entry(p, zvol_state_t, zv_hlink); 166 mutex_enter(&zv->zv_state_lock); 167 if (zv->zv_hash == hash && strcmp(zv->zv_name, name) == 0) { 168 /* 169 * this is the right zvol, take the locks in the 170 * right order 171 */ 172 if (mode != RW_NONE && 173 !rw_tryenter(&zv->zv_suspend_lock, mode)) { 174 mutex_exit(&zv->zv_state_lock); 175 rw_enter(&zv->zv_suspend_lock, mode); 176 mutex_enter(&zv->zv_state_lock); 177 /* 178 * zvol cannot be renamed as we continue 179 * to hold zvol_state_lock 180 */ 181 ASSERT(zv->zv_hash == hash && 182 strcmp(zv->zv_name, name) == 0); 183 } 184 rw_exit(&zvol_state_lock); 185 return (zv); 186 } 187 mutex_exit(&zv->zv_state_lock); 188 } 189 rw_exit(&zvol_state_lock); 190 191 return (NULL); 192 } 193 194 /* 195 * Find a zvol_state_t given the name. 196 * If found, return with zv_suspend_lock and zv_state_lock taken, otherwise, 197 * return (NULL) without the taking locks. The zv_suspend_lock is always taken 198 * before zv_state_lock. The mode argument indicates the mode (including none) 199 * for zv_suspend_lock to be taken. 200 */ 201 static zvol_state_t * 202 zvol_find_by_name(const char *name, int mode) 203 { 204 return (zvol_find_by_name_hash(name, zvol_name_hash(name), mode)); 205 } 206 207 /* 208 * ZFS_IOC_CREATE callback handles dmu zvol and zap object creation. 209 */ 210 void 211 zvol_create_cb(objset_t *os, void *arg, cred_t *cr, dmu_tx_t *tx) 212 { 213 zfs_creat_t *zct = arg; 214 nvlist_t *nvprops = zct->zct_props; 215 int error; 216 uint64_t volblocksize, volsize; 217 218 VERIFY0(nvlist_lookup_uint64(nvprops, 219 zfs_prop_to_name(ZFS_PROP_VOLSIZE), &volsize)); 220 if (nvlist_lookup_uint64(nvprops, 221 zfs_prop_to_name(ZFS_PROP_VOLBLOCKSIZE), &volblocksize) != 0) 222 volblocksize = zfs_prop_default_numeric(ZFS_PROP_VOLBLOCKSIZE); 223 224 /* 225 * These properties must be removed from the list so the generic 226 * property setting step won't apply to them. 227 */ 228 VERIFY0(nvlist_remove_all(nvprops, zfs_prop_to_name(ZFS_PROP_VOLSIZE))); 229 (void) nvlist_remove_all(nvprops, 230 zfs_prop_to_name(ZFS_PROP_VOLBLOCKSIZE)); 231 232 error = dmu_object_claim(os, ZVOL_OBJ, DMU_OT_ZVOL, volblocksize, 233 DMU_OT_NONE, 0, tx); 234 ASSERT0(error); 235 236 error = zap_create_claim(os, ZVOL_ZAP_OBJ, DMU_OT_ZVOL_PROP, 237 DMU_OT_NONE, 0, tx); 238 ASSERT0(error); 239 240 error = zap_update(os, ZVOL_ZAP_OBJ, "size", 8, 1, &volsize, tx); 241 ASSERT0(error); 242 } 243 244 /* 245 * ZFS_IOC_OBJSET_STATS entry point. 246 */ 247 int 248 zvol_get_stats(objset_t *os, nvlist_t *nv) 249 { 250 int error; 251 dmu_object_info_t *doi; 252 uint64_t val; 253 254 error = zap_lookup(os, ZVOL_ZAP_OBJ, "size", 8, 1, &val); 255 if (error) 256 return (error); 257 258 dsl_prop_nvlist_add_uint64(nv, ZFS_PROP_VOLSIZE, val); 259 doi = kmem_alloc(sizeof (dmu_object_info_t), KM_SLEEP); 260 error = dmu_object_info(os, ZVOL_OBJ, doi); 261 262 if (error == 0) { 263 dsl_prop_nvlist_add_uint64(nv, ZFS_PROP_VOLBLOCKSIZE, 264 doi->doi_data_block_size); 265 } 266 267 kmem_free(doi, sizeof (dmu_object_info_t)); 268 269 return (error); 270 } 271 272 /* 273 * Sanity check volume size. 274 */ 275 int 276 zvol_check_volsize(uint64_t volsize, uint64_t blocksize) 277 { 278 if (volsize == 0) 279 return (SET_ERROR(EINVAL)); 280 281 if (volsize % blocksize != 0) 282 return (SET_ERROR(EINVAL)); 283 284 #ifdef _ILP32 285 if (volsize - 1 > SPEC_MAXOFFSET_T) 286 return (SET_ERROR(EOVERFLOW)); 287 #endif 288 return (0); 289 } 290 291 /* 292 * Ensure the zap is flushed then inform the VFS of the capacity change. 293 */ 294 static int 295 zvol_update_volsize(uint64_t volsize, objset_t *os) 296 { 297 dmu_tx_t *tx; 298 int error; 299 uint64_t txg; 300 301 tx = dmu_tx_create(os); 302 dmu_tx_hold_zap(tx, ZVOL_ZAP_OBJ, TRUE, NULL); 303 dmu_tx_mark_netfree(tx); 304 error = dmu_tx_assign(tx, DMU_TX_WAIT); 305 if (error) { 306 dmu_tx_abort(tx); 307 return (error); 308 } 309 txg = dmu_tx_get_txg(tx); 310 311 error = zap_update(os, ZVOL_ZAP_OBJ, "size", 8, 1, 312 &volsize, tx); 313 dmu_tx_commit(tx); 314 315 txg_wait_synced(dmu_objset_pool(os), txg); 316 317 if (error == 0) 318 error = dmu_free_long_range(os, 319 ZVOL_OBJ, volsize, DMU_OBJECT_END); 320 321 return (error); 322 } 323 324 /* 325 * Set ZFS_PROP_VOLSIZE set entry point. Note that modifying the volume 326 * size will result in a udev "change" event being generated. 327 */ 328 int 329 zvol_set_volsize(const char *name, uint64_t volsize) 330 { 331 objset_t *os = NULL; 332 uint64_t readonly; 333 int error; 334 boolean_t owned = B_FALSE; 335 336 error = dsl_prop_get_integer(name, 337 zfs_prop_to_name(ZFS_PROP_READONLY), &readonly, NULL); 338 if (error != 0) 339 return (error); 340 if (readonly) 341 return (SET_ERROR(EROFS)); 342 343 zvol_state_t *zv = zvol_find_by_name(name, RW_READER); 344 345 ASSERT(zv == NULL || (MUTEX_HELD(&zv->zv_state_lock) && 346 RW_READ_HELD(&zv->zv_suspend_lock))); 347 348 if (zv == NULL || zv->zv_objset == NULL) { 349 if (zv != NULL) 350 rw_exit(&zv->zv_suspend_lock); 351 if ((error = dmu_objset_own(name, DMU_OST_ZVOL, B_FALSE, B_TRUE, 352 FTAG, &os)) != 0) { 353 if (zv != NULL) 354 mutex_exit(&zv->zv_state_lock); 355 return (error); 356 } 357 owned = B_TRUE; 358 if (zv != NULL) 359 zv->zv_objset = os; 360 } else { 361 os = zv->zv_objset; 362 } 363 364 dmu_object_info_t *doi = kmem_alloc(sizeof (*doi), KM_SLEEP); 365 366 if ((error = dmu_object_info(os, ZVOL_OBJ, doi)) || 367 (error = zvol_check_volsize(volsize, doi->doi_data_block_size))) 368 goto out; 369 370 error = zvol_update_volsize(volsize, os); 371 if (error == 0 && zv != NULL) { 372 zv->zv_volsize = volsize; 373 zv->zv_changed = 1; 374 } 375 out: 376 kmem_free(doi, sizeof (dmu_object_info_t)); 377 378 if (owned) { 379 dmu_objset_disown(os, B_TRUE, FTAG); 380 if (zv != NULL) 381 zv->zv_objset = NULL; 382 } else { 383 rw_exit(&zv->zv_suspend_lock); 384 } 385 386 if (zv != NULL) 387 mutex_exit(&zv->zv_state_lock); 388 389 if (error == 0 && zv != NULL) 390 zvol_os_update_volsize(zv, volsize); 391 392 return (error); 393 } 394 395 /* 396 * Update volthreading. 397 */ 398 int 399 zvol_set_volthreading(const char *name, boolean_t value) 400 { 401 zvol_state_t *zv = zvol_find_by_name(name, RW_NONE); 402 if (zv == NULL) 403 return (SET_ERROR(ENOENT)); 404 zv->zv_threading = value; 405 mutex_exit(&zv->zv_state_lock); 406 return (0); 407 } 408 409 /* 410 * Update zvol ro property. 411 */ 412 int 413 zvol_set_ro(const char *name, boolean_t value) 414 { 415 zvol_state_t *zv = zvol_find_by_name(name, RW_NONE); 416 if (zv == NULL) 417 return (-1); 418 if (value) { 419 zvol_os_set_disk_ro(zv, 1); 420 zv->zv_flags |= ZVOL_RDONLY; 421 } else { 422 zvol_os_set_disk_ro(zv, 0); 423 zv->zv_flags &= ~ZVOL_RDONLY; 424 } 425 mutex_exit(&zv->zv_state_lock); 426 return (0); 427 } 428 429 /* 430 * Sanity check volume block size. 431 */ 432 int 433 zvol_check_volblocksize(const char *name, uint64_t volblocksize) 434 { 435 /* Record sizes above 128k need the feature to be enabled */ 436 if (volblocksize > SPA_OLD_MAXBLOCKSIZE) { 437 spa_t *spa; 438 int error; 439 440 if ((error = spa_open(name, &spa, FTAG)) != 0) 441 return (error); 442 443 if (!spa_feature_is_enabled(spa, SPA_FEATURE_LARGE_BLOCKS)) { 444 spa_close(spa, FTAG); 445 return (SET_ERROR(ENOTSUP)); 446 } 447 448 /* 449 * We don't allow setting the property above 1MB, 450 * unless the tunable has been changed. 451 */ 452 if (volblocksize > zfs_max_recordsize) { 453 spa_close(spa, FTAG); 454 return (SET_ERROR(EDOM)); 455 } 456 457 spa_close(spa, FTAG); 458 } 459 460 if (volblocksize < SPA_MINBLOCKSIZE || 461 volblocksize > SPA_MAXBLOCKSIZE || 462 !ISP2(volblocksize)) 463 return (SET_ERROR(EDOM)); 464 465 return (0); 466 } 467 468 /* 469 * Replay a TX_TRUNCATE ZIL transaction if asked. TX_TRUNCATE is how we 470 * implement DKIOCFREE/free-long-range. 471 */ 472 static int 473 zvol_replay_truncate(void *arg1, void *arg2, boolean_t byteswap) 474 { 475 zvol_state_t *zv = arg1; 476 lr_truncate_t *lr = arg2; 477 uint64_t offset, length; 478 479 ASSERT3U(lr->lr_common.lrc_reclen, >=, sizeof (*lr)); 480 481 if (byteswap) 482 byteswap_uint64_array(lr, sizeof (*lr)); 483 484 offset = lr->lr_offset; 485 length = lr->lr_length; 486 487 dmu_tx_t *tx = dmu_tx_create(zv->zv_objset); 488 dmu_tx_mark_netfree(tx); 489 int error = dmu_tx_assign(tx, DMU_TX_WAIT); 490 if (error != 0) { 491 dmu_tx_abort(tx); 492 } else { 493 (void) zil_replaying(zv->zv_zilog, tx); 494 dmu_tx_commit(tx); 495 error = dmu_free_long_range(zv->zv_objset, ZVOL_OBJ, offset, 496 length); 497 } 498 499 return (error); 500 } 501 502 /* 503 * Replay a TX_WRITE ZIL transaction that didn't get committed 504 * after a system failure 505 */ 506 static int 507 zvol_replay_write(void *arg1, void *arg2, boolean_t byteswap) 508 { 509 zvol_state_t *zv = arg1; 510 lr_write_t *lr = arg2; 511 objset_t *os = zv->zv_objset; 512 char *data = (char *)(lr + 1); /* data follows lr_write_t */ 513 uint64_t offset, length; 514 dmu_tx_t *tx; 515 int error; 516 517 ASSERT3U(lr->lr_common.lrc_reclen, >=, sizeof (*lr)); 518 519 if (byteswap) 520 byteswap_uint64_array(lr, sizeof (*lr)); 521 522 offset = lr->lr_offset; 523 length = lr->lr_length; 524 525 /* If it's a dmu_sync() block, write the whole block */ 526 if (lr->lr_common.lrc_reclen == sizeof (lr_write_t)) { 527 uint64_t blocksize = BP_GET_LSIZE(&lr->lr_blkptr); 528 if (length < blocksize) { 529 offset -= offset % blocksize; 530 length = blocksize; 531 } 532 } 533 534 tx = dmu_tx_create(os); 535 dmu_tx_hold_write(tx, ZVOL_OBJ, offset, length); 536 error = dmu_tx_assign(tx, DMU_TX_WAIT); 537 if (error) { 538 dmu_tx_abort(tx); 539 } else { 540 dmu_write(os, ZVOL_OBJ, offset, length, data, tx); 541 (void) zil_replaying(zv->zv_zilog, tx); 542 dmu_tx_commit(tx); 543 } 544 545 return (error); 546 } 547 548 /* 549 * Replay a TX_CLONE_RANGE ZIL transaction that didn't get committed 550 * after a system failure 551 */ 552 static int 553 zvol_replay_clone_range(void *arg1, void *arg2, boolean_t byteswap) 554 { 555 zvol_state_t *zv = arg1; 556 lr_clone_range_t *lr = arg2; 557 objset_t *os = zv->zv_objset; 558 dmu_tx_t *tx; 559 int error; 560 uint64_t blksz; 561 uint64_t off; 562 uint64_t len; 563 564 ASSERT3U(lr->lr_common.lrc_reclen, >=, sizeof (*lr)); 565 ASSERT3U(lr->lr_common.lrc_reclen, >=, offsetof(lr_clone_range_t, 566 lr_bps[lr->lr_nbps])); 567 568 if (byteswap) 569 byteswap_uint64_array(lr, sizeof (*lr)); 570 571 ASSERT(spa_feature_is_enabled(dmu_objset_spa(os), 572 SPA_FEATURE_BLOCK_CLONING)); 573 574 off = lr->lr_offset; 575 len = lr->lr_length; 576 blksz = lr->lr_blksz; 577 578 if ((off % blksz) != 0) { 579 return (SET_ERROR(EINVAL)); 580 } 581 582 error = dnode_hold(os, ZVOL_OBJ, zv, &zv->zv_dn); 583 if (error != 0 || !zv->zv_dn) 584 return (error); 585 tx = dmu_tx_create(os); 586 dmu_tx_hold_clone_by_dnode(tx, zv->zv_dn, off, len, blksz); 587 error = dmu_tx_assign(tx, DMU_TX_WAIT); 588 if (error != 0) { 589 dmu_tx_abort(tx); 590 goto out; 591 } 592 error = dmu_brt_clone(zv->zv_objset, ZVOL_OBJ, off, len, 593 tx, lr->lr_bps, lr->lr_nbps); 594 if (error != 0) { 595 dmu_tx_commit(tx); 596 goto out; 597 } 598 599 /* 600 * zil_replaying() not only check if we are replaying ZIL, but also 601 * updates the ZIL header to record replay progress. 602 */ 603 VERIFY(zil_replaying(zv->zv_zilog, tx)); 604 dmu_tx_commit(tx); 605 606 out: 607 dnode_rele(zv->zv_dn, zv); 608 zv->zv_dn = NULL; 609 return (error); 610 } 611 612 int 613 zvol_clone_range(zvol_state_t *zv_src, uint64_t inoff, zvol_state_t *zv_dst, 614 uint64_t outoff, uint64_t len) 615 { 616 zilog_t *zilog_dst; 617 zfs_locked_range_t *inlr, *outlr; 618 objset_t *inos, *outos; 619 dmu_tx_t *tx; 620 blkptr_t *bps; 621 size_t maxblocks; 622 int error = 0; 623 624 rw_enter(&zv_dst->zv_suspend_lock, RW_READER); 625 if (zv_dst->zv_zilog == NULL) { 626 rw_exit(&zv_dst->zv_suspend_lock); 627 rw_enter(&zv_dst->zv_suspend_lock, RW_WRITER); 628 if (zv_dst->zv_zilog == NULL) { 629 zv_dst->zv_zilog = zil_open(zv_dst->zv_objset, 630 zvol_get_data, &zv_dst->zv_kstat.dk_zil_sums); 631 zv_dst->zv_flags |= ZVOL_WRITTEN_TO; 632 VERIFY0((zv_dst->zv_zilog->zl_header->zh_flags & 633 ZIL_REPLAY_NEEDED)); 634 } 635 rw_downgrade(&zv_dst->zv_suspend_lock); 636 } 637 if (zv_src != zv_dst) 638 rw_enter(&zv_src->zv_suspend_lock, RW_READER); 639 640 inos = zv_src->zv_objset; 641 outos = zv_dst->zv_objset; 642 643 /* 644 * Sanity checks 645 */ 646 if (!spa_feature_is_enabled(dmu_objset_spa(outos), 647 SPA_FEATURE_BLOCK_CLONING)) { 648 error = SET_ERROR(EOPNOTSUPP); 649 goto out; 650 } 651 if (dmu_objset_spa(inos) != dmu_objset_spa(outos)) { 652 error = SET_ERROR(EXDEV); 653 goto out; 654 } 655 if (inos->os_encrypted != outos->os_encrypted) { 656 error = SET_ERROR(EXDEV); 657 goto out; 658 } 659 if (zv_src->zv_volblocksize != zv_dst->zv_volblocksize) { 660 error = SET_ERROR(EINVAL); 661 goto out; 662 } 663 if (inoff >= zv_src->zv_volsize || outoff >= zv_dst->zv_volsize) { 664 goto out; 665 } 666 667 /* 668 * Do not read beyond boundary 669 */ 670 if (len > zv_src->zv_volsize - inoff) 671 len = zv_src->zv_volsize - inoff; 672 if (len > zv_dst->zv_volsize - outoff) 673 len = zv_dst->zv_volsize - outoff; 674 if (len == 0) 675 goto out; 676 677 /* 678 * No overlapping if we are cloning within the same file 679 */ 680 if (zv_src == zv_dst) { 681 if (inoff < outoff + len && outoff < inoff + len) { 682 error = SET_ERROR(EINVAL); 683 goto out; 684 } 685 } 686 687 /* 688 * Offsets and length must be at block boundaries 689 */ 690 if ((inoff % zv_src->zv_volblocksize) != 0 || 691 (outoff % zv_dst->zv_volblocksize) != 0) { 692 error = SET_ERROR(EINVAL); 693 goto out; 694 } 695 696 /* 697 * Length must be multiple of block size 698 */ 699 if ((len % zv_src->zv_volblocksize) != 0) { 700 error = SET_ERROR(EINVAL); 701 goto out; 702 } 703 704 zilog_dst = zv_dst->zv_zilog; 705 maxblocks = zil_max_log_data(zilog_dst, sizeof (lr_clone_range_t)) / 706 sizeof (bps[0]); 707 bps = vmem_alloc(sizeof (bps[0]) * maxblocks, KM_SLEEP); 708 /* 709 * Maintain predictable lock order. 710 */ 711 if (zv_src < zv_dst || (zv_src == zv_dst && inoff < outoff)) { 712 inlr = zfs_rangelock_enter(&zv_src->zv_rangelock, inoff, len, 713 RL_READER); 714 outlr = zfs_rangelock_enter(&zv_dst->zv_rangelock, outoff, len, 715 RL_WRITER); 716 } else { 717 outlr = zfs_rangelock_enter(&zv_dst->zv_rangelock, outoff, len, 718 RL_WRITER); 719 inlr = zfs_rangelock_enter(&zv_src->zv_rangelock, inoff, len, 720 RL_READER); 721 } 722 723 while (len > 0) { 724 uint64_t size, last_synced_txg; 725 size_t nbps = maxblocks; 726 size = MIN(zv_src->zv_volblocksize * maxblocks, len); 727 last_synced_txg = spa_last_synced_txg( 728 dmu_objset_spa(zv_src->zv_objset)); 729 error = dmu_read_l0_bps(zv_src->zv_objset, ZVOL_OBJ, inoff, 730 size, bps, &nbps); 731 if (error != 0) { 732 /* 733 * If we are trying to clone a block that was created 734 * in the current transaction group, the error will be 735 * EAGAIN here. Based on zfs_bclone_wait_dirty either 736 * return a shortened range to the caller so it can 737 * fallback, or wait for the next TXG and check again. 738 */ 739 if (error == EAGAIN && zfs_bclone_wait_dirty) { 740 txg_wait_synced(dmu_objset_pool 741 (zv_src->zv_objset), last_synced_txg + 1); 742 continue; 743 } 744 break; 745 } 746 747 tx = dmu_tx_create(zv_dst->zv_objset); 748 dmu_tx_hold_clone_by_dnode(tx, zv_dst->zv_dn, outoff, size, 749 zv_src->zv_volblocksize); 750 error = dmu_tx_assign(tx, DMU_TX_WAIT); 751 if (error != 0) { 752 dmu_tx_abort(tx); 753 break; 754 } 755 error = dmu_brt_clone(zv_dst->zv_objset, ZVOL_OBJ, outoff, size, 756 tx, bps, nbps); 757 if (error != 0) { 758 dmu_tx_commit(tx); 759 break; 760 } 761 zvol_log_clone_range(zilog_dst, tx, TX_CLONE_RANGE, outoff, 762 size, zv_src->zv_volblocksize, bps, nbps); 763 dmu_tx_commit(tx); 764 inoff += size; 765 outoff += size; 766 len -= size; 767 } 768 vmem_free(bps, sizeof (bps[0]) * maxblocks); 769 zfs_rangelock_exit(outlr); 770 zfs_rangelock_exit(inlr); 771 if (error == 0 && zv_dst->zv_objset->os_sync == ZFS_SYNC_ALWAYS) { 772 error = zil_commit(zilog_dst, ZVOL_OBJ); 773 } 774 out: 775 if (zv_src != zv_dst) 776 rw_exit(&zv_src->zv_suspend_lock); 777 rw_exit(&zv_dst->zv_suspend_lock); 778 return (error); 779 } 780 781 /* 782 * Handles TX_CLONE_RANGE transactions. 783 */ 784 void 785 zvol_log_clone_range(zilog_t *zilog, dmu_tx_t *tx, int txtype, uint64_t off, 786 uint64_t len, uint64_t blksz, const blkptr_t *bps, size_t nbps) 787 { 788 itx_t *itx; 789 lr_clone_range_t *lr; 790 uint64_t partlen, max_log_data; 791 size_t partnbps; 792 793 if (zil_replaying(zilog, tx)) 794 return; 795 796 max_log_data = zil_max_log_data(zilog, sizeof (lr_clone_range_t)); 797 798 while (nbps > 0) { 799 partnbps = MIN(nbps, max_log_data / sizeof (bps[0])); 800 partlen = partnbps * blksz; 801 ASSERT3U(partlen, <, len + blksz); 802 partlen = MIN(partlen, len); 803 804 itx = zil_itx_create(txtype, 805 sizeof (*lr) + sizeof (bps[0]) * partnbps); 806 lr = (lr_clone_range_t *)&itx->itx_lr; 807 lr->lr_foid = ZVOL_OBJ; 808 lr->lr_offset = off; 809 lr->lr_length = partlen; 810 lr->lr_blksz = blksz; 811 lr->lr_nbps = partnbps; 812 memcpy(lr->lr_bps, bps, sizeof (bps[0]) * partnbps); 813 814 zil_itx_assign(zilog, itx, tx); 815 816 bps += partnbps; 817 ASSERT3U(nbps, >=, partnbps); 818 nbps -= partnbps; 819 off += partlen; 820 ASSERT3U(len, >=, partlen); 821 len -= partlen; 822 } 823 } 824 825 static int 826 zvol_replay_err(void *arg1, void *arg2, boolean_t byteswap) 827 { 828 (void) arg1, (void) arg2, (void) byteswap; 829 return (SET_ERROR(ENOTSUP)); 830 } 831 832 /* 833 * Callback vectors for replaying records. 834 * Only TX_WRITE and TX_TRUNCATE are needed for zvol. 835 */ 836 zil_replay_func_t *const zvol_replay_vector[TX_MAX_TYPE] = { 837 zvol_replay_err, /* no such transaction type */ 838 zvol_replay_err, /* TX_CREATE */ 839 zvol_replay_err, /* TX_MKDIR */ 840 zvol_replay_err, /* TX_MKXATTR */ 841 zvol_replay_err, /* TX_SYMLINK */ 842 zvol_replay_err, /* TX_REMOVE */ 843 zvol_replay_err, /* TX_RMDIR */ 844 zvol_replay_err, /* TX_LINK */ 845 zvol_replay_err, /* TX_RENAME */ 846 zvol_replay_write, /* TX_WRITE */ 847 zvol_replay_truncate, /* TX_TRUNCATE */ 848 zvol_replay_err, /* TX_SETATTR */ 849 zvol_replay_err, /* TX_ACL_V0 */ 850 zvol_replay_err, /* TX_ACL */ 851 zvol_replay_err, /* TX_CREATE_ACL */ 852 zvol_replay_err, /* TX_CREATE_ATTR */ 853 zvol_replay_err, /* TX_CREATE_ACL_ATTR */ 854 zvol_replay_err, /* TX_MKDIR_ACL */ 855 zvol_replay_err, /* TX_MKDIR_ATTR */ 856 zvol_replay_err, /* TX_MKDIR_ACL_ATTR */ 857 zvol_replay_err, /* TX_WRITE2 */ 858 zvol_replay_err, /* TX_SETSAXATTR */ 859 zvol_replay_err, /* TX_RENAME_EXCHANGE */ 860 zvol_replay_err, /* TX_RENAME_WHITEOUT */ 861 zvol_replay_clone_range, /* TX_CLONE_RANGE */ 862 }; 863 864 /* 865 * zvol_log_write() handles TX_WRITE transactions. 866 */ 867 void 868 zvol_log_write(zvol_state_t *zv, dmu_tx_t *tx, uint64_t offset, 869 uint64_t size, boolean_t commit) 870 { 871 uint32_t blocksize = zv->zv_volblocksize; 872 zilog_t *zilog = zv->zv_zilog; 873 itx_wr_state_t write_state; 874 uint64_t log_size = 0; 875 876 if (zil_replaying(zilog, tx)) 877 return; 878 879 write_state = zil_write_state(zilog, size, blocksize, B_FALSE, commit); 880 881 while (size) { 882 itx_t *itx; 883 lr_write_t *lr; 884 itx_wr_state_t wr_state = write_state; 885 ssize_t len = size; 886 887 if (wr_state == WR_COPIED && size > zil_max_copied_data(zilog)) 888 wr_state = WR_NEED_COPY; 889 else if (wr_state == WR_INDIRECT) 890 len = MIN(blocksize - P2PHASE(offset, blocksize), size); 891 892 itx = zil_itx_create(TX_WRITE, sizeof (*lr) + 893 (wr_state == WR_COPIED ? len : 0)); 894 lr = (lr_write_t *)&itx->itx_lr; 895 if (wr_state == WR_COPIED && 896 dmu_read_by_dnode(zv->zv_dn, offset, len, lr + 1, 897 DMU_READ_NO_PREFETCH | DMU_KEEP_CACHING) != 0) { 898 zil_itx_destroy(itx, 0); 899 itx = zil_itx_create(TX_WRITE, sizeof (*lr)); 900 lr = (lr_write_t *)&itx->itx_lr; 901 wr_state = WR_NEED_COPY; 902 } 903 904 log_size += itx->itx_size; 905 if (wr_state == WR_NEED_COPY) 906 log_size += len; 907 908 itx->itx_wr_state = wr_state; 909 lr->lr_foid = ZVOL_OBJ; 910 lr->lr_offset = offset; 911 lr->lr_length = len; 912 lr->lr_blkoff = 0; 913 BP_ZERO(&lr->lr_blkptr); 914 915 itx->itx_private = zv; 916 917 zil_itx_assign(zilog, itx, tx); 918 919 offset += len; 920 size -= len; 921 } 922 923 dsl_pool_wrlog_count(zilog->zl_dmu_pool, log_size, tx->tx_txg); 924 } 925 926 /* 927 * Log a DKIOCFREE/free-long-range to the ZIL with TX_TRUNCATE. 928 */ 929 void 930 zvol_log_truncate(zvol_state_t *zv, dmu_tx_t *tx, uint64_t off, uint64_t len) 931 { 932 itx_t *itx; 933 lr_truncate_t *lr; 934 zilog_t *zilog = zv->zv_zilog; 935 936 if (zil_replaying(zilog, tx)) 937 return; 938 939 itx = zil_itx_create(TX_TRUNCATE, sizeof (*lr)); 940 lr = (lr_truncate_t *)&itx->itx_lr; 941 lr->lr_foid = ZVOL_OBJ; 942 lr->lr_offset = off; 943 lr->lr_length = len; 944 945 zil_itx_assign(zilog, itx, tx); 946 } 947 948 949 static void 950 zvol_get_done(zgd_t *zgd, int error) 951 { 952 (void) error; 953 if (zgd->zgd_db) 954 dmu_buf_rele(zgd->zgd_db, zgd); 955 956 zfs_rangelock_exit(zgd->zgd_lr); 957 958 kmem_free(zgd, sizeof (zgd_t)); 959 } 960 961 /* 962 * Get data to generate a TX_WRITE intent log record. 963 */ 964 int 965 zvol_get_data(void *arg, uint64_t arg2, lr_write_t *lr, char *buf, 966 struct lwb *lwb, zio_t *zio) 967 { 968 zvol_state_t *zv = arg; 969 uint64_t offset = lr->lr_offset; 970 uint64_t size = lr->lr_length; 971 dmu_buf_t *db; 972 zgd_t *zgd; 973 int error; 974 975 ASSERT3P(lwb, !=, NULL); 976 ASSERT3U(size, !=, 0); 977 978 zgd = kmem_zalloc(sizeof (zgd_t), KM_SLEEP); 979 zgd->zgd_lwb = lwb; 980 981 /* 982 * Write records come in two flavors: immediate and indirect. 983 * For small writes it's cheaper to store the data with the 984 * log record (immediate); for large writes it's cheaper to 985 * sync the data and get a pointer to it (indirect) so that 986 * we don't have to write the data twice. 987 */ 988 if (buf != NULL) { /* immediate write */ 989 zgd->zgd_lr = zfs_rangelock_enter(&zv->zv_rangelock, offset, 990 size, RL_READER); 991 error = dmu_read_by_dnode(zv->zv_dn, offset, size, buf, 992 DMU_READ_NO_PREFETCH | DMU_KEEP_CACHING); 993 } else { /* indirect write */ 994 ASSERT3P(zio, !=, NULL); 995 /* 996 * Have to lock the whole block to ensure when it's written out 997 * and its checksum is being calculated that no one can change 998 * the data. Contrarily to zfs_get_data we need not re-check 999 * blocksize after we get the lock because it cannot be changed. 1000 */ 1001 size = zv->zv_volblocksize; 1002 offset = P2ALIGN_TYPED(offset, size, uint64_t); 1003 zgd->zgd_lr = zfs_rangelock_enter(&zv->zv_rangelock, offset, 1004 size, RL_READER); 1005 error = dmu_buf_hold_noread_by_dnode(zv->zv_dn, offset, zgd, 1006 &db); 1007 if (error == 0) { 1008 blkptr_t *bp = &lr->lr_blkptr; 1009 1010 zgd->zgd_db = db; 1011 zgd->zgd_bp = bp; 1012 1013 ASSERT(db != NULL); 1014 ASSERT(db->db_offset == offset); 1015 ASSERT(db->db_size == size); 1016 1017 error = dmu_sync(zio, lr->lr_common.lrc_txg, 1018 zvol_get_done, zgd); 1019 1020 if (error == 0) 1021 return (0); 1022 } 1023 } 1024 1025 zvol_get_done(zgd, error); 1026 1027 return (error); 1028 } 1029 1030 /* 1031 * The zvol_state_t's are inserted into zvol_state_list and zvol_htable. 1032 */ 1033 1034 void 1035 zvol_insert(zvol_state_t *zv) 1036 { 1037 ASSERT(RW_WRITE_HELD(&zvol_state_lock)); 1038 list_insert_head(&zvol_state_list, zv); 1039 hlist_add_head(&zv->zv_hlink, ZVOL_HT_HEAD(zv->zv_hash)); 1040 } 1041 1042 /* 1043 * Simply remove the zvol from to list of zvols. 1044 */ 1045 static void 1046 zvol_remove(zvol_state_t *zv) 1047 { 1048 ASSERT(RW_WRITE_HELD(&zvol_state_lock)); 1049 list_remove(&zvol_state_list, zv); 1050 hlist_del(&zv->zv_hlink); 1051 } 1052 1053 /* 1054 * Setup zv after we just own the zv->objset 1055 */ 1056 static int 1057 zvol_setup_zv(zvol_state_t *zv) 1058 { 1059 uint64_t volsize; 1060 int error; 1061 uint64_t ro; 1062 objset_t *os = zv->zv_objset; 1063 1064 ASSERT(MUTEX_HELD(&zv->zv_state_lock)); 1065 ASSERT(RW_LOCK_HELD(&zv->zv_suspend_lock)); 1066 1067 zv->zv_zilog = NULL; 1068 zv->zv_flags &= ~ZVOL_WRITTEN_TO; 1069 1070 error = dsl_prop_get_integer(zv->zv_name, "readonly", &ro, NULL); 1071 if (error) 1072 return (error); 1073 1074 error = zap_lookup(os, ZVOL_ZAP_OBJ, "size", 8, 1, &volsize); 1075 if (error) 1076 return (error); 1077 1078 error = dnode_hold(os, ZVOL_OBJ, zv, &zv->zv_dn); 1079 if (error) 1080 return (error); 1081 1082 zvol_os_set_capacity(zv, volsize >> 9); 1083 zv->zv_volsize = volsize; 1084 1085 if (ro || dmu_objset_is_snapshot(os) || 1086 !spa_writeable(dmu_objset_spa(os))) { 1087 zvol_os_set_disk_ro(zv, 1); 1088 zv->zv_flags |= ZVOL_RDONLY; 1089 } else { 1090 zvol_os_set_disk_ro(zv, 0); 1091 zv->zv_flags &= ~ZVOL_RDONLY; 1092 } 1093 return (0); 1094 } 1095 1096 /* 1097 * Shutdown every zv_objset related stuff except zv_objset itself. 1098 * The is the reverse of zvol_setup_zv. 1099 */ 1100 static void 1101 zvol_shutdown_zv(zvol_state_t *zv) 1102 { 1103 ASSERT(MUTEX_HELD(&zv->zv_state_lock) && 1104 RW_LOCK_HELD(&zv->zv_suspend_lock)); 1105 1106 if (zv->zv_flags & ZVOL_WRITTEN_TO) { 1107 ASSERT(zv->zv_zilog != NULL); 1108 zil_close(zv->zv_zilog); 1109 } 1110 1111 zv->zv_zilog = NULL; 1112 1113 dnode_rele(zv->zv_dn, zv); 1114 zv->zv_dn = NULL; 1115 1116 /* 1117 * Evict cached data. We must write out any dirty data before 1118 * disowning the dataset. 1119 */ 1120 if (zv->zv_flags & ZVOL_WRITTEN_TO) 1121 txg_wait_synced(dmu_objset_pool(zv->zv_objset), 0); 1122 dmu_objset_evict_dbufs(zv->zv_objset); 1123 } 1124 1125 /* 1126 * return the proper tag for rollback and recv 1127 */ 1128 void * 1129 zvol_tag(zvol_state_t *zv) 1130 { 1131 ASSERT(RW_WRITE_HELD(&zv->zv_suspend_lock)); 1132 return (zv->zv_open_count > 0 ? zv : NULL); 1133 } 1134 1135 /* 1136 * Suspend the zvol for recv and rollback. 1137 */ 1138 zvol_state_t * 1139 zvol_suspend(const char *name) 1140 { 1141 zvol_state_t *zv; 1142 1143 zv = zvol_find_by_name(name, RW_WRITER); 1144 1145 if (zv == NULL) 1146 return (NULL); 1147 1148 /* block all I/O, release in zvol_resume. */ 1149 ASSERT(MUTEX_HELD(&zv->zv_state_lock)); 1150 ASSERT(RW_WRITE_HELD(&zv->zv_suspend_lock)); 1151 1152 atomic_inc(&zv->zv_suspend_ref); 1153 1154 if (zv->zv_open_count > 0) 1155 zvol_shutdown_zv(zv); 1156 1157 /* 1158 * do not hold zv_state_lock across suspend/resume to 1159 * avoid locking up zvol lookups 1160 */ 1161 mutex_exit(&zv->zv_state_lock); 1162 1163 /* zv_suspend_lock is released in zvol_resume() */ 1164 return (zv); 1165 } 1166 1167 int 1168 zvol_resume(zvol_state_t *zv) 1169 { 1170 int error = 0; 1171 1172 ASSERT(RW_WRITE_HELD(&zv->zv_suspend_lock)); 1173 1174 mutex_enter(&zv->zv_state_lock); 1175 1176 if (zv->zv_open_count > 0) { 1177 VERIFY0(dmu_objset_hold(zv->zv_name, zv, &zv->zv_objset)); 1178 VERIFY3P(zv->zv_objset->os_dsl_dataset->ds_owner, ==, zv); 1179 VERIFY(dsl_dataset_long_held(zv->zv_objset->os_dsl_dataset)); 1180 dmu_objset_rele(zv->zv_objset, zv); 1181 1182 error = zvol_setup_zv(zv); 1183 } 1184 1185 mutex_exit(&zv->zv_state_lock); 1186 1187 rw_exit(&zv->zv_suspend_lock); 1188 /* 1189 * We need this because we don't hold zvol_state_lock while releasing 1190 * zv_suspend_lock. zvol_remove_minors_impl thus cannot check 1191 * zv_suspend_lock to determine it is safe to free because rwlock is 1192 * not inherent atomic. 1193 */ 1194 atomic_dec(&zv->zv_suspend_ref); 1195 1196 if (zv->zv_flags & ZVOL_REMOVING) 1197 cv_broadcast(&zv->zv_removing_cv); 1198 1199 return (error); 1200 } 1201 1202 int 1203 zvol_first_open(zvol_state_t *zv, boolean_t readonly) 1204 { 1205 objset_t *os; 1206 int error; 1207 1208 ASSERT(RW_READ_HELD(&zv->zv_suspend_lock)); 1209 ASSERT(MUTEX_HELD(&zv->zv_state_lock)); 1210 ASSERT(mutex_owned(&spa_namespace_lock)); 1211 1212 boolean_t ro = (readonly || (strchr(zv->zv_name, '@') != NULL)); 1213 error = dmu_objset_own(zv->zv_name, DMU_OST_ZVOL, ro, B_TRUE, zv, &os); 1214 if (error) 1215 return (error); 1216 1217 zv->zv_objset = os; 1218 1219 error = zvol_setup_zv(zv); 1220 if (error) { 1221 dmu_objset_disown(os, 1, zv); 1222 zv->zv_objset = NULL; 1223 } 1224 1225 return (error); 1226 } 1227 1228 void 1229 zvol_last_close(zvol_state_t *zv) 1230 { 1231 ASSERT(RW_READ_HELD(&zv->zv_suspend_lock)); 1232 ASSERT(MUTEX_HELD(&zv->zv_state_lock)); 1233 1234 if (zv->zv_flags & ZVOL_REMOVING) 1235 cv_broadcast(&zv->zv_removing_cv); 1236 1237 zvol_shutdown_zv(zv); 1238 1239 dmu_objset_disown(zv->zv_objset, 1, zv); 1240 zv->zv_objset = NULL; 1241 } 1242 1243 typedef struct minors_job { 1244 list_t *list; 1245 list_node_t link; 1246 /* input */ 1247 char *name; 1248 /* output */ 1249 int error; 1250 } minors_job_t; 1251 1252 /* 1253 * Prefetch zvol dnodes for the minors_job 1254 */ 1255 static void 1256 zvol_prefetch_minors_impl(void *arg) 1257 { 1258 minors_job_t *job = arg; 1259 char *dsname = job->name; 1260 objset_t *os = NULL; 1261 1262 job->error = dmu_objset_own(dsname, DMU_OST_ZVOL, B_TRUE, B_TRUE, 1263 FTAG, &os); 1264 if (job->error == 0) { 1265 dmu_prefetch_dnode(os, ZVOL_OBJ, ZIO_PRIORITY_SYNC_READ); 1266 dmu_objset_disown(os, B_TRUE, FTAG); 1267 } 1268 } 1269 1270 /* 1271 * Mask errors to continue dmu_objset_find() traversal 1272 */ 1273 static int 1274 zvol_create_snap_minor_cb(const char *dsname, void *arg) 1275 { 1276 minors_job_t *j = arg; 1277 list_t *minors_list = j->list; 1278 const char *name = j->name; 1279 1280 ASSERT0(MUTEX_HELD(&spa_namespace_lock)); 1281 1282 /* skip the designated dataset */ 1283 if (name && strcmp(dsname, name) == 0) 1284 return (0); 1285 1286 /* at this point, the dsname should name a snapshot */ 1287 if (strchr(dsname, '@') == 0) { 1288 dprintf("zvol_create_snap_minor_cb(): " 1289 "%s is not a snapshot name\n", dsname); 1290 } else { 1291 minors_job_t *job; 1292 char *n = kmem_strdup(dsname); 1293 if (n == NULL) 1294 return (0); 1295 1296 job = kmem_alloc(sizeof (minors_job_t), KM_SLEEP); 1297 job->name = n; 1298 job->list = minors_list; 1299 job->error = 0; 1300 list_insert_tail(minors_list, job); 1301 /* don't care if dispatch fails, because job->error is 0 */ 1302 taskq_dispatch(system_taskq, zvol_prefetch_minors_impl, job, 1303 TQ_SLEEP); 1304 } 1305 1306 return (0); 1307 } 1308 1309 /* 1310 * If spa_keystore_load_wkey() is called for an encrypted zvol, 1311 * we need to look for any clones also using the key. This function 1312 * is "best effort" - so we just skip over it if there are failures. 1313 */ 1314 static void 1315 zvol_add_clones(const char *dsname, list_t *minors_list) 1316 { 1317 /* Also check if it has clones */ 1318 dsl_dir_t *dd = NULL; 1319 dsl_pool_t *dp = NULL; 1320 1321 if (dsl_pool_hold(dsname, FTAG, &dp) != 0) 1322 return; 1323 1324 if (!spa_feature_is_enabled(dp->dp_spa, 1325 SPA_FEATURE_ENCRYPTION)) 1326 goto out; 1327 1328 if (dsl_dir_hold(dp, dsname, FTAG, &dd, NULL) != 0) 1329 goto out; 1330 1331 if (dsl_dir_phys(dd)->dd_clones == 0) 1332 goto out; 1333 1334 zap_cursor_t *zc = kmem_alloc(sizeof (zap_cursor_t), KM_SLEEP); 1335 zap_attribute_t *za = zap_attribute_alloc(); 1336 objset_t *mos = dd->dd_pool->dp_meta_objset; 1337 1338 for (zap_cursor_init(zc, mos, dsl_dir_phys(dd)->dd_clones); 1339 zap_cursor_retrieve(zc, za) == 0; 1340 zap_cursor_advance(zc)) { 1341 dsl_dataset_t *clone; 1342 minors_job_t *job; 1343 1344 if (dsl_dataset_hold_obj(dd->dd_pool, 1345 za->za_first_integer, FTAG, &clone) == 0) { 1346 1347 char name[ZFS_MAX_DATASET_NAME_LEN]; 1348 dsl_dataset_name(clone, name); 1349 1350 char *n = kmem_strdup(name); 1351 job = kmem_alloc(sizeof (minors_job_t), KM_SLEEP); 1352 job->name = n; 1353 job->list = minors_list; 1354 job->error = 0; 1355 list_insert_tail(minors_list, job); 1356 1357 dsl_dataset_rele(clone, FTAG); 1358 } 1359 } 1360 zap_cursor_fini(zc); 1361 zap_attribute_free(za); 1362 kmem_free(zc, sizeof (zap_cursor_t)); 1363 1364 out: 1365 if (dd != NULL) 1366 dsl_dir_rele(dd, FTAG); 1367 dsl_pool_rele(dp, FTAG); 1368 } 1369 1370 /* 1371 * Mask errors to continue dmu_objset_find() traversal 1372 */ 1373 static int 1374 zvol_create_minors_cb(const char *dsname, void *arg) 1375 { 1376 uint64_t snapdev; 1377 int error; 1378 list_t *minors_list = arg; 1379 1380 ASSERT0(MUTEX_HELD(&spa_namespace_lock)); 1381 1382 error = dsl_prop_get_integer(dsname, "snapdev", &snapdev, NULL); 1383 if (error) 1384 return (0); 1385 1386 /* 1387 * Given the name and the 'snapdev' property, create device minor nodes 1388 * with the linkages to zvols/snapshots as needed. 1389 * If the name represents a zvol, create a minor node for the zvol, then 1390 * check if its snapshots are 'visible', and if so, iterate over the 1391 * snapshots and create device minor nodes for those. 1392 */ 1393 if (strchr(dsname, '@') == 0) { 1394 minors_job_t *job; 1395 char *n = kmem_strdup(dsname); 1396 if (n == NULL) 1397 return (0); 1398 1399 job = kmem_alloc(sizeof (minors_job_t), KM_SLEEP); 1400 job->name = n; 1401 job->list = minors_list; 1402 job->error = 0; 1403 list_insert_tail(minors_list, job); 1404 /* don't care if dispatch fails, because job->error is 0 */ 1405 taskq_dispatch(system_taskq, zvol_prefetch_minors_impl, job, 1406 TQ_SLEEP); 1407 1408 zvol_add_clones(dsname, minors_list); 1409 1410 if (snapdev == ZFS_SNAPDEV_VISIBLE) { 1411 /* 1412 * traverse snapshots only, do not traverse children, 1413 * and skip the 'dsname' 1414 */ 1415 (void) dmu_objset_find(dsname, 1416 zvol_create_snap_minor_cb, (void *)job, 1417 DS_FIND_SNAPSHOTS); 1418 } 1419 } else { 1420 dprintf("zvol_create_minors_cb(): %s is not a zvol name\n", 1421 dsname); 1422 } 1423 1424 return (0); 1425 } 1426 1427 static void 1428 zvol_task_update_status(zvol_task_t *task, uint64_t total, uint64_t done, 1429 int error) 1430 { 1431 1432 task->zt_total += total; 1433 task->zt_done += done; 1434 if (task->zt_total != task->zt_done) { 1435 task->zt_status = -1; 1436 if (error) 1437 task->zt_error = error; 1438 } 1439 } 1440 1441 static void 1442 zvol_task_report_status(zvol_task_t *task) 1443 { 1444 #ifdef ZFS_DEBUG 1445 static const char *const msg[] = { 1446 "create", 1447 "remove", 1448 "rename", 1449 "set snapdev", 1450 "set volmode", 1451 "unknown", 1452 }; 1453 1454 if (task->zt_status == 0) 1455 return; 1456 1457 zvol_async_op_t op = MIN(task->zt_op, ZVOL_ASYNC_MAX); 1458 if (task->zt_error) { 1459 dprintf("The %s minors zvol task was not ok, last error %d\n", 1460 msg[op], task->zt_error); 1461 } else { 1462 dprintf("The %s minors zvol task was not ok\n", msg[op]); 1463 } 1464 #else 1465 (void) task; 1466 #endif 1467 } 1468 1469 /* 1470 * Create minors for the specified dataset, including children and snapshots. 1471 * Pay attention to the 'snapdev' property and iterate over the snapshots 1472 * only if they are 'visible'. This approach allows one to assure that the 1473 * snapshot metadata is read from disk only if it is needed. 1474 * 1475 * The name can represent a dataset to be recursively scanned for zvols and 1476 * their snapshots, or a single zvol snapshot. If the name represents a 1477 * dataset, the scan is performed in two nested stages: 1478 * - scan the dataset for zvols, and 1479 * - for each zvol, create a minor node, then check if the zvol's snapshots 1480 * are 'visible', and only then iterate over the snapshots if needed 1481 * 1482 * If the name represents a snapshot, a check is performed if the snapshot is 1483 * 'visible' (which also verifies that the parent is a zvol), and if so, 1484 * a minor node for that snapshot is created. 1485 */ 1486 static void 1487 zvol_create_minors_impl(zvol_task_t *task) 1488 { 1489 const char *name = task->zt_name1; 1490 list_t minors_list; 1491 minors_job_t *job; 1492 uint64_t snapdev; 1493 int total = 0, done = 0, last_error, error; 1494 1495 /* 1496 * Note: the dsl_pool_config_lock must not be held. 1497 * Minor node creation needs to obtain the zvol_state_lock. 1498 * zvol_open() obtains the zvol_state_lock and then the dsl pool 1499 * config lock. Therefore, we can't have the config lock now if 1500 * we are going to wait for the zvol_state_lock, because it 1501 * would be a lock order inversion which could lead to deadlock. 1502 */ 1503 1504 if (zvol_inhibit_dev) { 1505 return; 1506 } 1507 1508 /* 1509 * This is the list for prefetch jobs. Whenever we found a match 1510 * during dmu_objset_find, we insert a minors_job to the list and do 1511 * taskq_dispatch to parallel prefetch zvol dnodes. Note we don't need 1512 * any lock because all list operation is done on the current thread. 1513 * 1514 * We will use this list to do zvol_os_create_minor after prefetch 1515 * so we don't have to traverse using dmu_objset_find again. 1516 */ 1517 list_create(&minors_list, sizeof (minors_job_t), 1518 offsetof(minors_job_t, link)); 1519 1520 1521 if (strchr(name, '@') != NULL) { 1522 error = dsl_prop_get_integer(name, "snapdev", &snapdev, NULL); 1523 if (error == 0 && snapdev == ZFS_SNAPDEV_VISIBLE) { 1524 error = zvol_os_create_minor(name); 1525 if (error == 0) { 1526 done++; 1527 } else { 1528 last_error = error; 1529 } 1530 total++; 1531 } 1532 } else { 1533 fstrans_cookie_t cookie = spl_fstrans_mark(); 1534 (void) dmu_objset_find(name, zvol_create_minors_cb, 1535 &minors_list, DS_FIND_CHILDREN); 1536 spl_fstrans_unmark(cookie); 1537 } 1538 1539 taskq_wait_outstanding(system_taskq, 0); 1540 1541 /* 1542 * Prefetch is completed, we can do zvol_os_create_minor 1543 * sequentially. 1544 */ 1545 while ((job = list_remove_head(&minors_list)) != NULL) { 1546 if (!job->error) { 1547 error = zvol_os_create_minor(job->name); 1548 if (error == 0) { 1549 done++; 1550 } else { 1551 last_error = error; 1552 } 1553 } else if (job->error == EINVAL) { 1554 /* 1555 * The objset, with the name requested by current job 1556 * exist, but have the type different from zvol. 1557 * Just ignore this sort of errors. 1558 */ 1559 done++; 1560 } else { 1561 last_error = job->error; 1562 } 1563 total++; 1564 kmem_strfree(job->name); 1565 kmem_free(job, sizeof (minors_job_t)); 1566 } 1567 1568 list_destroy(&minors_list); 1569 zvol_task_update_status(task, total, done, last_error); 1570 } 1571 1572 /* 1573 * Remove minors for specified dataset including children and snapshots. 1574 */ 1575 1576 /* 1577 * Remove the minor for a given zvol. This will do it all: 1578 * - flag the zvol for removal, so new requests are rejected 1579 * - wait until outstanding requests are completed 1580 * - remove it from lists 1581 * - free it 1582 * It's also usable as a taskq task, and smells nice too. 1583 */ 1584 static void 1585 zvol_remove_minor_task(void *arg) 1586 { 1587 zvol_state_t *zv = (zvol_state_t *)arg; 1588 1589 ASSERT(!RW_LOCK_HELD(&zvol_state_lock)); 1590 ASSERT(!MUTEX_HELD(&zv->zv_state_lock)); 1591 1592 mutex_enter(&zv->zv_state_lock); 1593 while (zv->zv_open_count > 0 || atomic_read(&zv->zv_suspend_ref)) { 1594 zv->zv_flags |= ZVOL_REMOVING; 1595 cv_wait(&zv->zv_removing_cv, &zv->zv_state_lock); 1596 } 1597 mutex_exit(&zv->zv_state_lock); 1598 1599 rw_enter(&zvol_state_lock, RW_WRITER); 1600 mutex_enter(&zv->zv_state_lock); 1601 1602 zvol_remove(zv); 1603 zvol_os_clear_private(zv); 1604 1605 mutex_exit(&zv->zv_state_lock); 1606 rw_exit(&zvol_state_lock); 1607 1608 zvol_os_free(zv); 1609 } 1610 1611 static void 1612 zvol_free_task(void *arg) 1613 { 1614 zvol_os_free(arg); 1615 } 1616 1617 static void 1618 zvol_remove_minors_impl(zvol_task_t *task) 1619 { 1620 zvol_state_t *zv, *zv_next; 1621 const char *name = task ? task->zt_name1 : NULL; 1622 int namelen = ((name) ? strlen(name) : 0); 1623 taskqid_t t; 1624 list_t delay_list, free_list; 1625 1626 if (zvol_inhibit_dev) 1627 return; 1628 1629 list_create(&delay_list, sizeof (zvol_state_t), 1630 offsetof(zvol_state_t, zv_next)); 1631 list_create(&free_list, sizeof (zvol_state_t), 1632 offsetof(zvol_state_t, zv_next)); 1633 1634 rw_enter(&zvol_state_lock, RW_WRITER); 1635 1636 for (zv = list_head(&zvol_state_list); zv != NULL; zv = zv_next) { 1637 zv_next = list_next(&zvol_state_list, zv); 1638 1639 mutex_enter(&zv->zv_state_lock); 1640 if (name == NULL || strcmp(zv->zv_name, name) == 0 || 1641 (strncmp(zv->zv_name, name, namelen) == 0 && 1642 (zv->zv_name[namelen] == '/' || 1643 zv->zv_name[namelen] == '@'))) { 1644 /* 1645 * By holding zv_state_lock here, we guarantee that no 1646 * one is currently using this zv 1647 */ 1648 1649 /* 1650 * If in use, try to throw everyone off and try again 1651 * later. 1652 */ 1653 if (zv->zv_open_count > 0 || 1654 atomic_read(&zv->zv_suspend_ref)) { 1655 zv->zv_flags |= ZVOL_REMOVING; 1656 t = taskq_dispatch( 1657 zv->zv_objset->os_spa->spa_zvol_taskq, 1658 zvol_remove_minor_task, zv, TQ_SLEEP); 1659 if (t == TASKQID_INVALID) { 1660 /* 1661 * Couldn't create the task, so we'll 1662 * do it in place once the loop is 1663 * finished. 1664 */ 1665 list_insert_head(&delay_list, zv); 1666 } 1667 mutex_exit(&zv->zv_state_lock); 1668 continue; 1669 } 1670 1671 zvol_remove(zv); 1672 1673 /* 1674 * Cleared while holding zvol_state_lock as a writer 1675 * which will prevent zvol_open() from opening it. 1676 */ 1677 zvol_os_clear_private(zv); 1678 1679 /* Drop zv_state_lock before zvol_free() */ 1680 mutex_exit(&zv->zv_state_lock); 1681 1682 /* Try parallel zv_free, if failed do it in place */ 1683 t = taskq_dispatch(system_taskq, zvol_free_task, zv, 1684 TQ_SLEEP); 1685 if (t == TASKQID_INVALID) 1686 list_insert_head(&free_list, zv); 1687 } else { 1688 mutex_exit(&zv->zv_state_lock); 1689 } 1690 } 1691 rw_exit(&zvol_state_lock); 1692 1693 /* Wait for zvols that we couldn't create a remove task for */ 1694 while ((zv = list_remove_head(&delay_list)) != NULL) 1695 zvol_remove_minor_task(zv); 1696 1697 /* Free any that we couldn't free in parallel earlier */ 1698 while ((zv = list_remove_head(&free_list)) != NULL) 1699 zvol_os_free(zv); 1700 } 1701 1702 /* Remove minor for this specific volume only */ 1703 static int 1704 zvol_remove_minor_impl(const char *name) 1705 { 1706 zvol_state_t *zv = NULL, *zv_next; 1707 1708 if (zvol_inhibit_dev) 1709 return (0); 1710 1711 rw_enter(&zvol_state_lock, RW_WRITER); 1712 1713 for (zv = list_head(&zvol_state_list); zv != NULL; zv = zv_next) { 1714 zv_next = list_next(&zvol_state_list, zv); 1715 1716 mutex_enter(&zv->zv_state_lock); 1717 if (strcmp(zv->zv_name, name) == 0) 1718 /* Found, leave the the loop with zv_lock held */ 1719 break; 1720 mutex_exit(&zv->zv_state_lock); 1721 } 1722 1723 if (zv == NULL) { 1724 rw_exit(&zvol_state_lock); 1725 return (SET_ERROR(ENOENT)); 1726 } 1727 1728 ASSERT(MUTEX_HELD(&zv->zv_state_lock)); 1729 1730 if (zv->zv_open_count > 0 || atomic_read(&zv->zv_suspend_ref)) { 1731 /* 1732 * In use, so try to throw everyone off, then wait 1733 * until finished. 1734 */ 1735 zv->zv_flags |= ZVOL_REMOVING; 1736 mutex_exit(&zv->zv_state_lock); 1737 rw_exit(&zvol_state_lock); 1738 zvol_remove_minor_task(zv); 1739 return (0); 1740 } 1741 1742 zvol_remove(zv); 1743 zvol_os_clear_private(zv); 1744 1745 mutex_exit(&zv->zv_state_lock); 1746 rw_exit(&zvol_state_lock); 1747 1748 zvol_os_free(zv); 1749 1750 return (0); 1751 } 1752 1753 /* 1754 * Rename minors for specified dataset including children and snapshots. 1755 */ 1756 static void 1757 zvol_rename_minors_impl(zvol_task_t *task) 1758 { 1759 zvol_state_t *zv, *zv_next; 1760 const char *oldname = task->zt_name1; 1761 const char *newname = task->zt_name2; 1762 int total = 0, done = 0, last_error, error, oldnamelen; 1763 1764 if (zvol_inhibit_dev) 1765 return; 1766 1767 oldnamelen = strlen(oldname); 1768 1769 rw_enter(&zvol_state_lock, RW_READER); 1770 1771 for (zv = list_head(&zvol_state_list); zv != NULL; zv = zv_next) { 1772 zv_next = list_next(&zvol_state_list, zv); 1773 1774 mutex_enter(&zv->zv_state_lock); 1775 1776 if (strcmp(zv->zv_name, oldname) == 0) { 1777 error = zvol_os_rename_minor(zv, newname); 1778 } else if (strncmp(zv->zv_name, oldname, oldnamelen) == 0 && 1779 (zv->zv_name[oldnamelen] == '/' || 1780 zv->zv_name[oldnamelen] == '@')) { 1781 char *name = kmem_asprintf("%s%c%s", newname, 1782 zv->zv_name[oldnamelen], 1783 zv->zv_name + oldnamelen + 1); 1784 error = zvol_os_rename_minor(zv, name); 1785 kmem_strfree(name); 1786 } 1787 if (error) { 1788 last_error = error; 1789 } else { 1790 done++; 1791 } 1792 total++; 1793 mutex_exit(&zv->zv_state_lock); 1794 } 1795 1796 rw_exit(&zvol_state_lock); 1797 zvol_task_update_status(task, total, done, last_error); 1798 } 1799 1800 typedef struct zvol_snapdev_cb_arg { 1801 zvol_task_t *task; 1802 uint64_t snapdev; 1803 } zvol_snapdev_cb_arg_t; 1804 1805 static int 1806 zvol_set_snapdev_cb(const char *dsname, void *param) 1807 { 1808 zvol_snapdev_cb_arg_t *arg = param; 1809 int error = 0; 1810 1811 if (strchr(dsname, '@') == NULL) 1812 return (0); 1813 1814 switch (arg->snapdev) { 1815 case ZFS_SNAPDEV_VISIBLE: 1816 error = zvol_os_create_minor(dsname); 1817 break; 1818 case ZFS_SNAPDEV_HIDDEN: 1819 error = zvol_remove_minor_impl(dsname); 1820 break; 1821 } 1822 1823 zvol_task_update_status(arg->task, 1, error == 0, error); 1824 return (0); 1825 } 1826 1827 static void 1828 zvol_set_snapdev_impl(zvol_task_t *task) 1829 { 1830 const char *name = task->zt_name1; 1831 uint64_t snapdev = task->zt_value; 1832 1833 zvol_snapdev_cb_arg_t arg = {task, snapdev}; 1834 fstrans_cookie_t cookie = spl_fstrans_mark(); 1835 /* 1836 * The zvol_set_snapdev_sync() sets snapdev appropriately 1837 * in the dataset hierarchy. Here, we only scan snapshots. 1838 */ 1839 dmu_objset_find(name, zvol_set_snapdev_cb, &arg, DS_FIND_SNAPSHOTS); 1840 spl_fstrans_unmark(cookie); 1841 } 1842 1843 static void 1844 zvol_set_volmode_impl(zvol_task_t *task) 1845 { 1846 const char *name = task->zt_name1; 1847 uint64_t volmode = task->zt_value; 1848 fstrans_cookie_t cookie; 1849 uint64_t old_volmode; 1850 zvol_state_t *zv; 1851 int error; 1852 1853 if (strchr(name, '@') != NULL) 1854 return; 1855 1856 /* 1857 * It's unfortunate we need to remove minors before we create new ones: 1858 * this is necessary because our backing gendisk (zvol_state->zv_disk) 1859 * could be different when we set, for instance, volmode from "geom" 1860 * to "dev" (or vice versa). 1861 */ 1862 zv = zvol_find_by_name(name, RW_NONE); 1863 if (zv == NULL && volmode == ZFS_VOLMODE_NONE) 1864 return; 1865 if (zv != NULL) { 1866 old_volmode = zv->zv_volmode; 1867 mutex_exit(&zv->zv_state_lock); 1868 if (old_volmode == volmode) 1869 return; 1870 zvol_wait_close(zv); 1871 } 1872 cookie = spl_fstrans_mark(); 1873 switch (volmode) { 1874 case ZFS_VOLMODE_NONE: 1875 error = zvol_remove_minor_impl(name); 1876 break; 1877 case ZFS_VOLMODE_GEOM: 1878 case ZFS_VOLMODE_DEV: 1879 error = zvol_remove_minor_impl(name); 1880 /* 1881 * The remove minor function call above, might be not 1882 * needed, if volmode was switched from 'none' value. 1883 * Ignore error in this case. 1884 */ 1885 if (error == ENOENT) 1886 error = 0; 1887 else if (error) 1888 break; 1889 error = zvol_os_create_minor(name); 1890 break; 1891 case ZFS_VOLMODE_DEFAULT: 1892 error = zvol_remove_minor_impl(name); 1893 if (zvol_volmode == ZFS_VOLMODE_NONE) 1894 break; 1895 else /* if zvol_volmode is invalid defaults to "geom" */ 1896 error = zvol_os_create_minor(name); 1897 break; 1898 } 1899 zvol_task_update_status(task, 1, error == 0, error); 1900 spl_fstrans_unmark(cookie); 1901 } 1902 1903 /* 1904 * The worker thread function performed asynchronously. 1905 */ 1906 static void 1907 zvol_task_cb(void *arg) 1908 { 1909 zvol_task_t *task = arg; 1910 1911 switch (task->zt_op) { 1912 case ZVOL_ASYNC_CREATE_MINORS: 1913 zvol_create_minors_impl(task); 1914 break; 1915 case ZVOL_ASYNC_REMOVE_MINORS: 1916 zvol_remove_minors_impl(task); 1917 break; 1918 case ZVOL_ASYNC_RENAME_MINORS: 1919 zvol_rename_minors_impl(task); 1920 break; 1921 case ZVOL_ASYNC_SET_SNAPDEV: 1922 zvol_set_snapdev_impl(task); 1923 break; 1924 case ZVOL_ASYNC_SET_VOLMODE: 1925 zvol_set_volmode_impl(task); 1926 break; 1927 default: 1928 VERIFY(0); 1929 break; 1930 } 1931 1932 zvol_task_report_status(task); 1933 kmem_free(task, sizeof (zvol_task_t)); 1934 } 1935 1936 typedef struct zvol_set_prop_int_arg { 1937 const char *zsda_name; 1938 uint64_t zsda_value; 1939 zprop_source_t zsda_source; 1940 zfs_prop_t zsda_prop; 1941 } zvol_set_prop_int_arg_t; 1942 1943 /* 1944 * Sanity check the dataset for safe use by the sync task. No additional 1945 * conditions are imposed. 1946 */ 1947 static int 1948 zvol_set_common_check(void *arg, dmu_tx_t *tx) 1949 { 1950 zvol_set_prop_int_arg_t *zsda = arg; 1951 dsl_pool_t *dp = dmu_tx_pool(tx); 1952 dsl_dir_t *dd; 1953 int error; 1954 1955 error = dsl_dir_hold(dp, zsda->zsda_name, FTAG, &dd, NULL); 1956 if (error != 0) 1957 return (error); 1958 1959 dsl_dir_rele(dd, FTAG); 1960 1961 return (error); 1962 } 1963 1964 static int 1965 zvol_set_common_sync_cb(dsl_pool_t *dp, dsl_dataset_t *ds, void *arg) 1966 { 1967 zvol_set_prop_int_arg_t *zsda = arg; 1968 char dsname[ZFS_MAX_DATASET_NAME_LEN]; 1969 zvol_task_t *task; 1970 uint64_t prop; 1971 1972 const char *prop_name = zfs_prop_to_name(zsda->zsda_prop); 1973 dsl_dataset_name(ds, dsname); 1974 1975 if (dsl_prop_get_int_ds(ds, prop_name, &prop) != 0) 1976 return (0); 1977 1978 task = kmem_zalloc(sizeof (zvol_task_t), KM_SLEEP); 1979 if (zsda->zsda_prop == ZFS_PROP_VOLMODE) { 1980 task->zt_op = ZVOL_ASYNC_SET_VOLMODE; 1981 } else if (zsda->zsda_prop == ZFS_PROP_SNAPDEV) { 1982 task->zt_op = ZVOL_ASYNC_SET_SNAPDEV; 1983 } else { 1984 kmem_free(task, sizeof (zvol_task_t)); 1985 return (0); 1986 } 1987 task->zt_value = prop; 1988 strlcpy(task->zt_name1, dsname, sizeof (task->zt_name1)); 1989 (void) taskq_dispatch(dp->dp_spa->spa_zvol_taskq, zvol_task_cb, 1990 task, TQ_SLEEP); 1991 return (0); 1992 } 1993 1994 /* 1995 * Traverse all child datasets and apply the property appropriately. 1996 * We call dsl_prop_set_sync_impl() here to set the value only on the toplevel 1997 * dataset and read the effective "property" on every child in the callback 1998 * function: this is because the value is not guaranteed to be the same in the 1999 * whole dataset hierarchy. 2000 */ 2001 static void 2002 zvol_set_common_sync(void *arg, dmu_tx_t *tx) 2003 { 2004 zvol_set_prop_int_arg_t *zsda = arg; 2005 dsl_pool_t *dp = dmu_tx_pool(tx); 2006 dsl_dir_t *dd; 2007 dsl_dataset_t *ds; 2008 int error; 2009 2010 VERIFY0(dsl_dir_hold(dp, zsda->zsda_name, FTAG, &dd, NULL)); 2011 2012 error = dsl_dataset_hold(dp, zsda->zsda_name, FTAG, &ds); 2013 if (error == 0) { 2014 dsl_prop_set_sync_impl(ds, zfs_prop_to_name(zsda->zsda_prop), 2015 zsda->zsda_source, sizeof (zsda->zsda_value), 1, 2016 &zsda->zsda_value, tx); 2017 dsl_dataset_rele(ds, FTAG); 2018 } 2019 2020 dmu_objset_find_dp(dp, dd->dd_object, zvol_set_common_sync_cb, 2021 zsda, DS_FIND_CHILDREN); 2022 2023 dsl_dir_rele(dd, FTAG); 2024 } 2025 2026 int 2027 zvol_set_common(const char *ddname, zfs_prop_t prop, zprop_source_t source, 2028 uint64_t val) 2029 { 2030 zvol_set_prop_int_arg_t zsda; 2031 2032 zsda.zsda_name = ddname; 2033 zsda.zsda_source = source; 2034 zsda.zsda_value = val; 2035 zsda.zsda_prop = prop; 2036 2037 return (dsl_sync_task(ddname, zvol_set_common_check, 2038 zvol_set_common_sync, &zsda, 0, ZFS_SPACE_CHECK_NONE)); 2039 } 2040 2041 void 2042 zvol_create_minors(const char *name) 2043 { 2044 spa_t *spa; 2045 zvol_task_t *task; 2046 taskqid_t id; 2047 2048 if (spa_open(name, &spa, FTAG) != 0) 2049 return; 2050 2051 task = kmem_zalloc(sizeof (zvol_task_t), KM_SLEEP); 2052 task->zt_op = ZVOL_ASYNC_CREATE_MINORS; 2053 strlcpy(task->zt_name1, name, sizeof (task->zt_name1)); 2054 id = taskq_dispatch(spa->spa_zvol_taskq, zvol_task_cb, task, TQ_SLEEP); 2055 if (id != TASKQID_INVALID) 2056 taskq_wait_id(spa->spa_zvol_taskq, id); 2057 2058 spa_close(spa, FTAG); 2059 } 2060 2061 void 2062 zvol_remove_minors(spa_t *spa, const char *name, boolean_t async) 2063 { 2064 zvol_task_t *task; 2065 taskqid_t id; 2066 2067 task = kmem_zalloc(sizeof (zvol_task_t), KM_SLEEP); 2068 task->zt_op = ZVOL_ASYNC_REMOVE_MINORS; 2069 strlcpy(task->zt_name1, name, sizeof (task->zt_name1)); 2070 id = taskq_dispatch(spa->spa_zvol_taskq, zvol_task_cb, task, TQ_SLEEP); 2071 if ((async == B_FALSE) && (id != TASKQID_INVALID)) 2072 taskq_wait_id(spa->spa_zvol_taskq, id); 2073 } 2074 2075 void 2076 zvol_rename_minors(spa_t *spa, const char *name1, const char *name2, 2077 boolean_t async) 2078 { 2079 zvol_task_t *task; 2080 taskqid_t id; 2081 2082 task = kmem_zalloc(sizeof (zvol_task_t), KM_SLEEP); 2083 task->zt_op = ZVOL_ASYNC_RENAME_MINORS; 2084 strlcpy(task->zt_name1, name1, sizeof (task->zt_name1)); 2085 strlcpy(task->zt_name2, name2, sizeof (task->zt_name2)); 2086 id = taskq_dispatch(spa->spa_zvol_taskq, zvol_task_cb, task, TQ_SLEEP); 2087 if ((async == B_FALSE) && (id != TASKQID_INVALID)) 2088 taskq_wait_id(spa->spa_zvol_taskq, id); 2089 } 2090 2091 boolean_t 2092 zvol_is_zvol(const char *name) 2093 { 2094 2095 return (zvol_os_is_zvol(name)); 2096 } 2097 2098 int 2099 zvol_init_impl(void) 2100 { 2101 int i; 2102 2103 /* 2104 * zvol_threads is the module param the user passes in. 2105 * 2106 * zvol_actual_threads is what we use internally, since the user can 2107 * pass zvol_thread = 0 to mean "use all the CPUs" (the default). 2108 */ 2109 static unsigned int zvol_actual_threads; 2110 2111 if (zvol_threads == 0) { 2112 /* 2113 * See dde9380a1 for why 32 was chosen here. This should 2114 * probably be refined to be some multiple of the number 2115 * of CPUs. 2116 */ 2117 zvol_actual_threads = MAX(max_ncpus, 32); 2118 } else { 2119 zvol_actual_threads = MIN(MAX(zvol_threads, 1), 1024); 2120 } 2121 2122 /* 2123 * Use at least 32 zvol_threads but for many core system, 2124 * prefer 6 threads per taskq, but no more taskqs 2125 * than threads in them on large systems. 2126 * 2127 * taskq total 2128 * cpus taskqs threads threads 2129 * ------- ------- ------- ------- 2130 * 1 1 32 32 2131 * 2 1 32 32 2132 * 4 1 32 32 2133 * 8 2 16 32 2134 * 16 3 11 33 2135 * 32 5 7 35 2136 * 64 8 8 64 2137 * 128 11 12 132 2138 * 256 16 16 256 2139 */ 2140 zv_taskq_t *ztqs = &zvol_taskqs; 2141 int num_tqs = MIN(max_ncpus, zvol_num_taskqs); 2142 if (num_tqs == 0) { 2143 num_tqs = 1 + max_ncpus / 6; 2144 while (num_tqs * num_tqs > zvol_actual_threads) 2145 num_tqs--; 2146 } 2147 2148 int per_tq_thread = zvol_actual_threads / num_tqs; 2149 if (per_tq_thread * num_tqs < zvol_actual_threads) 2150 per_tq_thread++; 2151 2152 ztqs->tqs_cnt = num_tqs; 2153 ztqs->tqs_taskq = kmem_alloc(num_tqs * sizeof (taskq_t *), KM_SLEEP); 2154 2155 for (uint_t i = 0; i < num_tqs; i++) { 2156 char name[32]; 2157 (void) snprintf(name, sizeof (name), "%s_tq-%u", 2158 ZVOL_DRIVER, i); 2159 ztqs->tqs_taskq[i] = taskq_create(name, per_tq_thread, 2160 maxclsyspri, per_tq_thread, INT_MAX, 2161 TASKQ_PREPOPULATE | TASKQ_DYNAMIC); 2162 if (ztqs->tqs_taskq[i] == NULL) { 2163 for (int j = i - 1; j >= 0; j--) 2164 taskq_destroy(ztqs->tqs_taskq[j]); 2165 kmem_free(ztqs->tqs_taskq, ztqs->tqs_cnt * 2166 sizeof (taskq_t *)); 2167 ztqs->tqs_taskq = NULL; 2168 return (SET_ERROR(ENOMEM)); 2169 } 2170 } 2171 2172 list_create(&zvol_state_list, sizeof (zvol_state_t), 2173 offsetof(zvol_state_t, zv_next)); 2174 rw_init(&zvol_state_lock, NULL, RW_DEFAULT, NULL); 2175 2176 zvol_htable = kmem_alloc(ZVOL_HT_SIZE * sizeof (struct hlist_head), 2177 KM_SLEEP); 2178 for (i = 0; i < ZVOL_HT_SIZE; i++) 2179 INIT_HLIST_HEAD(&zvol_htable[i]); 2180 2181 return (0); 2182 } 2183 2184 void 2185 zvol_fini_impl(void) 2186 { 2187 zv_taskq_t *ztqs = &zvol_taskqs; 2188 2189 zvol_remove_minors_impl(NULL); 2190 2191 /* 2192 * The call to "zvol_remove_minors_impl" may dispatch entries to 2193 * the system_taskq, but it doesn't wait for those entries to 2194 * complete before it returns. Thus, we must wait for all of the 2195 * removals to finish, before we can continue. 2196 */ 2197 taskq_wait_outstanding(system_taskq, 0); 2198 2199 kmem_free(zvol_htable, ZVOL_HT_SIZE * sizeof (struct hlist_head)); 2200 list_destroy(&zvol_state_list); 2201 rw_destroy(&zvol_state_lock); 2202 2203 if (ztqs->tqs_taskq == NULL) { 2204 ASSERT0(ztqs->tqs_cnt); 2205 } else { 2206 for (uint_t i = 0; i < ztqs->tqs_cnt; i++) { 2207 ASSERT3P(ztqs->tqs_taskq[i], !=, NULL); 2208 taskq_destroy(ztqs->tqs_taskq[i]); 2209 } 2210 kmem_free(ztqs->tqs_taskq, ztqs->tqs_cnt * 2211 sizeof (taskq_t *)); 2212 ztqs->tqs_taskq = NULL; 2213 } 2214 } 2215 2216 ZFS_MODULE_PARAM(zfs_vol, zvol_, inhibit_dev, UINT, ZMOD_RW, 2217 "Do not create zvol device nodes"); 2218 ZFS_MODULE_PARAM(zfs_vol, zvol_, prefetch_bytes, UINT, ZMOD_RW, 2219 "Prefetch N bytes at zvol start+end"); 2220 ZFS_MODULE_PARAM(zfs_vol, zvol_vol, mode, UINT, ZMOD_RW, 2221 "Default volmode property value"); 2222 ZFS_MODULE_PARAM(zfs_vol, zvol_, threads, UINT, ZMOD_RW, 2223 "Number of threads for I/O requests. Set to 0 to use all active CPUs"); 2224 ZFS_MODULE_PARAM(zfs_vol, zvol_, num_taskqs, UINT, ZMOD_RW, 2225 "Number of zvol taskqs"); 2226 ZFS_MODULE_PARAM(zfs_vol, zvol_, request_sync, UINT, ZMOD_RW, 2227 "Synchronously handle bio requests"); 2228