1 // SPDX-License-Identifier: CDDL-1.0 2 /* 3 * CDDL HEADER START 4 * 5 * The contents of this file are subject to the terms of the 6 * Common Development and Distribution License (the "License"). 7 * You may not use this file except in compliance with the License. 8 * 9 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 10 * or https://opensource.org/licenses/CDDL-1.0. 11 * See the License for the specific language governing permissions 12 * and limitations under the License. 13 * 14 * When distributing Covered Code, include this CDDL HEADER in each 15 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 16 * If applicable, add the following below this CDDL HEADER, with the 17 * fields enclosed by brackets "[]" replaced with your own identifying 18 * information: Portions Copyright [yyyy] [name of copyright owner] 19 * 20 * CDDL HEADER END 21 */ 22 /* 23 * Copyright (C) 2008-2010 Lawrence Livermore National Security, LLC. 24 * Produced at Lawrence Livermore National Laboratory (cf, DISCLAIMER). 25 * Rewritten for Linux by Brian Behlendorf <behlendorf1@llnl.gov>. 26 * LLNL-CODE-403049. 27 * 28 * ZFS volume emulation driver. 29 * 30 * Makes a DMU object look like a volume of arbitrary size, up to 2^64 bytes. 31 * Volumes are accessed through the symbolic links named: 32 * 33 * /dev/<pool_name>/<dataset_name> 34 * 35 * Volumes are persistent through reboot and module load. No user command 36 * needs to be run before opening and using a device. 37 * 38 * Copyright 2014 Nexenta Systems, Inc. All rights reserved. 39 * Copyright (c) 2016 Actifio, Inc. All rights reserved. 40 * Copyright (c) 2012, 2019 by Delphix. All rights reserved. 41 * Copyright (c) 2024, Klara, Inc. 42 */ 43 44 /* 45 * Note on locking of zvol state structures. 46 * 47 * These structures are used to maintain internal state used to emulate block 48 * devices on top of zvols. In particular, management of device minor number 49 * operations - create, remove, rename, and set_snapdev - involves access to 50 * these structures. The zvol_state_lock is primarily used to protect the 51 * zvol_state_list. The zv->zv_state_lock is used to protect the contents 52 * of the zvol_state_t structures, as well as to make sure that when the 53 * time comes to remove the structure from the list, it is not in use, and 54 * therefore, it can be taken off zvol_state_list and freed. 55 * 56 * The zv_suspend_lock was introduced to allow for suspending I/O to a zvol, 57 * e.g. for the duration of receive and rollback operations. This lock can be 58 * held for significant periods of time. Given that it is undesirable to hold 59 * mutexes for long periods of time, the following lock ordering applies: 60 * - take zvol_state_lock if necessary, to protect zvol_state_list 61 * - take zv_suspend_lock if necessary, by the code path in question 62 * - take zv_state_lock to protect zvol_state_t 63 * 64 * The minor operations are issued to spa->spa_zvol_taskq queues, that are 65 * single-threaded (to preserve order of minor operations), and are executed 66 * through the zvol_task_cb that dispatches the specific operations. Therefore, 67 * these operations are serialized per pool. Consequently, we can be certain 68 * that for a given zvol, there is only one operation at a time in progress. 69 * That is why one can be sure that first, zvol_state_t for a given zvol is 70 * allocated and placed on zvol_state_list, and then other minor operations 71 * for this zvol are going to proceed in the order of issue. 72 * 73 */ 74 75 #include <sys/dataset_kstats.h> 76 #include <sys/dbuf.h> 77 #include <sys/dmu_traverse.h> 78 #include <sys/dsl_dataset.h> 79 #include <sys/dsl_prop.h> 80 #include <sys/dsl_dir.h> 81 #include <sys/zap.h> 82 #include <sys/zfeature.h> 83 #include <sys/zil_impl.h> 84 #include <sys/dmu_tx.h> 85 #include <sys/zio.h> 86 #include <sys/zfs_rlock.h> 87 #include <sys/spa_impl.h> 88 #include <sys/zvol.h> 89 #include <sys/zvol_impl.h> 90 91 unsigned int zvol_inhibit_dev = 0; 92 unsigned int zvol_prefetch_bytes = (128 * 1024); 93 unsigned int zvol_volmode = ZFS_VOLMODE_GEOM; 94 unsigned int zvol_threads = 0; 95 unsigned int zvol_num_taskqs = 0; 96 unsigned int zvol_request_sync = 0; 97 98 struct hlist_head *zvol_htable; 99 static list_t zvol_state_list; 100 krwlock_t zvol_state_lock; 101 extern int zfs_bclone_wait_dirty; 102 zv_taskq_t zvol_taskqs; 103 104 typedef enum { 105 ZVOL_ASYNC_REMOVE_MINORS, 106 ZVOL_ASYNC_RENAME_MINORS, 107 ZVOL_ASYNC_SET_SNAPDEV, 108 ZVOL_ASYNC_SET_VOLMODE, 109 ZVOL_ASYNC_MAX 110 } zvol_async_op_t; 111 112 typedef struct { 113 zvol_async_op_t op; 114 char name1[MAXNAMELEN]; 115 char name2[MAXNAMELEN]; 116 uint64_t value; 117 } zvol_task_t; 118 119 zv_request_task_t * 120 zv_request_task_create(zv_request_t zvr) 121 { 122 zv_request_task_t *task; 123 task = kmem_alloc(sizeof (zv_request_task_t), KM_SLEEP); 124 taskq_init_ent(&task->ent); 125 task->zvr = zvr; 126 return (task); 127 } 128 129 void 130 zv_request_task_free(zv_request_task_t *task) 131 { 132 kmem_free(task, sizeof (*task)); 133 } 134 135 uint64_t 136 zvol_name_hash(const char *name) 137 { 138 uint64_t crc = -1ULL; 139 ASSERT(zfs_crc64_table[128] == ZFS_CRC64_POLY); 140 for (const uint8_t *p = (const uint8_t *)name; *p != 0; p++) 141 crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ (*p)) & 0xFF]; 142 return (crc); 143 } 144 145 /* 146 * Find a zvol_state_t given the name and hash generated by zvol_name_hash. 147 * If found, return with zv_suspend_lock and zv_state_lock taken, otherwise, 148 * return (NULL) without the taking locks. The zv_suspend_lock is always taken 149 * before zv_state_lock. The mode argument indicates the mode (including none) 150 * for zv_suspend_lock to be taken. 151 */ 152 zvol_state_t * 153 zvol_find_by_name_hash(const char *name, uint64_t hash, int mode) 154 { 155 zvol_state_t *zv; 156 struct hlist_node *p = NULL; 157 158 rw_enter(&zvol_state_lock, RW_READER); 159 hlist_for_each(p, ZVOL_HT_HEAD(hash)) { 160 zv = hlist_entry(p, zvol_state_t, zv_hlink); 161 mutex_enter(&zv->zv_state_lock); 162 if (zv->zv_hash == hash && strcmp(zv->zv_name, name) == 0) { 163 /* 164 * this is the right zvol, take the locks in the 165 * right order 166 */ 167 if (mode != RW_NONE && 168 !rw_tryenter(&zv->zv_suspend_lock, mode)) { 169 mutex_exit(&zv->zv_state_lock); 170 rw_enter(&zv->zv_suspend_lock, mode); 171 mutex_enter(&zv->zv_state_lock); 172 /* 173 * zvol cannot be renamed as we continue 174 * to hold zvol_state_lock 175 */ 176 ASSERT(zv->zv_hash == hash && 177 strcmp(zv->zv_name, name) == 0); 178 } 179 rw_exit(&zvol_state_lock); 180 return (zv); 181 } 182 mutex_exit(&zv->zv_state_lock); 183 } 184 rw_exit(&zvol_state_lock); 185 186 return (NULL); 187 } 188 189 /* 190 * Find a zvol_state_t given the name. 191 * If found, return with zv_suspend_lock and zv_state_lock taken, otherwise, 192 * return (NULL) without the taking locks. The zv_suspend_lock is always taken 193 * before zv_state_lock. The mode argument indicates the mode (including none) 194 * for zv_suspend_lock to be taken. 195 */ 196 static zvol_state_t * 197 zvol_find_by_name(const char *name, int mode) 198 { 199 return (zvol_find_by_name_hash(name, zvol_name_hash(name), mode)); 200 } 201 202 /* 203 * ZFS_IOC_CREATE callback handles dmu zvol and zap object creation. 204 */ 205 void 206 zvol_create_cb(objset_t *os, void *arg, cred_t *cr, dmu_tx_t *tx) 207 { 208 zfs_creat_t *zct = arg; 209 nvlist_t *nvprops = zct->zct_props; 210 int error; 211 uint64_t volblocksize, volsize; 212 213 VERIFY(nvlist_lookup_uint64(nvprops, 214 zfs_prop_to_name(ZFS_PROP_VOLSIZE), &volsize) == 0); 215 if (nvlist_lookup_uint64(nvprops, 216 zfs_prop_to_name(ZFS_PROP_VOLBLOCKSIZE), &volblocksize) != 0) 217 volblocksize = zfs_prop_default_numeric(ZFS_PROP_VOLBLOCKSIZE); 218 219 /* 220 * These properties must be removed from the list so the generic 221 * property setting step won't apply to them. 222 */ 223 VERIFY(nvlist_remove_all(nvprops, 224 zfs_prop_to_name(ZFS_PROP_VOLSIZE)) == 0); 225 (void) nvlist_remove_all(nvprops, 226 zfs_prop_to_name(ZFS_PROP_VOLBLOCKSIZE)); 227 228 error = dmu_object_claim(os, ZVOL_OBJ, DMU_OT_ZVOL, volblocksize, 229 DMU_OT_NONE, 0, tx); 230 ASSERT(error == 0); 231 232 error = zap_create_claim(os, ZVOL_ZAP_OBJ, DMU_OT_ZVOL_PROP, 233 DMU_OT_NONE, 0, tx); 234 ASSERT(error == 0); 235 236 error = zap_update(os, ZVOL_ZAP_OBJ, "size", 8, 1, &volsize, tx); 237 ASSERT(error == 0); 238 } 239 240 /* 241 * ZFS_IOC_OBJSET_STATS entry point. 242 */ 243 int 244 zvol_get_stats(objset_t *os, nvlist_t *nv) 245 { 246 int error; 247 dmu_object_info_t *doi; 248 uint64_t val; 249 250 error = zap_lookup(os, ZVOL_ZAP_OBJ, "size", 8, 1, &val); 251 if (error) 252 return (SET_ERROR(error)); 253 254 dsl_prop_nvlist_add_uint64(nv, ZFS_PROP_VOLSIZE, val); 255 doi = kmem_alloc(sizeof (dmu_object_info_t), KM_SLEEP); 256 error = dmu_object_info(os, ZVOL_OBJ, doi); 257 258 if (error == 0) { 259 dsl_prop_nvlist_add_uint64(nv, ZFS_PROP_VOLBLOCKSIZE, 260 doi->doi_data_block_size); 261 } 262 263 kmem_free(doi, sizeof (dmu_object_info_t)); 264 265 return (SET_ERROR(error)); 266 } 267 268 /* 269 * Sanity check volume size. 270 */ 271 int 272 zvol_check_volsize(uint64_t volsize, uint64_t blocksize) 273 { 274 if (volsize == 0) 275 return (SET_ERROR(EINVAL)); 276 277 if (volsize % blocksize != 0) 278 return (SET_ERROR(EINVAL)); 279 280 #ifdef _ILP32 281 if (volsize - 1 > SPEC_MAXOFFSET_T) 282 return (SET_ERROR(EOVERFLOW)); 283 #endif 284 return (0); 285 } 286 287 /* 288 * Ensure the zap is flushed then inform the VFS of the capacity change. 289 */ 290 static int 291 zvol_update_volsize(uint64_t volsize, objset_t *os) 292 { 293 dmu_tx_t *tx; 294 int error; 295 uint64_t txg; 296 297 tx = dmu_tx_create(os); 298 dmu_tx_hold_zap(tx, ZVOL_ZAP_OBJ, TRUE, NULL); 299 dmu_tx_mark_netfree(tx); 300 error = dmu_tx_assign(tx, DMU_TX_WAIT); 301 if (error) { 302 dmu_tx_abort(tx); 303 return (SET_ERROR(error)); 304 } 305 txg = dmu_tx_get_txg(tx); 306 307 error = zap_update(os, ZVOL_ZAP_OBJ, "size", 8, 1, 308 &volsize, tx); 309 dmu_tx_commit(tx); 310 311 txg_wait_synced(dmu_objset_pool(os), txg); 312 313 if (error == 0) 314 error = dmu_free_long_range(os, 315 ZVOL_OBJ, volsize, DMU_OBJECT_END); 316 317 return (error); 318 } 319 320 /* 321 * Set ZFS_PROP_VOLSIZE set entry point. Note that modifying the volume 322 * size will result in a udev "change" event being generated. 323 */ 324 int 325 zvol_set_volsize(const char *name, uint64_t volsize) 326 { 327 objset_t *os = NULL; 328 uint64_t readonly; 329 int error; 330 boolean_t owned = B_FALSE; 331 332 error = dsl_prop_get_integer(name, 333 zfs_prop_to_name(ZFS_PROP_READONLY), &readonly, NULL); 334 if (error != 0) 335 return (SET_ERROR(error)); 336 if (readonly) 337 return (SET_ERROR(EROFS)); 338 339 zvol_state_t *zv = zvol_find_by_name(name, RW_READER); 340 341 ASSERT(zv == NULL || (MUTEX_HELD(&zv->zv_state_lock) && 342 RW_READ_HELD(&zv->zv_suspend_lock))); 343 344 if (zv == NULL || zv->zv_objset == NULL) { 345 if (zv != NULL) 346 rw_exit(&zv->zv_suspend_lock); 347 if ((error = dmu_objset_own(name, DMU_OST_ZVOL, B_FALSE, B_TRUE, 348 FTAG, &os)) != 0) { 349 if (zv != NULL) 350 mutex_exit(&zv->zv_state_lock); 351 return (SET_ERROR(error)); 352 } 353 owned = B_TRUE; 354 if (zv != NULL) 355 zv->zv_objset = os; 356 } else { 357 os = zv->zv_objset; 358 } 359 360 dmu_object_info_t *doi = kmem_alloc(sizeof (*doi), KM_SLEEP); 361 362 if ((error = dmu_object_info(os, ZVOL_OBJ, doi)) || 363 (error = zvol_check_volsize(volsize, doi->doi_data_block_size))) 364 goto out; 365 366 error = zvol_update_volsize(volsize, os); 367 if (error == 0 && zv != NULL) { 368 zv->zv_volsize = volsize; 369 zv->zv_changed = 1; 370 } 371 out: 372 kmem_free(doi, sizeof (dmu_object_info_t)); 373 374 if (owned) { 375 dmu_objset_disown(os, B_TRUE, FTAG); 376 if (zv != NULL) 377 zv->zv_objset = NULL; 378 } else { 379 rw_exit(&zv->zv_suspend_lock); 380 } 381 382 if (zv != NULL) 383 mutex_exit(&zv->zv_state_lock); 384 385 if (error == 0 && zv != NULL) 386 zvol_os_update_volsize(zv, volsize); 387 388 return (SET_ERROR(error)); 389 } 390 391 /* 392 * Update volthreading. 393 */ 394 int 395 zvol_set_volthreading(const char *name, boolean_t value) 396 { 397 zvol_state_t *zv = zvol_find_by_name(name, RW_NONE); 398 if (zv == NULL) 399 return (ENOENT); 400 zv->zv_threading = value; 401 mutex_exit(&zv->zv_state_lock); 402 return (0); 403 } 404 405 /* 406 * Update zvol ro property. 407 */ 408 int 409 zvol_set_ro(const char *name, boolean_t value) 410 { 411 zvol_state_t *zv = zvol_find_by_name(name, RW_NONE); 412 if (zv == NULL) 413 return (-1); 414 if (value) { 415 zvol_os_set_disk_ro(zv, 1); 416 zv->zv_flags |= ZVOL_RDONLY; 417 } else { 418 zvol_os_set_disk_ro(zv, 0); 419 zv->zv_flags &= ~ZVOL_RDONLY; 420 } 421 mutex_exit(&zv->zv_state_lock); 422 return (0); 423 } 424 425 /* 426 * Sanity check volume block size. 427 */ 428 int 429 zvol_check_volblocksize(const char *name, uint64_t volblocksize) 430 { 431 /* Record sizes above 128k need the feature to be enabled */ 432 if (volblocksize > SPA_OLD_MAXBLOCKSIZE) { 433 spa_t *spa; 434 int error; 435 436 if ((error = spa_open(name, &spa, FTAG)) != 0) 437 return (error); 438 439 if (!spa_feature_is_enabled(spa, SPA_FEATURE_LARGE_BLOCKS)) { 440 spa_close(spa, FTAG); 441 return (SET_ERROR(ENOTSUP)); 442 } 443 444 /* 445 * We don't allow setting the property above 1MB, 446 * unless the tunable has been changed. 447 */ 448 if (volblocksize > zfs_max_recordsize) 449 return (SET_ERROR(EDOM)); 450 451 spa_close(spa, FTAG); 452 } 453 454 if (volblocksize < SPA_MINBLOCKSIZE || 455 volblocksize > SPA_MAXBLOCKSIZE || 456 !ISP2(volblocksize)) 457 return (SET_ERROR(EDOM)); 458 459 return (0); 460 } 461 462 /* 463 * Replay a TX_TRUNCATE ZIL transaction if asked. TX_TRUNCATE is how we 464 * implement DKIOCFREE/free-long-range. 465 */ 466 static int 467 zvol_replay_truncate(void *arg1, void *arg2, boolean_t byteswap) 468 { 469 zvol_state_t *zv = arg1; 470 lr_truncate_t *lr = arg2; 471 uint64_t offset, length; 472 473 ASSERT3U(lr->lr_common.lrc_reclen, >=, sizeof (*lr)); 474 475 if (byteswap) 476 byteswap_uint64_array(lr, sizeof (*lr)); 477 478 offset = lr->lr_offset; 479 length = lr->lr_length; 480 481 dmu_tx_t *tx = dmu_tx_create(zv->zv_objset); 482 dmu_tx_mark_netfree(tx); 483 int error = dmu_tx_assign(tx, DMU_TX_WAIT); 484 if (error != 0) { 485 dmu_tx_abort(tx); 486 } else { 487 (void) zil_replaying(zv->zv_zilog, tx); 488 dmu_tx_commit(tx); 489 error = dmu_free_long_range(zv->zv_objset, ZVOL_OBJ, offset, 490 length); 491 } 492 493 return (error); 494 } 495 496 /* 497 * Replay a TX_WRITE ZIL transaction that didn't get committed 498 * after a system failure 499 */ 500 static int 501 zvol_replay_write(void *arg1, void *arg2, boolean_t byteswap) 502 { 503 zvol_state_t *zv = arg1; 504 lr_write_t *lr = arg2; 505 objset_t *os = zv->zv_objset; 506 char *data = (char *)(lr + 1); /* data follows lr_write_t */ 507 uint64_t offset, length; 508 dmu_tx_t *tx; 509 int error; 510 511 ASSERT3U(lr->lr_common.lrc_reclen, >=, sizeof (*lr)); 512 513 if (byteswap) 514 byteswap_uint64_array(lr, sizeof (*lr)); 515 516 offset = lr->lr_offset; 517 length = lr->lr_length; 518 519 /* If it's a dmu_sync() block, write the whole block */ 520 if (lr->lr_common.lrc_reclen == sizeof (lr_write_t)) { 521 uint64_t blocksize = BP_GET_LSIZE(&lr->lr_blkptr); 522 if (length < blocksize) { 523 offset -= offset % blocksize; 524 length = blocksize; 525 } 526 } 527 528 tx = dmu_tx_create(os); 529 dmu_tx_hold_write(tx, ZVOL_OBJ, offset, length); 530 error = dmu_tx_assign(tx, DMU_TX_WAIT); 531 if (error) { 532 dmu_tx_abort(tx); 533 } else { 534 dmu_write(os, ZVOL_OBJ, offset, length, data, tx); 535 (void) zil_replaying(zv->zv_zilog, tx); 536 dmu_tx_commit(tx); 537 } 538 539 return (error); 540 } 541 542 /* 543 * Replay a TX_CLONE_RANGE ZIL transaction that didn't get committed 544 * after a system failure 545 */ 546 static int 547 zvol_replay_clone_range(void *arg1, void *arg2, boolean_t byteswap) 548 { 549 zvol_state_t *zv = arg1; 550 lr_clone_range_t *lr = arg2; 551 objset_t *os = zv->zv_objset; 552 dmu_tx_t *tx; 553 int error; 554 uint64_t blksz; 555 uint64_t off; 556 uint64_t len; 557 558 ASSERT3U(lr->lr_common.lrc_reclen, >=, sizeof (*lr)); 559 ASSERT3U(lr->lr_common.lrc_reclen, >=, offsetof(lr_clone_range_t, 560 lr_bps[lr->lr_nbps])); 561 562 if (byteswap) 563 byteswap_uint64_array(lr, sizeof (*lr)); 564 565 ASSERT(spa_feature_is_enabled(dmu_objset_spa(os), 566 SPA_FEATURE_BLOCK_CLONING)); 567 568 off = lr->lr_offset; 569 len = lr->lr_length; 570 blksz = lr->lr_blksz; 571 572 if ((off % blksz) != 0) { 573 return (SET_ERROR(EINVAL)); 574 } 575 576 error = dnode_hold(os, ZVOL_OBJ, zv, &zv->zv_dn); 577 if (error != 0 || !zv->zv_dn) 578 return (error); 579 tx = dmu_tx_create(os); 580 dmu_tx_hold_clone_by_dnode(tx, zv->zv_dn, off, len, blksz); 581 error = dmu_tx_assign(tx, DMU_TX_WAIT); 582 if (error != 0) { 583 dmu_tx_abort(tx); 584 goto out; 585 } 586 error = dmu_brt_clone(zv->zv_objset, ZVOL_OBJ, off, len, 587 tx, lr->lr_bps, lr->lr_nbps); 588 if (error != 0) { 589 dmu_tx_commit(tx); 590 goto out; 591 } 592 593 /* 594 * zil_replaying() not only check if we are replaying ZIL, but also 595 * updates the ZIL header to record replay progress. 596 */ 597 VERIFY(zil_replaying(zv->zv_zilog, tx)); 598 dmu_tx_commit(tx); 599 600 out: 601 dnode_rele(zv->zv_dn, zv); 602 zv->zv_dn = NULL; 603 return (error); 604 } 605 606 int 607 zvol_clone_range(zvol_state_t *zv_src, uint64_t inoff, zvol_state_t *zv_dst, 608 uint64_t outoff, uint64_t len) 609 { 610 zilog_t *zilog_dst; 611 zfs_locked_range_t *inlr, *outlr; 612 objset_t *inos, *outos; 613 dmu_tx_t *tx; 614 blkptr_t *bps; 615 size_t maxblocks; 616 int error = EINVAL; 617 618 rw_enter(&zv_dst->zv_suspend_lock, RW_READER); 619 if (zv_dst->zv_zilog == NULL) { 620 rw_exit(&zv_dst->zv_suspend_lock); 621 rw_enter(&zv_dst->zv_suspend_lock, RW_WRITER); 622 if (zv_dst->zv_zilog == NULL) { 623 zv_dst->zv_zilog = zil_open(zv_dst->zv_objset, 624 zvol_get_data, &zv_dst->zv_kstat.dk_zil_sums); 625 zv_dst->zv_flags |= ZVOL_WRITTEN_TO; 626 VERIFY0((zv_dst->zv_zilog->zl_header->zh_flags & 627 ZIL_REPLAY_NEEDED)); 628 } 629 rw_downgrade(&zv_dst->zv_suspend_lock); 630 } 631 if (zv_src != zv_dst) 632 rw_enter(&zv_src->zv_suspend_lock, RW_READER); 633 634 inos = zv_src->zv_objset; 635 outos = zv_dst->zv_objset; 636 637 /* 638 * Sanity checks 639 */ 640 if (!spa_feature_is_enabled(dmu_objset_spa(outos), 641 SPA_FEATURE_BLOCK_CLONING)) { 642 error = EOPNOTSUPP; 643 goto out; 644 } 645 if (dmu_objset_spa(inos) != dmu_objset_spa(outos)) { 646 error = EXDEV; 647 goto out; 648 } 649 if (inos->os_encrypted != outos->os_encrypted) { 650 error = EXDEV; 651 goto out; 652 } 653 if (zv_src->zv_volblocksize != zv_dst->zv_volblocksize) { 654 error = EINVAL; 655 goto out; 656 } 657 if (inoff >= zv_src->zv_volsize || outoff >= zv_dst->zv_volsize) { 658 error = 0; 659 goto out; 660 } 661 662 /* 663 * Do not read beyond boundary 664 */ 665 if (len > zv_src->zv_volsize - inoff) 666 len = zv_src->zv_volsize - inoff; 667 if (len > zv_dst->zv_volsize - outoff) 668 len = zv_dst->zv_volsize - outoff; 669 if (len == 0) { 670 error = 0; 671 goto out; 672 } 673 674 /* 675 * No overlapping if we are cloning within the same file 676 */ 677 if (zv_src == zv_dst) { 678 if (inoff < outoff + len && outoff < inoff + len) { 679 error = EINVAL; 680 goto out; 681 } 682 } 683 684 /* 685 * Offsets and length must be at block boundaries 686 */ 687 if ((inoff % zv_src->zv_volblocksize) != 0 || 688 (outoff % zv_dst->zv_volblocksize) != 0) { 689 error = EINVAL; 690 goto out; 691 } 692 693 /* 694 * Length must be multiple of block size 695 */ 696 if ((len % zv_src->zv_volblocksize) != 0) { 697 error = EINVAL; 698 goto out; 699 } 700 701 zilog_dst = zv_dst->zv_zilog; 702 maxblocks = zil_max_log_data(zilog_dst, sizeof (lr_clone_range_t)) / 703 sizeof (bps[0]); 704 bps = vmem_alloc(sizeof (bps[0]) * maxblocks, KM_SLEEP); 705 /* 706 * Maintain predictable lock order. 707 */ 708 if (zv_src < zv_dst || (zv_src == zv_dst && inoff < outoff)) { 709 inlr = zfs_rangelock_enter(&zv_src->zv_rangelock, inoff, len, 710 RL_READER); 711 outlr = zfs_rangelock_enter(&zv_dst->zv_rangelock, outoff, len, 712 RL_WRITER); 713 } else { 714 outlr = zfs_rangelock_enter(&zv_dst->zv_rangelock, outoff, len, 715 RL_WRITER); 716 inlr = zfs_rangelock_enter(&zv_src->zv_rangelock, inoff, len, 717 RL_READER); 718 } 719 720 while (len > 0) { 721 uint64_t size, last_synced_txg; 722 size_t nbps = maxblocks; 723 size = MIN(zv_src->zv_volblocksize * maxblocks, len); 724 last_synced_txg = spa_last_synced_txg( 725 dmu_objset_spa(zv_src->zv_objset)); 726 error = dmu_read_l0_bps(zv_src->zv_objset, ZVOL_OBJ, inoff, 727 size, bps, &nbps); 728 if (error != 0) { 729 /* 730 * If we are trying to clone a block that was created 731 * in the current transaction group, the error will be 732 * EAGAIN here. Based on zfs_bclone_wait_dirty either 733 * return a shortened range to the caller so it can 734 * fallback, or wait for the next TXG and check again. 735 */ 736 if (error == EAGAIN && zfs_bclone_wait_dirty) { 737 txg_wait_synced(dmu_objset_pool 738 (zv_src->zv_objset), last_synced_txg + 1); 739 continue; 740 } 741 break; 742 } 743 744 tx = dmu_tx_create(zv_dst->zv_objset); 745 dmu_tx_hold_clone_by_dnode(tx, zv_dst->zv_dn, outoff, size, 746 zv_src->zv_volblocksize); 747 error = dmu_tx_assign(tx, DMU_TX_WAIT); 748 if (error != 0) { 749 dmu_tx_abort(tx); 750 break; 751 } 752 error = dmu_brt_clone(zv_dst->zv_objset, ZVOL_OBJ, outoff, size, 753 tx, bps, nbps); 754 if (error != 0) { 755 dmu_tx_commit(tx); 756 break; 757 } 758 zvol_log_clone_range(zilog_dst, tx, TX_CLONE_RANGE, outoff, 759 size, zv_src->zv_volblocksize, bps, nbps); 760 dmu_tx_commit(tx); 761 inoff += size; 762 outoff += size; 763 len -= size; 764 } 765 vmem_free(bps, sizeof (bps[0]) * maxblocks); 766 zfs_rangelock_exit(outlr); 767 zfs_rangelock_exit(inlr); 768 if (error == 0 && zv_dst->zv_objset->os_sync == ZFS_SYNC_ALWAYS) { 769 zil_commit(zilog_dst, ZVOL_OBJ); 770 } 771 out: 772 if (zv_src != zv_dst) 773 rw_exit(&zv_src->zv_suspend_lock); 774 rw_exit(&zv_dst->zv_suspend_lock); 775 return (SET_ERROR(error)); 776 } 777 778 /* 779 * Handles TX_CLONE_RANGE transactions. 780 */ 781 void 782 zvol_log_clone_range(zilog_t *zilog, dmu_tx_t *tx, int txtype, uint64_t off, 783 uint64_t len, uint64_t blksz, const blkptr_t *bps, size_t nbps) 784 { 785 itx_t *itx; 786 lr_clone_range_t *lr; 787 uint64_t partlen, max_log_data; 788 size_t partnbps; 789 790 if (zil_replaying(zilog, tx)) 791 return; 792 793 max_log_data = zil_max_log_data(zilog, sizeof (lr_clone_range_t)); 794 795 while (nbps > 0) { 796 partnbps = MIN(nbps, max_log_data / sizeof (bps[0])); 797 partlen = partnbps * blksz; 798 ASSERT3U(partlen, <, len + blksz); 799 partlen = MIN(partlen, len); 800 801 itx = zil_itx_create(txtype, 802 sizeof (*lr) + sizeof (bps[0]) * partnbps); 803 lr = (lr_clone_range_t *)&itx->itx_lr; 804 lr->lr_foid = ZVOL_OBJ; 805 lr->lr_offset = off; 806 lr->lr_length = partlen; 807 lr->lr_blksz = blksz; 808 lr->lr_nbps = partnbps; 809 memcpy(lr->lr_bps, bps, sizeof (bps[0]) * partnbps); 810 811 zil_itx_assign(zilog, itx, tx); 812 813 bps += partnbps; 814 ASSERT3U(nbps, >=, partnbps); 815 nbps -= partnbps; 816 off += partlen; 817 ASSERT3U(len, >=, partlen); 818 len -= partlen; 819 } 820 } 821 822 static int 823 zvol_replay_err(void *arg1, void *arg2, boolean_t byteswap) 824 { 825 (void) arg1, (void) arg2, (void) byteswap; 826 return (SET_ERROR(ENOTSUP)); 827 } 828 829 /* 830 * Callback vectors for replaying records. 831 * Only TX_WRITE and TX_TRUNCATE are needed for zvol. 832 */ 833 zil_replay_func_t *const zvol_replay_vector[TX_MAX_TYPE] = { 834 zvol_replay_err, /* no such transaction type */ 835 zvol_replay_err, /* TX_CREATE */ 836 zvol_replay_err, /* TX_MKDIR */ 837 zvol_replay_err, /* TX_MKXATTR */ 838 zvol_replay_err, /* TX_SYMLINK */ 839 zvol_replay_err, /* TX_REMOVE */ 840 zvol_replay_err, /* TX_RMDIR */ 841 zvol_replay_err, /* TX_LINK */ 842 zvol_replay_err, /* TX_RENAME */ 843 zvol_replay_write, /* TX_WRITE */ 844 zvol_replay_truncate, /* TX_TRUNCATE */ 845 zvol_replay_err, /* TX_SETATTR */ 846 zvol_replay_err, /* TX_ACL_V0 */ 847 zvol_replay_err, /* TX_ACL */ 848 zvol_replay_err, /* TX_CREATE_ACL */ 849 zvol_replay_err, /* TX_CREATE_ATTR */ 850 zvol_replay_err, /* TX_CREATE_ACL_ATTR */ 851 zvol_replay_err, /* TX_MKDIR_ACL */ 852 zvol_replay_err, /* TX_MKDIR_ATTR */ 853 zvol_replay_err, /* TX_MKDIR_ACL_ATTR */ 854 zvol_replay_err, /* TX_WRITE2 */ 855 zvol_replay_err, /* TX_SETSAXATTR */ 856 zvol_replay_err, /* TX_RENAME_EXCHANGE */ 857 zvol_replay_err, /* TX_RENAME_WHITEOUT */ 858 zvol_replay_clone_range, /* TX_CLONE_RANGE */ 859 }; 860 861 /* 862 * zvol_log_write() handles synchronous writes using TX_WRITE ZIL transactions. 863 * 864 * We store data in the log buffers if it's small enough. 865 * Otherwise we will later flush the data out via dmu_sync(). 866 */ 867 static const ssize_t zvol_immediate_write_sz = 32768; 868 869 void 870 zvol_log_write(zvol_state_t *zv, dmu_tx_t *tx, uint64_t offset, 871 uint64_t size, boolean_t commit) 872 { 873 uint32_t blocksize = zv->zv_volblocksize; 874 zilog_t *zilog = zv->zv_zilog; 875 itx_wr_state_t write_state; 876 uint64_t log_size = 0; 877 878 if (zil_replaying(zilog, tx)) 879 return; 880 881 if (zilog->zl_logbias == ZFS_LOGBIAS_THROUGHPUT) 882 write_state = WR_INDIRECT; 883 else if (!spa_has_slogs(zilog->zl_spa) && 884 size >= blocksize && blocksize > zvol_immediate_write_sz) 885 write_state = WR_INDIRECT; 886 else if (commit) 887 write_state = WR_COPIED; 888 else 889 write_state = WR_NEED_COPY; 890 891 while (size) { 892 itx_t *itx; 893 lr_write_t *lr; 894 itx_wr_state_t wr_state = write_state; 895 ssize_t len = size; 896 897 if (wr_state == WR_COPIED && size > zil_max_copied_data(zilog)) 898 wr_state = WR_NEED_COPY; 899 else if (wr_state == WR_INDIRECT) 900 len = MIN(blocksize - P2PHASE(offset, blocksize), size); 901 902 itx = zil_itx_create(TX_WRITE, sizeof (*lr) + 903 (wr_state == WR_COPIED ? len : 0)); 904 lr = (lr_write_t *)&itx->itx_lr; 905 if (wr_state == WR_COPIED && 906 dmu_read_by_dnode(zv->zv_dn, offset, len, lr + 1, 907 DMU_READ_NO_PREFETCH | DMU_KEEP_CACHING) != 0) { 908 zil_itx_destroy(itx); 909 itx = zil_itx_create(TX_WRITE, sizeof (*lr)); 910 lr = (lr_write_t *)&itx->itx_lr; 911 wr_state = WR_NEED_COPY; 912 } 913 914 log_size += itx->itx_size; 915 if (wr_state == WR_NEED_COPY) 916 log_size += len; 917 918 itx->itx_wr_state = wr_state; 919 lr->lr_foid = ZVOL_OBJ; 920 lr->lr_offset = offset; 921 lr->lr_length = len; 922 lr->lr_blkoff = 0; 923 BP_ZERO(&lr->lr_blkptr); 924 925 itx->itx_private = zv; 926 927 (void) zil_itx_assign(zilog, itx, tx); 928 929 offset += len; 930 size -= len; 931 } 932 933 dsl_pool_wrlog_count(zilog->zl_dmu_pool, log_size, tx->tx_txg); 934 } 935 936 /* 937 * Log a DKIOCFREE/free-long-range to the ZIL with TX_TRUNCATE. 938 */ 939 void 940 zvol_log_truncate(zvol_state_t *zv, dmu_tx_t *tx, uint64_t off, uint64_t len) 941 { 942 itx_t *itx; 943 lr_truncate_t *lr; 944 zilog_t *zilog = zv->zv_zilog; 945 946 if (zil_replaying(zilog, tx)) 947 return; 948 949 itx = zil_itx_create(TX_TRUNCATE, sizeof (*lr)); 950 lr = (lr_truncate_t *)&itx->itx_lr; 951 lr->lr_foid = ZVOL_OBJ; 952 lr->lr_offset = off; 953 lr->lr_length = len; 954 955 zil_itx_assign(zilog, itx, tx); 956 } 957 958 959 static void 960 zvol_get_done(zgd_t *zgd, int error) 961 { 962 (void) error; 963 if (zgd->zgd_db) 964 dmu_buf_rele(zgd->zgd_db, zgd); 965 966 zfs_rangelock_exit(zgd->zgd_lr); 967 968 kmem_free(zgd, sizeof (zgd_t)); 969 } 970 971 /* 972 * Get data to generate a TX_WRITE intent log record. 973 */ 974 int 975 zvol_get_data(void *arg, uint64_t arg2, lr_write_t *lr, char *buf, 976 struct lwb *lwb, zio_t *zio) 977 { 978 zvol_state_t *zv = arg; 979 uint64_t offset = lr->lr_offset; 980 uint64_t size = lr->lr_length; 981 dmu_buf_t *db; 982 zgd_t *zgd; 983 int error; 984 985 ASSERT3P(lwb, !=, NULL); 986 ASSERT3U(size, !=, 0); 987 988 zgd = kmem_zalloc(sizeof (zgd_t), KM_SLEEP); 989 zgd->zgd_lwb = lwb; 990 991 /* 992 * Write records come in two flavors: immediate and indirect. 993 * For small writes it's cheaper to store the data with the 994 * log record (immediate); for large writes it's cheaper to 995 * sync the data and get a pointer to it (indirect) so that 996 * we don't have to write the data twice. 997 */ 998 if (buf != NULL) { /* immediate write */ 999 zgd->zgd_lr = zfs_rangelock_enter(&zv->zv_rangelock, offset, 1000 size, RL_READER); 1001 error = dmu_read_by_dnode(zv->zv_dn, offset, size, buf, 1002 DMU_READ_NO_PREFETCH | DMU_KEEP_CACHING); 1003 } else { /* indirect write */ 1004 ASSERT3P(zio, !=, NULL); 1005 /* 1006 * Have to lock the whole block to ensure when it's written out 1007 * and its checksum is being calculated that no one can change 1008 * the data. Contrarily to zfs_get_data we need not re-check 1009 * blocksize after we get the lock because it cannot be changed. 1010 */ 1011 size = zv->zv_volblocksize; 1012 offset = P2ALIGN_TYPED(offset, size, uint64_t); 1013 zgd->zgd_lr = zfs_rangelock_enter(&zv->zv_rangelock, offset, 1014 size, RL_READER); 1015 error = dmu_buf_hold_noread_by_dnode(zv->zv_dn, offset, zgd, 1016 &db); 1017 if (error == 0) { 1018 blkptr_t *bp = &lr->lr_blkptr; 1019 1020 zgd->zgd_db = db; 1021 zgd->zgd_bp = bp; 1022 1023 ASSERT(db != NULL); 1024 ASSERT(db->db_offset == offset); 1025 ASSERT(db->db_size == size); 1026 1027 error = dmu_sync(zio, lr->lr_common.lrc_txg, 1028 zvol_get_done, zgd); 1029 1030 if (error == 0) 1031 return (0); 1032 } 1033 } 1034 1035 zvol_get_done(zgd, error); 1036 1037 return (SET_ERROR(error)); 1038 } 1039 1040 /* 1041 * The zvol_state_t's are inserted into zvol_state_list and zvol_htable. 1042 */ 1043 1044 void 1045 zvol_insert(zvol_state_t *zv) 1046 { 1047 ASSERT(RW_WRITE_HELD(&zvol_state_lock)); 1048 list_insert_head(&zvol_state_list, zv); 1049 hlist_add_head(&zv->zv_hlink, ZVOL_HT_HEAD(zv->zv_hash)); 1050 } 1051 1052 /* 1053 * Simply remove the zvol from to list of zvols. 1054 */ 1055 static void 1056 zvol_remove(zvol_state_t *zv) 1057 { 1058 ASSERT(RW_WRITE_HELD(&zvol_state_lock)); 1059 list_remove(&zvol_state_list, zv); 1060 hlist_del(&zv->zv_hlink); 1061 } 1062 1063 /* 1064 * Setup zv after we just own the zv->objset 1065 */ 1066 static int 1067 zvol_setup_zv(zvol_state_t *zv) 1068 { 1069 uint64_t volsize; 1070 int error; 1071 uint64_t ro; 1072 objset_t *os = zv->zv_objset; 1073 1074 ASSERT(MUTEX_HELD(&zv->zv_state_lock)); 1075 ASSERT(RW_LOCK_HELD(&zv->zv_suspend_lock)); 1076 1077 zv->zv_zilog = NULL; 1078 zv->zv_flags &= ~ZVOL_WRITTEN_TO; 1079 1080 error = dsl_prop_get_integer(zv->zv_name, "readonly", &ro, NULL); 1081 if (error) 1082 return (SET_ERROR(error)); 1083 1084 error = zap_lookup(os, ZVOL_ZAP_OBJ, "size", 8, 1, &volsize); 1085 if (error) 1086 return (SET_ERROR(error)); 1087 1088 error = dnode_hold(os, ZVOL_OBJ, zv, &zv->zv_dn); 1089 if (error) 1090 return (SET_ERROR(error)); 1091 1092 zvol_os_set_capacity(zv, volsize >> 9); 1093 zv->zv_volsize = volsize; 1094 1095 if (ro || dmu_objset_is_snapshot(os) || 1096 !spa_writeable(dmu_objset_spa(os))) { 1097 zvol_os_set_disk_ro(zv, 1); 1098 zv->zv_flags |= ZVOL_RDONLY; 1099 } else { 1100 zvol_os_set_disk_ro(zv, 0); 1101 zv->zv_flags &= ~ZVOL_RDONLY; 1102 } 1103 return (0); 1104 } 1105 1106 /* 1107 * Shutdown every zv_objset related stuff except zv_objset itself. 1108 * The is the reverse of zvol_setup_zv. 1109 */ 1110 static void 1111 zvol_shutdown_zv(zvol_state_t *zv) 1112 { 1113 ASSERT(MUTEX_HELD(&zv->zv_state_lock) && 1114 RW_LOCK_HELD(&zv->zv_suspend_lock)); 1115 1116 if (zv->zv_flags & ZVOL_WRITTEN_TO) { 1117 ASSERT(zv->zv_zilog != NULL); 1118 zil_close(zv->zv_zilog); 1119 } 1120 1121 zv->zv_zilog = NULL; 1122 1123 dnode_rele(zv->zv_dn, zv); 1124 zv->zv_dn = NULL; 1125 1126 /* 1127 * Evict cached data. We must write out any dirty data before 1128 * disowning the dataset. 1129 */ 1130 if (zv->zv_flags & ZVOL_WRITTEN_TO) 1131 txg_wait_synced(dmu_objset_pool(zv->zv_objset), 0); 1132 (void) dmu_objset_evict_dbufs(zv->zv_objset); 1133 } 1134 1135 /* 1136 * return the proper tag for rollback and recv 1137 */ 1138 void * 1139 zvol_tag(zvol_state_t *zv) 1140 { 1141 ASSERT(RW_WRITE_HELD(&zv->zv_suspend_lock)); 1142 return (zv->zv_open_count > 0 ? zv : NULL); 1143 } 1144 1145 /* 1146 * Suspend the zvol for recv and rollback. 1147 */ 1148 zvol_state_t * 1149 zvol_suspend(const char *name) 1150 { 1151 zvol_state_t *zv; 1152 1153 zv = zvol_find_by_name(name, RW_WRITER); 1154 1155 if (zv == NULL) 1156 return (NULL); 1157 1158 /* block all I/O, release in zvol_resume. */ 1159 ASSERT(MUTEX_HELD(&zv->zv_state_lock)); 1160 ASSERT(RW_WRITE_HELD(&zv->zv_suspend_lock)); 1161 1162 atomic_inc(&zv->zv_suspend_ref); 1163 1164 if (zv->zv_open_count > 0) 1165 zvol_shutdown_zv(zv); 1166 1167 /* 1168 * do not hold zv_state_lock across suspend/resume to 1169 * avoid locking up zvol lookups 1170 */ 1171 mutex_exit(&zv->zv_state_lock); 1172 1173 /* zv_suspend_lock is released in zvol_resume() */ 1174 return (zv); 1175 } 1176 1177 int 1178 zvol_resume(zvol_state_t *zv) 1179 { 1180 int error = 0; 1181 1182 ASSERT(RW_WRITE_HELD(&zv->zv_suspend_lock)); 1183 1184 mutex_enter(&zv->zv_state_lock); 1185 1186 if (zv->zv_open_count > 0) { 1187 VERIFY0(dmu_objset_hold(zv->zv_name, zv, &zv->zv_objset)); 1188 VERIFY3P(zv->zv_objset->os_dsl_dataset->ds_owner, ==, zv); 1189 VERIFY(dsl_dataset_long_held(zv->zv_objset->os_dsl_dataset)); 1190 dmu_objset_rele(zv->zv_objset, zv); 1191 1192 error = zvol_setup_zv(zv); 1193 } 1194 1195 mutex_exit(&zv->zv_state_lock); 1196 1197 rw_exit(&zv->zv_suspend_lock); 1198 /* 1199 * We need this because we don't hold zvol_state_lock while releasing 1200 * zv_suspend_lock. zvol_remove_minors_impl thus cannot check 1201 * zv_suspend_lock to determine it is safe to free because rwlock is 1202 * not inherent atomic. 1203 */ 1204 atomic_dec(&zv->zv_suspend_ref); 1205 1206 if (zv->zv_flags & ZVOL_REMOVING) 1207 cv_broadcast(&zv->zv_removing_cv); 1208 1209 return (SET_ERROR(error)); 1210 } 1211 1212 int 1213 zvol_first_open(zvol_state_t *zv, boolean_t readonly) 1214 { 1215 objset_t *os; 1216 int error; 1217 1218 ASSERT(RW_READ_HELD(&zv->zv_suspend_lock)); 1219 ASSERT(MUTEX_HELD(&zv->zv_state_lock)); 1220 ASSERT(mutex_owned(&spa_namespace_lock)); 1221 1222 boolean_t ro = (readonly || (strchr(zv->zv_name, '@') != NULL)); 1223 error = dmu_objset_own(zv->zv_name, DMU_OST_ZVOL, ro, B_TRUE, zv, &os); 1224 if (error) 1225 return (SET_ERROR(error)); 1226 1227 zv->zv_objset = os; 1228 1229 error = zvol_setup_zv(zv); 1230 if (error) { 1231 dmu_objset_disown(os, 1, zv); 1232 zv->zv_objset = NULL; 1233 } 1234 1235 return (error); 1236 } 1237 1238 void 1239 zvol_last_close(zvol_state_t *zv) 1240 { 1241 ASSERT(RW_READ_HELD(&zv->zv_suspend_lock)); 1242 ASSERT(MUTEX_HELD(&zv->zv_state_lock)); 1243 1244 if (zv->zv_flags & ZVOL_REMOVING) 1245 cv_broadcast(&zv->zv_removing_cv); 1246 1247 zvol_shutdown_zv(zv); 1248 1249 dmu_objset_disown(zv->zv_objset, 1, zv); 1250 zv->zv_objset = NULL; 1251 } 1252 1253 typedef struct minors_job { 1254 list_t *list; 1255 list_node_t link; 1256 /* input */ 1257 char *name; 1258 /* output */ 1259 int error; 1260 } minors_job_t; 1261 1262 /* 1263 * Prefetch zvol dnodes for the minors_job 1264 */ 1265 static void 1266 zvol_prefetch_minors_impl(void *arg) 1267 { 1268 minors_job_t *job = arg; 1269 char *dsname = job->name; 1270 objset_t *os = NULL; 1271 1272 job->error = dmu_objset_own(dsname, DMU_OST_ZVOL, B_TRUE, B_TRUE, 1273 FTAG, &os); 1274 if (job->error == 0) { 1275 dmu_prefetch_dnode(os, ZVOL_OBJ, ZIO_PRIORITY_SYNC_READ); 1276 dmu_objset_disown(os, B_TRUE, FTAG); 1277 } 1278 } 1279 1280 /* 1281 * Mask errors to continue dmu_objset_find() traversal 1282 */ 1283 static int 1284 zvol_create_snap_minor_cb(const char *dsname, void *arg) 1285 { 1286 minors_job_t *j = arg; 1287 list_t *minors_list = j->list; 1288 const char *name = j->name; 1289 1290 ASSERT0(MUTEX_HELD(&spa_namespace_lock)); 1291 1292 /* skip the designated dataset */ 1293 if (name && strcmp(dsname, name) == 0) 1294 return (0); 1295 1296 /* at this point, the dsname should name a snapshot */ 1297 if (strchr(dsname, '@') == 0) { 1298 dprintf("zvol_create_snap_minor_cb(): " 1299 "%s is not a snapshot name\n", dsname); 1300 } else { 1301 minors_job_t *job; 1302 char *n = kmem_strdup(dsname); 1303 if (n == NULL) 1304 return (0); 1305 1306 job = kmem_alloc(sizeof (minors_job_t), KM_SLEEP); 1307 job->name = n; 1308 job->list = minors_list; 1309 job->error = 0; 1310 list_insert_tail(minors_list, job); 1311 /* don't care if dispatch fails, because job->error is 0 */ 1312 taskq_dispatch(system_taskq, zvol_prefetch_minors_impl, job, 1313 TQ_SLEEP); 1314 } 1315 1316 return (0); 1317 } 1318 1319 /* 1320 * If spa_keystore_load_wkey() is called for an encrypted zvol, 1321 * we need to look for any clones also using the key. This function 1322 * is "best effort" - so we just skip over it if there are failures. 1323 */ 1324 static void 1325 zvol_add_clones(const char *dsname, list_t *minors_list) 1326 { 1327 /* Also check if it has clones */ 1328 dsl_dir_t *dd = NULL; 1329 dsl_pool_t *dp = NULL; 1330 1331 if (dsl_pool_hold(dsname, FTAG, &dp) != 0) 1332 return; 1333 1334 if (!spa_feature_is_enabled(dp->dp_spa, 1335 SPA_FEATURE_ENCRYPTION)) 1336 goto out; 1337 1338 if (dsl_dir_hold(dp, dsname, FTAG, &dd, NULL) != 0) 1339 goto out; 1340 1341 if (dsl_dir_phys(dd)->dd_clones == 0) 1342 goto out; 1343 1344 zap_cursor_t *zc = kmem_alloc(sizeof (zap_cursor_t), KM_SLEEP); 1345 zap_attribute_t *za = zap_attribute_alloc(); 1346 objset_t *mos = dd->dd_pool->dp_meta_objset; 1347 1348 for (zap_cursor_init(zc, mos, dsl_dir_phys(dd)->dd_clones); 1349 zap_cursor_retrieve(zc, za) == 0; 1350 zap_cursor_advance(zc)) { 1351 dsl_dataset_t *clone; 1352 minors_job_t *job; 1353 1354 if (dsl_dataset_hold_obj(dd->dd_pool, 1355 za->za_first_integer, FTAG, &clone) == 0) { 1356 1357 char name[ZFS_MAX_DATASET_NAME_LEN]; 1358 dsl_dataset_name(clone, name); 1359 1360 char *n = kmem_strdup(name); 1361 job = kmem_alloc(sizeof (minors_job_t), KM_SLEEP); 1362 job->name = n; 1363 job->list = minors_list; 1364 job->error = 0; 1365 list_insert_tail(minors_list, job); 1366 1367 dsl_dataset_rele(clone, FTAG); 1368 } 1369 } 1370 zap_cursor_fini(zc); 1371 zap_attribute_free(za); 1372 kmem_free(zc, sizeof (zap_cursor_t)); 1373 1374 out: 1375 if (dd != NULL) 1376 dsl_dir_rele(dd, FTAG); 1377 dsl_pool_rele(dp, FTAG); 1378 } 1379 1380 /* 1381 * Mask errors to continue dmu_objset_find() traversal 1382 */ 1383 static int 1384 zvol_create_minors_cb(const char *dsname, void *arg) 1385 { 1386 uint64_t snapdev; 1387 int error; 1388 list_t *minors_list = arg; 1389 1390 ASSERT0(MUTEX_HELD(&spa_namespace_lock)); 1391 1392 error = dsl_prop_get_integer(dsname, "snapdev", &snapdev, NULL); 1393 if (error) 1394 return (0); 1395 1396 /* 1397 * Given the name and the 'snapdev' property, create device minor nodes 1398 * with the linkages to zvols/snapshots as needed. 1399 * If the name represents a zvol, create a minor node for the zvol, then 1400 * check if its snapshots are 'visible', and if so, iterate over the 1401 * snapshots and create device minor nodes for those. 1402 */ 1403 if (strchr(dsname, '@') == 0) { 1404 minors_job_t *job; 1405 char *n = kmem_strdup(dsname); 1406 if (n == NULL) 1407 return (0); 1408 1409 job = kmem_alloc(sizeof (minors_job_t), KM_SLEEP); 1410 job->name = n; 1411 job->list = minors_list; 1412 job->error = 0; 1413 list_insert_tail(minors_list, job); 1414 /* don't care if dispatch fails, because job->error is 0 */ 1415 taskq_dispatch(system_taskq, zvol_prefetch_minors_impl, job, 1416 TQ_SLEEP); 1417 1418 zvol_add_clones(dsname, minors_list); 1419 1420 if (snapdev == ZFS_SNAPDEV_VISIBLE) { 1421 /* 1422 * traverse snapshots only, do not traverse children, 1423 * and skip the 'dsname' 1424 */ 1425 (void) dmu_objset_find(dsname, 1426 zvol_create_snap_minor_cb, (void *)job, 1427 DS_FIND_SNAPSHOTS); 1428 } 1429 } else { 1430 dprintf("zvol_create_minors_cb(): %s is not a zvol name\n", 1431 dsname); 1432 } 1433 1434 return (0); 1435 } 1436 1437 /* 1438 * Create minors for the specified dataset, including children and snapshots. 1439 * Pay attention to the 'snapdev' property and iterate over the snapshots 1440 * only if they are 'visible'. This approach allows one to assure that the 1441 * snapshot metadata is read from disk only if it is needed. 1442 * 1443 * The name can represent a dataset to be recursively scanned for zvols and 1444 * their snapshots, or a single zvol snapshot. If the name represents a 1445 * dataset, the scan is performed in two nested stages: 1446 * - scan the dataset for zvols, and 1447 * - for each zvol, create a minor node, then check if the zvol's snapshots 1448 * are 'visible', and only then iterate over the snapshots if needed 1449 * 1450 * If the name represents a snapshot, a check is performed if the snapshot is 1451 * 'visible' (which also verifies that the parent is a zvol), and if so, 1452 * a minor node for that snapshot is created. 1453 */ 1454 void 1455 zvol_create_minors_recursive(const char *name) 1456 { 1457 list_t minors_list; 1458 minors_job_t *job; 1459 1460 if (zvol_inhibit_dev) 1461 return; 1462 1463 /* 1464 * This is the list for prefetch jobs. Whenever we found a match 1465 * during dmu_objset_find, we insert a minors_job to the list and do 1466 * taskq_dispatch to parallel prefetch zvol dnodes. Note we don't need 1467 * any lock because all list operation is done on the current thread. 1468 * 1469 * We will use this list to do zvol_os_create_minor after prefetch 1470 * so we don't have to traverse using dmu_objset_find again. 1471 */ 1472 list_create(&minors_list, sizeof (minors_job_t), 1473 offsetof(minors_job_t, link)); 1474 1475 1476 if (strchr(name, '@') != NULL) { 1477 uint64_t snapdev; 1478 1479 int error = dsl_prop_get_integer(name, "snapdev", 1480 &snapdev, NULL); 1481 1482 if (error == 0 && snapdev == ZFS_SNAPDEV_VISIBLE) 1483 (void) zvol_os_create_minor(name); 1484 } else { 1485 fstrans_cookie_t cookie = spl_fstrans_mark(); 1486 (void) dmu_objset_find(name, zvol_create_minors_cb, 1487 &minors_list, DS_FIND_CHILDREN); 1488 spl_fstrans_unmark(cookie); 1489 } 1490 1491 taskq_wait_outstanding(system_taskq, 0); 1492 1493 /* 1494 * Prefetch is completed, we can do zvol_os_create_minor 1495 * sequentially. 1496 */ 1497 while ((job = list_remove_head(&minors_list)) != NULL) { 1498 if (!job->error) 1499 (void) zvol_os_create_minor(job->name); 1500 kmem_strfree(job->name); 1501 kmem_free(job, sizeof (minors_job_t)); 1502 } 1503 1504 list_destroy(&minors_list); 1505 } 1506 1507 void 1508 zvol_create_minor(const char *name) 1509 { 1510 /* 1511 * Note: the dsl_pool_config_lock must not be held. 1512 * Minor node creation needs to obtain the zvol_state_lock. 1513 * zvol_open() obtains the zvol_state_lock and then the dsl pool 1514 * config lock. Therefore, we can't have the config lock now if 1515 * we are going to wait for the zvol_state_lock, because it 1516 * would be a lock order inversion which could lead to deadlock. 1517 */ 1518 1519 if (zvol_inhibit_dev) 1520 return; 1521 1522 if (strchr(name, '@') != NULL) { 1523 uint64_t snapdev; 1524 1525 int error = dsl_prop_get_integer(name, 1526 "snapdev", &snapdev, NULL); 1527 1528 if (error == 0 && snapdev == ZFS_SNAPDEV_VISIBLE) 1529 (void) zvol_os_create_minor(name); 1530 } else { 1531 (void) zvol_os_create_minor(name); 1532 } 1533 } 1534 1535 /* 1536 * Remove minors for specified dataset including children and snapshots. 1537 */ 1538 1539 /* 1540 * Remove the minor for a given zvol. This will do it all: 1541 * - flag the zvol for removal, so new requests are rejected 1542 * - wait until outstanding requests are completed 1543 * - remove it from lists 1544 * - free it 1545 * It's also usable as a taskq task, and smells nice too. 1546 */ 1547 static void 1548 zvol_remove_minor_task(void *arg) 1549 { 1550 zvol_state_t *zv = (zvol_state_t *)arg; 1551 1552 ASSERT(!RW_LOCK_HELD(&zvol_state_lock)); 1553 ASSERT(!MUTEX_HELD(&zv->zv_state_lock)); 1554 1555 mutex_enter(&zv->zv_state_lock); 1556 while (zv->zv_open_count > 0 || atomic_read(&zv->zv_suspend_ref)) { 1557 zv->zv_flags |= ZVOL_REMOVING; 1558 cv_wait(&zv->zv_removing_cv, &zv->zv_state_lock); 1559 } 1560 mutex_exit(&zv->zv_state_lock); 1561 1562 rw_enter(&zvol_state_lock, RW_WRITER); 1563 mutex_enter(&zv->zv_state_lock); 1564 1565 zvol_remove(zv); 1566 zvol_os_clear_private(zv); 1567 1568 mutex_exit(&zv->zv_state_lock); 1569 rw_exit(&zvol_state_lock); 1570 1571 zvol_os_free(zv); 1572 } 1573 1574 static void 1575 zvol_free_task(void *arg) 1576 { 1577 zvol_os_free(arg); 1578 } 1579 1580 void 1581 zvol_remove_minors_impl(const char *name) 1582 { 1583 zvol_state_t *zv, *zv_next; 1584 int namelen = ((name) ? strlen(name) : 0); 1585 taskqid_t t; 1586 list_t delay_list, free_list; 1587 1588 if (zvol_inhibit_dev) 1589 return; 1590 1591 list_create(&delay_list, sizeof (zvol_state_t), 1592 offsetof(zvol_state_t, zv_next)); 1593 list_create(&free_list, sizeof (zvol_state_t), 1594 offsetof(zvol_state_t, zv_next)); 1595 1596 rw_enter(&zvol_state_lock, RW_WRITER); 1597 1598 for (zv = list_head(&zvol_state_list); zv != NULL; zv = zv_next) { 1599 zv_next = list_next(&zvol_state_list, zv); 1600 1601 mutex_enter(&zv->zv_state_lock); 1602 if (name == NULL || strcmp(zv->zv_name, name) == 0 || 1603 (strncmp(zv->zv_name, name, namelen) == 0 && 1604 (zv->zv_name[namelen] == '/' || 1605 zv->zv_name[namelen] == '@'))) { 1606 /* 1607 * By holding zv_state_lock here, we guarantee that no 1608 * one is currently using this zv 1609 */ 1610 1611 /* 1612 * If in use, try to throw everyone off and try again 1613 * later. 1614 */ 1615 if (zv->zv_open_count > 0 || 1616 atomic_read(&zv->zv_suspend_ref)) { 1617 zv->zv_flags |= ZVOL_REMOVING; 1618 t = taskq_dispatch( 1619 zv->zv_objset->os_spa->spa_zvol_taskq, 1620 zvol_remove_minor_task, zv, TQ_SLEEP); 1621 if (t == TASKQID_INVALID) { 1622 /* 1623 * Couldn't create the task, so we'll 1624 * do it in place once the loop is 1625 * finished. 1626 */ 1627 list_insert_head(&delay_list, zv); 1628 } 1629 mutex_exit(&zv->zv_state_lock); 1630 continue; 1631 } 1632 1633 zvol_remove(zv); 1634 1635 /* 1636 * Cleared while holding zvol_state_lock as a writer 1637 * which will prevent zvol_open() from opening it. 1638 */ 1639 zvol_os_clear_private(zv); 1640 1641 /* Drop zv_state_lock before zvol_free() */ 1642 mutex_exit(&zv->zv_state_lock); 1643 1644 /* Try parallel zv_free, if failed do it in place */ 1645 t = taskq_dispatch(system_taskq, zvol_free_task, zv, 1646 TQ_SLEEP); 1647 if (t == TASKQID_INVALID) 1648 list_insert_head(&free_list, zv); 1649 } else { 1650 mutex_exit(&zv->zv_state_lock); 1651 } 1652 } 1653 rw_exit(&zvol_state_lock); 1654 1655 /* Wait for zvols that we couldn't create a remove task for */ 1656 while ((zv = list_remove_head(&delay_list)) != NULL) 1657 zvol_remove_minor_task(zv); 1658 1659 /* Free any that we couldn't free in parallel earlier */ 1660 while ((zv = list_remove_head(&free_list)) != NULL) 1661 zvol_os_free(zv); 1662 } 1663 1664 /* Remove minor for this specific volume only */ 1665 static void 1666 zvol_remove_minor_impl(const char *name) 1667 { 1668 zvol_state_t *zv = NULL, *zv_next; 1669 1670 if (zvol_inhibit_dev) 1671 return; 1672 1673 rw_enter(&zvol_state_lock, RW_WRITER); 1674 1675 for (zv = list_head(&zvol_state_list); zv != NULL; zv = zv_next) { 1676 zv_next = list_next(&zvol_state_list, zv); 1677 1678 mutex_enter(&zv->zv_state_lock); 1679 if (strcmp(zv->zv_name, name) == 0) 1680 /* Found, leave the the loop with zv_lock held */ 1681 break; 1682 mutex_exit(&zv->zv_state_lock); 1683 } 1684 1685 if (zv == NULL) { 1686 rw_exit(&zvol_state_lock); 1687 return; 1688 } 1689 1690 ASSERT(MUTEX_HELD(&zv->zv_state_lock)); 1691 1692 if (zv->zv_open_count > 0 || atomic_read(&zv->zv_suspend_ref)) { 1693 /* 1694 * In use, so try to throw everyone off, then wait 1695 * until finished. 1696 */ 1697 zv->zv_flags |= ZVOL_REMOVING; 1698 mutex_exit(&zv->zv_state_lock); 1699 rw_exit(&zvol_state_lock); 1700 zvol_remove_minor_task(zv); 1701 return; 1702 } 1703 1704 zvol_remove(zv); 1705 zvol_os_clear_private(zv); 1706 1707 mutex_exit(&zv->zv_state_lock); 1708 rw_exit(&zvol_state_lock); 1709 1710 zvol_os_free(zv); 1711 } 1712 1713 /* 1714 * Rename minors for specified dataset including children and snapshots. 1715 */ 1716 static void 1717 zvol_rename_minors_impl(const char *oldname, const char *newname) 1718 { 1719 zvol_state_t *zv, *zv_next; 1720 int oldnamelen; 1721 1722 if (zvol_inhibit_dev) 1723 return; 1724 1725 oldnamelen = strlen(oldname); 1726 1727 rw_enter(&zvol_state_lock, RW_READER); 1728 1729 for (zv = list_head(&zvol_state_list); zv != NULL; zv = zv_next) { 1730 zv_next = list_next(&zvol_state_list, zv); 1731 1732 mutex_enter(&zv->zv_state_lock); 1733 1734 if (strcmp(zv->zv_name, oldname) == 0) { 1735 zvol_os_rename_minor(zv, newname); 1736 } else if (strncmp(zv->zv_name, oldname, oldnamelen) == 0 && 1737 (zv->zv_name[oldnamelen] == '/' || 1738 zv->zv_name[oldnamelen] == '@')) { 1739 char *name = kmem_asprintf("%s%c%s", newname, 1740 zv->zv_name[oldnamelen], 1741 zv->zv_name + oldnamelen + 1); 1742 zvol_os_rename_minor(zv, name); 1743 kmem_strfree(name); 1744 } 1745 1746 mutex_exit(&zv->zv_state_lock); 1747 } 1748 1749 rw_exit(&zvol_state_lock); 1750 } 1751 1752 typedef struct zvol_snapdev_cb_arg { 1753 uint64_t snapdev; 1754 } zvol_snapdev_cb_arg_t; 1755 1756 static int 1757 zvol_set_snapdev_cb(const char *dsname, void *param) 1758 { 1759 zvol_snapdev_cb_arg_t *arg = param; 1760 1761 if (strchr(dsname, '@') == NULL) 1762 return (0); 1763 1764 switch (arg->snapdev) { 1765 case ZFS_SNAPDEV_VISIBLE: 1766 (void) zvol_os_create_minor(dsname); 1767 break; 1768 case ZFS_SNAPDEV_HIDDEN: 1769 (void) zvol_remove_minor_impl(dsname); 1770 break; 1771 } 1772 1773 return (0); 1774 } 1775 1776 static void 1777 zvol_set_snapdev_impl(char *name, uint64_t snapdev) 1778 { 1779 zvol_snapdev_cb_arg_t arg = {snapdev}; 1780 fstrans_cookie_t cookie = spl_fstrans_mark(); 1781 /* 1782 * The zvol_set_snapdev_sync() sets snapdev appropriately 1783 * in the dataset hierarchy. Here, we only scan snapshots. 1784 */ 1785 dmu_objset_find(name, zvol_set_snapdev_cb, &arg, DS_FIND_SNAPSHOTS); 1786 spl_fstrans_unmark(cookie); 1787 } 1788 1789 static void 1790 zvol_set_volmode_impl(char *name, uint64_t volmode) 1791 { 1792 fstrans_cookie_t cookie; 1793 uint64_t old_volmode; 1794 zvol_state_t *zv; 1795 1796 if (strchr(name, '@') != NULL) 1797 return; 1798 1799 /* 1800 * It's unfortunate we need to remove minors before we create new ones: 1801 * this is necessary because our backing gendisk (zvol_state->zv_disk) 1802 * could be different when we set, for instance, volmode from "geom" 1803 * to "dev" (or vice versa). 1804 */ 1805 zv = zvol_find_by_name(name, RW_NONE); 1806 if (zv == NULL && volmode == ZFS_VOLMODE_NONE) 1807 return; 1808 if (zv != NULL) { 1809 old_volmode = zv->zv_volmode; 1810 mutex_exit(&zv->zv_state_lock); 1811 if (old_volmode == volmode) 1812 return; 1813 zvol_wait_close(zv); 1814 } 1815 cookie = spl_fstrans_mark(); 1816 switch (volmode) { 1817 case ZFS_VOLMODE_NONE: 1818 (void) zvol_remove_minor_impl(name); 1819 break; 1820 case ZFS_VOLMODE_GEOM: 1821 case ZFS_VOLMODE_DEV: 1822 (void) zvol_remove_minor_impl(name); 1823 (void) zvol_os_create_minor(name); 1824 break; 1825 case ZFS_VOLMODE_DEFAULT: 1826 (void) zvol_remove_minor_impl(name); 1827 if (zvol_volmode == ZFS_VOLMODE_NONE) 1828 break; 1829 else /* if zvol_volmode is invalid defaults to "geom" */ 1830 (void) zvol_os_create_minor(name); 1831 break; 1832 } 1833 spl_fstrans_unmark(cookie); 1834 } 1835 1836 static zvol_task_t * 1837 zvol_task_alloc(zvol_async_op_t op, const char *name1, const char *name2, 1838 uint64_t value) 1839 { 1840 zvol_task_t *task; 1841 1842 /* Never allow tasks on hidden names. */ 1843 if (name1[0] == '$') 1844 return (NULL); 1845 1846 task = kmem_zalloc(sizeof (zvol_task_t), KM_SLEEP); 1847 task->op = op; 1848 task->value = value; 1849 1850 strlcpy(task->name1, name1, sizeof (task->name1)); 1851 if (name2 != NULL) 1852 strlcpy(task->name2, name2, sizeof (task->name2)); 1853 1854 return (task); 1855 } 1856 1857 static void 1858 zvol_task_free(zvol_task_t *task) 1859 { 1860 kmem_free(task, sizeof (zvol_task_t)); 1861 } 1862 1863 /* 1864 * The worker thread function performed asynchronously. 1865 */ 1866 static void 1867 zvol_task_cb(void *arg) 1868 { 1869 zvol_task_t *task = arg; 1870 1871 switch (task->op) { 1872 case ZVOL_ASYNC_REMOVE_MINORS: 1873 zvol_remove_minors_impl(task->name1); 1874 break; 1875 case ZVOL_ASYNC_RENAME_MINORS: 1876 zvol_rename_minors_impl(task->name1, task->name2); 1877 break; 1878 case ZVOL_ASYNC_SET_SNAPDEV: 1879 zvol_set_snapdev_impl(task->name1, task->value); 1880 break; 1881 case ZVOL_ASYNC_SET_VOLMODE: 1882 zvol_set_volmode_impl(task->name1, task->value); 1883 break; 1884 default: 1885 VERIFY(0); 1886 break; 1887 } 1888 1889 zvol_task_free(task); 1890 } 1891 1892 typedef struct zvol_set_prop_int_arg { 1893 const char *zsda_name; 1894 uint64_t zsda_value; 1895 zprop_source_t zsda_source; 1896 zfs_prop_t zsda_prop; 1897 } zvol_set_prop_int_arg_t; 1898 1899 /* 1900 * Sanity check the dataset for safe use by the sync task. No additional 1901 * conditions are imposed. 1902 */ 1903 static int 1904 zvol_set_common_check(void *arg, dmu_tx_t *tx) 1905 { 1906 zvol_set_prop_int_arg_t *zsda = arg; 1907 dsl_pool_t *dp = dmu_tx_pool(tx); 1908 dsl_dir_t *dd; 1909 int error; 1910 1911 error = dsl_dir_hold(dp, zsda->zsda_name, FTAG, &dd, NULL); 1912 if (error != 0) 1913 return (error); 1914 1915 dsl_dir_rele(dd, FTAG); 1916 1917 return (error); 1918 } 1919 1920 static int 1921 zvol_set_common_sync_cb(dsl_pool_t *dp, dsl_dataset_t *ds, void *arg) 1922 { 1923 zvol_set_prop_int_arg_t *zsda = arg; 1924 char dsname[ZFS_MAX_DATASET_NAME_LEN]; 1925 zvol_task_t *task; 1926 uint64_t prop; 1927 1928 const char *prop_name = zfs_prop_to_name(zsda->zsda_prop); 1929 dsl_dataset_name(ds, dsname); 1930 1931 if (dsl_prop_get_int_ds(ds, prop_name, &prop) != 0) 1932 return (0); 1933 1934 switch (zsda->zsda_prop) { 1935 case ZFS_PROP_VOLMODE: 1936 task = zvol_task_alloc(ZVOL_ASYNC_SET_VOLMODE, dsname, 1937 NULL, prop); 1938 break; 1939 case ZFS_PROP_SNAPDEV: 1940 task = zvol_task_alloc(ZVOL_ASYNC_SET_SNAPDEV, dsname, 1941 NULL, prop); 1942 break; 1943 default: 1944 task = NULL; 1945 break; 1946 } 1947 1948 if (task == NULL) 1949 return (0); 1950 1951 (void) taskq_dispatch(dp->dp_spa->spa_zvol_taskq, zvol_task_cb, 1952 task, TQ_SLEEP); 1953 return (0); 1954 } 1955 1956 /* 1957 * Traverse all child datasets and apply the property appropriately. 1958 * We call dsl_prop_set_sync_impl() here to set the value only on the toplevel 1959 * dataset and read the effective "property" on every child in the callback 1960 * function: this is because the value is not guaranteed to be the same in the 1961 * whole dataset hierarchy. 1962 */ 1963 static void 1964 zvol_set_common_sync(void *arg, dmu_tx_t *tx) 1965 { 1966 zvol_set_prop_int_arg_t *zsda = arg; 1967 dsl_pool_t *dp = dmu_tx_pool(tx); 1968 dsl_dir_t *dd; 1969 dsl_dataset_t *ds; 1970 int error; 1971 1972 VERIFY0(dsl_dir_hold(dp, zsda->zsda_name, FTAG, &dd, NULL)); 1973 1974 error = dsl_dataset_hold(dp, zsda->zsda_name, FTAG, &ds); 1975 if (error == 0) { 1976 dsl_prop_set_sync_impl(ds, zfs_prop_to_name(zsda->zsda_prop), 1977 zsda->zsda_source, sizeof (zsda->zsda_value), 1, 1978 &zsda->zsda_value, tx); 1979 dsl_dataset_rele(ds, FTAG); 1980 } 1981 1982 dmu_objset_find_dp(dp, dd->dd_object, zvol_set_common_sync_cb, 1983 zsda, DS_FIND_CHILDREN); 1984 1985 dsl_dir_rele(dd, FTAG); 1986 } 1987 1988 int 1989 zvol_set_common(const char *ddname, zfs_prop_t prop, zprop_source_t source, 1990 uint64_t val) 1991 { 1992 zvol_set_prop_int_arg_t zsda; 1993 1994 zsda.zsda_name = ddname; 1995 zsda.zsda_source = source; 1996 zsda.zsda_value = val; 1997 zsda.zsda_prop = prop; 1998 1999 return (dsl_sync_task(ddname, zvol_set_common_check, 2000 zvol_set_common_sync, &zsda, 0, ZFS_SPACE_CHECK_NONE)); 2001 } 2002 2003 void 2004 zvol_remove_minors(spa_t *spa, const char *name, boolean_t async) 2005 { 2006 zvol_task_t *task; 2007 taskqid_t id; 2008 2009 task = zvol_task_alloc(ZVOL_ASYNC_REMOVE_MINORS, name, NULL, ~0ULL); 2010 if (task == NULL) 2011 return; 2012 2013 id = taskq_dispatch(spa->spa_zvol_taskq, zvol_task_cb, task, TQ_SLEEP); 2014 if ((async == B_FALSE) && (id != TASKQID_INVALID)) 2015 taskq_wait_id(spa->spa_zvol_taskq, id); 2016 } 2017 2018 void 2019 zvol_rename_minors(spa_t *spa, const char *name1, const char *name2, 2020 boolean_t async) 2021 { 2022 zvol_task_t *task; 2023 taskqid_t id; 2024 2025 task = zvol_task_alloc(ZVOL_ASYNC_RENAME_MINORS, name1, name2, ~0ULL); 2026 if (task == NULL) 2027 return; 2028 2029 id = taskq_dispatch(spa->spa_zvol_taskq, zvol_task_cb, task, TQ_SLEEP); 2030 if ((async == B_FALSE) && (id != TASKQID_INVALID)) 2031 taskq_wait_id(spa->spa_zvol_taskq, id); 2032 } 2033 2034 boolean_t 2035 zvol_is_zvol(const char *name) 2036 { 2037 2038 return (zvol_os_is_zvol(name)); 2039 } 2040 2041 int 2042 zvol_init_impl(void) 2043 { 2044 int i; 2045 2046 /* 2047 * zvol_threads is the module param the user passes in. 2048 * 2049 * zvol_actual_threads is what we use internally, since the user can 2050 * pass zvol_thread = 0 to mean "use all the CPUs" (the default). 2051 */ 2052 static unsigned int zvol_actual_threads; 2053 2054 if (zvol_threads == 0) { 2055 /* 2056 * See dde9380a1 for why 32 was chosen here. This should 2057 * probably be refined to be some multiple of the number 2058 * of CPUs. 2059 */ 2060 zvol_actual_threads = MAX(max_ncpus, 32); 2061 } else { 2062 zvol_actual_threads = MIN(MAX(zvol_threads, 1), 1024); 2063 } 2064 2065 /* 2066 * Use at least 32 zvol_threads but for many core system, 2067 * prefer 6 threads per taskq, but no more taskqs 2068 * than threads in them on large systems. 2069 * 2070 * taskq total 2071 * cpus taskqs threads threads 2072 * ------- ------- ------- ------- 2073 * 1 1 32 32 2074 * 2 1 32 32 2075 * 4 1 32 32 2076 * 8 2 16 32 2077 * 16 3 11 33 2078 * 32 5 7 35 2079 * 64 8 8 64 2080 * 128 11 12 132 2081 * 256 16 16 256 2082 */ 2083 zv_taskq_t *ztqs = &zvol_taskqs; 2084 int num_tqs = MIN(max_ncpus, zvol_num_taskqs); 2085 if (num_tqs == 0) { 2086 num_tqs = 1 + max_ncpus / 6; 2087 while (num_tqs * num_tqs > zvol_actual_threads) 2088 num_tqs--; 2089 } 2090 2091 int per_tq_thread = zvol_actual_threads / num_tqs; 2092 if (per_tq_thread * num_tqs < zvol_actual_threads) 2093 per_tq_thread++; 2094 2095 ztqs->tqs_cnt = num_tqs; 2096 ztqs->tqs_taskq = kmem_alloc(num_tqs * sizeof (taskq_t *), KM_SLEEP); 2097 2098 for (uint_t i = 0; i < num_tqs; i++) { 2099 char name[32]; 2100 (void) snprintf(name, sizeof (name), "%s_tq-%u", 2101 ZVOL_DRIVER, i); 2102 ztqs->tqs_taskq[i] = taskq_create(name, per_tq_thread, 2103 maxclsyspri, per_tq_thread, INT_MAX, 2104 TASKQ_PREPOPULATE | TASKQ_DYNAMIC); 2105 if (ztqs->tqs_taskq[i] == NULL) { 2106 for (int j = i - 1; j >= 0; j--) 2107 taskq_destroy(ztqs->tqs_taskq[j]); 2108 kmem_free(ztqs->tqs_taskq, ztqs->tqs_cnt * 2109 sizeof (taskq_t *)); 2110 ztqs->tqs_taskq = NULL; 2111 return (SET_ERROR(ENOMEM)); 2112 } 2113 } 2114 2115 list_create(&zvol_state_list, sizeof (zvol_state_t), 2116 offsetof(zvol_state_t, zv_next)); 2117 rw_init(&zvol_state_lock, NULL, RW_DEFAULT, NULL); 2118 2119 zvol_htable = kmem_alloc(ZVOL_HT_SIZE * sizeof (struct hlist_head), 2120 KM_SLEEP); 2121 for (i = 0; i < ZVOL_HT_SIZE; i++) 2122 INIT_HLIST_HEAD(&zvol_htable[i]); 2123 2124 return (0); 2125 } 2126 2127 void 2128 zvol_fini_impl(void) 2129 { 2130 zv_taskq_t *ztqs = &zvol_taskqs; 2131 2132 zvol_remove_minors_impl(NULL); 2133 2134 /* 2135 * The call to "zvol_remove_minors_impl" may dispatch entries to 2136 * the system_taskq, but it doesn't wait for those entries to 2137 * complete before it returns. Thus, we must wait for all of the 2138 * removals to finish, before we can continue. 2139 */ 2140 taskq_wait_outstanding(system_taskq, 0); 2141 2142 kmem_free(zvol_htable, ZVOL_HT_SIZE * sizeof (struct hlist_head)); 2143 list_destroy(&zvol_state_list); 2144 rw_destroy(&zvol_state_lock); 2145 2146 if (ztqs->tqs_taskq == NULL) { 2147 ASSERT3U(ztqs->tqs_cnt, ==, 0); 2148 } else { 2149 for (uint_t i = 0; i < ztqs->tqs_cnt; i++) { 2150 ASSERT3P(ztqs->tqs_taskq[i], !=, NULL); 2151 taskq_destroy(ztqs->tqs_taskq[i]); 2152 } 2153 kmem_free(ztqs->tqs_taskq, ztqs->tqs_cnt * 2154 sizeof (taskq_t *)); 2155 ztqs->tqs_taskq = NULL; 2156 } 2157 } 2158 2159 ZFS_MODULE_PARAM(zfs_vol, zvol_, inhibit_dev, UINT, ZMOD_RW, 2160 "Do not create zvol device nodes"); 2161 ZFS_MODULE_PARAM(zfs_vol, zvol_, prefetch_bytes, UINT, ZMOD_RW, 2162 "Prefetch N bytes at zvol start+end"); 2163 ZFS_MODULE_PARAM(zfs_vol, zvol_vol, mode, UINT, ZMOD_RW, 2164 "Default volmode property value"); 2165 ZFS_MODULE_PARAM(zfs_vol, zvol_, threads, UINT, ZMOD_RW, 2166 "Number of threads for I/O requests. Set to 0 to use all active CPUs"); 2167 ZFS_MODULE_PARAM(zfs_vol, zvol_, num_taskqs, UINT, ZMOD_RW, 2168 "Number of zvol taskqs"); 2169 ZFS_MODULE_PARAM(zfs_vol, zvol_, request_sync, UINT, ZMOD_RW, 2170 "Synchronously handle bio requests"); 2171