1 // SPDX-License-Identifier: CDDL-1.0 2 /* 3 * CDDL HEADER START 4 * 5 * The contents of this file are subject to the terms of the 6 * Common Development and Distribution License (the "License"). 7 * You may not use this file except in compliance with the License. 8 * 9 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 10 * or https://opensource.org/licenses/CDDL-1.0. 11 * See the License for the specific language governing permissions 12 * and limitations under the License. 13 * 14 * When distributing Covered Code, include this CDDL HEADER in each 15 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 16 * If applicable, add the following below this CDDL HEADER, with the 17 * fields enclosed by brackets "[]" replaced with your own identifying 18 * information: Portions Copyright [yyyy] [name of copyright owner] 19 * 20 * CDDL HEADER END 21 */ 22 /* 23 * Copyright (C) 2008-2010 Lawrence Livermore National Security, LLC. 24 * Produced at Lawrence Livermore National Laboratory (cf, DISCLAIMER). 25 * Rewritten for Linux by Brian Behlendorf <behlendorf1@llnl.gov>. 26 * LLNL-CODE-403049. 27 * 28 * ZFS volume emulation driver. 29 * 30 * Makes a DMU object look like a volume of arbitrary size, up to 2^64 bytes. 31 * Volumes are accessed through the symbolic links named: 32 * 33 * /dev/<pool_name>/<dataset_name> 34 * 35 * Volumes are persistent through reboot and module load. No user command 36 * needs to be run before opening and using a device. 37 * 38 * Copyright 2014 Nexenta Systems, Inc. All rights reserved. 39 * Copyright (c) 2016 Actifio, Inc. All rights reserved. 40 * Copyright (c) 2012, 2019 by Delphix. All rights reserved. 41 * Copyright (c) 2024, 2025, Klara, Inc. 42 */ 43 44 /* 45 * Note on locking of zvol state structures. 46 * 47 * zvol_state_t represents the connection between a single dataset 48 * (DMU_OST_ZVOL) and the device "minor" (some OS-specific representation of a 49 * "disk" or "device" or "volume", eg, a /dev/zdXX node, a GEOM object, etc). 50 * 51 * The global zvol_state_lock is used to protect access to zvol_state_list and 52 * zvol_htable, which are the primary way to obtain a zvol_state_t from a name. 53 * It should not be used for anything not name-relateds, and you should avoid 54 * sleeping or waiting while its held. See zvol_find_by_name(), zvol_insert(), 55 * zvol_remove(). 56 * 57 * The zv_state_lock is used to protect the contents of the associated 58 * zvol_state_t. Most of the zvol_state_t is dedicated to control and 59 * configuration; almost none of it is needed for data operations (that is, 60 * read, write, flush) so this lock is rarely taken during general IO. It 61 * should be released quickly; you should avoid sleeping or waiting while its 62 * held. 63 * 64 * zv_suspend_lock is used to suspend IO/data operations to a zvol. The read 65 * half should held for the duration of an IO operation. The write half should 66 * be taken when something to wait for IO to complete and the block further IO, 67 * eg for the duration of receive and rollback operations. This lock can be 68 * held for long periods of time. 69 * 70 * Thus, the following lock ordering appies. 71 * - take zvol_state_lock if necessary, to protect zvol_state_list 72 * - take zv_suspend_lock if necessary, by the code path in question 73 * - take zv_state_lock to protect zvol_state_t 74 * 75 * The minor operations are issued to spa->spa_zvol_taskq queues, that are 76 * single-threaded (to preserve order of minor operations), and are executed 77 * through the zvol_task_cb that dispatches the specific operations. Therefore, 78 * these operations are serialized per pool. Consequently, we can be certain 79 * that for a given zvol, there is only one operation at a time in progress. 80 * That is why one can be sure that first, zvol_state_t for a given zvol is 81 * allocated and placed on zvol_state_list, and then other minor operations for 82 * this zvol are going to proceed in the order of issue. 83 */ 84 85 #include <sys/dataset_kstats.h> 86 #include <sys/dbuf.h> 87 #include <sys/dmu_traverse.h> 88 #include <sys/dsl_dataset.h> 89 #include <sys/dsl_prop.h> 90 #include <sys/dsl_dir.h> 91 #include <sys/zap.h> 92 #include <sys/zfeature.h> 93 #include <sys/zil_impl.h> 94 #include <sys/dmu_tx.h> 95 #include <sys/zio.h> 96 #include <sys/zfs_rlock.h> 97 #include <sys/spa_impl.h> 98 #include <sys/zvol.h> 99 #include <sys/zvol_impl.h> 100 101 unsigned int zvol_inhibit_dev = 0; 102 unsigned int zvol_prefetch_bytes = (128 * 1024); 103 unsigned int zvol_volmode = ZFS_VOLMODE_GEOM; 104 unsigned int zvol_threads = 0; 105 unsigned int zvol_num_taskqs = 0; 106 unsigned int zvol_request_sync = 0; 107 108 struct hlist_head *zvol_htable; 109 static list_t zvol_state_list; 110 krwlock_t zvol_state_lock; 111 extern int zfs_bclone_wait_dirty; 112 zv_taskq_t zvol_taskqs; 113 114 typedef enum { 115 ZVOL_ASYNC_CREATE_MINORS, 116 ZVOL_ASYNC_REMOVE_MINORS, 117 ZVOL_ASYNC_RENAME_MINORS, 118 ZVOL_ASYNC_SET_SNAPDEV, 119 ZVOL_ASYNC_SET_VOLMODE, 120 ZVOL_ASYNC_MAX 121 } zvol_async_op_t; 122 123 typedef struct { 124 zvol_async_op_t zt_op; 125 char zt_name1[MAXNAMELEN]; 126 char zt_name2[MAXNAMELEN]; 127 uint64_t zt_value; 128 uint32_t zt_total; 129 uint32_t zt_done; 130 int32_t zt_status; 131 int zt_error; 132 } zvol_task_t; 133 134 zv_request_task_t * 135 zv_request_task_create(zv_request_t zvr) 136 { 137 zv_request_task_t *task; 138 task = kmem_alloc(sizeof (zv_request_task_t), KM_SLEEP); 139 taskq_init_ent(&task->ent); 140 task->zvr = zvr; 141 return (task); 142 } 143 144 void 145 zv_request_task_free(zv_request_task_t *task) 146 { 147 kmem_free(task, sizeof (*task)); 148 } 149 150 uint64_t 151 zvol_name_hash(const char *name) 152 { 153 uint64_t crc = -1ULL; 154 ASSERT(zfs_crc64_table[128] == ZFS_CRC64_POLY); 155 for (const uint8_t *p = (const uint8_t *)name; *p != 0; p++) 156 crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ (*p)) & 0xFF]; 157 return (crc); 158 } 159 160 /* 161 * Find a zvol_state_t given the name and hash generated by zvol_name_hash. 162 * If found, return with zv_suspend_lock and zv_state_lock taken, otherwise, 163 * return (NULL) without the taking locks. The zv_suspend_lock is always taken 164 * before zv_state_lock. The mode argument indicates the mode (including none) 165 * for zv_suspend_lock to be taken. 166 */ 167 zvol_state_t * 168 zvol_find_by_name_hash(const char *name, uint64_t hash, int mode) 169 { 170 zvol_state_t *zv; 171 struct hlist_node *p = NULL; 172 173 rw_enter(&zvol_state_lock, RW_READER); 174 hlist_for_each(p, ZVOL_HT_HEAD(hash)) { 175 zv = hlist_entry(p, zvol_state_t, zv_hlink); 176 mutex_enter(&zv->zv_state_lock); 177 if (zv->zv_hash == hash && strcmp(zv->zv_name, name) == 0) { 178 /* 179 * this is the right zvol, take the locks in the 180 * right order 181 */ 182 if (mode != RW_NONE && 183 !rw_tryenter(&zv->zv_suspend_lock, mode)) { 184 mutex_exit(&zv->zv_state_lock); 185 rw_enter(&zv->zv_suspend_lock, mode); 186 mutex_enter(&zv->zv_state_lock); 187 /* 188 * zvol cannot be renamed as we continue 189 * to hold zvol_state_lock 190 */ 191 ASSERT(zv->zv_hash == hash && 192 strcmp(zv->zv_name, name) == 0); 193 } 194 rw_exit(&zvol_state_lock); 195 return (zv); 196 } 197 mutex_exit(&zv->zv_state_lock); 198 } 199 rw_exit(&zvol_state_lock); 200 201 return (NULL); 202 } 203 204 /* 205 * Find a zvol_state_t given the name. 206 * If found, return with zv_suspend_lock and zv_state_lock taken, otherwise, 207 * return (NULL) without the taking locks. The zv_suspend_lock is always taken 208 * before zv_state_lock. The mode argument indicates the mode (including none) 209 * for zv_suspend_lock to be taken. 210 */ 211 static zvol_state_t * 212 zvol_find_by_name(const char *name, int mode) 213 { 214 return (zvol_find_by_name_hash(name, zvol_name_hash(name), mode)); 215 } 216 217 /* 218 * ZFS_IOC_CREATE callback handles dmu zvol and zap object creation. 219 */ 220 void 221 zvol_create_cb(objset_t *os, void *arg, cred_t *cr, dmu_tx_t *tx) 222 { 223 zfs_creat_t *zct = arg; 224 nvlist_t *nvprops = zct->zct_props; 225 int error; 226 uint64_t volblocksize, volsize; 227 228 VERIFY0(nvlist_lookup_uint64(nvprops, 229 zfs_prop_to_name(ZFS_PROP_VOLSIZE), &volsize)); 230 if (nvlist_lookup_uint64(nvprops, 231 zfs_prop_to_name(ZFS_PROP_VOLBLOCKSIZE), &volblocksize) != 0) 232 volblocksize = zfs_prop_default_numeric(ZFS_PROP_VOLBLOCKSIZE); 233 234 /* 235 * These properties must be removed from the list so the generic 236 * property setting step won't apply to them. 237 */ 238 VERIFY0(nvlist_remove_all(nvprops, zfs_prop_to_name(ZFS_PROP_VOLSIZE))); 239 (void) nvlist_remove_all(nvprops, 240 zfs_prop_to_name(ZFS_PROP_VOLBLOCKSIZE)); 241 242 error = dmu_object_claim(os, ZVOL_OBJ, DMU_OT_ZVOL, volblocksize, 243 DMU_OT_NONE, 0, tx); 244 ASSERT0(error); 245 246 error = zap_create_claim(os, ZVOL_ZAP_OBJ, DMU_OT_ZVOL_PROP, 247 DMU_OT_NONE, 0, tx); 248 ASSERT0(error); 249 250 error = zap_update(os, ZVOL_ZAP_OBJ, "size", 8, 1, &volsize, tx); 251 ASSERT0(error); 252 } 253 254 /* 255 * ZFS_IOC_OBJSET_STATS entry point. 256 */ 257 int 258 zvol_get_stats(objset_t *os, nvlist_t *nv) 259 { 260 int error; 261 dmu_object_info_t *doi; 262 uint64_t val; 263 264 error = zap_lookup(os, ZVOL_ZAP_OBJ, "size", 8, 1, &val); 265 if (error) 266 return (error); 267 268 dsl_prop_nvlist_add_uint64(nv, ZFS_PROP_VOLSIZE, val); 269 doi = kmem_alloc(sizeof (dmu_object_info_t), KM_SLEEP); 270 error = dmu_object_info(os, ZVOL_OBJ, doi); 271 272 if (error == 0) { 273 dsl_prop_nvlist_add_uint64(nv, ZFS_PROP_VOLBLOCKSIZE, 274 doi->doi_data_block_size); 275 } 276 277 kmem_free(doi, sizeof (dmu_object_info_t)); 278 279 return (error); 280 } 281 282 /* 283 * Sanity check volume size. 284 */ 285 int 286 zvol_check_volsize(uint64_t volsize, uint64_t blocksize) 287 { 288 if (volsize == 0) 289 return (SET_ERROR(EINVAL)); 290 291 if (volsize % blocksize != 0) 292 return (SET_ERROR(EINVAL)); 293 294 #ifdef _ILP32 295 if (volsize - 1 > SPEC_MAXOFFSET_T) 296 return (SET_ERROR(EOVERFLOW)); 297 #endif 298 return (0); 299 } 300 301 /* 302 * Ensure the zap is flushed then inform the VFS of the capacity change. 303 */ 304 static int 305 zvol_update_volsize(uint64_t volsize, objset_t *os) 306 { 307 dmu_tx_t *tx; 308 int error; 309 uint64_t txg; 310 311 tx = dmu_tx_create(os); 312 dmu_tx_hold_zap(tx, ZVOL_ZAP_OBJ, TRUE, NULL); 313 dmu_tx_mark_netfree(tx); 314 error = dmu_tx_assign(tx, DMU_TX_WAIT); 315 if (error) { 316 dmu_tx_abort(tx); 317 return (error); 318 } 319 txg = dmu_tx_get_txg(tx); 320 321 error = zap_update(os, ZVOL_ZAP_OBJ, "size", 8, 1, 322 &volsize, tx); 323 dmu_tx_commit(tx); 324 325 txg_wait_synced(dmu_objset_pool(os), txg); 326 327 if (error == 0) 328 error = dmu_free_long_range(os, 329 ZVOL_OBJ, volsize, DMU_OBJECT_END); 330 331 return (error); 332 } 333 334 /* 335 * Set ZFS_PROP_VOLSIZE set entry point. Note that modifying the volume 336 * size will result in a udev "change" event being generated. 337 */ 338 int 339 zvol_set_volsize(const char *name, uint64_t volsize) 340 { 341 objset_t *os = NULL; 342 uint64_t readonly; 343 int error; 344 boolean_t owned = B_FALSE; 345 346 error = dsl_prop_get_integer(name, 347 zfs_prop_to_name(ZFS_PROP_READONLY), &readonly, NULL); 348 if (error != 0) 349 return (error); 350 if (readonly) 351 return (SET_ERROR(EROFS)); 352 353 zvol_state_t *zv = zvol_find_by_name(name, RW_READER); 354 355 ASSERT(zv == NULL || (MUTEX_HELD(&zv->zv_state_lock) && 356 RW_READ_HELD(&zv->zv_suspend_lock))); 357 358 if (zv == NULL || zv->zv_objset == NULL) { 359 if (zv != NULL) 360 rw_exit(&zv->zv_suspend_lock); 361 if ((error = dmu_objset_own(name, DMU_OST_ZVOL, B_FALSE, B_TRUE, 362 FTAG, &os)) != 0) { 363 if (zv != NULL) 364 mutex_exit(&zv->zv_state_lock); 365 return (error); 366 } 367 owned = B_TRUE; 368 if (zv != NULL) 369 zv->zv_objset = os; 370 } else { 371 os = zv->zv_objset; 372 } 373 374 dmu_object_info_t *doi = kmem_alloc(sizeof (*doi), KM_SLEEP); 375 376 if ((error = dmu_object_info(os, ZVOL_OBJ, doi)) || 377 (error = zvol_check_volsize(volsize, doi->doi_data_block_size))) 378 goto out; 379 380 error = zvol_update_volsize(volsize, os); 381 if (error == 0 && zv != NULL) { 382 zv->zv_volsize = volsize; 383 zv->zv_changed = 1; 384 } 385 out: 386 kmem_free(doi, sizeof (dmu_object_info_t)); 387 388 if (owned) { 389 dmu_objset_disown(os, B_TRUE, FTAG); 390 if (zv != NULL) 391 zv->zv_objset = NULL; 392 } else { 393 rw_exit(&zv->zv_suspend_lock); 394 } 395 396 if (zv != NULL) 397 mutex_exit(&zv->zv_state_lock); 398 399 if (error == 0 && zv != NULL) 400 zvol_os_update_volsize(zv, volsize); 401 402 return (error); 403 } 404 405 /* 406 * Update volthreading. 407 */ 408 int 409 zvol_set_volthreading(const char *name, boolean_t value) 410 { 411 zvol_state_t *zv = zvol_find_by_name(name, RW_NONE); 412 if (zv == NULL) 413 return (SET_ERROR(ENOENT)); 414 zv->zv_threading = value; 415 mutex_exit(&zv->zv_state_lock); 416 return (0); 417 } 418 419 /* 420 * Update zvol ro property. 421 */ 422 int 423 zvol_set_ro(const char *name, boolean_t value) 424 { 425 zvol_state_t *zv = zvol_find_by_name(name, RW_NONE); 426 if (zv == NULL) 427 return (-1); 428 if (value) { 429 zvol_os_set_disk_ro(zv, 1); 430 zv->zv_flags |= ZVOL_RDONLY; 431 } else { 432 zvol_os_set_disk_ro(zv, 0); 433 zv->zv_flags &= ~ZVOL_RDONLY; 434 } 435 mutex_exit(&zv->zv_state_lock); 436 return (0); 437 } 438 439 /* 440 * Sanity check volume block size. 441 */ 442 int 443 zvol_check_volblocksize(const char *name, uint64_t volblocksize) 444 { 445 /* Record sizes above 128k need the feature to be enabled */ 446 if (volblocksize > SPA_OLD_MAXBLOCKSIZE) { 447 spa_t *spa; 448 int error; 449 450 if ((error = spa_open(name, &spa, FTAG)) != 0) 451 return (error); 452 453 if (!spa_feature_is_enabled(spa, SPA_FEATURE_LARGE_BLOCKS)) { 454 spa_close(spa, FTAG); 455 return (SET_ERROR(ENOTSUP)); 456 } 457 458 /* 459 * We don't allow setting the property above 1MB, 460 * unless the tunable has been changed. 461 */ 462 if (volblocksize > zfs_max_recordsize) { 463 spa_close(spa, FTAG); 464 return (SET_ERROR(EDOM)); 465 } 466 467 spa_close(spa, FTAG); 468 } 469 470 if (volblocksize < SPA_MINBLOCKSIZE || 471 volblocksize > SPA_MAXBLOCKSIZE || 472 !ISP2(volblocksize)) 473 return (SET_ERROR(EDOM)); 474 475 return (0); 476 } 477 478 /* 479 * Replay a TX_TRUNCATE ZIL transaction if asked. TX_TRUNCATE is how we 480 * implement DKIOCFREE/free-long-range. 481 */ 482 static int 483 zvol_replay_truncate(void *arg1, void *arg2, boolean_t byteswap) 484 { 485 zvol_state_t *zv = arg1; 486 lr_truncate_t *lr = arg2; 487 uint64_t offset, length; 488 489 ASSERT3U(lr->lr_common.lrc_reclen, >=, sizeof (*lr)); 490 491 if (byteswap) 492 byteswap_uint64_array(lr, sizeof (*lr)); 493 494 offset = lr->lr_offset; 495 length = lr->lr_length; 496 497 dmu_tx_t *tx = dmu_tx_create(zv->zv_objset); 498 dmu_tx_mark_netfree(tx); 499 int error = dmu_tx_assign(tx, DMU_TX_WAIT); 500 if (error != 0) { 501 dmu_tx_abort(tx); 502 } else { 503 (void) zil_replaying(zv->zv_zilog, tx); 504 dmu_tx_commit(tx); 505 error = dmu_free_long_range(zv->zv_objset, ZVOL_OBJ, offset, 506 length); 507 } 508 509 return (error); 510 } 511 512 /* 513 * Replay a TX_WRITE ZIL transaction that didn't get committed 514 * after a system failure 515 */ 516 static int 517 zvol_replay_write(void *arg1, void *arg2, boolean_t byteswap) 518 { 519 zvol_state_t *zv = arg1; 520 lr_write_t *lr = arg2; 521 objset_t *os = zv->zv_objset; 522 char *data = (char *)(lr + 1); /* data follows lr_write_t */ 523 uint64_t offset, length; 524 dmu_tx_t *tx; 525 int error; 526 527 ASSERT3U(lr->lr_common.lrc_reclen, >=, sizeof (*lr)); 528 529 if (byteswap) 530 byteswap_uint64_array(lr, sizeof (*lr)); 531 532 offset = lr->lr_offset; 533 length = lr->lr_length; 534 535 /* If it's a dmu_sync() block, write the whole block */ 536 if (lr->lr_common.lrc_reclen == sizeof (lr_write_t)) { 537 uint64_t blocksize = BP_GET_LSIZE(&lr->lr_blkptr); 538 if (length < blocksize) { 539 offset -= offset % blocksize; 540 length = blocksize; 541 } 542 } 543 544 tx = dmu_tx_create(os); 545 dmu_tx_hold_write(tx, ZVOL_OBJ, offset, length); 546 error = dmu_tx_assign(tx, DMU_TX_WAIT); 547 if (error) { 548 dmu_tx_abort(tx); 549 } else { 550 dmu_write(os, ZVOL_OBJ, offset, length, data, tx); 551 (void) zil_replaying(zv->zv_zilog, tx); 552 dmu_tx_commit(tx); 553 } 554 555 return (error); 556 } 557 558 /* 559 * Replay a TX_CLONE_RANGE ZIL transaction that didn't get committed 560 * after a system failure 561 */ 562 static int 563 zvol_replay_clone_range(void *arg1, void *arg2, boolean_t byteswap) 564 { 565 zvol_state_t *zv = arg1; 566 lr_clone_range_t *lr = arg2; 567 objset_t *os = zv->zv_objset; 568 dmu_tx_t *tx; 569 int error; 570 uint64_t blksz; 571 uint64_t off; 572 uint64_t len; 573 574 ASSERT3U(lr->lr_common.lrc_reclen, >=, sizeof (*lr)); 575 ASSERT3U(lr->lr_common.lrc_reclen, >=, offsetof(lr_clone_range_t, 576 lr_bps[lr->lr_nbps])); 577 578 if (byteswap) 579 byteswap_uint64_array(lr, sizeof (*lr)); 580 581 ASSERT(spa_feature_is_enabled(dmu_objset_spa(os), 582 SPA_FEATURE_BLOCK_CLONING)); 583 584 off = lr->lr_offset; 585 len = lr->lr_length; 586 blksz = lr->lr_blksz; 587 588 if ((off % blksz) != 0) { 589 return (SET_ERROR(EINVAL)); 590 } 591 592 error = dnode_hold(os, ZVOL_OBJ, zv, &zv->zv_dn); 593 if (error != 0 || !zv->zv_dn) 594 return (error); 595 tx = dmu_tx_create(os); 596 dmu_tx_hold_clone_by_dnode(tx, zv->zv_dn, off, len, blksz); 597 error = dmu_tx_assign(tx, DMU_TX_WAIT); 598 if (error != 0) { 599 dmu_tx_abort(tx); 600 goto out; 601 } 602 error = dmu_brt_clone(zv->zv_objset, ZVOL_OBJ, off, len, 603 tx, lr->lr_bps, lr->lr_nbps); 604 if (error != 0) { 605 dmu_tx_commit(tx); 606 goto out; 607 } 608 609 /* 610 * zil_replaying() not only check if we are replaying ZIL, but also 611 * updates the ZIL header to record replay progress. 612 */ 613 VERIFY(zil_replaying(zv->zv_zilog, tx)); 614 dmu_tx_commit(tx); 615 616 out: 617 dnode_rele(zv->zv_dn, zv); 618 zv->zv_dn = NULL; 619 return (error); 620 } 621 622 int 623 zvol_clone_range(zvol_state_t *zv_src, uint64_t inoff, zvol_state_t *zv_dst, 624 uint64_t outoff, uint64_t len) 625 { 626 zilog_t *zilog_dst; 627 zfs_locked_range_t *inlr, *outlr; 628 objset_t *inos, *outos; 629 dmu_tx_t *tx; 630 blkptr_t *bps; 631 size_t maxblocks; 632 int error = 0; 633 634 rw_enter(&zv_dst->zv_suspend_lock, RW_READER); 635 if (zv_dst->zv_zilog == NULL) { 636 rw_exit(&zv_dst->zv_suspend_lock); 637 rw_enter(&zv_dst->zv_suspend_lock, RW_WRITER); 638 if (zv_dst->zv_zilog == NULL) { 639 zv_dst->zv_zilog = zil_open(zv_dst->zv_objset, 640 zvol_get_data, &zv_dst->zv_kstat.dk_zil_sums); 641 zv_dst->zv_flags |= ZVOL_WRITTEN_TO; 642 VERIFY0((zv_dst->zv_zilog->zl_header->zh_flags & 643 ZIL_REPLAY_NEEDED)); 644 } 645 rw_downgrade(&zv_dst->zv_suspend_lock); 646 } 647 if (zv_src != zv_dst) 648 rw_enter(&zv_src->zv_suspend_lock, RW_READER); 649 650 inos = zv_src->zv_objset; 651 outos = zv_dst->zv_objset; 652 653 /* 654 * Sanity checks 655 */ 656 if (!spa_feature_is_enabled(dmu_objset_spa(outos), 657 SPA_FEATURE_BLOCK_CLONING)) { 658 error = SET_ERROR(EOPNOTSUPP); 659 goto out; 660 } 661 if (dmu_objset_spa(inos) != dmu_objset_spa(outos)) { 662 error = SET_ERROR(EXDEV); 663 goto out; 664 } 665 if (inos->os_encrypted != outos->os_encrypted) { 666 error = SET_ERROR(EXDEV); 667 goto out; 668 } 669 if (zv_src->zv_volblocksize != zv_dst->zv_volblocksize) { 670 error = SET_ERROR(EINVAL); 671 goto out; 672 } 673 if (inoff >= zv_src->zv_volsize || outoff >= zv_dst->zv_volsize) { 674 goto out; 675 } 676 677 /* 678 * Do not read beyond boundary 679 */ 680 if (len > zv_src->zv_volsize - inoff) 681 len = zv_src->zv_volsize - inoff; 682 if (len > zv_dst->zv_volsize - outoff) 683 len = zv_dst->zv_volsize - outoff; 684 if (len == 0) 685 goto out; 686 687 /* 688 * No overlapping if we are cloning within the same file 689 */ 690 if (zv_src == zv_dst) { 691 if (inoff < outoff + len && outoff < inoff + len) { 692 error = SET_ERROR(EINVAL); 693 goto out; 694 } 695 } 696 697 /* 698 * Offsets and length must be at block boundaries 699 */ 700 if ((inoff % zv_src->zv_volblocksize) != 0 || 701 (outoff % zv_dst->zv_volblocksize) != 0) { 702 error = SET_ERROR(EINVAL); 703 goto out; 704 } 705 706 /* 707 * Length must be multiple of block size 708 */ 709 if ((len % zv_src->zv_volblocksize) != 0) { 710 error = SET_ERROR(EINVAL); 711 goto out; 712 } 713 714 zilog_dst = zv_dst->zv_zilog; 715 maxblocks = zil_max_log_data(zilog_dst, sizeof (lr_clone_range_t)) / 716 sizeof (bps[0]); 717 bps = vmem_alloc(sizeof (bps[0]) * maxblocks, KM_SLEEP); 718 /* 719 * Maintain predictable lock order. 720 */ 721 if (zv_src < zv_dst || (zv_src == zv_dst && inoff < outoff)) { 722 inlr = zfs_rangelock_enter(&zv_src->zv_rangelock, inoff, len, 723 RL_READER); 724 outlr = zfs_rangelock_enter(&zv_dst->zv_rangelock, outoff, len, 725 RL_WRITER); 726 } else { 727 outlr = zfs_rangelock_enter(&zv_dst->zv_rangelock, outoff, len, 728 RL_WRITER); 729 inlr = zfs_rangelock_enter(&zv_src->zv_rangelock, inoff, len, 730 RL_READER); 731 } 732 733 while (len > 0) { 734 uint64_t size, last_synced_txg; 735 size_t nbps = maxblocks; 736 size = MIN(zv_src->zv_volblocksize * maxblocks, len); 737 last_synced_txg = spa_last_synced_txg( 738 dmu_objset_spa(zv_src->zv_objset)); 739 error = dmu_read_l0_bps(zv_src->zv_objset, ZVOL_OBJ, inoff, 740 size, bps, &nbps); 741 if (error != 0) { 742 /* 743 * If we are trying to clone a block that was created 744 * in the current transaction group, the error will be 745 * EAGAIN here. Based on zfs_bclone_wait_dirty either 746 * return a shortened range to the caller so it can 747 * fallback, or wait for the next TXG and check again. 748 */ 749 if (error == EAGAIN && zfs_bclone_wait_dirty) { 750 txg_wait_synced(dmu_objset_pool 751 (zv_src->zv_objset), last_synced_txg + 1); 752 continue; 753 } 754 break; 755 } 756 757 tx = dmu_tx_create(zv_dst->zv_objset); 758 dmu_tx_hold_clone_by_dnode(tx, zv_dst->zv_dn, outoff, size, 759 zv_src->zv_volblocksize); 760 error = dmu_tx_assign(tx, DMU_TX_WAIT); 761 if (error != 0) { 762 dmu_tx_abort(tx); 763 break; 764 } 765 error = dmu_brt_clone(zv_dst->zv_objset, ZVOL_OBJ, outoff, size, 766 tx, bps, nbps); 767 if (error != 0) { 768 dmu_tx_commit(tx); 769 break; 770 } 771 zvol_log_clone_range(zilog_dst, tx, TX_CLONE_RANGE, outoff, 772 size, zv_src->zv_volblocksize, bps, nbps); 773 dmu_tx_commit(tx); 774 inoff += size; 775 outoff += size; 776 len -= size; 777 } 778 vmem_free(bps, sizeof (bps[0]) * maxblocks); 779 zfs_rangelock_exit(outlr); 780 zfs_rangelock_exit(inlr); 781 if (error == 0 && zv_dst->zv_objset->os_sync == ZFS_SYNC_ALWAYS) { 782 error = zil_commit(zilog_dst, ZVOL_OBJ); 783 } 784 out: 785 if (zv_src != zv_dst) 786 rw_exit(&zv_src->zv_suspend_lock); 787 rw_exit(&zv_dst->zv_suspend_lock); 788 return (error); 789 } 790 791 /* 792 * Handles TX_CLONE_RANGE transactions. 793 */ 794 void 795 zvol_log_clone_range(zilog_t *zilog, dmu_tx_t *tx, int txtype, uint64_t off, 796 uint64_t len, uint64_t blksz, const blkptr_t *bps, size_t nbps) 797 { 798 itx_t *itx; 799 lr_clone_range_t *lr; 800 uint64_t partlen, max_log_data; 801 size_t partnbps; 802 803 if (zil_replaying(zilog, tx)) 804 return; 805 806 max_log_data = zil_max_log_data(zilog, sizeof (lr_clone_range_t)); 807 808 while (nbps > 0) { 809 partnbps = MIN(nbps, max_log_data / sizeof (bps[0])); 810 partlen = partnbps * blksz; 811 ASSERT3U(partlen, <, len + blksz); 812 partlen = MIN(partlen, len); 813 814 itx = zil_itx_create(txtype, 815 sizeof (*lr) + sizeof (bps[0]) * partnbps); 816 lr = (lr_clone_range_t *)&itx->itx_lr; 817 lr->lr_foid = ZVOL_OBJ; 818 lr->lr_offset = off; 819 lr->lr_length = partlen; 820 lr->lr_blksz = blksz; 821 lr->lr_nbps = partnbps; 822 memcpy(lr->lr_bps, bps, sizeof (bps[0]) * partnbps); 823 824 zil_itx_assign(zilog, itx, tx); 825 826 bps += partnbps; 827 ASSERT3U(nbps, >=, partnbps); 828 nbps -= partnbps; 829 off += partlen; 830 ASSERT3U(len, >=, partlen); 831 len -= partlen; 832 } 833 } 834 835 static int 836 zvol_replay_err(void *arg1, void *arg2, boolean_t byteswap) 837 { 838 (void) arg1, (void) arg2, (void) byteswap; 839 return (SET_ERROR(ENOTSUP)); 840 } 841 842 /* 843 * Callback vectors for replaying records. 844 * Only TX_WRITE and TX_TRUNCATE are needed for zvol. 845 */ 846 zil_replay_func_t *const zvol_replay_vector[TX_MAX_TYPE] = { 847 zvol_replay_err, /* no such transaction type */ 848 zvol_replay_err, /* TX_CREATE */ 849 zvol_replay_err, /* TX_MKDIR */ 850 zvol_replay_err, /* TX_MKXATTR */ 851 zvol_replay_err, /* TX_SYMLINK */ 852 zvol_replay_err, /* TX_REMOVE */ 853 zvol_replay_err, /* TX_RMDIR */ 854 zvol_replay_err, /* TX_LINK */ 855 zvol_replay_err, /* TX_RENAME */ 856 zvol_replay_write, /* TX_WRITE */ 857 zvol_replay_truncate, /* TX_TRUNCATE */ 858 zvol_replay_err, /* TX_SETATTR */ 859 zvol_replay_err, /* TX_ACL_V0 */ 860 zvol_replay_err, /* TX_ACL */ 861 zvol_replay_err, /* TX_CREATE_ACL */ 862 zvol_replay_err, /* TX_CREATE_ATTR */ 863 zvol_replay_err, /* TX_CREATE_ACL_ATTR */ 864 zvol_replay_err, /* TX_MKDIR_ACL */ 865 zvol_replay_err, /* TX_MKDIR_ATTR */ 866 zvol_replay_err, /* TX_MKDIR_ACL_ATTR */ 867 zvol_replay_err, /* TX_WRITE2 */ 868 zvol_replay_err, /* TX_SETSAXATTR */ 869 zvol_replay_err, /* TX_RENAME_EXCHANGE */ 870 zvol_replay_err, /* TX_RENAME_WHITEOUT */ 871 zvol_replay_clone_range, /* TX_CLONE_RANGE */ 872 }; 873 874 /* 875 * zvol_log_write() handles TX_WRITE transactions. 876 */ 877 void 878 zvol_log_write(zvol_state_t *zv, dmu_tx_t *tx, uint64_t offset, 879 uint64_t size, boolean_t commit) 880 { 881 uint32_t blocksize = zv->zv_volblocksize; 882 zilog_t *zilog = zv->zv_zilog; 883 itx_wr_state_t write_state; 884 uint64_t log_size = 0; 885 886 if (zil_replaying(zilog, tx)) 887 return; 888 889 write_state = zil_write_state(zilog, size, blocksize, B_FALSE, commit); 890 891 while (size) { 892 itx_t *itx; 893 lr_write_t *lr; 894 itx_wr_state_t wr_state = write_state; 895 ssize_t len = size; 896 897 if (wr_state == WR_COPIED && size > zil_max_copied_data(zilog)) 898 wr_state = WR_NEED_COPY; 899 else if (wr_state == WR_INDIRECT) 900 len = MIN(blocksize - P2PHASE(offset, blocksize), size); 901 902 itx = zil_itx_create(TX_WRITE, sizeof (*lr) + 903 (wr_state == WR_COPIED ? len : 0)); 904 lr = (lr_write_t *)&itx->itx_lr; 905 if (wr_state == WR_COPIED && 906 dmu_read_by_dnode(zv->zv_dn, offset, len, lr + 1, 907 DMU_READ_NO_PREFETCH | DMU_KEEP_CACHING) != 0) { 908 zil_itx_destroy(itx, 0); 909 itx = zil_itx_create(TX_WRITE, sizeof (*lr)); 910 lr = (lr_write_t *)&itx->itx_lr; 911 wr_state = WR_NEED_COPY; 912 } 913 914 log_size += itx->itx_size; 915 if (wr_state == WR_NEED_COPY) 916 log_size += len; 917 918 itx->itx_wr_state = wr_state; 919 lr->lr_foid = ZVOL_OBJ; 920 lr->lr_offset = offset; 921 lr->lr_length = len; 922 lr->lr_blkoff = 0; 923 BP_ZERO(&lr->lr_blkptr); 924 925 itx->itx_private = zv; 926 927 zil_itx_assign(zilog, itx, tx); 928 929 offset += len; 930 size -= len; 931 } 932 933 dsl_pool_wrlog_count(zilog->zl_dmu_pool, log_size, tx->tx_txg); 934 } 935 936 /* 937 * Log a DKIOCFREE/free-long-range to the ZIL with TX_TRUNCATE. 938 */ 939 void 940 zvol_log_truncate(zvol_state_t *zv, dmu_tx_t *tx, uint64_t off, uint64_t len) 941 { 942 itx_t *itx; 943 lr_truncate_t *lr; 944 zilog_t *zilog = zv->zv_zilog; 945 946 if (zil_replaying(zilog, tx)) 947 return; 948 949 itx = zil_itx_create(TX_TRUNCATE, sizeof (*lr)); 950 lr = (lr_truncate_t *)&itx->itx_lr; 951 lr->lr_foid = ZVOL_OBJ; 952 lr->lr_offset = off; 953 lr->lr_length = len; 954 955 zil_itx_assign(zilog, itx, tx); 956 } 957 958 959 static void 960 zvol_get_done(zgd_t *zgd, int error) 961 { 962 (void) error; 963 if (zgd->zgd_db) 964 dmu_buf_rele(zgd->zgd_db, zgd); 965 966 zfs_rangelock_exit(zgd->zgd_lr); 967 968 kmem_free(zgd, sizeof (zgd_t)); 969 } 970 971 /* 972 * Get data to generate a TX_WRITE intent log record. 973 */ 974 int 975 zvol_get_data(void *arg, uint64_t arg2, lr_write_t *lr, char *buf, 976 struct lwb *lwb, zio_t *zio) 977 { 978 zvol_state_t *zv = arg; 979 uint64_t offset = lr->lr_offset; 980 uint64_t size = lr->lr_length; 981 dmu_buf_t *db; 982 zgd_t *zgd; 983 int error; 984 985 ASSERT3P(lwb, !=, NULL); 986 ASSERT3U(size, !=, 0); 987 988 zgd = kmem_zalloc(sizeof (zgd_t), KM_SLEEP); 989 zgd->zgd_lwb = lwb; 990 991 /* 992 * Write records come in two flavors: immediate and indirect. 993 * For small writes it's cheaper to store the data with the 994 * log record (immediate); for large writes it's cheaper to 995 * sync the data and get a pointer to it (indirect) so that 996 * we don't have to write the data twice. 997 */ 998 if (buf != NULL) { /* immediate write */ 999 zgd->zgd_lr = zfs_rangelock_enter(&zv->zv_rangelock, offset, 1000 size, RL_READER); 1001 error = dmu_read_by_dnode(zv->zv_dn, offset, size, buf, 1002 DMU_READ_NO_PREFETCH | DMU_KEEP_CACHING); 1003 } else { /* indirect write */ 1004 ASSERT3P(zio, !=, NULL); 1005 /* 1006 * Have to lock the whole block to ensure when it's written out 1007 * and its checksum is being calculated that no one can change 1008 * the data. Contrarily to zfs_get_data we need not re-check 1009 * blocksize after we get the lock because it cannot be changed. 1010 */ 1011 size = zv->zv_volblocksize; 1012 offset = P2ALIGN_TYPED(offset, size, uint64_t); 1013 zgd->zgd_lr = zfs_rangelock_enter(&zv->zv_rangelock, offset, 1014 size, RL_READER); 1015 error = dmu_buf_hold_noread_by_dnode(zv->zv_dn, offset, zgd, 1016 &db); 1017 if (error == 0) { 1018 blkptr_t *bp = &lr->lr_blkptr; 1019 1020 zgd->zgd_db = db; 1021 zgd->zgd_bp = bp; 1022 1023 ASSERT(db != NULL); 1024 ASSERT(db->db_offset == offset); 1025 ASSERT(db->db_size == size); 1026 1027 error = dmu_sync(zio, lr->lr_common.lrc_txg, 1028 zvol_get_done, zgd); 1029 1030 if (error == 0) 1031 return (0); 1032 } 1033 } 1034 1035 zvol_get_done(zgd, error); 1036 1037 return (error); 1038 } 1039 1040 /* 1041 * The zvol_state_t's are inserted into zvol_state_list and zvol_htable. 1042 */ 1043 1044 void 1045 zvol_insert(zvol_state_t *zv) 1046 { 1047 ASSERT(RW_WRITE_HELD(&zvol_state_lock)); 1048 list_insert_head(&zvol_state_list, zv); 1049 hlist_add_head(&zv->zv_hlink, ZVOL_HT_HEAD(zv->zv_hash)); 1050 } 1051 1052 /* 1053 * Simply remove the zvol from to list of zvols. 1054 */ 1055 static void 1056 zvol_remove(zvol_state_t *zv) 1057 { 1058 ASSERT(RW_WRITE_HELD(&zvol_state_lock)); 1059 list_remove(&zvol_state_list, zv); 1060 hlist_del(&zv->zv_hlink); 1061 } 1062 1063 /* 1064 * Setup zv after we just own the zv->objset 1065 */ 1066 static int 1067 zvol_setup_zv(zvol_state_t *zv) 1068 { 1069 uint64_t volsize; 1070 int error; 1071 uint64_t ro; 1072 objset_t *os = zv->zv_objset; 1073 1074 ASSERT(MUTEX_HELD(&zv->zv_state_lock)); 1075 ASSERT(RW_LOCK_HELD(&zv->zv_suspend_lock)); 1076 1077 zv->zv_zilog = NULL; 1078 zv->zv_flags &= ~ZVOL_WRITTEN_TO; 1079 1080 error = dsl_prop_get_integer(zv->zv_name, "readonly", &ro, NULL); 1081 if (error) 1082 return (error); 1083 1084 error = zap_lookup(os, ZVOL_ZAP_OBJ, "size", 8, 1, &volsize); 1085 if (error) 1086 return (error); 1087 1088 error = dnode_hold(os, ZVOL_OBJ, zv, &zv->zv_dn); 1089 if (error) 1090 return (error); 1091 1092 zvol_os_set_capacity(zv, volsize >> 9); 1093 zv->zv_volsize = volsize; 1094 1095 if (ro || dmu_objset_is_snapshot(os) || 1096 !spa_writeable(dmu_objset_spa(os))) { 1097 zvol_os_set_disk_ro(zv, 1); 1098 zv->zv_flags |= ZVOL_RDONLY; 1099 } else { 1100 zvol_os_set_disk_ro(zv, 0); 1101 zv->zv_flags &= ~ZVOL_RDONLY; 1102 } 1103 return (0); 1104 } 1105 1106 /* 1107 * Shutdown every zv_objset related stuff except zv_objset itself. 1108 * The is the reverse of zvol_setup_zv. 1109 */ 1110 static void 1111 zvol_shutdown_zv(zvol_state_t *zv) 1112 { 1113 ASSERT(MUTEX_HELD(&zv->zv_state_lock) && 1114 RW_LOCK_HELD(&zv->zv_suspend_lock)); 1115 1116 if (zv->zv_flags & ZVOL_WRITTEN_TO) { 1117 ASSERT(zv->zv_zilog != NULL); 1118 zil_close(zv->zv_zilog); 1119 } 1120 1121 zv->zv_zilog = NULL; 1122 1123 dnode_rele(zv->zv_dn, zv); 1124 zv->zv_dn = NULL; 1125 1126 /* 1127 * Evict cached data. We must write out any dirty data before 1128 * disowning the dataset. 1129 */ 1130 if (zv->zv_flags & ZVOL_WRITTEN_TO) 1131 txg_wait_synced(dmu_objset_pool(zv->zv_objset), 0); 1132 dmu_objset_evict_dbufs(zv->zv_objset); 1133 } 1134 1135 /* 1136 * return the proper tag for rollback and recv 1137 */ 1138 void * 1139 zvol_tag(zvol_state_t *zv) 1140 { 1141 ASSERT(RW_WRITE_HELD(&zv->zv_suspend_lock)); 1142 return (zv->zv_open_count > 0 ? zv : NULL); 1143 } 1144 1145 /* 1146 * Suspend the zvol for recv and rollback. 1147 */ 1148 zvol_state_t * 1149 zvol_suspend(const char *name) 1150 { 1151 zvol_state_t *zv; 1152 1153 zv = zvol_find_by_name(name, RW_WRITER); 1154 1155 if (zv == NULL) 1156 return (NULL); 1157 1158 /* block all I/O, release in zvol_resume. */ 1159 ASSERT(MUTEX_HELD(&zv->zv_state_lock)); 1160 ASSERT(RW_WRITE_HELD(&zv->zv_suspend_lock)); 1161 1162 atomic_inc(&zv->zv_suspend_ref); 1163 1164 if (zv->zv_open_count > 0) 1165 zvol_shutdown_zv(zv); 1166 1167 /* 1168 * do not hold zv_state_lock across suspend/resume to 1169 * avoid locking up zvol lookups 1170 */ 1171 mutex_exit(&zv->zv_state_lock); 1172 1173 /* zv_suspend_lock is released in zvol_resume() */ 1174 return (zv); 1175 } 1176 1177 int 1178 zvol_resume(zvol_state_t *zv) 1179 { 1180 int error = 0; 1181 1182 ASSERT(RW_WRITE_HELD(&zv->zv_suspend_lock)); 1183 1184 mutex_enter(&zv->zv_state_lock); 1185 1186 if (zv->zv_open_count > 0) { 1187 VERIFY0(dmu_objset_hold(zv->zv_name, zv, &zv->zv_objset)); 1188 VERIFY3P(zv->zv_objset->os_dsl_dataset->ds_owner, ==, zv); 1189 VERIFY(dsl_dataset_long_held(zv->zv_objset->os_dsl_dataset)); 1190 dmu_objset_rele(zv->zv_objset, zv); 1191 1192 error = zvol_setup_zv(zv); 1193 } 1194 1195 mutex_exit(&zv->zv_state_lock); 1196 1197 rw_exit(&zv->zv_suspend_lock); 1198 /* 1199 * We need this because we don't hold zvol_state_lock while releasing 1200 * zv_suspend_lock. zvol_remove_minors_impl thus cannot check 1201 * zv_suspend_lock to determine it is safe to free because rwlock is 1202 * not inherent atomic. 1203 */ 1204 atomic_dec(&zv->zv_suspend_ref); 1205 1206 if (zv->zv_flags & ZVOL_REMOVING) 1207 cv_broadcast(&zv->zv_removing_cv); 1208 1209 return (error); 1210 } 1211 1212 int 1213 zvol_first_open(zvol_state_t *zv, boolean_t readonly) 1214 { 1215 objset_t *os; 1216 int error; 1217 1218 ASSERT(RW_READ_HELD(&zv->zv_suspend_lock)); 1219 ASSERT(MUTEX_HELD(&zv->zv_state_lock)); 1220 ASSERT(mutex_owned(&spa_namespace_lock)); 1221 1222 boolean_t ro = (readonly || (strchr(zv->zv_name, '@') != NULL)); 1223 error = dmu_objset_own(zv->zv_name, DMU_OST_ZVOL, ro, B_TRUE, zv, &os); 1224 if (error) 1225 return (error); 1226 1227 zv->zv_objset = os; 1228 1229 error = zvol_setup_zv(zv); 1230 if (error) { 1231 dmu_objset_disown(os, 1, zv); 1232 zv->zv_objset = NULL; 1233 } 1234 1235 return (error); 1236 } 1237 1238 void 1239 zvol_last_close(zvol_state_t *zv) 1240 { 1241 ASSERT(RW_READ_HELD(&zv->zv_suspend_lock)); 1242 ASSERT(MUTEX_HELD(&zv->zv_state_lock)); 1243 1244 if (zv->zv_flags & ZVOL_REMOVING) 1245 cv_broadcast(&zv->zv_removing_cv); 1246 1247 zvol_shutdown_zv(zv); 1248 1249 dmu_objset_disown(zv->zv_objset, 1, zv); 1250 zv->zv_objset = NULL; 1251 } 1252 1253 typedef struct minors_job { 1254 list_t *list; 1255 list_node_t link; 1256 /* input */ 1257 char *name; 1258 /* output */ 1259 int error; 1260 } minors_job_t; 1261 1262 /* 1263 * Prefetch zvol dnodes for the minors_job 1264 */ 1265 static void 1266 zvol_prefetch_minors_impl(void *arg) 1267 { 1268 minors_job_t *job = arg; 1269 char *dsname = job->name; 1270 objset_t *os = NULL; 1271 1272 job->error = dmu_objset_own(dsname, DMU_OST_ZVOL, B_TRUE, B_TRUE, 1273 FTAG, &os); 1274 if (job->error == 0) { 1275 dmu_prefetch_dnode(os, ZVOL_OBJ, ZIO_PRIORITY_SYNC_READ); 1276 dmu_objset_disown(os, B_TRUE, FTAG); 1277 } 1278 } 1279 1280 /* 1281 * Mask errors to continue dmu_objset_find() traversal 1282 */ 1283 static int 1284 zvol_create_snap_minor_cb(const char *dsname, void *arg) 1285 { 1286 minors_job_t *j = arg; 1287 list_t *minors_list = j->list; 1288 const char *name = j->name; 1289 1290 ASSERT0(MUTEX_HELD(&spa_namespace_lock)); 1291 1292 /* skip the designated dataset */ 1293 if (name && strcmp(dsname, name) == 0) 1294 return (0); 1295 1296 /* at this point, the dsname should name a snapshot */ 1297 if (strchr(dsname, '@') == 0) { 1298 dprintf("zvol_create_snap_minor_cb(): " 1299 "%s is not a snapshot name\n", dsname); 1300 } else { 1301 minors_job_t *job; 1302 char *n = kmem_strdup(dsname); 1303 if (n == NULL) 1304 return (0); 1305 1306 job = kmem_alloc(sizeof (minors_job_t), KM_SLEEP); 1307 job->name = n; 1308 job->list = minors_list; 1309 job->error = 0; 1310 list_insert_tail(minors_list, job); 1311 /* don't care if dispatch fails, because job->error is 0 */ 1312 taskq_dispatch(system_taskq, zvol_prefetch_minors_impl, job, 1313 TQ_SLEEP); 1314 } 1315 1316 return (0); 1317 } 1318 1319 /* 1320 * If spa_keystore_load_wkey() is called for an encrypted zvol, 1321 * we need to look for any clones also using the key. This function 1322 * is "best effort" - so we just skip over it if there are failures. 1323 */ 1324 static void 1325 zvol_add_clones(const char *dsname, list_t *minors_list) 1326 { 1327 /* Also check if it has clones */ 1328 dsl_dir_t *dd = NULL; 1329 dsl_pool_t *dp = NULL; 1330 1331 if (dsl_pool_hold(dsname, FTAG, &dp) != 0) 1332 return; 1333 1334 if (!spa_feature_is_enabled(dp->dp_spa, 1335 SPA_FEATURE_ENCRYPTION)) 1336 goto out; 1337 1338 if (dsl_dir_hold(dp, dsname, FTAG, &dd, NULL) != 0) 1339 goto out; 1340 1341 if (dsl_dir_phys(dd)->dd_clones == 0) 1342 goto out; 1343 1344 zap_cursor_t *zc = kmem_alloc(sizeof (zap_cursor_t), KM_SLEEP); 1345 zap_attribute_t *za = zap_attribute_alloc(); 1346 objset_t *mos = dd->dd_pool->dp_meta_objset; 1347 1348 for (zap_cursor_init(zc, mos, dsl_dir_phys(dd)->dd_clones); 1349 zap_cursor_retrieve(zc, za) == 0; 1350 zap_cursor_advance(zc)) { 1351 dsl_dataset_t *clone; 1352 minors_job_t *job; 1353 1354 if (dsl_dataset_hold_obj(dd->dd_pool, 1355 za->za_first_integer, FTAG, &clone) == 0) { 1356 1357 char name[ZFS_MAX_DATASET_NAME_LEN]; 1358 dsl_dataset_name(clone, name); 1359 1360 char *n = kmem_strdup(name); 1361 job = kmem_alloc(sizeof (minors_job_t), KM_SLEEP); 1362 job->name = n; 1363 job->list = minors_list; 1364 job->error = 0; 1365 list_insert_tail(minors_list, job); 1366 1367 dsl_dataset_rele(clone, FTAG); 1368 } 1369 } 1370 zap_cursor_fini(zc); 1371 zap_attribute_free(za); 1372 kmem_free(zc, sizeof (zap_cursor_t)); 1373 1374 out: 1375 if (dd != NULL) 1376 dsl_dir_rele(dd, FTAG); 1377 dsl_pool_rele(dp, FTAG); 1378 } 1379 1380 /* 1381 * Mask errors to continue dmu_objset_find() traversal 1382 */ 1383 static int 1384 zvol_create_minors_cb(const char *dsname, void *arg) 1385 { 1386 uint64_t snapdev; 1387 int error; 1388 list_t *minors_list = arg; 1389 1390 ASSERT0(MUTEX_HELD(&spa_namespace_lock)); 1391 1392 error = dsl_prop_get_integer(dsname, "snapdev", &snapdev, NULL); 1393 if (error) 1394 return (0); 1395 1396 /* 1397 * Given the name and the 'snapdev' property, create device minor nodes 1398 * with the linkages to zvols/snapshots as needed. 1399 * If the name represents a zvol, create a minor node for the zvol, then 1400 * check if its snapshots are 'visible', and if so, iterate over the 1401 * snapshots and create device minor nodes for those. 1402 */ 1403 if (strchr(dsname, '@') == 0) { 1404 minors_job_t *job; 1405 char *n = kmem_strdup(dsname); 1406 if (n == NULL) 1407 return (0); 1408 1409 job = kmem_alloc(sizeof (minors_job_t), KM_SLEEP); 1410 job->name = n; 1411 job->list = minors_list; 1412 job->error = 0; 1413 list_insert_tail(minors_list, job); 1414 /* don't care if dispatch fails, because job->error is 0 */ 1415 taskq_dispatch(system_taskq, zvol_prefetch_minors_impl, job, 1416 TQ_SLEEP); 1417 1418 zvol_add_clones(dsname, minors_list); 1419 1420 if (snapdev == ZFS_SNAPDEV_VISIBLE) { 1421 /* 1422 * traverse snapshots only, do not traverse children, 1423 * and skip the 'dsname' 1424 */ 1425 (void) dmu_objset_find(dsname, 1426 zvol_create_snap_minor_cb, (void *)job, 1427 DS_FIND_SNAPSHOTS); 1428 } 1429 } else { 1430 dprintf("zvol_create_minors_cb(): %s is not a zvol name\n", 1431 dsname); 1432 } 1433 1434 return (0); 1435 } 1436 1437 static void 1438 zvol_task_update_status(zvol_task_t *task, uint64_t total, uint64_t done, 1439 int error) 1440 { 1441 1442 task->zt_total += total; 1443 task->zt_done += done; 1444 if (task->zt_total != task->zt_done) { 1445 task->zt_status = -1; 1446 if (error) 1447 task->zt_error = error; 1448 } 1449 } 1450 1451 static void 1452 zvol_task_report_status(zvol_task_t *task) 1453 { 1454 #ifdef ZFS_DEBUG 1455 static const char *const msg[] = { 1456 "create", 1457 "remove", 1458 "rename", 1459 "set snapdev", 1460 "set volmode", 1461 "unknown", 1462 }; 1463 1464 if (task->zt_status == 0) 1465 return; 1466 1467 zvol_async_op_t op = MIN(task->zt_op, ZVOL_ASYNC_MAX); 1468 if (task->zt_error) { 1469 dprintf("The %s minors zvol task was not ok, last error %d\n", 1470 msg[op], task->zt_error); 1471 } else { 1472 dprintf("The %s minors zvol task was not ok\n", msg[op]); 1473 } 1474 #else 1475 (void) task; 1476 #endif 1477 } 1478 1479 /* 1480 * Create minors for the specified dataset, including children and snapshots. 1481 * Pay attention to the 'snapdev' property and iterate over the snapshots 1482 * only if they are 'visible'. This approach allows one to assure that the 1483 * snapshot metadata is read from disk only if it is needed. 1484 * 1485 * The name can represent a dataset to be recursively scanned for zvols and 1486 * their snapshots, or a single zvol snapshot. If the name represents a 1487 * dataset, the scan is performed in two nested stages: 1488 * - scan the dataset for zvols, and 1489 * - for each zvol, create a minor node, then check if the zvol's snapshots 1490 * are 'visible', and only then iterate over the snapshots if needed 1491 * 1492 * If the name represents a snapshot, a check is performed if the snapshot is 1493 * 'visible' (which also verifies that the parent is a zvol), and if so, 1494 * a minor node for that snapshot is created. 1495 */ 1496 static void 1497 zvol_create_minors_impl(zvol_task_t *task) 1498 { 1499 const char *name = task->zt_name1; 1500 list_t minors_list; 1501 minors_job_t *job; 1502 uint64_t snapdev; 1503 int total = 0, done = 0, last_error, error; 1504 1505 /* 1506 * Note: the dsl_pool_config_lock must not be held. 1507 * Minor node creation needs to obtain the zvol_state_lock. 1508 * zvol_open() obtains the zvol_state_lock and then the dsl pool 1509 * config lock. Therefore, we can't have the config lock now if 1510 * we are going to wait for the zvol_state_lock, because it 1511 * would be a lock order inversion which could lead to deadlock. 1512 */ 1513 1514 if (zvol_inhibit_dev) { 1515 return; 1516 } 1517 1518 /* 1519 * This is the list for prefetch jobs. Whenever we found a match 1520 * during dmu_objset_find, we insert a minors_job to the list and do 1521 * taskq_dispatch to parallel prefetch zvol dnodes. Note we don't need 1522 * any lock because all list operation is done on the current thread. 1523 * 1524 * We will use this list to do zvol_os_create_minor after prefetch 1525 * so we don't have to traverse using dmu_objset_find again. 1526 */ 1527 list_create(&minors_list, sizeof (minors_job_t), 1528 offsetof(minors_job_t, link)); 1529 1530 1531 if (strchr(name, '@') != NULL) { 1532 error = dsl_prop_get_integer(name, "snapdev", &snapdev, NULL); 1533 if (error == 0 && snapdev == ZFS_SNAPDEV_VISIBLE) { 1534 error = zvol_os_create_minor(name); 1535 if (error == 0) { 1536 done++; 1537 } else { 1538 last_error = error; 1539 } 1540 total++; 1541 } 1542 } else { 1543 fstrans_cookie_t cookie = spl_fstrans_mark(); 1544 (void) dmu_objset_find(name, zvol_create_minors_cb, 1545 &minors_list, DS_FIND_CHILDREN); 1546 spl_fstrans_unmark(cookie); 1547 } 1548 1549 taskq_wait_outstanding(system_taskq, 0); 1550 1551 /* 1552 * Prefetch is completed, we can do zvol_os_create_minor 1553 * sequentially. 1554 */ 1555 while ((job = list_remove_head(&minors_list)) != NULL) { 1556 if (!job->error) { 1557 error = zvol_os_create_minor(job->name); 1558 if (error == 0) { 1559 done++; 1560 } else { 1561 last_error = error; 1562 } 1563 } else if (job->error == EINVAL) { 1564 /* 1565 * The objset, with the name requested by current job 1566 * exist, but have the type different from zvol. 1567 * Just ignore this sort of errors. 1568 */ 1569 done++; 1570 } else { 1571 last_error = job->error; 1572 } 1573 total++; 1574 kmem_strfree(job->name); 1575 kmem_free(job, sizeof (minors_job_t)); 1576 } 1577 1578 list_destroy(&minors_list); 1579 zvol_task_update_status(task, total, done, last_error); 1580 } 1581 1582 /* 1583 * Remove minors for specified dataset and, optionally, its children and 1584 * snapshots. 1585 */ 1586 static void 1587 zvol_remove_minors_impl(zvol_task_t *task) 1588 { 1589 zvol_state_t *zv, *zv_next; 1590 const char *name = task ? task->zt_name1 : NULL; 1591 int namelen = ((name) ? strlen(name) : 0); 1592 boolean_t children = task ? !!task->zt_value : B_TRUE; 1593 1594 if (zvol_inhibit_dev) 1595 return; 1596 1597 /* 1598 * We collect up zvols that we want to remove on a separate list, so 1599 * that we don't have to hold zvol_state_lock for the whole time. 1600 * 1601 * We can't remove them from the global lists until we're completely 1602 * done with them, because that would make them appear to ZFS-side ops 1603 * that they don't exist, and the name might be reused, which can't be 1604 * good. 1605 */ 1606 list_t remove_list; 1607 list_create(&remove_list, sizeof (zvol_state_t), 1608 offsetof(zvol_state_t, zv_remove_node)); 1609 1610 rw_enter(&zvol_state_lock, RW_READER); 1611 1612 for (zv = list_head(&zvol_state_list); zv != NULL; zv = zv_next) { 1613 zv_next = list_next(&zvol_state_list, zv); 1614 1615 mutex_enter(&zv->zv_state_lock); 1616 if (zv->zv_flags & ZVOL_REMOVING) { 1617 /* Another thread is handling shutdown, skip it. */ 1618 mutex_exit(&zv->zv_state_lock); 1619 continue; 1620 } 1621 1622 /* 1623 * This zvol should be removed if: 1624 * - no name was offered (ie removing all at shutdown); or 1625 * - name matches exactly; or 1626 * - we were asked to remove children, and 1627 * - the start of the name matches, and 1628 * - there is a '/' immediately after the matched name; or 1629 * - there is a '@' immediately after the matched name 1630 */ 1631 if (name == NULL || strcmp(zv->zv_name, name) == 0 || 1632 (children && strncmp(zv->zv_name, name, namelen) == 0 && 1633 (zv->zv_name[namelen] == '/' || 1634 zv->zv_name[namelen] == '@'))) { 1635 1636 /* 1637 * Matched, so mark it removal. We want to take the 1638 * write half of the suspend lock to make sure that 1639 * the zvol is not suspended, and give any data ops 1640 * chance to finish. 1641 */ 1642 mutex_exit(&zv->zv_state_lock); 1643 rw_enter(&zv->zv_suspend_lock, RW_WRITER); 1644 mutex_enter(&zv->zv_state_lock); 1645 1646 if (zv->zv_flags & ZVOL_REMOVING) { 1647 /* Another thread has taken it, let them. */ 1648 mutex_exit(&zv->zv_state_lock); 1649 rw_exit(&zv->zv_suspend_lock); 1650 continue; 1651 } 1652 1653 /* 1654 * Mark it and unlock. New entries will see the flag 1655 * and return ENXIO. 1656 */ 1657 zv->zv_flags |= ZVOL_REMOVING; 1658 mutex_exit(&zv->zv_state_lock); 1659 rw_exit(&zv->zv_suspend_lock); 1660 1661 /* Put it on the list for the next stage. */ 1662 list_insert_head(&remove_list, zv); 1663 } else 1664 mutex_exit(&zv->zv_state_lock); 1665 } 1666 1667 rw_exit(&zvol_state_lock); 1668 1669 /* Didn't match any, nothing to do! */ 1670 if (list_is_empty(&remove_list)) { 1671 if (task) 1672 task->zt_error = SET_ERROR(ENOENT); 1673 return; 1674 } 1675 1676 /* Actually shut them all down. */ 1677 for (zv = list_head(&remove_list); zv != NULL; zv = zv_next) { 1678 zv_next = list_next(&remove_list, zv); 1679 1680 mutex_enter(&zv->zv_state_lock); 1681 1682 /* 1683 * Still open or suspended, just wait. This can happen if, for 1684 * example, we managed to acquire zv_state_lock in the moments 1685 * where zvol_open() or zvol_release() are trading locks to 1686 * call zvol_first_open() or zvol_last_close(). 1687 */ 1688 while (zv->zv_open_count > 0 || 1689 atomic_read(&zv->zv_suspend_ref)) 1690 cv_wait(&zv->zv_removing_cv, &zv->zv_state_lock); 1691 1692 /* 1693 * No users, shut down the OS side. This may not remove the 1694 * minor from view immediately, depending on the kernel 1695 * specifics, but it will ensure that it is unusable and that 1696 * this zvol_state_t can never again be reached from an OS-side 1697 * operation. 1698 */ 1699 zvol_os_remove_minor(zv); 1700 mutex_exit(&zv->zv_state_lock); 1701 1702 /* Remove it from the name lookup lists */ 1703 rw_enter(&zvol_state_lock, RW_WRITER); 1704 zvol_remove(zv); 1705 rw_exit(&zvol_state_lock); 1706 } 1707 1708 /* 1709 * Our own references on remove_list is the last one, free them and 1710 * we're done. 1711 */ 1712 while ((zv = list_remove_head(&remove_list)) != NULL) 1713 zvol_os_free(zv); 1714 1715 list_destroy(&remove_list); 1716 } 1717 1718 /* Remove minor for this specific volume only */ 1719 static int 1720 zvol_remove_minor_impl(const char *name) 1721 { 1722 if (zvol_inhibit_dev) 1723 return (0); 1724 1725 zvol_task_t task; 1726 memset(&task, 0, sizeof (zvol_task_t)); 1727 strlcpy(task.zt_name1, name, sizeof (task.zt_name1)); 1728 task.zt_value = B_FALSE; 1729 1730 zvol_remove_minors_impl(&task); 1731 1732 return (task.zt_error); 1733 } 1734 1735 /* 1736 * Rename minors for specified dataset including children and snapshots. 1737 */ 1738 static void 1739 zvol_rename_minors_impl(zvol_task_t *task) 1740 { 1741 zvol_state_t *zv, *zv_next; 1742 const char *oldname = task->zt_name1; 1743 const char *newname = task->zt_name2; 1744 int total = 0, done = 0, last_error, error, oldnamelen; 1745 1746 if (zvol_inhibit_dev) 1747 return; 1748 1749 oldnamelen = strlen(oldname); 1750 1751 rw_enter(&zvol_state_lock, RW_READER); 1752 1753 for (zv = list_head(&zvol_state_list); zv != NULL; zv = zv_next) { 1754 zv_next = list_next(&zvol_state_list, zv); 1755 1756 mutex_enter(&zv->zv_state_lock); 1757 1758 if (strcmp(zv->zv_name, oldname) == 0) { 1759 error = zvol_os_rename_minor(zv, newname); 1760 } else if (strncmp(zv->zv_name, oldname, oldnamelen) == 0 && 1761 (zv->zv_name[oldnamelen] == '/' || 1762 zv->zv_name[oldnamelen] == '@')) { 1763 char *name = kmem_asprintf("%s%c%s", newname, 1764 zv->zv_name[oldnamelen], 1765 zv->zv_name + oldnamelen + 1); 1766 error = zvol_os_rename_minor(zv, name); 1767 kmem_strfree(name); 1768 } 1769 if (error) { 1770 last_error = error; 1771 } else { 1772 done++; 1773 } 1774 total++; 1775 mutex_exit(&zv->zv_state_lock); 1776 } 1777 1778 rw_exit(&zvol_state_lock); 1779 zvol_task_update_status(task, total, done, last_error); 1780 } 1781 1782 typedef struct zvol_snapdev_cb_arg { 1783 zvol_task_t *task; 1784 uint64_t snapdev; 1785 } zvol_snapdev_cb_arg_t; 1786 1787 static int 1788 zvol_set_snapdev_cb(const char *dsname, void *param) 1789 { 1790 zvol_snapdev_cb_arg_t *arg = param; 1791 int error = 0; 1792 1793 if (strchr(dsname, '@') == NULL) 1794 return (0); 1795 1796 switch (arg->snapdev) { 1797 case ZFS_SNAPDEV_VISIBLE: 1798 error = zvol_os_create_minor(dsname); 1799 break; 1800 case ZFS_SNAPDEV_HIDDEN: 1801 error = zvol_remove_minor_impl(dsname); 1802 break; 1803 } 1804 1805 zvol_task_update_status(arg->task, 1, error == 0, error); 1806 return (0); 1807 } 1808 1809 static void 1810 zvol_set_snapdev_impl(zvol_task_t *task) 1811 { 1812 const char *name = task->zt_name1; 1813 uint64_t snapdev = task->zt_value; 1814 1815 zvol_snapdev_cb_arg_t arg = {task, snapdev}; 1816 fstrans_cookie_t cookie = spl_fstrans_mark(); 1817 /* 1818 * The zvol_set_snapdev_sync() sets snapdev appropriately 1819 * in the dataset hierarchy. Here, we only scan snapshots. 1820 */ 1821 dmu_objset_find(name, zvol_set_snapdev_cb, &arg, DS_FIND_SNAPSHOTS); 1822 spl_fstrans_unmark(cookie); 1823 } 1824 1825 static void 1826 zvol_set_volmode_impl(zvol_task_t *task) 1827 { 1828 const char *name = task->zt_name1; 1829 uint64_t volmode = task->zt_value; 1830 fstrans_cookie_t cookie; 1831 uint64_t old_volmode; 1832 zvol_state_t *zv; 1833 int error; 1834 1835 if (strchr(name, '@') != NULL) 1836 return; 1837 1838 /* 1839 * It's unfortunate we need to remove minors before we create new ones: 1840 * this is necessary because our backing gendisk (zvol_state->zv_disk) 1841 * could be different when we set, for instance, volmode from "geom" 1842 * to "dev" (or vice versa). 1843 */ 1844 zv = zvol_find_by_name(name, RW_NONE); 1845 if (zv == NULL && volmode == ZFS_VOLMODE_NONE) 1846 return; 1847 if (zv != NULL) { 1848 old_volmode = zv->zv_volmode; 1849 mutex_exit(&zv->zv_state_lock); 1850 if (old_volmode == volmode) 1851 return; 1852 zvol_wait_close(zv); 1853 } 1854 cookie = spl_fstrans_mark(); 1855 switch (volmode) { 1856 case ZFS_VOLMODE_NONE: 1857 error = zvol_remove_minor_impl(name); 1858 break; 1859 case ZFS_VOLMODE_GEOM: 1860 case ZFS_VOLMODE_DEV: 1861 error = zvol_remove_minor_impl(name); 1862 /* 1863 * The remove minor function call above, might be not 1864 * needed, if volmode was switched from 'none' value. 1865 * Ignore error in this case. 1866 */ 1867 if (error == ENOENT) 1868 error = 0; 1869 else if (error) 1870 break; 1871 error = zvol_os_create_minor(name); 1872 break; 1873 case ZFS_VOLMODE_DEFAULT: 1874 error = zvol_remove_minor_impl(name); 1875 if (zvol_volmode == ZFS_VOLMODE_NONE) 1876 break; 1877 else /* if zvol_volmode is invalid defaults to "geom" */ 1878 error = zvol_os_create_minor(name); 1879 break; 1880 } 1881 zvol_task_update_status(task, 1, error == 0, error); 1882 spl_fstrans_unmark(cookie); 1883 } 1884 1885 /* 1886 * The worker thread function performed asynchronously. 1887 */ 1888 static void 1889 zvol_task_cb(void *arg) 1890 { 1891 zvol_task_t *task = arg; 1892 1893 switch (task->zt_op) { 1894 case ZVOL_ASYNC_CREATE_MINORS: 1895 zvol_create_minors_impl(task); 1896 break; 1897 case ZVOL_ASYNC_REMOVE_MINORS: 1898 zvol_remove_minors_impl(task); 1899 break; 1900 case ZVOL_ASYNC_RENAME_MINORS: 1901 zvol_rename_minors_impl(task); 1902 break; 1903 case ZVOL_ASYNC_SET_SNAPDEV: 1904 zvol_set_snapdev_impl(task); 1905 break; 1906 case ZVOL_ASYNC_SET_VOLMODE: 1907 zvol_set_volmode_impl(task); 1908 break; 1909 default: 1910 VERIFY(0); 1911 break; 1912 } 1913 1914 zvol_task_report_status(task); 1915 kmem_free(task, sizeof (zvol_task_t)); 1916 } 1917 1918 typedef struct zvol_set_prop_int_arg { 1919 const char *zsda_name; 1920 uint64_t zsda_value; 1921 zprop_source_t zsda_source; 1922 zfs_prop_t zsda_prop; 1923 } zvol_set_prop_int_arg_t; 1924 1925 /* 1926 * Sanity check the dataset for safe use by the sync task. No additional 1927 * conditions are imposed. 1928 */ 1929 static int 1930 zvol_set_common_check(void *arg, dmu_tx_t *tx) 1931 { 1932 zvol_set_prop_int_arg_t *zsda = arg; 1933 dsl_pool_t *dp = dmu_tx_pool(tx); 1934 dsl_dir_t *dd; 1935 int error; 1936 1937 error = dsl_dir_hold(dp, zsda->zsda_name, FTAG, &dd, NULL); 1938 if (error != 0) 1939 return (error); 1940 1941 dsl_dir_rele(dd, FTAG); 1942 1943 return (error); 1944 } 1945 1946 static int 1947 zvol_set_common_sync_cb(dsl_pool_t *dp, dsl_dataset_t *ds, void *arg) 1948 { 1949 zvol_set_prop_int_arg_t *zsda = arg; 1950 char dsname[ZFS_MAX_DATASET_NAME_LEN]; 1951 zvol_task_t *task; 1952 uint64_t prop; 1953 1954 const char *prop_name = zfs_prop_to_name(zsda->zsda_prop); 1955 dsl_dataset_name(ds, dsname); 1956 1957 if (dsl_prop_get_int_ds(ds, prop_name, &prop) != 0) 1958 return (0); 1959 1960 task = kmem_zalloc(sizeof (zvol_task_t), KM_SLEEP); 1961 if (zsda->zsda_prop == ZFS_PROP_VOLMODE) { 1962 task->zt_op = ZVOL_ASYNC_SET_VOLMODE; 1963 } else if (zsda->zsda_prop == ZFS_PROP_SNAPDEV) { 1964 task->zt_op = ZVOL_ASYNC_SET_SNAPDEV; 1965 } else { 1966 kmem_free(task, sizeof (zvol_task_t)); 1967 return (0); 1968 } 1969 task->zt_value = prop; 1970 strlcpy(task->zt_name1, dsname, sizeof (task->zt_name1)); 1971 (void) taskq_dispatch(dp->dp_spa->spa_zvol_taskq, zvol_task_cb, 1972 task, TQ_SLEEP); 1973 return (0); 1974 } 1975 1976 /* 1977 * Traverse all child datasets and apply the property appropriately. 1978 * We call dsl_prop_set_sync_impl() here to set the value only on the toplevel 1979 * dataset and read the effective "property" on every child in the callback 1980 * function: this is because the value is not guaranteed to be the same in the 1981 * whole dataset hierarchy. 1982 */ 1983 static void 1984 zvol_set_common_sync(void *arg, dmu_tx_t *tx) 1985 { 1986 zvol_set_prop_int_arg_t *zsda = arg; 1987 dsl_pool_t *dp = dmu_tx_pool(tx); 1988 dsl_dir_t *dd; 1989 dsl_dataset_t *ds; 1990 int error; 1991 1992 VERIFY0(dsl_dir_hold(dp, zsda->zsda_name, FTAG, &dd, NULL)); 1993 1994 error = dsl_dataset_hold(dp, zsda->zsda_name, FTAG, &ds); 1995 if (error == 0) { 1996 dsl_prop_set_sync_impl(ds, zfs_prop_to_name(zsda->zsda_prop), 1997 zsda->zsda_source, sizeof (zsda->zsda_value), 1, 1998 &zsda->zsda_value, tx); 1999 dsl_dataset_rele(ds, FTAG); 2000 } 2001 2002 dmu_objset_find_dp(dp, dd->dd_object, zvol_set_common_sync_cb, 2003 zsda, DS_FIND_CHILDREN); 2004 2005 dsl_dir_rele(dd, FTAG); 2006 } 2007 2008 int 2009 zvol_set_common(const char *ddname, zfs_prop_t prop, zprop_source_t source, 2010 uint64_t val) 2011 { 2012 zvol_set_prop_int_arg_t zsda; 2013 2014 zsda.zsda_name = ddname; 2015 zsda.zsda_source = source; 2016 zsda.zsda_value = val; 2017 zsda.zsda_prop = prop; 2018 2019 return (dsl_sync_task(ddname, zvol_set_common_check, 2020 zvol_set_common_sync, &zsda, 0, ZFS_SPACE_CHECK_NONE)); 2021 } 2022 2023 void 2024 zvol_create_minors(const char *name) 2025 { 2026 spa_t *spa; 2027 zvol_task_t *task; 2028 taskqid_t id; 2029 2030 if (spa_open(name, &spa, FTAG) != 0) 2031 return; 2032 2033 task = kmem_zalloc(sizeof (zvol_task_t), KM_SLEEP); 2034 task->zt_op = ZVOL_ASYNC_CREATE_MINORS; 2035 strlcpy(task->zt_name1, name, sizeof (task->zt_name1)); 2036 id = taskq_dispatch(spa->spa_zvol_taskq, zvol_task_cb, task, TQ_SLEEP); 2037 if (id != TASKQID_INVALID) 2038 taskq_wait_id(spa->spa_zvol_taskq, id); 2039 2040 spa_close(spa, FTAG); 2041 } 2042 2043 void 2044 zvol_remove_minors(spa_t *spa, const char *name, boolean_t async) 2045 { 2046 zvol_task_t *task; 2047 taskqid_t id; 2048 2049 task = kmem_zalloc(sizeof (zvol_task_t), KM_SLEEP); 2050 task->zt_op = ZVOL_ASYNC_REMOVE_MINORS; 2051 strlcpy(task->zt_name1, name, sizeof (task->zt_name1)); 2052 task->zt_value = B_TRUE; 2053 id = taskq_dispatch(spa->spa_zvol_taskq, zvol_task_cb, task, TQ_SLEEP); 2054 if ((async == B_FALSE) && (id != TASKQID_INVALID)) 2055 taskq_wait_id(spa->spa_zvol_taskq, id); 2056 } 2057 2058 void 2059 zvol_rename_minors(spa_t *spa, const char *name1, const char *name2, 2060 boolean_t async) 2061 { 2062 zvol_task_t *task; 2063 taskqid_t id; 2064 2065 task = kmem_zalloc(sizeof (zvol_task_t), KM_SLEEP); 2066 task->zt_op = ZVOL_ASYNC_RENAME_MINORS; 2067 strlcpy(task->zt_name1, name1, sizeof (task->zt_name1)); 2068 strlcpy(task->zt_name2, name2, sizeof (task->zt_name2)); 2069 id = taskq_dispatch(spa->spa_zvol_taskq, zvol_task_cb, task, TQ_SLEEP); 2070 if ((async == B_FALSE) && (id != TASKQID_INVALID)) 2071 taskq_wait_id(spa->spa_zvol_taskq, id); 2072 } 2073 2074 boolean_t 2075 zvol_is_zvol(const char *name) 2076 { 2077 2078 return (zvol_os_is_zvol(name)); 2079 } 2080 2081 int 2082 zvol_init_impl(void) 2083 { 2084 int i; 2085 2086 /* 2087 * zvol_threads is the module param the user passes in. 2088 * 2089 * zvol_actual_threads is what we use internally, since the user can 2090 * pass zvol_thread = 0 to mean "use all the CPUs" (the default). 2091 */ 2092 static unsigned int zvol_actual_threads; 2093 2094 if (zvol_threads == 0) { 2095 /* 2096 * See dde9380a1 for why 32 was chosen here. This should 2097 * probably be refined to be some multiple of the number 2098 * of CPUs. 2099 */ 2100 zvol_actual_threads = MAX(max_ncpus, 32); 2101 } else { 2102 zvol_actual_threads = MIN(MAX(zvol_threads, 1), 1024); 2103 } 2104 2105 /* 2106 * Use at least 32 zvol_threads but for many core system, 2107 * prefer 6 threads per taskq, but no more taskqs 2108 * than threads in them on large systems. 2109 * 2110 * taskq total 2111 * cpus taskqs threads threads 2112 * ------- ------- ------- ------- 2113 * 1 1 32 32 2114 * 2 1 32 32 2115 * 4 1 32 32 2116 * 8 2 16 32 2117 * 16 3 11 33 2118 * 32 5 7 35 2119 * 64 8 8 64 2120 * 128 11 12 132 2121 * 256 16 16 256 2122 */ 2123 zv_taskq_t *ztqs = &zvol_taskqs; 2124 int num_tqs = MIN(max_ncpus, zvol_num_taskqs); 2125 if (num_tqs == 0) { 2126 num_tqs = 1 + max_ncpus / 6; 2127 while (num_tqs * num_tqs > zvol_actual_threads) 2128 num_tqs--; 2129 } 2130 2131 int per_tq_thread = zvol_actual_threads / num_tqs; 2132 if (per_tq_thread * num_tqs < zvol_actual_threads) 2133 per_tq_thread++; 2134 2135 ztqs->tqs_cnt = num_tqs; 2136 ztqs->tqs_taskq = kmem_alloc(num_tqs * sizeof (taskq_t *), KM_SLEEP); 2137 2138 for (uint_t i = 0; i < num_tqs; i++) { 2139 char name[32]; 2140 (void) snprintf(name, sizeof (name), "%s_tq-%u", 2141 ZVOL_DRIVER, i); 2142 ztqs->tqs_taskq[i] = taskq_create(name, per_tq_thread, 2143 maxclsyspri, per_tq_thread, INT_MAX, 2144 TASKQ_PREPOPULATE | TASKQ_DYNAMIC); 2145 if (ztqs->tqs_taskq[i] == NULL) { 2146 for (int j = i - 1; j >= 0; j--) 2147 taskq_destroy(ztqs->tqs_taskq[j]); 2148 kmem_free(ztqs->tqs_taskq, ztqs->tqs_cnt * 2149 sizeof (taskq_t *)); 2150 ztqs->tqs_taskq = NULL; 2151 return (SET_ERROR(ENOMEM)); 2152 } 2153 } 2154 2155 list_create(&zvol_state_list, sizeof (zvol_state_t), 2156 offsetof(zvol_state_t, zv_next)); 2157 rw_init(&zvol_state_lock, NULL, RW_DEFAULT, NULL); 2158 2159 zvol_htable = kmem_alloc(ZVOL_HT_SIZE * sizeof (struct hlist_head), 2160 KM_SLEEP); 2161 for (i = 0; i < ZVOL_HT_SIZE; i++) 2162 INIT_HLIST_HEAD(&zvol_htable[i]); 2163 2164 return (0); 2165 } 2166 2167 void 2168 zvol_fini_impl(void) 2169 { 2170 zv_taskq_t *ztqs = &zvol_taskqs; 2171 2172 zvol_remove_minors_impl(NULL); 2173 2174 kmem_free(zvol_htable, ZVOL_HT_SIZE * sizeof (struct hlist_head)); 2175 list_destroy(&zvol_state_list); 2176 rw_destroy(&zvol_state_lock); 2177 2178 if (ztqs->tqs_taskq == NULL) { 2179 ASSERT0(ztqs->tqs_cnt); 2180 } else { 2181 for (uint_t i = 0; i < ztqs->tqs_cnt; i++) { 2182 ASSERT3P(ztqs->tqs_taskq[i], !=, NULL); 2183 taskq_destroy(ztqs->tqs_taskq[i]); 2184 } 2185 kmem_free(ztqs->tqs_taskq, ztqs->tqs_cnt * 2186 sizeof (taskq_t *)); 2187 ztqs->tqs_taskq = NULL; 2188 } 2189 } 2190 2191 ZFS_MODULE_PARAM(zfs_vol, zvol_, inhibit_dev, UINT, ZMOD_RW, 2192 "Do not create zvol device nodes"); 2193 ZFS_MODULE_PARAM(zfs_vol, zvol_, prefetch_bytes, UINT, ZMOD_RW, 2194 "Prefetch N bytes at zvol start+end"); 2195 ZFS_MODULE_PARAM(zfs_vol, zvol_vol, mode, UINT, ZMOD_RW, 2196 "Default volmode property value"); 2197 ZFS_MODULE_PARAM(zfs_vol, zvol_, threads, UINT, ZMOD_RW, 2198 "Number of threads for I/O requests. Set to 0 to use all active CPUs"); 2199 ZFS_MODULE_PARAM(zfs_vol, zvol_, num_taskqs, UINT, ZMOD_RW, 2200 "Number of zvol taskqs"); 2201 ZFS_MODULE_PARAM(zfs_vol, zvol_, request_sync, UINT, ZMOD_RW, 2202 "Synchronously handle bio requests"); 2203