1 // SPDX-License-Identifier: CDDL-1.0 2 /* 3 * CDDL HEADER START 4 * 5 * The contents of this file are subject to the terms of the 6 * Common Development and Distribution License (the "License"). 7 * You may not use this file except in compliance with the License. 8 * 9 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 10 * or https://opensource.org/licenses/CDDL-1.0. 11 * See the License for the specific language governing permissions 12 * and limitations under the License. 13 * 14 * When distributing Covered Code, include this CDDL HEADER in each 15 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 16 * If applicable, add the following below this CDDL HEADER, with the 17 * fields enclosed by brackets "[]" replaced with your own identifying 18 * information: Portions Copyright [yyyy] [name of copyright owner] 19 * 20 * CDDL HEADER END 21 */ 22 /* 23 * Copyright (C) 2008-2010 Lawrence Livermore National Security, LLC. 24 * Produced at Lawrence Livermore National Laboratory (cf, DISCLAIMER). 25 * Rewritten for Linux by Brian Behlendorf <behlendorf1@llnl.gov>. 26 * LLNL-CODE-403049. 27 * 28 * ZFS volume emulation driver. 29 * 30 * Makes a DMU object look like a volume of arbitrary size, up to 2^64 bytes. 31 * Volumes are accessed through the symbolic links named: 32 * 33 * /dev/<pool_name>/<dataset_name> 34 * 35 * Volumes are persistent through reboot and module load. No user command 36 * needs to be run before opening and using a device. 37 * 38 * Copyright 2014 Nexenta Systems, Inc. All rights reserved. 39 * Copyright (c) 2016 Actifio, Inc. All rights reserved. 40 * Copyright (c) 2012, 2019 by Delphix. All rights reserved. 41 * Copyright (c) 2024, 2025, Klara, Inc. 42 */ 43 44 /* 45 * Note on locking of zvol state structures. 46 * 47 * zvol_state_t represents the connection between a single dataset 48 * (DMU_OST_ZVOL) and the device "minor" (some OS-specific representation of a 49 * "disk" or "device" or "volume", eg, a /dev/zdXX node, a GEOM object, etc). 50 * 51 * The global zvol_state_lock is used to protect access to zvol_state_list and 52 * zvol_htable, which are the primary way to obtain a zvol_state_t from a name. 53 * It should not be used for anything not name-relateds, and you should avoid 54 * sleeping or waiting while its held. See zvol_find_by_name(), zvol_insert(), 55 * zvol_remove(). 56 * 57 * The zv_state_lock is used to protect the contents of the associated 58 * zvol_state_t. Most of the zvol_state_t is dedicated to control and 59 * configuration; almost none of it is needed for data operations (that is, 60 * read, write, flush) so this lock is rarely taken during general IO. It 61 * should be released quickly; you should avoid sleeping or waiting while its 62 * held. 63 * 64 * zv_suspend_lock is used to suspend IO/data operations to a zvol. The read 65 * half should held for the duration of an IO operation. The write half should 66 * be taken when something to wait for IO to complete and the block further IO, 67 * eg for the duration of receive and rollback operations. This lock can be 68 * held for long periods of time. 69 * 70 * Thus, the following lock ordering appies. 71 * - take zvol_state_lock if necessary, to protect zvol_state_list 72 * - take zv_suspend_lock if necessary, by the code path in question 73 * - take zv_state_lock to protect zvol_state_t 74 * 75 * The minor operations are issued to spa->spa_zvol_taskq queues, that are 76 * single-threaded (to preserve order of minor operations), and are executed 77 * through the zvol_task_cb that dispatches the specific operations. Therefore, 78 * these operations are serialized per pool. Consequently, we can be certain 79 * that for a given zvol, there is only one operation at a time in progress. 80 * That is why one can be sure that first, zvol_state_t for a given zvol is 81 * allocated and placed on zvol_state_list, and then other minor operations for 82 * this zvol are going to proceed in the order of issue. 83 */ 84 85 #include <sys/dataset_kstats.h> 86 #include <sys/dbuf.h> 87 #include <sys/dmu_traverse.h> 88 #include <sys/dsl_dataset.h> 89 #include <sys/dsl_prop.h> 90 #include <sys/dsl_dir.h> 91 #include <sys/zap.h> 92 #include <sys/zfeature.h> 93 #include <sys/zil_impl.h> 94 #include <sys/dmu_tx.h> 95 #include <sys/zio.h> 96 #include <sys/zfs_rlock.h> 97 #include <sys/spa_impl.h> 98 #include <sys/zvol.h> 99 #include <sys/zvol_impl.h> 100 101 unsigned int zvol_inhibit_dev = 0; 102 unsigned int zvol_prefetch_bytes = (128 * 1024); 103 unsigned int zvol_volmode = ZFS_VOLMODE_GEOM; 104 unsigned int zvol_threads = 0; 105 unsigned int zvol_num_taskqs = 0; 106 unsigned int zvol_request_sync = 0; 107 108 struct hlist_head *zvol_htable; 109 static list_t zvol_state_list; 110 krwlock_t zvol_state_lock; 111 extern int zfs_bclone_wait_dirty; 112 zv_taskq_t zvol_taskqs; 113 114 typedef enum { 115 ZVOL_ASYNC_CREATE_MINORS, 116 ZVOL_ASYNC_REMOVE_MINORS, 117 ZVOL_ASYNC_RENAME_MINORS, 118 ZVOL_ASYNC_SET_SNAPDEV, 119 ZVOL_ASYNC_SET_VOLMODE, 120 ZVOL_ASYNC_MAX 121 } zvol_async_op_t; 122 123 typedef struct { 124 zvol_async_op_t zt_op; 125 char zt_name1[MAXNAMELEN]; 126 char zt_name2[MAXNAMELEN]; 127 uint64_t zt_value; 128 uint32_t zt_total; 129 uint32_t zt_done; 130 int32_t zt_status; 131 int zt_error; 132 } zvol_task_t; 133 134 zv_request_task_t * 135 zv_request_task_create(zv_request_t zvr) 136 { 137 zv_request_task_t *task; 138 task = kmem_alloc(sizeof (zv_request_task_t), KM_SLEEP); 139 taskq_init_ent(&task->ent); 140 task->zvr = zvr; 141 return (task); 142 } 143 144 void 145 zv_request_task_free(zv_request_task_t *task) 146 { 147 kmem_free(task, sizeof (*task)); 148 } 149 150 uint64_t 151 zvol_name_hash(const char *name) 152 { 153 uint64_t crc = -1ULL; 154 ASSERT(zfs_crc64_table[128] == ZFS_CRC64_POLY); 155 for (const uint8_t *p = (const uint8_t *)name; *p != 0; p++) 156 crc = (crc >> 8) ^ zfs_crc64_table[(crc ^ (*p)) & 0xFF]; 157 return (crc); 158 } 159 160 /* 161 * Find a zvol_state_t given the name and hash generated by zvol_name_hash. 162 * If found, return with zv_suspend_lock and zv_state_lock taken, otherwise, 163 * return (NULL) without the taking locks. The zv_suspend_lock is always taken 164 * before zv_state_lock. The mode argument indicates the mode (including none) 165 * for zv_suspend_lock to be taken. 166 */ 167 zvol_state_t * 168 zvol_find_by_name_hash(const char *name, uint64_t hash, int mode) 169 { 170 zvol_state_t *zv; 171 struct hlist_node *p = NULL; 172 173 rw_enter(&zvol_state_lock, RW_READER); 174 hlist_for_each(p, ZVOL_HT_HEAD(hash)) { 175 zv = hlist_entry(p, zvol_state_t, zv_hlink); 176 mutex_enter(&zv->zv_state_lock); 177 if (zv->zv_hash == hash && strcmp(zv->zv_name, name) == 0) { 178 /* 179 * this is the right zvol, take the locks in the 180 * right order 181 */ 182 if (mode != RW_NONE && 183 !rw_tryenter(&zv->zv_suspend_lock, mode)) { 184 mutex_exit(&zv->zv_state_lock); 185 rw_enter(&zv->zv_suspend_lock, mode); 186 mutex_enter(&zv->zv_state_lock); 187 /* 188 * zvol cannot be renamed as we continue 189 * to hold zvol_state_lock 190 */ 191 ASSERT(zv->zv_hash == hash && 192 strcmp(zv->zv_name, name) == 0); 193 } 194 rw_exit(&zvol_state_lock); 195 return (zv); 196 } 197 mutex_exit(&zv->zv_state_lock); 198 } 199 rw_exit(&zvol_state_lock); 200 201 return (NULL); 202 } 203 204 /* 205 * Find a zvol_state_t given the name. 206 * If found, return with zv_suspend_lock and zv_state_lock taken, otherwise, 207 * return (NULL) without the taking locks. The zv_suspend_lock is always taken 208 * before zv_state_lock. The mode argument indicates the mode (including none) 209 * for zv_suspend_lock to be taken. 210 */ 211 static zvol_state_t * 212 zvol_find_by_name(const char *name, int mode) 213 { 214 return (zvol_find_by_name_hash(name, zvol_name_hash(name), mode)); 215 } 216 217 /* 218 * ZFS_IOC_CREATE callback handles dmu zvol and zap object creation. 219 */ 220 void 221 zvol_create_cb(objset_t *os, void *arg, cred_t *cr, dmu_tx_t *tx) 222 { 223 zfs_creat_t *zct = arg; 224 nvlist_t *nvprops = zct->zct_props; 225 int error; 226 uint64_t volblocksize, volsize; 227 228 VERIFY0(nvlist_lookup_uint64(nvprops, 229 zfs_prop_to_name(ZFS_PROP_VOLSIZE), &volsize)); 230 if (nvlist_lookup_uint64(nvprops, 231 zfs_prop_to_name(ZFS_PROP_VOLBLOCKSIZE), &volblocksize) != 0) 232 volblocksize = zfs_prop_default_numeric(ZFS_PROP_VOLBLOCKSIZE); 233 234 /* 235 * These properties must be removed from the list so the generic 236 * property setting step won't apply to them. 237 */ 238 VERIFY0(nvlist_remove_all(nvprops, zfs_prop_to_name(ZFS_PROP_VOLSIZE))); 239 (void) nvlist_remove_all(nvprops, 240 zfs_prop_to_name(ZFS_PROP_VOLBLOCKSIZE)); 241 242 error = dmu_object_claim(os, ZVOL_OBJ, DMU_OT_ZVOL, volblocksize, 243 DMU_OT_NONE, 0, tx); 244 ASSERT0(error); 245 246 error = zap_create_claim(os, ZVOL_ZAP_OBJ, DMU_OT_ZVOL_PROP, 247 DMU_OT_NONE, 0, tx); 248 ASSERT0(error); 249 250 error = zap_update(os, ZVOL_ZAP_OBJ, "size", 8, 1, &volsize, tx); 251 ASSERT0(error); 252 } 253 254 /* 255 * ZFS_IOC_OBJSET_STATS entry point. 256 */ 257 int 258 zvol_get_stats(objset_t *os, nvlist_t *nv) 259 { 260 int error; 261 dmu_object_info_t *doi; 262 uint64_t val; 263 264 error = zap_lookup(os, ZVOL_ZAP_OBJ, "size", 8, 1, &val); 265 if (error) 266 return (error); 267 268 dsl_prop_nvlist_add_uint64(nv, ZFS_PROP_VOLSIZE, val); 269 doi = kmem_alloc(sizeof (dmu_object_info_t), KM_SLEEP); 270 error = dmu_object_info(os, ZVOL_OBJ, doi); 271 272 if (error == 0) { 273 dsl_prop_nvlist_add_uint64(nv, ZFS_PROP_VOLBLOCKSIZE, 274 doi->doi_data_block_size); 275 } 276 277 kmem_free(doi, sizeof (dmu_object_info_t)); 278 279 return (error); 280 } 281 282 /* 283 * Sanity check volume size. 284 */ 285 int 286 zvol_check_volsize(uint64_t volsize, uint64_t blocksize) 287 { 288 if (volsize == 0) 289 return (SET_ERROR(EINVAL)); 290 291 if (volsize % blocksize != 0) 292 return (SET_ERROR(EINVAL)); 293 294 #ifdef _ILP32 295 if (volsize - 1 > SPEC_MAXOFFSET_T) 296 return (SET_ERROR(EOVERFLOW)); 297 #endif 298 return (0); 299 } 300 301 /* 302 * Ensure the zap is flushed then inform the VFS of the capacity change. 303 */ 304 static int 305 zvol_update_volsize(uint64_t volsize, objset_t *os) 306 { 307 dmu_tx_t *tx; 308 int error; 309 uint64_t txg; 310 311 tx = dmu_tx_create(os); 312 dmu_tx_hold_zap(tx, ZVOL_ZAP_OBJ, TRUE, NULL); 313 dmu_tx_mark_netfree(tx); 314 error = dmu_tx_assign(tx, DMU_TX_WAIT); 315 if (error) { 316 dmu_tx_abort(tx); 317 return (error); 318 } 319 txg = dmu_tx_get_txg(tx); 320 321 error = zap_update(os, ZVOL_ZAP_OBJ, "size", 8, 1, 322 &volsize, tx); 323 dmu_tx_commit(tx); 324 325 txg_wait_synced(dmu_objset_pool(os), txg); 326 327 if (error == 0) 328 error = dmu_free_long_range(os, 329 ZVOL_OBJ, volsize, DMU_OBJECT_END); 330 331 return (error); 332 } 333 334 /* 335 * Set ZFS_PROP_VOLSIZE set entry point. Note that modifying the volume 336 * size will result in a udev "change" event being generated. 337 */ 338 int 339 zvol_set_volsize(const char *name, uint64_t volsize) 340 { 341 objset_t *os = NULL; 342 uint64_t readonly; 343 int error; 344 boolean_t owned = B_FALSE; 345 346 error = dsl_prop_get_integer(name, 347 zfs_prop_to_name(ZFS_PROP_READONLY), &readonly, NULL); 348 if (error != 0) 349 return (error); 350 if (readonly) 351 return (SET_ERROR(EROFS)); 352 353 zvol_state_t *zv = zvol_find_by_name(name, RW_READER); 354 355 ASSERT(zv == NULL || (MUTEX_HELD(&zv->zv_state_lock) && 356 RW_READ_HELD(&zv->zv_suspend_lock))); 357 358 if (zv == NULL || zv->zv_objset == NULL) { 359 if (zv != NULL) 360 rw_exit(&zv->zv_suspend_lock); 361 if ((error = dmu_objset_own(name, DMU_OST_ZVOL, B_FALSE, B_TRUE, 362 FTAG, &os)) != 0) { 363 if (zv != NULL) 364 mutex_exit(&zv->zv_state_lock); 365 return (error); 366 } 367 owned = B_TRUE; 368 if (zv != NULL) 369 zv->zv_objset = os; 370 } else { 371 os = zv->zv_objset; 372 } 373 374 dmu_object_info_t *doi = kmem_alloc(sizeof (*doi), KM_SLEEP); 375 376 if ((error = dmu_object_info(os, ZVOL_OBJ, doi)) || 377 (error = zvol_check_volsize(volsize, doi->doi_data_block_size))) 378 goto out; 379 380 error = zvol_update_volsize(volsize, os); 381 if (error == 0 && zv != NULL) { 382 zv->zv_volsize = volsize; 383 zv->zv_changed = 1; 384 } 385 out: 386 kmem_free(doi, sizeof (dmu_object_info_t)); 387 388 if (owned) { 389 dmu_objset_disown(os, B_TRUE, FTAG); 390 if (zv != NULL) 391 zv->zv_objset = NULL; 392 } else { 393 rw_exit(&zv->zv_suspend_lock); 394 } 395 396 if (zv != NULL) 397 mutex_exit(&zv->zv_state_lock); 398 399 if (error == 0 && zv != NULL) 400 zvol_os_update_volsize(zv, volsize); 401 402 return (error); 403 } 404 405 /* 406 * Update volthreading. 407 */ 408 int 409 zvol_set_volthreading(const char *name, boolean_t value) 410 { 411 zvol_state_t *zv = zvol_find_by_name(name, RW_NONE); 412 if (zv == NULL) 413 return (-1); 414 zv->zv_threading = value; 415 mutex_exit(&zv->zv_state_lock); 416 return (0); 417 } 418 419 /* 420 * Update zvol ro property. 421 */ 422 int 423 zvol_set_ro(const char *name, boolean_t value) 424 { 425 zvol_state_t *zv = zvol_find_by_name(name, RW_NONE); 426 if (zv == NULL) 427 return (-1); 428 if (value) { 429 zvol_os_set_disk_ro(zv, 1); 430 zv->zv_flags |= ZVOL_RDONLY; 431 } else { 432 zvol_os_set_disk_ro(zv, 0); 433 zv->zv_flags &= ~ZVOL_RDONLY; 434 } 435 mutex_exit(&zv->zv_state_lock); 436 return (0); 437 } 438 439 /* 440 * Sanity check volume block size. 441 */ 442 int 443 zvol_check_volblocksize(const char *name, uint64_t volblocksize) 444 { 445 /* Record sizes above 128k need the feature to be enabled */ 446 if (volblocksize > SPA_OLD_MAXBLOCKSIZE) { 447 spa_t *spa; 448 int error; 449 450 if ((error = spa_open(name, &spa, FTAG)) != 0) 451 return (error); 452 453 if (!spa_feature_is_enabled(spa, SPA_FEATURE_LARGE_BLOCKS)) { 454 spa_close(spa, FTAG); 455 return (SET_ERROR(ENOTSUP)); 456 } 457 458 /* 459 * We don't allow setting the property above 1MB, 460 * unless the tunable has been changed. 461 */ 462 if (volblocksize > zfs_max_recordsize) { 463 spa_close(spa, FTAG); 464 return (SET_ERROR(EDOM)); 465 } 466 467 spa_close(spa, FTAG); 468 } 469 470 if (volblocksize < SPA_MINBLOCKSIZE || 471 volblocksize > SPA_MAXBLOCKSIZE || 472 !ISP2(volblocksize)) 473 return (SET_ERROR(EDOM)); 474 475 return (0); 476 } 477 478 /* 479 * Replay a TX_TRUNCATE ZIL transaction if asked. TX_TRUNCATE is how we 480 * implement DKIOCFREE/free-long-range. 481 */ 482 static int 483 zvol_replay_truncate(void *arg1, void *arg2, boolean_t byteswap) 484 { 485 zvol_state_t *zv = arg1; 486 lr_truncate_t *lr = arg2; 487 uint64_t offset, length; 488 489 ASSERT3U(lr->lr_common.lrc_reclen, >=, sizeof (*lr)); 490 491 if (byteswap) 492 byteswap_uint64_array(lr, sizeof (*lr)); 493 494 offset = lr->lr_offset; 495 length = lr->lr_length; 496 497 dmu_tx_t *tx = dmu_tx_create(zv->zv_objset); 498 dmu_tx_mark_netfree(tx); 499 int error = dmu_tx_assign(tx, DMU_TX_WAIT); 500 if (error != 0) { 501 dmu_tx_abort(tx); 502 } else { 503 (void) zil_replaying(zv->zv_zilog, tx); 504 dmu_tx_commit(tx); 505 error = dmu_free_long_range(zv->zv_objset, ZVOL_OBJ, offset, 506 length); 507 } 508 509 return (error); 510 } 511 512 /* 513 * Replay a TX_WRITE ZIL transaction that didn't get committed 514 * after a system failure 515 */ 516 static int 517 zvol_replay_write(void *arg1, void *arg2, boolean_t byteswap) 518 { 519 zvol_state_t *zv = arg1; 520 lr_write_t *lr = arg2; 521 objset_t *os = zv->zv_objset; 522 char *data = (char *)(lr + 1); /* data follows lr_write_t */ 523 uint64_t offset, length; 524 dmu_tx_t *tx; 525 int error; 526 527 ASSERT3U(lr->lr_common.lrc_reclen, >=, sizeof (*lr)); 528 529 if (byteswap) 530 byteswap_uint64_array(lr, sizeof (*lr)); 531 532 offset = lr->lr_offset; 533 length = lr->lr_length; 534 535 /* If it's a dmu_sync() block, write the whole block */ 536 if (lr->lr_common.lrc_reclen == sizeof (lr_write_t)) { 537 uint64_t blocksize = BP_GET_LSIZE(&lr->lr_blkptr); 538 if (length < blocksize) { 539 offset -= offset % blocksize; 540 length = blocksize; 541 } 542 } 543 544 tx = dmu_tx_create(os); 545 dmu_tx_hold_write(tx, ZVOL_OBJ, offset, length); 546 error = dmu_tx_assign(tx, DMU_TX_WAIT); 547 if (error) { 548 dmu_tx_abort(tx); 549 } else { 550 dmu_write(os, ZVOL_OBJ, offset, length, data, tx, 551 DMU_READ_PREFETCH); 552 (void) zil_replaying(zv->zv_zilog, tx); 553 dmu_tx_commit(tx); 554 } 555 556 return (error); 557 } 558 559 /* 560 * Replay a TX_CLONE_RANGE ZIL transaction that didn't get committed 561 * after a system failure 562 */ 563 static int 564 zvol_replay_clone_range(void *arg1, void *arg2, boolean_t byteswap) 565 { 566 zvol_state_t *zv = arg1; 567 lr_clone_range_t *lr = arg2; 568 objset_t *os = zv->zv_objset; 569 dmu_tx_t *tx; 570 int error; 571 uint64_t blksz; 572 uint64_t off; 573 uint64_t len; 574 575 ASSERT3U(lr->lr_common.lrc_reclen, >=, sizeof (*lr)); 576 ASSERT3U(lr->lr_common.lrc_reclen, >=, offsetof(lr_clone_range_t, 577 lr_bps[lr->lr_nbps])); 578 579 if (byteswap) 580 byteswap_uint64_array(lr, sizeof (*lr)); 581 582 ASSERT(spa_feature_is_enabled(dmu_objset_spa(os), 583 SPA_FEATURE_BLOCK_CLONING)); 584 585 off = lr->lr_offset; 586 len = lr->lr_length; 587 blksz = lr->lr_blksz; 588 589 if ((off % blksz) != 0) { 590 return (SET_ERROR(EINVAL)); 591 } 592 593 error = dnode_hold(os, ZVOL_OBJ, zv, &zv->zv_dn); 594 if (error != 0 || !zv->zv_dn) 595 return (error); 596 tx = dmu_tx_create(os); 597 dmu_tx_hold_clone_by_dnode(tx, zv->zv_dn, off, len, blksz); 598 error = dmu_tx_assign(tx, DMU_TX_WAIT); 599 if (error != 0) { 600 dmu_tx_abort(tx); 601 goto out; 602 } 603 error = dmu_brt_clone(zv->zv_objset, ZVOL_OBJ, off, len, 604 tx, lr->lr_bps, lr->lr_nbps); 605 if (error != 0) { 606 dmu_tx_commit(tx); 607 goto out; 608 } 609 610 /* 611 * zil_replaying() not only check if we are replaying ZIL, but also 612 * updates the ZIL header to record replay progress. 613 */ 614 VERIFY(zil_replaying(zv->zv_zilog, tx)); 615 dmu_tx_commit(tx); 616 617 out: 618 dnode_rele(zv->zv_dn, zv); 619 zv->zv_dn = NULL; 620 return (error); 621 } 622 623 int 624 zvol_clone_range(zvol_state_t *zv_src, uint64_t inoff, zvol_state_t *zv_dst, 625 uint64_t outoff, uint64_t len) 626 { 627 zilog_t *zilog_dst; 628 zfs_locked_range_t *inlr, *outlr; 629 objset_t *inos, *outos; 630 dmu_tx_t *tx; 631 blkptr_t *bps; 632 size_t maxblocks; 633 int error = 0; 634 635 rw_enter(&zv_dst->zv_suspend_lock, RW_READER); 636 if (zv_dst->zv_zilog == NULL) { 637 rw_exit(&zv_dst->zv_suspend_lock); 638 rw_enter(&zv_dst->zv_suspend_lock, RW_WRITER); 639 if (zv_dst->zv_zilog == NULL) { 640 zv_dst->zv_zilog = zil_open(zv_dst->zv_objset, 641 zvol_get_data, &zv_dst->zv_kstat.dk_zil_sums); 642 zv_dst->zv_flags |= ZVOL_WRITTEN_TO; 643 VERIFY0((zv_dst->zv_zilog->zl_header->zh_flags & 644 ZIL_REPLAY_NEEDED)); 645 } 646 rw_downgrade(&zv_dst->zv_suspend_lock); 647 } 648 if (zv_src != zv_dst) 649 rw_enter(&zv_src->zv_suspend_lock, RW_READER); 650 651 inos = zv_src->zv_objset; 652 outos = zv_dst->zv_objset; 653 654 /* 655 * Sanity checks 656 */ 657 if (!spa_feature_is_enabled(dmu_objset_spa(outos), 658 SPA_FEATURE_BLOCK_CLONING)) { 659 error = SET_ERROR(EOPNOTSUPP); 660 goto out; 661 } 662 if (dmu_objset_spa(inos) != dmu_objset_spa(outos)) { 663 error = SET_ERROR(EXDEV); 664 goto out; 665 } 666 if (inos->os_encrypted != outos->os_encrypted) { 667 error = SET_ERROR(EXDEV); 668 goto out; 669 } 670 if (zv_src->zv_volblocksize != zv_dst->zv_volblocksize) { 671 error = SET_ERROR(EINVAL); 672 goto out; 673 } 674 if (inoff >= zv_src->zv_volsize || outoff >= zv_dst->zv_volsize) { 675 goto out; 676 } 677 678 /* 679 * Do not read beyond boundary 680 */ 681 if (len > zv_src->zv_volsize - inoff) 682 len = zv_src->zv_volsize - inoff; 683 if (len > zv_dst->zv_volsize - outoff) 684 len = zv_dst->zv_volsize - outoff; 685 if (len == 0) 686 goto out; 687 688 /* 689 * No overlapping if we are cloning within the same file 690 */ 691 if (zv_src == zv_dst) { 692 if (inoff < outoff + len && outoff < inoff + len) { 693 error = SET_ERROR(EINVAL); 694 goto out; 695 } 696 } 697 698 /* 699 * Offsets and length must be at block boundaries 700 */ 701 if ((inoff % zv_src->zv_volblocksize) != 0 || 702 (outoff % zv_dst->zv_volblocksize) != 0) { 703 error = SET_ERROR(EINVAL); 704 goto out; 705 } 706 707 /* 708 * Length must be multiple of block size 709 */ 710 if ((len % zv_src->zv_volblocksize) != 0) { 711 error = SET_ERROR(EINVAL); 712 goto out; 713 } 714 715 zilog_dst = zv_dst->zv_zilog; 716 maxblocks = zil_max_log_data(zilog_dst, sizeof (lr_clone_range_t)) / 717 sizeof (bps[0]); 718 bps = vmem_alloc(sizeof (bps[0]) * maxblocks, KM_SLEEP); 719 /* 720 * Maintain predictable lock order. 721 */ 722 if (zv_src < zv_dst || (zv_src == zv_dst && inoff < outoff)) { 723 inlr = zfs_rangelock_enter(&zv_src->zv_rangelock, inoff, len, 724 RL_READER); 725 outlr = zfs_rangelock_enter(&zv_dst->zv_rangelock, outoff, len, 726 RL_WRITER); 727 } else { 728 outlr = zfs_rangelock_enter(&zv_dst->zv_rangelock, outoff, len, 729 RL_WRITER); 730 inlr = zfs_rangelock_enter(&zv_src->zv_rangelock, inoff, len, 731 RL_READER); 732 } 733 734 while (len > 0) { 735 uint64_t size, last_synced_txg; 736 size_t nbps = maxblocks; 737 size = MIN(zv_src->zv_volblocksize * maxblocks, len); 738 last_synced_txg = spa_last_synced_txg( 739 dmu_objset_spa(zv_src->zv_objset)); 740 error = dmu_read_l0_bps(zv_src->zv_objset, ZVOL_OBJ, inoff, 741 size, bps, &nbps); 742 if (error != 0) { 743 /* 744 * If we are trying to clone a block that was created 745 * in the current transaction group, the error will be 746 * EAGAIN here. Based on zfs_bclone_wait_dirty either 747 * return a shortened range to the caller so it can 748 * fallback, or wait for the next TXG and check again. 749 */ 750 if (error == EAGAIN && zfs_bclone_wait_dirty) { 751 txg_wait_synced(dmu_objset_pool 752 (zv_src->zv_objset), last_synced_txg + 1); 753 continue; 754 } 755 break; 756 } 757 758 tx = dmu_tx_create(zv_dst->zv_objset); 759 dmu_tx_hold_clone_by_dnode(tx, zv_dst->zv_dn, outoff, size, 760 zv_src->zv_volblocksize); 761 error = dmu_tx_assign(tx, DMU_TX_WAIT); 762 if (error != 0) { 763 dmu_tx_abort(tx); 764 break; 765 } 766 error = dmu_brt_clone(zv_dst->zv_objset, ZVOL_OBJ, outoff, size, 767 tx, bps, nbps); 768 if (error != 0) { 769 dmu_tx_commit(tx); 770 break; 771 } 772 zvol_log_clone_range(zilog_dst, tx, TX_CLONE_RANGE, outoff, 773 size, zv_src->zv_volblocksize, bps, nbps); 774 dmu_tx_commit(tx); 775 inoff += size; 776 outoff += size; 777 len -= size; 778 } 779 vmem_free(bps, sizeof (bps[0]) * maxblocks); 780 zfs_rangelock_exit(outlr); 781 zfs_rangelock_exit(inlr); 782 if (error == 0 && zv_dst->zv_objset->os_sync == ZFS_SYNC_ALWAYS) { 783 error = zil_commit(zilog_dst, ZVOL_OBJ); 784 } 785 out: 786 if (zv_src != zv_dst) 787 rw_exit(&zv_src->zv_suspend_lock); 788 rw_exit(&zv_dst->zv_suspend_lock); 789 return (error); 790 } 791 792 /* 793 * Handles TX_CLONE_RANGE transactions. 794 */ 795 void 796 zvol_log_clone_range(zilog_t *zilog, dmu_tx_t *tx, int txtype, uint64_t off, 797 uint64_t len, uint64_t blksz, const blkptr_t *bps, size_t nbps) 798 { 799 itx_t *itx; 800 lr_clone_range_t *lr; 801 uint64_t partlen, max_log_data; 802 size_t partnbps; 803 804 if (zil_replaying(zilog, tx)) 805 return; 806 807 max_log_data = zil_max_log_data(zilog, sizeof (lr_clone_range_t)); 808 809 while (nbps > 0) { 810 partnbps = MIN(nbps, max_log_data / sizeof (bps[0])); 811 partlen = partnbps * blksz; 812 ASSERT3U(partlen, <, len + blksz); 813 partlen = MIN(partlen, len); 814 815 itx = zil_itx_create(txtype, 816 sizeof (*lr) + sizeof (bps[0]) * partnbps); 817 lr = (lr_clone_range_t *)&itx->itx_lr; 818 lr->lr_foid = ZVOL_OBJ; 819 lr->lr_offset = off; 820 lr->lr_length = partlen; 821 lr->lr_blksz = blksz; 822 lr->lr_nbps = partnbps; 823 memcpy(lr->lr_bps, bps, sizeof (bps[0]) * partnbps); 824 825 zil_itx_assign(zilog, itx, tx); 826 827 bps += partnbps; 828 ASSERT3U(nbps, >=, partnbps); 829 nbps -= partnbps; 830 off += partlen; 831 ASSERT3U(len, >=, partlen); 832 len -= partlen; 833 } 834 } 835 836 static int 837 zvol_replay_err(void *arg1, void *arg2, boolean_t byteswap) 838 { 839 (void) arg1, (void) arg2, (void) byteswap; 840 return (SET_ERROR(ENOTSUP)); 841 } 842 843 /* 844 * Callback vectors for replaying records. 845 * Only TX_WRITE and TX_TRUNCATE are needed for zvol. 846 */ 847 zil_replay_func_t *const zvol_replay_vector[TX_MAX_TYPE] = { 848 zvol_replay_err, /* no such transaction type */ 849 zvol_replay_err, /* TX_CREATE */ 850 zvol_replay_err, /* TX_MKDIR */ 851 zvol_replay_err, /* TX_MKXATTR */ 852 zvol_replay_err, /* TX_SYMLINK */ 853 zvol_replay_err, /* TX_REMOVE */ 854 zvol_replay_err, /* TX_RMDIR */ 855 zvol_replay_err, /* TX_LINK */ 856 zvol_replay_err, /* TX_RENAME */ 857 zvol_replay_write, /* TX_WRITE */ 858 zvol_replay_truncate, /* TX_TRUNCATE */ 859 zvol_replay_err, /* TX_SETATTR */ 860 zvol_replay_err, /* TX_ACL_V0 */ 861 zvol_replay_err, /* TX_ACL */ 862 zvol_replay_err, /* TX_CREATE_ACL */ 863 zvol_replay_err, /* TX_CREATE_ATTR */ 864 zvol_replay_err, /* TX_CREATE_ACL_ATTR */ 865 zvol_replay_err, /* TX_MKDIR_ACL */ 866 zvol_replay_err, /* TX_MKDIR_ATTR */ 867 zvol_replay_err, /* TX_MKDIR_ACL_ATTR */ 868 zvol_replay_err, /* TX_WRITE2 */ 869 zvol_replay_err, /* TX_SETSAXATTR */ 870 zvol_replay_err, /* TX_RENAME_EXCHANGE */ 871 zvol_replay_err, /* TX_RENAME_WHITEOUT */ 872 zvol_replay_clone_range, /* TX_CLONE_RANGE */ 873 }; 874 875 /* 876 * zvol_log_write() handles TX_WRITE transactions. 877 */ 878 void 879 zvol_log_write(zvol_state_t *zv, dmu_tx_t *tx, uint64_t offset, 880 uint64_t size, boolean_t commit) 881 { 882 uint32_t blocksize = zv->zv_volblocksize; 883 zilog_t *zilog = zv->zv_zilog; 884 itx_wr_state_t write_state; 885 uint64_t log_size = 0; 886 887 if (zil_replaying(zilog, tx)) 888 return; 889 890 write_state = zil_write_state(zilog, size, blocksize, B_FALSE, commit); 891 892 while (size) { 893 itx_t *itx; 894 lr_write_t *lr; 895 itx_wr_state_t wr_state = write_state; 896 ssize_t len = size; 897 898 if (wr_state == WR_COPIED && size > zil_max_copied_data(zilog)) 899 wr_state = WR_NEED_COPY; 900 else if (wr_state == WR_INDIRECT) 901 len = MIN(blocksize - P2PHASE(offset, blocksize), size); 902 903 itx = zil_itx_create(TX_WRITE, sizeof (*lr) + 904 (wr_state == WR_COPIED ? len : 0)); 905 lr = (lr_write_t *)&itx->itx_lr; 906 if (wr_state == WR_COPIED && 907 dmu_read_by_dnode(zv->zv_dn, offset, len, lr + 1, 908 DMU_READ_NO_PREFETCH | DMU_KEEP_CACHING) != 0) { 909 zil_itx_destroy(itx, 0); 910 itx = zil_itx_create(TX_WRITE, sizeof (*lr)); 911 lr = (lr_write_t *)&itx->itx_lr; 912 wr_state = WR_NEED_COPY; 913 } 914 915 log_size += itx->itx_size; 916 if (wr_state == WR_NEED_COPY) 917 log_size += len; 918 919 itx->itx_wr_state = wr_state; 920 lr->lr_foid = ZVOL_OBJ; 921 lr->lr_offset = offset; 922 lr->lr_length = len; 923 lr->lr_blkoff = 0; 924 BP_ZERO(&lr->lr_blkptr); 925 926 itx->itx_private = zv; 927 928 zil_itx_assign(zilog, itx, tx); 929 930 offset += len; 931 size -= len; 932 } 933 934 dsl_pool_wrlog_count(zilog->zl_dmu_pool, log_size, tx->tx_txg); 935 } 936 937 /* 938 * Log a DKIOCFREE/free-long-range to the ZIL with TX_TRUNCATE. 939 */ 940 void 941 zvol_log_truncate(zvol_state_t *zv, dmu_tx_t *tx, uint64_t off, uint64_t len) 942 { 943 itx_t *itx; 944 lr_truncate_t *lr; 945 zilog_t *zilog = zv->zv_zilog; 946 947 if (zil_replaying(zilog, tx)) 948 return; 949 950 itx = zil_itx_create(TX_TRUNCATE, sizeof (*lr)); 951 lr = (lr_truncate_t *)&itx->itx_lr; 952 lr->lr_foid = ZVOL_OBJ; 953 lr->lr_offset = off; 954 lr->lr_length = len; 955 956 zil_itx_assign(zilog, itx, tx); 957 } 958 959 960 static void 961 zvol_get_done(zgd_t *zgd, int error) 962 { 963 (void) error; 964 if (zgd->zgd_db) 965 dmu_buf_rele(zgd->zgd_db, zgd); 966 967 zfs_rangelock_exit(zgd->zgd_lr); 968 969 kmem_free(zgd, sizeof (zgd_t)); 970 } 971 972 /* 973 * Get data to generate a TX_WRITE intent log record. 974 */ 975 int 976 zvol_get_data(void *arg, uint64_t arg2, lr_write_t *lr, char *buf, 977 struct lwb *lwb, zio_t *zio) 978 { 979 zvol_state_t *zv = arg; 980 uint64_t offset = lr->lr_offset; 981 uint64_t size = lr->lr_length; 982 dmu_buf_t *db; 983 zgd_t *zgd; 984 int error; 985 986 ASSERT3P(lwb, !=, NULL); 987 ASSERT3U(size, !=, 0); 988 989 zgd = kmem_zalloc(sizeof (zgd_t), KM_SLEEP); 990 zgd->zgd_lwb = lwb; 991 992 /* 993 * Write records come in two flavors: immediate and indirect. 994 * For small writes it's cheaper to store the data with the 995 * log record (immediate); for large writes it's cheaper to 996 * sync the data and get a pointer to it (indirect) so that 997 * we don't have to write the data twice. 998 */ 999 if (buf != NULL) { /* immediate write */ 1000 zgd->zgd_lr = zfs_rangelock_enter(&zv->zv_rangelock, offset, 1001 size, RL_READER); 1002 error = dmu_read_by_dnode(zv->zv_dn, offset, size, buf, 1003 DMU_READ_NO_PREFETCH | DMU_KEEP_CACHING); 1004 } else { /* indirect write */ 1005 ASSERT3P(zio, !=, NULL); 1006 /* 1007 * Have to lock the whole block to ensure when it's written out 1008 * and its checksum is being calculated that no one can change 1009 * the data. Contrarily to zfs_get_data we need not re-check 1010 * blocksize after we get the lock because it cannot be changed. 1011 */ 1012 size = zv->zv_volblocksize; 1013 offset = P2ALIGN_TYPED(offset, size, uint64_t); 1014 zgd->zgd_lr = zfs_rangelock_enter(&zv->zv_rangelock, offset, 1015 size, RL_READER); 1016 error = dmu_buf_hold_noread_by_dnode(zv->zv_dn, offset, zgd, 1017 &db); 1018 if (error == 0) { 1019 blkptr_t *bp = &lr->lr_blkptr; 1020 1021 zgd->zgd_db = db; 1022 zgd->zgd_bp = bp; 1023 1024 ASSERT(db != NULL); 1025 ASSERT(db->db_offset == offset); 1026 ASSERT(db->db_size == size); 1027 1028 error = dmu_sync(zio, lr->lr_common.lrc_txg, 1029 zvol_get_done, zgd); 1030 1031 if (error == 0) 1032 return (0); 1033 } 1034 } 1035 1036 zvol_get_done(zgd, error); 1037 1038 return (error); 1039 } 1040 1041 /* 1042 * The zvol_state_t's are inserted into zvol_state_list and zvol_htable. 1043 */ 1044 1045 void 1046 zvol_insert(zvol_state_t *zv) 1047 { 1048 ASSERT(RW_WRITE_HELD(&zvol_state_lock)); 1049 list_insert_head(&zvol_state_list, zv); 1050 hlist_add_head(&zv->zv_hlink, ZVOL_HT_HEAD(zv->zv_hash)); 1051 } 1052 1053 /* 1054 * Simply remove the zvol from to list of zvols. 1055 */ 1056 static void 1057 zvol_remove(zvol_state_t *zv) 1058 { 1059 ASSERT(RW_WRITE_HELD(&zvol_state_lock)); 1060 list_remove(&zvol_state_list, zv); 1061 hlist_del(&zv->zv_hlink); 1062 } 1063 1064 /* 1065 * Setup zv after we just own the zv->objset 1066 */ 1067 static int 1068 zvol_setup_zv(zvol_state_t *zv) 1069 { 1070 uint64_t volsize; 1071 int error; 1072 uint64_t ro; 1073 objset_t *os = zv->zv_objset; 1074 1075 ASSERT(MUTEX_HELD(&zv->zv_state_lock)); 1076 ASSERT(RW_LOCK_HELD(&zv->zv_suspend_lock)); 1077 1078 zv->zv_zilog = NULL; 1079 zv->zv_flags &= ~ZVOL_WRITTEN_TO; 1080 1081 error = dsl_prop_get_integer(zv->zv_name, "readonly", &ro, NULL); 1082 if (error) 1083 return (error); 1084 1085 error = zap_lookup(os, ZVOL_ZAP_OBJ, "size", 8, 1, &volsize); 1086 if (error) 1087 return (error); 1088 1089 error = dnode_hold(os, ZVOL_OBJ, zv, &zv->zv_dn); 1090 if (error) 1091 return (error); 1092 1093 zvol_os_set_capacity(zv, volsize >> 9); 1094 zv->zv_volsize = volsize; 1095 1096 if (ro || dmu_objset_is_snapshot(os) || 1097 !spa_writeable(dmu_objset_spa(os))) { 1098 zvol_os_set_disk_ro(zv, 1); 1099 zv->zv_flags |= ZVOL_RDONLY; 1100 } else { 1101 zvol_os_set_disk_ro(zv, 0); 1102 zv->zv_flags &= ~ZVOL_RDONLY; 1103 } 1104 return (0); 1105 } 1106 1107 /* 1108 * Shutdown every zv_objset related stuff except zv_objset itself. 1109 * The is the reverse of zvol_setup_zv. 1110 */ 1111 static void 1112 zvol_shutdown_zv(zvol_state_t *zv) 1113 { 1114 ASSERT(MUTEX_HELD(&zv->zv_state_lock) && 1115 RW_LOCK_HELD(&zv->zv_suspend_lock)); 1116 1117 if (zv->zv_flags & ZVOL_WRITTEN_TO) { 1118 ASSERT(zv->zv_zilog != NULL); 1119 zil_close(zv->zv_zilog); 1120 } 1121 1122 zv->zv_zilog = NULL; 1123 1124 dnode_rele(zv->zv_dn, zv); 1125 zv->zv_dn = NULL; 1126 1127 /* 1128 * Evict cached data. We must write out any dirty data before 1129 * disowning the dataset. 1130 */ 1131 if (zv->zv_flags & ZVOL_WRITTEN_TO) 1132 txg_wait_synced(dmu_objset_pool(zv->zv_objset), 0); 1133 dmu_objset_evict_dbufs(zv->zv_objset); 1134 } 1135 1136 /* 1137 * return the proper tag for rollback and recv 1138 */ 1139 void * 1140 zvol_tag(zvol_state_t *zv) 1141 { 1142 ASSERT(RW_WRITE_HELD(&zv->zv_suspend_lock)); 1143 return (zv->zv_open_count > 0 ? zv : NULL); 1144 } 1145 1146 /* 1147 * Suspend the zvol for recv and rollback. 1148 */ 1149 int 1150 zvol_suspend(const char *name, zvol_state_t **zvp) 1151 { 1152 zvol_state_t *zv; 1153 1154 zv = zvol_find_by_name(name, RW_WRITER); 1155 1156 if (zv == NULL) 1157 return (SET_ERROR(ENOENT)); 1158 1159 /* block all I/O, release in zvol_resume. */ 1160 ASSERT(MUTEX_HELD(&zv->zv_state_lock)); 1161 ASSERT(RW_WRITE_HELD(&zv->zv_suspend_lock)); 1162 1163 /* 1164 * If it's being removed, unlock and return error. It doesn't make any 1165 * sense to try to suspend a zvol being removed, but being here also 1166 * means that zvol_remove_minors_impl() is about to call zvol_remove() 1167 * and then destroy the zvol_state_t, so returning a pointer to it for 1168 * the caller to mess with would be a disaster anyway. 1169 */ 1170 if (zv->zv_flags & ZVOL_REMOVING) { 1171 mutex_exit(&zv->zv_state_lock); 1172 rw_exit(&zv->zv_suspend_lock); 1173 /* NB: Returning EIO here to match zfsvfs_teardown() */ 1174 return (SET_ERROR(EIO)); 1175 } 1176 1177 atomic_inc(&zv->zv_suspend_ref); 1178 1179 if (zv->zv_open_count > 0) 1180 zvol_shutdown_zv(zv); 1181 1182 /* 1183 * do not hold zv_state_lock across suspend/resume to 1184 * avoid locking up zvol lookups 1185 */ 1186 mutex_exit(&zv->zv_state_lock); 1187 1188 /* zv_suspend_lock is released in zvol_resume() */ 1189 *zvp = zv; 1190 return (0); 1191 } 1192 1193 int 1194 zvol_resume(zvol_state_t *zv) 1195 { 1196 int error = 0; 1197 1198 ASSERT(RW_WRITE_HELD(&zv->zv_suspend_lock)); 1199 1200 mutex_enter(&zv->zv_state_lock); 1201 1202 if (zv->zv_open_count > 0) { 1203 VERIFY0(dmu_objset_hold(zv->zv_name, zv, &zv->zv_objset)); 1204 VERIFY3P(zv->zv_objset->os_dsl_dataset->ds_owner, ==, zv); 1205 VERIFY(dsl_dataset_long_held(zv->zv_objset->os_dsl_dataset)); 1206 dmu_objset_rele(zv->zv_objset, zv); 1207 1208 error = zvol_setup_zv(zv); 1209 } 1210 1211 mutex_exit(&zv->zv_state_lock); 1212 1213 rw_exit(&zv->zv_suspend_lock); 1214 /* 1215 * We need this because we don't hold zvol_state_lock while releasing 1216 * zv_suspend_lock. zvol_remove_minors_impl thus cannot check 1217 * zv_suspend_lock to determine it is safe to free because rwlock is 1218 * not inherent atomic. 1219 */ 1220 atomic_dec(&zv->zv_suspend_ref); 1221 1222 if (zv->zv_flags & ZVOL_REMOVING) 1223 cv_broadcast(&zv->zv_removing_cv); 1224 1225 return (error); 1226 } 1227 1228 int 1229 zvol_first_open(zvol_state_t *zv, boolean_t readonly) 1230 { 1231 objset_t *os; 1232 int error; 1233 1234 ASSERT(RW_READ_HELD(&zv->zv_suspend_lock)); 1235 ASSERT(MUTEX_HELD(&zv->zv_state_lock)); 1236 ASSERT(spa_namespace_held()); 1237 1238 boolean_t ro = (readonly || (strchr(zv->zv_name, '@') != NULL)); 1239 error = dmu_objset_own(zv->zv_name, DMU_OST_ZVOL, ro, B_TRUE, zv, &os); 1240 if (error) 1241 return (error); 1242 1243 zv->zv_objset = os; 1244 1245 error = zvol_setup_zv(zv); 1246 if (error) { 1247 dmu_objset_disown(os, 1, zv); 1248 zv->zv_objset = NULL; 1249 } 1250 1251 return (error); 1252 } 1253 1254 void 1255 zvol_last_close(zvol_state_t *zv) 1256 { 1257 ASSERT(RW_READ_HELD(&zv->zv_suspend_lock)); 1258 ASSERT(MUTEX_HELD(&zv->zv_state_lock)); 1259 1260 if (zv->zv_flags & ZVOL_REMOVING) 1261 cv_broadcast(&zv->zv_removing_cv); 1262 1263 zvol_shutdown_zv(zv); 1264 1265 dmu_objset_disown(zv->zv_objset, 1, zv); 1266 zv->zv_objset = NULL; 1267 } 1268 1269 typedef struct minors_job { 1270 list_t *list; 1271 list_node_t link; 1272 /* input */ 1273 char *name; 1274 /* output */ 1275 int error; 1276 } minors_job_t; 1277 1278 /* 1279 * Prefetch zvol dnodes for the minors_job 1280 */ 1281 static void 1282 zvol_prefetch_minors_impl(void *arg) 1283 { 1284 minors_job_t *job = arg; 1285 char *dsname = job->name; 1286 objset_t *os = NULL; 1287 1288 job->error = dmu_objset_own(dsname, DMU_OST_ZVOL, B_TRUE, B_TRUE, 1289 FTAG, &os); 1290 if (job->error == 0) { 1291 dmu_prefetch_dnode(os, ZVOL_OBJ, ZIO_PRIORITY_SYNC_READ); 1292 dmu_objset_disown(os, B_TRUE, FTAG); 1293 } 1294 } 1295 1296 /* 1297 * Mask errors to continue dmu_objset_find() traversal 1298 */ 1299 static int 1300 zvol_create_snap_minor_cb(const char *dsname, void *arg) 1301 { 1302 minors_job_t *j = arg; 1303 list_t *minors_list = j->list; 1304 const char *name = j->name; 1305 1306 ASSERT0(spa_namespace_held()); 1307 1308 /* skip the designated dataset */ 1309 if (name && strcmp(dsname, name) == 0) 1310 return (0); 1311 1312 /* at this point, the dsname should name a snapshot */ 1313 if (strchr(dsname, '@') == 0) { 1314 dprintf("zvol_create_snap_minor_cb(): " 1315 "%s is not a snapshot name\n", dsname); 1316 } else { 1317 minors_job_t *job; 1318 char *n = kmem_strdup(dsname); 1319 if (n == NULL) 1320 return (0); 1321 1322 job = kmem_alloc(sizeof (minors_job_t), KM_SLEEP); 1323 job->name = n; 1324 job->list = minors_list; 1325 job->error = 0; 1326 list_insert_tail(minors_list, job); 1327 /* don't care if dispatch fails, because job->error is 0 */ 1328 taskq_dispatch(system_taskq, zvol_prefetch_minors_impl, job, 1329 TQ_SLEEP); 1330 } 1331 1332 return (0); 1333 } 1334 1335 /* 1336 * If spa_keystore_load_wkey() is called for an encrypted zvol, 1337 * we need to look for any clones also using the key. This function 1338 * is "best effort" - so we just skip over it if there are failures. 1339 */ 1340 static void 1341 zvol_add_clones(const char *dsname, list_t *minors_list) 1342 { 1343 /* Also check if it has clones */ 1344 dsl_dir_t *dd = NULL; 1345 dsl_pool_t *dp = NULL; 1346 1347 if (dsl_pool_hold(dsname, FTAG, &dp) != 0) 1348 return; 1349 1350 if (!spa_feature_is_enabled(dp->dp_spa, 1351 SPA_FEATURE_ENCRYPTION)) 1352 goto out; 1353 1354 if (dsl_dir_hold(dp, dsname, FTAG, &dd, NULL) != 0) 1355 goto out; 1356 1357 if (dsl_dir_phys(dd)->dd_clones == 0) 1358 goto out; 1359 1360 zap_cursor_t *zc = kmem_alloc(sizeof (zap_cursor_t), KM_SLEEP); 1361 zap_attribute_t *za = zap_attribute_alloc(); 1362 objset_t *mos = dd->dd_pool->dp_meta_objset; 1363 1364 for (zap_cursor_init(zc, mos, dsl_dir_phys(dd)->dd_clones); 1365 zap_cursor_retrieve(zc, za) == 0; 1366 zap_cursor_advance(zc)) { 1367 dsl_dataset_t *clone; 1368 minors_job_t *job; 1369 1370 if (dsl_dataset_hold_obj(dd->dd_pool, 1371 za->za_first_integer, FTAG, &clone) == 0) { 1372 1373 char name[ZFS_MAX_DATASET_NAME_LEN]; 1374 dsl_dataset_name(clone, name); 1375 1376 char *n = kmem_strdup(name); 1377 job = kmem_alloc(sizeof (minors_job_t), KM_SLEEP); 1378 job->name = n; 1379 job->list = minors_list; 1380 job->error = 0; 1381 list_insert_tail(minors_list, job); 1382 1383 dsl_dataset_rele(clone, FTAG); 1384 } 1385 } 1386 zap_cursor_fini(zc); 1387 zap_attribute_free(za); 1388 kmem_free(zc, sizeof (zap_cursor_t)); 1389 1390 out: 1391 if (dd != NULL) 1392 dsl_dir_rele(dd, FTAG); 1393 dsl_pool_rele(dp, FTAG); 1394 } 1395 1396 /* 1397 * Mask errors to continue dmu_objset_find() traversal 1398 */ 1399 static int 1400 zvol_create_minors_cb(const char *dsname, void *arg) 1401 { 1402 uint64_t snapdev; 1403 int error; 1404 list_t *minors_list = arg; 1405 1406 ASSERT0(spa_namespace_held()); 1407 1408 error = dsl_prop_get_integer(dsname, "snapdev", &snapdev, NULL); 1409 if (error) 1410 return (0); 1411 1412 /* 1413 * Given the name and the 'snapdev' property, create device minor nodes 1414 * with the linkages to zvols/snapshots as needed. 1415 * If the name represents a zvol, create a minor node for the zvol, then 1416 * check if its snapshots are 'visible', and if so, iterate over the 1417 * snapshots and create device minor nodes for those. 1418 */ 1419 if (strchr(dsname, '@') == 0) { 1420 minors_job_t *job; 1421 char *n = kmem_strdup(dsname); 1422 if (n == NULL) 1423 return (0); 1424 1425 job = kmem_alloc(sizeof (minors_job_t), KM_SLEEP); 1426 job->name = n; 1427 job->list = minors_list; 1428 job->error = 0; 1429 list_insert_tail(minors_list, job); 1430 /* don't care if dispatch fails, because job->error is 0 */ 1431 taskq_dispatch(system_taskq, zvol_prefetch_minors_impl, job, 1432 TQ_SLEEP); 1433 1434 zvol_add_clones(dsname, minors_list); 1435 1436 if (snapdev == ZFS_SNAPDEV_VISIBLE) { 1437 /* 1438 * traverse snapshots only, do not traverse children, 1439 * and skip the 'dsname' 1440 */ 1441 (void) dmu_objset_find(dsname, 1442 zvol_create_snap_minor_cb, (void *)job, 1443 DS_FIND_SNAPSHOTS); 1444 } 1445 } else { 1446 dprintf("zvol_create_minors_cb(): %s is not a zvol name\n", 1447 dsname); 1448 } 1449 1450 return (0); 1451 } 1452 1453 static void 1454 zvol_task_update_status(zvol_task_t *task, uint64_t total, uint64_t done, 1455 int error) 1456 { 1457 1458 task->zt_total += total; 1459 task->zt_done += done; 1460 if (task->zt_total != task->zt_done) { 1461 task->zt_status = -1; 1462 if (error) 1463 task->zt_error = error; 1464 } 1465 } 1466 1467 static void 1468 zvol_task_report_status(zvol_task_t *task) 1469 { 1470 #ifdef ZFS_DEBUG 1471 static const char *const msg[] = { 1472 "create", 1473 "remove", 1474 "rename", 1475 "set snapdev", 1476 "set volmode", 1477 "unknown", 1478 }; 1479 1480 if (task->zt_status == 0) 1481 return; 1482 1483 zvol_async_op_t op = MIN(task->zt_op, ZVOL_ASYNC_MAX); 1484 if (task->zt_error) { 1485 dprintf("The %s minors zvol task was not ok, last error %d\n", 1486 msg[op], task->zt_error); 1487 } else { 1488 dprintf("The %s minors zvol task was not ok\n", msg[op]); 1489 } 1490 #else 1491 (void) task; 1492 #endif 1493 } 1494 1495 /* 1496 * Create minors for the specified dataset, including children and snapshots. 1497 * Pay attention to the 'snapdev' property and iterate over the snapshots 1498 * only if they are 'visible'. This approach allows one to assure that the 1499 * snapshot metadata is read from disk only if it is needed. 1500 * 1501 * The name can represent a dataset to be recursively scanned for zvols and 1502 * their snapshots, or a single zvol snapshot. If the name represents a 1503 * dataset, the scan is performed in two nested stages: 1504 * - scan the dataset for zvols, and 1505 * - for each zvol, create a minor node, then check if the zvol's snapshots 1506 * are 'visible', and only then iterate over the snapshots if needed 1507 * 1508 * If the name represents a snapshot, a check is performed if the snapshot is 1509 * 'visible' (which also verifies that the parent is a zvol), and if so, 1510 * a minor node for that snapshot is created. 1511 */ 1512 static void 1513 zvol_create_minors_impl(zvol_task_t *task) 1514 { 1515 const char *name = task->zt_name1; 1516 list_t minors_list; 1517 minors_job_t *job; 1518 uint64_t snapdev; 1519 int total = 0, done = 0, last_error, error; 1520 1521 /* 1522 * Note: the dsl_pool_config_lock must not be held. 1523 * Minor node creation needs to obtain the zvol_state_lock. 1524 * zvol_open() obtains the zvol_state_lock and then the dsl pool 1525 * config lock. Therefore, we can't have the config lock now if 1526 * we are going to wait for the zvol_state_lock, because it 1527 * would be a lock order inversion which could lead to deadlock. 1528 */ 1529 1530 if (zvol_inhibit_dev) { 1531 return; 1532 } 1533 1534 /* 1535 * This is the list for prefetch jobs. Whenever we found a match 1536 * during dmu_objset_find, we insert a minors_job to the list and do 1537 * taskq_dispatch to parallel prefetch zvol dnodes. Note we don't need 1538 * any lock because all list operation is done on the current thread. 1539 * 1540 * We will use this list to do zvol_os_create_minor after prefetch 1541 * so we don't have to traverse using dmu_objset_find again. 1542 */ 1543 list_create(&minors_list, sizeof (minors_job_t), 1544 offsetof(minors_job_t, link)); 1545 1546 1547 if (strchr(name, '@') != NULL) { 1548 error = dsl_prop_get_integer(name, "snapdev", &snapdev, NULL); 1549 if (error == 0 && snapdev == ZFS_SNAPDEV_VISIBLE) { 1550 error = zvol_os_create_minor(name); 1551 if (error == 0) { 1552 done++; 1553 } else { 1554 last_error = error; 1555 } 1556 total++; 1557 } 1558 } else { 1559 fstrans_cookie_t cookie = spl_fstrans_mark(); 1560 (void) dmu_objset_find(name, zvol_create_minors_cb, 1561 &minors_list, DS_FIND_CHILDREN); 1562 spl_fstrans_unmark(cookie); 1563 } 1564 1565 taskq_wait_outstanding(system_taskq, 0); 1566 1567 /* 1568 * Prefetch is completed, we can do zvol_os_create_minor 1569 * sequentially. 1570 */ 1571 while ((job = list_remove_head(&minors_list)) != NULL) { 1572 if (!job->error) { 1573 error = zvol_os_create_minor(job->name); 1574 if (error == 0) { 1575 done++; 1576 } else { 1577 last_error = error; 1578 } 1579 } else if (job->error == EINVAL) { 1580 /* 1581 * The objset, with the name requested by current job 1582 * exist, but have the type different from zvol. 1583 * Just ignore this sort of errors. 1584 */ 1585 done++; 1586 } else { 1587 last_error = job->error; 1588 } 1589 total++; 1590 kmem_strfree(job->name); 1591 kmem_free(job, sizeof (minors_job_t)); 1592 } 1593 1594 list_destroy(&minors_list); 1595 zvol_task_update_status(task, total, done, last_error); 1596 } 1597 1598 /* 1599 * Remove minors for specified dataset and, optionally, its children and 1600 * snapshots. 1601 */ 1602 static void 1603 zvol_remove_minors_impl(zvol_task_t *task) 1604 { 1605 zvol_state_t *zv, *zv_next; 1606 const char *name = task ? task->zt_name1 : NULL; 1607 int namelen = ((name) ? strlen(name) : 0); 1608 boolean_t children = task ? !!task->zt_value : B_TRUE; 1609 1610 if (zvol_inhibit_dev) 1611 return; 1612 1613 /* 1614 * We collect up zvols that we want to remove on a separate list, so 1615 * that we don't have to hold zvol_state_lock for the whole time. 1616 * 1617 * We can't remove them from the global lists until we're completely 1618 * done with them, because that would make them appear to ZFS-side ops 1619 * that they don't exist, and the name might be reused, which can't be 1620 * good. 1621 */ 1622 list_t remove_list; 1623 list_create(&remove_list, sizeof (zvol_state_t), 1624 offsetof(zvol_state_t, zv_remove_node)); 1625 1626 rw_enter(&zvol_state_lock, RW_READER); 1627 1628 for (zv = list_head(&zvol_state_list); zv != NULL; zv = zv_next) { 1629 zv_next = list_next(&zvol_state_list, zv); 1630 1631 mutex_enter(&zv->zv_state_lock); 1632 if (zv->zv_flags & ZVOL_REMOVING) { 1633 /* Another thread is handling shutdown, skip it. */ 1634 mutex_exit(&zv->zv_state_lock); 1635 continue; 1636 } 1637 1638 /* 1639 * This zvol should be removed if: 1640 * - no name was offered (ie removing all at shutdown); or 1641 * - name matches exactly; or 1642 * - we were asked to remove children, and 1643 * - the start of the name matches, and 1644 * - there is a '/' immediately after the matched name; or 1645 * - there is a '@' immediately after the matched name 1646 */ 1647 if (name == NULL || strcmp(zv->zv_name, name) == 0 || 1648 (children && strncmp(zv->zv_name, name, namelen) == 0 && 1649 (zv->zv_name[namelen] == '/' || 1650 zv->zv_name[namelen] == '@'))) { 1651 1652 /* 1653 * Matched, so mark it removal. We want to take the 1654 * write half of the suspend lock to make sure that 1655 * the zvol is not suspended, and give any data ops 1656 * chance to finish. 1657 */ 1658 mutex_exit(&zv->zv_state_lock); 1659 rw_enter(&zv->zv_suspend_lock, RW_WRITER); 1660 mutex_enter(&zv->zv_state_lock); 1661 1662 if (zv->zv_flags & ZVOL_REMOVING) { 1663 /* Another thread has taken it, let them. */ 1664 mutex_exit(&zv->zv_state_lock); 1665 rw_exit(&zv->zv_suspend_lock); 1666 continue; 1667 } 1668 1669 /* 1670 * Mark it and unlock. New entries will see the flag 1671 * and return ENXIO. 1672 */ 1673 zv->zv_flags |= ZVOL_REMOVING; 1674 mutex_exit(&zv->zv_state_lock); 1675 rw_exit(&zv->zv_suspend_lock); 1676 1677 /* Put it on the list for the next stage. */ 1678 list_insert_head(&remove_list, zv); 1679 } else 1680 mutex_exit(&zv->zv_state_lock); 1681 } 1682 1683 rw_exit(&zvol_state_lock); 1684 1685 /* Didn't match any, nothing to do! */ 1686 if (list_is_empty(&remove_list)) { 1687 if (task) 1688 task->zt_error = SET_ERROR(ENOENT); 1689 return; 1690 } 1691 1692 /* Actually shut them all down. */ 1693 for (zv = list_head(&remove_list); zv != NULL; zv = zv_next) { 1694 zv_next = list_next(&remove_list, zv); 1695 1696 mutex_enter(&zv->zv_state_lock); 1697 1698 /* 1699 * Still open or suspended, just wait. This can happen if, for 1700 * example, we managed to acquire zv_state_lock in the moments 1701 * where zvol_open() or zvol_release() are trading locks to 1702 * call zvol_first_open() or zvol_last_close(). 1703 */ 1704 while (zv->zv_open_count > 0 || 1705 atomic_read(&zv->zv_suspend_ref)) 1706 cv_wait(&zv->zv_removing_cv, &zv->zv_state_lock); 1707 1708 /* 1709 * No users, shut down the OS side. This may not remove the 1710 * minor from view immediately, depending on the kernel 1711 * specifics, but it will ensure that it is unusable and that 1712 * this zvol_state_t can never again be reached from an OS-side 1713 * operation. 1714 */ 1715 zvol_os_remove_minor(zv); 1716 mutex_exit(&zv->zv_state_lock); 1717 1718 /* Remove it from the name lookup lists */ 1719 rw_enter(&zvol_state_lock, RW_WRITER); 1720 zvol_remove(zv); 1721 rw_exit(&zvol_state_lock); 1722 } 1723 1724 /* 1725 * Our own references on remove_list is the last one, free them and 1726 * we're done. 1727 */ 1728 while ((zv = list_remove_head(&remove_list)) != NULL) 1729 zvol_os_free(zv); 1730 1731 list_destroy(&remove_list); 1732 } 1733 1734 /* Remove minor for this specific volume only */ 1735 static int 1736 zvol_remove_minor_impl(const char *name) 1737 { 1738 if (zvol_inhibit_dev) 1739 return (0); 1740 1741 zvol_task_t task; 1742 memset(&task, 0, sizeof (zvol_task_t)); 1743 strlcpy(task.zt_name1, name, sizeof (task.zt_name1)); 1744 task.zt_value = B_FALSE; 1745 1746 zvol_remove_minors_impl(&task); 1747 1748 return (task.zt_error); 1749 } 1750 1751 /* 1752 * Rename minors for specified dataset including children and snapshots. 1753 */ 1754 static void 1755 zvol_rename_minors_impl(zvol_task_t *task) 1756 { 1757 zvol_state_t *zv, *zv_next; 1758 const char *oldname = task->zt_name1; 1759 const char *newname = task->zt_name2; 1760 int total = 0, done = 0, last_error, error, oldnamelen; 1761 1762 if (zvol_inhibit_dev) 1763 return; 1764 1765 oldnamelen = strlen(oldname); 1766 1767 rw_enter(&zvol_state_lock, RW_READER); 1768 1769 for (zv = list_head(&zvol_state_list); zv != NULL; zv = zv_next) { 1770 zv_next = list_next(&zvol_state_list, zv); 1771 1772 mutex_enter(&zv->zv_state_lock); 1773 1774 if (strcmp(zv->zv_name, oldname) == 0) { 1775 error = zvol_os_rename_minor(zv, newname); 1776 } else if (strncmp(zv->zv_name, oldname, oldnamelen) == 0 && 1777 (zv->zv_name[oldnamelen] == '/' || 1778 zv->zv_name[oldnamelen] == '@')) { 1779 char *name = kmem_asprintf("%s%c%s", newname, 1780 zv->zv_name[oldnamelen], 1781 zv->zv_name + oldnamelen + 1); 1782 error = zvol_os_rename_minor(zv, name); 1783 kmem_strfree(name); 1784 } 1785 if (error) { 1786 last_error = error; 1787 } else { 1788 done++; 1789 } 1790 total++; 1791 mutex_exit(&zv->zv_state_lock); 1792 } 1793 1794 rw_exit(&zvol_state_lock); 1795 zvol_task_update_status(task, total, done, last_error); 1796 } 1797 1798 typedef struct zvol_snapdev_cb_arg { 1799 zvol_task_t *task; 1800 uint64_t snapdev; 1801 } zvol_snapdev_cb_arg_t; 1802 1803 static int 1804 zvol_set_snapdev_cb(const char *dsname, void *param) 1805 { 1806 zvol_snapdev_cb_arg_t *arg = param; 1807 int error = 0; 1808 1809 if (strchr(dsname, '@') == NULL) 1810 return (0); 1811 1812 switch (arg->snapdev) { 1813 case ZFS_SNAPDEV_VISIBLE: 1814 error = zvol_os_create_minor(dsname); 1815 break; 1816 case ZFS_SNAPDEV_HIDDEN: 1817 error = zvol_remove_minor_impl(dsname); 1818 break; 1819 } 1820 1821 zvol_task_update_status(arg->task, 1, error == 0, error); 1822 return (0); 1823 } 1824 1825 static void 1826 zvol_set_snapdev_impl(zvol_task_t *task) 1827 { 1828 const char *name = task->zt_name1; 1829 uint64_t snapdev = task->zt_value; 1830 1831 zvol_snapdev_cb_arg_t arg = {task, snapdev}; 1832 fstrans_cookie_t cookie = spl_fstrans_mark(); 1833 /* 1834 * The zvol_set_snapdev_sync() sets snapdev appropriately 1835 * in the dataset hierarchy. Here, we only scan snapshots. 1836 */ 1837 dmu_objset_find(name, zvol_set_snapdev_cb, &arg, DS_FIND_SNAPSHOTS); 1838 spl_fstrans_unmark(cookie); 1839 } 1840 1841 static void 1842 zvol_set_volmode_impl(zvol_task_t *task) 1843 { 1844 const char *name = task->zt_name1; 1845 uint64_t volmode = task->zt_value; 1846 fstrans_cookie_t cookie; 1847 uint64_t old_volmode; 1848 zvol_state_t *zv; 1849 int error; 1850 1851 if (strchr(name, '@') != NULL) 1852 return; 1853 1854 /* 1855 * It's unfortunate we need to remove minors before we create new ones: 1856 * this is necessary because our backing gendisk (zvol_state->zv_disk) 1857 * could be different when we set, for instance, volmode from "geom" 1858 * to "dev" (or vice versa). 1859 */ 1860 zv = zvol_find_by_name(name, RW_NONE); 1861 if (zv == NULL && volmode == ZFS_VOLMODE_NONE) 1862 return; 1863 if (zv != NULL) { 1864 old_volmode = zv->zv_volmode; 1865 mutex_exit(&zv->zv_state_lock); 1866 if (old_volmode == volmode) 1867 return; 1868 zvol_wait_close(zv); 1869 } 1870 cookie = spl_fstrans_mark(); 1871 switch (volmode) { 1872 case ZFS_VOLMODE_NONE: 1873 error = zvol_remove_minor_impl(name); 1874 break; 1875 case ZFS_VOLMODE_GEOM: 1876 case ZFS_VOLMODE_DEV: 1877 error = zvol_remove_minor_impl(name); 1878 /* 1879 * The remove minor function call above, might be not 1880 * needed, if volmode was switched from 'none' value. 1881 * Ignore error in this case. 1882 */ 1883 if (error == ENOENT) 1884 error = 0; 1885 else if (error) 1886 break; 1887 error = zvol_os_create_minor(name); 1888 break; 1889 case ZFS_VOLMODE_DEFAULT: 1890 error = zvol_remove_minor_impl(name); 1891 if (zvol_volmode == ZFS_VOLMODE_NONE) 1892 break; 1893 else /* if zvol_volmode is invalid defaults to "geom" */ 1894 error = zvol_os_create_minor(name); 1895 break; 1896 } 1897 zvol_task_update_status(task, 1, error == 0, error); 1898 spl_fstrans_unmark(cookie); 1899 } 1900 1901 /* 1902 * The worker thread function performed asynchronously. 1903 */ 1904 static void 1905 zvol_task_cb(void *arg) 1906 { 1907 zvol_task_t *task = arg; 1908 1909 switch (task->zt_op) { 1910 case ZVOL_ASYNC_CREATE_MINORS: 1911 zvol_create_minors_impl(task); 1912 break; 1913 case ZVOL_ASYNC_REMOVE_MINORS: 1914 zvol_remove_minors_impl(task); 1915 break; 1916 case ZVOL_ASYNC_RENAME_MINORS: 1917 zvol_rename_minors_impl(task); 1918 break; 1919 case ZVOL_ASYNC_SET_SNAPDEV: 1920 zvol_set_snapdev_impl(task); 1921 break; 1922 case ZVOL_ASYNC_SET_VOLMODE: 1923 zvol_set_volmode_impl(task); 1924 break; 1925 default: 1926 VERIFY(0); 1927 break; 1928 } 1929 1930 zvol_task_report_status(task); 1931 kmem_free(task, sizeof (zvol_task_t)); 1932 } 1933 1934 typedef struct zvol_set_prop_int_arg { 1935 const char *zsda_name; 1936 uint64_t zsda_value; 1937 zprop_source_t zsda_source; 1938 zfs_prop_t zsda_prop; 1939 } zvol_set_prop_int_arg_t; 1940 1941 /* 1942 * Sanity check the dataset for safe use by the sync task. No additional 1943 * conditions are imposed. 1944 */ 1945 static int 1946 zvol_set_common_check(void *arg, dmu_tx_t *tx) 1947 { 1948 zvol_set_prop_int_arg_t *zsda = arg; 1949 dsl_pool_t *dp = dmu_tx_pool(tx); 1950 dsl_dir_t *dd; 1951 int error; 1952 1953 error = dsl_dir_hold(dp, zsda->zsda_name, FTAG, &dd, NULL); 1954 if (error != 0) 1955 return (error); 1956 1957 dsl_dir_rele(dd, FTAG); 1958 1959 return (error); 1960 } 1961 1962 static int 1963 zvol_set_common_sync_cb(dsl_pool_t *dp, dsl_dataset_t *ds, void *arg) 1964 { 1965 zvol_set_prop_int_arg_t *zsda = arg; 1966 char dsname[ZFS_MAX_DATASET_NAME_LEN]; 1967 zvol_task_t *task; 1968 uint64_t prop; 1969 1970 const char *prop_name = zfs_prop_to_name(zsda->zsda_prop); 1971 dsl_dataset_name(ds, dsname); 1972 1973 if (dsl_prop_get_int_ds(ds, prop_name, &prop) != 0) 1974 return (0); 1975 1976 task = kmem_zalloc(sizeof (zvol_task_t), KM_SLEEP); 1977 if (zsda->zsda_prop == ZFS_PROP_VOLMODE) { 1978 task->zt_op = ZVOL_ASYNC_SET_VOLMODE; 1979 } else if (zsda->zsda_prop == ZFS_PROP_SNAPDEV) { 1980 task->zt_op = ZVOL_ASYNC_SET_SNAPDEV; 1981 } else { 1982 kmem_free(task, sizeof (zvol_task_t)); 1983 return (0); 1984 } 1985 task->zt_value = prop; 1986 strlcpy(task->zt_name1, dsname, sizeof (task->zt_name1)); 1987 (void) taskq_dispatch(dp->dp_spa->spa_zvol_taskq, zvol_task_cb, 1988 task, TQ_SLEEP); 1989 return (0); 1990 } 1991 1992 /* 1993 * Traverse all child datasets and apply the property appropriately. 1994 * We call dsl_prop_set_sync_impl() here to set the value only on the toplevel 1995 * dataset and read the effective "property" on every child in the callback 1996 * function: this is because the value is not guaranteed to be the same in the 1997 * whole dataset hierarchy. 1998 */ 1999 static void 2000 zvol_set_common_sync(void *arg, dmu_tx_t *tx) 2001 { 2002 zvol_set_prop_int_arg_t *zsda = arg; 2003 dsl_pool_t *dp = dmu_tx_pool(tx); 2004 dsl_dir_t *dd; 2005 dsl_dataset_t *ds; 2006 int error; 2007 2008 VERIFY0(dsl_dir_hold(dp, zsda->zsda_name, FTAG, &dd, NULL)); 2009 2010 error = dsl_dataset_hold(dp, zsda->zsda_name, FTAG, &ds); 2011 if (error == 0) { 2012 dsl_prop_set_sync_impl(ds, zfs_prop_to_name(zsda->zsda_prop), 2013 zsda->zsda_source, sizeof (zsda->zsda_value), 1, 2014 &zsda->zsda_value, tx); 2015 dsl_dataset_rele(ds, FTAG); 2016 } 2017 2018 dmu_objset_find_dp(dp, dd->dd_object, zvol_set_common_sync_cb, 2019 zsda, DS_FIND_CHILDREN); 2020 2021 dsl_dir_rele(dd, FTAG); 2022 } 2023 2024 int 2025 zvol_set_common(const char *ddname, zfs_prop_t prop, zprop_source_t source, 2026 uint64_t val) 2027 { 2028 zvol_set_prop_int_arg_t zsda; 2029 2030 zsda.zsda_name = ddname; 2031 zsda.zsda_source = source; 2032 zsda.zsda_value = val; 2033 zsda.zsda_prop = prop; 2034 2035 return (dsl_sync_task(ddname, zvol_set_common_check, 2036 zvol_set_common_sync, &zsda, 0, ZFS_SPACE_CHECK_NONE)); 2037 } 2038 2039 void 2040 zvol_create_minors(const char *name) 2041 { 2042 spa_t *spa; 2043 zvol_task_t *task; 2044 taskqid_t id; 2045 2046 if (spa_open(name, &spa, FTAG) != 0) 2047 return; 2048 2049 task = kmem_zalloc(sizeof (zvol_task_t), KM_SLEEP); 2050 task->zt_op = ZVOL_ASYNC_CREATE_MINORS; 2051 strlcpy(task->zt_name1, name, sizeof (task->zt_name1)); 2052 id = taskq_dispatch(spa->spa_zvol_taskq, zvol_task_cb, task, TQ_SLEEP); 2053 if (id != TASKQID_INVALID) 2054 taskq_wait_id(spa->spa_zvol_taskq, id); 2055 2056 spa_close(spa, FTAG); 2057 } 2058 2059 void 2060 zvol_remove_minors(spa_t *spa, const char *name, boolean_t async) 2061 { 2062 zvol_task_t *task; 2063 taskqid_t id; 2064 2065 task = kmem_zalloc(sizeof (zvol_task_t), KM_SLEEP); 2066 task->zt_op = ZVOL_ASYNC_REMOVE_MINORS; 2067 strlcpy(task->zt_name1, name, sizeof (task->zt_name1)); 2068 task->zt_value = B_TRUE; 2069 id = taskq_dispatch(spa->spa_zvol_taskq, zvol_task_cb, task, TQ_SLEEP); 2070 if ((async == B_FALSE) && (id != TASKQID_INVALID)) 2071 taskq_wait_id(spa->spa_zvol_taskq, id); 2072 } 2073 2074 void 2075 zvol_rename_minors(spa_t *spa, const char *name1, const char *name2, 2076 boolean_t async) 2077 { 2078 zvol_task_t *task; 2079 taskqid_t id; 2080 2081 task = kmem_zalloc(sizeof (zvol_task_t), KM_SLEEP); 2082 task->zt_op = ZVOL_ASYNC_RENAME_MINORS; 2083 strlcpy(task->zt_name1, name1, sizeof (task->zt_name1)); 2084 strlcpy(task->zt_name2, name2, sizeof (task->zt_name2)); 2085 id = taskq_dispatch(spa->spa_zvol_taskq, zvol_task_cb, task, TQ_SLEEP); 2086 if ((async == B_FALSE) && (id != TASKQID_INVALID)) 2087 taskq_wait_id(spa->spa_zvol_taskq, id); 2088 } 2089 2090 boolean_t 2091 zvol_is_zvol(const char *name) 2092 { 2093 2094 return (zvol_os_is_zvol(name)); 2095 } 2096 2097 int 2098 zvol_init_impl(void) 2099 { 2100 int i; 2101 2102 /* 2103 * zvol_threads is the module param the user passes in. 2104 * 2105 * zvol_actual_threads is what we use internally, since the user can 2106 * pass zvol_thread = 0 to mean "use all the CPUs" (the default). 2107 */ 2108 static unsigned int zvol_actual_threads; 2109 2110 if (zvol_threads == 0) { 2111 /* 2112 * See dde9380a1 for why 32 was chosen here. This should 2113 * probably be refined to be some multiple of the number 2114 * of CPUs. 2115 */ 2116 zvol_actual_threads = MAX(max_ncpus, 32); 2117 } else { 2118 zvol_actual_threads = MIN(MAX(zvol_threads, 1), 1024); 2119 } 2120 2121 /* 2122 * Use at least 32 zvol_threads but for many core system, 2123 * prefer 6 threads per taskq, but no more taskqs 2124 * than threads in them on large systems. 2125 * 2126 * taskq total 2127 * cpus taskqs threads threads 2128 * ------- ------- ------- ------- 2129 * 1 1 32 32 2130 * 2 1 32 32 2131 * 4 1 32 32 2132 * 8 2 16 32 2133 * 16 3 11 33 2134 * 32 5 7 35 2135 * 64 8 8 64 2136 * 128 11 12 132 2137 * 256 16 16 256 2138 */ 2139 zv_taskq_t *ztqs = &zvol_taskqs; 2140 int num_tqs = MIN(max_ncpus, zvol_num_taskqs); 2141 if (num_tqs == 0) { 2142 num_tqs = 1 + max_ncpus / 6; 2143 while (num_tqs * num_tqs > zvol_actual_threads) 2144 num_tqs--; 2145 } 2146 2147 int per_tq_thread = zvol_actual_threads / num_tqs; 2148 if (per_tq_thread * num_tqs < zvol_actual_threads) 2149 per_tq_thread++; 2150 2151 ztqs->tqs_cnt = num_tqs; 2152 ztqs->tqs_taskq = kmem_alloc(num_tqs * sizeof (taskq_t *), KM_SLEEP); 2153 2154 for (uint_t i = 0; i < num_tqs; i++) { 2155 char name[32]; 2156 (void) snprintf(name, sizeof (name), "%s_tq-%u", 2157 ZVOL_DRIVER, i); 2158 ztqs->tqs_taskq[i] = taskq_create(name, per_tq_thread, 2159 maxclsyspri, per_tq_thread, INT_MAX, 2160 TASKQ_PREPOPULATE | TASKQ_DYNAMIC); 2161 if (ztqs->tqs_taskq[i] == NULL) { 2162 for (int j = i - 1; j >= 0; j--) 2163 taskq_destroy(ztqs->tqs_taskq[j]); 2164 kmem_free(ztqs->tqs_taskq, ztqs->tqs_cnt * 2165 sizeof (taskq_t *)); 2166 ztqs->tqs_taskq = NULL; 2167 return (SET_ERROR(ENOMEM)); 2168 } 2169 } 2170 2171 list_create(&zvol_state_list, sizeof (zvol_state_t), 2172 offsetof(zvol_state_t, zv_next)); 2173 rw_init(&zvol_state_lock, NULL, RW_DEFAULT, NULL); 2174 2175 zvol_htable = kmem_alloc(ZVOL_HT_SIZE * sizeof (struct hlist_head), 2176 KM_SLEEP); 2177 for (i = 0; i < ZVOL_HT_SIZE; i++) 2178 INIT_HLIST_HEAD(&zvol_htable[i]); 2179 2180 return (0); 2181 } 2182 2183 void 2184 zvol_fini_impl(void) 2185 { 2186 zv_taskq_t *ztqs = &zvol_taskqs; 2187 2188 zvol_remove_minors_impl(NULL); 2189 2190 kmem_free(zvol_htable, ZVOL_HT_SIZE * sizeof (struct hlist_head)); 2191 list_destroy(&zvol_state_list); 2192 rw_destroy(&zvol_state_lock); 2193 2194 if (ztqs->tqs_taskq == NULL) { 2195 ASSERT0(ztqs->tqs_cnt); 2196 } else { 2197 for (uint_t i = 0; i < ztqs->tqs_cnt; i++) { 2198 ASSERT3P(ztqs->tqs_taskq[i], !=, NULL); 2199 taskq_destroy(ztqs->tqs_taskq[i]); 2200 } 2201 kmem_free(ztqs->tqs_taskq, ztqs->tqs_cnt * 2202 sizeof (taskq_t *)); 2203 ztqs->tqs_taskq = NULL; 2204 } 2205 } 2206 2207 ZFS_MODULE_PARAM(zfs_vol, zvol_, inhibit_dev, UINT, ZMOD_RW, 2208 "Do not create zvol device nodes"); 2209 ZFS_MODULE_PARAM(zfs_vol, zvol_, prefetch_bytes, UINT, ZMOD_RW, 2210 "Prefetch N bytes at zvol start+end"); 2211 ZFS_MODULE_PARAM(zfs_vol, zvol_vol, mode, UINT, ZMOD_RW, 2212 "Default volmode property value"); 2213 ZFS_MODULE_PARAM(zfs_vol, zvol_, threads, UINT, ZMOD_RW, 2214 "Number of threads for I/O requests. Set to 0 to use all active CPUs"); 2215 ZFS_MODULE_PARAM(zfs_vol, zvol_, num_taskqs, UINT, ZMOD_RW, 2216 "Number of zvol taskqs"); 2217 ZFS_MODULE_PARAM(zfs_vol, zvol_, request_sync, UINT, ZMOD_RW, 2218 "Synchronously handle bio requests"); 2219