1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or https://opensource.org/licenses/CDDL-1.0. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 22 /* 23 * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved. 24 * Copyright (c) 2012, 2018 by Delphix. All rights reserved. 25 * Copyright (c) 2015 by Chunwei Chen. All rights reserved. 26 * Copyright 2017 Nexenta Systems, Inc. 27 * Copyright (c) 2021, 2022 by Pawel Jakub Dawidek 28 */ 29 30 /* Portions Copyright 2007 Jeremy Teo */ 31 /* Portions Copyright 2010 Robert Milkowski */ 32 33 #include <sys/types.h> 34 #include <sys/param.h> 35 #include <sys/time.h> 36 #include <sys/sysmacros.h> 37 #include <sys/vfs.h> 38 #include <sys/uio_impl.h> 39 #include <sys/file.h> 40 #include <sys/stat.h> 41 #include <sys/kmem.h> 42 #include <sys/cmn_err.h> 43 #include <sys/errno.h> 44 #include <sys/zfs_dir.h> 45 #include <sys/zfs_acl.h> 46 #include <sys/zfs_ioctl.h> 47 #include <sys/fs/zfs.h> 48 #include <sys/dmu.h> 49 #include <sys/dmu_objset.h> 50 #include <sys/dsl_crypt.h> 51 #include <sys/spa.h> 52 #include <sys/txg.h> 53 #include <sys/dbuf.h> 54 #include <sys/policy.h> 55 #include <sys/zfeature.h> 56 #include <sys/zfs_vnops.h> 57 #include <sys/zfs_quota.h> 58 #include <sys/zfs_vfsops.h> 59 #include <sys/zfs_znode.h> 60 61 62 int 63 zfs_fsync(znode_t *zp, int syncflag, cred_t *cr) 64 { 65 int error = 0; 66 zfsvfs_t *zfsvfs = ZTOZSB(zp); 67 68 if (zfsvfs->z_os->os_sync != ZFS_SYNC_DISABLED) { 69 if ((error = zfs_enter_verify_zp(zfsvfs, zp, FTAG)) != 0) 70 return (error); 71 atomic_inc_32(&zp->z_sync_writes_cnt); 72 zil_commit(zfsvfs->z_log, zp->z_id); 73 atomic_dec_32(&zp->z_sync_writes_cnt); 74 zfs_exit(zfsvfs, FTAG); 75 } 76 return (error); 77 } 78 79 80 #if defined(SEEK_HOLE) && defined(SEEK_DATA) 81 /* 82 * Lseek support for finding holes (cmd == SEEK_HOLE) and 83 * data (cmd == SEEK_DATA). "off" is an in/out parameter. 84 */ 85 static int 86 zfs_holey_common(znode_t *zp, ulong_t cmd, loff_t *off) 87 { 88 zfs_locked_range_t *lr; 89 uint64_t noff = (uint64_t)*off; /* new offset */ 90 uint64_t file_sz; 91 int error; 92 boolean_t hole; 93 94 file_sz = zp->z_size; 95 if (noff >= file_sz) { 96 return (SET_ERROR(ENXIO)); 97 } 98 99 if (cmd == F_SEEK_HOLE) 100 hole = B_TRUE; 101 else 102 hole = B_FALSE; 103 104 /* Flush any mmap()'d data to disk */ 105 if (zn_has_cached_data(zp, 0, file_sz - 1)) 106 zn_flush_cached_data(zp, B_FALSE); 107 108 lr = zfs_rangelock_enter(&zp->z_rangelock, 0, UINT64_MAX, RL_READER); 109 error = dmu_offset_next(ZTOZSB(zp)->z_os, zp->z_id, hole, &noff); 110 zfs_rangelock_exit(lr); 111 112 if (error == ESRCH) 113 return (SET_ERROR(ENXIO)); 114 115 /* File was dirty, so fall back to using generic logic */ 116 if (error == EBUSY) { 117 if (hole) 118 *off = file_sz; 119 120 return (0); 121 } 122 123 /* 124 * We could find a hole that begins after the logical end-of-file, 125 * because dmu_offset_next() only works on whole blocks. If the 126 * EOF falls mid-block, then indicate that the "virtual hole" 127 * at the end of the file begins at the logical EOF, rather than 128 * at the end of the last block. 129 */ 130 if (noff > file_sz) { 131 ASSERT(hole); 132 noff = file_sz; 133 } 134 135 if (noff < *off) 136 return (error); 137 *off = noff; 138 return (error); 139 } 140 141 int 142 zfs_holey(znode_t *zp, ulong_t cmd, loff_t *off) 143 { 144 zfsvfs_t *zfsvfs = ZTOZSB(zp); 145 int error; 146 147 if ((error = zfs_enter_verify_zp(zfsvfs, zp, FTAG)) != 0) 148 return (error); 149 150 error = zfs_holey_common(zp, cmd, off); 151 152 zfs_exit(zfsvfs, FTAG); 153 return (error); 154 } 155 #endif /* SEEK_HOLE && SEEK_DATA */ 156 157 int 158 zfs_access(znode_t *zp, int mode, int flag, cred_t *cr) 159 { 160 zfsvfs_t *zfsvfs = ZTOZSB(zp); 161 int error; 162 163 if ((error = zfs_enter_verify_zp(zfsvfs, zp, FTAG)) != 0) 164 return (error); 165 166 if (flag & V_ACE_MASK) 167 #if defined(__linux__) 168 error = zfs_zaccess(zp, mode, flag, B_FALSE, cr, 169 zfs_init_idmap); 170 #else 171 error = zfs_zaccess(zp, mode, flag, B_FALSE, cr, 172 NULL); 173 #endif 174 else 175 #if defined(__linux__) 176 error = zfs_zaccess_rwx(zp, mode, flag, cr, zfs_init_idmap); 177 #else 178 error = zfs_zaccess_rwx(zp, mode, flag, cr, NULL); 179 #endif 180 181 zfs_exit(zfsvfs, FTAG); 182 return (error); 183 } 184 185 static uint64_t zfs_vnops_read_chunk_size = 1024 * 1024; /* Tunable */ 186 187 /* 188 * Read bytes from specified file into supplied buffer. 189 * 190 * IN: zp - inode of file to be read from. 191 * uio - structure supplying read location, range info, 192 * and return buffer. 193 * ioflag - O_SYNC flags; used to provide FRSYNC semantics. 194 * O_DIRECT flag; used to bypass page cache. 195 * cr - credentials of caller. 196 * 197 * OUT: uio - updated offset and range, buffer filled. 198 * 199 * RETURN: 0 on success, error code on failure. 200 * 201 * Side Effects: 202 * inode - atime updated if byte count > 0 203 */ 204 int 205 zfs_read(struct znode *zp, zfs_uio_t *uio, int ioflag, cred_t *cr) 206 { 207 (void) cr; 208 int error = 0; 209 boolean_t frsync = B_FALSE; 210 211 zfsvfs_t *zfsvfs = ZTOZSB(zp); 212 if ((error = zfs_enter_verify_zp(zfsvfs, zp, FTAG)) != 0) 213 return (error); 214 215 if (zp->z_pflags & ZFS_AV_QUARANTINED) { 216 zfs_exit(zfsvfs, FTAG); 217 return (SET_ERROR(EACCES)); 218 } 219 220 /* We don't copy out anything useful for directories. */ 221 if (Z_ISDIR(ZTOTYPE(zp))) { 222 zfs_exit(zfsvfs, FTAG); 223 return (SET_ERROR(EISDIR)); 224 } 225 226 /* 227 * Validate file offset 228 */ 229 if (zfs_uio_offset(uio) < (offset_t)0) { 230 zfs_exit(zfsvfs, FTAG); 231 return (SET_ERROR(EINVAL)); 232 } 233 234 /* 235 * Fasttrack empty reads 236 */ 237 if (zfs_uio_resid(uio) == 0) { 238 zfs_exit(zfsvfs, FTAG); 239 return (0); 240 } 241 242 #ifdef FRSYNC 243 /* 244 * If we're in FRSYNC mode, sync out this znode before reading it. 245 * Only do this for non-snapshots. 246 * 247 * Some platforms do not support FRSYNC and instead map it 248 * to O_SYNC, which results in unnecessary calls to zil_commit. We 249 * only honor FRSYNC requests on platforms which support it. 250 */ 251 frsync = !!(ioflag & FRSYNC); 252 #endif 253 if (zfsvfs->z_log && 254 (frsync || zfsvfs->z_os->os_sync == ZFS_SYNC_ALWAYS)) 255 zil_commit(zfsvfs->z_log, zp->z_id); 256 257 /* 258 * Lock the range against changes. 259 */ 260 zfs_locked_range_t *lr = zfs_rangelock_enter(&zp->z_rangelock, 261 zfs_uio_offset(uio), zfs_uio_resid(uio), RL_READER); 262 263 /* 264 * If we are reading past end-of-file we can skip 265 * to the end; but we might still need to set atime. 266 */ 267 if (zfs_uio_offset(uio) >= zp->z_size) { 268 error = 0; 269 goto out; 270 } 271 272 ASSERT(zfs_uio_offset(uio) < zp->z_size); 273 #if defined(__linux__) 274 ssize_t start_offset = zfs_uio_offset(uio); 275 #endif 276 ssize_t n = MIN(zfs_uio_resid(uio), zp->z_size - zfs_uio_offset(uio)); 277 ssize_t start_resid = n; 278 279 while (n > 0) { 280 ssize_t nbytes = MIN(n, zfs_vnops_read_chunk_size - 281 P2PHASE(zfs_uio_offset(uio), zfs_vnops_read_chunk_size)); 282 #ifdef UIO_NOCOPY 283 if (zfs_uio_segflg(uio) == UIO_NOCOPY) 284 error = mappedread_sf(zp, nbytes, uio); 285 else 286 #endif 287 if (zn_has_cached_data(zp, zfs_uio_offset(uio), 288 zfs_uio_offset(uio) + nbytes - 1) && !(ioflag & O_DIRECT)) { 289 error = mappedread(zp, nbytes, uio); 290 } else { 291 error = dmu_read_uio_dbuf(sa_get_db(zp->z_sa_hdl), 292 uio, nbytes); 293 } 294 295 if (error) { 296 /* convert checksum errors into IO errors */ 297 if (error == ECKSUM) 298 error = SET_ERROR(EIO); 299 300 #if defined(__linux__) 301 /* 302 * if we actually read some bytes, bubbling EFAULT 303 * up to become EAGAIN isn't what we want here... 304 * 305 * ...on Linux, at least. On FBSD, doing this breaks. 306 */ 307 if (error == EFAULT && 308 (zfs_uio_offset(uio) - start_offset) != 0) 309 error = 0; 310 #endif 311 break; 312 } 313 314 n -= nbytes; 315 } 316 317 int64_t nread = start_resid - n; 318 dataset_kstats_update_read_kstats(&zfsvfs->z_kstat, nread); 319 task_io_account_read(nread); 320 out: 321 zfs_rangelock_exit(lr); 322 323 ZFS_ACCESSTIME_STAMP(zfsvfs, zp); 324 zfs_exit(zfsvfs, FTAG); 325 return (error); 326 } 327 328 static void 329 zfs_clear_setid_bits_if_necessary(zfsvfs_t *zfsvfs, znode_t *zp, cred_t *cr, 330 uint64_t *clear_setid_bits_txgp, dmu_tx_t *tx) 331 { 332 zilog_t *zilog = zfsvfs->z_log; 333 const uint64_t uid = KUID_TO_SUID(ZTOUID(zp)); 334 335 ASSERT(clear_setid_bits_txgp != NULL); 336 ASSERT(tx != NULL); 337 338 /* 339 * Clear Set-UID/Set-GID bits on successful write if not 340 * privileged and at least one of the execute bits is set. 341 * 342 * It would be nice to do this after all writes have 343 * been done, but that would still expose the ISUID/ISGID 344 * to another app after the partial write is committed. 345 * 346 * Note: we don't call zfs_fuid_map_id() here because 347 * user 0 is not an ephemeral uid. 348 */ 349 mutex_enter(&zp->z_acl_lock); 350 if ((zp->z_mode & (S_IXUSR | (S_IXUSR >> 3) | (S_IXUSR >> 6))) != 0 && 351 (zp->z_mode & (S_ISUID | S_ISGID)) != 0 && 352 secpolicy_vnode_setid_retain(zp, cr, 353 ((zp->z_mode & S_ISUID) != 0 && uid == 0)) != 0) { 354 uint64_t newmode; 355 356 zp->z_mode &= ~(S_ISUID | S_ISGID); 357 newmode = zp->z_mode; 358 (void) sa_update(zp->z_sa_hdl, SA_ZPL_MODE(zfsvfs), 359 (void *)&newmode, sizeof (uint64_t), tx); 360 361 mutex_exit(&zp->z_acl_lock); 362 363 /* 364 * Make sure SUID/SGID bits will be removed when we replay the 365 * log. If the setid bits are keep coming back, don't log more 366 * than one TX_SETATTR per transaction group. 367 */ 368 if (*clear_setid_bits_txgp != dmu_tx_get_txg(tx)) { 369 vattr_t va = {0}; 370 371 va.va_mask = ATTR_MODE; 372 va.va_nodeid = zp->z_id; 373 va.va_mode = newmode; 374 zfs_log_setattr(zilog, tx, TX_SETATTR, zp, &va, 375 ATTR_MODE, NULL); 376 *clear_setid_bits_txgp = dmu_tx_get_txg(tx); 377 } 378 } else { 379 mutex_exit(&zp->z_acl_lock); 380 } 381 } 382 383 /* 384 * Write the bytes to a file. 385 * 386 * IN: zp - znode of file to be written to. 387 * uio - structure supplying write location, range info, 388 * and data buffer. 389 * ioflag - O_APPEND flag set if in append mode. 390 * O_DIRECT flag; used to bypass page cache. 391 * cr - credentials of caller. 392 * 393 * OUT: uio - updated offset and range. 394 * 395 * RETURN: 0 if success 396 * error code if failure 397 * 398 * Timestamps: 399 * ip - ctime|mtime updated if byte count > 0 400 */ 401 int 402 zfs_write(znode_t *zp, zfs_uio_t *uio, int ioflag, cred_t *cr) 403 { 404 int error = 0, error1; 405 ssize_t start_resid = zfs_uio_resid(uio); 406 uint64_t clear_setid_bits_txg = 0; 407 408 /* 409 * Fasttrack empty write 410 */ 411 ssize_t n = start_resid; 412 if (n == 0) 413 return (0); 414 415 zfsvfs_t *zfsvfs = ZTOZSB(zp); 416 if ((error = zfs_enter_verify_zp(zfsvfs, zp, FTAG)) != 0) 417 return (error); 418 419 sa_bulk_attr_t bulk[4]; 420 int count = 0; 421 uint64_t mtime[2], ctime[2]; 422 SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_MTIME(zfsvfs), NULL, &mtime, 16); 423 SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_CTIME(zfsvfs), NULL, &ctime, 16); 424 SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_SIZE(zfsvfs), NULL, 425 &zp->z_size, 8); 426 SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_FLAGS(zfsvfs), NULL, 427 &zp->z_pflags, 8); 428 429 /* 430 * Callers might not be able to detect properly that we are read-only, 431 * so check it explicitly here. 432 */ 433 if (zfs_is_readonly(zfsvfs)) { 434 zfs_exit(zfsvfs, FTAG); 435 return (SET_ERROR(EROFS)); 436 } 437 438 /* 439 * If immutable or not appending then return EPERM. 440 * Intentionally allow ZFS_READONLY through here. 441 * See zfs_zaccess_common() 442 */ 443 if ((zp->z_pflags & ZFS_IMMUTABLE) || 444 ((zp->z_pflags & ZFS_APPENDONLY) && !(ioflag & O_APPEND) && 445 (zfs_uio_offset(uio) < zp->z_size))) { 446 zfs_exit(zfsvfs, FTAG); 447 return (SET_ERROR(EPERM)); 448 } 449 450 /* 451 * Validate file offset 452 */ 453 offset_t woff = ioflag & O_APPEND ? zp->z_size : zfs_uio_offset(uio); 454 if (woff < 0) { 455 zfs_exit(zfsvfs, FTAG); 456 return (SET_ERROR(EINVAL)); 457 } 458 459 /* 460 * Pre-fault the pages to ensure slow (eg NFS) pages 461 * don't hold up txg. 462 */ 463 ssize_t pfbytes = MIN(n, DMU_MAX_ACCESS >> 1); 464 if (zfs_uio_prefaultpages(pfbytes, uio)) { 465 zfs_exit(zfsvfs, FTAG); 466 return (SET_ERROR(EFAULT)); 467 } 468 469 /* 470 * If in append mode, set the io offset pointer to eof. 471 */ 472 zfs_locked_range_t *lr; 473 if (ioflag & O_APPEND) { 474 /* 475 * Obtain an appending range lock to guarantee file append 476 * semantics. We reset the write offset once we have the lock. 477 */ 478 lr = zfs_rangelock_enter(&zp->z_rangelock, 0, n, RL_APPEND); 479 woff = lr->lr_offset; 480 if (lr->lr_length == UINT64_MAX) { 481 /* 482 * We overlocked the file because this write will cause 483 * the file block size to increase. 484 * Note that zp_size cannot change with this lock held. 485 */ 486 woff = zp->z_size; 487 } 488 zfs_uio_setoffset(uio, woff); 489 } else { 490 /* 491 * Note that if the file block size will change as a result of 492 * this write, then this range lock will lock the entire file 493 * so that we can re-write the block safely. 494 */ 495 lr = zfs_rangelock_enter(&zp->z_rangelock, woff, n, RL_WRITER); 496 } 497 498 if (zn_rlimit_fsize_uio(zp, uio)) { 499 zfs_rangelock_exit(lr); 500 zfs_exit(zfsvfs, FTAG); 501 return (SET_ERROR(EFBIG)); 502 } 503 504 const rlim64_t limit = MAXOFFSET_T; 505 506 if (woff >= limit) { 507 zfs_rangelock_exit(lr); 508 zfs_exit(zfsvfs, FTAG); 509 return (SET_ERROR(EFBIG)); 510 } 511 512 if (n > limit - woff) 513 n = limit - woff; 514 515 uint64_t end_size = MAX(zp->z_size, woff + n); 516 zilog_t *zilog = zfsvfs->z_log; 517 boolean_t commit = (ioflag & (O_SYNC | O_DSYNC)) || 518 (zfsvfs->z_os->os_sync == ZFS_SYNC_ALWAYS); 519 520 const uint64_t uid = KUID_TO_SUID(ZTOUID(zp)); 521 const uint64_t gid = KGID_TO_SGID(ZTOGID(zp)); 522 const uint64_t projid = zp->z_projid; 523 524 /* 525 * Write the file in reasonable size chunks. Each chunk is written 526 * in a separate transaction; this keeps the intent log records small 527 * and allows us to do more fine-grained space accounting. 528 */ 529 while (n > 0) { 530 woff = zfs_uio_offset(uio); 531 532 if (zfs_id_overblockquota(zfsvfs, DMU_USERUSED_OBJECT, uid) || 533 zfs_id_overblockquota(zfsvfs, DMU_GROUPUSED_OBJECT, gid) || 534 (projid != ZFS_DEFAULT_PROJID && 535 zfs_id_overblockquota(zfsvfs, DMU_PROJECTUSED_OBJECT, 536 projid))) { 537 error = SET_ERROR(EDQUOT); 538 break; 539 } 540 541 uint64_t blksz; 542 if (lr->lr_length == UINT64_MAX && zp->z_size <= zp->z_blksz) { 543 if (zp->z_blksz > zfsvfs->z_max_blksz && 544 !ISP2(zp->z_blksz)) { 545 /* 546 * File's blocksize is already larger than the 547 * "recordsize" property. Only let it grow to 548 * the next power of 2. 549 */ 550 blksz = 1 << highbit64(zp->z_blksz); 551 } else { 552 blksz = zfsvfs->z_max_blksz; 553 } 554 blksz = MIN(blksz, P2ROUNDUP(end_size, 555 SPA_MINBLOCKSIZE)); 556 blksz = MAX(blksz, zp->z_blksz); 557 } else { 558 blksz = zp->z_blksz; 559 } 560 561 arc_buf_t *abuf = NULL; 562 ssize_t nbytes = n; 563 if (n >= blksz && woff >= zp->z_size && 564 P2PHASE(woff, blksz) == 0 && 565 (blksz >= SPA_OLD_MAXBLOCKSIZE || n < 4 * blksz)) { 566 /* 567 * This write covers a full block. "Borrow" a buffer 568 * from the dmu so that we can fill it before we enter 569 * a transaction. This avoids the possibility of 570 * holding up the transaction if the data copy hangs 571 * up on a pagefault (e.g., from an NFS server mapping). 572 */ 573 abuf = dmu_request_arcbuf(sa_get_db(zp->z_sa_hdl), 574 blksz); 575 ASSERT(abuf != NULL); 576 ASSERT(arc_buf_size(abuf) == blksz); 577 if ((error = zfs_uiocopy(abuf->b_data, blksz, 578 UIO_WRITE, uio, &nbytes))) { 579 dmu_return_arcbuf(abuf); 580 break; 581 } 582 ASSERT3S(nbytes, ==, blksz); 583 } else { 584 nbytes = MIN(n, (DMU_MAX_ACCESS >> 1) - 585 P2PHASE(woff, blksz)); 586 if (pfbytes < nbytes) { 587 if (zfs_uio_prefaultpages(nbytes, uio)) { 588 error = SET_ERROR(EFAULT); 589 break; 590 } 591 pfbytes = nbytes; 592 } 593 } 594 595 /* 596 * Start a transaction. 597 */ 598 dmu_tx_t *tx = dmu_tx_create(zfsvfs->z_os); 599 dmu_tx_hold_sa(tx, zp->z_sa_hdl, B_FALSE); 600 dmu_buf_impl_t *db = (dmu_buf_impl_t *)sa_get_db(zp->z_sa_hdl); 601 DB_DNODE_ENTER(db); 602 dmu_tx_hold_write_by_dnode(tx, DB_DNODE(db), woff, nbytes); 603 DB_DNODE_EXIT(db); 604 zfs_sa_upgrade_txholds(tx, zp); 605 error = dmu_tx_assign(tx, TXG_WAIT); 606 if (error) { 607 dmu_tx_abort(tx); 608 if (abuf != NULL) 609 dmu_return_arcbuf(abuf); 610 break; 611 } 612 613 /* 614 * NB: We must call zfs_clear_setid_bits_if_necessary before 615 * committing the transaction! 616 */ 617 618 /* 619 * If rangelock_enter() over-locked we grow the blocksize 620 * and then reduce the lock range. This will only happen 621 * on the first iteration since rangelock_reduce() will 622 * shrink down lr_length to the appropriate size. 623 */ 624 if (lr->lr_length == UINT64_MAX) { 625 zfs_grow_blocksize(zp, blksz, tx); 626 zfs_rangelock_reduce(lr, woff, n); 627 } 628 629 ssize_t tx_bytes; 630 if (abuf == NULL) { 631 tx_bytes = zfs_uio_resid(uio); 632 zfs_uio_fault_disable(uio, B_TRUE); 633 error = dmu_write_uio_dbuf(sa_get_db(zp->z_sa_hdl), 634 uio, nbytes, tx); 635 zfs_uio_fault_disable(uio, B_FALSE); 636 #ifdef __linux__ 637 if (error == EFAULT) { 638 zfs_clear_setid_bits_if_necessary(zfsvfs, zp, 639 cr, &clear_setid_bits_txg, tx); 640 dmu_tx_commit(tx); 641 /* 642 * Account for partial writes before 643 * continuing the loop. 644 * Update needs to occur before the next 645 * zfs_uio_prefaultpages, or prefaultpages may 646 * error, and we may break the loop early. 647 */ 648 n -= tx_bytes - zfs_uio_resid(uio); 649 pfbytes -= tx_bytes - zfs_uio_resid(uio); 650 continue; 651 } 652 #endif 653 /* 654 * On FreeBSD, EFAULT should be propagated back to the 655 * VFS, which will handle faulting and will retry. 656 */ 657 if (error != 0 && error != EFAULT) { 658 zfs_clear_setid_bits_if_necessary(zfsvfs, zp, 659 cr, &clear_setid_bits_txg, tx); 660 dmu_tx_commit(tx); 661 break; 662 } 663 tx_bytes -= zfs_uio_resid(uio); 664 } else { 665 /* 666 * Thus, we're writing a full block at a block-aligned 667 * offset and extending the file past EOF. 668 * 669 * dmu_assign_arcbuf_by_dbuf() will directly assign the 670 * arc buffer to a dbuf. 671 */ 672 error = dmu_assign_arcbuf_by_dbuf( 673 sa_get_db(zp->z_sa_hdl), woff, abuf, tx); 674 if (error != 0) { 675 /* 676 * XXX This might not be necessary if 677 * dmu_assign_arcbuf_by_dbuf is guaranteed 678 * to be atomic. 679 */ 680 zfs_clear_setid_bits_if_necessary(zfsvfs, zp, 681 cr, &clear_setid_bits_txg, tx); 682 dmu_return_arcbuf(abuf); 683 dmu_tx_commit(tx); 684 break; 685 } 686 ASSERT3S(nbytes, <=, zfs_uio_resid(uio)); 687 zfs_uioskip(uio, nbytes); 688 tx_bytes = nbytes; 689 } 690 if (tx_bytes && 691 zn_has_cached_data(zp, woff, woff + tx_bytes - 1) && 692 !(ioflag & O_DIRECT)) { 693 update_pages(zp, woff, tx_bytes, zfsvfs->z_os); 694 } 695 696 /* 697 * If we made no progress, we're done. If we made even 698 * partial progress, update the znode and ZIL accordingly. 699 */ 700 if (tx_bytes == 0) { 701 (void) sa_update(zp->z_sa_hdl, SA_ZPL_SIZE(zfsvfs), 702 (void *)&zp->z_size, sizeof (uint64_t), tx); 703 dmu_tx_commit(tx); 704 ASSERT(error != 0); 705 break; 706 } 707 708 zfs_clear_setid_bits_if_necessary(zfsvfs, zp, cr, 709 &clear_setid_bits_txg, tx); 710 711 zfs_tstamp_update_setup(zp, CONTENT_MODIFIED, mtime, ctime); 712 713 /* 714 * Update the file size (zp_size) if it has changed; 715 * account for possible concurrent updates. 716 */ 717 while ((end_size = zp->z_size) < zfs_uio_offset(uio)) { 718 (void) atomic_cas_64(&zp->z_size, end_size, 719 zfs_uio_offset(uio)); 720 ASSERT(error == 0 || error == EFAULT); 721 } 722 /* 723 * If we are replaying and eof is non zero then force 724 * the file size to the specified eof. Note, there's no 725 * concurrency during replay. 726 */ 727 if (zfsvfs->z_replay && zfsvfs->z_replay_eof != 0) 728 zp->z_size = zfsvfs->z_replay_eof; 729 730 error1 = sa_bulk_update(zp->z_sa_hdl, bulk, count, tx); 731 if (error1 != 0) 732 /* Avoid clobbering EFAULT. */ 733 error = error1; 734 735 /* 736 * NB: During replay, the TX_SETATTR record logged by 737 * zfs_clear_setid_bits_if_necessary must precede any of 738 * the TX_WRITE records logged here. 739 */ 740 zfs_log_write(zilog, tx, TX_WRITE, zp, woff, tx_bytes, commit, 741 NULL, NULL); 742 743 dmu_tx_commit(tx); 744 745 if (error != 0) 746 break; 747 ASSERT3S(tx_bytes, ==, nbytes); 748 n -= nbytes; 749 pfbytes -= nbytes; 750 } 751 752 zfs_znode_update_vfs(zp); 753 zfs_rangelock_exit(lr); 754 755 /* 756 * If we're in replay mode, or we made no progress, or the 757 * uio data is inaccessible return an error. Otherwise, it's 758 * at least a partial write, so it's successful. 759 */ 760 if (zfsvfs->z_replay || zfs_uio_resid(uio) == start_resid || 761 error == EFAULT) { 762 zfs_exit(zfsvfs, FTAG); 763 return (error); 764 } 765 766 if (commit) 767 zil_commit(zilog, zp->z_id); 768 769 const int64_t nwritten = start_resid - zfs_uio_resid(uio); 770 dataset_kstats_update_write_kstats(&zfsvfs->z_kstat, nwritten); 771 task_io_account_write(nwritten); 772 773 zfs_exit(zfsvfs, FTAG); 774 return (0); 775 } 776 777 int 778 zfs_getsecattr(znode_t *zp, vsecattr_t *vsecp, int flag, cred_t *cr) 779 { 780 zfsvfs_t *zfsvfs = ZTOZSB(zp); 781 int error; 782 boolean_t skipaclchk = (flag & ATTR_NOACLCHECK) ? B_TRUE : B_FALSE; 783 784 if ((error = zfs_enter_verify_zp(zfsvfs, zp, FTAG)) != 0) 785 return (error); 786 error = zfs_getacl(zp, vsecp, skipaclchk, cr); 787 zfs_exit(zfsvfs, FTAG); 788 789 return (error); 790 } 791 792 int 793 zfs_setsecattr(znode_t *zp, vsecattr_t *vsecp, int flag, cred_t *cr) 794 { 795 zfsvfs_t *zfsvfs = ZTOZSB(zp); 796 int error; 797 boolean_t skipaclchk = (flag & ATTR_NOACLCHECK) ? B_TRUE : B_FALSE; 798 zilog_t *zilog; 799 800 if ((error = zfs_enter_verify_zp(zfsvfs, zp, FTAG)) != 0) 801 return (error); 802 zilog = zfsvfs->z_log; 803 error = zfs_setacl(zp, vsecp, skipaclchk, cr); 804 805 if (zfsvfs->z_os->os_sync == ZFS_SYNC_ALWAYS) 806 zil_commit(zilog, 0); 807 808 zfs_exit(zfsvfs, FTAG); 809 return (error); 810 } 811 812 #ifdef ZFS_DEBUG 813 static int zil_fault_io = 0; 814 #endif 815 816 static void zfs_get_done(zgd_t *zgd, int error); 817 818 /* 819 * Get data to generate a TX_WRITE intent log record. 820 */ 821 int 822 zfs_get_data(void *arg, uint64_t gen, lr_write_t *lr, char *buf, 823 struct lwb *lwb, zio_t *zio) 824 { 825 zfsvfs_t *zfsvfs = arg; 826 objset_t *os = zfsvfs->z_os; 827 znode_t *zp; 828 uint64_t object = lr->lr_foid; 829 uint64_t offset = lr->lr_offset; 830 uint64_t size = lr->lr_length; 831 dmu_buf_t *db; 832 zgd_t *zgd; 833 int error = 0; 834 uint64_t zp_gen; 835 836 ASSERT3P(lwb, !=, NULL); 837 ASSERT3U(size, !=, 0); 838 839 /* 840 * Nothing to do if the file has been removed 841 */ 842 if (zfs_zget(zfsvfs, object, &zp) != 0) 843 return (SET_ERROR(ENOENT)); 844 if (zp->z_unlinked) { 845 /* 846 * Release the vnode asynchronously as we currently have the 847 * txg stopped from syncing. 848 */ 849 zfs_zrele_async(zp); 850 return (SET_ERROR(ENOENT)); 851 } 852 /* check if generation number matches */ 853 if (sa_lookup(zp->z_sa_hdl, SA_ZPL_GEN(zfsvfs), &zp_gen, 854 sizeof (zp_gen)) != 0) { 855 zfs_zrele_async(zp); 856 return (SET_ERROR(EIO)); 857 } 858 if (zp_gen != gen) { 859 zfs_zrele_async(zp); 860 return (SET_ERROR(ENOENT)); 861 } 862 863 zgd = kmem_zalloc(sizeof (zgd_t), KM_SLEEP); 864 zgd->zgd_lwb = lwb; 865 zgd->zgd_private = zp; 866 867 /* 868 * Write records come in two flavors: immediate and indirect. 869 * For small writes it's cheaper to store the data with the 870 * log record (immediate); for large writes it's cheaper to 871 * sync the data and get a pointer to it (indirect) so that 872 * we don't have to write the data twice. 873 */ 874 if (buf != NULL) { /* immediate write */ 875 zgd->zgd_lr = zfs_rangelock_enter(&zp->z_rangelock, 876 offset, size, RL_READER); 877 /* test for truncation needs to be done while range locked */ 878 if (offset >= zp->z_size) { 879 error = SET_ERROR(ENOENT); 880 } else { 881 error = dmu_read(os, object, offset, size, buf, 882 DMU_READ_NO_PREFETCH); 883 } 884 ASSERT(error == 0 || error == ENOENT); 885 } else { /* indirect write */ 886 ASSERT3P(zio, !=, NULL); 887 /* 888 * Have to lock the whole block to ensure when it's 889 * written out and its checksum is being calculated 890 * that no one can change the data. We need to re-check 891 * blocksize after we get the lock in case it's changed! 892 */ 893 for (;;) { 894 uint64_t blkoff; 895 size = zp->z_blksz; 896 blkoff = ISP2(size) ? P2PHASE(offset, size) : offset; 897 offset -= blkoff; 898 zgd->zgd_lr = zfs_rangelock_enter(&zp->z_rangelock, 899 offset, size, RL_READER); 900 if (zp->z_blksz == size) 901 break; 902 offset += blkoff; 903 zfs_rangelock_exit(zgd->zgd_lr); 904 } 905 /* test for truncation needs to be done while range locked */ 906 if (lr->lr_offset >= zp->z_size) 907 error = SET_ERROR(ENOENT); 908 #ifdef ZFS_DEBUG 909 if (zil_fault_io) { 910 error = SET_ERROR(EIO); 911 zil_fault_io = 0; 912 } 913 #endif 914 if (error == 0) 915 error = dmu_buf_hold_noread(os, object, offset, zgd, 916 &db); 917 918 if (error == 0) { 919 blkptr_t *bp = &lr->lr_blkptr; 920 921 zgd->zgd_db = db; 922 zgd->zgd_bp = bp; 923 924 ASSERT(db->db_offset == offset); 925 ASSERT(db->db_size == size); 926 927 error = dmu_sync(zio, lr->lr_common.lrc_txg, 928 zfs_get_done, zgd); 929 ASSERT(error || lr->lr_length <= size); 930 931 /* 932 * On success, we need to wait for the write I/O 933 * initiated by dmu_sync() to complete before we can 934 * release this dbuf. We will finish everything up 935 * in the zfs_get_done() callback. 936 */ 937 if (error == 0) 938 return (0); 939 940 if (error == EALREADY) { 941 lr->lr_common.lrc_txtype = TX_WRITE2; 942 /* 943 * TX_WRITE2 relies on the data previously 944 * written by the TX_WRITE that caused 945 * EALREADY. We zero out the BP because 946 * it is the old, currently-on-disk BP. 947 */ 948 zgd->zgd_bp = NULL; 949 BP_ZERO(bp); 950 error = 0; 951 } 952 } 953 } 954 955 zfs_get_done(zgd, error); 956 957 return (error); 958 } 959 960 961 static void 962 zfs_get_done(zgd_t *zgd, int error) 963 { 964 (void) error; 965 znode_t *zp = zgd->zgd_private; 966 967 if (zgd->zgd_db) 968 dmu_buf_rele(zgd->zgd_db, zgd); 969 970 zfs_rangelock_exit(zgd->zgd_lr); 971 972 /* 973 * Release the vnode asynchronously as we currently have the 974 * txg stopped from syncing. 975 */ 976 zfs_zrele_async(zp); 977 978 kmem_free(zgd, sizeof (zgd_t)); 979 } 980 981 static int 982 zfs_enter_two(zfsvfs_t *zfsvfs1, zfsvfs_t *zfsvfs2, const char *tag) 983 { 984 int error; 985 986 /* Swap. Not sure if the order of zfs_enter()s is important. */ 987 if (zfsvfs1 > zfsvfs2) { 988 zfsvfs_t *tmpzfsvfs; 989 990 tmpzfsvfs = zfsvfs2; 991 zfsvfs2 = zfsvfs1; 992 zfsvfs1 = tmpzfsvfs; 993 } 994 995 error = zfs_enter(zfsvfs1, tag); 996 if (error != 0) 997 return (error); 998 if (zfsvfs1 != zfsvfs2) { 999 error = zfs_enter(zfsvfs2, tag); 1000 if (error != 0) { 1001 zfs_exit(zfsvfs1, tag); 1002 return (error); 1003 } 1004 } 1005 1006 return (0); 1007 } 1008 1009 static void 1010 zfs_exit_two(zfsvfs_t *zfsvfs1, zfsvfs_t *zfsvfs2, const char *tag) 1011 { 1012 1013 zfs_exit(zfsvfs1, tag); 1014 if (zfsvfs1 != zfsvfs2) 1015 zfs_exit(zfsvfs2, tag); 1016 } 1017 1018 /* 1019 * We split each clone request in chunks that can fit into a single ZIL 1020 * log entry. Each ZIL log entry can fit 130816 bytes for a block cloning 1021 * operation (see zil_max_log_data() and zfs_log_clone_range()). This gives 1022 * us room for storing 1022 block pointers. 1023 * 1024 * On success, the function return the number of bytes copied in *lenp. 1025 * Note, it doesn't return how much bytes are left to be copied. 1026 * On errors which are caused by any file system limitations or 1027 * brt limitations `EINVAL` is returned. In the most cases a user 1028 * requested bad parameters, it could be possible to clone the file but 1029 * some parameters don't match the requirements. 1030 */ 1031 int 1032 zfs_clone_range(znode_t *inzp, uint64_t *inoffp, znode_t *outzp, 1033 uint64_t *outoffp, uint64_t *lenp, cred_t *cr) 1034 { 1035 zfsvfs_t *inzfsvfs, *outzfsvfs; 1036 objset_t *inos, *outos; 1037 zfs_locked_range_t *inlr, *outlr; 1038 dmu_buf_impl_t *db; 1039 dmu_tx_t *tx; 1040 zilog_t *zilog; 1041 uint64_t inoff, outoff, len, done; 1042 uint64_t outsize, size; 1043 int error; 1044 int count = 0; 1045 sa_bulk_attr_t bulk[3]; 1046 uint64_t mtime[2], ctime[2]; 1047 uint64_t uid, gid, projid; 1048 blkptr_t *bps; 1049 size_t maxblocks, nbps; 1050 uint_t inblksz; 1051 uint64_t clear_setid_bits_txg = 0; 1052 1053 inoff = *inoffp; 1054 outoff = *outoffp; 1055 len = *lenp; 1056 done = 0; 1057 1058 inzfsvfs = ZTOZSB(inzp); 1059 outzfsvfs = ZTOZSB(outzp); 1060 1061 /* 1062 * We need to call zfs_enter() potentially on two different datasets, 1063 * so we need a dedicated function for that. 1064 */ 1065 error = zfs_enter_two(inzfsvfs, outzfsvfs, FTAG); 1066 if (error != 0) 1067 return (error); 1068 1069 inos = inzfsvfs->z_os; 1070 outos = outzfsvfs->z_os; 1071 1072 /* 1073 * Both source and destination have to belong to the same storage pool. 1074 */ 1075 if (dmu_objset_spa(inos) != dmu_objset_spa(outos)) { 1076 zfs_exit_two(inzfsvfs, outzfsvfs, FTAG); 1077 return (SET_ERROR(EXDEV)); 1078 } 1079 1080 /* 1081 * outos and inos belongs to the same storage pool. 1082 * see a few lines above, only one check. 1083 */ 1084 if (!spa_feature_is_enabled(dmu_objset_spa(outos), 1085 SPA_FEATURE_BLOCK_CLONING)) { 1086 zfs_exit_two(inzfsvfs, outzfsvfs, FTAG); 1087 return (SET_ERROR(EOPNOTSUPP)); 1088 } 1089 1090 ASSERT(!outzfsvfs->z_replay); 1091 1092 /* 1093 * Block cloning from an unencrypted dataset into an encrypted 1094 * dataset and vice versa is not supported. 1095 */ 1096 if (inos->os_encrypted != outos->os_encrypted) { 1097 zfs_exit_two(inzfsvfs, outzfsvfs, FTAG); 1098 return (SET_ERROR(EXDEV)); 1099 } 1100 1101 /* 1102 * Cloning across encrypted datasets is possible only if they 1103 * share the same master key. 1104 */ 1105 if (inos != outos && inos->os_encrypted && 1106 !dmu_objset_crypto_key_equal(inos, outos)) { 1107 zfs_exit_two(inzfsvfs, outzfsvfs, FTAG); 1108 return (SET_ERROR(EXDEV)); 1109 } 1110 1111 error = zfs_verify_zp(inzp); 1112 if (error == 0) 1113 error = zfs_verify_zp(outzp); 1114 if (error != 0) { 1115 zfs_exit_two(inzfsvfs, outzfsvfs, FTAG); 1116 return (error); 1117 } 1118 1119 /* 1120 * We don't copy source file's flags that's why we don't allow to clone 1121 * files that are in quarantine. 1122 */ 1123 if (inzp->z_pflags & ZFS_AV_QUARANTINED) { 1124 zfs_exit_two(inzfsvfs, outzfsvfs, FTAG); 1125 return (SET_ERROR(EACCES)); 1126 } 1127 1128 if (inoff >= inzp->z_size) { 1129 *lenp = 0; 1130 zfs_exit_two(inzfsvfs, outzfsvfs, FTAG); 1131 return (0); 1132 } 1133 if (len > inzp->z_size - inoff) { 1134 len = inzp->z_size - inoff; 1135 } 1136 if (len == 0) { 1137 *lenp = 0; 1138 zfs_exit_two(inzfsvfs, outzfsvfs, FTAG); 1139 return (0); 1140 } 1141 1142 /* 1143 * Callers might not be able to detect properly that we are read-only, 1144 * so check it explicitly here. 1145 */ 1146 if (zfs_is_readonly(outzfsvfs)) { 1147 zfs_exit_two(inzfsvfs, outzfsvfs, FTAG); 1148 return (SET_ERROR(EROFS)); 1149 } 1150 1151 /* 1152 * If immutable or not appending then return EPERM. 1153 * Intentionally allow ZFS_READONLY through here. 1154 * See zfs_zaccess_common() 1155 */ 1156 if ((outzp->z_pflags & ZFS_IMMUTABLE) != 0) { 1157 zfs_exit_two(inzfsvfs, outzfsvfs, FTAG); 1158 return (SET_ERROR(EPERM)); 1159 } 1160 1161 /* 1162 * No overlapping if we are cloning within the same file. 1163 */ 1164 if (inzp == outzp) { 1165 if (inoff < outoff + len && outoff < inoff + len) { 1166 zfs_exit_two(inzfsvfs, outzfsvfs, FTAG); 1167 return (SET_ERROR(EINVAL)); 1168 } 1169 } 1170 1171 /* 1172 * Maintain predictable lock order. 1173 */ 1174 if (inzp < outzp || (inzp == outzp && inoff < outoff)) { 1175 inlr = zfs_rangelock_enter(&inzp->z_rangelock, inoff, len, 1176 RL_READER); 1177 outlr = zfs_rangelock_enter(&outzp->z_rangelock, outoff, len, 1178 RL_WRITER); 1179 } else { 1180 outlr = zfs_rangelock_enter(&outzp->z_rangelock, outoff, len, 1181 RL_WRITER); 1182 inlr = zfs_rangelock_enter(&inzp->z_rangelock, inoff, len, 1183 RL_READER); 1184 } 1185 1186 inblksz = inzp->z_blksz; 1187 1188 /* 1189 * We cannot clone into a file with different block size if we can't 1190 * grow it (block size is already bigger, has more than one block, or 1191 * not locked for growth). There are other possible reasons for the 1192 * grow to fail, but we cover what we can before opening transaction 1193 * and the rest detect after we try to do it. 1194 */ 1195 if (inblksz < outzp->z_blksz) { 1196 error = SET_ERROR(EINVAL); 1197 goto unlock; 1198 } 1199 if (inblksz != outzp->z_blksz && (outzp->z_size > outzp->z_blksz || 1200 outlr->lr_length != UINT64_MAX)) { 1201 error = SET_ERROR(EINVAL); 1202 goto unlock; 1203 } 1204 1205 /* 1206 * Block size must be power-of-2 if destination offset != 0. 1207 * There can be no multiple blocks of non-power-of-2 size. 1208 */ 1209 if (outoff != 0 && !ISP2(inblksz)) { 1210 error = SET_ERROR(EINVAL); 1211 goto unlock; 1212 } 1213 1214 /* 1215 * Offsets and len must be at block boundries. 1216 */ 1217 if ((inoff % inblksz) != 0 || (outoff % inblksz) != 0) { 1218 error = SET_ERROR(EINVAL); 1219 goto unlock; 1220 } 1221 /* 1222 * Length must be multipe of blksz, except for the end of the file. 1223 */ 1224 if ((len % inblksz) != 0 && 1225 (len < inzp->z_size - inoff || len < outzp->z_size - outoff)) { 1226 error = SET_ERROR(EINVAL); 1227 goto unlock; 1228 } 1229 1230 /* 1231 * If we are copying only one block and it is smaller than recordsize 1232 * property, do not allow destination to grow beyond one block if it 1233 * is not there yet. Otherwise the destination will get stuck with 1234 * that block size forever, that can be as small as 512 bytes, no 1235 * matter how big the destination grow later. 1236 */ 1237 if (len <= inblksz && inblksz < outzfsvfs->z_max_blksz && 1238 outzp->z_size <= inblksz && outoff + len > inblksz) { 1239 error = SET_ERROR(EINVAL); 1240 goto unlock; 1241 } 1242 1243 error = zn_rlimit_fsize(outoff + len); 1244 if (error != 0) { 1245 goto unlock; 1246 } 1247 1248 if (inoff >= MAXOFFSET_T || outoff >= MAXOFFSET_T) { 1249 error = SET_ERROR(EFBIG); 1250 goto unlock; 1251 } 1252 1253 SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_MTIME(outzfsvfs), NULL, 1254 &mtime, 16); 1255 SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_CTIME(outzfsvfs), NULL, 1256 &ctime, 16); 1257 SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_SIZE(outzfsvfs), NULL, 1258 &outzp->z_size, 8); 1259 1260 zilog = outzfsvfs->z_log; 1261 maxblocks = zil_max_log_data(zilog, sizeof (lr_clone_range_t)) / 1262 sizeof (bps[0]); 1263 1264 uid = KUID_TO_SUID(ZTOUID(outzp)); 1265 gid = KGID_TO_SGID(ZTOGID(outzp)); 1266 projid = outzp->z_projid; 1267 1268 bps = vmem_alloc(sizeof (bps[0]) * maxblocks, KM_SLEEP); 1269 1270 /* 1271 * Clone the file in reasonable size chunks. Each chunk is cloned 1272 * in a separate transaction; this keeps the intent log records small 1273 * and allows us to do more fine-grained space accounting. 1274 */ 1275 while (len > 0) { 1276 size = MIN(inblksz * maxblocks, len); 1277 1278 if (zfs_id_overblockquota(outzfsvfs, DMU_USERUSED_OBJECT, 1279 uid) || 1280 zfs_id_overblockquota(outzfsvfs, DMU_GROUPUSED_OBJECT, 1281 gid) || 1282 (projid != ZFS_DEFAULT_PROJID && 1283 zfs_id_overblockquota(outzfsvfs, DMU_PROJECTUSED_OBJECT, 1284 projid))) { 1285 error = SET_ERROR(EDQUOT); 1286 break; 1287 } 1288 1289 nbps = maxblocks; 1290 error = dmu_read_l0_bps(inos, inzp->z_id, inoff, size, bps, 1291 &nbps); 1292 if (error != 0) { 1293 /* 1294 * If we are trying to clone a block that was created 1295 * in the current transaction group, error will be 1296 * EAGAIN here, which we can just return to the caller 1297 * so it can fallback if it likes. 1298 */ 1299 break; 1300 } 1301 1302 /* 1303 * Start a transaction. 1304 */ 1305 tx = dmu_tx_create(outos); 1306 dmu_tx_hold_sa(tx, outzp->z_sa_hdl, B_FALSE); 1307 db = (dmu_buf_impl_t *)sa_get_db(outzp->z_sa_hdl); 1308 DB_DNODE_ENTER(db); 1309 dmu_tx_hold_clone_by_dnode(tx, DB_DNODE(db), outoff, size); 1310 DB_DNODE_EXIT(db); 1311 zfs_sa_upgrade_txholds(tx, outzp); 1312 error = dmu_tx_assign(tx, TXG_WAIT); 1313 if (error != 0) { 1314 dmu_tx_abort(tx); 1315 break; 1316 } 1317 1318 /* 1319 * Copy source znode's block size. This is done only if the 1320 * whole znode is locked (see zfs_rangelock_cb()) and only 1321 * on the first iteration since zfs_rangelock_reduce() will 1322 * shrink down lr_length to the appropriate size. 1323 */ 1324 if (outlr->lr_length == UINT64_MAX) { 1325 zfs_grow_blocksize(outzp, inblksz, tx); 1326 1327 /* 1328 * Block growth may fail for many reasons we can not 1329 * predict here. If it happen the cloning is doomed. 1330 */ 1331 if (inblksz != outzp->z_blksz) { 1332 error = SET_ERROR(EINVAL); 1333 dmu_tx_abort(tx); 1334 break; 1335 } 1336 1337 /* 1338 * Round range lock up to the block boundary, so we 1339 * prevent appends until we are done. 1340 */ 1341 zfs_rangelock_reduce(outlr, outoff, 1342 ((len - 1) / inblksz + 1) * inblksz); 1343 } 1344 1345 error = dmu_brt_clone(outos, outzp->z_id, outoff, size, tx, 1346 bps, nbps); 1347 if (error != 0) { 1348 dmu_tx_commit(tx); 1349 break; 1350 } 1351 1352 if (zn_has_cached_data(outzp, outoff, outoff + size - 1)) { 1353 update_pages(outzp, outoff, size, outos); 1354 } 1355 1356 zfs_clear_setid_bits_if_necessary(outzfsvfs, outzp, cr, 1357 &clear_setid_bits_txg, tx); 1358 1359 zfs_tstamp_update_setup(outzp, CONTENT_MODIFIED, mtime, ctime); 1360 1361 /* 1362 * Update the file size (zp_size) if it has changed; 1363 * account for possible concurrent updates. 1364 */ 1365 while ((outsize = outzp->z_size) < outoff + size) { 1366 (void) atomic_cas_64(&outzp->z_size, outsize, 1367 outoff + size); 1368 } 1369 1370 error = sa_bulk_update(outzp->z_sa_hdl, bulk, count, tx); 1371 1372 zfs_log_clone_range(zilog, tx, TX_CLONE_RANGE, outzp, outoff, 1373 size, inblksz, bps, nbps); 1374 1375 dmu_tx_commit(tx); 1376 1377 if (error != 0) 1378 break; 1379 1380 inoff += size; 1381 outoff += size; 1382 len -= size; 1383 done += size; 1384 } 1385 1386 vmem_free(bps, sizeof (bps[0]) * maxblocks); 1387 zfs_znode_update_vfs(outzp); 1388 1389 unlock: 1390 zfs_rangelock_exit(outlr); 1391 zfs_rangelock_exit(inlr); 1392 1393 if (done > 0) { 1394 /* 1395 * If we have made at least partial progress, reset the error. 1396 */ 1397 error = 0; 1398 1399 ZFS_ACCESSTIME_STAMP(inzfsvfs, inzp); 1400 1401 if (outos->os_sync == ZFS_SYNC_ALWAYS) { 1402 zil_commit(zilog, outzp->z_id); 1403 } 1404 1405 *inoffp += done; 1406 *outoffp += done; 1407 *lenp = done; 1408 } else { 1409 /* 1410 * If we made no progress, there must be a good reason. 1411 * EOF is handled explicitly above, before the loop. 1412 */ 1413 ASSERT3S(error, !=, 0); 1414 } 1415 1416 zfs_exit_two(inzfsvfs, outzfsvfs, FTAG); 1417 1418 return (error); 1419 } 1420 1421 /* 1422 * Usual pattern would be to call zfs_clone_range() from zfs_replay_clone(), 1423 * but we cannot do that, because when replaying we don't have source znode 1424 * available. This is why we need a dedicated replay function. 1425 */ 1426 int 1427 zfs_clone_range_replay(znode_t *zp, uint64_t off, uint64_t len, uint64_t blksz, 1428 const blkptr_t *bps, size_t nbps) 1429 { 1430 zfsvfs_t *zfsvfs; 1431 dmu_buf_impl_t *db; 1432 dmu_tx_t *tx; 1433 int error; 1434 int count = 0; 1435 sa_bulk_attr_t bulk[3]; 1436 uint64_t mtime[2], ctime[2]; 1437 1438 ASSERT3U(off, <, MAXOFFSET_T); 1439 ASSERT3U(len, >, 0); 1440 ASSERT3U(nbps, >, 0); 1441 1442 zfsvfs = ZTOZSB(zp); 1443 1444 ASSERT(spa_feature_is_enabled(dmu_objset_spa(zfsvfs->z_os), 1445 SPA_FEATURE_BLOCK_CLONING)); 1446 1447 if ((error = zfs_enter_verify_zp(zfsvfs, zp, FTAG)) != 0) 1448 return (error); 1449 1450 ASSERT(zfsvfs->z_replay); 1451 ASSERT(!zfs_is_readonly(zfsvfs)); 1452 1453 if ((off % blksz) != 0) { 1454 zfs_exit(zfsvfs, FTAG); 1455 return (SET_ERROR(EINVAL)); 1456 } 1457 1458 SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_MTIME(zfsvfs), NULL, &mtime, 16); 1459 SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_CTIME(zfsvfs), NULL, &ctime, 16); 1460 SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_SIZE(zfsvfs), NULL, 1461 &zp->z_size, 8); 1462 1463 /* 1464 * Start a transaction. 1465 */ 1466 tx = dmu_tx_create(zfsvfs->z_os); 1467 1468 dmu_tx_hold_sa(tx, zp->z_sa_hdl, B_FALSE); 1469 db = (dmu_buf_impl_t *)sa_get_db(zp->z_sa_hdl); 1470 DB_DNODE_ENTER(db); 1471 dmu_tx_hold_clone_by_dnode(tx, DB_DNODE(db), off, len); 1472 DB_DNODE_EXIT(db); 1473 zfs_sa_upgrade_txholds(tx, zp); 1474 error = dmu_tx_assign(tx, TXG_WAIT); 1475 if (error != 0) { 1476 dmu_tx_abort(tx); 1477 zfs_exit(zfsvfs, FTAG); 1478 return (error); 1479 } 1480 1481 if (zp->z_blksz < blksz) 1482 zfs_grow_blocksize(zp, blksz, tx); 1483 1484 dmu_brt_clone(zfsvfs->z_os, zp->z_id, off, len, tx, bps, nbps); 1485 1486 zfs_tstamp_update_setup(zp, CONTENT_MODIFIED, mtime, ctime); 1487 1488 if (zp->z_size < off + len) 1489 zp->z_size = off + len; 1490 1491 error = sa_bulk_update(zp->z_sa_hdl, bulk, count, tx); 1492 1493 /* 1494 * zil_replaying() not only check if we are replaying ZIL, but also 1495 * updates the ZIL header to record replay progress. 1496 */ 1497 VERIFY(zil_replaying(zfsvfs->z_log, tx)); 1498 1499 dmu_tx_commit(tx); 1500 1501 zfs_znode_update_vfs(zp); 1502 1503 zfs_exit(zfsvfs, FTAG); 1504 1505 return (error); 1506 } 1507 1508 EXPORT_SYMBOL(zfs_access); 1509 EXPORT_SYMBOL(zfs_fsync); 1510 EXPORT_SYMBOL(zfs_holey); 1511 EXPORT_SYMBOL(zfs_read); 1512 EXPORT_SYMBOL(zfs_write); 1513 EXPORT_SYMBOL(zfs_getsecattr); 1514 EXPORT_SYMBOL(zfs_setsecattr); 1515 EXPORT_SYMBOL(zfs_clone_range); 1516 EXPORT_SYMBOL(zfs_clone_range_replay); 1517 1518 ZFS_MODULE_PARAM(zfs_vnops, zfs_vnops_, read_chunk_size, U64, ZMOD_RW, 1519 "Bytes to read per chunk"); 1520