1 // SPDX-License-Identifier: CDDL-1.0
2 /*
3 * CDDL HEADER START
4 *
5 * The contents of this file are subject to the terms of the
6 * Common Development and Distribution License (the "License").
7 * You may not use this file except in compliance with the License.
8 *
9 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
10 * or https://opensource.org/licenses/CDDL-1.0.
11 * See the License for the specific language governing permissions
12 * and limitations under the License.
13 *
14 * When distributing Covered Code, include this CDDL HEADER in each
15 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
16 * If applicable, add the following below this CDDL HEADER, with the
17 * fields enclosed by brackets "[]" replaced with your own identifying
18 * information: Portions Copyright [yyyy] [name of copyright owner]
19 *
20 * CDDL HEADER END
21 */
22
23 /*
24 * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
25 * Copyright (c) 2012, 2018 by Delphix. All rights reserved.
26 * Copyright (c) 2015 by Chunwei Chen. All rights reserved.
27 * Copyright 2017 Nexenta Systems, Inc.
28 * Copyright (c) 2021, 2022 by Pawel Jakub Dawidek
29 * Copyright (c) 2025, Rob Norris <robn@despairlabs.com>
30 * Copyright (c) 2025, Klara, Inc.
31 */
32
33 /* Portions Copyright 2007 Jeremy Teo */
34 /* Portions Copyright 2010 Robert Milkowski */
35
36 #include <sys/types.h>
37 #include <sys/param.h>
38 #include <sys/time.h>
39 #include <sys/sysmacros.h>
40 #include <sys/vfs.h>
41 #include <sys/file.h>
42 #include <sys/stat.h>
43 #include <sys/kmem.h>
44 #include <sys/cmn_err.h>
45 #include <sys/errno.h>
46 #include <sys/zfs_dir.h>
47 #include <sys/zfs_acl.h>
48 #include <sys/zfs_ioctl.h>
49 #include <sys/fs/zfs.h>
50 #include <sys/dmu.h>
51 #include <sys/dmu_objset.h>
52 #include <sys/dsl_crypt.h>
53 #include <sys/dsl_dataset.h>
54 #include <sys/spa.h>
55 #include <sys/txg.h>
56 #include <sys/brt.h>
57 #include <sys/dbuf.h>
58 #include <sys/policy.h>
59 #include <sys/zfeature.h>
60 #include <sys/zfs_vnops.h>
61 #include <sys/zfs_quota.h>
62 #include <sys/zfs_vfsops.h>
63 #include <sys/zfs_znode.h>
64
65 /*
66 * Enables access to the block cloning feature. If this setting is 0, then even
67 * if feature@block_cloning is enabled, using functions and system calls that
68 * attempt to clone blocks will act as though the feature is disabled.
69 */
70 int zfs_bclone_enabled = 1;
71
72 /*
73 * Restricts block cloning between datasets with different properties
74 * (checksum, compression, copies, dedup, or special_small_blocks).
75 */
76 int zfs_bclone_strict_properties = 1;
77
78 /*
79 * When set to 1 the FICLONE and FICLONERANGE ioctls will wait for any dirty
80 * data to be written to disk before proceeding. This ensures that the clone
81 * operation reliably succeeds, even if a file is modified and then immediately
82 * cloned. Note that for small files this may be slower than simply copying
83 * the file. When set to 0 the clone operation will immediately fail if it
84 * encounters any dirty blocks. By default waiting is enabled.
85 */
86 int zfs_bclone_wait_dirty = 1;
87
88 /*
89 * Enable Direct I/O. If this setting is 0, then all I/O requests will be
90 * directed through the ARC acting as though the dataset property direct was
91 * set to disabled.
92 *
93 * Disabled by default on FreeBSD until a potential range locking issue in
94 * zfs_getpages() can be resolved.
95 */
96 #ifdef __FreeBSD__
97 static int zfs_dio_enabled = 0;
98 #else
99 static int zfs_dio_enabled = 1;
100 #endif
101
102 /*
103 * Strictly enforce alignment for Direct I/O requests, returning EINVAL
104 * if not page-aligned instead of silently falling back to uncached I/O.
105 */
106 static int zfs_dio_strict = 0;
107
108
109 /*
110 * Maximum bytes to read per chunk in zfs_read().
111 */
112 #ifdef _ILP32
113 static uint64_t zfs_vnops_read_chunk_size = 1024 * 1024;
114 #else
115 static uint64_t zfs_vnops_read_chunk_size = DMU_MAX_ACCESS / 2;
116 #endif
117
118 int
zfs_fsync(znode_t * zp,int syncflag,cred_t * cr)119 zfs_fsync(znode_t *zp, int syncflag, cred_t *cr)
120 {
121 int error = 0;
122 zfsvfs_t *zfsvfs = ZTOZSB(zp);
123
124 if (zfsvfs->z_os->os_sync != ZFS_SYNC_DISABLED) {
125 if ((error = zfs_enter_verify_zp(zfsvfs, zp, FTAG)) != 0)
126 return (error);
127 error = zil_commit(zfsvfs->z_log, zp->z_id);
128 zfs_exit(zfsvfs, FTAG);
129 }
130 return (error);
131 }
132
133
134 #if defined(SEEK_HOLE) && defined(SEEK_DATA)
135 /*
136 * Lseek support for finding holes (cmd == SEEK_HOLE) and
137 * data (cmd == SEEK_DATA). "off" is an in/out parameter.
138 */
139 static int
zfs_holey_common(znode_t * zp,ulong_t cmd,loff_t * off)140 zfs_holey_common(znode_t *zp, ulong_t cmd, loff_t *off)
141 {
142 zfs_locked_range_t *lr;
143 uint64_t noff = (uint64_t)*off; /* new offset */
144 uint64_t file_sz;
145 int error;
146 boolean_t hole;
147
148 file_sz = zp->z_size;
149 if (noff >= file_sz) {
150 return (SET_ERROR(ENXIO));
151 }
152
153 if (cmd == F_SEEK_HOLE)
154 hole = B_TRUE;
155 else
156 hole = B_FALSE;
157
158 /* Flush any mmap()'d data to disk */
159 if (zn_has_cached_data(zp, 0, file_sz - 1))
160 zn_flush_cached_data(zp, B_TRUE);
161
162 lr = zfs_rangelock_enter(&zp->z_rangelock, 0, UINT64_MAX, RL_READER);
163 error = dmu_offset_next(ZTOZSB(zp)->z_os, zp->z_id, hole, &noff);
164 zfs_rangelock_exit(lr);
165
166 if (error == ESRCH)
167 return (SET_ERROR(ENXIO));
168
169 /* File was dirty, so fall back to using generic logic */
170 if (error == EBUSY) {
171 if (hole)
172 *off = file_sz;
173
174 return (0);
175 }
176
177 /*
178 * We could find a hole that begins after the logical end-of-file,
179 * because dmu_offset_next() only works on whole blocks. If the
180 * EOF falls mid-block, then indicate that the "virtual hole"
181 * at the end of the file begins at the logical EOF, rather than
182 * at the end of the last block.
183 */
184 if (noff > file_sz) {
185 ASSERT(hole);
186 noff = file_sz;
187 }
188
189 if (noff < *off)
190 return (error);
191 *off = noff;
192 return (error);
193 }
194
195 int
zfs_holey(znode_t * zp,ulong_t cmd,loff_t * off)196 zfs_holey(znode_t *zp, ulong_t cmd, loff_t *off)
197 {
198 zfsvfs_t *zfsvfs = ZTOZSB(zp);
199 int error;
200
201 if ((error = zfs_enter_verify_zp(zfsvfs, zp, FTAG)) != 0)
202 return (error);
203
204 error = zfs_holey_common(zp, cmd, off);
205
206 zfs_exit(zfsvfs, FTAG);
207 return (error);
208 }
209 #endif /* SEEK_HOLE && SEEK_DATA */
210
211 int
zfs_access(znode_t * zp,int mode,int flag,cred_t * cr)212 zfs_access(znode_t *zp, int mode, int flag, cred_t *cr)
213 {
214 zfsvfs_t *zfsvfs = ZTOZSB(zp);
215 int error;
216
217 if ((error = zfs_enter_verify_zp(zfsvfs, zp, FTAG)) != 0)
218 return (error);
219
220 if (flag & V_ACE_MASK)
221 #if defined(__linux__)
222 error = zfs_zaccess(zp, mode, flag, B_FALSE, cr,
223 zfs_init_idmap);
224 #else
225 error = zfs_zaccess(zp, mode, flag, B_FALSE, cr,
226 NULL);
227 #endif
228 else
229 #if defined(__linux__)
230 error = zfs_zaccess_rwx(zp, mode, flag, cr, zfs_init_idmap);
231 #else
232 error = zfs_zaccess_rwx(zp, mode, flag, cr, NULL);
233 #endif
234
235 zfs_exit(zfsvfs, FTAG);
236 return (error);
237 }
238
239 /*
240 * Determine if Direct I/O has been requested (either via the O_DIRECT flag or
241 * the "direct" dataset property). When inherited by the property only apply
242 * the O_DIRECT flag to correctly aligned IO requests. The rational for this
243 * is it allows the property to be safely set on a dataset without forcing
244 * all of the applications to be aware of the alignment restrictions. When
245 * O_DIRECT is explicitly requested by an application return EINVAL if the
246 * request is unaligned. In all cases, if the range for this request has
247 * been mmap'ed then we will perform buffered I/O to keep the mapped region
248 * synhronized with the ARC.
249 *
250 * It is possible that a file's pages could be mmap'ed after it is checked
251 * here. If so, that is handled coorarding in zfs_write(). See comments in the
252 * following area for how this is handled:
253 * zfs_write() -> update_pages()
254 */
255 static int
zfs_setup_direct(struct znode * zp,zfs_uio_t * uio,zfs_uio_rw_t rw,int * ioflagp)256 zfs_setup_direct(struct znode *zp, zfs_uio_t *uio, zfs_uio_rw_t rw,
257 int *ioflagp)
258 {
259 zfsvfs_t *zfsvfs = ZTOZSB(zp);
260 objset_t *os = zfsvfs->z_os;
261 int ioflag = *ioflagp;
262 int error = 0;
263
264 if (os->os_direct == ZFS_DIRECT_ALWAYS) {
265 /* Force either direct or uncached I/O. */
266 ioflag |= O_DIRECT;
267 }
268
269 if ((ioflag & O_DIRECT) == 0)
270 goto out;
271
272 if (!zfs_dio_enabled || os->os_direct == ZFS_DIRECT_DISABLED) {
273 /*
274 * Direct I/O is disabled. The I/O request will be directed
275 * through the ARC as uncached I/O.
276 */
277 goto out;
278 }
279
280 if (!zfs_uio_page_aligned(uio) ||
281 !zfs_uio_aligned(uio, PAGE_SIZE)) {
282 /*
283 * Misaligned requests can be executed through the ARC as
284 * uncached I/O. But if O_DIRECT was set by user and we
285 * were set to be strict, then it is a failure.
286 */
287 if ((*ioflagp & O_DIRECT) && zfs_dio_strict)
288 error = SET_ERROR(EINVAL);
289 goto out;
290 }
291
292 if (zn_has_cached_data(zp, zfs_uio_offset(uio),
293 zfs_uio_offset(uio) + zfs_uio_resid(uio) - 1)) {
294 /*
295 * The region is mmap'ed. The I/O request will be directed
296 * through the ARC as uncached I/O.
297 */
298 goto out;
299 }
300
301 /*
302 * For short writes the page mapping of Direct I/O makes no sense.
303 * Direct them through the ARC as uncached I/O.
304 */
305 if (rw == UIO_WRITE && zfs_uio_resid(uio) < zp->z_blksz)
306 goto out;
307
308 error = zfs_uio_get_dio_pages_alloc(uio, rw);
309 if (error)
310 goto out;
311 ASSERT(uio->uio_extflg & UIO_DIRECT);
312
313 out:
314 *ioflagp = ioflag;
315 return (error);
316 }
317
318 /*
319 * Read bytes from specified file into supplied buffer.
320 *
321 * IN: zp - inode of file to be read from.
322 * uio - structure supplying read location, range info,
323 * and return buffer.
324 * ioflag - O_SYNC flags; used to provide FRSYNC semantics.
325 * O_DIRECT flag; used to bypass page cache.
326 * cr - credentials of caller.
327 *
328 * OUT: uio - updated offset and range, buffer filled.
329 *
330 * RETURN: 0 on success, error code on failure.
331 *
332 * Side Effects:
333 * inode - atime updated if byte count > 0
334 */
335 int
zfs_read(struct znode * zp,zfs_uio_t * uio,int ioflag,cred_t * cr)336 zfs_read(struct znode *zp, zfs_uio_t *uio, int ioflag, cred_t *cr)
337 {
338 (void) cr;
339 int error = 0;
340 boolean_t frsync = B_FALSE;
341 boolean_t dio_checksum_failure = B_FALSE;
342
343 zfsvfs_t *zfsvfs = ZTOZSB(zp);
344 if ((error = zfs_enter_verify_zp(zfsvfs, zp, FTAG)) != 0)
345 return (error);
346
347 if (zp->z_pflags & ZFS_AV_QUARANTINED) {
348 zfs_exit(zfsvfs, FTAG);
349 return (SET_ERROR(EACCES));
350 }
351
352 /* We don't copy out anything useful for directories. */
353 if (Z_ISDIR(ZTOTYPE(zp))) {
354 zfs_exit(zfsvfs, FTAG);
355 return (SET_ERROR(EISDIR));
356 }
357
358 /*
359 * Validate file offset
360 */
361 if (zfs_uio_offset(uio) < (offset_t)0) {
362 zfs_exit(zfsvfs, FTAG);
363 return (SET_ERROR(EINVAL));
364 }
365
366 /*
367 * Fasttrack empty reads
368 */
369 if (zfs_uio_resid(uio) == 0) {
370 zfs_exit(zfsvfs, FTAG);
371 return (0);
372 }
373
374 #ifdef FRSYNC
375 /*
376 * If we're in FRSYNC mode, sync out this znode before reading it.
377 * Only do this for non-snapshots.
378 *
379 * Some platforms do not support FRSYNC and instead map it
380 * to O_SYNC, which results in unnecessary calls to zil_commit. We
381 * only honor FRSYNC requests on platforms which support it.
382 */
383 frsync = !!(ioflag & FRSYNC);
384 #endif
385 if (zfsvfs->z_log &&
386 (frsync || zfsvfs->z_os->os_sync == ZFS_SYNC_ALWAYS)) {
387 error = zil_commit(zfsvfs->z_log, zp->z_id);
388 if (error != 0) {
389 zfs_exit(zfsvfs, FTAG);
390 return (error);
391 }
392 }
393
394 /*
395 * Lock the range against changes.
396 */
397 zfs_locked_range_t *lr = zfs_rangelock_enter(&zp->z_rangelock,
398 zfs_uio_offset(uio), zfs_uio_resid(uio), RL_READER);
399
400 /*
401 * If we are reading past end-of-file we can skip
402 * to the end; but we might still need to set atime.
403 */
404 if (zfs_uio_offset(uio) >= zp->z_size) {
405 error = 0;
406 goto out;
407 }
408 ASSERT(zfs_uio_offset(uio) < zp->z_size);
409
410 /*
411 * Setting up Direct I/O if requested.
412 */
413 error = zfs_setup_direct(zp, uio, UIO_READ, &ioflag);
414 if (error) {
415 goto out;
416 }
417
418 #if defined(__linux__)
419 ssize_t start_offset = zfs_uio_offset(uio);
420 #endif
421 uint_t blksz = zp->z_blksz;
422 ssize_t chunk_size;
423 ssize_t n = MIN(zfs_uio_resid(uio), zp->z_size - zfs_uio_offset(uio));
424 ssize_t start_resid = n;
425 ssize_t dio_remaining_resid = 0;
426
427 dmu_flags_t dflags = DMU_READ_PREFETCH;
428 if (ioflag & O_DIRECT)
429 dflags |= DMU_UNCACHEDIO;
430 if (uio->uio_extflg & UIO_DIRECT) {
431 /*
432 * All pages for an O_DIRECT request ahve already been mapped
433 * so there's no compelling reason to handle this uio in
434 * smaller chunks.
435 */
436 chunk_size = DMU_MAX_ACCESS;
437
438 /*
439 * In the event that the O_DIRECT request is reading the entire
440 * file, it is possible file's length is not page sized
441 * aligned. However, lower layers expect that the Direct I/O
442 * request is page-aligned. In this case, as much of the file
443 * that can be read using Direct I/O happens and the remaining
444 * amount will be read through the ARC.
445 *
446 * This is still consistent with the semantics of Direct I/O in
447 * ZFS as at a minimum the I/O request must be page-aligned.
448 */
449 dio_remaining_resid = n - P2ALIGN_TYPED(n, PAGE_SIZE, ssize_t);
450 if (dio_remaining_resid != 0)
451 n -= dio_remaining_resid;
452 dflags |= DMU_DIRECTIO;
453 } else {
454 chunk_size = MIN(MAX(zfs_vnops_read_chunk_size, blksz),
455 DMU_MAX_ACCESS / 2);
456 }
457
458 while (n > 0) {
459 ssize_t nbytes = MIN(n, chunk_size -
460 P2PHASE(zfs_uio_offset(uio), blksz));
461 #ifdef UIO_NOCOPY
462 if (zfs_uio_segflg(uio) == UIO_NOCOPY)
463 error = mappedread_sf(zp, nbytes, uio);
464 else
465 #endif
466 if (zn_has_cached_data(zp, zfs_uio_offset(uio),
467 zfs_uio_offset(uio) + nbytes - 1)) {
468 error = mappedread(zp, nbytes, uio);
469 } else {
470 error = dmu_read_uio_dbuf(sa_get_db(zp->z_sa_hdl),
471 uio, nbytes, dflags);
472 }
473
474 if (error) {
475 /* convert checksum errors into IO errors */
476 if (error == ECKSUM) {
477 /*
478 * If a Direct I/O read returned a checksum
479 * verify error, then it must be treated as
480 * suspicious. The contents of the buffer could
481 * have beeen manipulated while the I/O was in
482 * flight. In this case, the remainder of I/O
483 * request will just be reissued through the
484 * ARC.
485 */
486 if (uio->uio_extflg & UIO_DIRECT) {
487 dio_checksum_failure = B_TRUE;
488 uio->uio_extflg &= ~UIO_DIRECT;
489 n += dio_remaining_resid;
490 dio_remaining_resid = 0;
491 continue;
492 } else {
493 error = SET_ERROR(EIO);
494 }
495 }
496
497 #if defined(__linux__)
498 /*
499 * if we actually read some bytes, bubbling EFAULT
500 * up to become EAGAIN isn't what we want here...
501 *
502 * ...on Linux, at least. On FBSD, doing this breaks.
503 */
504 if (error == EFAULT &&
505 (zfs_uio_offset(uio) - start_offset) != 0)
506 error = 0;
507 #endif
508 break;
509 }
510
511 n -= nbytes;
512 }
513
514 if (error == 0 && (uio->uio_extflg & UIO_DIRECT) &&
515 dio_remaining_resid != 0) {
516 /*
517 * Temporarily remove the UIO_DIRECT flag from the UIO so the
518 * remainder of the file can be read using the ARC.
519 */
520 uio->uio_extflg &= ~UIO_DIRECT;
521 dflags &= ~DMU_DIRECTIO;
522
523 if (zn_has_cached_data(zp, zfs_uio_offset(uio),
524 zfs_uio_offset(uio) + dio_remaining_resid - 1)) {
525 error = mappedread(zp, dio_remaining_resid, uio);
526 } else {
527 error = dmu_read_uio_dbuf(sa_get_db(zp->z_sa_hdl), uio,
528 dio_remaining_resid, dflags);
529 }
530 uio->uio_extflg |= UIO_DIRECT;
531 dflags |= DMU_DIRECTIO;
532
533 if (error != 0)
534 n += dio_remaining_resid;
535 } else if (error && (uio->uio_extflg & UIO_DIRECT)) {
536 n += dio_remaining_resid;
537 }
538 int64_t nread = start_resid - n;
539
540 dataset_kstats_update_read_kstats(&zfsvfs->z_kstat, nread);
541 out:
542 zfs_rangelock_exit(lr);
543
544 if (dio_checksum_failure == B_TRUE)
545 uio->uio_extflg |= UIO_DIRECT;
546
547 /*
548 * Cleanup for Direct I/O if requested.
549 */
550 if (uio->uio_extflg & UIO_DIRECT)
551 zfs_uio_free_dio_pages(uio, UIO_READ);
552
553 ZFS_ACCESSTIME_STAMP(zfsvfs, zp);
554 zfs_exit(zfsvfs, FTAG);
555 return (error);
556 }
557
558 static void
zfs_clear_setid_bits_if_necessary(zfsvfs_t * zfsvfs,znode_t * zp,cred_t * cr,uint64_t * clear_setid_bits_txgp,dmu_tx_t * tx)559 zfs_clear_setid_bits_if_necessary(zfsvfs_t *zfsvfs, znode_t *zp, cred_t *cr,
560 uint64_t *clear_setid_bits_txgp, dmu_tx_t *tx)
561 {
562 zilog_t *zilog = zfsvfs->z_log;
563 const uint64_t uid = KUID_TO_SUID(ZTOUID(zp));
564
565 ASSERT(clear_setid_bits_txgp != NULL);
566 ASSERT(tx != NULL);
567
568 /*
569 * Clear Set-UID/Set-GID bits on successful write if not
570 * privileged and at least one of the execute bits is set.
571 *
572 * It would be nice to do this after all writes have
573 * been done, but that would still expose the ISUID/ISGID
574 * to another app after the partial write is committed.
575 *
576 * Note: we don't call zfs_fuid_map_id() here because
577 * user 0 is not an ephemeral uid.
578 */
579 mutex_enter(&zp->z_acl_lock);
580 if ((zp->z_mode & (S_IXUSR | (S_IXUSR >> 3) | (S_IXUSR >> 6))) != 0 &&
581 (zp->z_mode & (S_ISUID | S_ISGID)) != 0 &&
582 secpolicy_vnode_setid_retain(zp, cr,
583 ((zp->z_mode & S_ISUID) != 0 && uid == 0)) != 0) {
584 uint64_t newmode;
585
586 zp->z_mode &= ~(S_ISUID | S_ISGID);
587 newmode = zp->z_mode;
588 (void) sa_update(zp->z_sa_hdl, SA_ZPL_MODE(zfsvfs),
589 (void *)&newmode, sizeof (uint64_t), tx);
590
591 mutex_exit(&zp->z_acl_lock);
592
593 /*
594 * Make sure SUID/SGID bits will be removed when we replay the
595 * log. If the setid bits are keep coming back, don't log more
596 * than one TX_SETATTR per transaction group.
597 */
598 if (*clear_setid_bits_txgp != dmu_tx_get_txg(tx)) {
599 vattr_t va = {0};
600
601 va.va_mask = ATTR_MODE;
602 va.va_nodeid = zp->z_id;
603 va.va_mode = newmode;
604 zfs_log_setattr(zilog, tx, TX_SETATTR, zp, &va,
605 ATTR_MODE, NULL);
606 *clear_setid_bits_txgp = dmu_tx_get_txg(tx);
607 }
608 } else {
609 mutex_exit(&zp->z_acl_lock);
610 }
611 }
612
613 /*
614 * Write the bytes to a file.
615 *
616 * IN: zp - znode of file to be written to.
617 * uio - structure supplying write location, range info,
618 * and data buffer.
619 * ioflag - O_APPEND flag set if in append mode.
620 * O_DIRECT flag; used to bypass page cache.
621 * cr - credentials of caller.
622 *
623 * OUT: uio - updated offset and range.
624 *
625 * RETURN: 0 if success
626 * error code if failure
627 *
628 * Timestamps:
629 * ip - ctime|mtime updated if byte count > 0
630 */
631 int
zfs_write(znode_t * zp,zfs_uio_t * uio,int ioflag,cred_t * cr)632 zfs_write(znode_t *zp, zfs_uio_t *uio, int ioflag, cred_t *cr)
633 {
634 int error = 0, error1;
635 ssize_t start_resid = zfs_uio_resid(uio);
636 uint64_t clear_setid_bits_txg = 0;
637 boolean_t o_direct_defer = B_FALSE;
638
639 /*
640 * Fasttrack empty write
641 */
642 ssize_t n = start_resid;
643 if (n == 0)
644 return (0);
645
646 zfsvfs_t *zfsvfs = ZTOZSB(zp);
647 if ((error = zfs_enter_verify_zp(zfsvfs, zp, FTAG)) != 0)
648 return (error);
649
650 sa_bulk_attr_t bulk[5];
651 int count = 0;
652 uint64_t mtime[2], ctime[2];
653 SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_MTIME(zfsvfs), NULL, &mtime, 16);
654 SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_CTIME(zfsvfs), NULL, &ctime, 16);
655 SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_SIZE(zfsvfs), NULL,
656 &zp->z_size, 8);
657 SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_FLAGS(zfsvfs), NULL,
658 &zp->z_pflags, 8);
659 if (zp->z_is_sa)
660 SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_SEQ(zfsvfs), NULL,
661 &zp->z_seq, 8);
662
663 /*
664 * Callers might not be able to detect properly that we are read-only,
665 * so check it explicitly here.
666 */
667 if (zfs_is_readonly(zfsvfs)) {
668 zfs_exit(zfsvfs, FTAG);
669 return (SET_ERROR(EROFS));
670 }
671
672 /*
673 * If immutable or not appending then return EPERM.
674 * Intentionally allow ZFS_READONLY through here.
675 * See zfs_zaccess_common()
676 */
677 if ((zp->z_pflags & ZFS_IMMUTABLE) ||
678 ((zp->z_pflags & ZFS_APPENDONLY) && !(ioflag & O_APPEND) &&
679 (zfs_uio_offset(uio) < zp->z_size))) {
680 zfs_exit(zfsvfs, FTAG);
681 return (SET_ERROR(EPERM));
682 }
683
684 /*
685 * Validate file offset
686 */
687 offset_t woff = ioflag & O_APPEND ? zp->z_size : zfs_uio_offset(uio);
688 if (woff < 0) {
689 zfs_exit(zfsvfs, FTAG);
690 return (SET_ERROR(EINVAL));
691 }
692
693 /*
694 * Setting up Direct I/O if requested.
695 */
696 error = zfs_setup_direct(zp, uio, UIO_WRITE, &ioflag);
697 if (error) {
698 zfs_exit(zfsvfs, FTAG);
699 return (SET_ERROR(error));
700 }
701
702 /*
703 * Pre-fault the pages to ensure slow (eg NFS) pages
704 * don't hold up txg.
705 */
706 ssize_t pfbytes = MIN(n, DMU_MAX_ACCESS >> 1);
707 if (zfs_uio_prefaultpages(pfbytes, uio)) {
708 zfs_exit(zfsvfs, FTAG);
709 return (SET_ERROR(EFAULT));
710 }
711
712 /*
713 * If in append mode, set the io offset pointer to eof.
714 */
715 zfs_locked_range_t *lr;
716 if (ioflag & O_APPEND) {
717 /*
718 * Obtain an appending range lock to guarantee file append
719 * semantics. We reset the write offset once we have the lock.
720 */
721 lr = zfs_rangelock_enter(&zp->z_rangelock, 0, n, RL_APPEND);
722 woff = lr->lr_offset;
723 if (lr->lr_length == UINT64_MAX) {
724 /*
725 * We overlocked the file because this write will cause
726 * the file block size to increase.
727 * Note that zp_size cannot change with this lock held.
728 */
729 woff = zp->z_size;
730 }
731 zfs_uio_setoffset(uio, woff);
732 /*
733 * We need to update the starting offset as well because it is
734 * set previously in the ZPL (Linux) and VNOPS (FreeBSD)
735 * layers.
736 */
737 zfs_uio_setsoffset(uio, woff);
738 } else {
739 /*
740 * Note that if the file block size will change as a result of
741 * this write, then this range lock will lock the entire file
742 * so that we can re-write the block safely.
743 */
744 lr = zfs_rangelock_enter(&zp->z_rangelock, woff, n, RL_WRITER);
745 }
746
747 if (zn_rlimit_fsize_uio(zp, uio)) {
748 zfs_rangelock_exit(lr);
749 zfs_exit(zfsvfs, FTAG);
750 return (SET_ERROR(EFBIG));
751 }
752
753 const rlim64_t limit = MAXOFFSET_T;
754
755 if (woff >= limit) {
756 zfs_rangelock_exit(lr);
757 zfs_exit(zfsvfs, FTAG);
758 return (SET_ERROR(EFBIG));
759 }
760
761 if (n > limit - woff)
762 n = limit - woff;
763
764 uint64_t end_size = MAX(zp->z_size, woff + n);
765 zilog_t *zilog = zfsvfs->z_log;
766 boolean_t commit = (ioflag & (O_SYNC | O_DSYNC)) ||
767 (zfsvfs->z_os->os_sync == ZFS_SYNC_ALWAYS);
768
769 const uint64_t uid = KUID_TO_SUID(ZTOUID(zp));
770 const uint64_t gid = KGID_TO_SGID(ZTOGID(zp));
771 const uint64_t projid = zp->z_projid;
772
773 /*
774 * In the event we are increasing the file block size
775 * (lr_length == UINT64_MAX), we will direct the write to the ARC.
776 * Because zfs_grow_blocksize() will read from the ARC in order to
777 * grow the dbuf, we avoid doing Direct I/O here as that would cause
778 * data written to disk to be overwritten by data in the ARC during
779 * the sync phase. Besides writing data twice to disk, we also
780 * want to avoid consistency concerns between data in the the ARC and
781 * on disk while growing the file's blocksize.
782 *
783 * We will only temporarily remove Direct I/O and put it back after
784 * we have grown the blocksize. We do this in the event a request
785 * is larger than max_blksz, so further requests to
786 * dmu_write_uio_dbuf() will still issue the requests using Direct
787 * IO.
788 *
789 * As an example:
790 * The first block to file is being written as a 4k request with
791 * a recorsize of 1K. The first 1K issued in the loop below will go
792 * through the ARC; however, the following 3 1K requests will
793 * use Direct I/O.
794 */
795 if (uio->uio_extflg & UIO_DIRECT && lr->lr_length == UINT64_MAX) {
796 uio->uio_extflg &= ~UIO_DIRECT;
797 o_direct_defer = B_TRUE;
798 }
799
800 /*
801 * Write the file in reasonable size chunks. Each chunk is written
802 * in a separate transaction; this keeps the intent log records small
803 * and allows us to do more fine-grained space accounting.
804 */
805 while (n > 0) {
806 woff = zfs_uio_offset(uio);
807
808 if (zfs_id_overblockquota(zfsvfs, DMU_USERUSED_OBJECT, uid) ||
809 zfs_id_overblockquota(zfsvfs, DMU_GROUPUSED_OBJECT, gid) ||
810 (projid != ZFS_DEFAULT_PROJID &&
811 zfs_id_overblockquota(zfsvfs, DMU_PROJECTUSED_OBJECT,
812 projid))) {
813 error = SET_ERROR(EDQUOT);
814 break;
815 }
816
817 uint64_t blksz;
818 if (lr->lr_length == UINT64_MAX && zp->z_size <= zp->z_blksz) {
819 if (zp->z_blksz > zfsvfs->z_max_blksz &&
820 !ISP2(zp->z_blksz)) {
821 /*
822 * File's blocksize is already larger than the
823 * "recordsize" property. Only let it grow to
824 * the next power of 2.
825 */
826 blksz = 1 << highbit64(zp->z_blksz);
827 } else {
828 blksz = zfsvfs->z_max_blksz;
829 }
830 blksz = MIN(blksz, P2ROUNDUP(end_size,
831 SPA_MINBLOCKSIZE));
832 blksz = MAX(blksz, zp->z_blksz);
833 } else {
834 blksz = zp->z_blksz;
835 }
836
837 arc_buf_t *abuf = NULL;
838 ssize_t nbytes = n;
839 if (n >= blksz && woff >= zp->z_size &&
840 P2PHASE(woff, blksz) == 0 &&
841 !(uio->uio_extflg & UIO_DIRECT) &&
842 (blksz >= SPA_OLD_MAXBLOCKSIZE || n < 4 * blksz)) {
843 /*
844 * This write covers a full block. "Borrow" a buffer
845 * from the dmu so that we can fill it before we enter
846 * a transaction. This avoids the possibility of
847 * holding up the transaction if the data copy hangs
848 * up on a pagefault (e.g., from an NFS server mapping).
849 */
850 abuf = dmu_request_arcbuf(sa_get_db(zp->z_sa_hdl),
851 blksz);
852 ASSERT(abuf != NULL);
853 ASSERT(arc_buf_size(abuf) == blksz);
854 if ((error = zfs_uiocopy(abuf->b_data, blksz,
855 UIO_WRITE, uio, &nbytes))) {
856 dmu_return_arcbuf(abuf);
857 break;
858 }
859 ASSERT3S(nbytes, ==, blksz);
860 } else {
861 nbytes = MIN(n, (DMU_MAX_ACCESS >> 1) -
862 P2PHASE(woff, blksz));
863 if (pfbytes < nbytes) {
864 if (zfs_uio_prefaultpages(nbytes, uio)) {
865 error = SET_ERROR(EFAULT);
866 break;
867 }
868 pfbytes = nbytes;
869 }
870 }
871
872 /*
873 * Start a transaction.
874 */
875 dmu_tx_t *tx = dmu_tx_create(zfsvfs->z_os);
876 dmu_tx_hold_sa(tx, zp->z_sa_hdl, ZFS_SEQ_MAY_GROW(zp));
877 dmu_buf_impl_t *db = (dmu_buf_impl_t *)sa_get_db(zp->z_sa_hdl);
878 DB_DNODE_ENTER(db);
879 dmu_tx_hold_write_by_dnode(tx, DB_DNODE(db), woff, nbytes);
880 DB_DNODE_EXIT(db);
881 zfs_sa_upgrade_txholds(tx, zp);
882 error = dmu_tx_assign(tx, DMU_TX_WAIT);
883 if (error) {
884 dmu_tx_abort(tx);
885 if (abuf != NULL)
886 dmu_return_arcbuf(abuf);
887 break;
888 }
889
890 /*
891 * NB: We must call zfs_clear_setid_bits_if_necessary before
892 * committing the transaction!
893 */
894
895 /*
896 * If rangelock_enter() over-locked we grow the blocksize
897 * and then reduce the lock range. This will only happen
898 * on the first iteration since rangelock_reduce() will
899 * shrink down lr_length to the appropriate size.
900 */
901 if (lr->lr_length == UINT64_MAX) {
902 zfs_grow_blocksize(zp, blksz, tx);
903 zfs_rangelock_reduce(lr, woff, n);
904 }
905
906 dmu_flags_t dflags = DMU_READ_PREFETCH;
907 if (ioflag & O_DIRECT)
908 dflags |= DMU_UNCACHEDIO;
909 if (uio->uio_extflg & UIO_DIRECT)
910 dflags |= DMU_DIRECTIO;
911
912 ssize_t tx_bytes;
913 if (abuf == NULL) {
914 tx_bytes = zfs_uio_resid(uio);
915 zfs_uio_fault_disable(uio, B_TRUE);
916 error = dmu_write_uio_dbuf(sa_get_db(zp->z_sa_hdl),
917 uio, nbytes, tx, dflags);
918 zfs_uio_fault_disable(uio, B_FALSE);
919 #ifdef __linux__
920 if (error == EFAULT) {
921 zfs_clear_setid_bits_if_necessary(zfsvfs, zp,
922 cr, &clear_setid_bits_txg, tx);
923 dmu_tx_commit(tx);
924 /*
925 * Account for partial writes before
926 * continuing the loop.
927 * Update needs to occur before the next
928 * zfs_uio_prefaultpages, or prefaultpages may
929 * error, and we may break the loop early.
930 */
931 n -= tx_bytes - zfs_uio_resid(uio);
932 pfbytes -= tx_bytes - zfs_uio_resid(uio);
933 continue;
934 }
935 #endif
936 /*
937 * On FreeBSD, EFAULT should be propagated back to the
938 * VFS, which will handle faulting and will retry.
939 */
940 if (error != 0 && error != EFAULT) {
941 zfs_clear_setid_bits_if_necessary(zfsvfs, zp,
942 cr, &clear_setid_bits_txg, tx);
943 dmu_tx_commit(tx);
944 break;
945 }
946 tx_bytes -= zfs_uio_resid(uio);
947 } else {
948 /*
949 * Thus, we're writing a full block at a block-aligned
950 * offset and extending the file past EOF.
951 *
952 * dmu_assign_arcbuf_by_dbuf() will directly assign the
953 * arc buffer to a dbuf.
954 */
955 error = dmu_assign_arcbuf_by_dbuf(
956 sa_get_db(zp->z_sa_hdl), woff, abuf, tx, dflags);
957 if (error != 0) {
958 /*
959 * XXX This might not be necessary if
960 * dmu_assign_arcbuf_by_dbuf is guaranteed
961 * to be atomic.
962 */
963 zfs_clear_setid_bits_if_necessary(zfsvfs, zp,
964 cr, &clear_setid_bits_txg, tx);
965 dmu_return_arcbuf(abuf);
966 dmu_tx_commit(tx);
967 break;
968 }
969 ASSERT3S(nbytes, <=, zfs_uio_resid(uio));
970 zfs_uioskip(uio, nbytes);
971 tx_bytes = nbytes;
972 }
973 /*
974 * There is a window where a file's pages can be mmap'ed after
975 * zfs_setup_direct() is called. This is due to the fact that
976 * the rangelock in this function is acquired after calling
977 * zfs_setup_direct(). This is done so that
978 * zfs_uio_prefaultpages() does not attempt to fault in pages
979 * on Linux for Direct I/O requests. This is not necessary as
980 * the pages are pinned in memory and can not be faulted out.
981 * Ideally, the rangelock would be held before calling
982 * zfs_setup_direct() and zfs_uio_prefaultpages(); however,
983 * this can lead to a deadlock as zfs_getpage() also acquires
984 * the rangelock as a RL_WRITER and prefaulting the pages can
985 * lead to zfs_getpage() being called.
986 *
987 * In the case of the pages being mapped after
988 * zfs_setup_direct() is called, the call to update_pages()
989 * will still be made to make sure there is consistency between
990 * the ARC and the Linux page cache. This is an ufortunate
991 * situation as the data will be read back into the ARC after
992 * the Direct I/O write has completed, but this is the penality
993 * for writing to a mmap'ed region of a file using Direct I/O.
994 */
995 if (tx_bytes &&
996 zn_has_cached_data(zp, woff, woff + tx_bytes - 1)) {
997 update_pages(zp, woff, tx_bytes, zfsvfs->z_os);
998 }
999
1000 /*
1001 * If we made no progress, we're done. If we made even
1002 * partial progress, update the znode and ZIL accordingly.
1003 */
1004 if (tx_bytes == 0) {
1005 (void) sa_update(zp->z_sa_hdl, SA_ZPL_SIZE(zfsvfs),
1006 (void *)&zp->z_size, sizeof (uint64_t), tx);
1007 dmu_tx_commit(tx);
1008 ASSERT(error != 0);
1009 break;
1010 }
1011
1012 zfs_clear_setid_bits_if_necessary(zfsvfs, zp, cr,
1013 &clear_setid_bits_txg, tx);
1014
1015 zfs_tstamp_update_setup(zp, CONTENT_MODIFIED, mtime, ctime);
1016 if (zp->z_is_sa)
1017 zp->z_has_seq = B_TRUE;
1018
1019 /*
1020 * Update the file size (zp_size) if it has changed;
1021 * account for possible concurrent updates.
1022 */
1023 while ((end_size = zp->z_size) < zfs_uio_offset(uio)) {
1024 (void) atomic_cas_64(&zp->z_size, end_size,
1025 zfs_uio_offset(uio));
1026 ASSERT(error == 0 || error == EFAULT);
1027 }
1028 /*
1029 * If we are replaying and eof is non zero then force
1030 * the file size to the specified eof. Note, there's no
1031 * concurrency during replay.
1032 */
1033 if (zfsvfs->z_replay && zfsvfs->z_replay_eof != 0)
1034 zp->z_size = zfsvfs->z_replay_eof;
1035
1036 ASSERT3S(count, <=, ARRAY_SIZE(bulk));
1037 error1 = sa_bulk_update(zp->z_sa_hdl, bulk, count, tx);
1038 if (error1 != 0)
1039 /* Avoid clobbering EFAULT. */
1040 error = error1;
1041
1042 /*
1043 * NB: During replay, the TX_SETATTR record logged by
1044 * zfs_clear_setid_bits_if_necessary must precede any of
1045 * the TX_WRITE records logged here.
1046 */
1047 zfs_log_write(zilog, tx, TX_WRITE, zp, woff, tx_bytes, commit,
1048 uio->uio_extflg & UIO_DIRECT ? B_TRUE : B_FALSE, NULL,
1049 NULL);
1050
1051 dmu_tx_commit(tx);
1052
1053 /*
1054 * Direct I/O was deferred in order to grow the first block.
1055 * At this point it can be re-enabled for subsequent writes.
1056 */
1057 if (o_direct_defer) {
1058 ASSERT(ioflag & O_DIRECT);
1059 uio->uio_extflg |= UIO_DIRECT;
1060 o_direct_defer = B_FALSE;
1061 }
1062
1063 if (error != 0)
1064 break;
1065 ASSERT3S(tx_bytes, ==, nbytes);
1066 n -= nbytes;
1067 pfbytes -= nbytes;
1068 }
1069
1070 if (o_direct_defer) {
1071 ASSERT(ioflag & O_DIRECT);
1072 uio->uio_extflg |= UIO_DIRECT;
1073 o_direct_defer = B_FALSE;
1074 }
1075
1076 zfs_znode_update_vfs(zp);
1077 zfs_rangelock_exit(lr);
1078
1079 /*
1080 * Cleanup for Direct I/O if requested.
1081 */
1082 if (uio->uio_extflg & UIO_DIRECT)
1083 zfs_uio_free_dio_pages(uio, UIO_WRITE);
1084
1085 /*
1086 * If we're in replay mode, or we made no progress, or the
1087 * uio data is inaccessible return an error. Otherwise, it's
1088 * at least a partial write, so it's successful.
1089 */
1090 if (zfsvfs->z_replay || zfs_uio_resid(uio) == start_resid ||
1091 error == EFAULT) {
1092 zfs_exit(zfsvfs, FTAG);
1093 return (error);
1094 }
1095
1096 if (commit) {
1097 error = zil_commit(zilog, zp->z_id);
1098 if (error != 0) {
1099 zfs_exit(zfsvfs, FTAG);
1100 return (error);
1101 }
1102 }
1103
1104 int64_t nwritten = start_resid - zfs_uio_resid(uio);
1105 dataset_kstats_update_write_kstats(&zfsvfs->z_kstat, nwritten);
1106
1107 zfs_exit(zfsvfs, FTAG);
1108 return (0);
1109 }
1110
1111 /*
1112 * Check if a block should be skipped during rewrite.
1113 * Returns B_TRUE if block should be skipped.
1114 */
1115 static boolean_t
zfs_rewrite_skip(dmu_buf_t * db,objset_t * os,uint64_t flags)1116 zfs_rewrite_skip(dmu_buf_t *db, objset_t *os, uint64_t flags)
1117 {
1118 /*
1119 * This may be slightly stale and racy, but should be OK for
1120 * the advisory use.
1121 */
1122 blkptr_t *bp = dmu_buf_get_blkptr(db);
1123 if (bp == NULL)
1124 return (B_TRUE);
1125
1126 if (flags & ZFS_REWRITE_SKIP_SNAPSHOT) {
1127 if (dmu_objset_block_is_shared(os, bp))
1128 return (B_TRUE);
1129 }
1130
1131 if (flags & ZFS_REWRITE_SKIP_BRT) {
1132 if (brt_maybe_exists(os->os_spa, bp))
1133 return (B_TRUE);
1134 }
1135
1136 return (B_FALSE);
1137 }
1138
1139 /*
1140 * Rewrite a range of file as-is without modification.
1141 *
1142 * IN: zp - znode of file to be rewritten.
1143 * off - Offset of the range to rewrite.
1144 * len - Length of the range to rewrite.
1145 * flags - Random rewrite parameters.
1146 * arg - flags-specific argument.
1147 *
1148 * RETURN: 0 if success
1149 * error code if failure
1150 */
1151 int
zfs_rewrite(znode_t * zp,uint64_t off,uint64_t len,uint64_t flags,uint64_t arg)1152 zfs_rewrite(znode_t *zp, uint64_t off, uint64_t len, uint64_t flags,
1153 uint64_t arg)
1154 {
1155 int error;
1156
1157 #define ZFS_REWRITE_VALID_FLAGS \
1158 (ZFS_REWRITE_PHYSICAL | ZFS_REWRITE_SKIP_SNAPSHOT | \
1159 ZFS_REWRITE_SKIP_BRT)
1160
1161 if ((flags & ~ZFS_REWRITE_VALID_FLAGS) != 0 || arg != 0)
1162 return (SET_ERROR(EINVAL));
1163
1164 zfsvfs_t *zfsvfs = ZTOZSB(zp);
1165 if ((error = zfs_enter_verify_zp(zfsvfs, zp, FTAG)) != 0)
1166 return (error);
1167
1168 /* Check if physical rewrite is allowed */
1169 spa_t *spa = zfsvfs->z_os->os_spa;
1170 if ((flags & ZFS_REWRITE_PHYSICAL) &&
1171 !spa_feature_is_enabled(spa, SPA_FEATURE_PHYSICAL_REWRITE)) {
1172 zfs_exit(zfsvfs, FTAG);
1173 return (SET_ERROR(ENOTSUP));
1174 }
1175
1176 if (zfs_is_readonly(zfsvfs)) {
1177 zfs_exit(zfsvfs, FTAG);
1178 return (SET_ERROR(EROFS));
1179 }
1180
1181 if (off >= zp->z_size) {
1182 zfs_exit(zfsvfs, FTAG);
1183 return (0);
1184 }
1185 if (len == 0 || len > zp->z_size - off)
1186 len = zp->z_size - off;
1187
1188 /* Flush any mmap()'d data to disk */
1189 if (zn_has_cached_data(zp, off, off + len - 1))
1190 zn_flush_cached_data(zp, B_TRUE);
1191
1192 zfs_locked_range_t *lr;
1193 lr = zfs_rangelock_enter(&zp->z_rangelock, off, len, RL_WRITER);
1194
1195 const uint64_t uid = KUID_TO_SUID(ZTOUID(zp));
1196 const uint64_t gid = KGID_TO_SGID(ZTOGID(zp));
1197 const uint64_t projid = zp->z_projid;
1198
1199 dmu_buf_impl_t *db = (dmu_buf_impl_t *)sa_get_db(zp->z_sa_hdl);
1200 DB_DNODE_ENTER(db);
1201 dnode_t *dn = DB_DNODE(db);
1202
1203 uint64_t n, noff = off, nr = 0, nw = 0;
1204 while (len > 0) {
1205 /*
1206 * Rewrite only actual data, skipping any holes. This might
1207 * be inaccurate for dirty files, but we don't really care.
1208 */
1209 if (noff == off) {
1210 /* Find next data in the file. */
1211 error = dnode_next_offset(dn, 0, &noff, 1, 1, 0);
1212 if (error || noff >= off + len) {
1213 if (error == ESRCH) /* No more data. */
1214 error = 0;
1215 break;
1216 }
1217 ASSERT3U(noff, >=, off);
1218 len -= noff - off;
1219 off = noff;
1220
1221 /* Find where the data end. */
1222 error = dnode_next_offset(dn, DNODE_FIND_HOLE, &noff,
1223 1, 1, 0);
1224 if (error != 0)
1225 noff = off + len;
1226 }
1227 ASSERT3U(noff, >, off);
1228
1229 if (zfs_id_overblockquota(zfsvfs, DMU_USERUSED_OBJECT, uid) ||
1230 zfs_id_overblockquota(zfsvfs, DMU_GROUPUSED_OBJECT, gid) ||
1231 (projid != ZFS_DEFAULT_PROJID &&
1232 zfs_id_overblockquota(zfsvfs, DMU_PROJECTUSED_OBJECT,
1233 projid))) {
1234 error = SET_ERROR(EDQUOT);
1235 break;
1236 }
1237
1238 n = MIN(MIN(len, noff - off),
1239 DMU_MAX_ACCESS / 2 - P2PHASE(off, zp->z_blksz));
1240
1241 dmu_tx_t *tx = dmu_tx_create(zfsvfs->z_os);
1242 dmu_tx_hold_write_by_dnode(tx, dn, off, n);
1243 error = dmu_tx_assign(tx, DMU_TX_WAIT);
1244 if (error) {
1245 dmu_tx_abort(tx);
1246 break;
1247 }
1248
1249 /* Mark all dbufs within range as dirty to trigger rewrite. */
1250 dmu_buf_t **dbp;
1251 int numbufs;
1252 error = dmu_buf_hold_array_by_dnode(dn, off, n, TRUE, FTAG,
1253 &numbufs, &dbp, DMU_READ_PREFETCH | DMU_UNCACHEDIO);
1254 if (error) {
1255 dmu_tx_commit(tx);
1256 break;
1257 }
1258 for (int i = 0; i < numbufs; i++) {
1259 nr += dbp[i]->db_size;
1260 if (dmu_buf_is_dirty(dbp[i], tx))
1261 continue;
1262
1263 if (zfs_rewrite_skip(dbp[i], zfsvfs->z_os, flags))
1264 continue;
1265
1266 nw += dbp[i]->db_size;
1267 if (flags & ZFS_REWRITE_PHYSICAL)
1268 dmu_buf_will_rewrite(dbp[i], tx);
1269 else
1270 dmu_buf_will_dirty(dbp[i], tx);
1271 }
1272 dmu_buf_rele_array(dbp, numbufs, FTAG);
1273
1274 dmu_tx_commit(tx);
1275
1276 len -= n;
1277 off += n;
1278
1279 if (issig()) {
1280 error = SET_ERROR(EINTR);
1281 break;
1282 }
1283 }
1284
1285 DB_DNODE_EXIT(db);
1286
1287 dataset_kstats_update_read_kstats(&zfsvfs->z_kstat, nr);
1288 dataset_kstats_update_write_kstats(&zfsvfs->z_kstat, nw);
1289
1290 zfs_rangelock_exit(lr);
1291 zfs_exit(zfsvfs, FTAG);
1292 return (error);
1293 }
1294
1295 int
zfs_getsecattr(znode_t * zp,vsecattr_t * vsecp,int flag,cred_t * cr)1296 zfs_getsecattr(znode_t *zp, vsecattr_t *vsecp, int flag, cred_t *cr)
1297 {
1298 zfsvfs_t *zfsvfs = ZTOZSB(zp);
1299 int error;
1300 boolean_t skipaclchk = (flag & ATTR_NOACLCHECK) ? B_TRUE : B_FALSE;
1301
1302 if ((error = zfs_enter_verify_zp(zfsvfs, zp, FTAG)) != 0)
1303 return (error);
1304 error = zfs_getacl(zp, vsecp, skipaclchk, cr);
1305 zfs_exit(zfsvfs, FTAG);
1306
1307 return (error);
1308 }
1309
1310 int
zfs_setsecattr(znode_t * zp,vsecattr_t * vsecp,int flag,cred_t * cr)1311 zfs_setsecattr(znode_t *zp, vsecattr_t *vsecp, int flag, cred_t *cr)
1312 {
1313 zfsvfs_t *zfsvfs = ZTOZSB(zp);
1314 int error;
1315 boolean_t skipaclchk = (flag & ATTR_NOACLCHECK) ? B_TRUE : B_FALSE;
1316 zilog_t *zilog;
1317
1318 if ((error = zfs_enter_verify_zp(zfsvfs, zp, FTAG)) != 0)
1319 return (error);
1320 zilog = zfsvfs->z_log;
1321 error = zfs_setacl(zp, vsecp, skipaclchk, cr);
1322
1323 if (error == 0 && zfsvfs->z_os->os_sync == ZFS_SYNC_ALWAYS)
1324 error = zil_commit(zilog, 0);
1325
1326 zfs_exit(zfsvfs, FTAG);
1327 return (error);
1328 }
1329
1330 /*
1331 * Get the optimal alignment to ensure direct IO can be performed without
1332 * incurring any RMW penalty on write. If direct IO is not enabled for this
1333 * file, returns an error.
1334 */
1335 int
zfs_get_direct_alignment(znode_t * zp,uint64_t * alignp)1336 zfs_get_direct_alignment(znode_t *zp, uint64_t *alignp)
1337 {
1338 zfsvfs_t *zfsvfs = ZTOZSB(zp);
1339
1340 if (!zfs_dio_enabled || zfsvfs->z_os->os_direct == ZFS_DIRECT_DISABLED)
1341 return (SET_ERROR(EOPNOTSUPP));
1342
1343 /*
1344 * If the file has multiple blocks, then its block size is fixed
1345 * forever, and so is the ideal alignment.
1346 *
1347 * If however it only has a single block, then we want to return the
1348 * max block size it could possibly grown to (ie, the dataset
1349 * recordsize). We do this so that a program querying alignment
1350 * immediately after the file is created gets a value that won't change
1351 * once the file has grown into the second block and beyond.
1352 *
1353 * Because we don't have a count of blocks easily available here, we
1354 * check if the apparent file size is smaller than its current block
1355 * size (meaning, the file hasn't yet grown into the current block
1356 * size) and then, check if the block size is smaller than the dataset
1357 * maximum (meaning, if the file grew past the current block size, the
1358 * block size could would be increased).
1359 */
1360 if (zp->z_size <= zp->z_blksz && zp->z_blksz < zfsvfs->z_max_blksz)
1361 *alignp = MAX(zfsvfs->z_max_blksz, PAGE_SIZE);
1362 else
1363 *alignp = MAX(zp->z_blksz, PAGE_SIZE);
1364
1365 return (0);
1366 }
1367
1368 #ifdef ZFS_DEBUG
1369 static int zil_fault_io = 0;
1370 #endif
1371
1372 static void zfs_get_done(zgd_t *zgd, int error);
1373
1374 /*
1375 * Get data to generate a TX_WRITE intent log record.
1376 */
1377 int
zfs_get_data(void * arg,uint64_t gen,lr_write_t * lr,char * buf,struct lwb * lwb,zio_t * zio)1378 zfs_get_data(void *arg, uint64_t gen, lr_write_t *lr, char *buf,
1379 struct lwb *lwb, zio_t *zio)
1380 {
1381 zfsvfs_t *zfsvfs = arg;
1382 objset_t *os = zfsvfs->z_os;
1383 znode_t *zp;
1384 uint64_t object = lr->lr_foid;
1385 uint64_t offset = lr->lr_offset;
1386 uint64_t size = lr->lr_length;
1387 zgd_t *zgd;
1388 int error = 0;
1389 uint64_t zp_gen;
1390
1391 ASSERT3P(lwb, !=, NULL);
1392 ASSERT3U(size, !=, 0);
1393
1394 /*
1395 * Nothing to do if the file has been removed
1396 */
1397 if (zfs_zget(zfsvfs, object, &zp) != 0)
1398 return (SET_ERROR(ENOENT));
1399 if (zp->z_unlinked) {
1400 /*
1401 * Release the vnode asynchronously as we currently have the
1402 * txg stopped from syncing.
1403 */
1404 zfs_zrele_async(zp);
1405 return (SET_ERROR(ENOENT));
1406 }
1407 /* check if generation number matches */
1408 if (sa_lookup(zp->z_sa_hdl, SA_ZPL_GEN(zfsvfs), &zp_gen,
1409 sizeof (zp_gen)) != 0) {
1410 zfs_zrele_async(zp);
1411 return (SET_ERROR(EIO));
1412 }
1413 if (zp_gen != gen) {
1414 zfs_zrele_async(zp);
1415 return (SET_ERROR(ENOENT));
1416 }
1417
1418 zgd = kmem_zalloc(sizeof (zgd_t), KM_SLEEP);
1419 zgd->zgd_lwb = lwb;
1420 zgd->zgd_private = zp;
1421
1422 /*
1423 * Write records come in two flavors: immediate and indirect.
1424 * For small writes it's cheaper to store the data with the
1425 * log record (immediate); for large writes it's cheaper to
1426 * sync the data and get a pointer to it (indirect) so that
1427 * we don't have to write the data twice.
1428 */
1429 if (buf != NULL) { /* immediate write */
1430 zgd->zgd_lr = zfs_rangelock_enter(&zp->z_rangelock, offset,
1431 size, RL_READER);
1432 /* test for truncation needs to be done while range locked */
1433 if (offset >= zp->z_size) {
1434 error = SET_ERROR(ENOENT);
1435 } else {
1436 error = dmu_read(os, object, offset, size, buf,
1437 DMU_READ_NO_PREFETCH | DMU_KEEP_CACHING);
1438 }
1439 ASSERT(error == 0 || error == ENOENT);
1440 } else { /* indirect write */
1441 ASSERT3P(zio, !=, NULL);
1442 /*
1443 * Have to lock the whole block to ensure when it's
1444 * written out and its checksum is being calculated
1445 * that no one can change the data. We need to re-check
1446 * blocksize after we get the lock in case it's changed!
1447 */
1448 for (;;) {
1449 uint64_t blkoff;
1450 size = zp->z_blksz;
1451 blkoff = ISP2(size) ? P2PHASE(offset, size) : offset;
1452 offset -= blkoff;
1453 zgd->zgd_lr = zfs_rangelock_enter(&zp->z_rangelock,
1454 offset, size, RL_READER);
1455 if (zp->z_blksz == size)
1456 break;
1457 offset += blkoff;
1458 zfs_rangelock_exit(zgd->zgd_lr);
1459 }
1460 /* test for truncation needs to be done while range locked */
1461 if (lr->lr_offset >= zp->z_size)
1462 error = SET_ERROR(ENOENT);
1463 #ifdef ZFS_DEBUG
1464 if (zil_fault_io) {
1465 error = SET_ERROR(EIO);
1466 zil_fault_io = 0;
1467 }
1468 #endif
1469
1470 dmu_buf_t *dbp;
1471 if (error == 0)
1472 error = dmu_buf_hold_noread(os, object, offset, zgd,
1473 &dbp);
1474
1475 if (error == 0) {
1476 zgd->zgd_db = dbp;
1477 dmu_buf_impl_t *db = (dmu_buf_impl_t *)dbp;
1478 boolean_t direct_write = B_FALSE;
1479 mutex_enter(&db->db_mtx);
1480 dbuf_dirty_record_t *dr =
1481 dbuf_find_dirty_eq(db, lr->lr_common.lrc_txg);
1482 if (dr != NULL && dr->dt.dl.dr_diowrite)
1483 direct_write = B_TRUE;
1484 mutex_exit(&db->db_mtx);
1485
1486 /*
1487 * All Direct I/O writes will have already completed and
1488 * the block pointer can be immediately stored in the
1489 * log record.
1490 */
1491 if (direct_write) {
1492 /*
1493 * A Direct I/O write always covers an entire
1494 * block.
1495 */
1496 ASSERT3U(dbp->db_size, ==, zp->z_blksz);
1497 lr->lr_blkptr = dr->dt.dl.dr_overridden_by;
1498 zfs_get_done(zgd, 0);
1499 return (0);
1500 }
1501
1502 blkptr_t *bp = &lr->lr_blkptr;
1503 zgd->zgd_bp = bp;
1504
1505 ASSERT3U(dbp->db_offset, ==, offset);
1506 ASSERT3U(dbp->db_size, ==, size);
1507
1508 error = dmu_sync(zio, lr->lr_common.lrc_txg,
1509 zfs_get_done, zgd);
1510 ASSERT(error || lr->lr_length <= size);
1511
1512 /*
1513 * On success, we need to wait for the write I/O
1514 * initiated by dmu_sync() to complete before we can
1515 * release this dbuf. We will finish everything up
1516 * in the zfs_get_done() callback.
1517 */
1518 if (error == 0)
1519 return (0);
1520
1521 if (error == EALREADY) {
1522 lr->lr_common.lrc_txtype = TX_WRITE2;
1523 /*
1524 * TX_WRITE2 relies on the data previously
1525 * written by the TX_WRITE that caused
1526 * EALREADY. We zero out the BP because
1527 * it is the old, currently-on-disk BP.
1528 */
1529 zgd->zgd_bp = NULL;
1530 BP_ZERO(bp);
1531 error = 0;
1532 }
1533 }
1534 }
1535
1536 zfs_get_done(zgd, error);
1537
1538 return (error);
1539 }
1540
1541 static void
zfs_get_done(zgd_t * zgd,int error)1542 zfs_get_done(zgd_t *zgd, int error)
1543 {
1544 (void) error;
1545 znode_t *zp = zgd->zgd_private;
1546
1547 if (zgd->zgd_db)
1548 dmu_buf_rele(zgd->zgd_db, zgd);
1549
1550 zfs_rangelock_exit(zgd->zgd_lr);
1551
1552 /*
1553 * Release the vnode asynchronously as we currently have the
1554 * txg stopped from syncing.
1555 */
1556 zfs_zrele_async(zp);
1557
1558 kmem_free(zgd, sizeof (zgd_t));
1559 }
1560
1561 static int
zfs_enter_two(zfsvfs_t * zfsvfs1,zfsvfs_t * zfsvfs2,const char * tag)1562 zfs_enter_two(zfsvfs_t *zfsvfs1, zfsvfs_t *zfsvfs2, const char *tag)
1563 {
1564 int error;
1565
1566 /* Swap. Not sure if the order of zfs_enter()s is important. */
1567 if (zfsvfs1 > zfsvfs2) {
1568 zfsvfs_t *tmpzfsvfs;
1569
1570 tmpzfsvfs = zfsvfs2;
1571 zfsvfs2 = zfsvfs1;
1572 zfsvfs1 = tmpzfsvfs;
1573 }
1574
1575 error = zfs_enter(zfsvfs1, tag);
1576 if (error != 0)
1577 return (error);
1578 if (zfsvfs1 != zfsvfs2) {
1579 error = zfs_enter(zfsvfs2, tag);
1580 if (error != 0) {
1581 zfs_exit(zfsvfs1, tag);
1582 return (error);
1583 }
1584 }
1585
1586 return (0);
1587 }
1588
1589 static void
zfs_exit_two(zfsvfs_t * zfsvfs1,zfsvfs_t * zfsvfs2,const char * tag)1590 zfs_exit_two(zfsvfs_t *zfsvfs1, zfsvfs_t *zfsvfs2, const char *tag)
1591 {
1592
1593 zfs_exit(zfsvfs1, tag);
1594 if (zfsvfs1 != zfsvfs2)
1595 zfs_exit(zfsvfs2, tag);
1596 }
1597
1598 /*
1599 * We split each clone request in chunks that can fit into a single ZIL
1600 * log entry. Each ZIL log entry can fit 130816 bytes for a block cloning
1601 * operation (see zil_max_log_data() and zfs_log_clone_range()). This gives
1602 * us room for storing 1022 block pointers.
1603 *
1604 * On success, the function return the number of bytes copied in *lenp.
1605 * Note, it doesn't return how much bytes are left to be copied.
1606 * On errors which are caused by any file system limitations or
1607 * brt limitations `EINVAL` is returned. In the most cases a user
1608 * requested bad parameters, it could be possible to clone the file but
1609 * some parameters don't match the requirements.
1610 */
1611 int
zfs_clone_range(znode_t * inzp,uint64_t * inoffp,znode_t * outzp,uint64_t * outoffp,uint64_t * lenp,cred_t * cr)1612 zfs_clone_range(znode_t *inzp, uint64_t *inoffp, znode_t *outzp,
1613 uint64_t *outoffp, uint64_t *lenp, cred_t *cr)
1614 {
1615 zfsvfs_t *inzfsvfs, *outzfsvfs;
1616 objset_t *inos, *outos;
1617 zfs_locked_range_t *inlr, *outlr;
1618 dmu_buf_impl_t *db;
1619 dmu_tx_t *tx;
1620 zilog_t *zilog;
1621 uint64_t inoff, outoff, len, done;
1622 uint64_t outsize, size;
1623 int error;
1624 int count = 0;
1625 sa_bulk_attr_t bulk[5];
1626 uint64_t mtime[2], ctime[2];
1627 uint64_t uid, gid, projid;
1628 blkptr_t *bps;
1629 size_t maxblocks, nbps;
1630 uint_t inblksz;
1631 uint64_t clear_setid_bits_txg = 0;
1632 uint64_t last_synced_txg = 0;
1633
1634 inoff = *inoffp;
1635 outoff = *outoffp;
1636 len = *lenp;
1637 done = 0;
1638
1639 inzfsvfs = ZTOZSB(inzp);
1640 outzfsvfs = ZTOZSB(outzp);
1641
1642 /*
1643 * We need to call zfs_enter() potentially on two different datasets,
1644 * so we need a dedicated function for that.
1645 */
1646 error = zfs_enter_two(inzfsvfs, outzfsvfs, FTAG);
1647 if (error != 0)
1648 return (error);
1649
1650 inos = inzfsvfs->z_os;
1651 outos = outzfsvfs->z_os;
1652
1653 /*
1654 * Both source and destination have to belong to the same storage pool.
1655 */
1656 if (dmu_objset_spa(inos) != dmu_objset_spa(outos)) {
1657 zfs_exit_two(inzfsvfs, outzfsvfs, FTAG);
1658 return (SET_ERROR(EXDEV));
1659 }
1660
1661 /*
1662 * outos and inos belongs to the same storage pool.
1663 * see a few lines above, only one check.
1664 */
1665 if (!spa_feature_is_enabled(dmu_objset_spa(outos),
1666 SPA_FEATURE_BLOCK_CLONING)) {
1667 zfs_exit_two(inzfsvfs, outzfsvfs, FTAG);
1668 return (SET_ERROR(EOPNOTSUPP));
1669 }
1670
1671 ASSERT(!outzfsvfs->z_replay);
1672
1673 /*
1674 * Block cloning from an unencrypted dataset into an encrypted
1675 * dataset and vice versa is not supported.
1676 */
1677 if (inos->os_encrypted != outos->os_encrypted) {
1678 zfs_exit_two(inzfsvfs, outzfsvfs, FTAG);
1679 return (SET_ERROR(EXDEV));
1680 }
1681
1682 /*
1683 * Cloning across encrypted datasets is possible only if they
1684 * share the same master key.
1685 */
1686 if (inos != outos && inos->os_encrypted &&
1687 !dmu_objset_crypto_key_equal(inos, outos)) {
1688 zfs_exit_two(inzfsvfs, outzfsvfs, FTAG);
1689 return (SET_ERROR(EXDEV));
1690 }
1691
1692 /*
1693 * Cloning between datasets with different properties is possible,
1694 * but it may cause confusions when copying data between them and
1695 * expecting new properties to apply.
1696 */
1697 if (zfs_bclone_strict_properties && inos != outos &&
1698 !inzfsvfs->z_issnap &&
1699 (inos->os_checksum != outos->os_checksum ||
1700 inos->os_compress != outos->os_compress ||
1701 inos->os_copies != outos->os_copies ||
1702 inos->os_dedup_checksum != outos->os_dedup_checksum)) {
1703 zfs_exit_two(inzfsvfs, outzfsvfs, FTAG);
1704 return (SET_ERROR(EXDEV));
1705 }
1706
1707 error = zfs_verify_zp(inzp);
1708 if (error == 0)
1709 error = zfs_verify_zp(outzp);
1710 if (error != 0) {
1711 zfs_exit_two(inzfsvfs, outzfsvfs, FTAG);
1712 return (error);
1713 }
1714
1715 /*
1716 * We don't copy source file's flags that's why we don't allow to clone
1717 * files that are in quarantine.
1718 */
1719 if (inzp->z_pflags & ZFS_AV_QUARANTINED) {
1720 zfs_exit_two(inzfsvfs, outzfsvfs, FTAG);
1721 return (SET_ERROR(EACCES));
1722 }
1723
1724 if (inoff >= inzp->z_size) {
1725 *lenp = 0;
1726 zfs_exit_two(inzfsvfs, outzfsvfs, FTAG);
1727 return (0);
1728 }
1729 if (len > inzp->z_size - inoff) {
1730 len = inzp->z_size - inoff;
1731 }
1732 if (len == 0) {
1733 *lenp = 0;
1734 zfs_exit_two(inzfsvfs, outzfsvfs, FTAG);
1735 return (0);
1736 }
1737
1738 /*
1739 * Callers might not be able to detect properly that we are read-only,
1740 * so check it explicitly here.
1741 */
1742 if (zfs_is_readonly(outzfsvfs)) {
1743 zfs_exit_two(inzfsvfs, outzfsvfs, FTAG);
1744 return (SET_ERROR(EROFS));
1745 }
1746
1747 /*
1748 * If immutable or not appending then return EPERM.
1749 * Intentionally allow ZFS_READONLY through here.
1750 * See zfs_zaccess_common()
1751 */
1752 if ((outzp->z_pflags & ZFS_IMMUTABLE) != 0) {
1753 zfs_exit_two(inzfsvfs, outzfsvfs, FTAG);
1754 return (SET_ERROR(EPERM));
1755 }
1756
1757 /*
1758 * No overlapping if we are cloning within the same file.
1759 */
1760 if (inzp == outzp) {
1761 if (inoff < outoff + len && outoff < inoff + len) {
1762 zfs_exit_two(inzfsvfs, outzfsvfs, FTAG);
1763 return (SET_ERROR(EINVAL));
1764 }
1765 }
1766
1767 /* Flush any mmap()'d data to disk */
1768 if (zn_has_cached_data(inzp, inoff, inoff + len - 1))
1769 zn_flush_cached_data(inzp, B_TRUE);
1770
1771 /*
1772 * Maintain predictable lock order.
1773 */
1774 if (inzp < outzp || (inzp == outzp && inoff < outoff)) {
1775 inlr = zfs_rangelock_enter(&inzp->z_rangelock, inoff, len,
1776 RL_READER);
1777 outlr = zfs_rangelock_enter(&outzp->z_rangelock, outoff, len,
1778 RL_WRITER);
1779 } else {
1780 outlr = zfs_rangelock_enter(&outzp->z_rangelock, outoff, len,
1781 RL_WRITER);
1782 inlr = zfs_rangelock_enter(&inzp->z_rangelock, inoff, len,
1783 RL_READER);
1784 }
1785
1786 inblksz = inzp->z_blksz;
1787
1788 /*
1789 * Cloning between datasets with different special_small_blocks would
1790 * bypass storage tier migration that would occur with a regular copy.
1791 */
1792 if (zfs_bclone_strict_properties && inos != outos &&
1793 !inzfsvfs->z_issnap && spa_has_special(dmu_objset_spa(inos))) {
1794 uint64_t in_smallblk = inos->os_zpl_special_smallblock;
1795 uint64_t out_smallblk = outos->os_zpl_special_smallblock;
1796 if (in_smallblk != out_smallblk) {
1797 uint64_t min_smallblk = MIN(in_smallblk, out_smallblk);
1798 uint64_t max_smallblk = MAX(in_smallblk, out_smallblk);
1799 if (min_smallblk < inblksz &&
1800 (inos->os_compress != ZIO_COMPRESS_OFF ||
1801 max_smallblk >= inblksz)) {
1802 error = SET_ERROR(EXDEV);
1803 goto unlock;
1804 }
1805 }
1806 }
1807
1808 /*
1809 * We cannot clone into a file with different block size if we can't
1810 * grow it (block size is already bigger, has more than one block, or
1811 * not locked for growth). There are other possible reasons for the
1812 * grow to fail, but we cover what we can before opening transaction
1813 * and the rest detect after we try to do it.
1814 */
1815 if (inblksz < outzp->z_blksz) {
1816 error = SET_ERROR(EINVAL);
1817 goto unlock;
1818 }
1819 if (inblksz != outzp->z_blksz && (outzp->z_size > outzp->z_blksz ||
1820 outlr->lr_length != UINT64_MAX)) {
1821 error = SET_ERROR(EINVAL);
1822 goto unlock;
1823 }
1824
1825 /*
1826 * Block size must be power-of-2 if destination offset != 0.
1827 * There can be no multiple blocks of non-power-of-2 size.
1828 */
1829 if (outoff != 0 && !ISP2(inblksz)) {
1830 error = SET_ERROR(EINVAL);
1831 goto unlock;
1832 }
1833
1834 /*
1835 * Offsets and len must be at block boundries.
1836 */
1837 if ((inoff % inblksz) != 0 || (outoff % inblksz) != 0) {
1838 error = SET_ERROR(EINVAL);
1839 goto unlock;
1840 }
1841 /*
1842 * Length must be multipe of blksz, except for the end of the file.
1843 */
1844 if ((len % inblksz) != 0 &&
1845 (len < inzp->z_size - inoff || len < outzp->z_size - outoff)) {
1846 error = SET_ERROR(EINVAL);
1847 goto unlock;
1848 }
1849
1850 /*
1851 * If we are copying only one block and it is smaller than recordsize
1852 * property, do not allow destination to grow beyond one block if it
1853 * is not there yet. Otherwise the destination will get stuck with
1854 * that block size forever, that can be as small as 512 bytes, no
1855 * matter how big the destination grow later.
1856 */
1857 if (len <= inblksz && inblksz < outzfsvfs->z_max_blksz &&
1858 outzp->z_size <= inblksz && outoff + len > inblksz) {
1859 error = SET_ERROR(EINVAL);
1860 goto unlock;
1861 }
1862
1863 error = zn_rlimit_fsize(outoff + len);
1864 if (error != 0) {
1865 goto unlock;
1866 }
1867
1868 if (inoff >= MAXOFFSET_T || outoff >= MAXOFFSET_T) {
1869 error = SET_ERROR(EFBIG);
1870 goto unlock;
1871 }
1872
1873 SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_MTIME(outzfsvfs), NULL,
1874 &mtime, 16);
1875 SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_CTIME(outzfsvfs), NULL,
1876 &ctime, 16);
1877 SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_SIZE(outzfsvfs), NULL,
1878 &outzp->z_size, 8);
1879 SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_FLAGS(outzfsvfs), NULL,
1880 &outzp->z_pflags, 8);
1881 if (outzp->z_is_sa)
1882 SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_SEQ(outzfsvfs), NULL,
1883 &outzp->z_seq, 8);
1884
1885 zilog = outzfsvfs->z_log;
1886 maxblocks = zil_max_log_data(zilog, sizeof (lr_clone_range_t)) /
1887 sizeof (bps[0]);
1888
1889 uid = KUID_TO_SUID(ZTOUID(outzp));
1890 gid = KGID_TO_SGID(ZTOGID(outzp));
1891 projid = outzp->z_projid;
1892
1893 bps = vmem_alloc(sizeof (bps[0]) * maxblocks, KM_SLEEP);
1894
1895 /*
1896 * Clone the file in reasonable size chunks. Each chunk is cloned
1897 * in a separate transaction; this keeps the intent log records small
1898 * and allows us to do more fine-grained space accounting.
1899 */
1900 while (len > 0) {
1901 size = MIN(inblksz * maxblocks, len);
1902
1903 if (zfs_id_overblockquota(outzfsvfs, DMU_USERUSED_OBJECT,
1904 uid) ||
1905 zfs_id_overblockquota(outzfsvfs, DMU_GROUPUSED_OBJECT,
1906 gid) ||
1907 (projid != ZFS_DEFAULT_PROJID &&
1908 zfs_id_overblockquota(outzfsvfs, DMU_PROJECTUSED_OBJECT,
1909 projid))) {
1910 error = SET_ERROR(EDQUOT);
1911 break;
1912 }
1913
1914 nbps = maxblocks;
1915 last_synced_txg = spa_last_synced_txg(dmu_objset_spa(inos));
1916 error = dmu_read_l0_bps(inos, inzp->z_id, inoff, size, bps,
1917 &nbps);
1918 if (error != 0) {
1919 /*
1920 * If we are trying to clone a block that was created
1921 * in the current transaction group, the error will be
1922 * EAGAIN here. Based on zfs_bclone_wait_dirty either
1923 * return a shortened range to the caller so it can
1924 * fallback, or wait for the next TXG and check again.
1925 */
1926 if (error == EAGAIN && zfs_bclone_wait_dirty) {
1927 txg_wait_flag_t wait_flags =
1928 spa_get_failmode(dmu_objset_spa(inos)) ==
1929 ZIO_FAILURE_MODE_CONTINUE ?
1930 TXG_WAIT_SUSPEND : 0;
1931 error = txg_wait_synced_flags(
1932 dmu_objset_pool(inos), last_synced_txg + 1,
1933 wait_flags);
1934 if (error == 0)
1935 continue;
1936 ASSERT3U(error, ==, ESHUTDOWN);
1937 error = SET_ERROR(EIO);
1938 }
1939
1940 break;
1941 }
1942
1943 /*
1944 * Start a transaction.
1945 */
1946 tx = dmu_tx_create(outos);
1947 dmu_tx_hold_sa(tx, outzp->z_sa_hdl, ZFS_SEQ_MAY_GROW(outzp));
1948 db = (dmu_buf_impl_t *)sa_get_db(outzp->z_sa_hdl);
1949 DB_DNODE_ENTER(db);
1950 dmu_tx_hold_clone_by_dnode(tx, DB_DNODE(db), outoff, size,
1951 inblksz);
1952 DB_DNODE_EXIT(db);
1953 zfs_sa_upgrade_txholds(tx, outzp);
1954 error = dmu_tx_assign(tx, DMU_TX_WAIT);
1955 if (error != 0) {
1956 dmu_tx_abort(tx);
1957 break;
1958 }
1959
1960 /*
1961 * Copy source znode's block size. This is done only if the
1962 * whole znode is locked (see zfs_rangelock_cb()) and only
1963 * on the first iteration since zfs_rangelock_reduce() will
1964 * shrink down lr_length to the appropriate size.
1965 */
1966 if (outlr->lr_length == UINT64_MAX) {
1967 zfs_grow_blocksize(outzp, inblksz, tx);
1968
1969 /*
1970 * Block growth may fail for many reasons we can not
1971 * predict here. If it happen the cloning is doomed.
1972 */
1973 if (inblksz != outzp->z_blksz) {
1974 error = SET_ERROR(EINVAL);
1975 dmu_tx_commit(tx);
1976 break;
1977 }
1978
1979 /*
1980 * Round range lock up to the block boundary, so we
1981 * prevent appends until we are done.
1982 */
1983 zfs_rangelock_reduce(outlr, outoff,
1984 ((len - 1) / inblksz + 1) * inblksz);
1985 }
1986
1987 error = dmu_brt_clone(outos, outzp->z_id, outoff, size, tx,
1988 bps, nbps);
1989 if (error != 0) {
1990 dmu_tx_commit(tx);
1991 break;
1992 }
1993
1994 if (zn_has_cached_data(outzp, outoff, outoff + size - 1)) {
1995 update_pages(outzp, outoff, size, outos);
1996 }
1997
1998 zfs_clear_setid_bits_if_necessary(outzfsvfs, outzp, cr,
1999 &clear_setid_bits_txg, tx);
2000
2001 zfs_tstamp_update_setup(outzp, CONTENT_MODIFIED, mtime, ctime);
2002 if (outzp->z_is_sa)
2003 outzp->z_has_seq = B_TRUE;
2004
2005 /*
2006 * Update the file size (zp_size) if it has changed;
2007 * account for possible concurrent updates.
2008 */
2009 while ((outsize = outzp->z_size) < outoff + size) {
2010 (void) atomic_cas_64(&outzp->z_size, outsize,
2011 outoff + size);
2012 }
2013
2014 ASSERT3S(count, <=, ARRAY_SIZE(bulk));
2015 error = sa_bulk_update(outzp->z_sa_hdl, bulk, count, tx);
2016
2017 zfs_log_clone_range(zilog, tx, TX_CLONE_RANGE, outzp, outoff,
2018 size, inblksz, bps, nbps);
2019
2020 dmu_tx_commit(tx);
2021
2022 if (error != 0)
2023 break;
2024
2025 inoff += size;
2026 outoff += size;
2027 len -= size;
2028 done += size;
2029
2030 if (issig()) {
2031 error = SET_ERROR(EINTR);
2032 break;
2033 }
2034 }
2035
2036 vmem_free(bps, sizeof (bps[0]) * maxblocks);
2037 zfs_znode_update_vfs(outzp);
2038
2039 unlock:
2040 zfs_rangelock_exit(outlr);
2041 zfs_rangelock_exit(inlr);
2042
2043 if (done > 0) {
2044 /*
2045 * If we have made at least partial progress, reset the error.
2046 */
2047 error = 0;
2048
2049 ZFS_ACCESSTIME_STAMP(inzfsvfs, inzp);
2050
2051 if (outos->os_sync == ZFS_SYNC_ALWAYS) {
2052 error = zil_commit(zilog, outzp->z_id);
2053 }
2054
2055 *inoffp += done;
2056 *outoffp += done;
2057 *lenp = done;
2058 } else {
2059 /*
2060 * If we made no progress, there must be a good reason.
2061 * EOF is handled explicitly above, before the loop.
2062 */
2063 ASSERT3S(error, !=, 0);
2064 }
2065
2066 zfs_exit_two(inzfsvfs, outzfsvfs, FTAG);
2067
2068 return (error);
2069 }
2070
2071 /*
2072 * Usual pattern would be to call zfs_clone_range() from zfs_replay_clone(),
2073 * but we cannot do that, because when replaying we don't have source znode
2074 * available. This is why we need a dedicated replay function.
2075 */
2076 int
zfs_clone_range_replay(znode_t * zp,uint64_t off,uint64_t len,uint64_t blksz,const blkptr_t * bps,size_t nbps)2077 zfs_clone_range_replay(znode_t *zp, uint64_t off, uint64_t len, uint64_t blksz,
2078 const blkptr_t *bps, size_t nbps)
2079 {
2080 zfsvfs_t *zfsvfs;
2081 dmu_buf_impl_t *db;
2082 dmu_tx_t *tx;
2083 int error;
2084 int count = 0;
2085 sa_bulk_attr_t bulk[5];
2086 uint64_t mtime[2], ctime[2];
2087
2088 ASSERT3U(off, <, MAXOFFSET_T);
2089 ASSERT3U(len, >, 0);
2090 ASSERT3U(nbps, >, 0);
2091
2092 zfsvfs = ZTOZSB(zp);
2093
2094 ASSERT(spa_feature_is_enabled(dmu_objset_spa(zfsvfs->z_os),
2095 SPA_FEATURE_BLOCK_CLONING));
2096
2097 if ((error = zfs_enter_verify_zp(zfsvfs, zp, FTAG)) != 0)
2098 return (error);
2099
2100 ASSERT(zfsvfs->z_replay);
2101 ASSERT(!zfs_is_readonly(zfsvfs));
2102
2103 if ((off % blksz) != 0) {
2104 zfs_exit(zfsvfs, FTAG);
2105 return (SET_ERROR(EINVAL));
2106 }
2107
2108 /*
2109 * Start a transaction.
2110 */
2111 tx = dmu_tx_create(zfsvfs->z_os);
2112
2113 dmu_tx_hold_sa(tx, zp->z_sa_hdl, ZFS_SEQ_MAY_GROW(zp));
2114 db = (dmu_buf_impl_t *)sa_get_db(zp->z_sa_hdl);
2115 DB_DNODE_ENTER(db);
2116 dmu_tx_hold_clone_by_dnode(tx, DB_DNODE(db), off, len, blksz);
2117 DB_DNODE_EXIT(db);
2118 zfs_sa_upgrade_txholds(tx, zp);
2119 error = dmu_tx_assign(tx, DMU_TX_WAIT);
2120 if (error != 0) {
2121 dmu_tx_abort(tx);
2122 zfs_exit(zfsvfs, FTAG);
2123 return (error);
2124 }
2125
2126 if (zp->z_blksz < blksz)
2127 zfs_grow_blocksize(zp, blksz, tx);
2128
2129 dmu_brt_clone(zfsvfs->z_os, zp->z_id, off, len, tx, bps, nbps);
2130
2131 SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_MTIME(zfsvfs), NULL, &mtime, 16);
2132 SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_CTIME(zfsvfs), NULL, &ctime, 16);
2133 SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_SIZE(zfsvfs), NULL,
2134 &zp->z_size, 8);
2135 SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_FLAGS(zfsvfs), NULL,
2136 &zp->z_pflags, 8);
2137 zfs_tstamp_update_setup(zp, CONTENT_MODIFIED, mtime, ctime);
2138 ZFS_PERSIST_SEQ(zp, bulk, count);
2139
2140 if (zp->z_size < off + len)
2141 zp->z_size = off + len;
2142
2143 ASSERT3S(count, <=, ARRAY_SIZE(bulk));
2144 error = sa_bulk_update(zp->z_sa_hdl, bulk, count, tx);
2145
2146 /*
2147 * zil_replaying() not only check if we are replaying ZIL, but also
2148 * updates the ZIL header to record replay progress.
2149 */
2150 VERIFY(zil_replaying(zfsvfs->z_log, tx));
2151
2152 dmu_tx_commit(tx);
2153
2154 zfs_znode_update_vfs(zp);
2155
2156 zfs_exit(zfsvfs, FTAG);
2157
2158 return (error);
2159 }
2160
2161 EXPORT_SYMBOL(zfs_access);
2162 EXPORT_SYMBOL(zfs_fsync);
2163 EXPORT_SYMBOL(zfs_holey);
2164 EXPORT_SYMBOL(zfs_read);
2165 EXPORT_SYMBOL(zfs_write);
2166 EXPORT_SYMBOL(zfs_getsecattr);
2167 EXPORT_SYMBOL(zfs_setsecattr);
2168 EXPORT_SYMBOL(zfs_clone_range);
2169 EXPORT_SYMBOL(zfs_clone_range_replay);
2170
2171 ZFS_MODULE_PARAM(zfs_vnops, zfs_vnops_, read_chunk_size, U64, ZMOD_RW,
2172 "Bytes to read per chunk");
2173
2174 ZFS_MODULE_PARAM(zfs, zfs_, bclone_enabled, INT, ZMOD_RW,
2175 "Enable block cloning");
2176
2177 ZFS_MODULE_PARAM(zfs, zfs_, bclone_strict_properties, INT, ZMOD_RW,
2178 "Restrict cross-dataset cloning with different properties");
2179
2180 ZFS_MODULE_PARAM(zfs, zfs_, bclone_wait_dirty, INT, ZMOD_RW,
2181 "Wait for dirty blocks when cloning");
2182
2183 ZFS_MODULE_PARAM(zfs, zfs_, dio_enabled, INT, ZMOD_RW,
2184 "Enable Direct I/O");
2185
2186 ZFS_MODULE_PARAM(zfs, zfs_, dio_strict, INT, ZMOD_RW,
2187 "Return errors on misaligned Direct I/O");
2188