xref: /freebsd/sys/contrib/openzfs/module/zfs/zfs_vnops.c (revision 7ab1a32cd43cbae61ad4dd435d6a482bbf61cb52)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or https://opensource.org/licenses/CDDL-1.0.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 
22 /*
23  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
24  * Copyright (c) 2012, 2018 by Delphix. All rights reserved.
25  * Copyright (c) 2015 by Chunwei Chen. All rights reserved.
26  * Copyright 2017 Nexenta Systems, Inc.
27  * Copyright (c) 2021, 2022 by Pawel Jakub Dawidek
28  */
29 
30 /* Portions Copyright 2007 Jeremy Teo */
31 /* Portions Copyright 2010 Robert Milkowski */
32 
33 #include <sys/types.h>
34 #include <sys/param.h>
35 #include <sys/time.h>
36 #include <sys/sysmacros.h>
37 #include <sys/vfs.h>
38 #include <sys/file.h>
39 #include <sys/stat.h>
40 #include <sys/kmem.h>
41 #include <sys/cmn_err.h>
42 #include <sys/errno.h>
43 #include <sys/zfs_dir.h>
44 #include <sys/zfs_acl.h>
45 #include <sys/zfs_ioctl.h>
46 #include <sys/fs/zfs.h>
47 #include <sys/dmu.h>
48 #include <sys/dmu_objset.h>
49 #include <sys/dsl_crypt.h>
50 #include <sys/spa.h>
51 #include <sys/txg.h>
52 #include <sys/dbuf.h>
53 #include <sys/policy.h>
54 #include <sys/zfeature.h>
55 #include <sys/zfs_vnops.h>
56 #include <sys/zfs_quota.h>
57 #include <sys/zfs_vfsops.h>
58 #include <sys/zfs_znode.h>
59 
60 /*
61  * Enable the experimental block cloning feature.  If this setting is 0, then
62  * even if feature@block_cloning is enabled, attempts to clone blocks will act
63  * as though the feature is disabled.
64  */
65 int zfs_bclone_enabled = 1;
66 
67 /*
68  * When set zfs_clone_range() waits for dirty data to be written to disk.
69  * This allows the clone operation to reliably succeed when a file is modified
70  * and then immediately cloned. For small files this may be slower than making
71  * a copy of the file and is therefore not the default.  However, in certain
72  * scenarios this behavior may be desirable so a tunable is provided.
73  */
74 static int zfs_bclone_wait_dirty = 0;
75 
76 /*
77  * Enable Direct I/O. If this setting is 0, then all I/O requests will be
78  * directed through the ARC acting as though the dataset property direct was
79  * set to disabled.
80  *
81  * Disabled by default on FreeBSD until a potential range locking issue in
82  * zfs_getpages() can be resolved.
83  */
84 #ifdef __FreeBSD__
85 static int zfs_dio_enabled = 0;
86 #else
87 static int zfs_dio_enabled = 1;
88 #endif
89 
90 
91 /*
92  * Maximum bytes to read per chunk in zfs_read().
93  */
94 static uint64_t zfs_vnops_read_chunk_size = 1024 * 1024;
95 
96 int
97 zfs_fsync(znode_t *zp, int syncflag, cred_t *cr)
98 {
99 	int error = 0;
100 	zfsvfs_t *zfsvfs = ZTOZSB(zp);
101 
102 	if (zfsvfs->z_os->os_sync != ZFS_SYNC_DISABLED) {
103 		if ((error = zfs_enter_verify_zp(zfsvfs, zp, FTAG)) != 0)
104 			return (error);
105 		atomic_inc_32(&zp->z_sync_writes_cnt);
106 		zil_commit(zfsvfs->z_log, zp->z_id);
107 		atomic_dec_32(&zp->z_sync_writes_cnt);
108 		zfs_exit(zfsvfs, FTAG);
109 	}
110 	return (error);
111 }
112 
113 
114 #if defined(SEEK_HOLE) && defined(SEEK_DATA)
115 /*
116  * Lseek support for finding holes (cmd == SEEK_HOLE) and
117  * data (cmd == SEEK_DATA). "off" is an in/out parameter.
118  */
119 static int
120 zfs_holey_common(znode_t *zp, ulong_t cmd, loff_t *off)
121 {
122 	zfs_locked_range_t *lr;
123 	uint64_t noff = (uint64_t)*off; /* new offset */
124 	uint64_t file_sz;
125 	int error;
126 	boolean_t hole;
127 
128 	file_sz = zp->z_size;
129 	if (noff >= file_sz)  {
130 		return (SET_ERROR(ENXIO));
131 	}
132 
133 	if (cmd == F_SEEK_HOLE)
134 		hole = B_TRUE;
135 	else
136 		hole = B_FALSE;
137 
138 	/* Flush any mmap()'d data to disk */
139 	if (zn_has_cached_data(zp, 0, file_sz - 1))
140 		zn_flush_cached_data(zp, B_TRUE);
141 
142 	lr = zfs_rangelock_enter(&zp->z_rangelock, 0, UINT64_MAX, RL_READER);
143 	error = dmu_offset_next(ZTOZSB(zp)->z_os, zp->z_id, hole, &noff);
144 	zfs_rangelock_exit(lr);
145 
146 	if (error == ESRCH)
147 		return (SET_ERROR(ENXIO));
148 
149 	/* File was dirty, so fall back to using generic logic */
150 	if (error == EBUSY) {
151 		if (hole)
152 			*off = file_sz;
153 
154 		return (0);
155 	}
156 
157 	/*
158 	 * We could find a hole that begins after the logical end-of-file,
159 	 * because dmu_offset_next() only works on whole blocks.  If the
160 	 * EOF falls mid-block, then indicate that the "virtual hole"
161 	 * at the end of the file begins at the logical EOF, rather than
162 	 * at the end of the last block.
163 	 */
164 	if (noff > file_sz) {
165 		ASSERT(hole);
166 		noff = file_sz;
167 	}
168 
169 	if (noff < *off)
170 		return (error);
171 	*off = noff;
172 	return (error);
173 }
174 
175 int
176 zfs_holey(znode_t *zp, ulong_t cmd, loff_t *off)
177 {
178 	zfsvfs_t *zfsvfs = ZTOZSB(zp);
179 	int error;
180 
181 	if ((error = zfs_enter_verify_zp(zfsvfs, zp, FTAG)) != 0)
182 		return (error);
183 
184 	error = zfs_holey_common(zp, cmd, off);
185 
186 	zfs_exit(zfsvfs, FTAG);
187 	return (error);
188 }
189 #endif /* SEEK_HOLE && SEEK_DATA */
190 
191 int
192 zfs_access(znode_t *zp, int mode, int flag, cred_t *cr)
193 {
194 	zfsvfs_t *zfsvfs = ZTOZSB(zp);
195 	int error;
196 
197 	if ((error = zfs_enter_verify_zp(zfsvfs, zp, FTAG)) != 0)
198 		return (error);
199 
200 	if (flag & V_ACE_MASK)
201 #if defined(__linux__)
202 		error = zfs_zaccess(zp, mode, flag, B_FALSE, cr,
203 		    zfs_init_idmap);
204 #else
205 		error = zfs_zaccess(zp, mode, flag, B_FALSE, cr,
206 		    NULL);
207 #endif
208 	else
209 #if defined(__linux__)
210 		error = zfs_zaccess_rwx(zp, mode, flag, cr, zfs_init_idmap);
211 #else
212 		error = zfs_zaccess_rwx(zp, mode, flag, cr, NULL);
213 #endif
214 
215 	zfs_exit(zfsvfs, FTAG);
216 	return (error);
217 }
218 
219 /*
220  * Determine if Direct I/O has been requested (either via the O_DIRECT flag or
221  * the "direct" dataset property). When inherited by the property only apply
222  * the O_DIRECT flag to correctly aligned IO requests. The rational for this
223  * is it allows the property to be safely set on a dataset without forcing
224  * all of the applications to be aware of the alignment restrictions. When
225  * O_DIRECT is explicitly requested by an application return EINVAL if the
226  * request is unaligned.  In all cases, if the range for this request has
227  * been mmap'ed then we will perform buffered I/O to keep the mapped region
228  * synhronized with the ARC.
229  *
230  * It is possible that a file's pages could be mmap'ed after it is checked
231  * here. If so, that is handled coorarding in zfs_write(). See comments in the
232  * following area for how this is handled:
233  * zfs_write() -> update_pages()
234  */
235 static int
236 zfs_setup_direct(struct znode *zp, zfs_uio_t *uio, zfs_uio_rw_t rw,
237     int *ioflagp)
238 {
239 	zfsvfs_t *zfsvfs = ZTOZSB(zp);
240 	objset_t *os = zfsvfs->z_os;
241 	int ioflag = *ioflagp;
242 	int error = 0;
243 
244 	if (!zfs_dio_enabled || os->os_direct == ZFS_DIRECT_DISABLED ||
245 	    zn_has_cached_data(zp, zfs_uio_offset(uio),
246 	    zfs_uio_offset(uio) + zfs_uio_resid(uio) - 1)) {
247 		/*
248 		 * Direct I/O is disabled or the region is mmap'ed. In either
249 		 * case the I/O request will just directed through the ARC.
250 		 */
251 		ioflag &= ~O_DIRECT;
252 		goto out;
253 	} else if (os->os_direct == ZFS_DIRECT_ALWAYS &&
254 	    zfs_uio_page_aligned(uio) &&
255 	    zfs_uio_aligned(uio, PAGE_SIZE)) {
256 		if ((rw == UIO_WRITE && zfs_uio_resid(uio) >= zp->z_blksz) ||
257 		    (rw == UIO_READ)) {
258 			ioflag |= O_DIRECT;
259 		}
260 	} else if (os->os_direct == ZFS_DIRECT_ALWAYS && (ioflag & O_DIRECT)) {
261 		/*
262 		 * Direct I/O was requested through the direct=always, but it
263 		 * is not properly PAGE_SIZE aligned. The request will be
264 		 * directed through the ARC.
265 		 */
266 		ioflag &= ~O_DIRECT;
267 	}
268 
269 	if (ioflag & O_DIRECT) {
270 		if (!zfs_uio_page_aligned(uio) ||
271 		    !zfs_uio_aligned(uio, PAGE_SIZE)) {
272 			error = SET_ERROR(EINVAL);
273 			goto out;
274 		}
275 
276 		error = zfs_uio_get_dio_pages_alloc(uio, rw);
277 		if (error) {
278 			goto out;
279 		}
280 	}
281 
282 	IMPLY(ioflag & O_DIRECT, uio->uio_extflg & UIO_DIRECT);
283 	ASSERT0(error);
284 
285 out:
286 	*ioflagp = ioflag;
287 	return (error);
288 }
289 
290 /*
291  * Read bytes from specified file into supplied buffer.
292  *
293  *	IN:	zp	- inode of file to be read from.
294  *		uio	- structure supplying read location, range info,
295  *			  and return buffer.
296  *		ioflag	- O_SYNC flags; used to provide FRSYNC semantics.
297  *			  O_DIRECT flag; used to bypass page cache.
298  *		cr	- credentials of caller.
299  *
300  *	OUT:	uio	- updated offset and range, buffer filled.
301  *
302  *	RETURN:	0 on success, error code on failure.
303  *
304  * Side Effects:
305  *	inode - atime updated if byte count > 0
306  */
307 int
308 zfs_read(struct znode *zp, zfs_uio_t *uio, int ioflag, cred_t *cr)
309 {
310 	(void) cr;
311 	int error = 0;
312 	boolean_t frsync = B_FALSE;
313 	boolean_t dio_checksum_failure = B_FALSE;
314 
315 	zfsvfs_t *zfsvfs = ZTOZSB(zp);
316 	if ((error = zfs_enter_verify_zp(zfsvfs, zp, FTAG)) != 0)
317 		return (error);
318 
319 	if (zp->z_pflags & ZFS_AV_QUARANTINED) {
320 		zfs_exit(zfsvfs, FTAG);
321 		return (SET_ERROR(EACCES));
322 	}
323 
324 	/* We don't copy out anything useful for directories. */
325 	if (Z_ISDIR(ZTOTYPE(zp))) {
326 		zfs_exit(zfsvfs, FTAG);
327 		return (SET_ERROR(EISDIR));
328 	}
329 
330 	/*
331 	 * Validate file offset
332 	 */
333 	if (zfs_uio_offset(uio) < (offset_t)0) {
334 		zfs_exit(zfsvfs, FTAG);
335 		return (SET_ERROR(EINVAL));
336 	}
337 
338 	/*
339 	 * Fasttrack empty reads
340 	 */
341 	if (zfs_uio_resid(uio) == 0) {
342 		zfs_exit(zfsvfs, FTAG);
343 		return (0);
344 	}
345 
346 #ifdef FRSYNC
347 	/*
348 	 * If we're in FRSYNC mode, sync out this znode before reading it.
349 	 * Only do this for non-snapshots.
350 	 *
351 	 * Some platforms do not support FRSYNC and instead map it
352 	 * to O_SYNC, which results in unnecessary calls to zil_commit. We
353 	 * only honor FRSYNC requests on platforms which support it.
354 	 */
355 	frsync = !!(ioflag & FRSYNC);
356 #endif
357 	if (zfsvfs->z_log &&
358 	    (frsync || zfsvfs->z_os->os_sync == ZFS_SYNC_ALWAYS))
359 		zil_commit(zfsvfs->z_log, zp->z_id);
360 
361 	/*
362 	 * Lock the range against changes.
363 	 */
364 	zfs_locked_range_t *lr = zfs_rangelock_enter(&zp->z_rangelock,
365 	    zfs_uio_offset(uio), zfs_uio_resid(uio), RL_READER);
366 
367 	/*
368 	 * If we are reading past end-of-file we can skip
369 	 * to the end; but we might still need to set atime.
370 	 */
371 	if (zfs_uio_offset(uio) >= zp->z_size) {
372 		error = 0;
373 		goto out;
374 	}
375 	ASSERT(zfs_uio_offset(uio) < zp->z_size);
376 
377 	/*
378 	 * Setting up Direct I/O if requested.
379 	 */
380 	error = zfs_setup_direct(zp, uio, UIO_READ, &ioflag);
381 	if (error) {
382 		goto out;
383 	}
384 
385 #if defined(__linux__)
386 	ssize_t start_offset = zfs_uio_offset(uio);
387 #endif
388 	ssize_t chunk_size = zfs_vnops_read_chunk_size;
389 	ssize_t n = MIN(zfs_uio_resid(uio), zp->z_size - zfs_uio_offset(uio));
390 	ssize_t start_resid = n;
391 	ssize_t dio_remaining_resid = 0;
392 
393 	if (uio->uio_extflg & UIO_DIRECT) {
394 		/*
395 		 * All pages for an O_DIRECT request ahve already been mapped
396 		 * so there's no compelling reason to handle this uio in
397 		 * smaller chunks.
398 		 */
399 		chunk_size = DMU_MAX_ACCESS;
400 
401 		/*
402 		 * In the event that the O_DIRECT request is reading the entire
403 		 * file, it is possible file's length is not page sized
404 		 * aligned. However, lower layers expect that the Direct I/O
405 		 * request is page-aligned. In this case, as much of the file
406 		 * that can be read using Direct I/O happens and the remaining
407 		 * amount will be read through the ARC.
408 		 *
409 		 * This is still consistent with the semantics of Direct I/O in
410 		 * ZFS as at a minimum the I/O request must be page-aligned.
411 		 */
412 		dio_remaining_resid = n - P2ALIGN_TYPED(n, PAGE_SIZE, ssize_t);
413 		if (dio_remaining_resid != 0)
414 			n -= dio_remaining_resid;
415 	}
416 
417 	while (n > 0) {
418 		ssize_t nbytes = MIN(n, chunk_size -
419 		    P2PHASE(zfs_uio_offset(uio), chunk_size));
420 #ifdef UIO_NOCOPY
421 		if (zfs_uio_segflg(uio) == UIO_NOCOPY)
422 			error = mappedread_sf(zp, nbytes, uio);
423 		else
424 #endif
425 		if (zn_has_cached_data(zp, zfs_uio_offset(uio),
426 		    zfs_uio_offset(uio) + nbytes - 1)) {
427 			error = mappedread(zp, nbytes, uio);
428 		} else {
429 			error = dmu_read_uio_dbuf(sa_get_db(zp->z_sa_hdl),
430 			    uio, nbytes);
431 		}
432 
433 		if (error) {
434 			/* convert checksum errors into IO errors */
435 			if (error == ECKSUM) {
436 				/*
437 				 * If a Direct I/O read returned a checksum
438 				 * verify error, then it must be treated as
439 				 * suspicious. The contents of the buffer could
440 				 * have beeen manipulated while the I/O was in
441 				 * flight. In this case, the remainder of I/O
442 				 * request will just be reissued through the
443 				 * ARC.
444 				 */
445 				if (uio->uio_extflg & UIO_DIRECT) {
446 					dio_checksum_failure = B_TRUE;
447 					uio->uio_extflg &= ~UIO_DIRECT;
448 					n += dio_remaining_resid;
449 					dio_remaining_resid = 0;
450 					continue;
451 				} else {
452 					error = SET_ERROR(EIO);
453 				}
454 			}
455 
456 #if defined(__linux__)
457 			/*
458 			 * if we actually read some bytes, bubbling EFAULT
459 			 * up to become EAGAIN isn't what we want here...
460 			 *
461 			 * ...on Linux, at least. On FBSD, doing this breaks.
462 			 */
463 			if (error == EFAULT &&
464 			    (zfs_uio_offset(uio) - start_offset) != 0)
465 				error = 0;
466 #endif
467 			break;
468 		}
469 
470 		n -= nbytes;
471 	}
472 
473 	if (error == 0 && (uio->uio_extflg & UIO_DIRECT) &&
474 	    dio_remaining_resid != 0) {
475 		/*
476 		 * Temporarily remove the UIO_DIRECT flag from the UIO so the
477 		 * remainder of the file can be read using the ARC.
478 		 */
479 		uio->uio_extflg &= ~UIO_DIRECT;
480 
481 		if (zn_has_cached_data(zp, zfs_uio_offset(uio),
482 		    zfs_uio_offset(uio) + dio_remaining_resid - 1)) {
483 			error = mappedread(zp, dio_remaining_resid, uio);
484 		} else {
485 			error = dmu_read_uio_dbuf(sa_get_db(zp->z_sa_hdl), uio,
486 			    dio_remaining_resid);
487 		}
488 		uio->uio_extflg |= UIO_DIRECT;
489 
490 		if (error != 0)
491 			n += dio_remaining_resid;
492 	} else if (error && (uio->uio_extflg & UIO_DIRECT)) {
493 		n += dio_remaining_resid;
494 	}
495 	int64_t nread = start_resid - n;
496 
497 	dataset_kstats_update_read_kstats(&zfsvfs->z_kstat, nread);
498 out:
499 	zfs_rangelock_exit(lr);
500 
501 	if (dio_checksum_failure == B_TRUE)
502 		uio->uio_extflg |= UIO_DIRECT;
503 
504 	/*
505 	 * Cleanup for Direct I/O if requested.
506 	 */
507 	if (uio->uio_extflg & UIO_DIRECT)
508 		zfs_uio_free_dio_pages(uio, UIO_READ);
509 
510 	ZFS_ACCESSTIME_STAMP(zfsvfs, zp);
511 	zfs_exit(zfsvfs, FTAG);
512 	return (error);
513 }
514 
515 static void
516 zfs_clear_setid_bits_if_necessary(zfsvfs_t *zfsvfs, znode_t *zp, cred_t *cr,
517     uint64_t *clear_setid_bits_txgp, dmu_tx_t *tx)
518 {
519 	zilog_t *zilog = zfsvfs->z_log;
520 	const uint64_t uid = KUID_TO_SUID(ZTOUID(zp));
521 
522 	ASSERT(clear_setid_bits_txgp != NULL);
523 	ASSERT(tx != NULL);
524 
525 	/*
526 	 * Clear Set-UID/Set-GID bits on successful write if not
527 	 * privileged and at least one of the execute bits is set.
528 	 *
529 	 * It would be nice to do this after all writes have
530 	 * been done, but that would still expose the ISUID/ISGID
531 	 * to another app after the partial write is committed.
532 	 *
533 	 * Note: we don't call zfs_fuid_map_id() here because
534 	 * user 0 is not an ephemeral uid.
535 	 */
536 	mutex_enter(&zp->z_acl_lock);
537 	if ((zp->z_mode & (S_IXUSR | (S_IXUSR >> 3) | (S_IXUSR >> 6))) != 0 &&
538 	    (zp->z_mode & (S_ISUID | S_ISGID)) != 0 &&
539 	    secpolicy_vnode_setid_retain(zp, cr,
540 	    ((zp->z_mode & S_ISUID) != 0 && uid == 0)) != 0) {
541 		uint64_t newmode;
542 
543 		zp->z_mode &= ~(S_ISUID | S_ISGID);
544 		newmode = zp->z_mode;
545 		(void) sa_update(zp->z_sa_hdl, SA_ZPL_MODE(zfsvfs),
546 		    (void *)&newmode, sizeof (uint64_t), tx);
547 
548 		mutex_exit(&zp->z_acl_lock);
549 
550 		/*
551 		 * Make sure SUID/SGID bits will be removed when we replay the
552 		 * log. If the setid bits are keep coming back, don't log more
553 		 * than one TX_SETATTR per transaction group.
554 		 */
555 		if (*clear_setid_bits_txgp != dmu_tx_get_txg(tx)) {
556 			vattr_t va = {0};
557 
558 			va.va_mask = ATTR_MODE;
559 			va.va_nodeid = zp->z_id;
560 			va.va_mode = newmode;
561 			zfs_log_setattr(zilog, tx, TX_SETATTR, zp, &va,
562 			    ATTR_MODE, NULL);
563 			*clear_setid_bits_txgp = dmu_tx_get_txg(tx);
564 		}
565 	} else {
566 		mutex_exit(&zp->z_acl_lock);
567 	}
568 }
569 
570 /*
571  * Write the bytes to a file.
572  *
573  *	IN:	zp	- znode of file to be written to.
574  *		uio	- structure supplying write location, range info,
575  *			  and data buffer.
576  *		ioflag	- O_APPEND flag set if in append mode.
577  *			  O_DIRECT flag; used to bypass page cache.
578  *		cr	- credentials of caller.
579  *
580  *	OUT:	uio	- updated offset and range.
581  *
582  *	RETURN:	0 if success
583  *		error code if failure
584  *
585  * Timestamps:
586  *	ip - ctime|mtime updated if byte count > 0
587  */
588 int
589 zfs_write(znode_t *zp, zfs_uio_t *uio, int ioflag, cred_t *cr)
590 {
591 	int error = 0, error1;
592 	ssize_t start_resid = zfs_uio_resid(uio);
593 	uint64_t clear_setid_bits_txg = 0;
594 	boolean_t o_direct_defer = B_FALSE;
595 
596 	/*
597 	 * Fasttrack empty write
598 	 */
599 	ssize_t n = start_resid;
600 	if (n == 0)
601 		return (0);
602 
603 	zfsvfs_t *zfsvfs = ZTOZSB(zp);
604 	if ((error = zfs_enter_verify_zp(zfsvfs, zp, FTAG)) != 0)
605 		return (error);
606 
607 	sa_bulk_attr_t bulk[4];
608 	int count = 0;
609 	uint64_t mtime[2], ctime[2];
610 	SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_MTIME(zfsvfs), NULL, &mtime, 16);
611 	SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_CTIME(zfsvfs), NULL, &ctime, 16);
612 	SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_SIZE(zfsvfs), NULL,
613 	    &zp->z_size, 8);
614 	SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_FLAGS(zfsvfs), NULL,
615 	    &zp->z_pflags, 8);
616 
617 	/*
618 	 * Callers might not be able to detect properly that we are read-only,
619 	 * so check it explicitly here.
620 	 */
621 	if (zfs_is_readonly(zfsvfs)) {
622 		zfs_exit(zfsvfs, FTAG);
623 		return (SET_ERROR(EROFS));
624 	}
625 
626 	/*
627 	 * If immutable or not appending then return EPERM.
628 	 * Intentionally allow ZFS_READONLY through here.
629 	 * See zfs_zaccess_common()
630 	 */
631 	if ((zp->z_pflags & ZFS_IMMUTABLE) ||
632 	    ((zp->z_pflags & ZFS_APPENDONLY) && !(ioflag & O_APPEND) &&
633 	    (zfs_uio_offset(uio) < zp->z_size))) {
634 		zfs_exit(zfsvfs, FTAG);
635 		return (SET_ERROR(EPERM));
636 	}
637 
638 	/*
639 	 * Validate file offset
640 	 */
641 	offset_t woff = ioflag & O_APPEND ? zp->z_size : zfs_uio_offset(uio);
642 	if (woff < 0) {
643 		zfs_exit(zfsvfs, FTAG);
644 		return (SET_ERROR(EINVAL));
645 	}
646 
647 	/*
648 	 * Setting up Direct I/O if requested.
649 	 */
650 	error = zfs_setup_direct(zp, uio, UIO_WRITE, &ioflag);
651 	if (error) {
652 		zfs_exit(zfsvfs, FTAG);
653 		return (SET_ERROR(error));
654 	}
655 
656 	/*
657 	 * Pre-fault the pages to ensure slow (eg NFS) pages
658 	 * don't hold up txg.
659 	 */
660 	ssize_t pfbytes = MIN(n, DMU_MAX_ACCESS >> 1);
661 	if (zfs_uio_prefaultpages(pfbytes, uio)) {
662 		zfs_exit(zfsvfs, FTAG);
663 		return (SET_ERROR(EFAULT));
664 	}
665 
666 	/*
667 	 * If in append mode, set the io offset pointer to eof.
668 	 */
669 	zfs_locked_range_t *lr;
670 	if (ioflag & O_APPEND) {
671 		/*
672 		 * Obtain an appending range lock to guarantee file append
673 		 * semantics.  We reset the write offset once we have the lock.
674 		 */
675 		lr = zfs_rangelock_enter(&zp->z_rangelock, 0, n, RL_APPEND);
676 		woff = lr->lr_offset;
677 		if (lr->lr_length == UINT64_MAX) {
678 			/*
679 			 * We overlocked the file because this write will cause
680 			 * the file block size to increase.
681 			 * Note that zp_size cannot change with this lock held.
682 			 */
683 			woff = zp->z_size;
684 		}
685 		zfs_uio_setoffset(uio, woff);
686 		/*
687 		 * We need to update the starting offset as well because it is
688 		 * set previously in the ZPL (Linux) and VNOPS (FreeBSD)
689 		 * layers.
690 		 */
691 		zfs_uio_setsoffset(uio, woff);
692 	} else {
693 		/*
694 		 * Note that if the file block size will change as a result of
695 		 * this write, then this range lock will lock the entire file
696 		 * so that we can re-write the block safely.
697 		 */
698 		lr = zfs_rangelock_enter(&zp->z_rangelock, woff, n, RL_WRITER);
699 	}
700 
701 	if (zn_rlimit_fsize_uio(zp, uio)) {
702 		zfs_rangelock_exit(lr);
703 		zfs_exit(zfsvfs, FTAG);
704 		return (SET_ERROR(EFBIG));
705 	}
706 
707 	const rlim64_t limit = MAXOFFSET_T;
708 
709 	if (woff >= limit) {
710 		zfs_rangelock_exit(lr);
711 		zfs_exit(zfsvfs, FTAG);
712 		return (SET_ERROR(EFBIG));
713 	}
714 
715 	if (n > limit - woff)
716 		n = limit - woff;
717 
718 	uint64_t end_size = MAX(zp->z_size, woff + n);
719 	zilog_t *zilog = zfsvfs->z_log;
720 	boolean_t commit = (ioflag & (O_SYNC | O_DSYNC)) ||
721 	    (zfsvfs->z_os->os_sync == ZFS_SYNC_ALWAYS);
722 
723 	const uint64_t uid = KUID_TO_SUID(ZTOUID(zp));
724 	const uint64_t gid = KGID_TO_SGID(ZTOGID(zp));
725 	const uint64_t projid = zp->z_projid;
726 
727 	/*
728 	 * In the event we are increasing the file block size
729 	 * (lr_length == UINT64_MAX), we will direct the write to the ARC.
730 	 * Because zfs_grow_blocksize() will read from the ARC in order to
731 	 * grow the dbuf, we avoid doing Direct I/O here as that would cause
732 	 * data written to disk to be overwritten by data in the ARC during
733 	 * the sync phase. Besides writing data twice to disk, we also
734 	 * want to avoid consistency concerns between data in the the ARC and
735 	 * on disk while growing the file's blocksize.
736 	 *
737 	 * We will only temporarily remove Direct I/O and put it back after
738 	 * we have grown the blocksize. We do this in the event a request
739 	 * is larger than max_blksz, so further requests to
740 	 * dmu_write_uio_dbuf() will still issue the requests using Direct
741 	 * IO.
742 	 *
743 	 * As an example:
744 	 * The first block to file is being written as a 4k request with
745 	 * a recorsize of 1K. The first 1K issued in the loop below will go
746 	 * through the ARC; however, the following 3 1K requests will
747 	 * use Direct I/O.
748 	 */
749 	if (uio->uio_extflg & UIO_DIRECT && lr->lr_length == UINT64_MAX) {
750 		uio->uio_extflg &= ~UIO_DIRECT;
751 		o_direct_defer = B_TRUE;
752 	}
753 
754 	/*
755 	 * Write the file in reasonable size chunks.  Each chunk is written
756 	 * in a separate transaction; this keeps the intent log records small
757 	 * and allows us to do more fine-grained space accounting.
758 	 */
759 	while (n > 0) {
760 		woff = zfs_uio_offset(uio);
761 
762 		if (zfs_id_overblockquota(zfsvfs, DMU_USERUSED_OBJECT, uid) ||
763 		    zfs_id_overblockquota(zfsvfs, DMU_GROUPUSED_OBJECT, gid) ||
764 		    (projid != ZFS_DEFAULT_PROJID &&
765 		    zfs_id_overblockquota(zfsvfs, DMU_PROJECTUSED_OBJECT,
766 		    projid))) {
767 			error = SET_ERROR(EDQUOT);
768 			break;
769 		}
770 
771 		uint64_t blksz;
772 		if (lr->lr_length == UINT64_MAX && zp->z_size <= zp->z_blksz) {
773 			if (zp->z_blksz > zfsvfs->z_max_blksz &&
774 			    !ISP2(zp->z_blksz)) {
775 				/*
776 				 * File's blocksize is already larger than the
777 				 * "recordsize" property.  Only let it grow to
778 				 * the next power of 2.
779 				 */
780 				blksz = 1 << highbit64(zp->z_blksz);
781 			} else {
782 				blksz = zfsvfs->z_max_blksz;
783 			}
784 			blksz = MIN(blksz, P2ROUNDUP(end_size,
785 			    SPA_MINBLOCKSIZE));
786 			blksz = MAX(blksz, zp->z_blksz);
787 		} else {
788 			blksz = zp->z_blksz;
789 		}
790 
791 		arc_buf_t *abuf = NULL;
792 		ssize_t nbytes = n;
793 		if (n >= blksz && woff >= zp->z_size &&
794 		    P2PHASE(woff, blksz) == 0 &&
795 		    !(uio->uio_extflg & UIO_DIRECT) &&
796 		    (blksz >= SPA_OLD_MAXBLOCKSIZE || n < 4 * blksz)) {
797 			/*
798 			 * This write covers a full block.  "Borrow" a buffer
799 			 * from the dmu so that we can fill it before we enter
800 			 * a transaction.  This avoids the possibility of
801 			 * holding up the transaction if the data copy hangs
802 			 * up on a pagefault (e.g., from an NFS server mapping).
803 			 */
804 			abuf = dmu_request_arcbuf(sa_get_db(zp->z_sa_hdl),
805 			    blksz);
806 			ASSERT(abuf != NULL);
807 			ASSERT(arc_buf_size(abuf) == blksz);
808 			if ((error = zfs_uiocopy(abuf->b_data, blksz,
809 			    UIO_WRITE, uio, &nbytes))) {
810 				dmu_return_arcbuf(abuf);
811 				break;
812 			}
813 			ASSERT3S(nbytes, ==, blksz);
814 		} else {
815 			nbytes = MIN(n, (DMU_MAX_ACCESS >> 1) -
816 			    P2PHASE(woff, blksz));
817 			if (pfbytes < nbytes) {
818 				if (zfs_uio_prefaultpages(nbytes, uio)) {
819 					error = SET_ERROR(EFAULT);
820 					break;
821 				}
822 				pfbytes = nbytes;
823 			}
824 		}
825 
826 		/*
827 		 * Start a transaction.
828 		 */
829 		dmu_tx_t *tx = dmu_tx_create(zfsvfs->z_os);
830 		dmu_tx_hold_sa(tx, zp->z_sa_hdl, B_FALSE);
831 		dmu_buf_impl_t *db = (dmu_buf_impl_t *)sa_get_db(zp->z_sa_hdl);
832 		DB_DNODE_ENTER(db);
833 		dmu_tx_hold_write_by_dnode(tx, DB_DNODE(db), woff, nbytes);
834 		DB_DNODE_EXIT(db);
835 		zfs_sa_upgrade_txholds(tx, zp);
836 		error = dmu_tx_assign(tx, TXG_WAIT);
837 		if (error) {
838 			dmu_tx_abort(tx);
839 			if (abuf != NULL)
840 				dmu_return_arcbuf(abuf);
841 			break;
842 		}
843 
844 		/*
845 		 * NB: We must call zfs_clear_setid_bits_if_necessary before
846 		 * committing the transaction!
847 		 */
848 
849 		/*
850 		 * If rangelock_enter() over-locked we grow the blocksize
851 		 * and then reduce the lock range.  This will only happen
852 		 * on the first iteration since rangelock_reduce() will
853 		 * shrink down lr_length to the appropriate size.
854 		 */
855 		if (lr->lr_length == UINT64_MAX) {
856 			zfs_grow_blocksize(zp, blksz, tx);
857 			zfs_rangelock_reduce(lr, woff, n);
858 		}
859 
860 		ssize_t tx_bytes;
861 		if (abuf == NULL) {
862 			tx_bytes = zfs_uio_resid(uio);
863 			zfs_uio_fault_disable(uio, B_TRUE);
864 			error = dmu_write_uio_dbuf(sa_get_db(zp->z_sa_hdl),
865 			    uio, nbytes, tx);
866 			zfs_uio_fault_disable(uio, B_FALSE);
867 #ifdef __linux__
868 			if (error == EFAULT) {
869 				zfs_clear_setid_bits_if_necessary(zfsvfs, zp,
870 				    cr, &clear_setid_bits_txg, tx);
871 				dmu_tx_commit(tx);
872 				/*
873 				 * Account for partial writes before
874 				 * continuing the loop.
875 				 * Update needs to occur before the next
876 				 * zfs_uio_prefaultpages, or prefaultpages may
877 				 * error, and we may break the loop early.
878 				 */
879 				n -= tx_bytes - zfs_uio_resid(uio);
880 				pfbytes -= tx_bytes - zfs_uio_resid(uio);
881 				continue;
882 			}
883 #endif
884 			/*
885 			 * On FreeBSD, EFAULT should be propagated back to the
886 			 * VFS, which will handle faulting and will retry.
887 			 */
888 			if (error != 0 && error != EFAULT) {
889 				zfs_clear_setid_bits_if_necessary(zfsvfs, zp,
890 				    cr, &clear_setid_bits_txg, tx);
891 				dmu_tx_commit(tx);
892 				break;
893 			}
894 			tx_bytes -= zfs_uio_resid(uio);
895 		} else {
896 			/*
897 			 * Thus, we're writing a full block at a block-aligned
898 			 * offset and extending the file past EOF.
899 			 *
900 			 * dmu_assign_arcbuf_by_dbuf() will directly assign the
901 			 * arc buffer to a dbuf.
902 			 */
903 			error = dmu_assign_arcbuf_by_dbuf(
904 			    sa_get_db(zp->z_sa_hdl), woff, abuf, tx);
905 			if (error != 0) {
906 				/*
907 				 * XXX This might not be necessary if
908 				 * dmu_assign_arcbuf_by_dbuf is guaranteed
909 				 * to be atomic.
910 				 */
911 				zfs_clear_setid_bits_if_necessary(zfsvfs, zp,
912 				    cr, &clear_setid_bits_txg, tx);
913 				dmu_return_arcbuf(abuf);
914 				dmu_tx_commit(tx);
915 				break;
916 			}
917 			ASSERT3S(nbytes, <=, zfs_uio_resid(uio));
918 			zfs_uioskip(uio, nbytes);
919 			tx_bytes = nbytes;
920 		}
921 		/*
922 		 * There is a window where a file's pages can be mmap'ed after
923 		 * zfs_setup_direct() is called. This is due to the fact that
924 		 * the rangelock in this function is acquired after calling
925 		 * zfs_setup_direct(). This is done so that
926 		 * zfs_uio_prefaultpages() does not attempt to fault in pages
927 		 * on Linux for Direct I/O requests. This is not necessary as
928 		 * the pages are pinned in memory and can not be faulted out.
929 		 * Ideally, the rangelock would be held before calling
930 		 * zfs_setup_direct() and zfs_uio_prefaultpages(); however,
931 		 * this can lead to a deadlock as zfs_getpage() also acquires
932 		 * the rangelock as a RL_WRITER and prefaulting the pages can
933 		 * lead to zfs_getpage() being called.
934 		 *
935 		 * In the case of the pages being mapped after
936 		 * zfs_setup_direct() is called, the call to update_pages()
937 		 * will still be made to make sure there is consistency between
938 		 * the ARC and the Linux page cache. This is an ufortunate
939 		 * situation as the data will be read back into the ARC after
940 		 * the Direct I/O write has completed, but this is the penality
941 		 * for writing to a mmap'ed region of a file using Direct I/O.
942 		 */
943 		if (tx_bytes &&
944 		    zn_has_cached_data(zp, woff, woff + tx_bytes - 1)) {
945 			update_pages(zp, woff, tx_bytes, zfsvfs->z_os);
946 		}
947 
948 		/*
949 		 * If we made no progress, we're done.  If we made even
950 		 * partial progress, update the znode and ZIL accordingly.
951 		 */
952 		if (tx_bytes == 0) {
953 			(void) sa_update(zp->z_sa_hdl, SA_ZPL_SIZE(zfsvfs),
954 			    (void *)&zp->z_size, sizeof (uint64_t), tx);
955 			dmu_tx_commit(tx);
956 			ASSERT(error != 0);
957 			break;
958 		}
959 
960 		zfs_clear_setid_bits_if_necessary(zfsvfs, zp, cr,
961 		    &clear_setid_bits_txg, tx);
962 
963 		zfs_tstamp_update_setup(zp, CONTENT_MODIFIED, mtime, ctime);
964 
965 		/*
966 		 * Update the file size (zp_size) if it has changed;
967 		 * account for possible concurrent updates.
968 		 */
969 		while ((end_size = zp->z_size) < zfs_uio_offset(uio)) {
970 			(void) atomic_cas_64(&zp->z_size, end_size,
971 			    zfs_uio_offset(uio));
972 			ASSERT(error == 0 || error == EFAULT);
973 		}
974 		/*
975 		 * If we are replaying and eof is non zero then force
976 		 * the file size to the specified eof. Note, there's no
977 		 * concurrency during replay.
978 		 */
979 		if (zfsvfs->z_replay && zfsvfs->z_replay_eof != 0)
980 			zp->z_size = zfsvfs->z_replay_eof;
981 
982 		error1 = sa_bulk_update(zp->z_sa_hdl, bulk, count, tx);
983 		if (error1 != 0)
984 			/* Avoid clobbering EFAULT. */
985 			error = error1;
986 
987 		/*
988 		 * NB: During replay, the TX_SETATTR record logged by
989 		 * zfs_clear_setid_bits_if_necessary must precede any of
990 		 * the TX_WRITE records logged here.
991 		 */
992 		zfs_log_write(zilog, tx, TX_WRITE, zp, woff, tx_bytes, commit,
993 		    uio->uio_extflg & UIO_DIRECT ? B_TRUE : B_FALSE, NULL,
994 		    NULL);
995 
996 		dmu_tx_commit(tx);
997 
998 		/*
999 		 * Direct I/O was deferred in order to grow the first block.
1000 		 * At this point it can be re-enabled for subsequent writes.
1001 		 */
1002 		if (o_direct_defer) {
1003 			ASSERT(ioflag & O_DIRECT);
1004 			uio->uio_extflg |= UIO_DIRECT;
1005 			o_direct_defer = B_FALSE;
1006 		}
1007 
1008 		if (error != 0)
1009 			break;
1010 		ASSERT3S(tx_bytes, ==, nbytes);
1011 		n -= nbytes;
1012 		pfbytes -= nbytes;
1013 	}
1014 
1015 	if (o_direct_defer) {
1016 		ASSERT(ioflag & O_DIRECT);
1017 		uio->uio_extflg |= UIO_DIRECT;
1018 		o_direct_defer = B_FALSE;
1019 	}
1020 
1021 	zfs_znode_update_vfs(zp);
1022 	zfs_rangelock_exit(lr);
1023 
1024 	/*
1025 	 * Cleanup for Direct I/O if requested.
1026 	 */
1027 	if (uio->uio_extflg & UIO_DIRECT)
1028 		zfs_uio_free_dio_pages(uio, UIO_WRITE);
1029 
1030 	/*
1031 	 * If we're in replay mode, or we made no progress, or the
1032 	 * uio data is inaccessible return an error.  Otherwise, it's
1033 	 * at least a partial write, so it's successful.
1034 	 */
1035 	if (zfsvfs->z_replay || zfs_uio_resid(uio) == start_resid ||
1036 	    error == EFAULT) {
1037 		zfs_exit(zfsvfs, FTAG);
1038 		return (error);
1039 	}
1040 
1041 	if (commit)
1042 		zil_commit(zilog, zp->z_id);
1043 
1044 	int64_t nwritten = start_resid - zfs_uio_resid(uio);
1045 	dataset_kstats_update_write_kstats(&zfsvfs->z_kstat, nwritten);
1046 
1047 	zfs_exit(zfsvfs, FTAG);
1048 	return (0);
1049 }
1050 
1051 int
1052 zfs_getsecattr(znode_t *zp, vsecattr_t *vsecp, int flag, cred_t *cr)
1053 {
1054 	zfsvfs_t *zfsvfs = ZTOZSB(zp);
1055 	int error;
1056 	boolean_t skipaclchk = (flag & ATTR_NOACLCHECK) ? B_TRUE : B_FALSE;
1057 
1058 	if ((error = zfs_enter_verify_zp(zfsvfs, zp, FTAG)) != 0)
1059 		return (error);
1060 	error = zfs_getacl(zp, vsecp, skipaclchk, cr);
1061 	zfs_exit(zfsvfs, FTAG);
1062 
1063 	return (error);
1064 }
1065 
1066 int
1067 zfs_setsecattr(znode_t *zp, vsecattr_t *vsecp, int flag, cred_t *cr)
1068 {
1069 	zfsvfs_t *zfsvfs = ZTOZSB(zp);
1070 	int error;
1071 	boolean_t skipaclchk = (flag & ATTR_NOACLCHECK) ? B_TRUE : B_FALSE;
1072 	zilog_t	*zilog;
1073 
1074 	if ((error = zfs_enter_verify_zp(zfsvfs, zp, FTAG)) != 0)
1075 		return (error);
1076 	zilog = zfsvfs->z_log;
1077 	error = zfs_setacl(zp, vsecp, skipaclchk, cr);
1078 
1079 	if (zfsvfs->z_os->os_sync == ZFS_SYNC_ALWAYS)
1080 		zil_commit(zilog, 0);
1081 
1082 	zfs_exit(zfsvfs, FTAG);
1083 	return (error);
1084 }
1085 
1086 #ifdef ZFS_DEBUG
1087 static int zil_fault_io = 0;
1088 #endif
1089 
1090 static void zfs_get_done(zgd_t *zgd, int error);
1091 
1092 /*
1093  * Get data to generate a TX_WRITE intent log record.
1094  */
1095 int
1096 zfs_get_data(void *arg, uint64_t gen, lr_write_t *lr, char *buf,
1097     struct lwb *lwb, zio_t *zio)
1098 {
1099 	zfsvfs_t *zfsvfs = arg;
1100 	objset_t *os = zfsvfs->z_os;
1101 	znode_t *zp;
1102 	uint64_t object = lr->lr_foid;
1103 	uint64_t offset = lr->lr_offset;
1104 	uint64_t size = lr->lr_length;
1105 	zgd_t *zgd;
1106 	int error = 0;
1107 	uint64_t zp_gen;
1108 
1109 	ASSERT3P(lwb, !=, NULL);
1110 	ASSERT3U(size, !=, 0);
1111 
1112 	/*
1113 	 * Nothing to do if the file has been removed
1114 	 */
1115 	if (zfs_zget(zfsvfs, object, &zp) != 0)
1116 		return (SET_ERROR(ENOENT));
1117 	if (zp->z_unlinked) {
1118 		/*
1119 		 * Release the vnode asynchronously as we currently have the
1120 		 * txg stopped from syncing.
1121 		 */
1122 		zfs_zrele_async(zp);
1123 		return (SET_ERROR(ENOENT));
1124 	}
1125 	/* check if generation number matches */
1126 	if (sa_lookup(zp->z_sa_hdl, SA_ZPL_GEN(zfsvfs), &zp_gen,
1127 	    sizeof (zp_gen)) != 0) {
1128 		zfs_zrele_async(zp);
1129 		return (SET_ERROR(EIO));
1130 	}
1131 	if (zp_gen != gen) {
1132 		zfs_zrele_async(zp);
1133 		return (SET_ERROR(ENOENT));
1134 	}
1135 
1136 	zgd = kmem_zalloc(sizeof (zgd_t), KM_SLEEP);
1137 	zgd->zgd_lwb = lwb;
1138 	zgd->zgd_private = zp;
1139 
1140 	/*
1141 	 * Write records come in two flavors: immediate and indirect.
1142 	 * For small writes it's cheaper to store the data with the
1143 	 * log record (immediate); for large writes it's cheaper to
1144 	 * sync the data and get a pointer to it (indirect) so that
1145 	 * we don't have to write the data twice.
1146 	 */
1147 	if (buf != NULL) { /* immediate write */
1148 		zgd->zgd_lr = zfs_rangelock_enter(&zp->z_rangelock, offset,
1149 		    size, RL_READER);
1150 		/* test for truncation needs to be done while range locked */
1151 		if (offset >= zp->z_size) {
1152 			error = SET_ERROR(ENOENT);
1153 		} else {
1154 			error = dmu_read(os, object, offset, size, buf,
1155 			    DMU_READ_NO_PREFETCH);
1156 		}
1157 		ASSERT(error == 0 || error == ENOENT);
1158 	} else { /* indirect write */
1159 		ASSERT3P(zio, !=, NULL);
1160 		/*
1161 		 * Have to lock the whole block to ensure when it's
1162 		 * written out and its checksum is being calculated
1163 		 * that no one can change the data. We need to re-check
1164 		 * blocksize after we get the lock in case it's changed!
1165 		 */
1166 		for (;;) {
1167 			uint64_t blkoff;
1168 			size = zp->z_blksz;
1169 			blkoff = ISP2(size) ? P2PHASE(offset, size) : offset;
1170 			offset -= blkoff;
1171 			zgd->zgd_lr = zfs_rangelock_enter(&zp->z_rangelock,
1172 			    offset, size, RL_READER);
1173 			if (zp->z_blksz == size)
1174 				break;
1175 			offset += blkoff;
1176 			zfs_rangelock_exit(zgd->zgd_lr);
1177 		}
1178 		/* test for truncation needs to be done while range locked */
1179 		if (lr->lr_offset >= zp->z_size)
1180 			error = SET_ERROR(ENOENT);
1181 #ifdef ZFS_DEBUG
1182 		if (zil_fault_io) {
1183 			error = SET_ERROR(EIO);
1184 			zil_fault_io = 0;
1185 		}
1186 #endif
1187 
1188 		dmu_buf_t *dbp;
1189 		if (error == 0)
1190 			error = dmu_buf_hold_noread(os, object, offset, zgd,
1191 			    &dbp);
1192 
1193 		if (error == 0) {
1194 			zgd->zgd_db = dbp;
1195 			dmu_buf_impl_t *db = (dmu_buf_impl_t *)dbp;
1196 			boolean_t direct_write = B_FALSE;
1197 			mutex_enter(&db->db_mtx);
1198 			dbuf_dirty_record_t *dr =
1199 			    dbuf_find_dirty_eq(db, lr->lr_common.lrc_txg);
1200 			if (dr != NULL && dr->dt.dl.dr_diowrite)
1201 				direct_write = B_TRUE;
1202 			mutex_exit(&db->db_mtx);
1203 
1204 			/*
1205 			 * All Direct I/O writes will have already completed and
1206 			 * the block pointer can be immediately stored in the
1207 			 * log record.
1208 			 */
1209 			if (direct_write) {
1210 				/*
1211 				 * A Direct I/O write always covers an entire
1212 				 * block.
1213 				 */
1214 				ASSERT3U(dbp->db_size, ==, zp->z_blksz);
1215 				lr->lr_blkptr = dr->dt.dl.dr_overridden_by;
1216 				zfs_get_done(zgd, 0);
1217 				return (0);
1218 			}
1219 
1220 			blkptr_t *bp = &lr->lr_blkptr;
1221 			zgd->zgd_bp = bp;
1222 
1223 			ASSERT3U(dbp->db_offset, ==, offset);
1224 			ASSERT3U(dbp->db_size, ==, size);
1225 
1226 			error = dmu_sync(zio, lr->lr_common.lrc_txg,
1227 			    zfs_get_done, zgd);
1228 			ASSERT(error || lr->lr_length <= size);
1229 
1230 			/*
1231 			 * On success, we need to wait for the write I/O
1232 			 * initiated by dmu_sync() to complete before we can
1233 			 * release this dbuf.  We will finish everything up
1234 			 * in the zfs_get_done() callback.
1235 			 */
1236 			if (error == 0)
1237 				return (0);
1238 
1239 			if (error == EALREADY) {
1240 				lr->lr_common.lrc_txtype = TX_WRITE2;
1241 				/*
1242 				 * TX_WRITE2 relies on the data previously
1243 				 * written by the TX_WRITE that caused
1244 				 * EALREADY.  We zero out the BP because
1245 				 * it is the old, currently-on-disk BP.
1246 				 */
1247 				zgd->zgd_bp = NULL;
1248 				BP_ZERO(bp);
1249 				error = 0;
1250 			}
1251 		}
1252 	}
1253 
1254 	zfs_get_done(zgd, error);
1255 
1256 	return (error);
1257 }
1258 
1259 static void
1260 zfs_get_done(zgd_t *zgd, int error)
1261 {
1262 	(void) error;
1263 	znode_t *zp = zgd->zgd_private;
1264 
1265 	if (zgd->zgd_db)
1266 		dmu_buf_rele(zgd->zgd_db, zgd);
1267 
1268 	zfs_rangelock_exit(zgd->zgd_lr);
1269 
1270 	/*
1271 	 * Release the vnode asynchronously as we currently have the
1272 	 * txg stopped from syncing.
1273 	 */
1274 	zfs_zrele_async(zp);
1275 
1276 	kmem_free(zgd, sizeof (zgd_t));
1277 }
1278 
1279 static int
1280 zfs_enter_two(zfsvfs_t *zfsvfs1, zfsvfs_t *zfsvfs2, const char *tag)
1281 {
1282 	int error;
1283 
1284 	/* Swap. Not sure if the order of zfs_enter()s is important. */
1285 	if (zfsvfs1 > zfsvfs2) {
1286 		zfsvfs_t *tmpzfsvfs;
1287 
1288 		tmpzfsvfs = zfsvfs2;
1289 		zfsvfs2 = zfsvfs1;
1290 		zfsvfs1 = tmpzfsvfs;
1291 	}
1292 
1293 	error = zfs_enter(zfsvfs1, tag);
1294 	if (error != 0)
1295 		return (error);
1296 	if (zfsvfs1 != zfsvfs2) {
1297 		error = zfs_enter(zfsvfs2, tag);
1298 		if (error != 0) {
1299 			zfs_exit(zfsvfs1, tag);
1300 			return (error);
1301 		}
1302 	}
1303 
1304 	return (0);
1305 }
1306 
1307 static void
1308 zfs_exit_two(zfsvfs_t *zfsvfs1, zfsvfs_t *zfsvfs2, const char *tag)
1309 {
1310 
1311 	zfs_exit(zfsvfs1, tag);
1312 	if (zfsvfs1 != zfsvfs2)
1313 		zfs_exit(zfsvfs2, tag);
1314 }
1315 
1316 /*
1317  * We split each clone request in chunks that can fit into a single ZIL
1318  * log entry. Each ZIL log entry can fit 130816 bytes for a block cloning
1319  * operation (see zil_max_log_data() and zfs_log_clone_range()). This gives
1320  * us room for storing 1022 block pointers.
1321  *
1322  * On success, the function return the number of bytes copied in *lenp.
1323  * Note, it doesn't return how much bytes are left to be copied.
1324  * On errors which are caused by any file system limitations or
1325  * brt limitations `EINVAL` is returned. In the most cases a user
1326  * requested bad parameters, it could be possible to clone the file but
1327  * some parameters don't match the requirements.
1328  */
1329 int
1330 zfs_clone_range(znode_t *inzp, uint64_t *inoffp, znode_t *outzp,
1331     uint64_t *outoffp, uint64_t *lenp, cred_t *cr)
1332 {
1333 	zfsvfs_t	*inzfsvfs, *outzfsvfs;
1334 	objset_t	*inos, *outos;
1335 	zfs_locked_range_t *inlr, *outlr;
1336 	dmu_buf_impl_t	*db;
1337 	dmu_tx_t	*tx;
1338 	zilog_t		*zilog;
1339 	uint64_t	inoff, outoff, len, done;
1340 	uint64_t	outsize, size;
1341 	int		error;
1342 	int		count = 0;
1343 	sa_bulk_attr_t	bulk[3];
1344 	uint64_t	mtime[2], ctime[2];
1345 	uint64_t	uid, gid, projid;
1346 	blkptr_t	*bps;
1347 	size_t		maxblocks, nbps;
1348 	uint_t		inblksz;
1349 	uint64_t	clear_setid_bits_txg = 0;
1350 	uint64_t	last_synced_txg = 0;
1351 
1352 	inoff = *inoffp;
1353 	outoff = *outoffp;
1354 	len = *lenp;
1355 	done = 0;
1356 
1357 	inzfsvfs = ZTOZSB(inzp);
1358 	outzfsvfs = ZTOZSB(outzp);
1359 
1360 	/*
1361 	 * We need to call zfs_enter() potentially on two different datasets,
1362 	 * so we need a dedicated function for that.
1363 	 */
1364 	error = zfs_enter_two(inzfsvfs, outzfsvfs, FTAG);
1365 	if (error != 0)
1366 		return (error);
1367 
1368 	inos = inzfsvfs->z_os;
1369 	outos = outzfsvfs->z_os;
1370 
1371 	/*
1372 	 * Both source and destination have to belong to the same storage pool.
1373 	 */
1374 	if (dmu_objset_spa(inos) != dmu_objset_spa(outos)) {
1375 		zfs_exit_two(inzfsvfs, outzfsvfs, FTAG);
1376 		return (SET_ERROR(EXDEV));
1377 	}
1378 
1379 	/*
1380 	 * outos and inos belongs to the same storage pool.
1381 	 * see a few lines above, only one check.
1382 	 */
1383 	if (!spa_feature_is_enabled(dmu_objset_spa(outos),
1384 	    SPA_FEATURE_BLOCK_CLONING)) {
1385 		zfs_exit_two(inzfsvfs, outzfsvfs, FTAG);
1386 		return (SET_ERROR(EOPNOTSUPP));
1387 	}
1388 
1389 	ASSERT(!outzfsvfs->z_replay);
1390 
1391 	/*
1392 	 * Block cloning from an unencrypted dataset into an encrypted
1393 	 * dataset and vice versa is not supported.
1394 	 */
1395 	if (inos->os_encrypted != outos->os_encrypted) {
1396 		zfs_exit_two(inzfsvfs, outzfsvfs, FTAG);
1397 		return (SET_ERROR(EXDEV));
1398 	}
1399 
1400 	/*
1401 	 * Cloning across encrypted datasets is possible only if they
1402 	 * share the same master key.
1403 	 */
1404 	if (inos != outos && inos->os_encrypted &&
1405 	    !dmu_objset_crypto_key_equal(inos, outos)) {
1406 		zfs_exit_two(inzfsvfs, outzfsvfs, FTAG);
1407 		return (SET_ERROR(EXDEV));
1408 	}
1409 
1410 	error = zfs_verify_zp(inzp);
1411 	if (error == 0)
1412 		error = zfs_verify_zp(outzp);
1413 	if (error != 0) {
1414 		zfs_exit_two(inzfsvfs, outzfsvfs, FTAG);
1415 		return (error);
1416 	}
1417 
1418 	/*
1419 	 * We don't copy source file's flags that's why we don't allow to clone
1420 	 * files that are in quarantine.
1421 	 */
1422 	if (inzp->z_pflags & ZFS_AV_QUARANTINED) {
1423 		zfs_exit_two(inzfsvfs, outzfsvfs, FTAG);
1424 		return (SET_ERROR(EACCES));
1425 	}
1426 
1427 	if (inoff >= inzp->z_size) {
1428 		*lenp = 0;
1429 		zfs_exit_two(inzfsvfs, outzfsvfs, FTAG);
1430 		return (0);
1431 	}
1432 	if (len > inzp->z_size - inoff) {
1433 		len = inzp->z_size - inoff;
1434 	}
1435 	if (len == 0) {
1436 		*lenp = 0;
1437 		zfs_exit_two(inzfsvfs, outzfsvfs, FTAG);
1438 		return (0);
1439 	}
1440 
1441 	/*
1442 	 * Callers might not be able to detect properly that we are read-only,
1443 	 * so check it explicitly here.
1444 	 */
1445 	if (zfs_is_readonly(outzfsvfs)) {
1446 		zfs_exit_two(inzfsvfs, outzfsvfs, FTAG);
1447 		return (SET_ERROR(EROFS));
1448 	}
1449 
1450 	/*
1451 	 * If immutable or not appending then return EPERM.
1452 	 * Intentionally allow ZFS_READONLY through here.
1453 	 * See zfs_zaccess_common()
1454 	 */
1455 	if ((outzp->z_pflags & ZFS_IMMUTABLE) != 0) {
1456 		zfs_exit_two(inzfsvfs, outzfsvfs, FTAG);
1457 		return (SET_ERROR(EPERM));
1458 	}
1459 
1460 	/*
1461 	 * No overlapping if we are cloning within the same file.
1462 	 */
1463 	if (inzp == outzp) {
1464 		if (inoff < outoff + len && outoff < inoff + len) {
1465 			zfs_exit_two(inzfsvfs, outzfsvfs, FTAG);
1466 			return (SET_ERROR(EINVAL));
1467 		}
1468 	}
1469 
1470 	/* Flush any mmap()'d data to disk */
1471 	if (zn_has_cached_data(inzp, inoff, inoff + len - 1))
1472 		zn_flush_cached_data(inzp, B_TRUE);
1473 
1474 	/*
1475 	 * Maintain predictable lock order.
1476 	 */
1477 	if (inzp < outzp || (inzp == outzp && inoff < outoff)) {
1478 		inlr = zfs_rangelock_enter(&inzp->z_rangelock, inoff, len,
1479 		    RL_READER);
1480 		outlr = zfs_rangelock_enter(&outzp->z_rangelock, outoff, len,
1481 		    RL_WRITER);
1482 	} else {
1483 		outlr = zfs_rangelock_enter(&outzp->z_rangelock, outoff, len,
1484 		    RL_WRITER);
1485 		inlr = zfs_rangelock_enter(&inzp->z_rangelock, inoff, len,
1486 		    RL_READER);
1487 	}
1488 
1489 	inblksz = inzp->z_blksz;
1490 
1491 	/*
1492 	 * We cannot clone into a file with different block size if we can't
1493 	 * grow it (block size is already bigger, has more than one block, or
1494 	 * not locked for growth).  There are other possible reasons for the
1495 	 * grow to fail, but we cover what we can before opening transaction
1496 	 * and the rest detect after we try to do it.
1497 	 */
1498 	if (inblksz < outzp->z_blksz) {
1499 		error = SET_ERROR(EINVAL);
1500 		goto unlock;
1501 	}
1502 	if (inblksz != outzp->z_blksz && (outzp->z_size > outzp->z_blksz ||
1503 	    outlr->lr_length != UINT64_MAX)) {
1504 		error = SET_ERROR(EINVAL);
1505 		goto unlock;
1506 	}
1507 
1508 	/*
1509 	 * Block size must be power-of-2 if destination offset != 0.
1510 	 * There can be no multiple blocks of non-power-of-2 size.
1511 	 */
1512 	if (outoff != 0 && !ISP2(inblksz)) {
1513 		error = SET_ERROR(EINVAL);
1514 		goto unlock;
1515 	}
1516 
1517 	/*
1518 	 * Offsets and len must be at block boundries.
1519 	 */
1520 	if ((inoff % inblksz) != 0 || (outoff % inblksz) != 0) {
1521 		error = SET_ERROR(EINVAL);
1522 		goto unlock;
1523 	}
1524 	/*
1525 	 * Length must be multipe of blksz, except for the end of the file.
1526 	 */
1527 	if ((len % inblksz) != 0 &&
1528 	    (len < inzp->z_size - inoff || len < outzp->z_size - outoff)) {
1529 		error = SET_ERROR(EINVAL);
1530 		goto unlock;
1531 	}
1532 
1533 	/*
1534 	 * If we are copying only one block and it is smaller than recordsize
1535 	 * property, do not allow destination to grow beyond one block if it
1536 	 * is not there yet.  Otherwise the destination will get stuck with
1537 	 * that block size forever, that can be as small as 512 bytes, no
1538 	 * matter how big the destination grow later.
1539 	 */
1540 	if (len <= inblksz && inblksz < outzfsvfs->z_max_blksz &&
1541 	    outzp->z_size <= inblksz && outoff + len > inblksz) {
1542 		error = SET_ERROR(EINVAL);
1543 		goto unlock;
1544 	}
1545 
1546 	error = zn_rlimit_fsize(outoff + len);
1547 	if (error != 0) {
1548 		goto unlock;
1549 	}
1550 
1551 	if (inoff >= MAXOFFSET_T || outoff >= MAXOFFSET_T) {
1552 		error = SET_ERROR(EFBIG);
1553 		goto unlock;
1554 	}
1555 
1556 	SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_MTIME(outzfsvfs), NULL,
1557 	    &mtime, 16);
1558 	SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_CTIME(outzfsvfs), NULL,
1559 	    &ctime, 16);
1560 	SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_SIZE(outzfsvfs), NULL,
1561 	    &outzp->z_size, 8);
1562 
1563 	zilog = outzfsvfs->z_log;
1564 	maxblocks = zil_max_log_data(zilog, sizeof (lr_clone_range_t)) /
1565 	    sizeof (bps[0]);
1566 
1567 	uid = KUID_TO_SUID(ZTOUID(outzp));
1568 	gid = KGID_TO_SGID(ZTOGID(outzp));
1569 	projid = outzp->z_projid;
1570 
1571 	bps = vmem_alloc(sizeof (bps[0]) * maxblocks, KM_SLEEP);
1572 
1573 	/*
1574 	 * Clone the file in reasonable size chunks.  Each chunk is cloned
1575 	 * in a separate transaction; this keeps the intent log records small
1576 	 * and allows us to do more fine-grained space accounting.
1577 	 */
1578 	while (len > 0) {
1579 		size = MIN(inblksz * maxblocks, len);
1580 
1581 		if (zfs_id_overblockquota(outzfsvfs, DMU_USERUSED_OBJECT,
1582 		    uid) ||
1583 		    zfs_id_overblockquota(outzfsvfs, DMU_GROUPUSED_OBJECT,
1584 		    gid) ||
1585 		    (projid != ZFS_DEFAULT_PROJID &&
1586 		    zfs_id_overblockquota(outzfsvfs, DMU_PROJECTUSED_OBJECT,
1587 		    projid))) {
1588 			error = SET_ERROR(EDQUOT);
1589 			break;
1590 		}
1591 
1592 		nbps = maxblocks;
1593 		last_synced_txg = spa_last_synced_txg(dmu_objset_spa(inos));
1594 		error = dmu_read_l0_bps(inos, inzp->z_id, inoff, size, bps,
1595 		    &nbps);
1596 		if (error != 0) {
1597 			/*
1598 			 * If we are trying to clone a block that was created
1599 			 * in the current transaction group, the error will be
1600 			 * EAGAIN here.  Based on zfs_bclone_wait_dirty either
1601 			 * return a shortened range to the caller so it can
1602 			 * fallback, or wait for the next TXG and check again.
1603 			 */
1604 			if (error == EAGAIN && zfs_bclone_wait_dirty) {
1605 				txg_wait_synced(dmu_objset_pool(inos),
1606 				    last_synced_txg + 1);
1607 				continue;
1608 			}
1609 
1610 			break;
1611 		}
1612 
1613 		/*
1614 		 * Start a transaction.
1615 		 */
1616 		tx = dmu_tx_create(outos);
1617 		dmu_tx_hold_sa(tx, outzp->z_sa_hdl, B_FALSE);
1618 		db = (dmu_buf_impl_t *)sa_get_db(outzp->z_sa_hdl);
1619 		DB_DNODE_ENTER(db);
1620 		dmu_tx_hold_clone_by_dnode(tx, DB_DNODE(db), outoff, size);
1621 		DB_DNODE_EXIT(db);
1622 		zfs_sa_upgrade_txholds(tx, outzp);
1623 		error = dmu_tx_assign(tx, TXG_WAIT);
1624 		if (error != 0) {
1625 			dmu_tx_abort(tx);
1626 			break;
1627 		}
1628 
1629 		/*
1630 		 * Copy source znode's block size. This is done only if the
1631 		 * whole znode is locked (see zfs_rangelock_cb()) and only
1632 		 * on the first iteration since zfs_rangelock_reduce() will
1633 		 * shrink down lr_length to the appropriate size.
1634 		 */
1635 		if (outlr->lr_length == UINT64_MAX) {
1636 			zfs_grow_blocksize(outzp, inblksz, tx);
1637 
1638 			/*
1639 			 * Block growth may fail for many reasons we can not
1640 			 * predict here.  If it happen the cloning is doomed.
1641 			 */
1642 			if (inblksz != outzp->z_blksz) {
1643 				error = SET_ERROR(EINVAL);
1644 				dmu_tx_abort(tx);
1645 				break;
1646 			}
1647 
1648 			/*
1649 			 * Round range lock up to the block boundary, so we
1650 			 * prevent appends until we are done.
1651 			 */
1652 			zfs_rangelock_reduce(outlr, outoff,
1653 			    ((len - 1) / inblksz + 1) * inblksz);
1654 		}
1655 
1656 		error = dmu_brt_clone(outos, outzp->z_id, outoff, size, tx,
1657 		    bps, nbps);
1658 		if (error != 0) {
1659 			dmu_tx_commit(tx);
1660 			break;
1661 		}
1662 
1663 		if (zn_has_cached_data(outzp, outoff, outoff + size - 1)) {
1664 			update_pages(outzp, outoff, size, outos);
1665 		}
1666 
1667 		zfs_clear_setid_bits_if_necessary(outzfsvfs, outzp, cr,
1668 		    &clear_setid_bits_txg, tx);
1669 
1670 		zfs_tstamp_update_setup(outzp, CONTENT_MODIFIED, mtime, ctime);
1671 
1672 		/*
1673 		 * Update the file size (zp_size) if it has changed;
1674 		 * account for possible concurrent updates.
1675 		 */
1676 		while ((outsize = outzp->z_size) < outoff + size) {
1677 			(void) atomic_cas_64(&outzp->z_size, outsize,
1678 			    outoff + size);
1679 		}
1680 
1681 		error = sa_bulk_update(outzp->z_sa_hdl, bulk, count, tx);
1682 
1683 		zfs_log_clone_range(zilog, tx, TX_CLONE_RANGE, outzp, outoff,
1684 		    size, inblksz, bps, nbps);
1685 
1686 		dmu_tx_commit(tx);
1687 
1688 		if (error != 0)
1689 			break;
1690 
1691 		inoff += size;
1692 		outoff += size;
1693 		len -= size;
1694 		done += size;
1695 
1696 		if (issig()) {
1697 			error = SET_ERROR(EINTR);
1698 			break;
1699 		}
1700 	}
1701 
1702 	vmem_free(bps, sizeof (bps[0]) * maxblocks);
1703 	zfs_znode_update_vfs(outzp);
1704 
1705 unlock:
1706 	zfs_rangelock_exit(outlr);
1707 	zfs_rangelock_exit(inlr);
1708 
1709 	if (done > 0) {
1710 		/*
1711 		 * If we have made at least partial progress, reset the error.
1712 		 */
1713 		error = 0;
1714 
1715 		ZFS_ACCESSTIME_STAMP(inzfsvfs, inzp);
1716 
1717 		if (outos->os_sync == ZFS_SYNC_ALWAYS) {
1718 			zil_commit(zilog, outzp->z_id);
1719 		}
1720 
1721 		*inoffp += done;
1722 		*outoffp += done;
1723 		*lenp = done;
1724 	} else {
1725 		/*
1726 		 * If we made no progress, there must be a good reason.
1727 		 * EOF is handled explicitly above, before the loop.
1728 		 */
1729 		ASSERT3S(error, !=, 0);
1730 	}
1731 
1732 	zfs_exit_two(inzfsvfs, outzfsvfs, FTAG);
1733 
1734 	return (error);
1735 }
1736 
1737 /*
1738  * Usual pattern would be to call zfs_clone_range() from zfs_replay_clone(),
1739  * but we cannot do that, because when replaying we don't have source znode
1740  * available. This is why we need a dedicated replay function.
1741  */
1742 int
1743 zfs_clone_range_replay(znode_t *zp, uint64_t off, uint64_t len, uint64_t blksz,
1744     const blkptr_t *bps, size_t nbps)
1745 {
1746 	zfsvfs_t	*zfsvfs;
1747 	dmu_buf_impl_t	*db;
1748 	dmu_tx_t	*tx;
1749 	int		error;
1750 	int		count = 0;
1751 	sa_bulk_attr_t	bulk[3];
1752 	uint64_t	mtime[2], ctime[2];
1753 
1754 	ASSERT3U(off, <, MAXOFFSET_T);
1755 	ASSERT3U(len, >, 0);
1756 	ASSERT3U(nbps, >, 0);
1757 
1758 	zfsvfs = ZTOZSB(zp);
1759 
1760 	ASSERT(spa_feature_is_enabled(dmu_objset_spa(zfsvfs->z_os),
1761 	    SPA_FEATURE_BLOCK_CLONING));
1762 
1763 	if ((error = zfs_enter_verify_zp(zfsvfs, zp, FTAG)) != 0)
1764 		return (error);
1765 
1766 	ASSERT(zfsvfs->z_replay);
1767 	ASSERT(!zfs_is_readonly(zfsvfs));
1768 
1769 	if ((off % blksz) != 0) {
1770 		zfs_exit(zfsvfs, FTAG);
1771 		return (SET_ERROR(EINVAL));
1772 	}
1773 
1774 	SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_MTIME(zfsvfs), NULL, &mtime, 16);
1775 	SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_CTIME(zfsvfs), NULL, &ctime, 16);
1776 	SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_SIZE(zfsvfs), NULL,
1777 	    &zp->z_size, 8);
1778 
1779 	/*
1780 	 * Start a transaction.
1781 	 */
1782 	tx = dmu_tx_create(zfsvfs->z_os);
1783 
1784 	dmu_tx_hold_sa(tx, zp->z_sa_hdl, B_FALSE);
1785 	db = (dmu_buf_impl_t *)sa_get_db(zp->z_sa_hdl);
1786 	DB_DNODE_ENTER(db);
1787 	dmu_tx_hold_clone_by_dnode(tx, DB_DNODE(db), off, len);
1788 	DB_DNODE_EXIT(db);
1789 	zfs_sa_upgrade_txholds(tx, zp);
1790 	error = dmu_tx_assign(tx, TXG_WAIT);
1791 	if (error != 0) {
1792 		dmu_tx_abort(tx);
1793 		zfs_exit(zfsvfs, FTAG);
1794 		return (error);
1795 	}
1796 
1797 	if (zp->z_blksz < blksz)
1798 		zfs_grow_blocksize(zp, blksz, tx);
1799 
1800 	dmu_brt_clone(zfsvfs->z_os, zp->z_id, off, len, tx, bps, nbps);
1801 
1802 	zfs_tstamp_update_setup(zp, CONTENT_MODIFIED, mtime, ctime);
1803 
1804 	if (zp->z_size < off + len)
1805 		zp->z_size = off + len;
1806 
1807 	error = sa_bulk_update(zp->z_sa_hdl, bulk, count, tx);
1808 
1809 	/*
1810 	 * zil_replaying() not only check if we are replaying ZIL, but also
1811 	 * updates the ZIL header to record replay progress.
1812 	 */
1813 	VERIFY(zil_replaying(zfsvfs->z_log, tx));
1814 
1815 	dmu_tx_commit(tx);
1816 
1817 	zfs_znode_update_vfs(zp);
1818 
1819 	zfs_exit(zfsvfs, FTAG);
1820 
1821 	return (error);
1822 }
1823 
1824 EXPORT_SYMBOL(zfs_access);
1825 EXPORT_SYMBOL(zfs_fsync);
1826 EXPORT_SYMBOL(zfs_holey);
1827 EXPORT_SYMBOL(zfs_read);
1828 EXPORT_SYMBOL(zfs_write);
1829 EXPORT_SYMBOL(zfs_getsecattr);
1830 EXPORT_SYMBOL(zfs_setsecattr);
1831 EXPORT_SYMBOL(zfs_clone_range);
1832 EXPORT_SYMBOL(zfs_clone_range_replay);
1833 
1834 ZFS_MODULE_PARAM(zfs_vnops, zfs_vnops_, read_chunk_size, U64, ZMOD_RW,
1835 	"Bytes to read per chunk");
1836 
1837 ZFS_MODULE_PARAM(zfs, zfs_, bclone_enabled, INT, ZMOD_RW,
1838 	"Enable block cloning");
1839 
1840 ZFS_MODULE_PARAM(zfs, zfs_, bclone_wait_dirty, INT, ZMOD_RW,
1841 	"Wait for dirty blocks when cloning");
1842 
1843 ZFS_MODULE_PARAM(zfs, zfs_, dio_enabled, INT, ZMOD_RW,
1844 	"Enable Direct I/O");
1845