xref: /freebsd/sys/contrib/openzfs/module/zfs/zfs_vnops.c (revision 833a452e9f082a7982a31c21f0da437dbbe0a39d)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 
22 /*
23  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
24  * Copyright (c) 2012, 2018 by Delphix. All rights reserved.
25  * Copyright (c) 2015 by Chunwei Chen. All rights reserved.
26  * Copyright 2017 Nexenta Systems, Inc.
27  */
28 
29 /* Portions Copyright 2007 Jeremy Teo */
30 /* Portions Copyright 2010 Robert Milkowski */
31 
32 #include <sys/types.h>
33 #include <sys/param.h>
34 #include <sys/time.h>
35 #include <sys/sysmacros.h>
36 #include <sys/vfs.h>
37 #include <sys/uio_impl.h>
38 #include <sys/file.h>
39 #include <sys/stat.h>
40 #include <sys/kmem.h>
41 #include <sys/cmn_err.h>
42 #include <sys/errno.h>
43 #include <sys/zfs_dir.h>
44 #include <sys/zfs_acl.h>
45 #include <sys/zfs_ioctl.h>
46 #include <sys/fs/zfs.h>
47 #include <sys/dmu.h>
48 #include <sys/dmu_objset.h>
49 #include <sys/spa.h>
50 #include <sys/txg.h>
51 #include <sys/dbuf.h>
52 #include <sys/policy.h>
53 #include <sys/zfs_vnops.h>
54 #include <sys/zfs_quota.h>
55 #include <sys/zfs_vfsops.h>
56 #include <sys/zfs_znode.h>
57 
58 
59 static ulong_t zfs_fsync_sync_cnt = 4;
60 
61 int
62 zfs_fsync(znode_t *zp, int syncflag, cred_t *cr)
63 {
64 	zfsvfs_t *zfsvfs = ZTOZSB(zp);
65 
66 	(void) tsd_set(zfs_fsyncer_key, (void *)zfs_fsync_sync_cnt);
67 
68 	if (zfsvfs->z_os->os_sync != ZFS_SYNC_DISABLED) {
69 		ZFS_ENTER(zfsvfs);
70 		ZFS_VERIFY_ZP(zp);
71 		zil_commit(zfsvfs->z_log, zp->z_id);
72 		ZFS_EXIT(zfsvfs);
73 	}
74 	tsd_set(zfs_fsyncer_key, NULL);
75 
76 	return (0);
77 }
78 
79 
80 #if defined(SEEK_HOLE) && defined(SEEK_DATA)
81 /*
82  * Lseek support for finding holes (cmd == SEEK_HOLE) and
83  * data (cmd == SEEK_DATA). "off" is an in/out parameter.
84  */
85 static int
86 zfs_holey_common(znode_t *zp, ulong_t cmd, loff_t *off)
87 {
88 	zfs_locked_range_t *lr;
89 	uint64_t noff = (uint64_t)*off; /* new offset */
90 	uint64_t file_sz;
91 	int error;
92 	boolean_t hole;
93 
94 	file_sz = zp->z_size;
95 	if (noff >= file_sz)  {
96 		return (SET_ERROR(ENXIO));
97 	}
98 
99 	if (cmd == F_SEEK_HOLE)
100 		hole = B_TRUE;
101 	else
102 		hole = B_FALSE;
103 
104 	/* Flush any mmap()'d data to disk */
105 	if (zn_has_cached_data(zp))
106 		zn_flush_cached_data(zp, B_FALSE);
107 
108 	lr = zfs_rangelock_enter(&zp->z_rangelock, 0, file_sz, RL_READER);
109 	error = dmu_offset_next(ZTOZSB(zp)->z_os, zp->z_id, hole, &noff);
110 	zfs_rangelock_exit(lr);
111 
112 	if (error == ESRCH)
113 		return (SET_ERROR(ENXIO));
114 
115 	/* File was dirty, so fall back to using generic logic */
116 	if (error == EBUSY) {
117 		if (hole)
118 			*off = file_sz;
119 
120 		return (0);
121 	}
122 
123 	/*
124 	 * We could find a hole that begins after the logical end-of-file,
125 	 * because dmu_offset_next() only works on whole blocks.  If the
126 	 * EOF falls mid-block, then indicate that the "virtual hole"
127 	 * at the end of the file begins at the logical EOF, rather than
128 	 * at the end of the last block.
129 	 */
130 	if (noff > file_sz) {
131 		ASSERT(hole);
132 		noff = file_sz;
133 	}
134 
135 	if (noff < *off)
136 		return (error);
137 	*off = noff;
138 	return (error);
139 }
140 
141 int
142 zfs_holey(znode_t *zp, ulong_t cmd, loff_t *off)
143 {
144 	zfsvfs_t *zfsvfs = ZTOZSB(zp);
145 	int error;
146 
147 	ZFS_ENTER(zfsvfs);
148 	ZFS_VERIFY_ZP(zp);
149 
150 	error = zfs_holey_common(zp, cmd, off);
151 
152 	ZFS_EXIT(zfsvfs);
153 	return (error);
154 }
155 #endif /* SEEK_HOLE && SEEK_DATA */
156 
157 /*ARGSUSED*/
158 int
159 zfs_access(znode_t *zp, int mode, int flag, cred_t *cr)
160 {
161 	zfsvfs_t *zfsvfs = ZTOZSB(zp);
162 	int error;
163 
164 	ZFS_ENTER(zfsvfs);
165 	ZFS_VERIFY_ZP(zp);
166 
167 	if (flag & V_ACE_MASK)
168 		error = zfs_zaccess(zp, mode, flag, B_FALSE, cr);
169 	else
170 		error = zfs_zaccess_rwx(zp, mode, flag, cr);
171 
172 	ZFS_EXIT(zfsvfs);
173 	return (error);
174 }
175 
176 static unsigned long zfs_vnops_read_chunk_size = 1024 * 1024; /* Tunable */
177 
178 /*
179  * Read bytes from specified file into supplied buffer.
180  *
181  *	IN:	zp	- inode of file to be read from.
182  *		uio	- structure supplying read location, range info,
183  *			  and return buffer.
184  *		ioflag	- O_SYNC flags; used to provide FRSYNC semantics.
185  *			  O_DIRECT flag; used to bypass page cache.
186  *		cr	- credentials of caller.
187  *
188  *	OUT:	uio	- updated offset and range, buffer filled.
189  *
190  *	RETURN:	0 on success, error code on failure.
191  *
192  * Side Effects:
193  *	inode - atime updated if byte count > 0
194  */
195 /* ARGSUSED */
196 int
197 zfs_read(struct znode *zp, zfs_uio_t *uio, int ioflag, cred_t *cr)
198 {
199 	int error = 0;
200 	boolean_t frsync = B_FALSE;
201 
202 	zfsvfs_t *zfsvfs = ZTOZSB(zp);
203 	ZFS_ENTER(zfsvfs);
204 	ZFS_VERIFY_ZP(zp);
205 
206 	if (zp->z_pflags & ZFS_AV_QUARANTINED) {
207 		ZFS_EXIT(zfsvfs);
208 		return (SET_ERROR(EACCES));
209 	}
210 
211 	/* We don't copy out anything useful for directories. */
212 	if (Z_ISDIR(ZTOTYPE(zp))) {
213 		ZFS_EXIT(zfsvfs);
214 		return (SET_ERROR(EISDIR));
215 	}
216 
217 	/*
218 	 * Validate file offset
219 	 */
220 	if (zfs_uio_offset(uio) < (offset_t)0) {
221 		ZFS_EXIT(zfsvfs);
222 		return (SET_ERROR(EINVAL));
223 	}
224 
225 	/*
226 	 * Fasttrack empty reads
227 	 */
228 	if (zfs_uio_resid(uio) == 0) {
229 		ZFS_EXIT(zfsvfs);
230 		return (0);
231 	}
232 
233 #ifdef FRSYNC
234 	/*
235 	 * If we're in FRSYNC mode, sync out this znode before reading it.
236 	 * Only do this for non-snapshots.
237 	 *
238 	 * Some platforms do not support FRSYNC and instead map it
239 	 * to O_SYNC, which results in unnecessary calls to zil_commit. We
240 	 * only honor FRSYNC requests on platforms which support it.
241 	 */
242 	frsync = !!(ioflag & FRSYNC);
243 #endif
244 	if (zfsvfs->z_log &&
245 	    (frsync || zfsvfs->z_os->os_sync == ZFS_SYNC_ALWAYS))
246 		zil_commit(zfsvfs->z_log, zp->z_id);
247 
248 	/*
249 	 * Lock the range against changes.
250 	 */
251 	zfs_locked_range_t *lr = zfs_rangelock_enter(&zp->z_rangelock,
252 	    zfs_uio_offset(uio), zfs_uio_resid(uio), RL_READER);
253 
254 	/*
255 	 * If we are reading past end-of-file we can skip
256 	 * to the end; but we might still need to set atime.
257 	 */
258 	if (zfs_uio_offset(uio) >= zp->z_size) {
259 		error = 0;
260 		goto out;
261 	}
262 
263 	ASSERT(zfs_uio_offset(uio) < zp->z_size);
264 #if defined(__linux__)
265 	ssize_t start_offset = zfs_uio_offset(uio);
266 #endif
267 	ssize_t n = MIN(zfs_uio_resid(uio), zp->z_size - zfs_uio_offset(uio));
268 	ssize_t start_resid = n;
269 
270 	while (n > 0) {
271 		ssize_t nbytes = MIN(n, zfs_vnops_read_chunk_size -
272 		    P2PHASE(zfs_uio_offset(uio), zfs_vnops_read_chunk_size));
273 #ifdef UIO_NOCOPY
274 		if (zfs_uio_segflg(uio) == UIO_NOCOPY)
275 			error = mappedread_sf(zp, nbytes, uio);
276 		else
277 #endif
278 		if (zn_has_cached_data(zp) && !(ioflag & O_DIRECT)) {
279 			error = mappedread(zp, nbytes, uio);
280 		} else {
281 			error = dmu_read_uio_dbuf(sa_get_db(zp->z_sa_hdl),
282 			    uio, nbytes);
283 		}
284 
285 		if (error) {
286 			/* convert checksum errors into IO errors */
287 			if (error == ECKSUM)
288 				error = SET_ERROR(EIO);
289 
290 #if defined(__linux__)
291 			/*
292 			 * if we actually read some bytes, bubbling EFAULT
293 			 * up to become EAGAIN isn't what we want here...
294 			 *
295 			 * ...on Linux, at least. On FBSD, doing this breaks.
296 			 */
297 			if (error == EFAULT &&
298 			    (zfs_uio_offset(uio) - start_offset) != 0)
299 				error = 0;
300 #endif
301 			break;
302 		}
303 
304 		n -= nbytes;
305 	}
306 
307 	int64_t nread = start_resid - n;
308 	dataset_kstats_update_read_kstats(&zfsvfs->z_kstat, nread);
309 	task_io_account_read(nread);
310 out:
311 	zfs_rangelock_exit(lr);
312 
313 	ZFS_ACCESSTIME_STAMP(zfsvfs, zp);
314 	ZFS_EXIT(zfsvfs);
315 	return (error);
316 }
317 
318 /*
319  * Write the bytes to a file.
320  *
321  *	IN:	zp	- znode of file to be written to.
322  *		uio	- structure supplying write location, range info,
323  *			  and data buffer.
324  *		ioflag	- O_APPEND flag set if in append mode.
325  *			  O_DIRECT flag; used to bypass page cache.
326  *		cr	- credentials of caller.
327  *
328  *	OUT:	uio	- updated offset and range.
329  *
330  *	RETURN:	0 if success
331  *		error code if failure
332  *
333  * Timestamps:
334  *	ip - ctime|mtime updated if byte count > 0
335  */
336 
337 /* ARGSUSED */
338 int
339 zfs_write(znode_t *zp, zfs_uio_t *uio, int ioflag, cred_t *cr)
340 {
341 	int error = 0, error1;
342 	ssize_t start_resid = zfs_uio_resid(uio);
343 
344 	/*
345 	 * Fasttrack empty write
346 	 */
347 	ssize_t n = start_resid;
348 	if (n == 0)
349 		return (0);
350 
351 	zfsvfs_t *zfsvfs = ZTOZSB(zp);
352 	ZFS_ENTER(zfsvfs);
353 	ZFS_VERIFY_ZP(zp);
354 
355 	sa_bulk_attr_t bulk[4];
356 	int count = 0;
357 	uint64_t mtime[2], ctime[2];
358 	SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_MTIME(zfsvfs), NULL, &mtime, 16);
359 	SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_CTIME(zfsvfs), NULL, &ctime, 16);
360 	SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_SIZE(zfsvfs), NULL,
361 	    &zp->z_size, 8);
362 	SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_FLAGS(zfsvfs), NULL,
363 	    &zp->z_pflags, 8);
364 
365 	/*
366 	 * Callers might not be able to detect properly that we are read-only,
367 	 * so check it explicitly here.
368 	 */
369 	if (zfs_is_readonly(zfsvfs)) {
370 		ZFS_EXIT(zfsvfs);
371 		return (SET_ERROR(EROFS));
372 	}
373 
374 	/*
375 	 * If immutable or not appending then return EPERM.
376 	 * Intentionally allow ZFS_READONLY through here.
377 	 * See zfs_zaccess_common()
378 	 */
379 	if ((zp->z_pflags & ZFS_IMMUTABLE) ||
380 	    ((zp->z_pflags & ZFS_APPENDONLY) && !(ioflag & O_APPEND) &&
381 	    (zfs_uio_offset(uio) < zp->z_size))) {
382 		ZFS_EXIT(zfsvfs);
383 		return (SET_ERROR(EPERM));
384 	}
385 
386 	/*
387 	 * Validate file offset
388 	 */
389 	offset_t woff = ioflag & O_APPEND ? zp->z_size : zfs_uio_offset(uio);
390 	if (woff < 0) {
391 		ZFS_EXIT(zfsvfs);
392 		return (SET_ERROR(EINVAL));
393 	}
394 
395 	const uint64_t max_blksz = zfsvfs->z_max_blksz;
396 
397 	/*
398 	 * Pre-fault the pages to ensure slow (eg NFS) pages
399 	 * don't hold up txg.
400 	 * Skip this if uio contains loaned arc_buf.
401 	 */
402 	if (zfs_uio_prefaultpages(MIN(n, max_blksz), uio)) {
403 		ZFS_EXIT(zfsvfs);
404 		return (SET_ERROR(EFAULT));
405 	}
406 
407 	/*
408 	 * If in append mode, set the io offset pointer to eof.
409 	 */
410 	zfs_locked_range_t *lr;
411 	if (ioflag & O_APPEND) {
412 		/*
413 		 * Obtain an appending range lock to guarantee file append
414 		 * semantics.  We reset the write offset once we have the lock.
415 		 */
416 		lr = zfs_rangelock_enter(&zp->z_rangelock, 0, n, RL_APPEND);
417 		woff = lr->lr_offset;
418 		if (lr->lr_length == UINT64_MAX) {
419 			/*
420 			 * We overlocked the file because this write will cause
421 			 * the file block size to increase.
422 			 * Note that zp_size cannot change with this lock held.
423 			 */
424 			woff = zp->z_size;
425 		}
426 		zfs_uio_setoffset(uio, woff);
427 	} else {
428 		/*
429 		 * Note that if the file block size will change as a result of
430 		 * this write, then this range lock will lock the entire file
431 		 * so that we can re-write the block safely.
432 		 */
433 		lr = zfs_rangelock_enter(&zp->z_rangelock, woff, n, RL_WRITER);
434 	}
435 
436 	if (zn_rlimit_fsize(zp, uio)) {
437 		zfs_rangelock_exit(lr);
438 		ZFS_EXIT(zfsvfs);
439 		return (SET_ERROR(EFBIG));
440 	}
441 
442 	const rlim64_t limit = MAXOFFSET_T;
443 
444 	if (woff >= limit) {
445 		zfs_rangelock_exit(lr);
446 		ZFS_EXIT(zfsvfs);
447 		return (SET_ERROR(EFBIG));
448 	}
449 
450 	if (n > limit - woff)
451 		n = limit - woff;
452 
453 	uint64_t end_size = MAX(zp->z_size, woff + n);
454 	zilog_t *zilog = zfsvfs->z_log;
455 
456 	const uint64_t uid = KUID_TO_SUID(ZTOUID(zp));
457 	const uint64_t gid = KGID_TO_SGID(ZTOGID(zp));
458 	const uint64_t projid = zp->z_projid;
459 
460 	/*
461 	 * Write the file in reasonable size chunks.  Each chunk is written
462 	 * in a separate transaction; this keeps the intent log records small
463 	 * and allows us to do more fine-grained space accounting.
464 	 */
465 	while (n > 0) {
466 		woff = zfs_uio_offset(uio);
467 
468 		if (zfs_id_overblockquota(zfsvfs, DMU_USERUSED_OBJECT, uid) ||
469 		    zfs_id_overblockquota(zfsvfs, DMU_GROUPUSED_OBJECT, gid) ||
470 		    (projid != ZFS_DEFAULT_PROJID &&
471 		    zfs_id_overblockquota(zfsvfs, DMU_PROJECTUSED_OBJECT,
472 		    projid))) {
473 			error = SET_ERROR(EDQUOT);
474 			break;
475 		}
476 
477 		arc_buf_t *abuf = NULL;
478 		if (n >= max_blksz && woff >= zp->z_size &&
479 		    P2PHASE(woff, max_blksz) == 0 &&
480 		    zp->z_blksz == max_blksz) {
481 			/*
482 			 * This write covers a full block.  "Borrow" a buffer
483 			 * from the dmu so that we can fill it before we enter
484 			 * a transaction.  This avoids the possibility of
485 			 * holding up the transaction if the data copy hangs
486 			 * up on a pagefault (e.g., from an NFS server mapping).
487 			 */
488 			size_t cbytes;
489 
490 			abuf = dmu_request_arcbuf(sa_get_db(zp->z_sa_hdl),
491 			    max_blksz);
492 			ASSERT(abuf != NULL);
493 			ASSERT(arc_buf_size(abuf) == max_blksz);
494 			if ((error = zfs_uiocopy(abuf->b_data, max_blksz,
495 			    UIO_WRITE, uio, &cbytes))) {
496 				dmu_return_arcbuf(abuf);
497 				break;
498 			}
499 			ASSERT3S(cbytes, ==, max_blksz);
500 		}
501 
502 		/*
503 		 * Start a transaction.
504 		 */
505 		dmu_tx_t *tx = dmu_tx_create(zfsvfs->z_os);
506 		dmu_tx_hold_sa(tx, zp->z_sa_hdl, B_FALSE);
507 		dmu_buf_impl_t *db = (dmu_buf_impl_t *)sa_get_db(zp->z_sa_hdl);
508 		DB_DNODE_ENTER(db);
509 		dmu_tx_hold_write_by_dnode(tx, DB_DNODE(db), woff,
510 		    MIN(n, max_blksz));
511 		DB_DNODE_EXIT(db);
512 		zfs_sa_upgrade_txholds(tx, zp);
513 		error = dmu_tx_assign(tx, TXG_WAIT);
514 		if (error) {
515 			dmu_tx_abort(tx);
516 			if (abuf != NULL)
517 				dmu_return_arcbuf(abuf);
518 			break;
519 		}
520 
521 		/*
522 		 * If rangelock_enter() over-locked we grow the blocksize
523 		 * and then reduce the lock range.  This will only happen
524 		 * on the first iteration since rangelock_reduce() will
525 		 * shrink down lr_length to the appropriate size.
526 		 */
527 		if (lr->lr_length == UINT64_MAX) {
528 			uint64_t new_blksz;
529 
530 			if (zp->z_blksz > max_blksz) {
531 				/*
532 				 * File's blocksize is already larger than the
533 				 * "recordsize" property.  Only let it grow to
534 				 * the next power of 2.
535 				 */
536 				ASSERT(!ISP2(zp->z_blksz));
537 				new_blksz = MIN(end_size,
538 				    1 << highbit64(zp->z_blksz));
539 			} else {
540 				new_blksz = MIN(end_size, max_blksz);
541 			}
542 			zfs_grow_blocksize(zp, new_blksz, tx);
543 			zfs_rangelock_reduce(lr, woff, n);
544 		}
545 
546 		/*
547 		 * XXX - should we really limit each write to z_max_blksz?
548 		 * Perhaps we should use SPA_MAXBLOCKSIZE chunks?
549 		 */
550 		const ssize_t nbytes =
551 		    MIN(n, max_blksz - P2PHASE(woff, max_blksz));
552 
553 		ssize_t tx_bytes;
554 		if (abuf == NULL) {
555 			tx_bytes = zfs_uio_resid(uio);
556 			zfs_uio_fault_disable(uio, B_TRUE);
557 			error = dmu_write_uio_dbuf(sa_get_db(zp->z_sa_hdl),
558 			    uio, nbytes, tx);
559 			zfs_uio_fault_disable(uio, B_FALSE);
560 #ifdef __linux__
561 			if (error == EFAULT) {
562 				dmu_tx_commit(tx);
563 				/*
564 				 * Account for partial writes before
565 				 * continuing the loop.
566 				 * Update needs to occur before the next
567 				 * zfs_uio_prefaultpages, or prefaultpages may
568 				 * error, and we may break the loop early.
569 				 */
570 				if (tx_bytes != zfs_uio_resid(uio))
571 					n -= tx_bytes - zfs_uio_resid(uio);
572 				if (zfs_uio_prefaultpages(MIN(n, max_blksz),
573 				    uio)) {
574 					break;
575 				}
576 				continue;
577 			}
578 #endif
579 			/*
580 			 * On FreeBSD, EFAULT should be propagated back to the
581 			 * VFS, which will handle faulting and will retry.
582 			 */
583 			if (error != 0 && error != EFAULT) {
584 				dmu_tx_commit(tx);
585 				break;
586 			}
587 			tx_bytes -= zfs_uio_resid(uio);
588 		} else {
589 			/* Implied by abuf != NULL: */
590 			ASSERT3S(n, >=, max_blksz);
591 			ASSERT0(P2PHASE(woff, max_blksz));
592 			/*
593 			 * We can simplify nbytes to MIN(n, max_blksz) since
594 			 * P2PHASE(woff, max_blksz) is 0, and knowing
595 			 * n >= max_blksz lets us simplify further:
596 			 */
597 			ASSERT3S(nbytes, ==, max_blksz);
598 			/*
599 			 * Thus, we're writing a full block at a block-aligned
600 			 * offset and extending the file past EOF.
601 			 *
602 			 * dmu_assign_arcbuf_by_dbuf() will directly assign the
603 			 * arc buffer to a dbuf.
604 			 */
605 			error = dmu_assign_arcbuf_by_dbuf(
606 			    sa_get_db(zp->z_sa_hdl), woff, abuf, tx);
607 			if (error != 0) {
608 				dmu_return_arcbuf(abuf);
609 				dmu_tx_commit(tx);
610 				break;
611 			}
612 			ASSERT3S(nbytes, <=, zfs_uio_resid(uio));
613 			zfs_uioskip(uio, nbytes);
614 			tx_bytes = nbytes;
615 		}
616 		if (tx_bytes && zn_has_cached_data(zp) &&
617 		    !(ioflag & O_DIRECT)) {
618 			update_pages(zp, woff, tx_bytes, zfsvfs->z_os);
619 		}
620 
621 		/*
622 		 * If we made no progress, we're done.  If we made even
623 		 * partial progress, update the znode and ZIL accordingly.
624 		 */
625 		if (tx_bytes == 0) {
626 			(void) sa_update(zp->z_sa_hdl, SA_ZPL_SIZE(zfsvfs),
627 			    (void *)&zp->z_size, sizeof (uint64_t), tx);
628 			dmu_tx_commit(tx);
629 			ASSERT(error != 0);
630 			break;
631 		}
632 
633 		/*
634 		 * Clear Set-UID/Set-GID bits on successful write if not
635 		 * privileged and at least one of the execute bits is set.
636 		 *
637 		 * It would be nice to do this after all writes have
638 		 * been done, but that would still expose the ISUID/ISGID
639 		 * to another app after the partial write is committed.
640 		 *
641 		 * Note: we don't call zfs_fuid_map_id() here because
642 		 * user 0 is not an ephemeral uid.
643 		 */
644 		mutex_enter(&zp->z_acl_lock);
645 		if ((zp->z_mode & (S_IXUSR | (S_IXUSR >> 3) |
646 		    (S_IXUSR >> 6))) != 0 &&
647 		    (zp->z_mode & (S_ISUID | S_ISGID)) != 0 &&
648 		    secpolicy_vnode_setid_retain(zp, cr,
649 		    ((zp->z_mode & S_ISUID) != 0 && uid == 0)) != 0) {
650 			uint64_t newmode;
651 			zp->z_mode &= ~(S_ISUID | S_ISGID);
652 			newmode = zp->z_mode;
653 			(void) sa_update(zp->z_sa_hdl, SA_ZPL_MODE(zfsvfs),
654 			    (void *)&newmode, sizeof (uint64_t), tx);
655 		}
656 		mutex_exit(&zp->z_acl_lock);
657 
658 		zfs_tstamp_update_setup(zp, CONTENT_MODIFIED, mtime, ctime);
659 
660 		/*
661 		 * Update the file size (zp_size) if it has changed;
662 		 * account for possible concurrent updates.
663 		 */
664 		while ((end_size = zp->z_size) < zfs_uio_offset(uio)) {
665 			(void) atomic_cas_64(&zp->z_size, end_size,
666 			    zfs_uio_offset(uio));
667 			ASSERT(error == 0 || error == EFAULT);
668 		}
669 		/*
670 		 * If we are replaying and eof is non zero then force
671 		 * the file size to the specified eof. Note, there's no
672 		 * concurrency during replay.
673 		 */
674 		if (zfsvfs->z_replay && zfsvfs->z_replay_eof != 0)
675 			zp->z_size = zfsvfs->z_replay_eof;
676 
677 		error1 = sa_bulk_update(zp->z_sa_hdl, bulk, count, tx);
678 		if (error1 != 0)
679 			/* Avoid clobbering EFAULT. */
680 			error = error1;
681 
682 		zfs_log_write(zilog, tx, TX_WRITE, zp, woff, tx_bytes, ioflag,
683 		    NULL, NULL);
684 		dmu_tx_commit(tx);
685 
686 		if (error != 0)
687 			break;
688 		ASSERT3S(tx_bytes, ==, nbytes);
689 		n -= nbytes;
690 
691 		if (n > 0) {
692 			if (zfs_uio_prefaultpages(MIN(n, max_blksz), uio)) {
693 				error = SET_ERROR(EFAULT);
694 				break;
695 			}
696 		}
697 	}
698 
699 	zfs_znode_update_vfs(zp);
700 	zfs_rangelock_exit(lr);
701 
702 	/*
703 	 * If we're in replay mode, or we made no progress, or the
704 	 * uio data is inaccessible return an error.  Otherwise, it's
705 	 * at least a partial write, so it's successful.
706 	 */
707 	if (zfsvfs->z_replay || zfs_uio_resid(uio) == start_resid ||
708 	    error == EFAULT) {
709 		ZFS_EXIT(zfsvfs);
710 		return (error);
711 	}
712 
713 	if (ioflag & (O_SYNC | O_DSYNC) ||
714 	    zfsvfs->z_os->os_sync == ZFS_SYNC_ALWAYS)
715 		zil_commit(zilog, zp->z_id);
716 
717 	const int64_t nwritten = start_resid - zfs_uio_resid(uio);
718 	dataset_kstats_update_write_kstats(&zfsvfs->z_kstat, nwritten);
719 	task_io_account_write(nwritten);
720 
721 	ZFS_EXIT(zfsvfs);
722 	return (0);
723 }
724 
725 /*ARGSUSED*/
726 int
727 zfs_getsecattr(znode_t *zp, vsecattr_t *vsecp, int flag, cred_t *cr)
728 {
729 	zfsvfs_t *zfsvfs = ZTOZSB(zp);
730 	int error;
731 	boolean_t skipaclchk = (flag & ATTR_NOACLCHECK) ? B_TRUE : B_FALSE;
732 
733 	ZFS_ENTER(zfsvfs);
734 	ZFS_VERIFY_ZP(zp);
735 	error = zfs_getacl(zp, vsecp, skipaclchk, cr);
736 	ZFS_EXIT(zfsvfs);
737 
738 	return (error);
739 }
740 
741 /*ARGSUSED*/
742 int
743 zfs_setsecattr(znode_t *zp, vsecattr_t *vsecp, int flag, cred_t *cr)
744 {
745 	zfsvfs_t *zfsvfs = ZTOZSB(zp);
746 	int error;
747 	boolean_t skipaclchk = (flag & ATTR_NOACLCHECK) ? B_TRUE : B_FALSE;
748 	zilog_t	*zilog = zfsvfs->z_log;
749 
750 	ZFS_ENTER(zfsvfs);
751 	ZFS_VERIFY_ZP(zp);
752 
753 	error = zfs_setacl(zp, vsecp, skipaclchk, cr);
754 
755 	if (zfsvfs->z_os->os_sync == ZFS_SYNC_ALWAYS)
756 		zil_commit(zilog, 0);
757 
758 	ZFS_EXIT(zfsvfs);
759 	return (error);
760 }
761 
762 #ifdef ZFS_DEBUG
763 static int zil_fault_io = 0;
764 #endif
765 
766 static void zfs_get_done(zgd_t *zgd, int error);
767 
768 /*
769  * Get data to generate a TX_WRITE intent log record.
770  */
771 int
772 zfs_get_data(void *arg, uint64_t gen, lr_write_t *lr, char *buf,
773     struct lwb *lwb, zio_t *zio)
774 {
775 	zfsvfs_t *zfsvfs = arg;
776 	objset_t *os = zfsvfs->z_os;
777 	znode_t *zp;
778 	uint64_t object = lr->lr_foid;
779 	uint64_t offset = lr->lr_offset;
780 	uint64_t size = lr->lr_length;
781 	dmu_buf_t *db;
782 	zgd_t *zgd;
783 	int error = 0;
784 	uint64_t zp_gen;
785 
786 	ASSERT3P(lwb, !=, NULL);
787 	ASSERT3P(zio, !=, NULL);
788 	ASSERT3U(size, !=, 0);
789 
790 	/*
791 	 * Nothing to do if the file has been removed
792 	 */
793 	if (zfs_zget(zfsvfs, object, &zp) != 0)
794 		return (SET_ERROR(ENOENT));
795 	if (zp->z_unlinked) {
796 		/*
797 		 * Release the vnode asynchronously as we currently have the
798 		 * txg stopped from syncing.
799 		 */
800 		zfs_zrele_async(zp);
801 		return (SET_ERROR(ENOENT));
802 	}
803 	/* check if generation number matches */
804 	if (sa_lookup(zp->z_sa_hdl, SA_ZPL_GEN(zfsvfs), &zp_gen,
805 	    sizeof (zp_gen)) != 0) {
806 		zfs_zrele_async(zp);
807 		return (SET_ERROR(EIO));
808 	}
809 	if (zp_gen != gen) {
810 		zfs_zrele_async(zp);
811 		return (SET_ERROR(ENOENT));
812 	}
813 
814 	zgd = (zgd_t *)kmem_zalloc(sizeof (zgd_t), KM_SLEEP);
815 	zgd->zgd_lwb = lwb;
816 	zgd->zgd_private = zp;
817 
818 	/*
819 	 * Write records come in two flavors: immediate and indirect.
820 	 * For small writes it's cheaper to store the data with the
821 	 * log record (immediate); for large writes it's cheaper to
822 	 * sync the data and get a pointer to it (indirect) so that
823 	 * we don't have to write the data twice.
824 	 */
825 	if (buf != NULL) { /* immediate write */
826 		zgd->zgd_lr = zfs_rangelock_enter(&zp->z_rangelock,
827 		    offset, size, RL_READER);
828 		/* test for truncation needs to be done while range locked */
829 		if (offset >= zp->z_size) {
830 			error = SET_ERROR(ENOENT);
831 		} else {
832 			error = dmu_read(os, object, offset, size, buf,
833 			    DMU_READ_NO_PREFETCH);
834 		}
835 		ASSERT(error == 0 || error == ENOENT);
836 	} else { /* indirect write */
837 		/*
838 		 * Have to lock the whole block to ensure when it's
839 		 * written out and its checksum is being calculated
840 		 * that no one can change the data. We need to re-check
841 		 * blocksize after we get the lock in case it's changed!
842 		 */
843 		for (;;) {
844 			uint64_t blkoff;
845 			size = zp->z_blksz;
846 			blkoff = ISP2(size) ? P2PHASE(offset, size) : offset;
847 			offset -= blkoff;
848 			zgd->zgd_lr = zfs_rangelock_enter(&zp->z_rangelock,
849 			    offset, size, RL_READER);
850 			if (zp->z_blksz == size)
851 				break;
852 			offset += blkoff;
853 			zfs_rangelock_exit(zgd->zgd_lr);
854 		}
855 		/* test for truncation needs to be done while range locked */
856 		if (lr->lr_offset >= zp->z_size)
857 			error = SET_ERROR(ENOENT);
858 #ifdef ZFS_DEBUG
859 		if (zil_fault_io) {
860 			error = SET_ERROR(EIO);
861 			zil_fault_io = 0;
862 		}
863 #endif
864 		if (error == 0)
865 			error = dmu_buf_hold(os, object, offset, zgd, &db,
866 			    DMU_READ_NO_PREFETCH);
867 
868 		if (error == 0) {
869 			blkptr_t *bp = &lr->lr_blkptr;
870 
871 			zgd->zgd_db = db;
872 			zgd->zgd_bp = bp;
873 
874 			ASSERT(db->db_offset == offset);
875 			ASSERT(db->db_size == size);
876 
877 			error = dmu_sync(zio, lr->lr_common.lrc_txg,
878 			    zfs_get_done, zgd);
879 			ASSERT(error || lr->lr_length <= size);
880 
881 			/*
882 			 * On success, we need to wait for the write I/O
883 			 * initiated by dmu_sync() to complete before we can
884 			 * release this dbuf.  We will finish everything up
885 			 * in the zfs_get_done() callback.
886 			 */
887 			if (error == 0)
888 				return (0);
889 
890 			if (error == EALREADY) {
891 				lr->lr_common.lrc_txtype = TX_WRITE2;
892 				/*
893 				 * TX_WRITE2 relies on the data previously
894 				 * written by the TX_WRITE that caused
895 				 * EALREADY.  We zero out the BP because
896 				 * it is the old, currently-on-disk BP.
897 				 */
898 				zgd->zgd_bp = NULL;
899 				BP_ZERO(bp);
900 				error = 0;
901 			}
902 		}
903 	}
904 
905 	zfs_get_done(zgd, error);
906 
907 	return (error);
908 }
909 
910 
911 /* ARGSUSED */
912 static void
913 zfs_get_done(zgd_t *zgd, int error)
914 {
915 	znode_t *zp = zgd->zgd_private;
916 
917 	if (zgd->zgd_db)
918 		dmu_buf_rele(zgd->zgd_db, zgd);
919 
920 	zfs_rangelock_exit(zgd->zgd_lr);
921 
922 	/*
923 	 * Release the vnode asynchronously as we currently have the
924 	 * txg stopped from syncing.
925 	 */
926 	zfs_zrele_async(zp);
927 
928 	kmem_free(zgd, sizeof (zgd_t));
929 }
930 
931 EXPORT_SYMBOL(zfs_access);
932 EXPORT_SYMBOL(zfs_fsync);
933 EXPORT_SYMBOL(zfs_holey);
934 EXPORT_SYMBOL(zfs_read);
935 EXPORT_SYMBOL(zfs_write);
936 EXPORT_SYMBOL(zfs_getsecattr);
937 EXPORT_SYMBOL(zfs_setsecattr);
938 
939 ZFS_MODULE_PARAM(zfs_vnops, zfs_vnops_, read_chunk_size, ULONG, ZMOD_RW,
940 	"Bytes to read per chunk");
941