xref: /illumos-gate/usr/src/uts/common/fs/zfs/zfs_vnops.c (revision 753d2d2e8e7fd0c9bcf736d9bf2f2faf4d6234cc)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 /*
22  * Copyright 2006 Sun Microsystems, Inc.  All rights reserved.
23  * Use is subject to license terms.
24  */
25 
26 #pragma ident	"%Z%%M%	%I%	%E% SMI"
27 
28 #include <sys/types.h>
29 #include <sys/param.h>
30 #include <sys/time.h>
31 #include <sys/systm.h>
32 #include <sys/sysmacros.h>
33 #include <sys/resource.h>
34 #include <sys/vfs.h>
35 #include <sys/vnode.h>
36 #include <sys/file.h>
37 #include <sys/stat.h>
38 #include <sys/kmem.h>
39 #include <sys/taskq.h>
40 #include <sys/uio.h>
41 #include <sys/vmsystm.h>
42 #include <sys/atomic.h>
43 #include <sys/vm.h>
44 #include <vm/seg_vn.h>
45 #include <vm/pvn.h>
46 #include <vm/as.h>
47 #include <sys/mman.h>
48 #include <sys/pathname.h>
49 #include <sys/cmn_err.h>
50 #include <sys/errno.h>
51 #include <sys/unistd.h>
52 #include <sys/zfs_vfsops.h>
53 #include <sys/zfs_dir.h>
54 #include <sys/zfs_acl.h>
55 #include <sys/zfs_ioctl.h>
56 #include <sys/fs/zfs.h>
57 #include <sys/dmu.h>
58 #include <sys/spa.h>
59 #include <sys/txg.h>
60 #include <sys/dbuf.h>
61 #include <sys/zap.h>
62 #include <sys/dirent.h>
63 #include <sys/policy.h>
64 #include <sys/sunddi.h>
65 #include <sys/filio.h>
66 #include "fs/fs_subr.h"
67 #include <sys/zfs_ctldir.h>
68 #include <sys/dnlc.h>
69 #include <sys/zfs_rlock.h>
70 
71 /*
72  * Programming rules.
73  *
74  * Each vnode op performs some logical unit of work.  To do this, the ZPL must
75  * properly lock its in-core state, create a DMU transaction, do the work,
76  * record this work in the intent log (ZIL), commit the DMU transaction,
77  * and wait the the intent log to commit if it's is a synchronous operation.
78  * Morover, the vnode ops must work in both normal and log replay context.
79  * The ordering of events is important to avoid deadlocks and references
80  * to freed memory.  The example below illustrates the following Big Rules:
81  *
82  *  (1) A check must be made in each zfs thread for a mounted file system.
83  *	This is done avoiding races using ZFS_ENTER(zfsvfs).
84  *	A ZFS_EXIT(zfsvfs) is needed before all returns.
85  *
86  *  (2)	VN_RELE() should always be the last thing except for zil_commit()
87  *	(if necessary) and ZFS_EXIT(). This is for 3 reasons:
88  *	First, if it's the last reference, the vnode/znode
89  *	can be freed, so the zp may point to freed memory.  Second, the last
90  *	reference will call zfs_zinactive(), which may induce a lot of work --
91  *	pushing cached pages (which acquires range locks) and syncing out
92  *	cached atime changes.  Third, zfs_zinactive() may require a new tx,
93  *	which could deadlock the system if you were already holding one.
94  *
95  *  (3)	All range locks must be grabbed before calling dmu_tx_assign(),
96  *	as they can span dmu_tx_assign() calls.
97  *
98  *  (4)	Always pass zfsvfs->z_assign as the second argument to dmu_tx_assign().
99  *	In normal operation, this will be TXG_NOWAIT.  During ZIL replay,
100  *	it will be a specific txg.  Either way, dmu_tx_assign() never blocks.
101  *	This is critical because we don't want to block while holding locks.
102  *	Note, in particular, that if a lock is sometimes acquired before
103  *	the tx assigns, and sometimes after (e.g. z_lock), then failing to
104  *	use a non-blocking assign can deadlock the system.  The scenario:
105  *
106  *	Thread A has grabbed a lock before calling dmu_tx_assign().
107  *	Thread B is in an already-assigned tx, and blocks for this lock.
108  *	Thread A calls dmu_tx_assign(TXG_WAIT) and blocks in txg_wait_open()
109  *	forever, because the previous txg can't quiesce until B's tx commits.
110  *
111  *	If dmu_tx_assign() returns ERESTART and zfsvfs->z_assign is TXG_NOWAIT,
112  *	then drop all locks, call dmu_tx_wait(), and try again.
113  *
114  *  (5)	If the operation succeeded, generate the intent log entry for it
115  *	before dropping locks.  This ensures that the ordering of events
116  *	in the intent log matches the order in which they actually occurred.
117  *
118  *  (6)	At the end of each vnode op, the DMU tx must always commit,
119  *	regardless of whether there were any errors.
120  *
121  *  (7)	After dropping all locks, invoke zil_commit(zilog, seq, foid)
122  *	to ensure that synchronous semantics are provided when necessary.
123  *
124  * In general, this is how things should be ordered in each vnode op:
125  *
126  *	ZFS_ENTER(zfsvfs);		// exit if unmounted
127  * top:
128  *	zfs_dirent_lock(&dl, ...)	// lock directory entry (may VN_HOLD())
129  *	rw_enter(...);			// grab any other locks you need
130  *	tx = dmu_tx_create(...);	// get DMU tx
131  *	dmu_tx_hold_*();		// hold each object you might modify
132  *	error = dmu_tx_assign(tx, zfsvfs->z_assign);	// try to assign
133  *	if (error) {
134  *		rw_exit(...);		// drop locks
135  *		zfs_dirent_unlock(dl);	// unlock directory entry
136  *		VN_RELE(...);		// release held vnodes
137  *		if (error == ERESTART && zfsvfs->z_assign == TXG_NOWAIT) {
138  *			dmu_tx_wait(tx);
139  *			dmu_tx_abort(tx);
140  *			goto top;
141  *		}
142  *		dmu_tx_abort(tx);	// abort DMU tx
143  *		ZFS_EXIT(zfsvfs);	// finished in zfs
144  *		return (error);		// really out of space
145  *	}
146  *	error = do_real_work();		// do whatever this VOP does
147  *	if (error == 0)
148  *		zfs_log_*(...);		// on success, make ZIL entry
149  *	dmu_tx_commit(tx);		// commit DMU tx -- error or not
150  *	rw_exit(...);			// drop locks
151  *	zfs_dirent_unlock(dl);		// unlock directory entry
152  *	VN_RELE(...);			// release held vnodes
153  *	zil_commit(zilog, seq, foid);	// synchronous when necessary
154  *	ZFS_EXIT(zfsvfs);		// finished in zfs
155  *	return (error);			// done, report error
156  */
157 
158 /* ARGSUSED */
159 static int
160 zfs_open(vnode_t **vpp, int flag, cred_t *cr)
161 {
162 	return (0);
163 }
164 
165 /* ARGSUSED */
166 static int
167 zfs_close(vnode_t *vp, int flag, int count, offset_t offset, cred_t *cr)
168 {
169 	/*
170 	 * Clean up any locks held by this process on the vp.
171 	 */
172 	cleanlocks(vp, ddi_get_pid(), 0);
173 	cleanshares(vp, ddi_get_pid());
174 
175 	return (0);
176 }
177 
178 /*
179  * Lseek support for finding holes (cmd == _FIO_SEEK_HOLE) and
180  * data (cmd == _FIO_SEEK_DATA). "off" is an in/out parameter.
181  */
182 static int
183 zfs_holey(vnode_t *vp, int cmd, offset_t *off)
184 {
185 	znode_t	*zp = VTOZ(vp);
186 	uint64_t noff = (uint64_t)*off; /* new offset */
187 	uint64_t file_sz;
188 	int error;
189 	boolean_t hole;
190 
191 	file_sz = zp->z_phys->zp_size;
192 	if (noff >= file_sz)  {
193 		return (ENXIO);
194 	}
195 
196 	if (cmd == _FIO_SEEK_HOLE)
197 		hole = B_TRUE;
198 	else
199 		hole = B_FALSE;
200 
201 	error = dmu_offset_next(zp->z_zfsvfs->z_os, zp->z_id, hole, &noff);
202 
203 	/* end of file? */
204 	if ((error == ESRCH) || (noff > file_sz)) {
205 		/*
206 		 * Handle the virtual hole at the end of file.
207 		 */
208 		if (hole) {
209 			*off = file_sz;
210 			return (0);
211 		}
212 		return (ENXIO);
213 	}
214 
215 	if (noff < *off)
216 		return (error);
217 	*off = noff;
218 	return (error);
219 }
220 
221 /* ARGSUSED */
222 static int
223 zfs_ioctl(vnode_t *vp, int com, intptr_t data, int flag, cred_t *cred,
224     int *rvalp)
225 {
226 	offset_t off;
227 	int error;
228 	zfsvfs_t *zfsvfs;
229 
230 	switch (com) {
231 	    case _FIOFFS:
232 		return (zfs_sync(vp->v_vfsp, 0, cred));
233 
234 		/*
235 		 * The following two ioctls are used by bfu.  Faking out,
236 		 * necessary to avoid bfu errors.
237 		 */
238 	    case _FIOGDIO:
239 	    case _FIOSDIO:
240 		return (0);
241 
242 	    case _FIO_SEEK_DATA:
243 	    case _FIO_SEEK_HOLE:
244 		if (ddi_copyin((void *)data, &off, sizeof (off), flag))
245 			return (EFAULT);
246 
247 		zfsvfs = VTOZ(vp)->z_zfsvfs;
248 		ZFS_ENTER(zfsvfs);
249 
250 		/* offset parameter is in/out */
251 		error = zfs_holey(vp, com, &off);
252 		ZFS_EXIT(zfsvfs);
253 		if (error)
254 			return (error);
255 		if (ddi_copyout(&off, (void *)data, sizeof (off), flag))
256 			return (EFAULT);
257 		return (0);
258 	}
259 	return (ENOTTY);
260 }
261 
262 /*
263  * When a file is memory mapped, we must keep the IO data synchronized
264  * between the DMU cache and the memory mapped pages.  What this means:
265  *
266  * On Write:	If we find a memory mapped page, we write to *both*
267  *		the page and the dmu buffer.
268  *
269  * NOTE: We will always "break up" the IO into PAGESIZE uiomoves when
270  *	the file is memory mapped.
271  */
272 static int
273 mappedwrite(vnode_t *vp, uint64_t woff, int nbytes, uio_t *uio, dmu_tx_t *tx)
274 {
275 	znode_t	*zp = VTOZ(vp);
276 	zfsvfs_t *zfsvfs = zp->z_zfsvfs;
277 	int64_t	start, off;
278 	int len = nbytes;
279 	int error = 0;
280 
281 	start = uio->uio_loffset;
282 	off = start & PAGEOFFSET;
283 	for (start &= PAGEMASK; len > 0; start += PAGESIZE) {
284 		page_t *pp;
285 		uint64_t bytes = MIN(PAGESIZE - off, len);
286 
287 		/*
288 		 * We don't want a new page to "appear" in the middle of
289 		 * the file update (because it may not get the write
290 		 * update data), so we grab a lock to block
291 		 * zfs_getpage().
292 		 */
293 		rw_enter(&zp->z_map_lock, RW_WRITER);
294 		if (pp = page_lookup(vp, start, SE_SHARED)) {
295 			caddr_t va;
296 
297 			rw_exit(&zp->z_map_lock);
298 			va = ppmapin(pp, PROT_READ | PROT_WRITE, (caddr_t)-1L);
299 			error = uiomove(va+off, bytes, UIO_WRITE, uio);
300 			if (error == 0) {
301 				dmu_write(zfsvfs->z_os, zp->z_id,
302 				    woff, bytes, va+off, tx);
303 			}
304 			ppmapout(va);
305 			page_unlock(pp);
306 		} else {
307 			error = dmu_write_uio(zfsvfs->z_os, zp->z_id,
308 			    woff, bytes, uio, tx);
309 			rw_exit(&zp->z_map_lock);
310 		}
311 		len -= bytes;
312 		woff += bytes;
313 		off = 0;
314 		if (error)
315 			break;
316 	}
317 	return (error);
318 }
319 
320 /*
321  * When a file is memory mapped, we must keep the IO data synchronized
322  * between the DMU cache and the memory mapped pages.  What this means:
323  *
324  * On Read:	We "read" preferentially from memory mapped pages,
325  *		else we default from the dmu buffer.
326  *
327  * NOTE: We will always "break up" the IO into PAGESIZE uiomoves when
328  *	the file is memory mapped.
329  */
330 static int
331 mappedread(vnode_t *vp, char *addr, int nbytes, uio_t *uio)
332 {
333 	int64_t	start, off, bytes;
334 	int len = nbytes;
335 	int error = 0;
336 
337 	start = uio->uio_loffset;
338 	off = start & PAGEOFFSET;
339 	for (start &= PAGEMASK; len > 0; start += PAGESIZE) {
340 		page_t *pp;
341 
342 		bytes = MIN(PAGESIZE - off, len);
343 		if (pp = page_lookup(vp, start, SE_SHARED)) {
344 			caddr_t va;
345 
346 			va = ppmapin(pp, PROT_READ, (caddr_t)-1L);
347 			error = uiomove(va + off, bytes, UIO_READ, uio);
348 			ppmapout(va);
349 			page_unlock(pp);
350 		} else {
351 			/* XXX use dmu_read here? */
352 			error = uiomove(addr, bytes, UIO_READ, uio);
353 		}
354 		len -= bytes;
355 		addr += bytes;
356 		off = 0;
357 		if (error)
358 			break;
359 	}
360 	return (error);
361 }
362 
363 uint_t zfs_read_chunk_size = 1024 * 1024; /* Tunable */
364 
365 /*
366  * Read bytes from specified file into supplied buffer.
367  *
368  *	IN:	vp	- vnode of file to be read from.
369  *		uio	- structure supplying read location, range info,
370  *			  and return buffer.
371  *		ioflag	- SYNC flags; used to provide FRSYNC semantics.
372  *		cr	- credentials of caller.
373  *
374  *	OUT:	uio	- updated offset and range, buffer filled.
375  *
376  *	RETURN:	0 if success
377  *		error code if failure
378  *
379  * Side Effects:
380  *	vp - atime updated if byte count > 0
381  */
382 /* ARGSUSED */
383 static int
384 zfs_read(vnode_t *vp, uio_t *uio, int ioflag, cred_t *cr, caller_context_t *ct)
385 {
386 	znode_t		*zp = VTOZ(vp);
387 	zfsvfs_t	*zfsvfs = zp->z_zfsvfs;
388 	uint64_t	delta;
389 	ssize_t		n, size, cnt, ndone;
390 	int		error, i, numbufs;
391 	dmu_buf_t	*dbp, **dbpp;
392 	rl_t		*rl;
393 
394 	ZFS_ENTER(zfsvfs);
395 
396 	/*
397 	 * Validate file offset
398 	 */
399 	if (uio->uio_loffset < (offset_t)0) {
400 		ZFS_EXIT(zfsvfs);
401 		return (EINVAL);
402 	}
403 
404 	/*
405 	 * Fasttrack empty reads
406 	 */
407 	if (uio->uio_resid == 0) {
408 		ZFS_EXIT(zfsvfs);
409 		return (0);
410 	}
411 
412 	/*
413 	 * Check for mandatory locks
414 	 */
415 	if (MANDMODE((mode_t)zp->z_phys->zp_mode)) {
416 		if (error = chklock(vp, FREAD,
417 		    uio->uio_loffset, uio->uio_resid, uio->uio_fmode, ct)) {
418 			ZFS_EXIT(zfsvfs);
419 			return (error);
420 		}
421 	}
422 
423 	/*
424 	 * If we're in FRSYNC mode, sync out this znode before reading it.
425 	 */
426 	if (ioflag & FRSYNC)
427 		zil_commit(zfsvfs->z_log, zp->z_last_itx, zp->z_id);
428 
429 	/*
430 	 * Lock the range against changes.
431 	 */
432 	rl = zfs_range_lock(zp, uio->uio_loffset, uio->uio_resid, RL_READER);
433 
434 	/*
435 	 * If we are reading past end-of-file we can skip
436 	 * to the end; but we might still need to set atime.
437 	 */
438 	if (uio->uio_loffset >= zp->z_phys->zp_size) {
439 		cnt = 0;
440 		error = 0;
441 		goto out;
442 	}
443 
444 	cnt = MIN(uio->uio_resid, zp->z_phys->zp_size - uio->uio_loffset);
445 
446 	for (ndone = 0; ndone < cnt; ndone += zfs_read_chunk_size) {
447 		ASSERT(uio->uio_loffset < zp->z_phys->zp_size);
448 		n = MIN(zfs_read_chunk_size,
449 		    zp->z_phys->zp_size - uio->uio_loffset);
450 		n = MIN(n, cnt);
451 		error = dmu_buf_hold_array_by_bonus(zp->z_dbuf,
452 		    uio->uio_loffset, n, TRUE, FTAG, &numbufs, &dbpp);
453 		if (error)
454 			goto out;
455 		/*
456 		 * Compute the adjustment to align the dmu buffers
457 		 * with the uio buffer.
458 		 */
459 		delta = uio->uio_loffset - dbpp[0]->db_offset;
460 
461 		for (i = 0; i < numbufs; i++) {
462 			if (n < 0)
463 				break;
464 			dbp = dbpp[i];
465 			size = dbp->db_size - delta;
466 			/*
467 			 * XXX -- this is correct, but may be suboptimal.
468 			 * If the pages are all clean, we don't need to
469 			 * go through mappedread().  Maybe the VMODSORT
470 			 * stuff can help us here.
471 			 */
472 			if (vn_has_cached_data(vp)) {
473 				error = mappedread(vp, (caddr_t)dbp->db_data +
474 				    delta, (n < size ? n : size), uio);
475 			} else {
476 				error = uiomove((caddr_t)dbp->db_data + delta,
477 					(n < size ? n : size), UIO_READ, uio);
478 			}
479 			if (error) {
480 				dmu_buf_rele_array(dbpp, numbufs, FTAG);
481 				goto out;
482 			}
483 			n -= dbp->db_size;
484 			if (delta) {
485 				n += delta;
486 				delta = 0;
487 			}
488 		}
489 		dmu_buf_rele_array(dbpp, numbufs, FTAG);
490 	}
491 out:
492 	zfs_range_unlock(rl);
493 
494 	ZFS_ACCESSTIME_STAMP(zfsvfs, zp);
495 	ZFS_EXIT(zfsvfs);
496 	return (error);
497 }
498 
499 /*
500  * Fault in the pages of the first n bytes specified by the uio structure.
501  * 1 byte in each page is touched and the uio struct is unmodified.
502  * Any error will exit this routine as this is only a best
503  * attempt to get the pages resident. This is a copy of ufs_trans_touch().
504  */
505 static void
506 zfs_prefault_write(ssize_t n, struct uio *uio)
507 {
508 	struct iovec *iov;
509 	ulong_t cnt, incr;
510 	caddr_t p;
511 	uint8_t tmp;
512 
513 	iov = uio->uio_iov;
514 
515 	while (n) {
516 		cnt = MIN(iov->iov_len, n);
517 		if (cnt == 0) {
518 			/* empty iov entry */
519 			iov++;
520 			continue;
521 		}
522 		n -= cnt;
523 		/*
524 		 * touch each page in this segment.
525 		 */
526 		p = iov->iov_base;
527 		while (cnt) {
528 			switch (uio->uio_segflg) {
529 			case UIO_USERSPACE:
530 			case UIO_USERISPACE:
531 				if (fuword8(p, &tmp))
532 					return;
533 				break;
534 			case UIO_SYSSPACE:
535 				if (kcopy(p, &tmp, 1))
536 					return;
537 				break;
538 			}
539 			incr = MIN(cnt, PAGESIZE);
540 			p += incr;
541 			cnt -= incr;
542 		}
543 		/*
544 		 * touch the last byte in case it straddles a page.
545 		 */
546 		p--;
547 		switch (uio->uio_segflg) {
548 		case UIO_USERSPACE:
549 		case UIO_USERISPACE:
550 			if (fuword8(p, &tmp))
551 				return;
552 			break;
553 		case UIO_SYSSPACE:
554 			if (kcopy(p, &tmp, 1))
555 				return;
556 			break;
557 		}
558 		iov++;
559 	}
560 }
561 
562 /*
563  * Write the bytes to a file.
564  *
565  *	IN:	vp	- vnode of file to be written to.
566  *		uio	- structure supplying write location, range info,
567  *			  and data buffer.
568  *		ioflag	- FAPPEND flag set if in append mode.
569  *		cr	- credentials of caller.
570  *
571  *	OUT:	uio	- updated offset and range.
572  *
573  *	RETURN:	0 if success
574  *		error code if failure
575  *
576  * Timestamps:
577  *	vp - ctime|mtime updated if byte count > 0
578  */
579 /* ARGSUSED */
580 static int
581 zfs_write(vnode_t *vp, uio_t *uio, int ioflag, cred_t *cr, caller_context_t *ct)
582 {
583 	znode_t		*zp = VTOZ(vp);
584 	rlim64_t	limit = uio->uio_llimit;
585 	ssize_t		start_resid = uio->uio_resid;
586 	ssize_t		tx_bytes;
587 	uint64_t	end_size;
588 	dmu_tx_t	*tx;
589 	zfsvfs_t	*zfsvfs = zp->z_zfsvfs;
590 	zilog_t		*zilog = zfsvfs->z_log;
591 	offset_t	woff;
592 	ssize_t		n, nbytes;
593 	rl_t		*rl;
594 	int		max_blksz = zfsvfs->z_max_blksz;
595 	int		error;
596 
597 	/*
598 	 * Fasttrack empty write
599 	 */
600 	n = start_resid;
601 	if (n == 0)
602 		return (0);
603 
604 	if (limit == RLIM64_INFINITY || limit > MAXOFFSET_T)
605 		limit = MAXOFFSET_T;
606 
607 	ZFS_ENTER(zfsvfs);
608 
609 	/*
610 	 * Pre-fault the pages to ensure slow (eg NFS) pages
611 	 * don't hold up txg.
612 	 */
613 	zfs_prefault_write(n, uio);
614 
615 	/*
616 	 * If in append mode, set the io offset pointer to eof.
617 	 */
618 	if (ioflag & FAPPEND) {
619 		/*
620 		 * Range lock for a file append:
621 		 * The value for the start of range will be determined by
622 		 * zfs_range_lock() (to guarantee append semantics).
623 		 * If this write will cause the block size to increase,
624 		 * zfs_range_lock() will lock the entire file, so we must
625 		 * later reduce the range after we grow the block size.
626 		 */
627 		rl = zfs_range_lock(zp, 0, n, RL_APPEND);
628 		if (rl->r_len == UINT64_MAX) {
629 			/* overlocked, zp_size can't change */
630 			woff = uio->uio_loffset = zp->z_phys->zp_size;
631 		} else {
632 			woff = uio->uio_loffset = rl->r_off;
633 		}
634 	} else {
635 		woff = uio->uio_loffset;
636 		/*
637 		 * Validate file offset
638 		 */
639 		if (woff < 0) {
640 			ZFS_EXIT(zfsvfs);
641 			return (EINVAL);
642 		}
643 
644 		/*
645 		 * If we need to grow the block size then zfs_range_lock()
646 		 * will lock a wider range than we request here.
647 		 * Later after growing the block size we reduce the range.
648 		 */
649 		rl = zfs_range_lock(zp, woff, n, RL_WRITER);
650 	}
651 
652 	if (woff >= limit) {
653 		error = EFBIG;
654 		goto no_tx_done;
655 	}
656 
657 	if ((woff + n) > limit || woff > (limit - n))
658 		n = limit - woff;
659 
660 	/*
661 	 * Check for mandatory locks
662 	 */
663 	if (MANDMODE((mode_t)zp->z_phys->zp_mode) &&
664 	    (error = chklock(vp, FWRITE, woff, n, uio->uio_fmode, ct)) != 0)
665 		goto no_tx_done;
666 	end_size = MAX(zp->z_phys->zp_size, woff + n);
667 top:
668 	tx = dmu_tx_create(zfsvfs->z_os);
669 	dmu_tx_hold_bonus(tx, zp->z_id);
670 	dmu_tx_hold_write(tx, zp->z_id, woff, MIN(n, max_blksz));
671 	error = dmu_tx_assign(tx, zfsvfs->z_assign);
672 	if (error) {
673 		if (error == ERESTART && zfsvfs->z_assign == TXG_NOWAIT) {
674 			dmu_tx_wait(tx);
675 			dmu_tx_abort(tx);
676 			goto top;
677 		}
678 		dmu_tx_abort(tx);
679 		goto no_tx_done;
680 	}
681 
682 	/*
683 	 * If zfs_range_lock() over-locked we grow the blocksize
684 	 * and then reduce the lock range.
685 	 */
686 	if (rl->r_len == UINT64_MAX) {
687 		uint64_t new_blksz;
688 
689 		if (zp->z_blksz > max_blksz) {
690 			ASSERT(!ISP2(zp->z_blksz));
691 			new_blksz = MIN(end_size, SPA_MAXBLOCKSIZE);
692 		} else {
693 			new_blksz = MIN(end_size, max_blksz);
694 		}
695 		zfs_grow_blocksize(zp, new_blksz, tx);
696 		zfs_range_reduce(rl, woff, n);
697 	}
698 
699 	/*
700 	 * The file data does not fit in the znode "cache", so we
701 	 * will be writing to the file block data buffers.
702 	 * Each buffer will be written in a separate transaction;
703 	 * this keeps the intent log records small and allows us
704 	 * to do more fine-grained space accounting.
705 	 */
706 	while (n > 0) {
707 		/*
708 		 * XXX - should we really limit each write to z_max_blksz?
709 		 * Perhaps we should use SPA_MAXBLOCKSIZE chunks?
710 		 */
711 		nbytes = MIN(n, max_blksz - P2PHASE(woff, max_blksz));
712 		rw_enter(&zp->z_map_lock, RW_READER);
713 
714 		tx_bytes = uio->uio_resid;
715 		if (vn_has_cached_data(vp)) {
716 			rw_exit(&zp->z_map_lock);
717 			error = mappedwrite(vp, woff, nbytes, uio, tx);
718 		} else {
719 			error = dmu_write_uio(zfsvfs->z_os, zp->z_id,
720 			    woff, nbytes, uio, tx);
721 			rw_exit(&zp->z_map_lock);
722 		}
723 		tx_bytes -= uio->uio_resid;
724 
725 		if (error) {
726 			/* XXX - do we need to "clean up" the dmu buffer? */
727 			break;
728 		}
729 
730 		ASSERT(tx_bytes == nbytes);
731 
732 		/*
733 		 * Clear Set-UID/Set-GID bits on successful write if not
734 		 * privileged and at least one of the excute bits is set.
735 		 *
736 		 * It would be nice to to this after all writes have
737 		 * been done, but that would still expose the ISUID/ISGID
738 		 * to another app after the partial write is committed.
739 		 */
740 
741 		mutex_enter(&zp->z_acl_lock);
742 		if ((zp->z_phys->zp_mode & (S_IXUSR | (S_IXUSR >> 3) |
743 		    (S_IXUSR >> 6))) != 0 &&
744 		    (zp->z_phys->zp_mode & (S_ISUID | S_ISGID)) != 0 &&
745 		    secpolicy_vnode_setid_retain(cr,
746 		    (zp->z_phys->zp_mode & S_ISUID) != 0 &&
747 		    zp->z_phys->zp_uid == 0) != 0) {
748 			    zp->z_phys->zp_mode &= ~(S_ISUID | S_ISGID);
749 		}
750 		mutex_exit(&zp->z_acl_lock);
751 
752 		n -= nbytes;
753 		if (n <= 0)
754 			break;
755 
756 		/*
757 		 * We have more work ahead of us, so wrap up this transaction
758 		 * and start another.  Exact same logic as tx_done below.
759 		 */
760 		while ((end_size = zp->z_phys->zp_size) < uio->uio_loffset) {
761 			dmu_buf_will_dirty(zp->z_dbuf, tx);
762 			(void) atomic_cas_64(&zp->z_phys->zp_size, end_size,
763 			    uio->uio_loffset);
764 		}
765 		zfs_time_stamper(zp, CONTENT_MODIFIED, tx);
766 		zfs_log_write(zilog, tx, TX_WRITE, zp, woff, tx_bytes,
767 		    ioflag, uio);
768 		dmu_tx_commit(tx);
769 
770 		/*
771 		 * Start another transaction.
772 		 */
773 		woff = uio->uio_loffset;
774 		tx = dmu_tx_create(zfsvfs->z_os);
775 		dmu_tx_hold_bonus(tx, zp->z_id);
776 		dmu_tx_hold_write(tx, zp->z_id, woff, MIN(n, max_blksz));
777 		error = dmu_tx_assign(tx, zfsvfs->z_assign);
778 		if (error) {
779 			if (error == ERESTART &&
780 			    zfsvfs->z_assign == TXG_NOWAIT) {
781 				dmu_tx_wait(tx);
782 				dmu_tx_abort(tx);
783 				goto top;
784 			}
785 			dmu_tx_abort(tx);
786 			goto no_tx_done;
787 		}
788 	}
789 
790 tx_done:
791 
792 	if (tx_bytes != 0) {
793 		/*
794 		 * Update the file size if it has changed; account
795 		 * for possible concurrent updates.
796 		 */
797 		while ((end_size = zp->z_phys->zp_size) < uio->uio_loffset) {
798 			dmu_buf_will_dirty(zp->z_dbuf, tx);
799 			(void) atomic_cas_64(&zp->z_phys->zp_size, end_size,
800 			    uio->uio_loffset);
801 		}
802 		zfs_time_stamper(zp, CONTENT_MODIFIED, tx);
803 		zfs_log_write(zilog, tx, TX_WRITE, zp, woff, tx_bytes,
804 		    ioflag, uio);
805 	}
806 	dmu_tx_commit(tx);
807 
808 
809 no_tx_done:
810 
811 	zfs_range_unlock(rl);
812 
813 	/*
814 	 * If we're in replay mode, or we made no progress, return error.
815 	 * Otherwise, it's at least a partial write, so it's successful.
816 	 */
817 	if (zfsvfs->z_assign >= TXG_INITIAL || uio->uio_resid == start_resid) {
818 		ZFS_EXIT(zfsvfs);
819 		return (error);
820 	}
821 
822 	if (ioflag & (FSYNC | FDSYNC))
823 		zil_commit(zilog, zp->z_last_itx, zp->z_id);
824 
825 	ZFS_EXIT(zfsvfs);
826 	return (0);
827 }
828 
829 void
830 zfs_get_done(dmu_buf_t *db, void *vrl)
831 {
832 	rl_t *rl = (rl_t *)vrl;
833 	vnode_t *vp = ZTOV(rl->r_zp);
834 
835 	dmu_buf_rele(db, rl);
836 	zfs_range_unlock(rl);
837 	VN_RELE(vp);
838 }
839 
840 /*
841  * Get data to generate a TX_WRITE intent log record.
842  */
843 int
844 zfs_get_data(void *arg, lr_write_t *lr, char *buf, zio_t *zio)
845 {
846 	zfsvfs_t *zfsvfs = arg;
847 	objset_t *os = zfsvfs->z_os;
848 	znode_t *zp;
849 	uint64_t off = lr->lr_offset;
850 	dmu_buf_t *db;
851 	rl_t *rl;
852 	int dlen = lr->lr_length;  		/* length of user data */
853 	int error = 0;
854 
855 	ASSERT(dlen != 0);
856 
857 	/*
858 	 * Nothing to do if the file has been removed
859 	 */
860 	if (zfs_zget(zfsvfs, lr->lr_foid, &zp) != 0)
861 		return (ENOENT);
862 	if (zp->z_reap) {
863 		VN_RELE(ZTOV(zp));
864 		return (ENOENT);
865 	}
866 
867 	/*
868 	 * Write records come in two flavors: immediate and indirect.
869 	 * For small writes it's cheaper to store the data with the
870 	 * log record (immediate); for large writes it's cheaper to
871 	 * sync the data and get a pointer to it (indirect) so that
872 	 * we don't have to write the data twice.
873 	 */
874 	if (buf != NULL) { /* immediate write */
875 		rl = zfs_range_lock(zp, off, dlen, RL_READER);
876 		/* test for truncation needs to be done while range locked */
877 		if (off >= zp->z_phys->zp_size) {
878 			error = ENOENT;
879 			goto out;
880 		}
881 		VERIFY(0 == dmu_read(os, lr->lr_foid, off, dlen, buf));
882 	} else { /* indirect write */
883 		uint64_t boff; /* block starting offset */
884 
885 		ASSERT3U(dlen, <=, zp->z_blksz);
886 		/*
887 		 * Have to lock the whole block to ensure when it's
888 		 * written out and it's checksum is being calculated
889 		 * that no one can change the data. We need to re-check
890 		 * blocksize after we get the lock in case it's changed!
891 		 */
892 		for (;;) {
893 			if (ISP2(zp->z_blksz)) {
894 				boff = P2ALIGN_TYPED(off, zp->z_blksz,
895 				    uint64_t);
896 			} else {
897 				boff = 0;
898 			}
899 			dlen = zp->z_blksz;
900 			rl = zfs_range_lock(zp, boff, dlen, RL_READER);
901 			if (zp->z_blksz == dlen)
902 				break;
903 			zfs_range_unlock(rl);
904 		}
905 		/* test for truncation needs to be done while range locked */
906 		if (off >= zp->z_phys->zp_size) {
907 			error = ENOENT;
908 			goto out;
909 		}
910 		VERIFY(0 == dmu_buf_hold(os, lr->lr_foid, boff, rl, &db));
911 		ASSERT(boff == db->db_offset);
912 		lr->lr_blkoff = off - boff;
913 		error = dmu_sync(zio, db, &lr->lr_blkptr,
914 		    lr->lr_common.lrc_txg, zio ? zfs_get_done : NULL, rl);
915 		/*
916 		 * If we get EINPROGRESS, then we need to wait for a
917 		 * write IO initiated by dmu_sync() to complete before
918 		 * we can release this dbuf.  We will finish everything
919 		 * up in the zfs_get_done() callback.
920 		 */
921 		if (error == EINPROGRESS)
922 			return (0);
923 		dmu_buf_rele(db, rl);
924 	}
925 out:
926 	zfs_range_unlock(rl);
927 	VN_RELE(ZTOV(zp));
928 	return (error);
929 }
930 
931 /*ARGSUSED*/
932 static int
933 zfs_access(vnode_t *vp, int mode, int flags, cred_t *cr)
934 {
935 	znode_t *zp = VTOZ(vp);
936 	zfsvfs_t *zfsvfs = zp->z_zfsvfs;
937 	int error;
938 
939 	ZFS_ENTER(zfsvfs);
940 	error = zfs_zaccess_rwx(zp, mode, cr);
941 	ZFS_EXIT(zfsvfs);
942 	return (error);
943 }
944 
945 /*
946  * Lookup an entry in a directory, or an extended attribute directory.
947  * If it exists, return a held vnode reference for it.
948  *
949  *	IN:	dvp	- vnode of directory to search.
950  *		nm	- name of entry to lookup.
951  *		pnp	- full pathname to lookup [UNUSED].
952  *		flags	- LOOKUP_XATTR set if looking for an attribute.
953  *		rdir	- root directory vnode [UNUSED].
954  *		cr	- credentials of caller.
955  *
956  *	OUT:	vpp	- vnode of located entry, NULL if not found.
957  *
958  *	RETURN:	0 if success
959  *		error code if failure
960  *
961  * Timestamps:
962  *	NA
963  */
964 /* ARGSUSED */
965 static int
966 zfs_lookup(vnode_t *dvp, char *nm, vnode_t **vpp, struct pathname *pnp,
967     int flags, vnode_t *rdir, cred_t *cr)
968 {
969 
970 	znode_t *zdp = VTOZ(dvp);
971 	zfsvfs_t *zfsvfs = zdp->z_zfsvfs;
972 	int	error;
973 
974 	ZFS_ENTER(zfsvfs);
975 
976 	*vpp = NULL;
977 
978 	if (flags & LOOKUP_XATTR) {
979 		/*
980 		 * We don't allow recursive attributes..
981 		 * Maybe someday we will.
982 		 */
983 		if (zdp->z_phys->zp_flags & ZFS_XATTR) {
984 			ZFS_EXIT(zfsvfs);
985 			return (EINVAL);
986 		}
987 
988 		if (error = zfs_get_xattrdir(VTOZ(dvp), vpp, cr)) {
989 			ZFS_EXIT(zfsvfs);
990 			return (error);
991 		}
992 
993 		/*
994 		 * Do we have permission to get into attribute directory?
995 		 */
996 
997 		if (error = zfs_zaccess(VTOZ(*vpp), ACE_EXECUTE, cr)) {
998 			VN_RELE(*vpp);
999 		}
1000 
1001 		ZFS_EXIT(zfsvfs);
1002 		return (error);
1003 	}
1004 
1005 	if (dvp->v_type != VDIR) {
1006 		ZFS_EXIT(zfsvfs);
1007 		return (ENOTDIR);
1008 	}
1009 
1010 	/*
1011 	 * Check accessibility of directory.
1012 	 */
1013 
1014 	if (error = zfs_zaccess(zdp, ACE_EXECUTE, cr)) {
1015 		ZFS_EXIT(zfsvfs);
1016 		return (error);
1017 	}
1018 
1019 	if ((error = zfs_dirlook(zdp, nm, vpp)) == 0) {
1020 
1021 		/*
1022 		 * Convert device special files
1023 		 */
1024 		if (IS_DEVVP(*vpp)) {
1025 			vnode_t	*svp;
1026 
1027 			svp = specvp(*vpp, (*vpp)->v_rdev, (*vpp)->v_type, cr);
1028 			VN_RELE(*vpp);
1029 			if (svp == NULL)
1030 				error = ENOSYS;
1031 			else
1032 				*vpp = svp;
1033 		}
1034 	}
1035 
1036 	ZFS_EXIT(zfsvfs);
1037 	return (error);
1038 }
1039 
1040 /*
1041  * Attempt to create a new entry in a directory.  If the entry
1042  * already exists, truncate the file if permissible, else return
1043  * an error.  Return the vp of the created or trunc'd file.
1044  *
1045  *	IN:	dvp	- vnode of directory to put new file entry in.
1046  *		name	- name of new file entry.
1047  *		vap	- attributes of new file.
1048  *		excl	- flag indicating exclusive or non-exclusive mode.
1049  *		mode	- mode to open file with.
1050  *		cr	- credentials of caller.
1051  *		flag	- large file flag [UNUSED].
1052  *
1053  *	OUT:	vpp	- vnode of created or trunc'd entry.
1054  *
1055  *	RETURN:	0 if success
1056  *		error code if failure
1057  *
1058  * Timestamps:
1059  *	dvp - ctime|mtime updated if new entry created
1060  *	 vp - ctime|mtime always, atime if new
1061  */
1062 /* ARGSUSED */
1063 static int
1064 zfs_create(vnode_t *dvp, char *name, vattr_t *vap, vcexcl_t excl,
1065     int mode, vnode_t **vpp, cred_t *cr, int flag)
1066 {
1067 	znode_t		*zp, *dzp = VTOZ(dvp);
1068 	zfsvfs_t	*zfsvfs = dzp->z_zfsvfs;
1069 	zilog_t		*zilog = zfsvfs->z_log;
1070 	objset_t	*os = zfsvfs->z_os;
1071 	zfs_dirlock_t	*dl;
1072 	dmu_tx_t	*tx;
1073 	int		error;
1074 	uint64_t	zoid;
1075 
1076 	ZFS_ENTER(zfsvfs);
1077 
1078 top:
1079 	*vpp = NULL;
1080 
1081 	if ((vap->va_mode & VSVTX) && secpolicy_vnode_stky_modify(cr))
1082 		vap->va_mode &= ~VSVTX;
1083 
1084 	if (*name == '\0') {
1085 		/*
1086 		 * Null component name refers to the directory itself.
1087 		 */
1088 		VN_HOLD(dvp);
1089 		zp = dzp;
1090 		dl = NULL;
1091 		error = 0;
1092 	} else {
1093 		/* possible VN_HOLD(zp) */
1094 		if (error = zfs_dirent_lock(&dl, dzp, name, &zp, 0)) {
1095 			if (strcmp(name, "..") == 0)
1096 				error = EISDIR;
1097 			ZFS_EXIT(zfsvfs);
1098 			return (error);
1099 		}
1100 	}
1101 
1102 	zoid = zp ? zp->z_id : -1ULL;
1103 
1104 	if (zp == NULL) {
1105 		/*
1106 		 * Create a new file object and update the directory
1107 		 * to reference it.
1108 		 */
1109 		if (error = zfs_zaccess(dzp, ACE_ADD_FILE, cr)) {
1110 			goto out;
1111 		}
1112 
1113 		/*
1114 		 * We only support the creation of regular files in
1115 		 * extended attribute directories.
1116 		 */
1117 		if ((dzp->z_phys->zp_flags & ZFS_XATTR) &&
1118 		    (vap->va_type != VREG)) {
1119 			error = EINVAL;
1120 			goto out;
1121 		}
1122 
1123 		tx = dmu_tx_create(os);
1124 		dmu_tx_hold_bonus(tx, DMU_NEW_OBJECT);
1125 		dmu_tx_hold_bonus(tx, dzp->z_id);
1126 		dmu_tx_hold_zap(tx, dzp->z_id, TRUE, name);
1127 		if (dzp->z_phys->zp_flags & ZFS_INHERIT_ACE)
1128 			dmu_tx_hold_write(tx, DMU_NEW_OBJECT,
1129 			    0, SPA_MAXBLOCKSIZE);
1130 		error = dmu_tx_assign(tx, zfsvfs->z_assign);
1131 		if (error) {
1132 			zfs_dirent_unlock(dl);
1133 			if (error == ERESTART &&
1134 			    zfsvfs->z_assign == TXG_NOWAIT) {
1135 				dmu_tx_wait(tx);
1136 				dmu_tx_abort(tx);
1137 				goto top;
1138 			}
1139 			dmu_tx_abort(tx);
1140 			ZFS_EXIT(zfsvfs);
1141 			return (error);
1142 		}
1143 		zfs_mknode(dzp, vap, &zoid, tx, cr, 0, &zp, 0);
1144 		ASSERT(zp->z_id == zoid);
1145 		(void) zfs_link_create(dl, zp, tx, ZNEW);
1146 		zfs_log_create(zilog, tx, TX_CREATE, dzp, zp, name);
1147 		dmu_tx_commit(tx);
1148 	} else {
1149 		/*
1150 		 * A directory entry already exists for this name.
1151 		 */
1152 		/*
1153 		 * Can't truncate an existing file if in exclusive mode.
1154 		 */
1155 		if (excl == EXCL) {
1156 			error = EEXIST;
1157 			goto out;
1158 		}
1159 		/*
1160 		 * Can't open a directory for writing.
1161 		 */
1162 		if ((ZTOV(zp)->v_type == VDIR) && (mode & S_IWRITE)) {
1163 			error = EISDIR;
1164 			goto out;
1165 		}
1166 		/*
1167 		 * Verify requested access to file.
1168 		 */
1169 		if (mode && (error = zfs_zaccess_rwx(zp, mode, cr))) {
1170 			goto out;
1171 		}
1172 
1173 		mutex_enter(&dzp->z_lock);
1174 		dzp->z_seq++;
1175 		mutex_exit(&dzp->z_lock);
1176 
1177 		/*
1178 		 * Truncate regular files if requested.
1179 		 */
1180 		if ((ZTOV(zp)->v_type == VREG) &&
1181 		    (zp->z_phys->zp_size != 0) &&
1182 		    (vap->va_mask & AT_SIZE) && (vap->va_size == 0)) {
1183 			error = zfs_freesp(zp, 0, 0, mode, TRUE);
1184 			if (error == ERESTART &&
1185 			    zfsvfs->z_assign == TXG_NOWAIT) {
1186 				/* NB: we already did dmu_tx_wait() */
1187 				zfs_dirent_unlock(dl);
1188 				VN_RELE(ZTOV(zp));
1189 				goto top;
1190 			}
1191 		}
1192 	}
1193 out:
1194 
1195 	if (dl)
1196 		zfs_dirent_unlock(dl);
1197 
1198 	if (error) {
1199 		if (zp)
1200 			VN_RELE(ZTOV(zp));
1201 	} else {
1202 		*vpp = ZTOV(zp);
1203 		/*
1204 		 * If vnode is for a device return a specfs vnode instead.
1205 		 */
1206 		if (IS_DEVVP(*vpp)) {
1207 			struct vnode *svp;
1208 
1209 			svp = specvp(*vpp, (*vpp)->v_rdev, (*vpp)->v_type, cr);
1210 			VN_RELE(*vpp);
1211 			if (svp == NULL) {
1212 				error = ENOSYS;
1213 			}
1214 			*vpp = svp;
1215 		}
1216 	}
1217 
1218 	ZFS_EXIT(zfsvfs);
1219 	return (error);
1220 }
1221 
1222 /*
1223  * Remove an entry from a directory.
1224  *
1225  *	IN:	dvp	- vnode of directory to remove entry from.
1226  *		name	- name of entry to remove.
1227  *		cr	- credentials of caller.
1228  *
1229  *	RETURN:	0 if success
1230  *		error code if failure
1231  *
1232  * Timestamps:
1233  *	dvp - ctime|mtime
1234  *	 vp - ctime (if nlink > 0)
1235  */
1236 static int
1237 zfs_remove(vnode_t *dvp, char *name, cred_t *cr)
1238 {
1239 	znode_t		*zp, *dzp = VTOZ(dvp);
1240 	znode_t		*xzp = NULL;
1241 	vnode_t		*vp;
1242 	zfsvfs_t	*zfsvfs = dzp->z_zfsvfs;
1243 	zilog_t		*zilog = zfsvfs->z_log;
1244 	uint64_t	acl_obj, xattr_obj;
1245 	zfs_dirlock_t	*dl;
1246 	dmu_tx_t	*tx;
1247 	int		may_delete_now, delete_now = FALSE;
1248 	int		reaped;
1249 	int		error;
1250 
1251 	ZFS_ENTER(zfsvfs);
1252 
1253 top:
1254 	/*
1255 	 * Attempt to lock directory; fail if entry doesn't exist.
1256 	 */
1257 	if (error = zfs_dirent_lock(&dl, dzp, name, &zp, ZEXISTS)) {
1258 		ZFS_EXIT(zfsvfs);
1259 		return (error);
1260 	}
1261 
1262 	vp = ZTOV(zp);
1263 
1264 	if (error = zfs_zaccess_delete(dzp, zp, cr)) {
1265 		goto out;
1266 	}
1267 
1268 	/*
1269 	 * Need to use rmdir for removing directories.
1270 	 */
1271 	if (vp->v_type == VDIR) {
1272 		error = EPERM;
1273 		goto out;
1274 	}
1275 
1276 	vnevent_remove(vp);
1277 
1278 	dnlc_remove(dvp, name);
1279 
1280 	mutex_enter(&vp->v_lock);
1281 	may_delete_now = vp->v_count == 1 && !vn_has_cached_data(vp);
1282 	mutex_exit(&vp->v_lock);
1283 
1284 	/*
1285 	 * We may delete the znode now, or we may put it on the delete queue;
1286 	 * it depends on whether we're the last link, and on whether there are
1287 	 * other holds on the vnode.  So we dmu_tx_hold() the right things to
1288 	 * allow for either case.
1289 	 */
1290 	tx = dmu_tx_create(zfsvfs->z_os);
1291 	dmu_tx_hold_zap(tx, dzp->z_id, FALSE, name);
1292 	dmu_tx_hold_bonus(tx, zp->z_id);
1293 	if (may_delete_now)
1294 		dmu_tx_hold_free(tx, zp->z_id, 0, DMU_OBJECT_END);
1295 
1296 	/* are there any extended attributes? */
1297 	if ((xattr_obj = zp->z_phys->zp_xattr) != 0) {
1298 		/*
1299 		 * XXX - There is a possibility that the delete
1300 		 * of the parent file could succeed, but then we get
1301 		 * an ENOSPC when we try to delete the xattrs...
1302 		 * so we would need to re-try the deletes periodically
1303 		 */
1304 		/* XXX - do we need this if we are deleting? */
1305 		dmu_tx_hold_bonus(tx, xattr_obj);
1306 	}
1307 
1308 	/* are there any additional acls */
1309 	if ((acl_obj = zp->z_phys->zp_acl.z_acl_extern_obj) != 0 &&
1310 	    may_delete_now)
1311 		dmu_tx_hold_free(tx, acl_obj, 0, DMU_OBJECT_END);
1312 
1313 	/* charge as an update -- would be nice not to charge at all */
1314 	dmu_tx_hold_zap(tx, zfsvfs->z_dqueue, FALSE, NULL);
1315 
1316 	error = dmu_tx_assign(tx, zfsvfs->z_assign);
1317 	if (error) {
1318 		zfs_dirent_unlock(dl);
1319 		VN_RELE(vp);
1320 		if (error == ERESTART && zfsvfs->z_assign == TXG_NOWAIT) {
1321 			dmu_tx_wait(tx);
1322 			dmu_tx_abort(tx);
1323 			goto top;
1324 		}
1325 		dmu_tx_abort(tx);
1326 		ZFS_EXIT(zfsvfs);
1327 		return (error);
1328 	}
1329 
1330 	/*
1331 	 * Remove the directory entry.
1332 	 */
1333 	error = zfs_link_destroy(dl, zp, tx, 0, &reaped);
1334 
1335 	if (error) {
1336 		dmu_tx_commit(tx);
1337 		goto out;
1338 	}
1339 
1340 	if (reaped) {
1341 		mutex_enter(&vp->v_lock);
1342 		delete_now = may_delete_now &&
1343 		    vp->v_count == 1 && !vn_has_cached_data(vp) &&
1344 		    zp->z_phys->zp_xattr == xattr_obj &&
1345 		    zp->z_phys->zp_acl.z_acl_extern_obj == acl_obj;
1346 		mutex_exit(&vp->v_lock);
1347 	}
1348 
1349 	if (delete_now) {
1350 		if (zp->z_phys->zp_xattr) {
1351 			error = zfs_zget(zfsvfs, zp->z_phys->zp_xattr, &xzp);
1352 			ASSERT3U(error, ==, 0);
1353 			ASSERT3U(xzp->z_phys->zp_links, ==, 2);
1354 			dmu_buf_will_dirty(xzp->z_dbuf, tx);
1355 			mutex_enter(&xzp->z_lock);
1356 			xzp->z_reap = 1;
1357 			xzp->z_phys->zp_links = 0;
1358 			mutex_exit(&xzp->z_lock);
1359 			zfs_dq_add(xzp, tx);
1360 			zp->z_phys->zp_xattr = 0; /* probably unnecessary */
1361 		}
1362 		mutex_enter(&zp->z_lock);
1363 		mutex_enter(&vp->v_lock);
1364 		vp->v_count--;
1365 		ASSERT3U(vp->v_count, ==, 0);
1366 		mutex_exit(&vp->v_lock);
1367 		mutex_exit(&zp->z_lock);
1368 		zfs_znode_delete(zp, tx);
1369 		VFS_RELE(zfsvfs->z_vfs);
1370 	} else if (reaped) {
1371 		zfs_dq_add(zp, tx);
1372 	}
1373 
1374 	zfs_log_remove(zilog, tx, TX_REMOVE, dzp, name);
1375 
1376 	dmu_tx_commit(tx);
1377 out:
1378 	zfs_dirent_unlock(dl);
1379 
1380 	if (!delete_now) {
1381 		VN_RELE(vp);
1382 	} else if (xzp) {
1383 		/* this rele delayed to prevent nesting transactions */
1384 		VN_RELE(ZTOV(xzp));
1385 	}
1386 
1387 	ZFS_EXIT(zfsvfs);
1388 	return (error);
1389 }
1390 
1391 /*
1392  * Create a new directory and insert it into dvp using the name
1393  * provided.  Return a pointer to the inserted directory.
1394  *
1395  *	IN:	dvp	- vnode of directory to add subdir to.
1396  *		dirname	- name of new directory.
1397  *		vap	- attributes of new directory.
1398  *		cr	- credentials of caller.
1399  *
1400  *	OUT:	vpp	- vnode of created directory.
1401  *
1402  *	RETURN:	0 if success
1403  *		error code if failure
1404  *
1405  * Timestamps:
1406  *	dvp - ctime|mtime updated
1407  *	 vp - ctime|mtime|atime updated
1408  */
1409 static int
1410 zfs_mkdir(vnode_t *dvp, char *dirname, vattr_t *vap, vnode_t **vpp, cred_t *cr)
1411 {
1412 	znode_t		*zp, *dzp = VTOZ(dvp);
1413 	zfsvfs_t	*zfsvfs = dzp->z_zfsvfs;
1414 	zilog_t		*zilog = zfsvfs->z_log;
1415 	zfs_dirlock_t	*dl;
1416 	uint64_t	zoid = 0;
1417 	dmu_tx_t	*tx;
1418 	int		error;
1419 
1420 	ASSERT(vap->va_type == VDIR);
1421 
1422 	ZFS_ENTER(zfsvfs);
1423 
1424 	if (dzp->z_phys->zp_flags & ZFS_XATTR) {
1425 		ZFS_EXIT(zfsvfs);
1426 		return (EINVAL);
1427 	}
1428 top:
1429 	*vpp = NULL;
1430 
1431 	/*
1432 	 * First make sure the new directory doesn't exist.
1433 	 */
1434 	if (error = zfs_dirent_lock(&dl, dzp, dirname, &zp, ZNEW)) {
1435 		ZFS_EXIT(zfsvfs);
1436 		return (error);
1437 	}
1438 
1439 	if (error = zfs_zaccess(dzp, ACE_ADD_SUBDIRECTORY, cr)) {
1440 		zfs_dirent_unlock(dl);
1441 		ZFS_EXIT(zfsvfs);
1442 		return (error);
1443 	}
1444 
1445 	/*
1446 	 * Add a new entry to the directory.
1447 	 */
1448 	tx = dmu_tx_create(zfsvfs->z_os);
1449 	dmu_tx_hold_zap(tx, dzp->z_id, TRUE, dirname);
1450 	dmu_tx_hold_zap(tx, DMU_NEW_OBJECT, FALSE, NULL);
1451 	if (dzp->z_phys->zp_flags & ZFS_INHERIT_ACE)
1452 		dmu_tx_hold_write(tx, DMU_NEW_OBJECT,
1453 		    0, SPA_MAXBLOCKSIZE);
1454 	error = dmu_tx_assign(tx, zfsvfs->z_assign);
1455 	if (error) {
1456 		zfs_dirent_unlock(dl);
1457 		if (error == ERESTART && zfsvfs->z_assign == TXG_NOWAIT) {
1458 			dmu_tx_wait(tx);
1459 			dmu_tx_abort(tx);
1460 			goto top;
1461 		}
1462 		dmu_tx_abort(tx);
1463 		ZFS_EXIT(zfsvfs);
1464 		return (error);
1465 	}
1466 
1467 	/*
1468 	 * Create new node.
1469 	 */
1470 	zfs_mknode(dzp, vap, &zoid, tx, cr, 0, &zp, 0);
1471 
1472 	/*
1473 	 * Now put new name in parent dir.
1474 	 */
1475 	(void) zfs_link_create(dl, zp, tx, ZNEW);
1476 
1477 	*vpp = ZTOV(zp);
1478 
1479 	zfs_log_create(zilog, tx, TX_MKDIR, dzp, zp, dirname);
1480 	dmu_tx_commit(tx);
1481 
1482 	zfs_dirent_unlock(dl);
1483 
1484 	ZFS_EXIT(zfsvfs);
1485 	return (0);
1486 }
1487 
1488 /*
1489  * Remove a directory subdir entry.  If the current working
1490  * directory is the same as the subdir to be removed, the
1491  * remove will fail.
1492  *
1493  *	IN:	dvp	- vnode of directory to remove from.
1494  *		name	- name of directory to be removed.
1495  *		cwd	- vnode of current working directory.
1496  *		cr	- credentials of caller.
1497  *
1498  *	RETURN:	0 if success
1499  *		error code if failure
1500  *
1501  * Timestamps:
1502  *	dvp - ctime|mtime updated
1503  */
1504 static int
1505 zfs_rmdir(vnode_t *dvp, char *name, vnode_t *cwd, cred_t *cr)
1506 {
1507 	znode_t		*dzp = VTOZ(dvp);
1508 	znode_t		*zp;
1509 	vnode_t		*vp;
1510 	zfsvfs_t	*zfsvfs = dzp->z_zfsvfs;
1511 	zilog_t		*zilog = zfsvfs->z_log;
1512 	zfs_dirlock_t	*dl;
1513 	dmu_tx_t	*tx;
1514 	int		error;
1515 
1516 	ZFS_ENTER(zfsvfs);
1517 
1518 top:
1519 	zp = NULL;
1520 
1521 	/*
1522 	 * Attempt to lock directory; fail if entry doesn't exist.
1523 	 */
1524 	if (error = zfs_dirent_lock(&dl, dzp, name, &zp, ZEXISTS)) {
1525 		ZFS_EXIT(zfsvfs);
1526 		return (error);
1527 	}
1528 
1529 	vp = ZTOV(zp);
1530 
1531 	if (error = zfs_zaccess_delete(dzp, zp, cr)) {
1532 		goto out;
1533 	}
1534 
1535 	if (vp->v_type != VDIR) {
1536 		error = ENOTDIR;
1537 		goto out;
1538 	}
1539 
1540 	if (vp == cwd) {
1541 		error = EINVAL;
1542 		goto out;
1543 	}
1544 
1545 	vnevent_rmdir(vp);
1546 
1547 	/*
1548 	 * Grab a lock on the parent pointer make sure we play well
1549 	 * with the treewalk and directory rename code.
1550 	 */
1551 	rw_enter(&zp->z_parent_lock, RW_WRITER);
1552 
1553 	tx = dmu_tx_create(zfsvfs->z_os);
1554 	dmu_tx_hold_zap(tx, dzp->z_id, FALSE, name);
1555 	dmu_tx_hold_bonus(tx, zp->z_id);
1556 	dmu_tx_hold_zap(tx, zfsvfs->z_dqueue, FALSE, NULL);
1557 	error = dmu_tx_assign(tx, zfsvfs->z_assign);
1558 	if (error) {
1559 		rw_exit(&zp->z_parent_lock);
1560 		zfs_dirent_unlock(dl);
1561 		VN_RELE(vp);
1562 		if (error == ERESTART && zfsvfs->z_assign == TXG_NOWAIT) {
1563 			dmu_tx_wait(tx);
1564 			dmu_tx_abort(tx);
1565 			goto top;
1566 		}
1567 		dmu_tx_abort(tx);
1568 		ZFS_EXIT(zfsvfs);
1569 		return (error);
1570 	}
1571 
1572 	error = zfs_link_destroy(dl, zp, tx, 0, NULL);
1573 
1574 	if (error == 0)
1575 		zfs_log_remove(zilog, tx, TX_RMDIR, dzp, name);
1576 
1577 	dmu_tx_commit(tx);
1578 
1579 	rw_exit(&zp->z_parent_lock);
1580 out:
1581 	zfs_dirent_unlock(dl);
1582 
1583 	VN_RELE(vp);
1584 
1585 	ZFS_EXIT(zfsvfs);
1586 	return (error);
1587 }
1588 
1589 /*
1590  * Read as many directory entries as will fit into the provided
1591  * buffer from the given directory cursor position (specified in
1592  * the uio structure.
1593  *
1594  *	IN:	vp	- vnode of directory to read.
1595  *		uio	- structure supplying read location, range info,
1596  *			  and return buffer.
1597  *		cr	- credentials of caller.
1598  *
1599  *	OUT:	uio	- updated offset and range, buffer filled.
1600  *		eofp	- set to true if end-of-file detected.
1601  *
1602  *	RETURN:	0 if success
1603  *		error code if failure
1604  *
1605  * Timestamps:
1606  *	vp - atime updated
1607  *
1608  * Note that the low 4 bits of the cookie returned by zap is always zero.
1609  * This allows us to use the low range for "special" directory entries:
1610  * We use 0 for '.', and 1 for '..'.  If this is the root of the filesystem,
1611  * we use the offset 2 for the '.zfs' directory.
1612  */
1613 /* ARGSUSED */
1614 static int
1615 zfs_readdir(vnode_t *vp, uio_t *uio, cred_t *cr, int *eofp)
1616 {
1617 	znode_t		*zp = VTOZ(vp);
1618 	iovec_t		*iovp;
1619 	dirent64_t	*odp;
1620 	zfsvfs_t	*zfsvfs = zp->z_zfsvfs;
1621 	objset_t	*os;
1622 	caddr_t		outbuf;
1623 	size_t		bufsize;
1624 	zap_cursor_t	zc;
1625 	zap_attribute_t	zap;
1626 	uint_t		bytes_wanted;
1627 	ushort_t	this_reclen;
1628 	uint64_t	offset; /* must be unsigned; checks for < 1 */
1629 	off64_t		*next;
1630 	int		local_eof;
1631 	int		outcount;
1632 	int		error;
1633 	uint8_t		prefetch;
1634 
1635 	ZFS_ENTER(zfsvfs);
1636 
1637 	/*
1638 	 * If we are not given an eof variable,
1639 	 * use a local one.
1640 	 */
1641 	if (eofp == NULL)
1642 		eofp = &local_eof;
1643 
1644 	/*
1645 	 * Check for valid iov_len.
1646 	 */
1647 	if (uio->uio_iov->iov_len <= 0) {
1648 		ZFS_EXIT(zfsvfs);
1649 		return (EINVAL);
1650 	}
1651 
1652 	/*
1653 	 * Quit if directory has been removed (posix)
1654 	 */
1655 	if ((*eofp = zp->z_reap) != 0) {
1656 		ZFS_EXIT(zfsvfs);
1657 		return (0);
1658 	}
1659 
1660 	error = 0;
1661 	os = zfsvfs->z_os;
1662 	offset = uio->uio_loffset;
1663 	prefetch = zp->z_zn_prefetch;
1664 
1665 	/*
1666 	 * Initialize the iterator cursor.
1667 	 */
1668 	if (offset <= 3) {
1669 		/*
1670 		 * Start iteration from the beginning of the directory.
1671 		 */
1672 		zap_cursor_init(&zc, os, zp->z_id);
1673 	} else {
1674 		/*
1675 		 * The offset is a serialized cursor.
1676 		 */
1677 		zap_cursor_init_serialized(&zc, os, zp->z_id, offset);
1678 	}
1679 
1680 	/*
1681 	 * Get space to change directory entries into fs independent format.
1682 	 */
1683 	iovp = uio->uio_iov;
1684 	bytes_wanted = iovp->iov_len;
1685 	if (uio->uio_segflg != UIO_SYSSPACE || uio->uio_iovcnt != 1) {
1686 		bufsize = bytes_wanted;
1687 		outbuf = kmem_alloc(bufsize, KM_SLEEP);
1688 		odp = (struct dirent64 *)outbuf;
1689 	} else {
1690 		bufsize = bytes_wanted;
1691 		odp = (struct dirent64 *)iovp->iov_base;
1692 	}
1693 
1694 	/*
1695 	 * Transform to file-system independent format
1696 	 */
1697 	outcount = 0;
1698 	while (outcount < bytes_wanted) {
1699 		/*
1700 		 * Special case `.', `..', and `.zfs'.
1701 		 */
1702 		if (offset == 0) {
1703 			(void) strcpy(zap.za_name, ".");
1704 			zap.za_first_integer = zp->z_id;
1705 			this_reclen = DIRENT64_RECLEN(1);
1706 		} else if (offset == 1) {
1707 			(void) strcpy(zap.za_name, "..");
1708 			zap.za_first_integer = zp->z_phys->zp_parent;
1709 			this_reclen = DIRENT64_RECLEN(2);
1710 		} else if (offset == 2 && zfs_show_ctldir(zp)) {
1711 			(void) strcpy(zap.za_name, ZFS_CTLDIR_NAME);
1712 			zap.za_first_integer = ZFSCTL_INO_ROOT;
1713 			this_reclen =
1714 			    DIRENT64_RECLEN(sizeof (ZFS_CTLDIR_NAME) - 1);
1715 		} else {
1716 			/*
1717 			 * Grab next entry.
1718 			 */
1719 			if (error = zap_cursor_retrieve(&zc, &zap)) {
1720 				if ((*eofp = (error == ENOENT)) != 0)
1721 					break;
1722 				else
1723 					goto update;
1724 			}
1725 
1726 			if (zap.za_integer_length != 8 ||
1727 			    zap.za_num_integers != 1) {
1728 				cmn_err(CE_WARN, "zap_readdir: bad directory "
1729 				    "entry, obj = %lld, offset = %lld\n",
1730 				    (u_longlong_t)zp->z_id,
1731 				    (u_longlong_t)offset);
1732 				error = ENXIO;
1733 				goto update;
1734 			}
1735 			this_reclen = DIRENT64_RECLEN(strlen(zap.za_name));
1736 		}
1737 
1738 		/*
1739 		 * Will this entry fit in the buffer?
1740 		 */
1741 		if (outcount + this_reclen > bufsize) {
1742 			/*
1743 			 * Did we manage to fit anything in the buffer?
1744 			 */
1745 			if (!outcount) {
1746 				error = EINVAL;
1747 				goto update;
1748 			}
1749 			break;
1750 		}
1751 		/*
1752 		 * Add this entry:
1753 		 */
1754 		odp->d_ino = (ino64_t)zap.za_first_integer;
1755 		odp->d_reclen = (ushort_t)this_reclen;
1756 		/* NOTE: d_off is the offset for the *next* entry */
1757 		next = &(odp->d_off);
1758 		(void) strncpy(odp->d_name, zap.za_name,
1759 		    DIRENT64_NAMELEN(this_reclen));
1760 		outcount += this_reclen;
1761 		odp = (dirent64_t *)((intptr_t)odp + this_reclen);
1762 
1763 		ASSERT(outcount <= bufsize);
1764 
1765 		/* Prefetch znode */
1766 		if (prefetch)
1767 			dmu_prefetch(os, zap.za_first_integer, 0, 0);
1768 
1769 		/*
1770 		 * Move to the next entry, fill in the previous offset.
1771 		 */
1772 		if (offset > 2 || (offset == 2 && !zfs_show_ctldir(zp))) {
1773 			zap_cursor_advance(&zc);
1774 			offset = zap_cursor_serialize(&zc);
1775 		} else {
1776 			offset += 1;
1777 		}
1778 		*next = offset;
1779 	}
1780 	zp->z_zn_prefetch = B_FALSE; /* a lookup will re-enable pre-fetching */
1781 
1782 	if (uio->uio_segflg == UIO_SYSSPACE && uio->uio_iovcnt == 1) {
1783 		iovp->iov_base += outcount;
1784 		iovp->iov_len -= outcount;
1785 		uio->uio_resid -= outcount;
1786 	} else if (error = uiomove(outbuf, (long)outcount, UIO_READ, uio)) {
1787 		/*
1788 		 * Reset the pointer.
1789 		 */
1790 		offset = uio->uio_loffset;
1791 	}
1792 
1793 update:
1794 	zap_cursor_fini(&zc);
1795 	if (uio->uio_segflg != UIO_SYSSPACE || uio->uio_iovcnt != 1)
1796 		kmem_free(outbuf, bufsize);
1797 
1798 	if (error == ENOENT)
1799 		error = 0;
1800 
1801 	ZFS_ACCESSTIME_STAMP(zfsvfs, zp);
1802 
1803 	uio->uio_loffset = offset;
1804 	ZFS_EXIT(zfsvfs);
1805 	return (error);
1806 }
1807 
1808 static int
1809 zfs_fsync(vnode_t *vp, int syncflag, cred_t *cr)
1810 {
1811 	znode_t	*zp = VTOZ(vp);
1812 	zfsvfs_t *zfsvfs = zp->z_zfsvfs;
1813 
1814 	/*
1815 	 * Regardless of whether this is required for standards conformance,
1816 	 * this is the logical behavior when fsync() is called on a file with
1817 	 * dirty pages.  We use B_ASYNC since the ZIL transactions are already
1818 	 * going to be pushed out as part of the zil_commit().
1819 	 */
1820 	if (vn_has_cached_data(vp) && !(syncflag & FNODSYNC) &&
1821 	    (vp->v_type == VREG) && !(IS_SWAPVP(vp)))
1822 		(void) VOP_PUTPAGE(vp, (offset_t)0, (size_t)0, B_ASYNC, cr);
1823 
1824 	ZFS_ENTER(zfsvfs);
1825 	zil_commit(zfsvfs->z_log, zp->z_last_itx, zp->z_id);
1826 	ZFS_EXIT(zfsvfs);
1827 	return (0);
1828 }
1829 
1830 /*
1831  * Get the requested file attributes and place them in the provided
1832  * vattr structure.
1833  *
1834  *	IN:	vp	- vnode of file.
1835  *		vap	- va_mask identifies requested attributes.
1836  *		flags	- [UNUSED]
1837  *		cr	- credentials of caller.
1838  *
1839  *	OUT:	vap	- attribute values.
1840  *
1841  *	RETURN:	0 (always succeeds)
1842  */
1843 /* ARGSUSED */
1844 static int
1845 zfs_getattr(vnode_t *vp, vattr_t *vap, int flags, cred_t *cr)
1846 {
1847 	znode_t *zp = VTOZ(vp);
1848 	zfsvfs_t *zfsvfs = zp->z_zfsvfs;
1849 	znode_phys_t *pzp = zp->z_phys;
1850 	int	error;
1851 
1852 	ZFS_ENTER(zfsvfs);
1853 
1854 	/*
1855 	 * Return all attributes.  It's cheaper to provide the answer
1856 	 * than to determine whether we were asked the question.
1857 	 */
1858 	mutex_enter(&zp->z_lock);
1859 
1860 	vap->va_type = vp->v_type;
1861 	vap->va_mode = pzp->zp_mode & MODEMASK;
1862 	vap->va_uid = zp->z_phys->zp_uid;
1863 	vap->va_gid = zp->z_phys->zp_gid;
1864 	vap->va_fsid = zp->z_zfsvfs->z_vfs->vfs_dev;
1865 	vap->va_nodeid = zp->z_id;
1866 	vap->va_nlink = MIN(pzp->zp_links, UINT32_MAX);	/* nlink_t limit! */
1867 	vap->va_size = pzp->zp_size;
1868 	vap->va_rdev = vp->v_rdev;
1869 	vap->va_seq = zp->z_seq;
1870 
1871 	ZFS_TIME_DECODE(&vap->va_atime, pzp->zp_atime);
1872 	ZFS_TIME_DECODE(&vap->va_mtime, pzp->zp_mtime);
1873 	ZFS_TIME_DECODE(&vap->va_ctime, pzp->zp_ctime);
1874 
1875 	/*
1876 	 * If ACL is trivial don't bother looking for ACE_READ_ATTRIBUTES.
1877 	 * Also, if we are the owner don't bother, since owner should
1878 	 * always be allowed to read basic attributes of file.
1879 	 */
1880 	if (!(zp->z_phys->zp_flags & ZFS_ACL_TRIVIAL) &&
1881 	    (zp->z_phys->zp_uid != crgetuid(cr))) {
1882 		if (error = zfs_zaccess(zp, ACE_READ_ATTRIBUTES, cr)) {
1883 			mutex_exit(&zp->z_lock);
1884 			ZFS_EXIT(zfsvfs);
1885 			return (error);
1886 		}
1887 	}
1888 
1889 	mutex_exit(&zp->z_lock);
1890 
1891 	dmu_object_size_from_db(zp->z_dbuf, &vap->va_blksize, &vap->va_nblocks);
1892 
1893 	if (zp->z_blksz == 0) {
1894 		/*
1895 		 * Block size hasn't been set; suggest maximal I/O transfers.
1896 		 */
1897 		vap->va_blksize = zfsvfs->z_max_blksz;
1898 	}
1899 
1900 	ZFS_EXIT(zfsvfs);
1901 	return (0);
1902 }
1903 
1904 /*
1905  * Set the file attributes to the values contained in the
1906  * vattr structure.
1907  *
1908  *	IN:	vp	- vnode of file to be modified.
1909  *		vap	- new attribute values.
1910  *		flags	- ATTR_UTIME set if non-default time values provided.
1911  *		cr	- credentials of caller.
1912  *
1913  *	RETURN:	0 if success
1914  *		error code if failure
1915  *
1916  * Timestamps:
1917  *	vp - ctime updated, mtime updated if size changed.
1918  */
1919 /* ARGSUSED */
1920 static int
1921 zfs_setattr(vnode_t *vp, vattr_t *vap, int flags, cred_t *cr,
1922 	caller_context_t *ct)
1923 {
1924 	struct znode	*zp = VTOZ(vp);
1925 	znode_phys_t	*pzp = zp->z_phys;
1926 	zfsvfs_t	*zfsvfs = zp->z_zfsvfs;
1927 	zilog_t		*zilog = zfsvfs->z_log;
1928 	dmu_tx_t	*tx;
1929 	vattr_t		oldva;
1930 	uint_t		mask = vap->va_mask;
1931 	uint_t		saved_mask;
1932 	int		trim_mask = FALSE;
1933 	uint64_t	new_mode;
1934 	znode_t		*attrzp;
1935 	int		need_policy = FALSE;
1936 	int		err;
1937 
1938 	if (mask == 0)
1939 		return (0);
1940 
1941 	if (mask & AT_NOSET)
1942 		return (EINVAL);
1943 
1944 	if (mask & AT_SIZE && vp->v_type == VDIR)
1945 		return (EISDIR);
1946 
1947 	if (mask & AT_SIZE && vp->v_type != VREG && vp->v_type != VFIFO)
1948 		return (EINVAL);
1949 
1950 	ZFS_ENTER(zfsvfs);
1951 
1952 top:
1953 	attrzp = NULL;
1954 
1955 	if (zfsvfs->z_vfs->vfs_flag & VFS_RDONLY) {
1956 		ZFS_EXIT(zfsvfs);
1957 		return (EROFS);
1958 	}
1959 
1960 	/*
1961 	 * First validate permissions
1962 	 */
1963 
1964 	if (mask & AT_SIZE) {
1965 		err = zfs_zaccess(zp, ACE_WRITE_DATA, cr);
1966 		if (err) {
1967 			ZFS_EXIT(zfsvfs);
1968 			return (err);
1969 		}
1970 		/*
1971 		 * XXX - Note, we are not providing any open
1972 		 * mode flags here (like FNDELAY), so we may
1973 		 * block if there are locks present... this
1974 		 * should be addressed in openat().
1975 		 */
1976 		do {
1977 			err = zfs_freesp(zp, vap->va_size, 0, 0, FALSE);
1978 			/* NB: we already did dmu_tx_wait() if necessary */
1979 		} while (err == ERESTART && zfsvfs->z_assign == TXG_NOWAIT);
1980 		if (err) {
1981 			ZFS_EXIT(zfsvfs);
1982 			return (err);
1983 		}
1984 	}
1985 
1986 	if (mask & (AT_ATIME|AT_MTIME))
1987 		need_policy = zfs_zaccess_v4_perm(zp, ACE_WRITE_ATTRIBUTES, cr);
1988 
1989 	if (mask & (AT_UID|AT_GID)) {
1990 		int	idmask = (mask & (AT_UID|AT_GID));
1991 		int	take_owner;
1992 		int	take_group;
1993 
1994 		/*
1995 		 * NOTE: even if a new mode is being set,
1996 		 * we may clear S_ISUID/S_ISGID bits.
1997 		 */
1998 
1999 		if (!(mask & AT_MODE))
2000 			vap->va_mode = pzp->zp_mode;
2001 
2002 		/*
2003 		 * Take ownership or chgrp to group we are a member of
2004 		 */
2005 
2006 		take_owner = (mask & AT_UID) && (vap->va_uid == crgetuid(cr));
2007 		take_group = (mask & AT_GID) && groupmember(vap->va_gid, cr);
2008 
2009 		/*
2010 		 * If both AT_UID and AT_GID are set then take_owner and
2011 		 * take_group must both be set in order to allow taking
2012 		 * ownership.
2013 		 *
2014 		 * Otherwise, send the check through secpolicy_vnode_setattr()
2015 		 *
2016 		 */
2017 
2018 		if (((idmask == (AT_UID|AT_GID)) && take_owner && take_group) ||
2019 		    ((idmask == AT_UID) && take_owner) ||
2020 		    ((idmask == AT_GID) && take_group)) {
2021 			if (zfs_zaccess_v4_perm(zp, ACE_WRITE_OWNER, cr) == 0) {
2022 				/*
2023 				 * Remove setuid/setgid for non-privileged users
2024 				 */
2025 				secpolicy_setid_clear(vap, cr);
2026 				trim_mask = TRUE;
2027 				saved_mask = vap->va_mask;
2028 			} else {
2029 				need_policy =  TRUE;
2030 			}
2031 		} else {
2032 			need_policy =  TRUE;
2033 		}
2034 	}
2035 
2036 	if (mask & AT_MODE)
2037 		need_policy = TRUE;
2038 
2039 	if (need_policy) {
2040 		mutex_enter(&zp->z_lock);
2041 		oldva.va_mode = pzp->zp_mode;
2042 		oldva.va_uid = zp->z_phys->zp_uid;
2043 		oldva.va_gid = zp->z_phys->zp_gid;
2044 		mutex_exit(&zp->z_lock);
2045 
2046 		/*
2047 		 * If trim_mask is set then take ownership
2048 		 * has been granted.  In that case remove
2049 		 * UID|GID from mask so that
2050 		 * secpolicy_vnode_setattr() doesn't revoke it.
2051 		 */
2052 		if (trim_mask)
2053 			vap->va_mask &= ~(AT_UID|AT_GID);
2054 
2055 		err = secpolicy_vnode_setattr(cr, vp, vap, &oldva, flags,
2056 		    (int (*)(void *, int, cred_t *))zfs_zaccess_rwx, zp);
2057 		if (err) {
2058 			ZFS_EXIT(zfsvfs);
2059 			return (err);
2060 		}
2061 
2062 		if (trim_mask)
2063 			vap->va_mask |= (saved_mask & (AT_UID|AT_GID));
2064 	}
2065 
2066 	/*
2067 	 * secpolicy_vnode_setattr, or take ownership may have
2068 	 * changed va_mask
2069 	 */
2070 	mask = vap->va_mask;
2071 
2072 	tx = dmu_tx_create(zfsvfs->z_os);
2073 	dmu_tx_hold_bonus(tx, zp->z_id);
2074 
2075 	if (mask & AT_MODE) {
2076 		uint64_t pmode = pzp->zp_mode;
2077 
2078 		new_mode = (pmode & S_IFMT) | (vap->va_mode & ~S_IFMT);
2079 
2080 		if (zp->z_phys->zp_acl.z_acl_extern_obj)
2081 			dmu_tx_hold_write(tx,
2082 			    pzp->zp_acl.z_acl_extern_obj, 0, SPA_MAXBLOCKSIZE);
2083 		else
2084 			dmu_tx_hold_write(tx, DMU_NEW_OBJECT,
2085 			    0, ZFS_ACL_SIZE(MAX_ACL_SIZE));
2086 	}
2087 
2088 	if ((mask & (AT_UID | AT_GID)) && zp->z_phys->zp_xattr != 0) {
2089 		err = zfs_zget(zp->z_zfsvfs, zp->z_phys->zp_xattr, &attrzp);
2090 		if (err) {
2091 			dmu_tx_abort(tx);
2092 			ZFS_EXIT(zfsvfs);
2093 			return (err);
2094 		}
2095 		dmu_tx_hold_bonus(tx, attrzp->z_id);
2096 	}
2097 
2098 	err = dmu_tx_assign(tx, zfsvfs->z_assign);
2099 	if (err) {
2100 		if (attrzp)
2101 			VN_RELE(ZTOV(attrzp));
2102 		if (err == ERESTART && zfsvfs->z_assign == TXG_NOWAIT) {
2103 			dmu_tx_wait(tx);
2104 			dmu_tx_abort(tx);
2105 			goto top;
2106 		}
2107 		dmu_tx_abort(tx);
2108 		ZFS_EXIT(zfsvfs);
2109 		return (err);
2110 	}
2111 
2112 	dmu_buf_will_dirty(zp->z_dbuf, tx);
2113 
2114 	/*
2115 	 * Set each attribute requested.
2116 	 * We group settings according to the locks they need to acquire.
2117 	 *
2118 	 * Note: you cannot set ctime directly, although it will be
2119 	 * updated as a side-effect of calling this function.
2120 	 */
2121 
2122 	mutex_enter(&zp->z_lock);
2123 
2124 	if (mask & AT_MODE) {
2125 		err = zfs_acl_chmod_setattr(zp, new_mode, tx);
2126 		ASSERT3U(err, ==, 0);
2127 	}
2128 
2129 	if (attrzp)
2130 		mutex_enter(&attrzp->z_lock);
2131 
2132 	if (mask & AT_UID) {
2133 		zp->z_phys->zp_uid = (uint64_t)vap->va_uid;
2134 		if (attrzp) {
2135 			attrzp->z_phys->zp_uid = (uint64_t)vap->va_uid;
2136 		}
2137 	}
2138 
2139 	if (mask & AT_GID) {
2140 		zp->z_phys->zp_gid = (uint64_t)vap->va_gid;
2141 		if (attrzp)
2142 			attrzp->z_phys->zp_gid = (uint64_t)vap->va_gid;
2143 	}
2144 
2145 	if (attrzp)
2146 		mutex_exit(&attrzp->z_lock);
2147 
2148 	if (mask & AT_ATIME)
2149 		ZFS_TIME_ENCODE(&vap->va_atime, pzp->zp_atime);
2150 
2151 	if (mask & AT_MTIME)
2152 		ZFS_TIME_ENCODE(&vap->va_mtime, pzp->zp_mtime);
2153 
2154 	if (mask & AT_SIZE)
2155 		zfs_time_stamper_locked(zp, CONTENT_MODIFIED, tx);
2156 	else if (mask != 0)
2157 		zfs_time_stamper_locked(zp, STATE_CHANGED, tx);
2158 
2159 	if (mask != 0)
2160 		zfs_log_setattr(zilog, tx, TX_SETATTR, zp, vap, mask);
2161 
2162 	mutex_exit(&zp->z_lock);
2163 
2164 	if (attrzp)
2165 		VN_RELE(ZTOV(attrzp));
2166 
2167 	dmu_tx_commit(tx);
2168 
2169 	ZFS_EXIT(zfsvfs);
2170 	return (err);
2171 }
2172 
2173 /*
2174  * Search back through the directory tree, using the ".." entries.
2175  * Lock each directory in the chain to prevent concurrent renames.
2176  * Fail any attempt to move a directory into one of its own descendants.
2177  * XXX - z_parent_lock can overlap with map or grow locks
2178  */
2179 typedef struct zfs_zlock {
2180 	krwlock_t	*zl_rwlock;	/* lock we acquired */
2181 	znode_t		*zl_znode;	/* znode we held */
2182 	struct zfs_zlock *zl_next;	/* next in list */
2183 } zfs_zlock_t;
2184 
2185 static int
2186 zfs_rename_lock(znode_t *szp, znode_t *tdzp, znode_t *sdzp, zfs_zlock_t **zlpp)
2187 {
2188 	zfs_zlock_t	*zl;
2189 	znode_t 	*zp = tdzp;
2190 	uint64_t	rootid = zp->z_zfsvfs->z_root;
2191 	uint64_t	*oidp = &zp->z_id;
2192 	krwlock_t	*rwlp = &szp->z_parent_lock;
2193 	krw_t		rw = RW_WRITER;
2194 
2195 	/*
2196 	 * First pass write-locks szp and compares to zp->z_id.
2197 	 * Later passes read-lock zp and compare to zp->z_parent.
2198 	 */
2199 	do {
2200 		zl = kmem_alloc(sizeof (*zl), KM_SLEEP);
2201 		zl->zl_rwlock = rwlp;
2202 		zl->zl_znode = NULL;
2203 		zl->zl_next = *zlpp;
2204 		*zlpp = zl;
2205 
2206 		rw_enter(rwlp, rw);
2207 
2208 		if (*oidp == szp->z_id)		/* We're a descendant of szp */
2209 			return (EINVAL);
2210 
2211 		if (*oidp == rootid)		/* We've hit the top */
2212 			return (0);
2213 
2214 		if (rw == RW_READER) {		/* i.e. not the first pass */
2215 			int error = zfs_zget(zp->z_zfsvfs, *oidp, &zp);
2216 			if (error)
2217 				return (error);
2218 			zl->zl_znode = zp;
2219 		}
2220 		oidp = &zp->z_phys->zp_parent;
2221 		rwlp = &zp->z_parent_lock;
2222 		rw = RW_READER;
2223 
2224 	} while (zp->z_id != sdzp->z_id);
2225 
2226 	return (0);
2227 }
2228 
2229 /*
2230  * Drop locks and release vnodes that were held by zfs_rename_lock().
2231  */
2232 static void
2233 zfs_rename_unlock(zfs_zlock_t **zlpp)
2234 {
2235 	zfs_zlock_t *zl;
2236 
2237 	while ((zl = *zlpp) != NULL) {
2238 		if (zl->zl_znode != NULL)
2239 			VN_RELE(ZTOV(zl->zl_znode));
2240 		rw_exit(zl->zl_rwlock);
2241 		*zlpp = zl->zl_next;
2242 		kmem_free(zl, sizeof (*zl));
2243 	}
2244 }
2245 
2246 /*
2247  * Move an entry from the provided source directory to the target
2248  * directory.  Change the entry name as indicated.
2249  *
2250  *	IN:	sdvp	- Source directory containing the "old entry".
2251  *		snm	- Old entry name.
2252  *		tdvp	- Target directory to contain the "new entry".
2253  *		tnm	- New entry name.
2254  *		cr	- credentials of caller.
2255  *
2256  *	RETURN:	0 if success
2257  *		error code if failure
2258  *
2259  * Timestamps:
2260  *	sdvp,tdvp - ctime|mtime updated
2261  */
2262 static int
2263 zfs_rename(vnode_t *sdvp, char *snm, vnode_t *tdvp, char *tnm, cred_t *cr)
2264 {
2265 	znode_t		*tdzp, *szp, *tzp;
2266 	znode_t		*sdzp = VTOZ(sdvp);
2267 	zfsvfs_t	*zfsvfs = sdzp->z_zfsvfs;
2268 	zilog_t		*zilog = zfsvfs->z_log;
2269 	vnode_t		*realvp;
2270 	zfs_dirlock_t	*sdl, *tdl;
2271 	dmu_tx_t	*tx;
2272 	zfs_zlock_t	*zl;
2273 	int		cmp, serr, terr, error;
2274 
2275 	ZFS_ENTER(zfsvfs);
2276 
2277 	/*
2278 	 * Make sure we have the real vp for the target directory.
2279 	 */
2280 	if (VOP_REALVP(tdvp, &realvp) == 0)
2281 		tdvp = realvp;
2282 
2283 	if (tdvp->v_vfsp != sdvp->v_vfsp) {
2284 		ZFS_EXIT(zfsvfs);
2285 		return (EXDEV);
2286 	}
2287 
2288 	tdzp = VTOZ(tdvp);
2289 top:
2290 	szp = NULL;
2291 	tzp = NULL;
2292 	zl = NULL;
2293 
2294 	/*
2295 	 * This is to prevent the creation of links into attribute space
2296 	 * by renaming a linked file into/outof an attribute directory.
2297 	 * See the comment in zfs_link() for why this is considered bad.
2298 	 */
2299 	if ((tdzp->z_phys->zp_flags & ZFS_XATTR) !=
2300 	    (sdzp->z_phys->zp_flags & ZFS_XATTR)) {
2301 		ZFS_EXIT(zfsvfs);
2302 		return (EINVAL);
2303 	}
2304 
2305 	/*
2306 	 * Lock source and target directory entries.  To prevent deadlock,
2307 	 * a lock ordering must be defined.  We lock the directory with
2308 	 * the smallest object id first, or if it's a tie, the one with
2309 	 * the lexically first name.
2310 	 */
2311 	if (sdzp->z_id < tdzp->z_id) {
2312 		cmp = -1;
2313 	} else if (sdzp->z_id > tdzp->z_id) {
2314 		cmp = 1;
2315 	} else {
2316 		cmp = strcmp(snm, tnm);
2317 		if (cmp == 0) {
2318 			/*
2319 			 * POSIX: "If the old argument and the new argument
2320 			 * both refer to links to the same existing file,
2321 			 * the rename() function shall return successfully
2322 			 * and perform no other action."
2323 			 */
2324 			ZFS_EXIT(zfsvfs);
2325 			return (0);
2326 		}
2327 	}
2328 	if (cmp < 0) {
2329 		serr = zfs_dirent_lock(&sdl, sdzp, snm, &szp, ZEXISTS);
2330 		terr = zfs_dirent_lock(&tdl, tdzp, tnm, &tzp, 0);
2331 	} else {
2332 		terr = zfs_dirent_lock(&tdl, tdzp, tnm, &tzp, 0);
2333 		serr = zfs_dirent_lock(&sdl, sdzp, snm, &szp, ZEXISTS);
2334 	}
2335 
2336 	if (serr) {
2337 		/*
2338 		 * Source entry invalid or not there.
2339 		 */
2340 		if (!terr) {
2341 			zfs_dirent_unlock(tdl);
2342 			if (tzp)
2343 				VN_RELE(ZTOV(tzp));
2344 		}
2345 		if (strcmp(snm, "..") == 0)
2346 			serr = EINVAL;
2347 		ZFS_EXIT(zfsvfs);
2348 		return (serr);
2349 	}
2350 	if (terr) {
2351 		zfs_dirent_unlock(sdl);
2352 		VN_RELE(ZTOV(szp));
2353 		if (strcmp(tnm, "..") == 0)
2354 			terr = EINVAL;
2355 		ZFS_EXIT(zfsvfs);
2356 		return (terr);
2357 	}
2358 
2359 	/*
2360 	 * Must have write access at the source to remove the old entry
2361 	 * and write access at the target to create the new entry.
2362 	 * Note that if target and source are the same, this can be
2363 	 * done in a single check.
2364 	 */
2365 
2366 	if (error = zfs_zaccess_rename(sdzp, szp, tdzp, tzp, cr))
2367 		goto out;
2368 
2369 	if (ZTOV(szp)->v_type == VDIR) {
2370 		/*
2371 		 * Check to make sure rename is valid.
2372 		 * Can't do a move like this: /usr/a/b to /usr/a/b/c/d
2373 		 */
2374 		if (error = zfs_rename_lock(szp, tdzp, sdzp, &zl))
2375 			goto out;
2376 	}
2377 
2378 	/*
2379 	 * Does target exist?
2380 	 */
2381 	if (tzp) {
2382 		/*
2383 		 * Source and target must be the same type.
2384 		 */
2385 		if (ZTOV(szp)->v_type == VDIR) {
2386 			if (ZTOV(tzp)->v_type != VDIR) {
2387 				error = ENOTDIR;
2388 				goto out;
2389 			}
2390 		} else {
2391 			if (ZTOV(tzp)->v_type == VDIR) {
2392 				error = EISDIR;
2393 				goto out;
2394 			}
2395 		}
2396 		/*
2397 		 * POSIX dictates that when the source and target
2398 		 * entries refer to the same file object, rename
2399 		 * must do nothing and exit without error.
2400 		 */
2401 		if (szp->z_id == tzp->z_id) {
2402 			error = 0;
2403 			goto out;
2404 		}
2405 	}
2406 
2407 	vnevent_rename_src(ZTOV(szp));
2408 	if (tzp)
2409 		vnevent_rename_dest(ZTOV(tzp));
2410 
2411 	tx = dmu_tx_create(zfsvfs->z_os);
2412 	dmu_tx_hold_bonus(tx, szp->z_id);	/* nlink changes */
2413 	dmu_tx_hold_bonus(tx, sdzp->z_id);	/* nlink changes */
2414 	dmu_tx_hold_zap(tx, sdzp->z_id, FALSE, snm);
2415 	dmu_tx_hold_zap(tx, tdzp->z_id, TRUE, tnm);
2416 	if (sdzp != tdzp)
2417 		dmu_tx_hold_bonus(tx, tdzp->z_id);	/* nlink changes */
2418 	if (tzp)
2419 		dmu_tx_hold_bonus(tx, tzp->z_id);	/* parent changes */
2420 	dmu_tx_hold_zap(tx, zfsvfs->z_dqueue, FALSE, NULL);
2421 	error = dmu_tx_assign(tx, zfsvfs->z_assign);
2422 	if (error) {
2423 		if (zl != NULL)
2424 			zfs_rename_unlock(&zl);
2425 		zfs_dirent_unlock(sdl);
2426 		zfs_dirent_unlock(tdl);
2427 		VN_RELE(ZTOV(szp));
2428 		if (tzp)
2429 			VN_RELE(ZTOV(tzp));
2430 		if (error == ERESTART && zfsvfs->z_assign == TXG_NOWAIT) {
2431 			dmu_tx_wait(tx);
2432 			dmu_tx_abort(tx);
2433 			goto top;
2434 		}
2435 		dmu_tx_abort(tx);
2436 		ZFS_EXIT(zfsvfs);
2437 		return (error);
2438 	}
2439 
2440 	if (tzp)	/* Attempt to remove the existing target */
2441 		error = zfs_link_destroy(tdl, tzp, tx, 0, NULL);
2442 
2443 	if (error == 0) {
2444 		error = zfs_link_create(tdl, szp, tx, ZRENAMING);
2445 		if (error == 0) {
2446 			error = zfs_link_destroy(sdl, szp, tx, ZRENAMING, NULL);
2447 			ASSERT(error == 0);
2448 			zfs_log_rename(zilog, tx, TX_RENAME, sdzp,
2449 			    sdl->dl_name, tdzp, tdl->dl_name, szp);
2450 		}
2451 	}
2452 
2453 	dmu_tx_commit(tx);
2454 out:
2455 	if (zl != NULL)
2456 		zfs_rename_unlock(&zl);
2457 
2458 	zfs_dirent_unlock(sdl);
2459 	zfs_dirent_unlock(tdl);
2460 
2461 	VN_RELE(ZTOV(szp));
2462 	if (tzp)
2463 		VN_RELE(ZTOV(tzp));
2464 
2465 	ZFS_EXIT(zfsvfs);
2466 	return (error);
2467 }
2468 
2469 /*
2470  * Insert the indicated symbolic reference entry into the directory.
2471  *
2472  *	IN:	dvp	- Directory to contain new symbolic link.
2473  *		link	- Name for new symlink entry.
2474  *		vap	- Attributes of new entry.
2475  *		target	- Target path of new symlink.
2476  *		cr	- credentials of caller.
2477  *
2478  *	RETURN:	0 if success
2479  *		error code if failure
2480  *
2481  * Timestamps:
2482  *	dvp - ctime|mtime updated
2483  */
2484 static int
2485 zfs_symlink(vnode_t *dvp, char *name, vattr_t *vap, char *link, cred_t *cr)
2486 {
2487 	znode_t		*zp, *dzp = VTOZ(dvp);
2488 	zfs_dirlock_t	*dl;
2489 	dmu_tx_t	*tx;
2490 	zfsvfs_t	*zfsvfs = dzp->z_zfsvfs;
2491 	zilog_t		*zilog = zfsvfs->z_log;
2492 	uint64_t	zoid;
2493 	int		len = strlen(link);
2494 	int		error;
2495 
2496 	ASSERT(vap->va_type == VLNK);
2497 
2498 	ZFS_ENTER(zfsvfs);
2499 top:
2500 	if (error = zfs_zaccess(dzp, ACE_ADD_FILE, cr)) {
2501 		ZFS_EXIT(zfsvfs);
2502 		return (error);
2503 	}
2504 
2505 	if (len > MAXPATHLEN) {
2506 		ZFS_EXIT(zfsvfs);
2507 		return (ENAMETOOLONG);
2508 	}
2509 
2510 	/*
2511 	 * Attempt to lock directory; fail if entry already exists.
2512 	 */
2513 	if (error = zfs_dirent_lock(&dl, dzp, name, &zp, ZNEW)) {
2514 		ZFS_EXIT(zfsvfs);
2515 		return (error);
2516 	}
2517 
2518 	tx = dmu_tx_create(zfsvfs->z_os);
2519 	dmu_tx_hold_write(tx, DMU_NEW_OBJECT, 0, MAX(1, len));
2520 	dmu_tx_hold_bonus(tx, dzp->z_id);
2521 	dmu_tx_hold_zap(tx, dzp->z_id, TRUE, name);
2522 	if (dzp->z_phys->zp_flags & ZFS_INHERIT_ACE)
2523 		dmu_tx_hold_write(tx, DMU_NEW_OBJECT, 0, SPA_MAXBLOCKSIZE);
2524 	error = dmu_tx_assign(tx, zfsvfs->z_assign);
2525 	if (error) {
2526 		zfs_dirent_unlock(dl);
2527 		if (error == ERESTART && zfsvfs->z_assign == TXG_NOWAIT) {
2528 			dmu_tx_wait(tx);
2529 			dmu_tx_abort(tx);
2530 			goto top;
2531 		}
2532 		dmu_tx_abort(tx);
2533 		ZFS_EXIT(zfsvfs);
2534 		return (error);
2535 	}
2536 
2537 	dmu_buf_will_dirty(dzp->z_dbuf, tx);
2538 
2539 	/*
2540 	 * Create a new object for the symlink.
2541 	 * Put the link content into bonus buffer if it will fit;
2542 	 * otherwise, store it just like any other file data.
2543 	 */
2544 	zoid = 0;
2545 	if (sizeof (znode_phys_t) + len <= dmu_bonus_max()) {
2546 		zfs_mknode(dzp, vap, &zoid, tx, cr, 0, &zp, len);
2547 		if (len != 0)
2548 			bcopy(link, zp->z_phys + 1, len);
2549 	} else {
2550 		dmu_buf_t *dbp;
2551 
2552 		zfs_mknode(dzp, vap, &zoid, tx, cr, 0, &zp, 0);
2553 
2554 		/*
2555 		 * Nothing can access the znode yet so no locking needed
2556 		 * for growing the znode's blocksize.
2557 		 */
2558 		zfs_grow_blocksize(zp, len, tx);
2559 
2560 		VERIFY(0 == dmu_buf_hold(zfsvfs->z_os, zoid, 0, FTAG, &dbp));
2561 		dmu_buf_will_dirty(dbp, tx);
2562 
2563 		ASSERT3U(len, <=, dbp->db_size);
2564 		bcopy(link, dbp->db_data, len);
2565 		dmu_buf_rele(dbp, FTAG);
2566 	}
2567 	zp->z_phys->zp_size = len;
2568 
2569 	/*
2570 	 * Insert the new object into the directory.
2571 	 */
2572 	(void) zfs_link_create(dl, zp, tx, ZNEW);
2573 out:
2574 	if (error == 0)
2575 		zfs_log_symlink(zilog, tx, TX_SYMLINK, dzp, zp, name, link);
2576 
2577 	dmu_tx_commit(tx);
2578 
2579 	zfs_dirent_unlock(dl);
2580 
2581 	VN_RELE(ZTOV(zp));
2582 
2583 	ZFS_EXIT(zfsvfs);
2584 	return (error);
2585 }
2586 
2587 /*
2588  * Return, in the buffer contained in the provided uio structure,
2589  * the symbolic path referred to by vp.
2590  *
2591  *	IN:	vp	- vnode of symbolic link.
2592  *		uoip	- structure to contain the link path.
2593  *		cr	- credentials of caller.
2594  *
2595  *	OUT:	uio	- structure to contain the link path.
2596  *
2597  *	RETURN:	0 if success
2598  *		error code if failure
2599  *
2600  * Timestamps:
2601  *	vp - atime updated
2602  */
2603 /* ARGSUSED */
2604 static int
2605 zfs_readlink(vnode_t *vp, uio_t *uio, cred_t *cr)
2606 {
2607 	znode_t		*zp = VTOZ(vp);
2608 	zfsvfs_t	*zfsvfs = zp->z_zfsvfs;
2609 	size_t		bufsz;
2610 	int		error;
2611 
2612 	ZFS_ENTER(zfsvfs);
2613 
2614 	bufsz = (size_t)zp->z_phys->zp_size;
2615 	if (bufsz + sizeof (znode_phys_t) <= zp->z_dbuf->db_size) {
2616 		error = uiomove(zp->z_phys + 1,
2617 		    MIN((size_t)bufsz, uio->uio_resid), UIO_READ, uio);
2618 	} else {
2619 		dmu_buf_t *dbp;
2620 		error = dmu_buf_hold(zfsvfs->z_os, zp->z_id, 0, FTAG, &dbp);
2621 		if (error) {
2622 			ZFS_EXIT(zfsvfs);
2623 			return (error);
2624 		}
2625 		error = uiomove(dbp->db_data,
2626 		    MIN((size_t)bufsz, uio->uio_resid), UIO_READ, uio);
2627 		dmu_buf_rele(dbp, FTAG);
2628 	}
2629 
2630 	ZFS_ACCESSTIME_STAMP(zfsvfs, zp);
2631 	ZFS_EXIT(zfsvfs);
2632 	return (error);
2633 }
2634 
2635 /*
2636  * Insert a new entry into directory tdvp referencing svp.
2637  *
2638  *	IN:	tdvp	- Directory to contain new entry.
2639  *		svp	- vnode of new entry.
2640  *		name	- name of new entry.
2641  *		cr	- credentials of caller.
2642  *
2643  *	RETURN:	0 if success
2644  *		error code if failure
2645  *
2646  * Timestamps:
2647  *	tdvp - ctime|mtime updated
2648  *	 svp - ctime updated
2649  */
2650 /* ARGSUSED */
2651 static int
2652 zfs_link(vnode_t *tdvp, vnode_t *svp, char *name, cred_t *cr)
2653 {
2654 	znode_t		*dzp = VTOZ(tdvp);
2655 	znode_t		*tzp, *szp;
2656 	zfsvfs_t	*zfsvfs = dzp->z_zfsvfs;
2657 	zilog_t		*zilog = zfsvfs->z_log;
2658 	zfs_dirlock_t	*dl;
2659 	dmu_tx_t	*tx;
2660 	vnode_t		*realvp;
2661 	int		error;
2662 
2663 	ASSERT(tdvp->v_type == VDIR);
2664 
2665 	ZFS_ENTER(zfsvfs);
2666 
2667 	if (VOP_REALVP(svp, &realvp) == 0)
2668 		svp = realvp;
2669 
2670 	if (svp->v_vfsp != tdvp->v_vfsp) {
2671 		ZFS_EXIT(zfsvfs);
2672 		return (EXDEV);
2673 	}
2674 
2675 	szp = VTOZ(svp);
2676 top:
2677 	/*
2678 	 * We do not support links between attributes and non-attributes
2679 	 * because of the potential security risk of creating links
2680 	 * into "normal" file space in order to circumvent restrictions
2681 	 * imposed in attribute space.
2682 	 */
2683 	if ((szp->z_phys->zp_flags & ZFS_XATTR) !=
2684 	    (dzp->z_phys->zp_flags & ZFS_XATTR)) {
2685 		ZFS_EXIT(zfsvfs);
2686 		return (EINVAL);
2687 	}
2688 
2689 	/*
2690 	 * POSIX dictates that we return EPERM here.
2691 	 * Better choices include ENOTSUP or EISDIR.
2692 	 */
2693 	if (svp->v_type == VDIR) {
2694 		ZFS_EXIT(zfsvfs);
2695 		return (EPERM);
2696 	}
2697 
2698 	if ((uid_t)szp->z_phys->zp_uid != crgetuid(cr) &&
2699 	    secpolicy_basic_link(cr) != 0) {
2700 		ZFS_EXIT(zfsvfs);
2701 		return (EPERM);
2702 	}
2703 
2704 	if (error = zfs_zaccess(dzp, ACE_ADD_FILE, cr)) {
2705 		ZFS_EXIT(zfsvfs);
2706 		return (error);
2707 	}
2708 
2709 	/*
2710 	 * Attempt to lock directory; fail if entry already exists.
2711 	 */
2712 	if (error = zfs_dirent_lock(&dl, dzp, name, &tzp, ZNEW)) {
2713 		ZFS_EXIT(zfsvfs);
2714 		return (error);
2715 	}
2716 
2717 	tx = dmu_tx_create(zfsvfs->z_os);
2718 	dmu_tx_hold_bonus(tx, szp->z_id);
2719 	dmu_tx_hold_zap(tx, dzp->z_id, TRUE, name);
2720 	error = dmu_tx_assign(tx, zfsvfs->z_assign);
2721 	if (error) {
2722 		zfs_dirent_unlock(dl);
2723 		if (error == ERESTART && zfsvfs->z_assign == TXG_NOWAIT) {
2724 			dmu_tx_wait(tx);
2725 			dmu_tx_abort(tx);
2726 			goto top;
2727 		}
2728 		dmu_tx_abort(tx);
2729 		ZFS_EXIT(zfsvfs);
2730 		return (error);
2731 	}
2732 
2733 	error = zfs_link_create(dl, szp, tx, 0);
2734 
2735 	if (error == 0)
2736 		zfs_log_link(zilog, tx, TX_LINK, dzp, szp, name);
2737 
2738 	dmu_tx_commit(tx);
2739 
2740 	zfs_dirent_unlock(dl);
2741 
2742 	ZFS_EXIT(zfsvfs);
2743 	return (error);
2744 }
2745 
2746 /*
2747  * zfs_null_putapage() is used when the file system has been force
2748  * unmounted. It just drops the pages.
2749  */
2750 /* ARGSUSED */
2751 static int
2752 zfs_null_putapage(vnode_t *vp, page_t *pp, u_offset_t *offp,
2753 		size_t *lenp, int flags, cred_t *cr)
2754 {
2755 	pvn_write_done(pp, B_INVAL|B_FORCE|B_ERROR);
2756 	return (0);
2757 }
2758 
2759 /*
2760  * Push a page out to disk, klustering if possible.
2761  *
2762  *	IN:	vp	- file to push page to.
2763  *		pp	- page to push.
2764  *		flags	- additional flags.
2765  *		cr	- credentials of caller.
2766  *
2767  *	OUT:	offp	- start of range pushed.
2768  *		lenp	- len of range pushed.
2769  *
2770  *	RETURN:	0 if success
2771  *		error code if failure
2772  *
2773  * NOTE: callers must have locked the page to be pushed.  On
2774  * exit, the page (and all other pages in the kluster) must be
2775  * unlocked.
2776  */
2777 /* ARGSUSED */
2778 static int
2779 zfs_putapage(vnode_t *vp, page_t *pp, u_offset_t *offp,
2780 		size_t *lenp, int flags, cred_t *cr)
2781 {
2782 	znode_t		*zp = VTOZ(vp);
2783 	zfsvfs_t	*zfsvfs = zp->z_zfsvfs;
2784 	zilog_t		*zilog = zfsvfs->z_log;
2785 	dmu_tx_t	*tx;
2786 	rl_t		*rl;
2787 	u_offset_t	off, koff;
2788 	size_t		len, klen;
2789 	int		err;
2790 
2791 	off = pp->p_offset;
2792 	len = PAGESIZE;
2793 	/*
2794 	 * If our blocksize is bigger than the page size, try to kluster
2795 	 * muiltiple pages so that we write a full block (thus avoiding
2796 	 * a read-modify-write).
2797 	 */
2798 	if (zp->z_blksz > PAGESIZE) {
2799 		uint64_t filesz = zp->z_phys->zp_size;
2800 
2801 		if (!ISP2(zp->z_blksz)) {
2802 			/* Only one block in the file. */
2803 			klen = P2ROUNDUP((ulong_t)zp->z_blksz, PAGESIZE);
2804 			koff = 0;
2805 		} else {
2806 			klen = zp->z_blksz;
2807 			koff = P2ALIGN(off, (u_offset_t)klen);
2808 		}
2809 		ASSERT(koff <= filesz);
2810 		if (koff + klen > filesz)
2811 			klen = P2ROUNDUP(filesz - koff, (uint64_t)PAGESIZE);
2812 		pp = pvn_write_kluster(vp, pp, &off, &len, koff, klen, flags);
2813 	}
2814 	ASSERT3U(btop(len), ==, btopr(len));
2815 top:
2816 	rl = zfs_range_lock(zp, off, len, RL_WRITER);
2817 	/*
2818 	 * Can't push pages past end-of-file.
2819 	 */
2820 	if (off >= zp->z_phys->zp_size) {
2821 		/* discard all pages */
2822 		flags |= B_INVAL;
2823 		err = 0;
2824 		goto out;
2825 	} else if (off + len > zp->z_phys->zp_size) {
2826 		int npages = btopr(zp->z_phys->zp_size - off);
2827 		page_t *trunc;
2828 
2829 		page_list_break(&pp, &trunc, npages);
2830 		/* discard pages past end of file */
2831 		if (trunc)
2832 			pvn_write_done(trunc, B_INVAL | flags);
2833 		len = zp->z_phys->zp_size - off;
2834 	}
2835 
2836 	tx = dmu_tx_create(zfsvfs->z_os);
2837 	dmu_tx_hold_write(tx, zp->z_id, off, len);
2838 	dmu_tx_hold_bonus(tx, zp->z_id);
2839 	err = dmu_tx_assign(tx, zfsvfs->z_assign);
2840 	if (err != 0) {
2841 		if (err == ERESTART && zfsvfs->z_assign == TXG_NOWAIT) {
2842 			zfs_range_unlock(rl);
2843 			dmu_tx_wait(tx);
2844 			dmu_tx_abort(tx);
2845 			err = 0;
2846 			goto top;
2847 		}
2848 		dmu_tx_abort(tx);
2849 		goto out;
2850 	}
2851 
2852 	if (zp->z_blksz <= PAGESIZE) {
2853 		caddr_t va = ppmapin(pp, PROT_READ, (caddr_t)-1);
2854 		ASSERT3U(len, <=, PAGESIZE);
2855 		dmu_write(zfsvfs->z_os, zp->z_id, off, len, va, tx);
2856 		ppmapout(va);
2857 	} else {
2858 		err = dmu_write_pages(zfsvfs->z_os, zp->z_id, off, len, pp, tx);
2859 	}
2860 
2861 	if (err == 0) {
2862 		zfs_time_stamper(zp, CONTENT_MODIFIED, tx);
2863 		(void) zfs_log_write(
2864 		    zilog, tx, TX_WRITE, zp, off, len, 0, NULL);
2865 		dmu_tx_commit(tx);
2866 	}
2867 
2868 out:
2869 	zfs_range_unlock(rl);
2870 	pvn_write_done(pp, (err ? B_ERROR : 0) | B_WRITE | flags);
2871 	if (offp)
2872 		*offp = off;
2873 	if (lenp)
2874 		*lenp = len;
2875 
2876 	return (err);
2877 }
2878 
2879 /*
2880  * Copy the portion of the file indicated from pages into the file.
2881  * The pages are stored in a page list attached to the files vnode.
2882  *
2883  *	IN:	vp	- vnode of file to push page data to.
2884  *		off	- position in file to put data.
2885  *		len	- amount of data to write.
2886  *		flags	- flags to control the operation.
2887  *		cr	- credentials of caller.
2888  *
2889  *	RETURN:	0 if success
2890  *		error code if failure
2891  *
2892  * Timestamps:
2893  *	vp - ctime|mtime updated
2894  */
2895 static int
2896 zfs_putpage(vnode_t *vp, offset_t off, size_t len, int flags, cred_t *cr)
2897 {
2898 	znode_t		*zp = VTOZ(vp);
2899 	zfsvfs_t	*zfsvfs = zp->z_zfsvfs;
2900 	page_t		*pp;
2901 	size_t		io_len;
2902 	u_offset_t	io_off;
2903 	uint64_t	filesz;
2904 	int		error = 0;
2905 
2906 	ZFS_ENTER(zfsvfs);
2907 
2908 	ASSERT(zp->z_dbuf_held && zp->z_phys);
2909 
2910 	if (len == 0) {
2911 		/*
2912 		 * Search the entire vp list for pages >= off.
2913 		 */
2914 		error = pvn_vplist_dirty(vp, (u_offset_t)off, zfs_putapage,
2915 		    flags, cr);
2916 		goto out;
2917 	}
2918 
2919 	filesz = zp->z_phys->zp_size; /* get consistent copy of zp_size */
2920 	if (off > filesz) {
2921 		/* past end of file */
2922 		ZFS_EXIT(zfsvfs);
2923 		return (0);
2924 	}
2925 
2926 	len = MIN(len, filesz - off);
2927 
2928 	for (io_off = off; io_off < off + len; io_off += io_len) {
2929 		if ((flags & B_INVAL) || ((flags & B_ASYNC) == 0)) {
2930 			pp = page_lookup(vp, io_off,
2931 				(flags & (B_INVAL | B_FREE)) ?
2932 					SE_EXCL : SE_SHARED);
2933 		} else {
2934 			pp = page_lookup_nowait(vp, io_off,
2935 				(flags & B_FREE) ? SE_EXCL : SE_SHARED);
2936 		}
2937 
2938 		if (pp != NULL && pvn_getdirty(pp, flags)) {
2939 			int err;
2940 
2941 			/*
2942 			 * Found a dirty page to push
2943 			 */
2944 			err = zfs_putapage(vp, pp, &io_off, &io_len, flags, cr);
2945 			if (err)
2946 				error = err;
2947 		} else {
2948 			io_len = PAGESIZE;
2949 		}
2950 	}
2951 out:
2952 	if ((flags & B_ASYNC) == 0)
2953 		zil_commit(zfsvfs->z_log, UINT64_MAX, zp->z_id);
2954 	ZFS_EXIT(zfsvfs);
2955 	return (error);
2956 }
2957 
2958 void
2959 zfs_inactive(vnode_t *vp, cred_t *cr)
2960 {
2961 	znode_t	*zp = VTOZ(vp);
2962 	zfsvfs_t *zfsvfs = zp->z_zfsvfs;
2963 	int error;
2964 
2965 	rw_enter(&zfsvfs->z_um_lock, RW_READER);
2966 	if (zfsvfs->z_unmounted2) {
2967 		ASSERT(zp->z_dbuf_held == 0);
2968 
2969 		if (vn_has_cached_data(vp)) {
2970 			(void) pvn_vplist_dirty(vp, 0, zfs_null_putapage,
2971 			    B_INVAL, cr);
2972 		}
2973 
2974 		mutex_enter(&zp->z_lock);
2975 		vp->v_count = 0; /* count arrives as 1 */
2976 		if (zp->z_dbuf == NULL) {
2977 			mutex_exit(&zp->z_lock);
2978 			zfs_znode_free(zp);
2979 		} else {
2980 			mutex_exit(&zp->z_lock);
2981 		}
2982 		rw_exit(&zfsvfs->z_um_lock);
2983 		VFS_RELE(zfsvfs->z_vfs);
2984 		return;
2985 	}
2986 
2987 	/*
2988 	 * Attempt to push any data in the page cache.  If this fails
2989 	 * we will get kicked out later in zfs_zinactive().
2990 	 */
2991 	if (vn_has_cached_data(vp)) {
2992 		(void) pvn_vplist_dirty(vp, 0, zfs_putapage, B_INVAL|B_ASYNC,
2993 		    cr);
2994 	}
2995 
2996 	if (zp->z_atime_dirty && zp->z_reap == 0) {
2997 		dmu_tx_t *tx = dmu_tx_create(zfsvfs->z_os);
2998 
2999 		dmu_tx_hold_bonus(tx, zp->z_id);
3000 		error = dmu_tx_assign(tx, TXG_WAIT);
3001 		if (error) {
3002 			dmu_tx_abort(tx);
3003 		} else {
3004 			dmu_buf_will_dirty(zp->z_dbuf, tx);
3005 			mutex_enter(&zp->z_lock);
3006 			zp->z_atime_dirty = 0;
3007 			mutex_exit(&zp->z_lock);
3008 			dmu_tx_commit(tx);
3009 		}
3010 	}
3011 
3012 	zfs_zinactive(zp);
3013 	rw_exit(&zfsvfs->z_um_lock);
3014 }
3015 
3016 /*
3017  * Bounds-check the seek operation.
3018  *
3019  *	IN:	vp	- vnode seeking within
3020  *		ooff	- old file offset
3021  *		noffp	- pointer to new file offset
3022  *
3023  *	RETURN:	0 if success
3024  *		EINVAL if new offset invalid
3025  */
3026 /* ARGSUSED */
3027 static int
3028 zfs_seek(vnode_t *vp, offset_t ooff, offset_t *noffp)
3029 {
3030 	if (vp->v_type == VDIR)
3031 		return (0);
3032 	return ((*noffp < 0 || *noffp > MAXOFFSET_T) ? EINVAL : 0);
3033 }
3034 
3035 /*
3036  * Pre-filter the generic locking function to trap attempts to place
3037  * a mandatory lock on a memory mapped file.
3038  */
3039 static int
3040 zfs_frlock(vnode_t *vp, int cmd, flock64_t *bfp, int flag, offset_t offset,
3041     flk_callback_t *flk_cbp, cred_t *cr)
3042 {
3043 	znode_t *zp = VTOZ(vp);
3044 	zfsvfs_t *zfsvfs = zp->z_zfsvfs;
3045 	int error;
3046 
3047 	ZFS_ENTER(zfsvfs);
3048 
3049 	/*
3050 	 * We are following the UFS semantics with respect to mapcnt
3051 	 * here: If we see that the file is mapped already, then we will
3052 	 * return an error, but we don't worry about races between this
3053 	 * function and zfs_map().
3054 	 */
3055 	if (zp->z_mapcnt > 0 && MANDMODE((mode_t)zp->z_phys->zp_mode)) {
3056 		ZFS_EXIT(zfsvfs);
3057 		return (EAGAIN);
3058 	}
3059 	error = fs_frlock(vp, cmd, bfp, flag, offset, flk_cbp, cr);
3060 	ZFS_EXIT(zfsvfs);
3061 	return (error);
3062 }
3063 
3064 /*
3065  * If we can't find a page in the cache, we will create a new page
3066  * and fill it with file data.  For efficiency, we may try to fill
3067  * multiple pages at once (klustering).
3068  */
3069 static int
3070 zfs_fillpage(vnode_t *vp, u_offset_t off, struct seg *seg,
3071     caddr_t addr, page_t *pl[], size_t plsz, enum seg_rw rw)
3072 {
3073 	znode_t *zp = VTOZ(vp);
3074 	page_t *pp, *cur_pp;
3075 	objset_t *os = zp->z_zfsvfs->z_os;
3076 	caddr_t va;
3077 	u_offset_t io_off, total;
3078 	uint64_t oid = zp->z_id;
3079 	size_t io_len;
3080 	uint64_t filesz;
3081 	int err;
3082 
3083 	/*
3084 	 * If we are only asking for a single page don't bother klustering.
3085 	 */
3086 	filesz = zp->z_phys->zp_size; /* get consistent copy of zp_size */
3087 	if (off >= filesz)
3088 		return (EFAULT);
3089 	if (plsz == PAGESIZE || zp->z_blksz <= PAGESIZE) {
3090 		io_off = off;
3091 		io_len = PAGESIZE;
3092 		pp = page_create_va(vp, io_off, io_len, PG_WAIT, seg, addr);
3093 	} else {
3094 		/*
3095 		 * Try to fill a kluster of pages (a blocks worth).
3096 		 */
3097 		size_t klen;
3098 		u_offset_t koff;
3099 
3100 		if (!ISP2(zp->z_blksz)) {
3101 			/* Only one block in the file. */
3102 			klen = P2ROUNDUP((ulong_t)zp->z_blksz, PAGESIZE);
3103 			koff = 0;
3104 		} else {
3105 			klen = plsz;
3106 			koff = P2ALIGN(off, (u_offset_t)klen);
3107 		}
3108 		ASSERT(koff <= filesz);
3109 		if (koff + klen > filesz)
3110 			klen = P2ROUNDUP(filesz, (uint64_t)PAGESIZE) - koff;
3111 		ASSERT3U(off, >=, koff);
3112 		ASSERT3U(off, <, koff + klen);
3113 		pp = pvn_read_kluster(vp, off, seg, addr, &io_off,
3114 			    &io_len, koff, klen, 0);
3115 	}
3116 	if (pp == NULL) {
3117 		/*
3118 		 * Some other thread entered the page before us.
3119 		 * Return to zfs_getpage to retry the lookup.
3120 		 */
3121 		*pl = NULL;
3122 		return (0);
3123 	}
3124 
3125 	/*
3126 	 * Fill the pages in the kluster.
3127 	 */
3128 	cur_pp = pp;
3129 	for (total = io_off + io_len; io_off < total; io_off += PAGESIZE) {
3130 		ASSERT3U(io_off, ==, cur_pp->p_offset);
3131 		va = ppmapin(cur_pp, PROT_READ | PROT_WRITE, (caddr_t)-1);
3132 		err = dmu_read(os, oid, io_off, PAGESIZE, va);
3133 		ppmapout(va);
3134 		if (err) {
3135 			/* On error, toss the entire kluster */
3136 			pvn_read_done(pp, B_ERROR);
3137 			return (err);
3138 		}
3139 		cur_pp = cur_pp->p_next;
3140 	}
3141 out:
3142 	/*
3143 	 * Fill in the page list array from the kluster.  If
3144 	 * there are too many pages in the kluster, return
3145 	 * as many pages as possible starting from the desired
3146 	 * offset `off'.
3147 	 * NOTE: the page list will always be null terminated.
3148 	 */
3149 	pvn_plist_init(pp, pl, plsz, off, io_len, rw);
3150 
3151 	return (0);
3152 }
3153 
3154 /*
3155  * Return pointers to the pages for the file region [off, off + len]
3156  * in the pl array.  If plsz is greater than len, this function may
3157  * also return page pointers from before or after the specified
3158  * region (i.e. some region [off', off' + plsz]).  These additional
3159  * pages are only returned if they are already in the cache, or were
3160  * created as part of a klustered read.
3161  *
3162  *	IN:	vp	- vnode of file to get data from.
3163  *		off	- position in file to get data from.
3164  *		len	- amount of data to retrieve.
3165  *		plsz	- length of provided page list.
3166  *		seg	- segment to obtain pages for.
3167  *		addr	- virtual address of fault.
3168  *		rw	- mode of created pages.
3169  *		cr	- credentials of caller.
3170  *
3171  *	OUT:	protp	- protection mode of created pages.
3172  *		pl	- list of pages created.
3173  *
3174  *	RETURN:	0 if success
3175  *		error code if failure
3176  *
3177  * Timestamps:
3178  *	vp - atime updated
3179  */
3180 /* ARGSUSED */
3181 static int
3182 zfs_getpage(vnode_t *vp, offset_t off, size_t len, uint_t *protp,
3183 	page_t *pl[], size_t plsz, struct seg *seg, caddr_t addr,
3184 	enum seg_rw rw, cred_t *cr)
3185 {
3186 	znode_t		*zp = VTOZ(vp);
3187 	zfsvfs_t	*zfsvfs = zp->z_zfsvfs;
3188 	page_t		*pp, **pl0 = pl;
3189 	int		need_unlock = 0, err = 0;
3190 	offset_t	orig_off;
3191 
3192 	ZFS_ENTER(zfsvfs);
3193 
3194 	if (protp)
3195 		*protp = PROT_ALL;
3196 
3197 	ASSERT(zp->z_dbuf_held && zp->z_phys);
3198 
3199 	/* no faultahead (for now) */
3200 	if (pl == NULL) {
3201 		ZFS_EXIT(zfsvfs);
3202 		return (0);
3203 	}
3204 
3205 	/* can't fault past EOF */
3206 	if (off >= zp->z_phys->zp_size) {
3207 		ZFS_EXIT(zfsvfs);
3208 		return (EFAULT);
3209 	}
3210 	orig_off = off;
3211 
3212 	/*
3213 	 * If we already own the lock, then we must be page faulting
3214 	 * in the middle of a write to this file (i.e., we are writing
3215 	 * to this file using data from a mapped region of the file).
3216 	 */
3217 	if (rw_owner(&zp->z_map_lock) != curthread) {
3218 		rw_enter(&zp->z_map_lock, RW_WRITER);
3219 		need_unlock = TRUE;
3220 	}
3221 
3222 	/*
3223 	 * Loop through the requested range [off, off + len] looking
3224 	 * for pages.  If we don't find a page, we will need to create
3225 	 * a new page and fill it with data from the file.
3226 	 */
3227 	while (len > 0) {
3228 		if (plsz < PAGESIZE)
3229 			break;
3230 		if (pp = page_lookup(vp, off, SE_SHARED)) {
3231 			*pl++ = pp;
3232 			off += PAGESIZE;
3233 			addr += PAGESIZE;
3234 			len -= PAGESIZE;
3235 			plsz -= PAGESIZE;
3236 		} else {
3237 			err = zfs_fillpage(vp, off, seg, addr, pl, plsz, rw);
3238 			if (err)
3239 				goto out;
3240 			/*
3241 			 * klustering may have changed our region
3242 			 * to be block aligned.
3243 			 */
3244 			if (((pp = *pl) != 0) && (off != pp->p_offset)) {
3245 				int delta = off - pp->p_offset;
3246 				len += delta;
3247 				off -= delta;
3248 				addr -= delta;
3249 			}
3250 			while (*pl) {
3251 				pl++;
3252 				off += PAGESIZE;
3253 				addr += PAGESIZE;
3254 				plsz -= PAGESIZE;
3255 				if (len > PAGESIZE)
3256 					len -= PAGESIZE;
3257 				else
3258 					len = 0;
3259 			}
3260 		}
3261 	}
3262 
3263 	/*
3264 	 * Fill out the page array with any pages already in the cache.
3265 	 */
3266 	while (plsz > 0) {
3267 		pp = page_lookup_nowait(vp, off, SE_SHARED);
3268 		if (pp == NULL)
3269 			break;
3270 		*pl++ = pp;
3271 		off += PAGESIZE;
3272 		plsz -= PAGESIZE;
3273 	}
3274 
3275 	ZFS_ACCESSTIME_STAMP(zfsvfs, zp);
3276 out:
3277 	/*
3278 	 * We can't grab the range lock for the page as reader which would
3279 	 * stop truncation as this leads to deadlock. So we need to recheck
3280 	 * the file size.
3281 	 */
3282 	if (orig_off >= zp->z_phys->zp_size)
3283 		err = EFAULT;
3284 	if (err) {
3285 		/*
3286 		 * Release any pages we have previously locked.
3287 		 */
3288 		while (pl > pl0)
3289 			page_unlock(*--pl);
3290 	}
3291 
3292 	*pl = NULL;
3293 
3294 	if (need_unlock)
3295 		rw_exit(&zp->z_map_lock);
3296 
3297 	ZFS_EXIT(zfsvfs);
3298 	return (err);
3299 }
3300 
3301 /*
3302  * Request a memory map for a section of a file.  This code interacts
3303  * with common code and the VM system as follows:
3304  *
3305  *	common code calls mmap(), which ends up in smmap_common()
3306  *
3307  *	this calls VOP_MAP(), which takes you into (say) zfs
3308  *
3309  *	zfs_map() calls as_map(), passing segvn_create() as the callback
3310  *
3311  *	segvn_create() creates the new segment and calls VOP_ADDMAP()
3312  *
3313  *	zfs_addmap() updates z_mapcnt
3314  */
3315 static int
3316 zfs_map(vnode_t *vp, offset_t off, struct as *as, caddr_t *addrp,
3317     size_t len, uchar_t prot, uchar_t maxprot, uint_t flags, cred_t *cr)
3318 {
3319 	znode_t *zp = VTOZ(vp);
3320 	zfsvfs_t *zfsvfs = zp->z_zfsvfs;
3321 	segvn_crargs_t	vn_a;
3322 	int		error;
3323 
3324 	ZFS_ENTER(zfsvfs);
3325 
3326 	if (vp->v_flag & VNOMAP) {
3327 		ZFS_EXIT(zfsvfs);
3328 		return (ENOSYS);
3329 	}
3330 
3331 	if (off < 0 || len > MAXOFFSET_T - off) {
3332 		ZFS_EXIT(zfsvfs);
3333 		return (ENXIO);
3334 	}
3335 
3336 	if (vp->v_type != VREG) {
3337 		ZFS_EXIT(zfsvfs);
3338 		return (ENODEV);
3339 	}
3340 
3341 	/*
3342 	 * If file is locked, disallow mapping.
3343 	 */
3344 	if (MANDMODE((mode_t)zp->z_phys->zp_mode) && vn_has_flocks(vp)) {
3345 		ZFS_EXIT(zfsvfs);
3346 		return (EAGAIN);
3347 	}
3348 
3349 	as_rangelock(as);
3350 	if ((flags & MAP_FIXED) == 0) {
3351 		map_addr(addrp, len, off, 1, flags);
3352 		if (*addrp == NULL) {
3353 			as_rangeunlock(as);
3354 			ZFS_EXIT(zfsvfs);
3355 			return (ENOMEM);
3356 		}
3357 	} else {
3358 		/*
3359 		 * User specified address - blow away any previous mappings
3360 		 */
3361 		(void) as_unmap(as, *addrp, len);
3362 	}
3363 
3364 	vn_a.vp = vp;
3365 	vn_a.offset = (u_offset_t)off;
3366 	vn_a.type = flags & MAP_TYPE;
3367 	vn_a.prot = prot;
3368 	vn_a.maxprot = maxprot;
3369 	vn_a.cred = cr;
3370 	vn_a.amp = NULL;
3371 	vn_a.flags = flags & ~MAP_TYPE;
3372 	vn_a.szc = 0;
3373 	vn_a.lgrp_mem_policy_flags = 0;
3374 
3375 	error = as_map(as, *addrp, len, segvn_create, &vn_a);
3376 
3377 	as_rangeunlock(as);
3378 	ZFS_EXIT(zfsvfs);
3379 	return (error);
3380 }
3381 
3382 /* ARGSUSED */
3383 static int
3384 zfs_addmap(vnode_t *vp, offset_t off, struct as *as, caddr_t addr,
3385     size_t len, uchar_t prot, uchar_t maxprot, uint_t flags, cred_t *cr)
3386 {
3387 	uint64_t pages = btopr(len);
3388 
3389 	atomic_add_64(&VTOZ(vp)->z_mapcnt, pages);
3390 	return (0);
3391 }
3392 
3393 /*
3394  * The reason we push dirty pages as part of zfs_delmap() is so that we get a
3395  * more accurate mtime for the associated file.  Since we don't have a way of
3396  * detecting when the data was actually modified, we have to resort to
3397  * heuristics.  If an explicit msync() is done, then we mark the mtime when the
3398  * last page is pushed.  The problem occurs when the msync() call is omitted,
3399  * which by far the most common case:
3400  *
3401  * 	open()
3402  * 	mmap()
3403  * 	<modify memory>
3404  * 	munmap()
3405  * 	close()
3406  * 	<time lapse>
3407  * 	putpage() via fsflush
3408  *
3409  * If we wait until fsflush to come along, we can have a modification time that
3410  * is some arbitrary point in the future.  In order to prevent this in the
3411  * common case, we flush pages whenever a (MAP_SHARED, PROT_WRITE) mapping is
3412  * torn down.
3413  */
3414 /* ARGSUSED */
3415 static int
3416 zfs_delmap(vnode_t *vp, offset_t off, struct as *as, caddr_t addr,
3417     size_t len, uint_t prot, uint_t maxprot, uint_t flags, cred_t *cr)
3418 {
3419 	uint64_t pages = btopr(len);
3420 
3421 	ASSERT3U(VTOZ(vp)->z_mapcnt, >=, pages);
3422 	atomic_add_64(&VTOZ(vp)->z_mapcnt, -pages);
3423 
3424 	if ((flags & MAP_SHARED) && (prot & PROT_WRITE) &&
3425 	    vn_has_cached_data(vp))
3426 		(void) VOP_PUTPAGE(vp, off, len, B_ASYNC, cr);
3427 
3428 	return (0);
3429 }
3430 
3431 /*
3432  * Free or allocate space in a file.  Currently, this function only
3433  * supports the `F_FREESP' command.  However, this command is somewhat
3434  * misnamed, as its functionality includes the ability to allocate as
3435  * well as free space.
3436  *
3437  *	IN:	vp	- vnode of file to free data in.
3438  *		cmd	- action to take (only F_FREESP supported).
3439  *		bfp	- section of file to free/alloc.
3440  *		flag	- current file open mode flags.
3441  *		offset	- current file offset.
3442  *		cr	- credentials of caller [UNUSED].
3443  *
3444  *	RETURN:	0 if success
3445  *		error code if failure
3446  *
3447  * Timestamps:
3448  *	vp - ctime|mtime updated
3449  */
3450 /* ARGSUSED */
3451 static int
3452 zfs_space(vnode_t *vp, int cmd, flock64_t *bfp, int flag,
3453     offset_t offset, cred_t *cr, caller_context_t *ct)
3454 {
3455 	znode_t		*zp = VTOZ(vp);
3456 	zfsvfs_t	*zfsvfs = zp->z_zfsvfs;
3457 	uint64_t	off, len;
3458 	int		error;
3459 
3460 	ZFS_ENTER(zfsvfs);
3461 
3462 top:
3463 	if (cmd != F_FREESP) {
3464 		ZFS_EXIT(zfsvfs);
3465 		return (EINVAL);
3466 	}
3467 
3468 	if (error = convoff(vp, bfp, 0, offset)) {
3469 		ZFS_EXIT(zfsvfs);
3470 		return (error);
3471 	}
3472 
3473 	if (bfp->l_len < 0) {
3474 		ZFS_EXIT(zfsvfs);
3475 		return (EINVAL);
3476 	}
3477 
3478 	off = bfp->l_start;
3479 	len = bfp->l_len; /* 0 means from off to end of file */
3480 
3481 	do {
3482 		error = zfs_freesp(zp, off, len, flag, TRUE);
3483 		/* NB: we already did dmu_tx_wait() if necessary */
3484 	} while (error == ERESTART && zfsvfs->z_assign == TXG_NOWAIT);
3485 
3486 	ZFS_EXIT(zfsvfs);
3487 	return (error);
3488 }
3489 
3490 static int
3491 zfs_fid(vnode_t *vp, fid_t *fidp)
3492 {
3493 	znode_t		*zp = VTOZ(vp);
3494 	zfsvfs_t	*zfsvfs = zp->z_zfsvfs;
3495 	uint32_t	gen = (uint32_t)zp->z_phys->zp_gen;
3496 	uint64_t	object = zp->z_id;
3497 	zfid_short_t	*zfid;
3498 	int		size, i;
3499 
3500 	ZFS_ENTER(zfsvfs);
3501 
3502 	size = (zfsvfs->z_parent != zfsvfs) ? LONG_FID_LEN : SHORT_FID_LEN;
3503 	if (fidp->fid_len < size) {
3504 		fidp->fid_len = size;
3505 		ZFS_EXIT(zfsvfs);
3506 		return (ENOSPC);
3507 	}
3508 
3509 	zfid = (zfid_short_t *)fidp;
3510 
3511 	zfid->zf_len = size;
3512 
3513 	for (i = 0; i < sizeof (zfid->zf_object); i++)
3514 		zfid->zf_object[i] = (uint8_t)(object >> (8 * i));
3515 
3516 	/* Must have a non-zero generation number to distinguish from .zfs */
3517 	if (gen == 0)
3518 		gen = 1;
3519 	for (i = 0; i < sizeof (zfid->zf_gen); i++)
3520 		zfid->zf_gen[i] = (uint8_t)(gen >> (8 * i));
3521 
3522 	if (size == LONG_FID_LEN) {
3523 		uint64_t	objsetid = dmu_objset_id(zfsvfs->z_os);
3524 		zfid_long_t	*zlfid;
3525 
3526 		zlfid = (zfid_long_t *)fidp;
3527 
3528 		for (i = 0; i < sizeof (zlfid->zf_setid); i++)
3529 			zlfid->zf_setid[i] = (uint8_t)(objsetid >> (8 * i));
3530 
3531 		/* XXX - this should be the generation number for the objset */
3532 		for (i = 0; i < sizeof (zlfid->zf_setgen); i++)
3533 			zlfid->zf_setgen[i] = 0;
3534 	}
3535 
3536 	ZFS_EXIT(zfsvfs);
3537 	return (0);
3538 }
3539 
3540 static int
3541 zfs_pathconf(vnode_t *vp, int cmd, ulong_t *valp, cred_t *cr)
3542 {
3543 	znode_t		*zp, *xzp;
3544 	zfsvfs_t	*zfsvfs;
3545 	zfs_dirlock_t	*dl;
3546 	int		error;
3547 
3548 	switch (cmd) {
3549 	case _PC_LINK_MAX:
3550 		*valp = ULONG_MAX;
3551 		return (0);
3552 
3553 	case _PC_FILESIZEBITS:
3554 		*valp = 64;
3555 		return (0);
3556 
3557 	case _PC_XATTR_EXISTS:
3558 		zp = VTOZ(vp);
3559 		zfsvfs = zp->z_zfsvfs;
3560 		ZFS_ENTER(zfsvfs);
3561 		*valp = 0;
3562 		error = zfs_dirent_lock(&dl, zp, "", &xzp,
3563 		    ZXATTR | ZEXISTS | ZSHARED);
3564 		if (error == 0) {
3565 			zfs_dirent_unlock(dl);
3566 			if (!zfs_dirempty(xzp))
3567 				*valp = 1;
3568 			VN_RELE(ZTOV(xzp));
3569 		} else if (error == ENOENT) {
3570 			/*
3571 			 * If there aren't extended attributes, it's the
3572 			 * same as having zero of them.
3573 			 */
3574 			error = 0;
3575 		}
3576 		ZFS_EXIT(zfsvfs);
3577 		return (error);
3578 
3579 	case _PC_ACL_ENABLED:
3580 		*valp = _ACL_ACE_ENABLED;
3581 		return (0);
3582 
3583 	case _PC_MIN_HOLE_SIZE:
3584 		*valp = (ulong_t)SPA_MINBLOCKSIZE;
3585 		return (0);
3586 
3587 	default:
3588 		return (fs_pathconf(vp, cmd, valp, cr));
3589 	}
3590 }
3591 
3592 /*ARGSUSED*/
3593 static int
3594 zfs_getsecattr(vnode_t *vp, vsecattr_t *vsecp, int flag, cred_t *cr)
3595 {
3596 	znode_t *zp = VTOZ(vp);
3597 	zfsvfs_t *zfsvfs = zp->z_zfsvfs;
3598 	int error;
3599 
3600 	ZFS_ENTER(zfsvfs);
3601 	error = zfs_getacl(zp, vsecp, cr);
3602 	ZFS_EXIT(zfsvfs);
3603 
3604 	return (error);
3605 }
3606 
3607 /*ARGSUSED*/
3608 static int
3609 zfs_setsecattr(vnode_t *vp, vsecattr_t *vsecp, int flag, cred_t *cr)
3610 {
3611 	znode_t *zp = VTOZ(vp);
3612 	zfsvfs_t *zfsvfs = zp->z_zfsvfs;
3613 	int error;
3614 
3615 	ZFS_ENTER(zfsvfs);
3616 	error = zfs_setacl(zp, vsecp, cr);
3617 	ZFS_EXIT(zfsvfs);
3618 	return (error);
3619 }
3620 
3621 /*
3622  * Predeclare these here so that the compiler assumes that
3623  * this is an "old style" function declaration that does
3624  * not include arguments => we won't get type mismatch errors
3625  * in the initializations that follow.
3626  */
3627 static int zfs_inval();
3628 static int zfs_isdir();
3629 
3630 static int
3631 zfs_inval()
3632 {
3633 	return (EINVAL);
3634 }
3635 
3636 static int
3637 zfs_isdir()
3638 {
3639 	return (EISDIR);
3640 }
3641 /*
3642  * Directory vnode operations template
3643  */
3644 vnodeops_t *zfs_dvnodeops;
3645 const fs_operation_def_t zfs_dvnodeops_template[] = {
3646 	VOPNAME_OPEN, zfs_open,
3647 	VOPNAME_CLOSE, zfs_close,
3648 	VOPNAME_READ, zfs_isdir,
3649 	VOPNAME_WRITE, zfs_isdir,
3650 	VOPNAME_IOCTL, zfs_ioctl,
3651 	VOPNAME_GETATTR, zfs_getattr,
3652 	VOPNAME_SETATTR, zfs_setattr,
3653 	VOPNAME_ACCESS, zfs_access,
3654 	VOPNAME_LOOKUP, zfs_lookup,
3655 	VOPNAME_CREATE, zfs_create,
3656 	VOPNAME_REMOVE, zfs_remove,
3657 	VOPNAME_LINK, zfs_link,
3658 	VOPNAME_RENAME, zfs_rename,
3659 	VOPNAME_MKDIR, zfs_mkdir,
3660 	VOPNAME_RMDIR, zfs_rmdir,
3661 	VOPNAME_READDIR, zfs_readdir,
3662 	VOPNAME_SYMLINK, zfs_symlink,
3663 	VOPNAME_FSYNC, zfs_fsync,
3664 	VOPNAME_INACTIVE, (fs_generic_func_p) zfs_inactive,
3665 	VOPNAME_FID, zfs_fid,
3666 	VOPNAME_SEEK, zfs_seek,
3667 	VOPNAME_PATHCONF, zfs_pathconf,
3668 	VOPNAME_GETSECATTR, zfs_getsecattr,
3669 	VOPNAME_SETSECATTR, zfs_setsecattr,
3670 	NULL, NULL
3671 };
3672 
3673 /*
3674  * Regular file vnode operations template
3675  */
3676 vnodeops_t *zfs_fvnodeops;
3677 const fs_operation_def_t zfs_fvnodeops_template[] = {
3678 	VOPNAME_OPEN, zfs_open,
3679 	VOPNAME_CLOSE, zfs_close,
3680 	VOPNAME_READ, zfs_read,
3681 	VOPNAME_WRITE, zfs_write,
3682 	VOPNAME_IOCTL, zfs_ioctl,
3683 	VOPNAME_GETATTR, zfs_getattr,
3684 	VOPNAME_SETATTR, zfs_setattr,
3685 	VOPNAME_ACCESS, zfs_access,
3686 	VOPNAME_LOOKUP, zfs_lookup,
3687 	VOPNAME_RENAME, zfs_rename,
3688 	VOPNAME_FSYNC, zfs_fsync,
3689 	VOPNAME_INACTIVE, (fs_generic_func_p)zfs_inactive,
3690 	VOPNAME_FID, zfs_fid,
3691 	VOPNAME_SEEK, zfs_seek,
3692 	VOPNAME_FRLOCK, zfs_frlock,
3693 	VOPNAME_SPACE, zfs_space,
3694 	VOPNAME_GETPAGE, zfs_getpage,
3695 	VOPNAME_PUTPAGE, zfs_putpage,
3696 	VOPNAME_MAP, (fs_generic_func_p) zfs_map,
3697 	VOPNAME_ADDMAP, (fs_generic_func_p) zfs_addmap,
3698 	VOPNAME_DELMAP, zfs_delmap,
3699 	VOPNAME_PATHCONF, zfs_pathconf,
3700 	VOPNAME_GETSECATTR, zfs_getsecattr,
3701 	VOPNAME_SETSECATTR, zfs_setsecattr,
3702 	VOPNAME_VNEVENT, fs_vnevent_support,
3703 	NULL, NULL
3704 };
3705 
3706 /*
3707  * Symbolic link vnode operations template
3708  */
3709 vnodeops_t *zfs_symvnodeops;
3710 const fs_operation_def_t zfs_symvnodeops_template[] = {
3711 	VOPNAME_GETATTR, zfs_getattr,
3712 	VOPNAME_SETATTR, zfs_setattr,
3713 	VOPNAME_ACCESS, zfs_access,
3714 	VOPNAME_RENAME, zfs_rename,
3715 	VOPNAME_READLINK, zfs_readlink,
3716 	VOPNAME_INACTIVE, (fs_generic_func_p) zfs_inactive,
3717 	VOPNAME_FID, zfs_fid,
3718 	VOPNAME_PATHCONF, zfs_pathconf,
3719 	VOPNAME_VNEVENT, fs_vnevent_support,
3720 	NULL, NULL
3721 };
3722 
3723 /*
3724  * Extended attribute directory vnode operations template
3725  *	This template is identical to the directory vnodes
3726  *	operation template except for restricted operations:
3727  *		VOP_MKDIR()
3728  *		VOP_SYMLINK()
3729  * Note that there are other restrictions embedded in:
3730  *	zfs_create()	- restrict type to VREG
3731  *	zfs_link()	- no links into/out of attribute space
3732  *	zfs_rename()	- no moves into/out of attribute space
3733  */
3734 vnodeops_t *zfs_xdvnodeops;
3735 const fs_operation_def_t zfs_xdvnodeops_template[] = {
3736 	VOPNAME_OPEN, zfs_open,
3737 	VOPNAME_CLOSE, zfs_close,
3738 	VOPNAME_IOCTL, zfs_ioctl,
3739 	VOPNAME_GETATTR, zfs_getattr,
3740 	VOPNAME_SETATTR, zfs_setattr,
3741 	VOPNAME_ACCESS, zfs_access,
3742 	VOPNAME_LOOKUP, zfs_lookup,
3743 	VOPNAME_CREATE, zfs_create,
3744 	VOPNAME_REMOVE, zfs_remove,
3745 	VOPNAME_LINK, zfs_link,
3746 	VOPNAME_RENAME, zfs_rename,
3747 	VOPNAME_MKDIR, zfs_inval,
3748 	VOPNAME_RMDIR, zfs_rmdir,
3749 	VOPNAME_READDIR, zfs_readdir,
3750 	VOPNAME_SYMLINK, zfs_inval,
3751 	VOPNAME_FSYNC, zfs_fsync,
3752 	VOPNAME_INACTIVE, (fs_generic_func_p) zfs_inactive,
3753 	VOPNAME_FID, zfs_fid,
3754 	VOPNAME_SEEK, zfs_seek,
3755 	VOPNAME_PATHCONF, zfs_pathconf,
3756 	VOPNAME_GETSECATTR, zfs_getsecattr,
3757 	VOPNAME_SETSECATTR, zfs_setsecattr,
3758 	VOPNAME_VNEVENT, fs_vnevent_support,
3759 	NULL, NULL
3760 };
3761 
3762 /*
3763  * Error vnode operations template
3764  */
3765 vnodeops_t *zfs_evnodeops;
3766 const fs_operation_def_t zfs_evnodeops_template[] = {
3767 	VOPNAME_INACTIVE, (fs_generic_func_p) zfs_inactive,
3768 	VOPNAME_PATHCONF, zfs_pathconf,
3769 	NULL, NULL
3770 };
3771