xref: /titanic_50/usr/src/uts/common/fs/zfs/zfs_vnops.c (revision 441d80aa4f613b6298fc8bd3151f4be02dbf84fc)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 /*
22  * Copyright 2006 Sun Microsystems, Inc.  All rights reserved.
23  * Use is subject to license terms.
24  */
25 
26 #pragma ident	"%Z%%M%	%I%	%E% SMI"
27 
28 #include <sys/types.h>
29 #include <sys/param.h>
30 #include <sys/time.h>
31 #include <sys/systm.h>
32 #include <sys/sysmacros.h>
33 #include <sys/resource.h>
34 #include <sys/vfs.h>
35 #include <sys/vnode.h>
36 #include <sys/file.h>
37 #include <sys/stat.h>
38 #include <sys/kmem.h>
39 #include <sys/taskq.h>
40 #include <sys/uio.h>
41 #include <sys/vmsystm.h>
42 #include <sys/atomic.h>
43 #include <vm/seg_vn.h>
44 #include <vm/pvn.h>
45 #include <vm/as.h>
46 #include <sys/mman.h>
47 #include <sys/pathname.h>
48 #include <sys/cmn_err.h>
49 #include <sys/errno.h>
50 #include <sys/unistd.h>
51 #include <sys/zfs_vfsops.h>
52 #include <sys/zfs_dir.h>
53 #include <sys/zfs_acl.h>
54 #include <sys/zfs_ioctl.h>
55 #include <sys/fs/zfs.h>
56 #include <sys/dmu.h>
57 #include <sys/spa.h>
58 #include <sys/txg.h>
59 #include <sys/refcount.h>  /* temporary for debugging purposes */
60 #include <sys/dbuf.h>
61 #include <sys/zap.h>
62 #include <sys/dirent.h>
63 #include <sys/policy.h>
64 #include <sys/sunddi.h>
65 #include <sys/filio.h>
66 #include "fs/fs_subr.h"
67 #include <sys/zfs_ctldir.h>
68 #include <sys/dnlc.h>
69 
70 /*
71  * Programming rules.
72  *
73  * Each vnode op performs some logical unit of work.  To do this, the ZPL must
74  * properly lock its in-core state, create a DMU transaction, do the work,
75  * record this work in the intent log (ZIL), commit the DMU transaction,
76  * and wait the the intent log to commit if it's is a synchronous operation.
77  * Morover, the vnode ops must work in both normal and log replay context.
78  * The ordering of events is important to avoid deadlocks and references
79  * to freed memory.  The example below illustrates the following Big Rules:
80  *
81  *  (1) A check must be made in each zfs thread for a mounted file system.
82  *	This is done avoiding races using ZFS_ENTER(zfsvfs).
83  *	A ZFS_EXIT(zfsvfs) is needed before all returns.
84  *
85  *  (2)	VN_RELE() should always be the last thing except for zil_commit()
86  *	and ZFS_EXIT(). This is for 3 reasons:
87  *	First, if it's the last reference, the vnode/znode
88  *	can be freed, so the zp may point to freed memory.  Second, the last
89  *	reference will call zfs_zinactive(), which may induce a lot of work --
90  *	pushing cached pages (which requires z_grow_lock) and syncing out
91  *	cached atime changes.  Third, zfs_zinactive() may require a new tx,
92  *	which could deadlock the system if you were already holding one.
93  *
94  *  (3)	Always pass zfsvfs->z_assign as the second argument to dmu_tx_assign().
95  *	In normal operation, this will be TXG_NOWAIT.  During ZIL replay,
96  *	it will be a specific txg.  Either way, dmu_tx_assign() never blocks.
97  *	This is critical because we don't want to block while holding locks.
98  *	Note, in particular, that if a lock is sometimes acquired before
99  *	the tx assigns, and sometimes after (e.g. z_lock), then failing to
100  *	use a non-blocking assign can deadlock the system.  The scenario:
101  *
102  *	Thread A has grabbed a lock before calling dmu_tx_assign().
103  *	Thread B is in an already-assigned tx, and blocks for this lock.
104  *	Thread A calls dmu_tx_assign(TXG_WAIT) and blocks in txg_wait_open()
105  *	forever, because the previous txg can't quiesce until B's tx commits.
106  *
107  *	If dmu_tx_assign() returns ERESTART and zfsvfs->z_assign is TXG_NOWAIT,
108  *	then drop all locks, call txg_wait_open(), and try again.
109  *
110  *  (4)	If the operation succeeded, generate the intent log entry for it
111  *	before dropping locks.  This ensures that the ordering of events
112  *	in the intent log matches the order in which they actually occurred.
113  *
114  *  (5)	At the end of each vnode op, the DMU tx must always commit,
115  *	regardless of whether there were any errors.
116  *
117  *  (6)	After dropping all locks, invoke zil_commit(zilog, seq, ioflag)
118  *	to ensure that synchronous semantics are provided when necessary.
119  *
120  * In general, this is how things should be ordered in each vnode op:
121  *
122  *	ZFS_ENTER(zfsvfs);		// exit if unmounted
123  * top:
124  *	zfs_dirent_lock(&dl, ...)	// lock directory entry (may VN_HOLD())
125  *	rw_enter(...);			// grab any other locks you need
126  *	tx = dmu_tx_create(...);	// get DMU tx
127  *	dmu_tx_hold_*();		// hold each object you might modify
128  *	error = dmu_tx_assign(tx, zfsvfs->z_assign);	// try to assign
129  *	if (error) {
130  *		dmu_tx_abort(tx);	// abort DMU tx
131  *		rw_exit(...);		// drop locks
132  *		zfs_dirent_unlock(dl);	// unlock directory entry
133  *		VN_RELE(...);		// release held vnodes
134  *		if (error == ERESTART && zfsvfs->z_assign == TXG_NOWAIT) {
135  *			txg_wait_open(dmu_objset_pool(os), 0);
136  *			goto top;
137  *		}
138  *		ZFS_EXIT(zfsvfs);	// finished in zfs
139  *		return (error);		// really out of space
140  *	}
141  *	error = do_real_work();		// do whatever this VOP does
142  *	if (error == 0)
143  *		seq = zfs_log_*(...);	// on success, make ZIL entry
144  *	dmu_tx_commit(tx);		// commit DMU tx -- error or not
145  *	rw_exit(...);			// drop locks
146  *	zfs_dirent_unlock(dl);		// unlock directory entry
147  *	VN_RELE(...);			// release held vnodes
148  *	zil_commit(zilog, seq, ioflag);	// synchronous when necessary
149  *	ZFS_EXIT(zfsvfs);		// finished in zfs
150  *	return (error);			// done, report error
151  */
152 
153 /* ARGSUSED */
154 static int
155 zfs_open(vnode_t **vpp, int flag, cred_t *cr)
156 {
157 	return (0);
158 }
159 
160 /* ARGSUSED */
161 static int
162 zfs_close(vnode_t *vp, int flag, int count, offset_t offset, cred_t *cr)
163 {
164 	/*
165 	 * Clean up any locks held by this process on the vp.
166 	 */
167 	cleanlocks(vp, ddi_get_pid(), 0);
168 	cleanshares(vp, ddi_get_pid());
169 
170 	return (0);
171 }
172 
173 /*
174  * Lseek support for finding holes (cmd == _FIO_SEEK_HOLE) and
175  * data (cmd == _FIO_SEEK_DATA). "off" is an in/out parameter.
176  */
177 static int
178 zfs_holey(vnode_t *vp, int cmd, offset_t *off)
179 {
180 	znode_t	*zp = VTOZ(vp);
181 	uint64_t noff = (uint64_t)*off; /* new offset */
182 	uint64_t file_sz;
183 	int error;
184 	boolean_t hole;
185 
186 	rw_enter(&zp->z_grow_lock, RW_READER);
187 	file_sz = zp->z_phys->zp_size;
188 	if (noff >= file_sz)  {
189 		rw_exit(&zp->z_grow_lock);
190 		return (ENXIO);
191 	}
192 
193 	if (cmd == _FIO_SEEK_HOLE)
194 		hole = B_TRUE;
195 	else
196 		hole = B_FALSE;
197 
198 	error = dmu_offset_next(zp->z_zfsvfs->z_os, zp->z_id, hole, &noff);
199 	rw_exit(&zp->z_grow_lock);
200 
201 	/* end of file? */
202 	if ((error == ESRCH) || (noff > file_sz)) {
203 		/*
204 		 * Handle the virtual hole at the end of file.
205 		 */
206 		if (hole) {
207 			*off = file_sz;
208 			return (0);
209 		}
210 		return (ENXIO);
211 	}
212 
213 	if (noff < *off)
214 		return (error);
215 	*off = noff;
216 	return (error);
217 }
218 
219 /* ARGSUSED */
220 static int
221 zfs_ioctl(vnode_t *vp, int com, intptr_t data, int flag, cred_t *cred,
222     int *rvalp)
223 {
224 	offset_t off;
225 	int error;
226 	zfsvfs_t *zfsvfs;
227 
228 	switch (com) {
229 	    case _FIOFFS:
230 		return (zfs_sync(vp->v_vfsp, 0, cred));
231 
232 	    case _FIO_SEEK_DATA:
233 	    case _FIO_SEEK_HOLE:
234 		if (ddi_copyin((void *)data, &off, sizeof (off), flag))
235 			return (EFAULT);
236 
237 		zfsvfs = VTOZ(vp)->z_zfsvfs;
238 		ZFS_ENTER(zfsvfs);
239 
240 		/* offset parameter is in/out */
241 		error = zfs_holey(vp, com, &off);
242 		ZFS_EXIT(zfsvfs);
243 		if (error)
244 			return (error);
245 		if (ddi_copyout(&off, (void *)data, sizeof (off), flag))
246 			return (EFAULT);
247 		return (0);
248 	}
249 	return (ENOTTY);
250 }
251 
252 /*
253  * When a file is memory mapped, we must keep the IO data synchronized
254  * between the DMU cache and the memory mapped pages.  What this means:
255  *
256  * On Write:	If we find a memory mapped page, we write to *both*
257  *		the page and the dmu buffer.
258  *
259  * NOTE: We will always "break up" the IO into PAGESIZE uiomoves when
260  *	the file is memory mapped.
261  */
262 static int
263 mappedwrite(vnode_t *vp, uint64_t woff, int nbytes, uio_t *uio, dmu_tx_t *tx)
264 {
265 	znode_t	*zp = VTOZ(vp);
266 	zfsvfs_t *zfsvfs = zp->z_zfsvfs;
267 	int64_t	start, off;
268 	int len = nbytes;
269 	int error = 0;
270 
271 	start = uio->uio_loffset;
272 	off = start & PAGEOFFSET;
273 	for (start &= PAGEMASK; len > 0; start += PAGESIZE) {
274 		page_t *pp;
275 		uint64_t bytes = MIN(PAGESIZE - off, len);
276 
277 		/*
278 		 * We don't want a new page to "appear" in the middle of
279 		 * the file update (because it may not get the write
280 		 * update data), so we grab a lock to block
281 		 * zfs_getpage().
282 		 */
283 		rw_enter(&zp->z_map_lock, RW_WRITER);
284 		if (pp = page_lookup(vp, start, SE_SHARED)) {
285 			caddr_t va;
286 
287 			rw_exit(&zp->z_map_lock);
288 			va = ppmapin(pp, PROT_READ | PROT_WRITE, (caddr_t)-1L);
289 			error = uiomove(va+off, bytes, UIO_WRITE, uio);
290 			if (error == 0) {
291 				dmu_write(zfsvfs->z_os, zp->z_id,
292 				    woff, bytes, va+off, tx);
293 			}
294 			ppmapout(va);
295 			page_unlock(pp);
296 		} else {
297 			error = dmu_write_uio(zfsvfs->z_os, zp->z_id,
298 			    woff, bytes, uio, tx);
299 			rw_exit(&zp->z_map_lock);
300 		}
301 		len -= bytes;
302 		woff += bytes;
303 		off = 0;
304 		if (error)
305 			break;
306 	}
307 	return (error);
308 }
309 
310 /*
311  * When a file is memory mapped, we must keep the IO data synchronized
312  * between the DMU cache and the memory mapped pages.  What this means:
313  *
314  * On Read:	We "read" preferentially from memory mapped pages,
315  *		else we default from the dmu buffer.
316  *
317  * NOTE: We will always "break up" the IO into PAGESIZE uiomoves when
318  *	the file is memory mapped.
319  */
320 static int
321 mappedread(vnode_t *vp, char *addr, int nbytes, uio_t *uio)
322 {
323 	int64_t	start, off, bytes;
324 	int len = nbytes;
325 	int error = 0;
326 
327 	start = uio->uio_loffset;
328 	off = start & PAGEOFFSET;
329 	for (start &= PAGEMASK; len > 0; start += PAGESIZE) {
330 		page_t *pp;
331 
332 		bytes = MIN(PAGESIZE - off, len);
333 		if (pp = page_lookup(vp, start, SE_SHARED)) {
334 			caddr_t va;
335 
336 			va = ppmapin(pp, PROT_READ | PROT_WRITE, (caddr_t)-1L);
337 			error = uiomove(va + off, bytes, UIO_READ, uio);
338 			ppmapout(va);
339 			page_unlock(pp);
340 		} else {
341 			/* XXX use dmu_read here? */
342 			error = uiomove(addr, bytes, UIO_READ, uio);
343 		}
344 		len -= bytes;
345 		addr += bytes;
346 		off = 0;
347 		if (error)
348 			break;
349 	}
350 	return (error);
351 }
352 
353 uint_t zfs_read_chunk_size = 1024 * 1024; /* Tunable */
354 
355 /*
356  * Read bytes from specified file into supplied buffer.
357  *
358  *	IN:	vp	- vnode of file to be read from.
359  *		uio	- structure supplying read location, range info,
360  *			  and return buffer.
361  *		ioflag	- SYNC flags; used to provide FRSYNC semantics.
362  *		cr	- credentials of caller.
363  *
364  *	OUT:	uio	- updated offset and range, buffer filled.
365  *
366  *	RETURN:	0 if success
367  *		error code if failure
368  *
369  * Side Effects:
370  *	vp - atime updated if byte count > 0
371  */
372 /* ARGSUSED */
373 static int
374 zfs_read(vnode_t *vp, uio_t *uio, int ioflag, cred_t *cr, caller_context_t *ct)
375 {
376 	znode_t		*zp = VTOZ(vp);
377 	zfsvfs_t	*zfsvfs = zp->z_zfsvfs;
378 	uint64_t	delta;
379 	ssize_t		n, size, cnt, ndone;
380 	int		error, i, numbufs;
381 	dmu_buf_t	*dbp, **dbpp;
382 
383 	ZFS_ENTER(zfsvfs);
384 
385 	/*
386 	 * Validate file offset
387 	 */
388 	if (uio->uio_loffset < (offset_t)0) {
389 		ZFS_EXIT(zfsvfs);
390 		return (EINVAL);
391 	}
392 
393 	/*
394 	 * Fasttrack empty reads
395 	 */
396 	if (uio->uio_resid == 0) {
397 		ZFS_EXIT(zfsvfs);
398 		return (0);
399 	}
400 
401 	/*
402 	 * Check for region locks
403 	 */
404 	if (MANDMODE((mode_t)zp->z_phys->zp_mode)) {
405 		if (error = chklock(vp, FREAD,
406 		    uio->uio_loffset, uio->uio_resid, uio->uio_fmode, ct)) {
407 			ZFS_EXIT(zfsvfs);
408 			return (error);
409 		}
410 	}
411 
412 	/*
413 	 * If we're in FRSYNC mode, sync out this znode before reading it.
414 	 */
415 	zil_commit(zfsvfs->z_log, zp->z_last_itx, ioflag & FRSYNC);
416 
417 	/*
418 	 * Make sure nobody restructures the file (changes block size)
419 	 * in the middle of the read.
420 	 */
421 	rw_enter(&zp->z_grow_lock, RW_READER);
422 	/*
423 	 * If we are reading past end-of-file we can skip
424 	 * to the end; but we might still need to set atime.
425 	 */
426 	if (uio->uio_loffset >= zp->z_phys->zp_size) {
427 		cnt = 0;
428 		error = 0;
429 		goto out;
430 	}
431 
432 	cnt = MIN(uio->uio_resid, zp->z_phys->zp_size - uio->uio_loffset);
433 
434 	for (ndone = 0; ndone < cnt; ndone += zfs_read_chunk_size) {
435 		ASSERT(uio->uio_loffset < zp->z_phys->zp_size);
436 		n = MIN(zfs_read_chunk_size,
437 		    zp->z_phys->zp_size - uio->uio_loffset);
438 		n = MIN(n, cnt);
439 		dbpp = dmu_buf_hold_array(zfsvfs->z_os, zp->z_id,
440 		    uio->uio_loffset, n, &numbufs);
441 		if (error = dmu_buf_read_array_canfail(dbpp, numbufs)) {
442 			dmu_buf_rele_array(dbpp, numbufs);
443 			goto out;
444 		}
445 		/*
446 		 * Compute the adjustment to align the dmu buffers
447 		 * with the uio buffer.
448 		 */
449 		delta = uio->uio_loffset - dbpp[0]->db_offset;
450 
451 		for (i = 0; i < numbufs; i++) {
452 			if (n < 0)
453 				break;
454 			dbp = dbpp[i];
455 			size = dbp->db_size - delta;
456 			/*
457 			 * XXX -- this is correct, but may be suboptimal.
458 			 * If the pages are all clean, we don't need to
459 			 * go through mappedread().  Maybe the VMODSORT
460 			 * stuff can help us here.
461 			 */
462 			if (vn_has_cached_data(vp)) {
463 				error = mappedread(vp, (caddr_t)dbp->db_data +
464 				    delta, (n < size ? n : size), uio);
465 			} else {
466 				error = uiomove((caddr_t)dbp->db_data + delta,
467 					(n < size ? n : size), UIO_READ, uio);
468 			}
469 			if (error) {
470 				dmu_buf_rele_array(dbpp, numbufs);
471 				goto out;
472 			}
473 			n -= dbp->db_size;
474 			if (delta) {
475 				n += delta;
476 				delta = 0;
477 			}
478 		}
479 		dmu_buf_rele_array(dbpp, numbufs);
480 	}
481 out:
482 	rw_exit(&zp->z_grow_lock);
483 
484 	ZFS_ACCESSTIME_STAMP(zfsvfs, zp);
485 	ZFS_EXIT(zfsvfs);
486 	return (error);
487 }
488 
489 /*
490  * Fault in the pages of the first n bytes specified by the uio structure.
491  * 1 byte in each page is touched and the uio struct is unmodified.
492  * Any error will exit this routine as this is only a best
493  * attempt to get the pages resident. This is a copy of ufs_trans_touch().
494  */
495 static void
496 zfs_prefault_write(ssize_t n, struct uio *uio)
497 {
498 	struct iovec *iov;
499 	ulong_t cnt, incr;
500 	caddr_t p;
501 	uint8_t tmp;
502 
503 	iov = uio->uio_iov;
504 
505 	while (n) {
506 		cnt = MIN(iov->iov_len, n);
507 		if (cnt == 0) {
508 			/* empty iov entry */
509 			iov++;
510 			continue;
511 		}
512 		n -= cnt;
513 		/*
514 		 * touch each page in this segment.
515 		 */
516 		p = iov->iov_base;
517 		while (cnt) {
518 			switch (uio->uio_segflg) {
519 			case UIO_USERSPACE:
520 			case UIO_USERISPACE:
521 				if (fuword8(p, &tmp))
522 					return;
523 				break;
524 			case UIO_SYSSPACE:
525 				if (kcopy(p, &tmp, 1))
526 					return;
527 				break;
528 			}
529 			incr = MIN(cnt, PAGESIZE);
530 			p += incr;
531 			cnt -= incr;
532 		}
533 		/*
534 		 * touch the last byte in case it straddles a page.
535 		 */
536 		p--;
537 		switch (uio->uio_segflg) {
538 		case UIO_USERSPACE:
539 		case UIO_USERISPACE:
540 			if (fuword8(p, &tmp))
541 				return;
542 			break;
543 		case UIO_SYSSPACE:
544 			if (kcopy(p, &tmp, 1))
545 				return;
546 			break;
547 		}
548 		iov++;
549 	}
550 }
551 
552 /*
553  * Write the bytes to a file.
554  *
555  *	IN:	vp	- vnode of file to be written to.
556  *		uio	- structure supplying write location, range info,
557  *			  and data buffer.
558  *		ioflag	- FAPPEND flag set if in append mode.
559  *		cr	- credentials of caller.
560  *
561  *	OUT:	uio	- updated offset and range.
562  *
563  *	RETURN:	0 if success
564  *		error code if failure
565  *
566  * Timestamps:
567  *	vp - ctime|mtime updated if byte count > 0
568  *
569  * Note: zfs_write() holds z_append_lock across calls to txg_wait_open().
570  * It has to because of the semantics of FAPPEND.  The implication is that
571  * we must never grab z_append_lock while in an assigned tx.
572  */
573 /* ARGSUSED */
574 static int
575 zfs_write(vnode_t *vp, uio_t *uio, int ioflag, cred_t *cr, caller_context_t *ct)
576 {
577 	znode_t		*zp = VTOZ(vp);
578 	rlim64_t	limit = uio->uio_llimit;
579 	ssize_t		start_resid = uio->uio_resid;
580 	ssize_t		tx_bytes;
581 	uint64_t	end_size;
582 	dmu_tx_t	*tx;
583 	zfsvfs_t	*zfsvfs = zp->z_zfsvfs;
584 	zilog_t		*zilog = zfsvfs->z_log;
585 	uint64_t	seq = 0;
586 	offset_t	woff;
587 	ssize_t		n, nbytes;
588 	int		max_blksz = zfsvfs->z_max_blksz;
589 	int		need_append_lock, error;
590 	krw_t		grow_rw = RW_READER;
591 
592 	if (limit == RLIM64_INFINITY || limit > MAXOFFSET_T)
593 		limit = MAXOFFSET_T;
594 
595 	n = start_resid;
596 
597 	/*
598 	 * Fasttrack empty write
599 	 */
600 	if (n == 0)
601 		return (0);
602 
603 	ZFS_ENTER(zfsvfs);
604 
605 	/*
606 	 * Pre-fault the pages to ensure slow (eg NFS) pages don't hold up txg
607 	 */
608 	zfs_prefault_write(MIN(start_resid, SPA_MAXBLOCKSIZE), uio);
609 
610 	/*
611 	 * If in append mode, set the io offset pointer to eof.
612 	 */
613 	need_append_lock = ioflag & FAPPEND;
614 	if (need_append_lock) {
615 		rw_enter(&zp->z_append_lock, RW_WRITER);
616 		woff = uio->uio_loffset = zp->z_phys->zp_size;
617 	} else {
618 		woff = uio->uio_loffset;
619 		/*
620 		 * Validate file offset
621 		 */
622 		if (woff < 0) {
623 			ZFS_EXIT(zfsvfs);
624 			return (EINVAL);
625 		}
626 
627 		/*
628 		 * If this write could change the file length,
629 		 * we need to synchronize with "appenders".
630 		 */
631 		if (woff < limit - n && woff + n > zp->z_phys->zp_size) {
632 			need_append_lock = TRUE;
633 			rw_enter(&zp->z_append_lock, RW_READER);
634 		}
635 	}
636 
637 	if (woff >= limit) {
638 		error = EFBIG;
639 		goto no_tx_done;
640 	}
641 
642 	if ((woff + n) > limit || woff > (limit - n))
643 		n = limit - woff;
644 
645 	/*
646 	 * Check for region locks
647 	 */
648 	if (MANDMODE((mode_t)zp->z_phys->zp_mode) &&
649 	    (error = chklock(vp, FWRITE, woff, n, uio->uio_fmode, ct)) != 0)
650 		goto no_tx_done;
651 top:
652 	/*
653 	 * Make sure nobody restructures the file (changes block size)
654 	 * in the middle of the write.
655 	 */
656 	rw_enter(&zp->z_grow_lock, grow_rw);
657 
658 	end_size = MAX(zp->z_phys->zp_size, woff + n);
659 	tx = dmu_tx_create(zfsvfs->z_os);
660 	dmu_tx_hold_bonus(tx, zp->z_id);
661 	dmu_tx_hold_write(tx, zp->z_id, woff, MIN(n, max_blksz));
662 	error = dmu_tx_assign(tx, zfsvfs->z_assign);
663 	if (error) {
664 		dmu_tx_abort(tx);
665 		rw_exit(&zp->z_grow_lock);
666 		if (error == ERESTART && zfsvfs->z_assign == TXG_NOWAIT) {
667 			txg_wait_open(dmu_objset_pool(zfsvfs->z_os), 0);
668 			goto top;
669 		}
670 		goto no_tx_done;
671 	}
672 
673 	if (end_size > zp->z_blksz &&
674 	    (!ISP2(zp->z_blksz) || zp->z_blksz < max_blksz)) {
675 		uint64_t new_blksz;
676 		/*
677 		 * This write will increase the file size beyond
678 		 * the current block size so increase the block size.
679 		 */
680 		if (grow_rw == RW_READER && !rw_tryupgrade(&zp->z_grow_lock)) {
681 			dmu_tx_commit(tx);
682 			rw_exit(&zp->z_grow_lock);
683 			grow_rw = RW_WRITER;
684 			goto top;
685 		}
686 		if (zp->z_blksz > max_blksz) {
687 			ASSERT(!ISP2(zp->z_blksz));
688 			new_blksz = MIN(end_size, SPA_MAXBLOCKSIZE);
689 		} else {
690 			new_blksz = MIN(end_size, max_blksz);
691 		}
692 		error = zfs_grow_blocksize(zp, new_blksz, tx);
693 		if (error) {
694 			tx_bytes = 0;
695 			goto tx_done;
696 		}
697 	}
698 
699 	if (grow_rw == RW_WRITER) {
700 		rw_downgrade(&zp->z_grow_lock);
701 		grow_rw = RW_READER;
702 	}
703 
704 	/*
705 	 * The file data does not fit in the znode "cache", so we
706 	 * will be writing to the file block data buffers.
707 	 * Each buffer will be written in a separate transaction;
708 	 * this keeps the intent log records small and allows us
709 	 * to do more fine-grained space accounting.
710 	 */
711 	while (n > 0) {
712 		/*
713 		 * XXX - should we really limit each write to z_max_blksz?
714 		 * Perhaps we should use SPA_MAXBLOCKSIZE chunks?
715 		 */
716 		nbytes = MIN(n, max_blksz - P2PHASE(woff, max_blksz));
717 		rw_enter(&zp->z_map_lock, RW_READER);
718 
719 		tx_bytes = uio->uio_resid;
720 		if (vn_has_cached_data(vp)) {
721 			rw_exit(&zp->z_map_lock);
722 			error = mappedwrite(vp, woff, nbytes, uio, tx);
723 		} else {
724 			error = dmu_write_uio(zfsvfs->z_os, zp->z_id,
725 			    woff, nbytes, uio, tx);
726 			rw_exit(&zp->z_map_lock);
727 		}
728 		tx_bytes -= uio->uio_resid;
729 
730 		if (error) {
731 			/* XXX - do we need to "clean up" the dmu buffer? */
732 			break;
733 		}
734 
735 		ASSERT(tx_bytes == nbytes);
736 
737 		n -= nbytes;
738 		if (n <= 0)
739 			break;
740 
741 		/*
742 		 * We have more work ahead of us, so wrap up this transaction
743 		 * and start another.  Exact same logic as tx_done below.
744 		 */
745 		while ((end_size = zp->z_phys->zp_size) < uio->uio_loffset) {
746 			dmu_buf_will_dirty(zp->z_dbuf, tx);
747 			(void) atomic_cas_64(&zp->z_phys->zp_size, end_size,
748 			    uio->uio_loffset);
749 		}
750 		zfs_time_stamper(zp, CONTENT_MODIFIED, tx);
751 		seq = zfs_log_write(zilog, tx, TX_WRITE, zp, woff, tx_bytes,
752 		    ioflag, uio);
753 		dmu_tx_commit(tx);
754 
755 		/* Pre-fault the next set of pages */
756 		zfs_prefault_write(MIN(n, SPA_MAXBLOCKSIZE), uio);
757 
758 		/*
759 		 * Start another transaction.
760 		 */
761 		woff = uio->uio_loffset;
762 		tx = dmu_tx_create(zfsvfs->z_os);
763 		dmu_tx_hold_bonus(tx, zp->z_id);
764 		dmu_tx_hold_write(tx, zp->z_id, woff, MIN(n, max_blksz));
765 		error = dmu_tx_assign(tx, zfsvfs->z_assign);
766 		if (error) {
767 			dmu_tx_abort(tx);
768 			rw_exit(&zp->z_grow_lock);
769 			if (error == ERESTART &&
770 			    zfsvfs->z_assign == TXG_NOWAIT) {
771 				txg_wait_open(dmu_objset_pool(zfsvfs->z_os), 0);
772 				goto top;
773 			}
774 			goto no_tx_done;
775 		}
776 	}
777 
778 tx_done:
779 
780 	if (tx_bytes != 0) {
781 		/*
782 		 * Update the file size if it has changed; account
783 		 * for possible concurrent updates.
784 		 */
785 		while ((end_size = zp->z_phys->zp_size) < uio->uio_loffset) {
786 			dmu_buf_will_dirty(zp->z_dbuf, tx);
787 			(void) atomic_cas_64(&zp->z_phys->zp_size, end_size,
788 			    uio->uio_loffset);
789 		}
790 		zfs_time_stamper(zp, CONTENT_MODIFIED, tx);
791 		seq = zfs_log_write(zilog, tx, TX_WRITE, zp, woff, tx_bytes,
792 		    ioflag, uio);
793 	}
794 	dmu_tx_commit(tx);
795 
796 	rw_exit(&zp->z_grow_lock);
797 
798 no_tx_done:
799 
800 	if (need_append_lock)
801 		rw_exit(&zp->z_append_lock);
802 
803 	/*
804 	 * If we're in replay mode, or we made no progress, return error.
805 	 * Otherwise, it's at least a partial write, so it's successful.
806 	 */
807 	if (zfsvfs->z_assign >= TXG_INITIAL || uio->uio_resid == start_resid) {
808 		ZFS_EXIT(zfsvfs);
809 		return (error);
810 	}
811 
812 	zil_commit(zilog, seq, ioflag & (FSYNC | FDSYNC));
813 
814 	ZFS_EXIT(zfsvfs);
815 	return (0);
816 }
817 
818 /*
819  * Get data to generate a TX_WRITE intent log record.
820  */
821 int
822 zfs_get_data(void *arg, lr_write_t *lr)
823 {
824 	zfsvfs_t *zfsvfs = arg;
825 	objset_t *os = zfsvfs->z_os;
826 	znode_t *zp;
827 	uint64_t off = lr->lr_offset;
828 	int dlen = lr->lr_length;  		/* length of user data */
829 	int reclen = lr->lr_common.lrc_reclen;
830 	int error = 0;
831 
832 	ASSERT(dlen != 0);
833 
834 	/*
835 	 * Nothing to do if the file has been removed or truncated.
836 	 */
837 	if (zfs_zget(zfsvfs, lr->lr_foid, &zp) != 0)
838 		return (ENOENT);
839 	if (off >= zp->z_phys->zp_size || zp->z_reap) {
840 		VN_RELE(ZTOV(zp));
841 		return (ENOENT);
842 	}
843 
844 	/*
845 	 * Write records come in two flavors: immediate and indirect.
846 	 * For small writes it's cheaper to store the data with the
847 	 * log record (immediate); for large writes it's cheaper to
848 	 * sync the data and get a pointer to it (indirect) so that
849 	 * we don't have to write the data twice.
850 	 */
851 	if (sizeof (lr_write_t) + dlen <= reclen) { /* immediate write */
852 		rw_enter(&zp->z_grow_lock, RW_READER);
853 		dmu_buf_t *db = dmu_buf_hold(os, lr->lr_foid, off);
854 		dmu_buf_read(db);
855 		bcopy((char *)db->db_data + off - db->db_offset, lr + 1, dlen);
856 		dmu_buf_rele(db);
857 		rw_exit(&zp->z_grow_lock);
858 	} else {
859 		/*
860 		 * We have to grab z_grow_lock as RW_WRITER because
861 		 * dmu_sync() can't handle concurrent dbuf_dirty() (6313856).
862 		 * z_grow_lock will be replaced with a range lock soon,
863 		 * which will eliminate the concurrency hit, but dmu_sync()
864 		 * really needs more thought.  It shouldn't have to rely on
865 		 * the caller to provide MT safety.
866 		 */
867 		rw_enter(&zp->z_grow_lock, RW_WRITER);
868 		txg_suspend(dmu_objset_pool(os));
869 		error = dmu_sync(os, lr->lr_foid, off, &lr->lr_blkoff,
870 		    &lr->lr_blkptr, lr->lr_common.lrc_txg);
871 		txg_resume(dmu_objset_pool(os));
872 		rw_exit(&zp->z_grow_lock);
873 	}
874 	VN_RELE(ZTOV(zp));
875 	return (error);
876 }
877 
878 /*ARGSUSED*/
879 static int
880 zfs_access(vnode_t *vp, int mode, int flags, cred_t *cr)
881 {
882 	znode_t *zp = VTOZ(vp);
883 	zfsvfs_t *zfsvfs = zp->z_zfsvfs;
884 	int error;
885 
886 	ZFS_ENTER(zfsvfs);
887 	error = zfs_zaccess_rwx(zp, mode, cr);
888 	ZFS_EXIT(zfsvfs);
889 	return (error);
890 }
891 
892 /*
893  * Lookup an entry in a directory, or an extended attribute directory.
894  * If it exists, return a held vnode reference for it.
895  *
896  *	IN:	dvp	- vnode of directory to search.
897  *		nm	- name of entry to lookup.
898  *		pnp	- full pathname to lookup [UNUSED].
899  *		flags	- LOOKUP_XATTR set if looking for an attribute.
900  *		rdir	- root directory vnode [UNUSED].
901  *		cr	- credentials of caller.
902  *
903  *	OUT:	vpp	- vnode of located entry, NULL if not found.
904  *
905  *	RETURN:	0 if success
906  *		error code if failure
907  *
908  * Timestamps:
909  *	NA
910  */
911 /* ARGSUSED */
912 static int
913 zfs_lookup(vnode_t *dvp, char *nm, vnode_t **vpp, struct pathname *pnp,
914     int flags, vnode_t *rdir, cred_t *cr)
915 {
916 
917 	znode_t *zdp = VTOZ(dvp);
918 	zfsvfs_t *zfsvfs = zdp->z_zfsvfs;
919 	int	error;
920 
921 	ZFS_ENTER(zfsvfs);
922 
923 	*vpp = NULL;
924 
925 	if (flags & LOOKUP_XATTR) {
926 		/*
927 		 * We don't allow recursive attributes..
928 		 * Maybe someday we will.
929 		 */
930 		if (zdp->z_phys->zp_flags & ZFS_XATTR) {
931 			ZFS_EXIT(zfsvfs);
932 			return (EINVAL);
933 		}
934 
935 		if (error = zfs_get_xattrdir(VTOZ(dvp), vpp, cr)) {
936 			ZFS_EXIT(zfsvfs);
937 			return (error);
938 		}
939 
940 		/*
941 		 * Do we have permission to get into attribute directory?
942 		 */
943 
944 		if (error = zfs_zaccess(VTOZ(*vpp), ACE_EXECUTE, cr)) {
945 			VN_RELE(*vpp);
946 		}
947 
948 		ZFS_EXIT(zfsvfs);
949 		return (error);
950 	}
951 
952 	if (dvp->v_type != VDIR)
953 		return (ENOTDIR);
954 
955 	/*
956 	 * Check accessibility of directory.
957 	 */
958 
959 	if (error = zfs_zaccess(zdp, ACE_EXECUTE, cr)) {
960 		ZFS_EXIT(zfsvfs);
961 		return (error);
962 	}
963 
964 	if ((error = zfs_dirlook(zdp, nm, vpp)) == 0) {
965 
966 		/*
967 		 * Convert device special files
968 		 */
969 		if (IS_DEVVP(*vpp)) {
970 			vnode_t	*svp;
971 
972 			svp = specvp(*vpp, (*vpp)->v_rdev, (*vpp)->v_type, cr);
973 			VN_RELE(*vpp);
974 			if (svp == NULL)
975 				error = ENOSYS;
976 			else
977 				*vpp = svp;
978 		}
979 	}
980 
981 	ZFS_EXIT(zfsvfs);
982 	return (error);
983 }
984 
985 /*
986  * Attempt to create a new entry in a directory.  If the entry
987  * already exists, truncate the file if permissible, else return
988  * an error.  Return the vp of the created or trunc'd file.
989  *
990  *	IN:	dvp	- vnode of directory to put new file entry in.
991  *		name	- name of new file entry.
992  *		vap	- attributes of new file.
993  *		excl	- flag indicating exclusive or non-exclusive mode.
994  *		mode	- mode to open file with.
995  *		cr	- credentials of caller.
996  *		flag	- large file flag [UNUSED].
997  *
998  *	OUT:	vpp	- vnode of created or trunc'd entry.
999  *
1000  *	RETURN:	0 if success
1001  *		error code if failure
1002  *
1003  * Timestamps:
1004  *	dvp - ctime|mtime updated if new entry created
1005  *	 vp - ctime|mtime always, atime if new
1006  */
1007 /* ARGSUSED */
1008 static int
1009 zfs_create(vnode_t *dvp, char *name, vattr_t *vap, vcexcl_t excl,
1010     int mode, vnode_t **vpp, cred_t *cr, int flag)
1011 {
1012 	znode_t		*zp, *dzp = VTOZ(dvp);
1013 	zfsvfs_t	*zfsvfs = dzp->z_zfsvfs;
1014 	zilog_t		*zilog = zfsvfs->z_log;
1015 	uint64_t	seq = 0;
1016 	objset_t	*os = zfsvfs->z_os;
1017 	zfs_dirlock_t	*dl;
1018 	dmu_tx_t	*tx;
1019 	int		error;
1020 	uint64_t	zoid;
1021 
1022 	ZFS_ENTER(zfsvfs);
1023 
1024 top:
1025 	*vpp = NULL;
1026 
1027 	if ((vap->va_mode & VSVTX) && secpolicy_vnode_stky_modify(cr))
1028 		vap->va_mode &= ~VSVTX;
1029 
1030 	if (*name == '\0') {
1031 		/*
1032 		 * Null component name refers to the directory itself.
1033 		 */
1034 		VN_HOLD(dvp);
1035 		zp = dzp;
1036 		dl = NULL;
1037 		error = 0;
1038 	} else {
1039 		/* possible VN_HOLD(zp) */
1040 		if (error = zfs_dirent_lock(&dl, dzp, name, &zp, 0)) {
1041 			if (strcmp(name, "..") == 0)
1042 				error = EISDIR;
1043 			ZFS_EXIT(zfsvfs);
1044 			return (error);
1045 		}
1046 	}
1047 
1048 	zoid = zp ? zp->z_id : -1ULL;
1049 
1050 	if (zp == NULL) {
1051 		/*
1052 		 * Create a new file object and update the directory
1053 		 * to reference it.
1054 		 */
1055 		if (error = zfs_zaccess(dzp, ACE_ADD_FILE, cr)) {
1056 			goto out;
1057 		}
1058 
1059 		/*
1060 		 * We only support the creation of regular files in
1061 		 * extended attribute directories.
1062 		 */
1063 		if ((dzp->z_phys->zp_flags & ZFS_XATTR) &&
1064 		    (vap->va_type != VREG)) {
1065 			error = EINVAL;
1066 			goto out;
1067 		}
1068 
1069 		tx = dmu_tx_create(os);
1070 		dmu_tx_hold_bonus(tx, DMU_NEW_OBJECT);
1071 		dmu_tx_hold_bonus(tx, dzp->z_id);
1072 		dmu_tx_hold_zap(tx, dzp->z_id, 1);
1073 		if (dzp->z_phys->zp_flags & ZFS_INHERIT_ACE)
1074 			dmu_tx_hold_write(tx, DMU_NEW_OBJECT,
1075 			    0, SPA_MAXBLOCKSIZE);
1076 		error = dmu_tx_assign(tx, zfsvfs->z_assign);
1077 		if (error) {
1078 			dmu_tx_abort(tx);
1079 			zfs_dirent_unlock(dl);
1080 			if (error == ERESTART &&
1081 			    zfsvfs->z_assign == TXG_NOWAIT) {
1082 				txg_wait_open(dmu_objset_pool(os), 0);
1083 				goto top;
1084 			}
1085 			ZFS_EXIT(zfsvfs);
1086 			return (error);
1087 		}
1088 		zfs_mknode(dzp, vap, &zoid, tx, cr, 0, &zp, 0);
1089 		ASSERT(zp->z_id == zoid);
1090 		(void) zfs_link_create(dl, zp, tx, ZNEW);
1091 		seq = zfs_log_create(zilog, tx, TX_CREATE, dzp, zp, name);
1092 		dmu_tx_commit(tx);
1093 	} else {
1094 		/*
1095 		 * A directory entry already exists for this name.
1096 		 */
1097 		/*
1098 		 * Can't truncate an existing file if in exclusive mode.
1099 		 */
1100 		if (excl == EXCL) {
1101 			error = EEXIST;
1102 			goto out;
1103 		}
1104 		/*
1105 		 * Can't open a directory for writing.
1106 		 */
1107 		if ((ZTOV(zp)->v_type == VDIR) && (mode & S_IWRITE)) {
1108 			error = EISDIR;
1109 			goto out;
1110 		}
1111 		/*
1112 		 * Verify requested access to file.
1113 		 */
1114 		if (mode && (error = zfs_zaccess_rwx(zp, mode, cr))) {
1115 			goto out;
1116 		}
1117 		/*
1118 		 * Truncate regular files if requested.
1119 		 */
1120 
1121 		/*
1122 		 * Need to update dzp->z_seq?
1123 		 */
1124 
1125 		mutex_enter(&dzp->z_lock);
1126 		dzp->z_seq++;
1127 		mutex_exit(&dzp->z_lock);
1128 
1129 		if ((ZTOV(zp)->v_type == VREG) && (zp->z_phys->zp_size != 0) &&
1130 		    (vap->va_mask & AT_SIZE) && (vap->va_size == 0)) {
1131 			/*
1132 			 * Truncate the file.
1133 			 */
1134 			tx = dmu_tx_create(os);
1135 			dmu_tx_hold_bonus(tx, zoid);
1136 			dmu_tx_hold_free(tx, zoid, 0, DMU_OBJECT_END);
1137 			error = dmu_tx_assign(tx, zfsvfs->z_assign);
1138 			if (error) {
1139 				dmu_tx_abort(tx);
1140 				if (dl)
1141 					zfs_dirent_unlock(dl);
1142 				VN_RELE(ZTOV(zp));
1143 				if (error == ERESTART &&
1144 				    zfsvfs->z_assign == TXG_NOWAIT) {
1145 					txg_wait_open(dmu_objset_pool(os), 0);
1146 					goto top;
1147 				}
1148 				ZFS_EXIT(zfsvfs);
1149 				return (error);
1150 			}
1151 			/*
1152 			 * Grab the grow_lock to serialize this change with
1153 			 * respect to other file manipulations.
1154 			 */
1155 			rw_enter(&zp->z_grow_lock, RW_WRITER);
1156 			error = zfs_freesp(zp, 0, 0, mode, tx, cr);
1157 			if (error == 0) {
1158 				zfs_time_stamper(zp, CONTENT_MODIFIED, tx);
1159 				seq = zfs_log_truncate(zilog, tx,
1160 				    TX_TRUNCATE, zp, 0, 0);
1161 			}
1162 			rw_exit(&zp->z_grow_lock);
1163 			dmu_tx_commit(tx);
1164 		}
1165 	}
1166 out:
1167 
1168 	if (dl)
1169 		zfs_dirent_unlock(dl);
1170 
1171 	if (error) {
1172 		if (zp)
1173 			VN_RELE(ZTOV(zp));
1174 	} else {
1175 		*vpp = ZTOV(zp);
1176 		/*
1177 		 * If vnode is for a device return a specfs vnode instead.
1178 		 */
1179 		if (IS_DEVVP(*vpp)) {
1180 			struct vnode *svp;
1181 
1182 			svp = specvp(*vpp, (*vpp)->v_rdev, (*vpp)->v_type, cr);
1183 			VN_RELE(*vpp);
1184 			if (svp == NULL) {
1185 				error = ENOSYS;
1186 			}
1187 			*vpp = svp;
1188 		}
1189 	}
1190 
1191 	zil_commit(zilog, seq, 0);
1192 
1193 	ZFS_EXIT(zfsvfs);
1194 	return (error);
1195 }
1196 
1197 /*
1198  * Remove an entry from a directory.
1199  *
1200  *	IN:	dvp	- vnode of directory to remove entry from.
1201  *		name	- name of entry to remove.
1202  *		cr	- credentials of caller.
1203  *
1204  *	RETURN:	0 if success
1205  *		error code if failure
1206  *
1207  * Timestamps:
1208  *	dvp - ctime|mtime
1209  *	 vp - ctime (if nlink > 0)
1210  */
1211 static int
1212 zfs_remove(vnode_t *dvp, char *name, cred_t *cr)
1213 {
1214 	znode_t		*zp, *dzp = VTOZ(dvp);
1215 	znode_t		*xzp = NULL;
1216 	vnode_t		*vp;
1217 	zfsvfs_t	*zfsvfs = dzp->z_zfsvfs;
1218 	zilog_t		*zilog = zfsvfs->z_log;
1219 	uint64_t	seq = 0;
1220 	uint64_t	acl_obj, xattr_obj;
1221 	zfs_dirlock_t	*dl;
1222 	dmu_tx_t	*tx;
1223 	int		may_delete_now, delete_now = FALSE;
1224 	int		reaped;
1225 	int		error;
1226 
1227 	ZFS_ENTER(zfsvfs);
1228 
1229 top:
1230 	/*
1231 	 * Attempt to lock directory; fail if entry doesn't exist.
1232 	 */
1233 	if (error = zfs_dirent_lock(&dl, dzp, name, &zp, ZEXISTS)) {
1234 		ZFS_EXIT(zfsvfs);
1235 		return (error);
1236 	}
1237 
1238 	vp = ZTOV(zp);
1239 
1240 	if (error = zfs_zaccess_delete(dzp, zp, cr)) {
1241 		goto out;
1242 	}
1243 
1244 	/*
1245 	 * Need to use rmdir for removing directories.
1246 	 */
1247 	if (vp->v_type == VDIR) {
1248 		error = EPERM;
1249 		goto out;
1250 	}
1251 
1252 	vnevent_remove(vp);
1253 
1254 	dnlc_remove(dvp, name);
1255 
1256 	mutex_enter(&vp->v_lock);
1257 	may_delete_now = vp->v_count == 1 && !vn_has_cached_data(vp);
1258 	mutex_exit(&vp->v_lock);
1259 
1260 	/*
1261 	 * We may delete the znode now, or we may put it on the delete queue;
1262 	 * it depends on whether we're the last link, and on whether there are
1263 	 * other holds on the vnode.  So we dmu_tx_hold() the right things to
1264 	 * allow for either case.
1265 	 */
1266 	tx = dmu_tx_create(zfsvfs->z_os);
1267 	dmu_tx_hold_zap(tx, dzp->z_id, -1);
1268 	dmu_tx_hold_bonus(tx, zp->z_id);
1269 	if (may_delete_now)
1270 		dmu_tx_hold_free(tx, zp->z_id, 0, DMU_OBJECT_END);
1271 
1272 	/* are there any extended attributes? */
1273 	if ((xattr_obj = zp->z_phys->zp_xattr) != 0) {
1274 		/*
1275 		 * XXX - There is a possibility that the delete
1276 		 * of the parent file could succeed, but then we get
1277 		 * an ENOSPC when we try to delete the xattrs...
1278 		 * so we would need to re-try the deletes periodically
1279 		 */
1280 		/* XXX - do we need this if we are deleting? */
1281 		dmu_tx_hold_bonus(tx, xattr_obj);
1282 	}
1283 
1284 	/* are there any additional acls */
1285 	if ((acl_obj = zp->z_phys->zp_acl.z_acl_extern_obj) != 0 &&
1286 	    may_delete_now)
1287 		dmu_tx_hold_free(tx, acl_obj, 0, DMU_OBJECT_END);
1288 
1289 	/* charge as an update -- would be nice not to charge at all */
1290 	dmu_tx_hold_zap(tx, zfsvfs->z_dqueue, -1);
1291 
1292 	error = dmu_tx_assign(tx, zfsvfs->z_assign);
1293 	if (error) {
1294 		dmu_tx_abort(tx);
1295 		zfs_dirent_unlock(dl);
1296 		VN_RELE(vp);
1297 		if (error == ERESTART && zfsvfs->z_assign == TXG_NOWAIT) {
1298 			txg_wait_open(dmu_objset_pool(zfsvfs->z_os), 0);
1299 			goto top;
1300 		}
1301 		ZFS_EXIT(zfsvfs);
1302 		return (error);
1303 	}
1304 
1305 	/*
1306 	 * Remove the directory entry.
1307 	 */
1308 	error = zfs_link_destroy(dl, zp, tx, 0, &reaped);
1309 
1310 	if (error) {
1311 		dmu_tx_commit(tx);
1312 		goto out;
1313 	}
1314 
1315 	if (reaped) {
1316 		mutex_enter(&vp->v_lock);
1317 		delete_now = may_delete_now &&
1318 		    vp->v_count == 1 && !vn_has_cached_data(vp) &&
1319 		    zp->z_phys->zp_xattr == xattr_obj &&
1320 		    zp->z_phys->zp_acl.z_acl_extern_obj == acl_obj;
1321 		mutex_exit(&vp->v_lock);
1322 	}
1323 
1324 	if (delete_now) {
1325 		if (zp->z_phys->zp_xattr) {
1326 			error = zfs_zget(zfsvfs, zp->z_phys->zp_xattr, &xzp);
1327 			ASSERT3U(error, ==, 0);
1328 			ASSERT3U(xzp->z_phys->zp_links, ==, 2);
1329 			dmu_buf_will_dirty(xzp->z_dbuf, tx);
1330 			mutex_enter(&xzp->z_lock);
1331 			xzp->z_reap = 1;
1332 			xzp->z_phys->zp_links = 0;
1333 			mutex_exit(&xzp->z_lock);
1334 			zfs_dq_add(xzp, tx);
1335 			zp->z_phys->zp_xattr = 0; /* probably unnecessary */
1336 		}
1337 		mutex_enter(&zp->z_lock);
1338 		mutex_enter(&vp->v_lock);
1339 		vp->v_count--;
1340 		ASSERT3U(vp->v_count, ==, 0);
1341 		mutex_exit(&vp->v_lock);
1342 		zp->z_active = 0;
1343 		mutex_exit(&zp->z_lock);
1344 		zfs_znode_delete(zp, tx);
1345 		VFS_RELE(zfsvfs->z_vfs);
1346 	} else if (reaped) {
1347 		zfs_dq_add(zp, tx);
1348 	}
1349 
1350 	seq = zfs_log_remove(zilog, tx, TX_REMOVE, dzp, name);
1351 
1352 	dmu_tx_commit(tx);
1353 out:
1354 	zfs_dirent_unlock(dl);
1355 
1356 	if (!delete_now) {
1357 		VN_RELE(vp);
1358 	} else if (xzp) {
1359 		/* this rele delayed to prevent nesting transactions */
1360 		VN_RELE(ZTOV(xzp));
1361 	}
1362 
1363 	zil_commit(zilog, seq, 0);
1364 
1365 	ZFS_EXIT(zfsvfs);
1366 	return (error);
1367 }
1368 
1369 /*
1370  * Create a new directory and insert it into dvp using the name
1371  * provided.  Return a pointer to the inserted directory.
1372  *
1373  *	IN:	dvp	- vnode of directory to add subdir to.
1374  *		dirname	- name of new directory.
1375  *		vap	- attributes of new directory.
1376  *		cr	- credentials of caller.
1377  *
1378  *	OUT:	vpp	- vnode of created directory.
1379  *
1380  *	RETURN:	0 if success
1381  *		error code if failure
1382  *
1383  * Timestamps:
1384  *	dvp - ctime|mtime updated
1385  *	 vp - ctime|mtime|atime updated
1386  */
1387 static int
1388 zfs_mkdir(vnode_t *dvp, char *dirname, vattr_t *vap, vnode_t **vpp, cred_t *cr)
1389 {
1390 	znode_t		*zp, *dzp = VTOZ(dvp);
1391 	zfsvfs_t	*zfsvfs = dzp->z_zfsvfs;
1392 	zilog_t		*zilog = zfsvfs->z_log;
1393 	uint64_t	seq = 0;
1394 	zfs_dirlock_t	*dl;
1395 	uint64_t	zoid = 0;
1396 	dmu_tx_t	*tx;
1397 	int		error;
1398 
1399 	ASSERT(vap->va_type == VDIR);
1400 
1401 	ZFS_ENTER(zfsvfs);
1402 
1403 	if (dzp->z_phys->zp_flags & ZFS_XATTR) {
1404 		ZFS_EXIT(zfsvfs);
1405 		return (EINVAL);
1406 	}
1407 top:
1408 	*vpp = NULL;
1409 
1410 	/*
1411 	 * First make sure the new directory doesn't exist.
1412 	 */
1413 	if (error = zfs_dirent_lock(&dl, dzp, dirname, &zp, ZNEW)) {
1414 		ZFS_EXIT(zfsvfs);
1415 		return (error);
1416 	}
1417 
1418 	if (error = zfs_zaccess(dzp, ACE_ADD_SUBDIRECTORY, cr)) {
1419 		zfs_dirent_unlock(dl);
1420 		ZFS_EXIT(zfsvfs);
1421 		return (error);
1422 	}
1423 
1424 	/*
1425 	 * Add a new entry to the directory.
1426 	 */
1427 	tx = dmu_tx_create(zfsvfs->z_os);
1428 	dmu_tx_hold_zap(tx, dzp->z_id, 1);
1429 	dmu_tx_hold_zap(tx, DMU_NEW_OBJECT, 0);
1430 	if (dzp->z_phys->zp_flags & ZFS_INHERIT_ACE)
1431 		dmu_tx_hold_write(tx, DMU_NEW_OBJECT,
1432 		    0, SPA_MAXBLOCKSIZE);
1433 	error = dmu_tx_assign(tx, zfsvfs->z_assign);
1434 	if (error) {
1435 		dmu_tx_abort(tx);
1436 		zfs_dirent_unlock(dl);
1437 		if (error == ERESTART && zfsvfs->z_assign == TXG_NOWAIT) {
1438 			txg_wait_open(dmu_objset_pool(zfsvfs->z_os), 0);
1439 			goto top;
1440 		}
1441 		ZFS_EXIT(zfsvfs);
1442 		return (error);
1443 	}
1444 
1445 	/*
1446 	 * Create new node.
1447 	 */
1448 	zfs_mknode(dzp, vap, &zoid, tx, cr, 0, &zp, 0);
1449 
1450 	/*
1451 	 * Now put new name in parent dir.
1452 	 */
1453 	(void) zfs_link_create(dl, zp, tx, ZNEW);
1454 
1455 	*vpp = ZTOV(zp);
1456 
1457 	seq = zfs_log_create(zilog, tx, TX_MKDIR, dzp, zp, dirname);
1458 	dmu_tx_commit(tx);
1459 
1460 	zfs_dirent_unlock(dl);
1461 
1462 	zil_commit(zilog, seq, 0);
1463 
1464 	ZFS_EXIT(zfsvfs);
1465 	return (0);
1466 }
1467 
1468 /*
1469  * Remove a directory subdir entry.  If the current working
1470  * directory is the same as the subdir to be removed, the
1471  * remove will fail.
1472  *
1473  *	IN:	dvp	- vnode of directory to remove from.
1474  *		name	- name of directory to be removed.
1475  *		cwd	- vnode of current working directory.
1476  *		cr	- credentials of caller.
1477  *
1478  *	RETURN:	0 if success
1479  *		error code if failure
1480  *
1481  * Timestamps:
1482  *	dvp - ctime|mtime updated
1483  */
1484 static int
1485 zfs_rmdir(vnode_t *dvp, char *name, vnode_t *cwd, cred_t *cr)
1486 {
1487 	znode_t		*dzp = VTOZ(dvp);
1488 	znode_t		*zp;
1489 	vnode_t		*vp;
1490 	zfsvfs_t	*zfsvfs = dzp->z_zfsvfs;
1491 	zilog_t		*zilog = zfsvfs->z_log;
1492 	uint64_t	seq = 0;
1493 	zfs_dirlock_t	*dl;
1494 	dmu_tx_t	*tx;
1495 	int		error;
1496 
1497 	ZFS_ENTER(zfsvfs);
1498 
1499 top:
1500 	zp = NULL;
1501 
1502 	/*
1503 	 * Attempt to lock directory; fail if entry doesn't exist.
1504 	 */
1505 	if (error = zfs_dirent_lock(&dl, dzp, name, &zp, ZEXISTS)) {
1506 		ZFS_EXIT(zfsvfs);
1507 		return (error);
1508 	}
1509 
1510 	vp = ZTOV(zp);
1511 
1512 	if (error = zfs_zaccess_delete(dzp, zp, cr)) {
1513 		goto out;
1514 	}
1515 
1516 	if (vp->v_type != VDIR) {
1517 		error = ENOTDIR;
1518 		goto out;
1519 	}
1520 
1521 	if (vp == cwd) {
1522 		error = EINVAL;
1523 		goto out;
1524 	}
1525 
1526 	vnevent_rmdir(vp);
1527 
1528 	/*
1529 	 * Grab a lock on the parent pointer make sure we play well
1530 	 * with the treewalk and directory rename code.
1531 	 */
1532 	rw_enter(&zp->z_parent_lock, RW_WRITER);
1533 
1534 	tx = dmu_tx_create(zfsvfs->z_os);
1535 	dmu_tx_hold_zap(tx, dzp->z_id, 1);
1536 	dmu_tx_hold_bonus(tx, zp->z_id);
1537 	dmu_tx_hold_zap(tx, zfsvfs->z_dqueue, 1);
1538 	error = dmu_tx_assign(tx, zfsvfs->z_assign);
1539 	if (error) {
1540 		dmu_tx_abort(tx);
1541 		rw_exit(&zp->z_parent_lock);
1542 		zfs_dirent_unlock(dl);
1543 		VN_RELE(vp);
1544 		if (error == ERESTART && zfsvfs->z_assign == TXG_NOWAIT) {
1545 			txg_wait_open(dmu_objset_pool(zfsvfs->z_os), 0);
1546 			goto top;
1547 		}
1548 		ZFS_EXIT(zfsvfs);
1549 		return (error);
1550 	}
1551 
1552 	error = zfs_link_destroy(dl, zp, tx, 0, NULL);
1553 
1554 	if (error == 0)
1555 		seq = zfs_log_remove(zilog, tx, TX_RMDIR, dzp, name);
1556 
1557 	dmu_tx_commit(tx);
1558 
1559 	rw_exit(&zp->z_parent_lock);
1560 out:
1561 	zfs_dirent_unlock(dl);
1562 
1563 	VN_RELE(vp);
1564 
1565 	zil_commit(zilog, seq, 0);
1566 
1567 	ZFS_EXIT(zfsvfs);
1568 	return (error);
1569 }
1570 
1571 /*
1572  * Read as many directory entries as will fit into the provided
1573  * buffer from the given directory cursor position (specified in
1574  * the uio structure.
1575  *
1576  *	IN:	vp	- vnode of directory to read.
1577  *		uio	- structure supplying read location, range info,
1578  *			  and return buffer.
1579  *		cr	- credentials of caller.
1580  *
1581  *	OUT:	uio	- updated offset and range, buffer filled.
1582  *		eofp	- set to true if end-of-file detected.
1583  *
1584  *	RETURN:	0 if success
1585  *		error code if failure
1586  *
1587  * Timestamps:
1588  *	vp - atime updated
1589  *
1590  * Note that the low 4 bits of the cookie returned by zap is always zero.
1591  * This allows us to use the low range for "special" directory entries:
1592  * We use 0 for '.', and 1 for '..'.  If this is the root of the filesystem,
1593  * we use the offset 2 for the '.zfs' directory.
1594  */
1595 /* ARGSUSED */
1596 static int
1597 zfs_readdir(vnode_t *vp, uio_t *uio, cred_t *cr, int *eofp)
1598 {
1599 	znode_t		*zp = VTOZ(vp);
1600 	iovec_t		*iovp;
1601 	dirent64_t	*odp;
1602 	zfsvfs_t	*zfsvfs = zp->z_zfsvfs;
1603 	objset_t	*os;
1604 	caddr_t		outbuf;
1605 	size_t		bufsize;
1606 	zap_cursor_t	zc;
1607 	zap_attribute_t	zap;
1608 	uint_t		bytes_wanted;
1609 	ushort_t	this_reclen;
1610 	uint64_t	offset; /* must be unsigned; checks for < 1 */
1611 	off64_t		*next;
1612 	int		local_eof;
1613 	int		outcount;
1614 	int		error;
1615 	uint8_t		prefetch;
1616 
1617 	ZFS_ENTER(zfsvfs);
1618 
1619 	/*
1620 	 * If we are not given an eof variable,
1621 	 * use a local one.
1622 	 */
1623 	if (eofp == NULL)
1624 		eofp = &local_eof;
1625 
1626 	/*
1627 	 * Check for valid iov_len.
1628 	 */
1629 	if (uio->uio_iov->iov_len <= 0) {
1630 		ZFS_EXIT(zfsvfs);
1631 		return (EINVAL);
1632 	}
1633 
1634 	/*
1635 	 * Quit if directory has been removed (posix)
1636 	 */
1637 	if ((*eofp = zp->z_reap) != 0) {
1638 		ZFS_EXIT(zfsvfs);
1639 		return (0);
1640 	}
1641 
1642 	error = 0;
1643 	os = zfsvfs->z_os;
1644 	offset = uio->uio_loffset;
1645 	prefetch = zp->z_zn_prefetch;
1646 
1647 	/*
1648 	 * Initialize the iterator cursor.
1649 	 */
1650 	if (offset <= 3) {
1651 		/*
1652 		 * Start iteration from the beginning of the directory.
1653 		 */
1654 		zap_cursor_init(&zc, os, zp->z_id);
1655 	} else {
1656 		/*
1657 		 * The offset is a serialized cursor.
1658 		 */
1659 		zap_cursor_init_serialized(&zc, os, zp->z_id, offset);
1660 	}
1661 
1662 	/*
1663 	 * Get space to change directory entries into fs independent format.
1664 	 */
1665 	iovp = uio->uio_iov;
1666 	bytes_wanted = iovp->iov_len;
1667 	if (uio->uio_segflg != UIO_SYSSPACE || uio->uio_iovcnt != 1) {
1668 		bufsize = bytes_wanted;
1669 		outbuf = kmem_alloc(bufsize, KM_SLEEP);
1670 		odp = (struct dirent64 *)outbuf;
1671 	} else {
1672 		bufsize = bytes_wanted;
1673 		odp = (struct dirent64 *)iovp->iov_base;
1674 	}
1675 
1676 	/*
1677 	 * Transform to file-system independent format
1678 	 */
1679 	outcount = 0;
1680 	while (outcount < bytes_wanted) {
1681 		/*
1682 		 * Special case `.', `..', and `.zfs'.
1683 		 */
1684 		if (offset == 0) {
1685 			(void) strcpy(zap.za_name, ".");
1686 			zap.za_first_integer = zp->z_id;
1687 			this_reclen = DIRENT64_RECLEN(1);
1688 		} else if (offset == 1) {
1689 			(void) strcpy(zap.za_name, "..");
1690 			zap.za_first_integer = zp->z_phys->zp_parent;
1691 			this_reclen = DIRENT64_RECLEN(2);
1692 		} else if (offset == 2 && zfs_show_ctldir(zp)) {
1693 			(void) strcpy(zap.za_name, ZFS_CTLDIR_NAME);
1694 			zap.za_first_integer = ZFSCTL_INO_ROOT;
1695 			this_reclen =
1696 			    DIRENT64_RECLEN(sizeof (ZFS_CTLDIR_NAME) - 1);
1697 		} else {
1698 			/*
1699 			 * Grab next entry.
1700 			 */
1701 			if (error = zap_cursor_retrieve(&zc, &zap)) {
1702 				if ((*eofp = (error == ENOENT)) != 0)
1703 					break;
1704 				else
1705 					goto update;
1706 			}
1707 
1708 			if (zap.za_integer_length != 8 ||
1709 			    zap.za_num_integers != 1) {
1710 				cmn_err(CE_WARN, "zap_readdir: bad directory "
1711 				    "entry, obj = %lld, offset = %lld\n",
1712 				    (u_longlong_t)zp->z_id,
1713 				    (u_longlong_t)offset);
1714 				error = ENXIO;
1715 				goto update;
1716 			}
1717 			this_reclen = DIRENT64_RECLEN(strlen(zap.za_name));
1718 		}
1719 
1720 		/*
1721 		 * Will this entry fit in the buffer?
1722 		 */
1723 		if (outcount + this_reclen > bufsize) {
1724 			/*
1725 			 * Did we manage to fit anything in the buffer?
1726 			 */
1727 			if (!outcount) {
1728 				error = EINVAL;
1729 				goto update;
1730 			}
1731 			break;
1732 		}
1733 		/*
1734 		 * Add this entry:
1735 		 */
1736 		odp->d_ino = (ino64_t)zap.za_first_integer;
1737 		odp->d_reclen = (ushort_t)this_reclen;
1738 		/* NOTE: d_off is the offset for the *next* entry */
1739 		next = &(odp->d_off);
1740 		(void) strncpy(odp->d_name, zap.za_name,
1741 		    DIRENT64_NAMELEN(this_reclen));
1742 		outcount += this_reclen;
1743 		odp = (dirent64_t *)((intptr_t)odp + this_reclen);
1744 
1745 		ASSERT(outcount <= bufsize);
1746 
1747 		/* Prefetch znode */
1748 		if (prefetch)
1749 			dmu_prefetch(os, zap.za_first_integer, 0, 0);
1750 
1751 		/*
1752 		 * Move to the next entry, fill in the previous offset.
1753 		 */
1754 		if (offset > 2 || (offset == 2 && !zfs_show_ctldir(zp))) {
1755 			zap_cursor_advance(&zc);
1756 			offset = zap_cursor_serialize(&zc);
1757 		} else {
1758 			offset += 1;
1759 		}
1760 		*next = offset;
1761 	}
1762 	zp->z_zn_prefetch = B_FALSE; /* a lookup will re-enable pre-fetching */
1763 
1764 	if (uio->uio_segflg == UIO_SYSSPACE && uio->uio_iovcnt == 1) {
1765 		iovp->iov_base += outcount;
1766 		iovp->iov_len -= outcount;
1767 		uio->uio_resid -= outcount;
1768 	} else if (error = uiomove(outbuf, (long)outcount, UIO_READ, uio)) {
1769 		/*
1770 		 * Reset the pointer.
1771 		 */
1772 		offset = uio->uio_loffset;
1773 	}
1774 
1775 update:
1776 	zap_cursor_fini(&zc);
1777 	if (uio->uio_segflg != UIO_SYSSPACE || uio->uio_iovcnt != 1)
1778 		kmem_free(outbuf, bufsize);
1779 
1780 	if (error == ENOENT)
1781 		error = 0;
1782 
1783 	ZFS_ACCESSTIME_STAMP(zfsvfs, zp);
1784 
1785 	uio->uio_loffset = offset;
1786 	ZFS_EXIT(zfsvfs);
1787 	return (error);
1788 }
1789 
1790 /* ARGSUSED */
1791 static int
1792 zfs_fsync(vnode_t *vp, int syncflag, cred_t *cr)
1793 {
1794 	znode_t	*zp = VTOZ(vp);
1795 	zfsvfs_t *zfsvfs = zp->z_zfsvfs;
1796 
1797 	ZFS_ENTER(zfsvfs);
1798 	zil_commit(zfsvfs->z_log, zp->z_last_itx, FSYNC);
1799 	ZFS_EXIT(zfsvfs);
1800 	return (0);
1801 }
1802 
1803 /*
1804  * Get the requested file attributes and place them in the provided
1805  * vattr structure.
1806  *
1807  *	IN:	vp	- vnode of file.
1808  *		vap	- va_mask identifies requested attributes.
1809  *		flags	- [UNUSED]
1810  *		cr	- credentials of caller.
1811  *
1812  *	OUT:	vap	- attribute values.
1813  *
1814  *	RETURN:	0 (always succeeds)
1815  */
1816 /* ARGSUSED */
1817 static int
1818 zfs_getattr(vnode_t *vp, vattr_t *vap, int flags, cred_t *cr)
1819 {
1820 	znode_t *zp = VTOZ(vp);
1821 	zfsvfs_t *zfsvfs = zp->z_zfsvfs;
1822 	znode_phys_t *pzp = zp->z_phys;
1823 	int	error;
1824 
1825 	ZFS_ENTER(zfsvfs);
1826 
1827 	/*
1828 	 * Return all attributes.  It's cheaper to provide the answer
1829 	 * than to determine whether we were asked the question.
1830 	 */
1831 	mutex_enter(&zp->z_lock);
1832 
1833 	vap->va_type = vp->v_type;
1834 	vap->va_mode = pzp->zp_mode & MODEMASK;
1835 	vap->va_uid = zp->z_phys->zp_uid;
1836 	vap->va_gid = zp->z_phys->zp_gid;
1837 	vap->va_fsid = zp->z_zfsvfs->z_vfs->vfs_dev;
1838 	vap->va_nodeid = zp->z_id;
1839 	vap->va_nlink = MIN(pzp->zp_links, UINT32_MAX);	/* nlink_t limit! */
1840 	vap->va_size = pzp->zp_size;
1841 	vap->va_rdev = pzp->zp_rdev;
1842 	vap->va_seq = zp->z_seq;
1843 
1844 	ZFS_TIME_DECODE(&vap->va_atime, pzp->zp_atime);
1845 	ZFS_TIME_DECODE(&vap->va_mtime, pzp->zp_mtime);
1846 	ZFS_TIME_DECODE(&vap->va_ctime, pzp->zp_ctime);
1847 
1848 	/*
1849 	 * If ACL is trivial don't bother looking for ACE_READ_ATTRIBUTES.
1850 	 * Also, if we are the owner don't bother, since owner should
1851 	 * always be allowed to read basic attributes of file.
1852 	 */
1853 	if (!(zp->z_phys->zp_flags & ZFS_ACL_TRIVIAL) &&
1854 	    (zp->z_phys->zp_uid != crgetuid(cr))) {
1855 		if (error = zfs_zaccess(zp, ACE_READ_ATTRIBUTES, cr)) {
1856 			mutex_exit(&zp->z_lock);
1857 			ZFS_EXIT(zfsvfs);
1858 			return (error);
1859 		}
1860 	}
1861 
1862 	mutex_exit(&zp->z_lock);
1863 
1864 	dmu_object_size_from_db(zp->z_dbuf, &vap->va_blksize, &vap->va_nblocks);
1865 
1866 	if (zp->z_blksz == 0) {
1867 		/*
1868 		 * Block size hasn't been set; suggest maximal I/O transfers.
1869 		 */
1870 		vap->va_blksize = zfsvfs->z_max_blksz;
1871 	}
1872 
1873 	ZFS_EXIT(zfsvfs);
1874 	return (0);
1875 }
1876 
1877 /*
1878  * Set the file attributes to the values contained in the
1879  * vattr structure.
1880  *
1881  *	IN:	vp	- vnode of file to be modified.
1882  *		vap	- new attribute values.
1883  *		flags	- ATTR_UTIME set if non-default time values provided.
1884  *		cr	- credentials of caller.
1885  *
1886  *	RETURN:	0 if success
1887  *		error code if failure
1888  *
1889  * Timestamps:
1890  *	vp - ctime updated, mtime updated if size changed.
1891  */
1892 /* ARGSUSED */
1893 static int
1894 zfs_setattr(vnode_t *vp, vattr_t *vap, int flags, cred_t *cr,
1895 	caller_context_t *ct)
1896 {
1897 	struct znode	*zp = VTOZ(vp);
1898 	znode_phys_t	*pzp = zp->z_phys;
1899 	zfsvfs_t	*zfsvfs = zp->z_zfsvfs;
1900 	zilog_t		*zilog = zfsvfs->z_log;
1901 	uint64_t	seq = 0;
1902 	dmu_tx_t	*tx;
1903 	uint_t		mask = vap->va_mask;
1904 	uint_t		mask_applied = 0;
1905 	vattr_t		oldva;
1906 	int		trim_mask = FALSE;
1907 	int		saved_mask;
1908 	uint64_t	new_mode;
1909 	znode_t		*attrzp;
1910 	int		have_grow_lock;
1911 	int		need_policy = FALSE;
1912 	int		err;
1913 
1914 	if (mask == 0)
1915 		return (0);
1916 
1917 	if (mask & AT_NOSET)
1918 		return (EINVAL);
1919 
1920 	if (mask & AT_SIZE && vp->v_type == VDIR)
1921 		return (EISDIR);
1922 
1923 	if (mask & AT_SIZE && vp->v_type != VREG && vp->v_type != VFIFO)
1924 		return (EINVAL);
1925 
1926 	ZFS_ENTER(zfsvfs);
1927 
1928 top:
1929 	have_grow_lock = FALSE;
1930 	attrzp = NULL;
1931 
1932 	if (zfsvfs->z_vfs->vfs_flag & VFS_RDONLY) {
1933 		ZFS_EXIT(zfsvfs);
1934 		return (EROFS);
1935 	}
1936 
1937 	/*
1938 	 * First validate permissions
1939 	 */
1940 
1941 	if (mask & AT_SIZE) {
1942 		err = zfs_zaccess(zp, ACE_WRITE_DATA, cr);
1943 		if (err) {
1944 			ZFS_EXIT(zfsvfs);
1945 			return (err);
1946 		}
1947 	}
1948 
1949 	if (mask & (AT_ATIME|AT_MTIME))
1950 		need_policy = zfs_zaccess_v4_perm(zp, ACE_WRITE_ATTRIBUTES, cr);
1951 
1952 	if (mask & (AT_UID|AT_GID)) {
1953 		int	idmask = (mask & (AT_UID|AT_GID));
1954 		int	take_owner;
1955 		int	take_group;
1956 
1957 		/*
1958 		 * NOTE: even if a new mode is being set,
1959 		 * we may clear S_ISUID/S_ISGID bits.
1960 		 */
1961 
1962 		if (!(mask & AT_MODE))
1963 			vap->va_mode = pzp->zp_mode;
1964 
1965 		/*
1966 		 * Take ownership or chgrp to group we are a member of
1967 		 */
1968 
1969 		take_owner = (mask & AT_UID) && (vap->va_uid == crgetuid(cr));
1970 		take_group = (mask & AT_GID) && groupmember(vap->va_gid, cr);
1971 
1972 		/*
1973 		 * If both AT_UID and AT_GID are set then take_owner and
1974 		 * take_group must both be set in order to allow taking
1975 		 * ownership.
1976 		 *
1977 		 * Otherwise, send the check through secpolicy_vnode_setattr()
1978 		 *
1979 		 */
1980 
1981 		if (((idmask == (AT_UID|AT_GID)) && take_owner && take_group) ||
1982 		    ((idmask == AT_UID) && take_owner) ||
1983 		    ((idmask == AT_GID) && take_group)) {
1984 			if (zfs_zaccess_v4_perm(zp, ACE_WRITE_OWNER, cr) == 0) {
1985 				/*
1986 				 * Remove setuid/setgid for non-privileged users
1987 				 */
1988 				secpolicy_setid_clear(vap, cr);
1989 				trim_mask = TRUE;
1990 				saved_mask = vap->va_mask;
1991 			} else {
1992 				need_policy =  TRUE;
1993 			}
1994 		} else {
1995 			need_policy =  TRUE;
1996 		}
1997 	}
1998 
1999 	if (mask & AT_MODE)
2000 		need_policy = TRUE;
2001 
2002 	if (need_policy) {
2003 		mutex_enter(&zp->z_lock);
2004 		oldva.va_mode = pzp->zp_mode;
2005 		oldva.va_uid = zp->z_phys->zp_uid;
2006 		oldva.va_gid = zp->z_phys->zp_gid;
2007 		mutex_exit(&zp->z_lock);
2008 
2009 		/*
2010 		 * If trim_mask is set then take ownership
2011 		 * has been granted.  In that case remove
2012 		 * UID|GID from mask so that
2013 		 * secpolicy_vnode_setattr() doesn't revoke it.
2014 		 */
2015 		if (trim_mask)
2016 			vap->va_mask &= ~(AT_UID|AT_GID);
2017 
2018 		err = secpolicy_vnode_setattr(cr, vp, vap, &oldva, flags,
2019 		    (int (*)(void *, int, cred_t *))zfs_zaccess_rwx, zp);
2020 		if (err) {
2021 			ZFS_EXIT(zfsvfs);
2022 			return (err);
2023 		}
2024 
2025 		if (trim_mask)
2026 			vap->va_mask |= (saved_mask & (AT_UID|AT_GID));
2027 	}
2028 
2029 	/*
2030 	 * secpolicy_vnode_setattr, or take ownership may have
2031 	 * changed va_mask
2032 	 */
2033 	mask = vap->va_mask;
2034 
2035 	tx = dmu_tx_create(zfsvfs->z_os);
2036 	dmu_tx_hold_bonus(tx, zp->z_id);
2037 
2038 	if (mask & AT_MODE) {
2039 
2040 		new_mode = (pzp->zp_mode & S_IFMT) | (vap->va_mode & ~S_IFMT);
2041 
2042 		if (zp->z_phys->zp_acl.z_acl_extern_obj)
2043 			dmu_tx_hold_write(tx,
2044 			    pzp->zp_acl.z_acl_extern_obj, 0, SPA_MAXBLOCKSIZE);
2045 		else
2046 			dmu_tx_hold_write(tx, DMU_NEW_OBJECT,
2047 			    0, ZFS_ACL_SIZE(MAX_ACL_SIZE));
2048 	}
2049 
2050 	if (mask & AT_SIZE) {
2051 		uint64_t off = vap->va_size;
2052 		/*
2053 		 * Grab the grow_lock to serialize this change with
2054 		 * respect to other file manipulations.
2055 		 */
2056 		rw_enter(&zp->z_grow_lock, RW_WRITER);
2057 		have_grow_lock = TRUE;
2058 		if (off < zp->z_phys->zp_size)
2059 			dmu_tx_hold_free(tx, zp->z_id, off, DMU_OBJECT_END);
2060 		else if (zp->z_phys->zp_size &&
2061 		    zp->z_blksz < zfsvfs->z_max_blksz && off > zp->z_blksz)
2062 			/* we will rewrite this block if we grow */
2063 			dmu_tx_hold_write(tx, zp->z_id, 0, zp->z_phys->zp_size);
2064 	}
2065 
2066 	if ((mask & (AT_UID | AT_GID)) && zp->z_phys->zp_xattr != 0) {
2067 		err = zfs_zget(zp->z_zfsvfs, zp->z_phys->zp_xattr, &attrzp);
2068 		if (err) {
2069 			dmu_tx_abort(tx);
2070 			if (have_grow_lock)
2071 				rw_exit(&zp->z_grow_lock);
2072 			ZFS_EXIT(zfsvfs);
2073 			return (err);
2074 		}
2075 		dmu_tx_hold_bonus(tx, attrzp->z_id);
2076 	}
2077 
2078 	err = dmu_tx_assign(tx, zfsvfs->z_assign);
2079 	if (err) {
2080 		if (attrzp)
2081 			VN_RELE(ZTOV(attrzp));
2082 		dmu_tx_abort(tx);
2083 		if (have_grow_lock)
2084 			rw_exit(&zp->z_grow_lock);
2085 		if (err == ERESTART && zfsvfs->z_assign == TXG_NOWAIT) {
2086 			txg_wait_open(dmu_objset_pool(zfsvfs->z_os), 0);
2087 			goto top;
2088 		}
2089 		ZFS_EXIT(zfsvfs);
2090 		return (err);
2091 	}
2092 
2093 	dmu_buf_will_dirty(zp->z_dbuf, tx);
2094 
2095 	/*
2096 	 * Set each attribute requested.
2097 	 * We group settings according to the locks they need to acquire.
2098 	 *
2099 	 * Note: you cannot set ctime directly, although it will be
2100 	 * updated as a side-effect of calling this function.
2101 	 */
2102 	if (mask & AT_SIZE) {
2103 		/*
2104 		 * XXX - Note, we are not providing any open
2105 		 * mode flags here (like FNDELAY), so we may
2106 		 * block if there are locks present... this
2107 		 * should be addressed in openat().
2108 		 */
2109 		err = zfs_freesp(zp, vap->va_size, 0, 0, tx, cr);
2110 		if (err) {
2111 			mutex_enter(&zp->z_lock);
2112 			goto out;
2113 		}
2114 		mask_applied |= AT_SIZE;
2115 	}
2116 
2117 	mask_applied = mask;	/* no errors after this point */
2118 
2119 	mutex_enter(&zp->z_lock);
2120 
2121 	if (mask & AT_MODE) {
2122 		err = zfs_acl_chmod_setattr(zp, new_mode, tx);
2123 		ASSERT3U(err, ==, 0);
2124 	}
2125 
2126 	if (attrzp)
2127 		mutex_enter(&attrzp->z_lock);
2128 
2129 	if (mask & AT_UID) {
2130 		zp->z_phys->zp_uid = (uint64_t)vap->va_uid;
2131 		if (attrzp) {
2132 			attrzp->z_phys->zp_uid = (uint64_t)vap->va_uid;
2133 		}
2134 	}
2135 
2136 	if (mask & AT_GID) {
2137 		zp->z_phys->zp_gid = (uint64_t)vap->va_gid;
2138 		if (attrzp)
2139 			attrzp->z_phys->zp_gid = (uint64_t)vap->va_gid;
2140 	}
2141 
2142 	if (attrzp)
2143 		mutex_exit(&attrzp->z_lock);
2144 
2145 	if (mask & AT_ATIME)
2146 		ZFS_TIME_ENCODE(&vap->va_atime, pzp->zp_atime);
2147 
2148 	if (mask & AT_MTIME)
2149 		ZFS_TIME_ENCODE(&vap->va_mtime, pzp->zp_mtime);
2150 
2151 	if (mask_applied & AT_SIZE)
2152 		zfs_time_stamper_locked(zp, CONTENT_MODIFIED, tx);
2153 	else if (mask_applied != 0)
2154 		zfs_time_stamper_locked(zp, STATE_CHANGED, tx);
2155 
2156 out:
2157 
2158 	if (mask_applied != 0)
2159 		seq = zfs_log_setattr(zilog, tx, TX_SETATTR, zp, vap,
2160 		    mask_applied);
2161 
2162 	mutex_exit(&zp->z_lock);
2163 
2164 	if (attrzp)
2165 		VN_RELE(ZTOV(attrzp));
2166 
2167 	if (have_grow_lock)
2168 		rw_exit(&zp->z_grow_lock);
2169 
2170 	dmu_tx_commit(tx);
2171 
2172 	zil_commit(zilog, seq, 0);
2173 
2174 	ZFS_EXIT(zfsvfs);
2175 	return (err);
2176 }
2177 
2178 /*
2179  * Search back through the directory tree, using the ".." entries.
2180  * Lock each directory in the chain to prevent concurrent renames.
2181  * Fail any attempt to move a directory into one of its own descendants.
2182  * XXX - z_parent_lock can overlap with map or grow locks
2183  */
2184 typedef struct zfs_zlock {
2185 	krwlock_t	*zl_rwlock;	/* lock we acquired */
2186 	znode_t		*zl_znode;	/* znode we held */
2187 	struct zfs_zlock *zl_next;	/* next in list */
2188 } zfs_zlock_t;
2189 
2190 static int
2191 zfs_rename_lock(znode_t *szp, znode_t *tdzp, znode_t *sdzp, zfs_zlock_t **zlpp)
2192 {
2193 	zfs_zlock_t	*zl;
2194 	znode_t 	*zp = tdzp;
2195 	uint64_t	rootid = zp->z_zfsvfs->z_root;
2196 	uint64_t	*oidp = &zp->z_id;
2197 	krwlock_t	*rwlp = &szp->z_parent_lock;
2198 	krw_t		rw = RW_WRITER;
2199 
2200 	/*
2201 	 * First pass write-locks szp and compares to zp->z_id.
2202 	 * Later passes read-lock zp and compare to zp->z_parent.
2203 	 */
2204 	do {
2205 		zl = kmem_alloc(sizeof (*zl), KM_SLEEP);
2206 		zl->zl_rwlock = rwlp;
2207 		zl->zl_znode = NULL;
2208 		zl->zl_next = *zlpp;
2209 		*zlpp = zl;
2210 
2211 		rw_enter(rwlp, rw);
2212 
2213 		if (*oidp == szp->z_id)		/* We're a descendant of szp */
2214 			return (EINVAL);
2215 
2216 		if (*oidp == rootid)		/* We've hit the top */
2217 			return (0);
2218 
2219 		if (rw == RW_READER) {		/* i.e. not the first pass */
2220 			int error = zfs_zget(zp->z_zfsvfs, *oidp, &zp);
2221 			if (error)
2222 				return (error);
2223 			zl->zl_znode = zp;
2224 		}
2225 		oidp = &zp->z_phys->zp_parent;
2226 		rwlp = &zp->z_parent_lock;
2227 		rw = RW_READER;
2228 
2229 	} while (zp->z_id != sdzp->z_id);
2230 
2231 	return (0);
2232 }
2233 
2234 /*
2235  * Drop locks and release vnodes that were held by zfs_rename_lock().
2236  */
2237 static void
2238 zfs_rename_unlock(zfs_zlock_t **zlpp)
2239 {
2240 	zfs_zlock_t *zl;
2241 
2242 	while ((zl = *zlpp) != NULL) {
2243 		if (zl->zl_znode != NULL)
2244 			VN_RELE(ZTOV(zl->zl_znode));
2245 		rw_exit(zl->zl_rwlock);
2246 		*zlpp = zl->zl_next;
2247 		kmem_free(zl, sizeof (*zl));
2248 	}
2249 }
2250 
2251 /*
2252  * Move an entry from the provided source directory to the target
2253  * directory.  Change the entry name as indicated.
2254  *
2255  *	IN:	sdvp	- Source directory containing the "old entry".
2256  *		snm	- Old entry name.
2257  *		tdvp	- Target directory to contain the "new entry".
2258  *		tnm	- New entry name.
2259  *		cr	- credentials of caller.
2260  *
2261  *	RETURN:	0 if success
2262  *		error code if failure
2263  *
2264  * Timestamps:
2265  *	sdvp,tdvp - ctime|mtime updated
2266  */
2267 static int
2268 zfs_rename(vnode_t *sdvp, char *snm, vnode_t *tdvp, char *tnm, cred_t *cr)
2269 {
2270 	znode_t		*tdzp, *szp, *tzp;
2271 	znode_t		*sdzp = VTOZ(sdvp);
2272 	zfsvfs_t	*zfsvfs = sdzp->z_zfsvfs;
2273 	zilog_t		*zilog = zfsvfs->z_log;
2274 	uint64_t	seq = 0;
2275 	vnode_t		*realvp;
2276 	zfs_dirlock_t	*sdl, *tdl;
2277 	dmu_tx_t	*tx;
2278 	zfs_zlock_t	*zl;
2279 	int		cmp, serr, terr, error;
2280 
2281 	ZFS_ENTER(zfsvfs);
2282 
2283 	/*
2284 	 * Make sure we have the real vp for the target directory.
2285 	 */
2286 	if (VOP_REALVP(tdvp, &realvp) == 0)
2287 		tdvp = realvp;
2288 
2289 	if (tdvp->v_vfsp != sdvp->v_vfsp) {
2290 		ZFS_EXIT(zfsvfs);
2291 		return (EXDEV);
2292 	}
2293 
2294 	tdzp = VTOZ(tdvp);
2295 top:
2296 	szp = NULL;
2297 	tzp = NULL;
2298 	zl = NULL;
2299 
2300 	/*
2301 	 * This is to prevent the creation of links into attribute space
2302 	 * by renaming a linked file into/outof an attribute directory.
2303 	 * See the comment in zfs_link() for why this is considered bad.
2304 	 */
2305 	if ((tdzp->z_phys->zp_flags & ZFS_XATTR) !=
2306 	    (sdzp->z_phys->zp_flags & ZFS_XATTR)) {
2307 		ZFS_EXIT(zfsvfs);
2308 		return (EINVAL);
2309 	}
2310 
2311 	/*
2312 	 * Lock source and target directory entries.  To prevent deadlock,
2313 	 * a lock ordering must be defined.  We lock the directory with
2314 	 * the smallest object id first, or if it's a tie, the one with
2315 	 * the lexically first name.
2316 	 */
2317 	if (sdzp->z_id < tdzp->z_id) {
2318 		cmp = -1;
2319 	} else if (sdzp->z_id > tdzp->z_id) {
2320 		cmp = 1;
2321 	} else {
2322 		cmp = strcmp(snm, tnm);
2323 		if (cmp == 0) {
2324 			/*
2325 			 * POSIX: "If the old argument and the new argument
2326 			 * both refer to links to the same existing file,
2327 			 * the rename() function shall return successfully
2328 			 * and perform no other action."
2329 			 */
2330 			ZFS_EXIT(zfsvfs);
2331 			return (0);
2332 		}
2333 	}
2334 	if (cmp < 0) {
2335 		serr = zfs_dirent_lock(&sdl, sdzp, snm, &szp, ZEXISTS);
2336 		terr = zfs_dirent_lock(&tdl, tdzp, tnm, &tzp, 0);
2337 	} else {
2338 		terr = zfs_dirent_lock(&tdl, tdzp, tnm, &tzp, 0);
2339 		serr = zfs_dirent_lock(&sdl, sdzp, snm, &szp, ZEXISTS);
2340 	}
2341 
2342 	if (serr) {
2343 		/*
2344 		 * Source entry invalid or not there.
2345 		 */
2346 		if (!terr) {
2347 			zfs_dirent_unlock(tdl);
2348 			if (tzp)
2349 				VN_RELE(ZTOV(tzp));
2350 		}
2351 		if (strcmp(snm, "..") == 0)
2352 			serr = EINVAL;
2353 		ZFS_EXIT(zfsvfs);
2354 		return (serr);
2355 	}
2356 	if (terr) {
2357 		zfs_dirent_unlock(sdl);
2358 		VN_RELE(ZTOV(szp));
2359 		if (strcmp(tnm, "..") == 0)
2360 			terr = EINVAL;
2361 		ZFS_EXIT(zfsvfs);
2362 		return (terr);
2363 	}
2364 
2365 	/*
2366 	 * Must have write access at the source to remove the old entry
2367 	 * and write access at the target to create the new entry.
2368 	 * Note that if target and source are the same, this can be
2369 	 * done in a single check.
2370 	 */
2371 
2372 	if (error = zfs_zaccess_rename(sdzp, szp, tdzp, tzp, cr))
2373 		goto out;
2374 
2375 	if (ZTOV(szp)->v_type == VDIR) {
2376 		/*
2377 		 * Check to make sure rename is valid.
2378 		 * Can't do a move like this: /usr/a/b to /usr/a/b/c/d
2379 		 */
2380 		if (error = zfs_rename_lock(szp, tdzp, sdzp, &zl))
2381 			goto out;
2382 	}
2383 
2384 	/*
2385 	 * Does target exist?
2386 	 */
2387 	if (tzp) {
2388 		/*
2389 		 * Source and target must be the same type.
2390 		 */
2391 		if (ZTOV(szp)->v_type == VDIR) {
2392 			if (ZTOV(tzp)->v_type != VDIR) {
2393 				error = ENOTDIR;
2394 				goto out;
2395 			}
2396 		} else {
2397 			if (ZTOV(tzp)->v_type == VDIR) {
2398 				error = EISDIR;
2399 				goto out;
2400 			}
2401 		}
2402 		/*
2403 		 * POSIX dictates that when the source and target
2404 		 * entries refer to the same file object, rename
2405 		 * must do nothing and exit without error.
2406 		 */
2407 		if (szp->z_id == tzp->z_id) {
2408 			error = 0;
2409 			goto out;
2410 		}
2411 	}
2412 
2413 	vnevent_rename_src(ZTOV(szp));
2414 	if (tzp)
2415 		vnevent_rename_dest(ZTOV(tzp));
2416 
2417 	tx = dmu_tx_create(zfsvfs->z_os);
2418 	dmu_tx_hold_bonus(tx, szp->z_id);	/* nlink changes */
2419 	dmu_tx_hold_bonus(tx, sdzp->z_id);	/* nlink changes */
2420 	if (sdzp != tdzp) {
2421 		dmu_tx_hold_zap(tx, sdzp->z_id, 1);
2422 		dmu_tx_hold_zap(tx, tdzp->z_id, 1);
2423 		dmu_tx_hold_bonus(tx, tdzp->z_id);	/* nlink changes */
2424 	} else {
2425 		dmu_tx_hold_zap(tx, sdzp->z_id, 2);
2426 	}
2427 	if (tzp) {
2428 		dmu_tx_hold_bonus(tx, tzp->z_id);	/* nlink changes */
2429 	}
2430 	dmu_tx_hold_zap(tx, zfsvfs->z_dqueue, 1);
2431 	error = dmu_tx_assign(tx, zfsvfs->z_assign);
2432 	if (error) {
2433 		dmu_tx_abort(tx);
2434 		if (zl != NULL)
2435 			zfs_rename_unlock(&zl);
2436 		zfs_dirent_unlock(sdl);
2437 		zfs_dirent_unlock(tdl);
2438 		VN_RELE(ZTOV(szp));
2439 		if (tzp)
2440 			VN_RELE(ZTOV(tzp));
2441 		if (error == ERESTART && zfsvfs->z_assign == TXG_NOWAIT) {
2442 			txg_wait_open(dmu_objset_pool(zfsvfs->z_os), 0);
2443 			goto top;
2444 		}
2445 		ZFS_EXIT(zfsvfs);
2446 		return (error);
2447 	}
2448 
2449 	if (tzp)	/* Attempt to remove the existing target */
2450 		error = zfs_link_destroy(tdl, tzp, tx, 0, NULL);
2451 
2452 	if (error == 0) {
2453 		error = zfs_link_create(tdl, szp, tx, ZRENAMING);
2454 		if (error == 0) {
2455 			error = zfs_link_destroy(sdl, szp, tx, ZRENAMING, NULL);
2456 			ASSERT(error == 0);
2457 			seq = zfs_log_rename(zilog, tx, TX_RENAME,
2458 			    sdzp, sdl->dl_name, tdzp, tdl->dl_name, szp);
2459 		}
2460 	}
2461 
2462 	dmu_tx_commit(tx);
2463 out:
2464 	if (zl != NULL)
2465 		zfs_rename_unlock(&zl);
2466 
2467 	zfs_dirent_unlock(sdl);
2468 	zfs_dirent_unlock(tdl);
2469 
2470 	VN_RELE(ZTOV(szp));
2471 	if (tzp)
2472 		VN_RELE(ZTOV(tzp));
2473 
2474 	zil_commit(zilog, seq, 0);
2475 
2476 	ZFS_EXIT(zfsvfs);
2477 	return (error);
2478 }
2479 
2480 /*
2481  * Insert the indicated symbolic reference entry into the directory.
2482  *
2483  *	IN:	dvp	- Directory to contain new symbolic link.
2484  *		link	- Name for new symlink entry.
2485  *		vap	- Attributes of new entry.
2486  *		target	- Target path of new symlink.
2487  *		cr	- credentials of caller.
2488  *
2489  *	RETURN:	0 if success
2490  *		error code if failure
2491  *
2492  * Timestamps:
2493  *	dvp - ctime|mtime updated
2494  */
2495 static int
2496 zfs_symlink(vnode_t *dvp, char *name, vattr_t *vap, char *link, cred_t *cr)
2497 {
2498 	znode_t		*zp, *dzp = VTOZ(dvp);
2499 	zfs_dirlock_t	*dl;
2500 	dmu_tx_t	*tx;
2501 	zfsvfs_t	*zfsvfs = dzp->z_zfsvfs;
2502 	zilog_t		*zilog = zfsvfs->z_log;
2503 	uint64_t	seq = 0;
2504 	uint64_t	zoid;
2505 	int		len = strlen(link);
2506 	int		error;
2507 
2508 	ASSERT(vap->va_type == VLNK);
2509 
2510 	ZFS_ENTER(zfsvfs);
2511 top:
2512 	if (error = zfs_zaccess(dzp, ACE_ADD_FILE, cr)) {
2513 		ZFS_EXIT(zfsvfs);
2514 		return (error);
2515 	}
2516 
2517 	if (len > MAXPATHLEN) {
2518 		ZFS_EXIT(zfsvfs);
2519 		return (ENAMETOOLONG);
2520 	}
2521 
2522 	/*
2523 	 * Attempt to lock directory; fail if entry already exists.
2524 	 */
2525 	if (error = zfs_dirent_lock(&dl, dzp, name, &zp, ZNEW)) {
2526 		ZFS_EXIT(zfsvfs);
2527 		return (error);
2528 	}
2529 
2530 	tx = dmu_tx_create(zfsvfs->z_os);
2531 	dmu_tx_hold_write(tx, DMU_NEW_OBJECT, 0, MAX(1, len));
2532 	dmu_tx_hold_bonus(tx, dzp->z_id);
2533 	dmu_tx_hold_zap(tx, dzp->z_id, 1);
2534 	if (dzp->z_phys->zp_flags & ZFS_INHERIT_ACE)
2535 		dmu_tx_hold_write(tx, DMU_NEW_OBJECT, 0, SPA_MAXBLOCKSIZE);
2536 	error = dmu_tx_assign(tx, zfsvfs->z_assign);
2537 	if (error) {
2538 		dmu_tx_abort(tx);
2539 		zfs_dirent_unlock(dl);
2540 		if (error == ERESTART && zfsvfs->z_assign == TXG_NOWAIT) {
2541 			txg_wait_open(dmu_objset_pool(zfsvfs->z_os), 0);
2542 			goto top;
2543 		}
2544 		ZFS_EXIT(zfsvfs);
2545 		return (error);
2546 	}
2547 
2548 	dmu_buf_will_dirty(dzp->z_dbuf, tx);
2549 
2550 	/*
2551 	 * Create a new object for the symlink.
2552 	 * Put the link content into bonus buffer if it will fit;
2553 	 * otherwise, store it just like any other file data.
2554 	 */
2555 	zoid = 0;
2556 	if (sizeof (znode_phys_t) + len <= dmu_bonus_max()) {
2557 		zfs_mknode(dzp, vap, &zoid, tx, cr, 0, &zp, len);
2558 		if (len != 0)
2559 			bcopy(link, zp->z_phys + 1, len);
2560 	} else {
2561 		dmu_buf_t *dbp;
2562 		zfs_mknode(dzp, vap, &zoid, tx, cr, 0, &zp, 0);
2563 
2564 		rw_enter(&zp->z_grow_lock, RW_WRITER);
2565 		error = zfs_grow_blocksize(zp, len, tx);
2566 		rw_exit(&zp->z_grow_lock);
2567 		if (error)
2568 			goto out;
2569 
2570 		dbp = dmu_buf_hold(zfsvfs->z_os, zoid, 0);
2571 		dmu_buf_will_dirty(dbp, tx);
2572 
2573 		ASSERT3U(len, <=, dbp->db_size);
2574 		bcopy(link, dbp->db_data, len);
2575 		dmu_buf_rele(dbp);
2576 	}
2577 	zp->z_phys->zp_size = len;
2578 
2579 	/*
2580 	 * Insert the new object into the directory.
2581 	 */
2582 	(void) zfs_link_create(dl, zp, tx, ZNEW);
2583 out:
2584 	if (error == 0)
2585 		seq = zfs_log_symlink(zilog, tx, TX_SYMLINK,
2586 		    dzp, zp, name, link);
2587 
2588 	dmu_tx_commit(tx);
2589 
2590 	zfs_dirent_unlock(dl);
2591 
2592 	VN_RELE(ZTOV(zp));
2593 
2594 	zil_commit(zilog, seq, 0);
2595 
2596 	ZFS_EXIT(zfsvfs);
2597 	return (error);
2598 }
2599 
2600 /*
2601  * Return, in the buffer contained in the provided uio structure,
2602  * the symbolic path referred to by vp.
2603  *
2604  *	IN:	vp	- vnode of symbolic link.
2605  *		uoip	- structure to contain the link path.
2606  *		cr	- credentials of caller.
2607  *
2608  *	OUT:	uio	- structure to contain the link path.
2609  *
2610  *	RETURN:	0 if success
2611  *		error code if failure
2612  *
2613  * Timestamps:
2614  *	vp - atime updated
2615  */
2616 /* ARGSUSED */
2617 static int
2618 zfs_readlink(vnode_t *vp, uio_t *uio, cred_t *cr)
2619 {
2620 	znode_t		*zp = VTOZ(vp);
2621 	zfsvfs_t	*zfsvfs = zp->z_zfsvfs;
2622 	size_t		bufsz;
2623 	int		error;
2624 
2625 	ZFS_ENTER(zfsvfs);
2626 
2627 	bufsz = (size_t)zp->z_phys->zp_size;
2628 	if (bufsz + sizeof (znode_phys_t) <= zp->z_dbuf->db_size) {
2629 		error = uiomove(zp->z_phys + 1,
2630 		    MIN((size_t)bufsz, uio->uio_resid), UIO_READ, uio);
2631 	} else {
2632 		dmu_buf_t *dbp = dmu_buf_hold(zfsvfs->z_os, zp->z_id, 0);
2633 		if ((error = dmu_buf_read_canfail(dbp)) != 0) {
2634 			dmu_buf_rele(dbp);
2635 			ZFS_EXIT(zfsvfs);
2636 			return (error);
2637 		}
2638 		error = uiomove(dbp->db_data,
2639 		    MIN((size_t)bufsz, uio->uio_resid), UIO_READ, uio);
2640 		dmu_buf_rele(dbp);
2641 	}
2642 
2643 	ZFS_ACCESSTIME_STAMP(zfsvfs, zp);
2644 	ZFS_EXIT(zfsvfs);
2645 	return (error);
2646 }
2647 
2648 /*
2649  * Insert a new entry into directory tdvp referencing svp.
2650  *
2651  *	IN:	tdvp	- Directory to contain new entry.
2652  *		svp	- vnode of new entry.
2653  *		name	- name of new entry.
2654  *		cr	- credentials of caller.
2655  *
2656  *	RETURN:	0 if success
2657  *		error code if failure
2658  *
2659  * Timestamps:
2660  *	tdvp - ctime|mtime updated
2661  *	 svp - ctime updated
2662  */
2663 /* ARGSUSED */
2664 static int
2665 zfs_link(vnode_t *tdvp, vnode_t *svp, char *name, cred_t *cr)
2666 {
2667 	znode_t		*dzp = VTOZ(tdvp);
2668 	znode_t		*tzp, *szp;
2669 	zfsvfs_t	*zfsvfs = dzp->z_zfsvfs;
2670 	zilog_t		*zilog = zfsvfs->z_log;
2671 	uint64_t	seq = 0;
2672 	zfs_dirlock_t	*dl;
2673 	dmu_tx_t	*tx;
2674 	vnode_t		*realvp;
2675 	int		error;
2676 
2677 	ASSERT(tdvp->v_type == VDIR);
2678 
2679 	ZFS_ENTER(zfsvfs);
2680 
2681 	if (VOP_REALVP(svp, &realvp) == 0)
2682 		svp = realvp;
2683 
2684 	if (svp->v_vfsp != tdvp->v_vfsp) {
2685 		ZFS_EXIT(zfsvfs);
2686 		return (EXDEV);
2687 	}
2688 
2689 	szp = VTOZ(svp);
2690 top:
2691 	/*
2692 	 * We do not support links between attributes and non-attributes
2693 	 * because of the potential security risk of creating links
2694 	 * into "normal" file space in order to circumvent restrictions
2695 	 * imposed in attribute space.
2696 	 */
2697 	if ((szp->z_phys->zp_flags & ZFS_XATTR) !=
2698 	    (dzp->z_phys->zp_flags & ZFS_XATTR)) {
2699 		ZFS_EXIT(zfsvfs);
2700 		return (EINVAL);
2701 	}
2702 
2703 	/*
2704 	 * POSIX dictates that we return EPERM here.
2705 	 * Better choices include ENOTSUP or EISDIR.
2706 	 */
2707 	if (svp->v_type == VDIR) {
2708 		ZFS_EXIT(zfsvfs);
2709 		return (EPERM);
2710 	}
2711 
2712 	if ((uid_t)szp->z_phys->zp_uid != crgetuid(cr) &&
2713 	    secpolicy_basic_link(cr) != 0) {
2714 		ZFS_EXIT(zfsvfs);
2715 		return (EPERM);
2716 	}
2717 
2718 	if (error = zfs_zaccess(dzp, ACE_ADD_FILE, cr)) {
2719 		ZFS_EXIT(zfsvfs);
2720 		return (error);
2721 	}
2722 
2723 	/*
2724 	 * Attempt to lock directory; fail if entry already exists.
2725 	 */
2726 	if (error = zfs_dirent_lock(&dl, dzp, name, &tzp, ZNEW)) {
2727 		ZFS_EXIT(zfsvfs);
2728 		return (error);
2729 	}
2730 
2731 	tx = dmu_tx_create(zfsvfs->z_os);
2732 	dmu_tx_hold_bonus(tx, szp->z_id);
2733 	dmu_tx_hold_zap(tx, dzp->z_id, 1);
2734 	error = dmu_tx_assign(tx, zfsvfs->z_assign);
2735 	if (error) {
2736 		dmu_tx_abort(tx);
2737 		zfs_dirent_unlock(dl);
2738 		if (error == ERESTART && zfsvfs->z_assign == TXG_NOWAIT) {
2739 			txg_wait_open(dmu_objset_pool(zfsvfs->z_os), 0);
2740 			goto top;
2741 		}
2742 		ZFS_EXIT(zfsvfs);
2743 		return (error);
2744 	}
2745 
2746 	error = zfs_link_create(dl, szp, tx, 0);
2747 
2748 	if (error == 0)
2749 		seq = zfs_log_link(zilog, tx, TX_LINK, dzp, szp, name);
2750 
2751 	dmu_tx_commit(tx);
2752 
2753 	zfs_dirent_unlock(dl);
2754 
2755 	zil_commit(zilog, seq, 0);
2756 
2757 	ZFS_EXIT(zfsvfs);
2758 	return (error);
2759 }
2760 
2761 /*
2762  * zfs_null_putapage() is used when the file system has been force
2763  * unmounted. It just drops the pages.
2764  */
2765 /* ARGSUSED */
2766 static int
2767 zfs_null_putapage(vnode_t *vp, page_t *pp, u_offset_t *offp,
2768 		size_t *lenp, int flags, cred_t *cr)
2769 {
2770 	pvn_write_done(pp, B_INVAL|B_FORCE|B_ERROR);
2771 	return (0);
2772 }
2773 
2774 /* ARGSUSED */
2775 static int
2776 zfs_putapage(vnode_t *vp, page_t *pp, u_offset_t *offp,
2777 		size_t *lenp, int flags, cred_t *cr)
2778 {
2779 	znode_t		*zp = VTOZ(vp);
2780 	zfsvfs_t	*zfsvfs = zp->z_zfsvfs;
2781 	zilog_t		*zilog = zfsvfs->z_log;
2782 	dmu_tx_t	*tx;
2783 	u_offset_t	off;
2784 	ssize_t		len;
2785 	caddr_t		va;
2786 	int		err;
2787 
2788 top:
2789 	rw_enter(&zp->z_grow_lock, RW_READER);
2790 
2791 	off = pp->p_offset;
2792 	len = MIN(PAGESIZE, zp->z_phys->zp_size - off);
2793 
2794 	tx = dmu_tx_create(zfsvfs->z_os);
2795 	dmu_tx_hold_write(tx, zp->z_id, off, len);
2796 	dmu_tx_hold_bonus(tx, zp->z_id);
2797 	err = dmu_tx_assign(tx, zfsvfs->z_assign);
2798 	if (err != 0) {
2799 		dmu_tx_abort(tx);
2800 		rw_exit(&zp->z_grow_lock);
2801 		if (err == ERESTART && zfsvfs->z_assign == TXG_NOWAIT) {
2802 			txg_wait_open(dmu_objset_pool(zfsvfs->z_os), 0);
2803 			goto top;
2804 		}
2805 		goto out;
2806 	}
2807 
2808 	va = ppmapin(pp, PROT_READ | PROT_WRITE, (caddr_t)-1);
2809 
2810 	dmu_write(zfsvfs->z_os, zp->z_id, off, len, va, tx);
2811 
2812 	ppmapout(va);
2813 
2814 	zfs_time_stamper(zp, CONTENT_MODIFIED, tx);
2815 	(void) zfs_log_write(zilog, tx, TX_WRITE, zp, off, len, 0, NULL);
2816 	dmu_tx_commit(tx);
2817 
2818 	rw_exit(&zp->z_grow_lock);
2819 
2820 	pvn_write_done(pp, B_WRITE | flags);
2821 	if (offp)
2822 		*offp = off;
2823 	if (lenp)
2824 		*lenp = len;
2825 
2826 out:
2827 	return (err);
2828 }
2829 
2830 /*
2831  * Copy the portion of the file indicated from pages into the file.
2832  * The pages are stored in a page list attached to the files vnode.
2833  *
2834  *	IN:	vp	- vnode of file to push page data to.
2835  *		off	- position in file to put data.
2836  *		len	- amount of data to write.
2837  *		flags	- flags to control the operation.
2838  *		cr	- credentials of caller.
2839  *
2840  *	RETURN:	0 if success
2841  *		error code if failure
2842  *
2843  * Timestamps:
2844  *	vp - ctime|mtime updated
2845  */
2846 static int
2847 zfs_putpage(vnode_t *vp, offset_t off, size_t len, int flags, cred_t *cr)
2848 {
2849 	znode_t		*zp = VTOZ(vp);
2850 	zfsvfs_t	*zfsvfs = zp->z_zfsvfs;
2851 	page_t		*pp;
2852 	size_t		io_len;
2853 	u_offset_t	io_off;
2854 	int		error = 0;
2855 
2856 	ZFS_ENTER(zfsvfs);
2857 
2858 	ASSERT(zp->z_dbuf_held && zp->z_phys);
2859 
2860 	if (len == 0) {
2861 		/*
2862 		 * Search the entire vp list for pages >= off.
2863 		 */
2864 		error = pvn_vplist_dirty(vp, (u_offset_t)off, zfs_putapage,
2865 		    flags, cr);
2866 		goto out;
2867 	}
2868 
2869 	if (off > zp->z_phys->zp_size) {
2870 		/* past end of file */
2871 		ZFS_EXIT(zfsvfs);
2872 		return (0);
2873 	}
2874 
2875 	len = MIN(len, zp->z_phys->zp_size - off);
2876 
2877 	for (io_off = off; io_off < off + len; io_off += io_len) {
2878 		if ((flags & B_INVAL) || ((flags & B_ASYNC) == 0)) {
2879 			pp  = page_lookup(vp, io_off,
2880 				(flags & (B_INVAL | B_FREE)) ?
2881 					SE_EXCL : SE_SHARED);
2882 		} else {
2883 			pp = page_lookup_nowait(vp, io_off,
2884 				(flags & B_FREE) ? SE_EXCL : SE_SHARED);
2885 		}
2886 
2887 		if (pp != NULL && pvn_getdirty(pp, flags)) {
2888 			int err;
2889 
2890 			/*
2891 			 * Found a dirty page to push
2892 			 */
2893 			if (err =
2894 			    zfs_putapage(vp, pp, &io_off, &io_len, flags, cr))
2895 				error = err;
2896 		} else {
2897 			io_len = PAGESIZE;
2898 		}
2899 	}
2900 out:
2901 	zil_commit(zfsvfs->z_log, UINT64_MAX, (flags & B_ASYNC) ? 0 : FDSYNC);
2902 	ZFS_EXIT(zfsvfs);
2903 	return (error);
2904 }
2905 
2906 void
2907 zfs_inactive(vnode_t *vp, cred_t *cr)
2908 {
2909 	znode_t	*zp = VTOZ(vp);
2910 	zfsvfs_t *zfsvfs = zp->z_zfsvfs;
2911 	int error;
2912 
2913 	rw_enter(&zfsvfs->z_um_lock, RW_READER);
2914 	if (zfsvfs->z_unmounted2) {
2915 		ASSERT(zp->z_dbuf_held == 0);
2916 
2917 		if (vn_has_cached_data(vp)) {
2918 			(void) pvn_vplist_dirty(vp, 0, zfs_null_putapage,
2919 			    B_INVAL, cr);
2920 		}
2921 
2922 		vp->v_count = 0; /* count arrives as 1 */
2923 		zfs_znode_free(zp);
2924 		rw_exit(&zfsvfs->z_um_lock);
2925 		VFS_RELE(zfsvfs->z_vfs);
2926 		return;
2927 	}
2928 
2929 	/*
2930 	 * Attempt to push any data in the page cache.  If this fails
2931 	 * we will get kicked out later in zfs_zinactive().
2932 	 */
2933 	if (vn_has_cached_data(vp)) {
2934 		(void) pvn_vplist_dirty(vp, 0, zfs_putapage, B_INVAL|B_ASYNC,
2935 		    cr);
2936 	}
2937 
2938 	if (zp->z_atime_dirty && zp->z_reap == 0) {
2939 		dmu_tx_t *tx = dmu_tx_create(zfsvfs->z_os);
2940 
2941 		dmu_tx_hold_bonus(tx, zp->z_id);
2942 		error = dmu_tx_assign(tx, TXG_WAIT);
2943 		if (error) {
2944 			dmu_tx_abort(tx);
2945 		} else {
2946 			dmu_buf_will_dirty(zp->z_dbuf, tx);
2947 			mutex_enter(&zp->z_lock);
2948 			zp->z_atime_dirty = 0;
2949 			mutex_exit(&zp->z_lock);
2950 			dmu_tx_commit(tx);
2951 		}
2952 	}
2953 
2954 	zfs_zinactive(zp);
2955 	rw_exit(&zfsvfs->z_um_lock);
2956 }
2957 
2958 /*
2959  * Bounds-check the seek operation.
2960  *
2961  *	IN:	vp	- vnode seeking within
2962  *		ooff	- old file offset
2963  *		noffp	- pointer to new file offset
2964  *
2965  *	RETURN:	0 if success
2966  *		EINVAL if new offset invalid
2967  */
2968 /* ARGSUSED */
2969 static int
2970 zfs_seek(vnode_t *vp, offset_t ooff, offset_t *noffp)
2971 {
2972 	if (vp->v_type == VDIR)
2973 		return (0);
2974 	return ((*noffp < 0 || *noffp > MAXOFFSET_T) ? EINVAL : 0);
2975 }
2976 
2977 /*
2978  * Pre-filter the generic locking function to trap attempts to place
2979  * a mandatory lock on a memory mapped file.
2980  */
2981 static int
2982 zfs_frlock(vnode_t *vp, int cmd, flock64_t *bfp, int flag, offset_t offset,
2983     flk_callback_t *flk_cbp, cred_t *cr)
2984 {
2985 	znode_t *zp = VTOZ(vp);
2986 	zfsvfs_t *zfsvfs = zp->z_zfsvfs;
2987 	uint_t cnt = 1;
2988 	int error;
2989 
2990 	ZFS_ENTER(zfsvfs);
2991 
2992 	/*
2993 	 * If file is being mapped, disallow frlock.  We set the mapcnt to
2994 	 * -1 here to signal that we are in the process of setting a lock.
2995 	 * This prevents a race with zfs_map().
2996 	 * XXX - well, sort of; since zfs_map() does not change z_mapcnt,
2997 	 * we could be in the middle of zfs_map() and still call fs_frlock().
2998 	 * Also, we are doing no checking in zfs_addmap() (where z_mapcnt
2999 	 * *is* manipulated).
3000 	 */
3001 	if (MANDMODE((mode_t)zp->z_phys->zp_mode) &&
3002 	    (int)(cnt = atomic_cas_32(&zp->z_mapcnt, 0, -1)) > 0) {
3003 		ZFS_EXIT(zfsvfs);
3004 		return (EAGAIN);
3005 	}
3006 	error = fs_frlock(vp, cmd, bfp, flag, offset, flk_cbp, cr);
3007 	ASSERT((cnt != 0) || ((int)atomic_cas_32(&zp->z_mapcnt, -1, 0) == -1));
3008 	ZFS_EXIT(zfsvfs);
3009 	return (error);
3010 }
3011 
3012 /*
3013  * If we can't find a page in the cache, we will create a new page
3014  * and fill it with file data.  For efficiency, we may try to fill
3015  * multiple pages as once (klustering).
3016  */
3017 static int
3018 zfs_fillpage(vnode_t *vp, u_offset_t off, struct seg *seg,
3019     caddr_t addr, page_t *pl[], size_t plsz, enum seg_rw rw)
3020 {
3021 	znode_t *zp = VTOZ(vp);
3022 	page_t *pp, *cur_pp;
3023 	objset_t *os = zp->z_zfsvfs->z_os;
3024 	caddr_t va;
3025 	u_offset_t io_off, total;
3026 	uint64_t oid = zp->z_id;
3027 	size_t io_len;
3028 	int err;
3029 
3030 	/*
3031 	 * If we are only asking for a single page don't bother klustering.
3032 	 */
3033 	if (plsz == PAGESIZE || zp->z_blksz <= PAGESIZE ||
3034 	    off > zp->z_phys->zp_size) {
3035 		io_off = off;
3036 		io_len = PAGESIZE;
3037 		pp = page_create_va(vp, io_off, io_len, PG_WAIT, seg, addr);
3038 	} else {
3039 		/*
3040 		 * Try to fill a kluster of pages (a blocks worth).
3041 		 */
3042 		size_t klen;
3043 		u_offset_t koff;
3044 
3045 		if (!ISP2(zp->z_blksz)) {
3046 			/* Only one block in the file. */
3047 			klen = P2ROUNDUP((ulong_t)zp->z_blksz, PAGESIZE);
3048 			koff = 0;
3049 		} else {
3050 			klen = plsz;
3051 			koff = P2ALIGN(off, (u_offset_t)klen);
3052 		}
3053 		if (klen > zp->z_phys->zp_size)
3054 			klen = P2ROUNDUP(zp->z_phys->zp_size,
3055 			    (uint64_t)PAGESIZE);
3056 		pp = pvn_read_kluster(vp, off, seg, addr, &io_off,
3057 			    &io_len, koff, klen, 0);
3058 	}
3059 	if (pp == NULL) {
3060 		/*
3061 		 * Some other thread entered the page before us.
3062 		 * Return to zfs_getpage to retry the lookup.
3063 		 */
3064 		*pl = NULL;
3065 		return (0);
3066 	}
3067 
3068 	/*
3069 	 * Fill the pages in the kluster.
3070 	 */
3071 	cur_pp = pp;
3072 	for (total = io_off + io_len; io_off < total; io_off += PAGESIZE) {
3073 		ASSERT(io_off == cur_pp->p_offset);
3074 		va = ppmapin(cur_pp, PROT_READ | PROT_WRITE, (caddr_t)-1);
3075 		err = dmu_read_canfail(os, oid, io_off, PAGESIZE, va);
3076 		ppmapout(va);
3077 		if (err) {
3078 			/* On error, toss the entire kluster */
3079 			pvn_read_done(pp, B_ERROR);
3080 			return (err);
3081 		}
3082 		cur_pp = cur_pp->p_next;
3083 	}
3084 out:
3085 	/*
3086 	 * Fill in the page list array from the kluster.  If
3087 	 * there are too many pages in the kluster, return
3088 	 * as many pages as possible starting from the desired
3089 	 * offset `off'.
3090 	 * NOTE: the page list will always be null terminated.
3091 	 */
3092 	pvn_plist_init(pp, pl, plsz, off, io_len, rw);
3093 
3094 	return (0);
3095 }
3096 
3097 /*
3098  * Return pointers to the pages for the file region [off, off + len]
3099  * in the pl array.  If plsz is greater than len, this function may
3100  * also return page pointers from before or after the specified
3101  * region (i.e. some region [off', off' + plsz]).  These additional
3102  * pages are only returned if they are already in the cache, or were
3103  * created as part of a klustered read.
3104  *
3105  *	IN:	vp	- vnode of file to get data from.
3106  *		off	- position in file to get data from.
3107  *		len	- amount of data to retrieve.
3108  *		plsz	- length of provided page list.
3109  *		seg	- segment to obtain pages for.
3110  *		addr	- virtual address of fault.
3111  *		rw	- mode of created pages.
3112  *		cr	- credentials of caller.
3113  *
3114  *	OUT:	protp	- protection mode of created pages.
3115  *		pl	- list of pages created.
3116  *
3117  *	RETURN:	0 if success
3118  *		error code if failure
3119  *
3120  * Timestamps:
3121  *	vp - atime updated
3122  */
3123 /* ARGSUSED */
3124 static int
3125 zfs_getpage(vnode_t *vp, offset_t off, size_t len, uint_t *protp,
3126 	page_t *pl[], size_t plsz, struct seg *seg, caddr_t addr,
3127 	enum seg_rw rw, cred_t *cr)
3128 {
3129 	znode_t		*zp = VTOZ(vp);
3130 	zfsvfs_t	*zfsvfs = zp->z_zfsvfs;
3131 	page_t		*pp, **pl0 = pl;
3132 	int		cnt = 0, need_unlock = 0, err = 0;
3133 
3134 	ZFS_ENTER(zfsvfs);
3135 
3136 	if (protp)
3137 		*protp = PROT_ALL;
3138 
3139 	ASSERT(zp->z_dbuf_held && zp->z_phys);
3140 
3141 	/* no faultahead (for now) */
3142 	if (pl == NULL) {
3143 		ZFS_EXIT(zfsvfs);
3144 		return (0);
3145 	}
3146 
3147 	/* can't fault past EOF */
3148 	if (off >= zp->z_phys->zp_size) {
3149 		ZFS_EXIT(zfsvfs);
3150 		return (EFAULT);
3151 	}
3152 
3153 	/*
3154 	 * Make sure nobody restructures the file (changes block size)
3155 	 * in the middle of the getpage.
3156 	 */
3157 	rw_enter(&zp->z_grow_lock, RW_READER);
3158 
3159 	/*
3160 	 * If we already own the lock, then we must be page faulting
3161 	 * in the middle of a write to this file (i.e., we are writing
3162 	 * to this file using data from a mapped region of the file).
3163 	 */
3164 	if (!rw_owner(&zp->z_map_lock)) {
3165 		rw_enter(&zp->z_map_lock, RW_WRITER);
3166 		need_unlock = TRUE;
3167 	}
3168 
3169 	/*
3170 	 * Loop through the requested range [off, off + len] looking
3171 	 * for pages.  If we don't find a page, we will need to create
3172 	 * a new page and fill it with data from the file.
3173 	 */
3174 	while (len > 0) {
3175 		if (plsz < PAGESIZE)
3176 			break;
3177 		if (pp = page_lookup(vp, off, SE_SHARED)) {
3178 			*pl++ = pp;
3179 			off += PAGESIZE;
3180 			addr += PAGESIZE;
3181 			len -= PAGESIZE;
3182 			plsz -= PAGESIZE;
3183 		} else {
3184 			err = zfs_fillpage(vp, off, seg, addr, pl, plsz, rw);
3185 			/*
3186 			 * klustering may have changed our region
3187 			 * to be block aligned.
3188 			 */
3189 			if (((pp = *pl) != 0) && (off != pp->p_offset)) {
3190 				int delta = off - pp->p_offset;
3191 				len += delta;
3192 				off -= delta;
3193 				addr -= delta;
3194 			}
3195 			while (*pl) {
3196 				pl++;
3197 				cnt++;
3198 				off += PAGESIZE;
3199 				addr += PAGESIZE;
3200 				plsz -= PAGESIZE;
3201 				if (len > PAGESIZE)
3202 					len -= PAGESIZE;
3203 				else
3204 					len = 0;
3205 			}
3206 		}
3207 		if (err)
3208 			goto out;
3209 	}
3210 
3211 	/*
3212 	 * Fill out the page array with any pages already in the cache.
3213 	 */
3214 	while (plsz > 0) {
3215 		pp = page_lookup_nowait(vp, off, SE_SHARED);
3216 		if (pp == NULL)
3217 			break;
3218 		*pl++ = pp;
3219 		off += PAGESIZE;
3220 		plsz -= PAGESIZE;
3221 	}
3222 
3223 	ZFS_ACCESSTIME_STAMP(zfsvfs, zp);
3224 out:
3225 	if (err) {
3226 		/*
3227 		 * Release any pages we have locked.
3228 		 */
3229 		while (pl > pl0)
3230 			page_unlock(*--pl);
3231 	}
3232 	*pl = NULL;
3233 
3234 	if (need_unlock)
3235 		rw_exit(&zp->z_map_lock);
3236 	rw_exit(&zp->z_grow_lock);
3237 
3238 	ZFS_EXIT(zfsvfs);
3239 	return (err);
3240 }
3241 
3242 static int
3243 zfs_map(vnode_t *vp, offset_t off, struct as *as, caddr_t *addrp,
3244     size_t len, uchar_t prot, uchar_t maxprot, uint_t flags, cred_t *cr)
3245 {
3246 	znode_t *zp = VTOZ(vp);
3247 	zfsvfs_t *zfsvfs = zp->z_zfsvfs;
3248 	segvn_crargs_t	vn_a;
3249 	int		error;
3250 
3251 	ZFS_ENTER(zfsvfs);
3252 
3253 	if (vp->v_flag & VNOMAP) {
3254 		ZFS_EXIT(zfsvfs);
3255 		return (ENOSYS);
3256 	}
3257 
3258 	if (off < 0 || len > MAXOFFSET_T - off) {
3259 		ZFS_EXIT(zfsvfs);
3260 		return (ENXIO);
3261 	}
3262 
3263 	if (vp->v_type != VREG) {
3264 		ZFS_EXIT(zfsvfs);
3265 		return (ENODEV);
3266 	}
3267 
3268 	/*
3269 	 * If file is locked, disallow mapping.
3270 	 * XXX - since we don't modify z_mapcnt here, there is nothing
3271 	 * to stop a file lock being placed immediately after we complete
3272 	 * this check.
3273 	 */
3274 	if (MANDMODE((mode_t)zp->z_phys->zp_mode)) {
3275 		if (vn_has_flocks(vp) || zp->z_mapcnt == -1) {
3276 			ZFS_EXIT(zfsvfs);
3277 			return (EAGAIN);
3278 		}
3279 	}
3280 
3281 	as_rangelock(as);
3282 	if ((flags & MAP_FIXED) == 0) {
3283 		map_addr(addrp, len, off, 1, flags);
3284 		if (*addrp == NULL) {
3285 			as_rangeunlock(as);
3286 			ZFS_EXIT(zfsvfs);
3287 			return (ENOMEM);
3288 		}
3289 	} else {
3290 		/*
3291 		 * User specified address - blow away any previous mappings
3292 		 */
3293 		(void) as_unmap(as, *addrp, len);
3294 	}
3295 
3296 	vn_a.vp = vp;
3297 	vn_a.offset = (u_offset_t)off;
3298 	vn_a.type = flags & MAP_TYPE;
3299 	vn_a.prot = prot;
3300 	vn_a.maxprot = maxprot;
3301 	vn_a.cred = cr;
3302 	vn_a.amp = NULL;
3303 	vn_a.flags = flags & ~MAP_TYPE;
3304 	vn_a.szc = 0;
3305 	vn_a.lgrp_mem_policy_flags = 0;
3306 
3307 	error = as_map(as, *addrp, len, segvn_create, &vn_a);
3308 
3309 	as_rangeunlock(as);
3310 	ZFS_EXIT(zfsvfs);
3311 	return (error);
3312 }
3313 
3314 /* ARGSUSED */
3315 static int
3316 zfs_addmap(vnode_t *vp, offset_t off, struct as *as, caddr_t addr,
3317     size_t len, uchar_t prot, uchar_t maxprot, uint_t flags, cred_t *cr)
3318 {
3319 	/*
3320 	 * XXX - shouldn't we be checking for file locks here?
3321 	 */
3322 	ASSERT3U(VTOZ(vp)->z_mapcnt, >=, 0);
3323 	atomic_add_32(&VTOZ(vp)->z_mapcnt, btopr(len));
3324 	return (0);
3325 }
3326 
3327 /* ARGSUSED */
3328 static int
3329 zfs_delmap(vnode_t *vp, offset_t off, struct as *as, caddr_t addr,
3330     size_t len, uint_t prot, uint_t maxprot, uint_t flags, cred_t *cr)
3331 {
3332 	atomic_add_32(&VTOZ(vp)->z_mapcnt, -btopr(len));
3333 	ASSERT3U(VTOZ(vp)->z_mapcnt, >=, 0);
3334 	return (0);
3335 }
3336 
3337 /*
3338  * Free or allocate space in a file.  Currently, this function only
3339  * supports the `F_FREESP' command.  However, this command is somewhat
3340  * misnamed, as its functionality includes the ability to allocate as
3341  * well as free space.
3342  *
3343  *	IN:	vp	- vnode of file to free data in.
3344  *		cmd	- action to take (only F_FREESP supported).
3345  *		bfp	- section of file to free/alloc.
3346  *		flag	- current file open mode flags.
3347  *		offset	- current file offset.
3348  *		cr	- credentials of caller [UNUSED].
3349  *
3350  *	RETURN:	0 if success
3351  *		error code if failure
3352  *
3353  * Timestamps:
3354  *	vp - ctime|mtime updated
3355  *
3356  * NOTE: This function is limited in that it will only permit space to
3357  *   be freed at the end of a file.  In essence, this function simply
3358  *   allows one to set the file size.
3359  */
3360 /* ARGSUSED */
3361 static int
3362 zfs_space(vnode_t *vp, int cmd, flock64_t *bfp, int flag,
3363     offset_t offset, cred_t *cr, caller_context_t *ct)
3364 {
3365 	dmu_tx_t	*tx;
3366 	znode_t		*zp = VTOZ(vp);
3367 	zfsvfs_t	*zfsvfs = zp->z_zfsvfs;
3368 	zilog_t		*zilog = zfsvfs->z_log;
3369 	uint64_t	seq = 0;
3370 	uint64_t	off, len;
3371 	int		error;
3372 
3373 	ZFS_ENTER(zfsvfs);
3374 
3375 top:
3376 	if (cmd != F_FREESP) {
3377 		ZFS_EXIT(zfsvfs);
3378 		return (EINVAL);
3379 	}
3380 
3381 	if (error = convoff(vp, bfp, 0, offset)) {
3382 		ZFS_EXIT(zfsvfs);
3383 		return (error);
3384 	}
3385 
3386 	if (bfp->l_len < 0) {
3387 		ZFS_EXIT(zfsvfs);
3388 		return (EINVAL);
3389 	}
3390 
3391 	off = bfp->l_start;
3392 	len = bfp->l_len;
3393 	tx = dmu_tx_create(zfsvfs->z_os);
3394 	/*
3395 	 * Grab the grow_lock to serialize this change with
3396 	 * respect to other file size changes.
3397 	 */
3398 	dmu_tx_hold_bonus(tx, zp->z_id);
3399 	rw_enter(&zp->z_grow_lock, RW_WRITER);
3400 	if (off + len > zp->z_blksz && zp->z_blksz < zfsvfs->z_max_blksz &&
3401 	    off >= zp->z_phys->zp_size) {
3402 		/*
3403 		 * We are increasing the length of the file,
3404 		 * and this may mean a block size increase.
3405 		 */
3406 		dmu_tx_hold_write(tx, zp->z_id, 0,
3407 		    MIN(off + len, zfsvfs->z_max_blksz));
3408 	} else if (off < zp->z_phys->zp_size) {
3409 		/*
3410 		 * If len == 0, we are truncating the file.
3411 		 */
3412 		dmu_tx_hold_free(tx, zp->z_id, off, len ? len : DMU_OBJECT_END);
3413 	}
3414 
3415 	error = dmu_tx_assign(tx, zfsvfs->z_assign);
3416 	if (error) {
3417 		dmu_tx_abort(tx);
3418 		rw_exit(&zp->z_grow_lock);
3419 		if (error == ERESTART && zfsvfs->z_assign == TXG_NOWAIT) {
3420 			txg_wait_open(dmu_objset_pool(zfsvfs->z_os), 0);
3421 			goto top;
3422 		}
3423 		ZFS_EXIT(zfsvfs);
3424 		return (error);
3425 	}
3426 
3427 	error = zfs_freesp(zp, off, len, flag, tx, cr);
3428 
3429 	if (error == 0) {
3430 		zfs_time_stamper(zp, CONTENT_MODIFIED, tx);
3431 		seq = zfs_log_truncate(zilog, tx, TX_TRUNCATE, zp, off, len);
3432 	}
3433 
3434 	rw_exit(&zp->z_grow_lock);
3435 
3436 	dmu_tx_commit(tx);
3437 
3438 	zil_commit(zilog, seq, 0);
3439 
3440 	ZFS_EXIT(zfsvfs);
3441 	return (error);
3442 }
3443 
3444 static int
3445 zfs_fid(vnode_t *vp, fid_t *fidp)
3446 {
3447 	znode_t		*zp = VTOZ(vp);
3448 	zfsvfs_t	*zfsvfs = zp->z_zfsvfs;
3449 	uint32_t	gen = (uint32_t)zp->z_phys->zp_gen;
3450 	uint64_t	object = zp->z_id;
3451 	zfid_short_t	*zfid;
3452 	int		size, i;
3453 
3454 	ZFS_ENTER(zfsvfs);
3455 
3456 	size = (zfsvfs->z_parent != zfsvfs) ? LONG_FID_LEN : SHORT_FID_LEN;
3457 	if (fidp->fid_len < size) {
3458 		fidp->fid_len = size;
3459 		return (ENOSPC);
3460 	}
3461 
3462 	zfid = (zfid_short_t *)fidp;
3463 
3464 	zfid->zf_len = size;
3465 
3466 	for (i = 0; i < sizeof (zfid->zf_object); i++)
3467 		zfid->zf_object[i] = (uint8_t)(object >> (8 * i));
3468 
3469 	/* Must have a non-zero generation number to distinguish from .zfs */
3470 	if (gen == 0)
3471 		gen = 1;
3472 	for (i = 0; i < sizeof (zfid->zf_gen); i++)
3473 		zfid->zf_gen[i] = (uint8_t)(gen >> (8 * i));
3474 
3475 	if (size == LONG_FID_LEN) {
3476 		uint64_t	objsetid = dmu_objset_id(zfsvfs->z_os);
3477 		zfid_long_t	*zlfid;
3478 
3479 		zlfid = (zfid_long_t *)fidp;
3480 
3481 		for (i = 0; i < sizeof (zlfid->zf_setid); i++)
3482 			zlfid->zf_setid[i] = (uint8_t)(objsetid >> (8 * i));
3483 
3484 		/* XXX - this should be the generation number for the objset */
3485 		for (i = 0; i < sizeof (zlfid->zf_setgen); i++)
3486 			zlfid->zf_setgen[i] = 0;
3487 	}
3488 
3489 	ZFS_EXIT(zfsvfs);
3490 	return (0);
3491 }
3492 
3493 static int
3494 zfs_pathconf(vnode_t *vp, int cmd, ulong_t *valp, cred_t *cr)
3495 {
3496 	znode_t		*zp, *xzp;
3497 	zfsvfs_t	*zfsvfs;
3498 	zfs_dirlock_t	*dl;
3499 	int		error;
3500 
3501 	switch (cmd) {
3502 	case _PC_LINK_MAX:
3503 		*valp = ULONG_MAX;
3504 		return (0);
3505 
3506 	case _PC_FILESIZEBITS:
3507 		*valp = 64;
3508 		return (0);
3509 
3510 	case _PC_XATTR_EXISTS:
3511 		zp = VTOZ(vp);
3512 		zfsvfs = zp->z_zfsvfs;
3513 		ZFS_ENTER(zfsvfs);
3514 		*valp = 0;
3515 		error = zfs_dirent_lock(&dl, zp, "", &xzp,
3516 		    ZXATTR | ZEXISTS | ZSHARED);
3517 		if (error == 0) {
3518 			zfs_dirent_unlock(dl);
3519 			if (!zfs_dirempty(xzp))
3520 				*valp = 1;
3521 			VN_RELE(ZTOV(xzp));
3522 		} else if (error == ENOENT) {
3523 			/*
3524 			 * If there aren't extended attributes, it's the
3525 			 * same as having zero of them.
3526 			 */
3527 			error = 0;
3528 		}
3529 		ZFS_EXIT(zfsvfs);
3530 		return (error);
3531 
3532 	case _PC_ACL_ENABLED:
3533 		*valp = _ACL_ACE_ENABLED;
3534 		return (0);
3535 
3536 	case _PC_MIN_HOLE_SIZE:
3537 		*valp = (ulong_t)SPA_MINBLOCKSIZE;
3538 		return (0);
3539 
3540 	default:
3541 		return (fs_pathconf(vp, cmd, valp, cr));
3542 	}
3543 }
3544 
3545 /*ARGSUSED*/
3546 static int
3547 zfs_getsecattr(vnode_t *vp, vsecattr_t *vsecp, int flag, cred_t *cr)
3548 {
3549 	znode_t *zp = VTOZ(vp);
3550 	zfsvfs_t *zfsvfs = zp->z_zfsvfs;
3551 	int error;
3552 
3553 	ZFS_ENTER(zfsvfs);
3554 	error = zfs_getacl(zp, vsecp, cr);
3555 	ZFS_EXIT(zfsvfs);
3556 
3557 	return (error);
3558 }
3559 
3560 /*ARGSUSED*/
3561 static int
3562 zfs_setsecattr(vnode_t *vp, vsecattr_t *vsecp, int flag, cred_t *cr)
3563 {
3564 	znode_t *zp = VTOZ(vp);
3565 	zfsvfs_t *zfsvfs = zp->z_zfsvfs;
3566 	int error;
3567 
3568 	ZFS_ENTER(zfsvfs);
3569 	error = zfs_setacl(zp, vsecp, cr);
3570 	ZFS_EXIT(zfsvfs);
3571 	return (error);
3572 }
3573 
3574 /*
3575  * Predeclare these here so that the compiler assumes that
3576  * this is an "old style" function declaration that does
3577  * not include arguments => we won't get type mismatch errors
3578  * in the initializations that follow.
3579  */
3580 static int zfs_inval();
3581 static int zfs_isdir();
3582 
3583 static int
3584 zfs_inval()
3585 {
3586 	return (EINVAL);
3587 }
3588 
3589 static int
3590 zfs_isdir()
3591 {
3592 	return (EISDIR);
3593 }
3594 /*
3595  * Directory vnode operations template
3596  */
3597 vnodeops_t *zfs_dvnodeops;
3598 const fs_operation_def_t zfs_dvnodeops_template[] = {
3599 	VOPNAME_OPEN, zfs_open,
3600 	VOPNAME_CLOSE, zfs_close,
3601 	VOPNAME_READ, zfs_isdir,
3602 	VOPNAME_WRITE, zfs_isdir,
3603 	VOPNAME_IOCTL, zfs_ioctl,
3604 	VOPNAME_GETATTR, zfs_getattr,
3605 	VOPNAME_SETATTR, zfs_setattr,
3606 	VOPNAME_ACCESS, zfs_access,
3607 	VOPNAME_LOOKUP, zfs_lookup,
3608 	VOPNAME_CREATE, zfs_create,
3609 	VOPNAME_REMOVE, zfs_remove,
3610 	VOPNAME_LINK, zfs_link,
3611 	VOPNAME_RENAME, zfs_rename,
3612 	VOPNAME_MKDIR, zfs_mkdir,
3613 	VOPNAME_RMDIR, zfs_rmdir,
3614 	VOPNAME_READDIR, zfs_readdir,
3615 	VOPNAME_SYMLINK, zfs_symlink,
3616 	VOPNAME_FSYNC, zfs_fsync,
3617 	VOPNAME_INACTIVE, (fs_generic_func_p) zfs_inactive,
3618 	VOPNAME_FID, zfs_fid,
3619 	VOPNAME_SEEK, zfs_seek,
3620 	VOPNAME_PATHCONF, zfs_pathconf,
3621 	VOPNAME_GETSECATTR, zfs_getsecattr,
3622 	VOPNAME_SETSECATTR, zfs_setsecattr,
3623 	NULL, NULL
3624 };
3625 
3626 /*
3627  * Regular file vnode operations template
3628  */
3629 vnodeops_t *zfs_fvnodeops;
3630 const fs_operation_def_t zfs_fvnodeops_template[] = {
3631 	VOPNAME_OPEN, zfs_open,
3632 	VOPNAME_CLOSE, zfs_close,
3633 	VOPNAME_READ, zfs_read,
3634 	VOPNAME_WRITE, zfs_write,
3635 	VOPNAME_IOCTL, zfs_ioctl,
3636 	VOPNAME_GETATTR, zfs_getattr,
3637 	VOPNAME_SETATTR, zfs_setattr,
3638 	VOPNAME_ACCESS, zfs_access,
3639 	VOPNAME_LOOKUP, zfs_lookup,
3640 	VOPNAME_RENAME, zfs_rename,
3641 	VOPNAME_FSYNC, zfs_fsync,
3642 	VOPNAME_INACTIVE, (fs_generic_func_p)zfs_inactive,
3643 	VOPNAME_FID, zfs_fid,
3644 	VOPNAME_SEEK, zfs_seek,
3645 	VOPNAME_FRLOCK, zfs_frlock,
3646 	VOPNAME_SPACE, zfs_space,
3647 	VOPNAME_GETPAGE, zfs_getpage,
3648 	VOPNAME_PUTPAGE, zfs_putpage,
3649 	VOPNAME_MAP, (fs_generic_func_p) zfs_map,
3650 	VOPNAME_ADDMAP, (fs_generic_func_p) zfs_addmap,
3651 	VOPNAME_DELMAP, zfs_delmap,
3652 	VOPNAME_PATHCONF, zfs_pathconf,
3653 	VOPNAME_GETSECATTR, zfs_getsecattr,
3654 	VOPNAME_SETSECATTR, zfs_setsecattr,
3655 	VOPNAME_VNEVENT, fs_vnevent_support,
3656 	NULL, NULL
3657 };
3658 
3659 /*
3660  * Symbolic link vnode operations template
3661  */
3662 vnodeops_t *zfs_symvnodeops;
3663 const fs_operation_def_t zfs_symvnodeops_template[] = {
3664 	VOPNAME_GETATTR, zfs_getattr,
3665 	VOPNAME_SETATTR, zfs_setattr,
3666 	VOPNAME_ACCESS, zfs_access,
3667 	VOPNAME_RENAME, zfs_rename,
3668 	VOPNAME_READLINK, zfs_readlink,
3669 	VOPNAME_INACTIVE, (fs_generic_func_p) zfs_inactive,
3670 	VOPNAME_FID, zfs_fid,
3671 	VOPNAME_PATHCONF, zfs_pathconf,
3672 	VOPNAME_VNEVENT, fs_vnevent_support,
3673 	NULL, NULL
3674 };
3675 
3676 /*
3677  * Extended attribute directory vnode operations template
3678  *	This template is identical to the directory vnodes
3679  *	operation template except for restricted operations:
3680  *		VOP_MKDIR()
3681  *		VOP_SYMLINK()
3682  * Note that there are other restrictions embedded in:
3683  *	zfs_create()	- restrict type to VREG
3684  *	zfs_link()	- no links into/out of attribute space
3685  *	zfs_rename()	- no moves into/out of attribute space
3686  */
3687 vnodeops_t *zfs_xdvnodeops;
3688 const fs_operation_def_t zfs_xdvnodeops_template[] = {
3689 	VOPNAME_OPEN, zfs_open,
3690 	VOPNAME_CLOSE, zfs_close,
3691 	VOPNAME_IOCTL, zfs_ioctl,
3692 	VOPNAME_GETATTR, zfs_getattr,
3693 	VOPNAME_SETATTR, zfs_setattr,
3694 	VOPNAME_ACCESS, zfs_access,
3695 	VOPNAME_LOOKUP, zfs_lookup,
3696 	VOPNAME_CREATE, zfs_create,
3697 	VOPNAME_REMOVE, zfs_remove,
3698 	VOPNAME_LINK, zfs_link,
3699 	VOPNAME_RENAME, zfs_rename,
3700 	VOPNAME_MKDIR, zfs_inval,
3701 	VOPNAME_RMDIR, zfs_rmdir,
3702 	VOPNAME_READDIR, zfs_readdir,
3703 	VOPNAME_SYMLINK, zfs_inval,
3704 	VOPNAME_FSYNC, zfs_fsync,
3705 	VOPNAME_INACTIVE, (fs_generic_func_p) zfs_inactive,
3706 	VOPNAME_FID, zfs_fid,
3707 	VOPNAME_SEEK, zfs_seek,
3708 	VOPNAME_PATHCONF, zfs_pathconf,
3709 	VOPNAME_GETSECATTR, zfs_getsecattr,
3710 	VOPNAME_SETSECATTR, zfs_setsecattr,
3711 	VOPNAME_VNEVENT, fs_vnevent_support,
3712 	NULL, NULL
3713 };
3714 
3715 /*
3716  * Error vnode operations template
3717  */
3718 vnodeops_t *zfs_evnodeops;
3719 const fs_operation_def_t zfs_evnodeops_template[] = {
3720 	VOPNAME_INACTIVE, (fs_generic_func_p) zfs_inactive,
3721 	VOPNAME_PATHCONF, zfs_pathconf,
3722 	NULL, NULL
3723 };
3724