xref: /illumos-gate/usr/src/uts/common/syscall/fcntl.c (revision 7e0e2549bfaa531aff576083ab0c07f84fa8fb27)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 
22 /*
23  * Copyright (c) 1994, 2010, Oracle and/or its affiliates. All rights reserved.
24  * Copyright (c) 2013, OmniTI Computer Consulting, Inc. All rights reserved.
25  */
26 
27 /*	Copyright (c) 1983, 1984, 1985, 1986, 1987, 1988, 1989 AT&T	*/
28 /*	  All Rights Reserved  	*/
29 
30 /*
31  * Portions of this source code were derived from Berkeley 4.3 BSD
32  * under license from the Regents of the University of California.
33  */
34 
35 
36 #include <sys/param.h>
37 #include <sys/isa_defs.h>
38 #include <sys/types.h>
39 #include <sys/sysmacros.h>
40 #include <sys/systm.h>
41 #include <sys/errno.h>
42 #include <sys/fcntl.h>
43 #include <sys/flock.h>
44 #include <sys/vnode.h>
45 #include <sys/file.h>
46 #include <sys/mode.h>
47 #include <sys/proc.h>
48 #include <sys/filio.h>
49 #include <sys/share.h>
50 #include <sys/debug.h>
51 #include <sys/rctl.h>
52 #include <sys/nbmlock.h>
53 
54 #include <sys/cmn_err.h>
55 
56 static int flock_check(vnode_t *, flock64_t *, offset_t, offset_t);
57 static int flock_get_start(vnode_t *, flock64_t *, offset_t, u_offset_t *);
58 static void fd_too_big(proc_t *);
59 
60 /*
61  * File control.
62  */
63 int
64 fcntl(int fdes, int cmd, intptr_t arg)
65 {
66 	int iarg;
67 	int error = 0;
68 	int retval;
69 	proc_t *p;
70 	file_t *fp;
71 	vnode_t *vp;
72 	u_offset_t offset;
73 	u_offset_t start;
74 	struct vattr vattr;
75 	int in_crit;
76 	int flag;
77 	struct flock sbf;
78 	struct flock64 bf;
79 	struct o_flock obf;
80 	struct flock64_32 bf64_32;
81 	struct fshare fsh;
82 	struct shrlock shr;
83 	struct shr_locowner shr_own;
84 	offset_t maxoffset;
85 	model_t datamodel;
86 	int fdres;
87 
88 #if defined(_ILP32) && !defined(lint) && defined(_SYSCALL32)
89 	ASSERT(sizeof (struct flock) == sizeof (struct flock32));
90 	ASSERT(sizeof (struct flock64) == sizeof (struct flock64_32));
91 #endif
92 #if defined(_LP64) && !defined(lint) && defined(_SYSCALL32)
93 	ASSERT(sizeof (struct flock) == sizeof (struct flock64_64));
94 	ASSERT(sizeof (struct flock64) == sizeof (struct flock64_64));
95 #endif
96 
97 	/*
98 	 * First, for speed, deal with the subset of cases
99 	 * that do not require getf() / releasef().
100 	 */
101 	switch (cmd) {
102 	case F_GETFD:
103 		if ((error = f_getfd_error(fdes, &flag)) == 0)
104 			retval = flag;
105 		goto out;
106 
107 	case F_SETFD:
108 		error = f_setfd_error(fdes, (int)arg);
109 		retval = 0;
110 		goto out;
111 
112 	case F_GETFL:
113 		if ((error = f_getfl(fdes, &flag)) == 0) {
114 			retval = (flag & (FMASK | FASYNC));
115 			if ((flag & (FSEARCH | FEXEC)) == 0)
116 				retval += FOPEN;
117 			else
118 				retval |= (flag & (FSEARCH | FEXEC));
119 		}
120 		goto out;
121 
122 	case F_GETXFL:
123 		if ((error = f_getfl(fdes, &flag)) == 0) {
124 			retval = flag;
125 			if ((flag & (FSEARCH | FEXEC)) == 0)
126 				retval += FOPEN;
127 		}
128 		goto out;
129 
130 	case F_BADFD:
131 		if ((error = f_badfd(fdes, &fdres, (int)arg)) == 0)
132 			retval = fdres;
133 		goto out;
134 	}
135 
136 	/*
137 	 * Second, for speed, deal with the subset of cases that
138 	 * require getf() / releasef() but do not require copyin.
139 	 */
140 	if ((fp = getf(fdes)) == NULL) {
141 		error = EBADF;
142 		goto out;
143 	}
144 	iarg = (int)arg;
145 
146 	switch (cmd) {
147 	case F_DUPFD:
148 	case F_DUPFD_CLOEXEC:
149 		p = curproc;
150 		if ((uint_t)iarg >= p->p_fno_ctl) {
151 			if (iarg >= 0)
152 				fd_too_big(p);
153 			error = EINVAL;
154 			goto done;
155 		}
156 		/*
157 		 * We need to increment the f_count reference counter
158 		 * before allocating a new file descriptor.
159 		 * Doing it other way round opens a window for race condition
160 		 * with closeandsetf() on the target file descriptor which can
161 		 * close the file still referenced by the original
162 		 * file descriptor.
163 		 */
164 		mutex_enter(&fp->f_tlock);
165 		fp->f_count++;
166 		mutex_exit(&fp->f_tlock);
167 		if ((retval = ufalloc_file(iarg, fp)) == -1) {
168 			/*
169 			 * New file descriptor can't be allocated.
170 			 * Revert the reference count.
171 			 */
172 			mutex_enter(&fp->f_tlock);
173 			fp->f_count--;
174 			mutex_exit(&fp->f_tlock);
175 			error = EMFILE;
176 		} else {
177 			if (cmd == F_DUPFD_CLOEXEC) {
178 				f_setfd(retval, FD_CLOEXEC);
179 			}
180 		}
181 		goto done;
182 
183 	case F_DUP2FD_CLOEXEC:
184 		if (fdes == iarg) {
185 			error = EINVAL;
186 			goto done;
187 		}
188 
189 		/*FALLTHROUGH*/
190 
191 	case F_DUP2FD:
192 		p = curproc;
193 		if (fdes == iarg) {
194 			retval = iarg;
195 		} else if ((uint_t)iarg >= p->p_fno_ctl) {
196 			if (iarg >= 0)
197 				fd_too_big(p);
198 			error = EBADF;
199 		} else {
200 			/*
201 			 * We can't hold our getf(fdes) across the call to
202 			 * closeandsetf() because it creates a window for
203 			 * deadlock: if one thread is doing dup2(a, b) while
204 			 * another is doing dup2(b, a), each one will block
205 			 * waiting for the other to call releasef().  The
206 			 * solution is to increment the file reference count
207 			 * (which we have to do anyway), then releasef(fdes),
208 			 * then closeandsetf().  Incrementing f_count ensures
209 			 * that fp won't disappear after we call releasef().
210 			 * When closeandsetf() fails, we try avoid calling
211 			 * closef() because of all the side effects.
212 			 */
213 			mutex_enter(&fp->f_tlock);
214 			fp->f_count++;
215 			mutex_exit(&fp->f_tlock);
216 			releasef(fdes);
217 			if ((error = closeandsetf(iarg, fp)) == 0) {
218 				if (cmd == F_DUP2FD_CLOEXEC) {
219 					f_setfd(iarg, FD_CLOEXEC);
220 				}
221 				retval = iarg;
222 			} else {
223 				mutex_enter(&fp->f_tlock);
224 				if (fp->f_count > 1) {
225 					fp->f_count--;
226 					mutex_exit(&fp->f_tlock);
227 				} else {
228 					mutex_exit(&fp->f_tlock);
229 					(void) closef(fp);
230 				}
231 			}
232 			goto out;
233 		}
234 		goto done;
235 
236 	case F_SETFL:
237 		vp = fp->f_vnode;
238 		flag = fp->f_flag;
239 		if ((iarg & (FNONBLOCK|FNDELAY)) == (FNONBLOCK|FNDELAY))
240 			iarg &= ~FNDELAY;
241 		if ((error = VOP_SETFL(vp, flag, iarg, fp->f_cred, NULL)) ==
242 		    0) {
243 			iarg &= FMASK;
244 			mutex_enter(&fp->f_tlock);
245 			fp->f_flag &= ~FMASK | (FREAD|FWRITE);
246 			fp->f_flag |= (iarg - FOPEN) & ~(FREAD|FWRITE);
247 			mutex_exit(&fp->f_tlock);
248 		}
249 		retval = 0;
250 		goto done;
251 	}
252 
253 	/*
254 	 * Finally, deal with the expensive cases.
255 	 */
256 	retval = 0;
257 	in_crit = 0;
258 	maxoffset = MAXOFF_T;
259 	datamodel = DATAMODEL_NATIVE;
260 #if defined(_SYSCALL32_IMPL)
261 	if ((datamodel = get_udatamodel()) == DATAMODEL_ILP32)
262 		maxoffset = MAXOFF32_T;
263 #endif
264 
265 	vp = fp->f_vnode;
266 	flag = fp->f_flag;
267 	offset = fp->f_offset;
268 
269 	switch (cmd) {
270 	/*
271 	 * The file system and vnode layers understand and implement
272 	 * locking with flock64 structures. So here once we pass through
273 	 * the test for compatibility as defined by LFS API, (for F_SETLK,
274 	 * F_SETLKW, F_GETLK, F_GETLKW, F_FREESP) we transform
275 	 * the flock structure to a flock64 structure and send it to the
276 	 * lower layers. Similarly in case of GETLK the returned flock64
277 	 * structure is transformed to a flock structure if everything fits
278 	 * in nicely, otherwise we return EOVERFLOW.
279 	 */
280 
281 	case F_GETLK:
282 	case F_O_GETLK:
283 	case F_SETLK:
284 	case F_SETLKW:
285 	case F_SETLK_NBMAND:
286 
287 		/*
288 		 * Copy in input fields only.
289 		 */
290 
291 		if (cmd == F_O_GETLK) {
292 			if (datamodel != DATAMODEL_ILP32) {
293 				error = EINVAL;
294 				break;
295 			}
296 
297 			if (copyin((void *)arg, &obf, sizeof (obf))) {
298 				error = EFAULT;
299 				break;
300 			}
301 			bf.l_type = obf.l_type;
302 			bf.l_whence = obf.l_whence;
303 			bf.l_start = (off64_t)obf.l_start;
304 			bf.l_len = (off64_t)obf.l_len;
305 			bf.l_sysid = (int)obf.l_sysid;
306 			bf.l_pid = obf.l_pid;
307 		} else if (datamodel == DATAMODEL_NATIVE) {
308 			if (copyin((void *)arg, &sbf, sizeof (sbf))) {
309 				error = EFAULT;
310 				break;
311 			}
312 			/*
313 			 * XXX	In an LP64 kernel with an LP64 application
314 			 *	there's no need to do a structure copy here
315 			 *	struct flock == struct flock64. However,
316 			 *	we did it this way to avoid more conditional
317 			 *	compilation.
318 			 */
319 			bf.l_type = sbf.l_type;
320 			bf.l_whence = sbf.l_whence;
321 			bf.l_start = (off64_t)sbf.l_start;
322 			bf.l_len = (off64_t)sbf.l_len;
323 			bf.l_sysid = sbf.l_sysid;
324 			bf.l_pid = sbf.l_pid;
325 		}
326 #if defined(_SYSCALL32_IMPL)
327 		else {
328 			struct flock32 sbf32;
329 			if (copyin((void *)arg, &sbf32, sizeof (sbf32))) {
330 				error = EFAULT;
331 				break;
332 			}
333 			bf.l_type = sbf32.l_type;
334 			bf.l_whence = sbf32.l_whence;
335 			bf.l_start = (off64_t)sbf32.l_start;
336 			bf.l_len = (off64_t)sbf32.l_len;
337 			bf.l_sysid = sbf32.l_sysid;
338 			bf.l_pid = sbf32.l_pid;
339 		}
340 #endif /* _SYSCALL32_IMPL */
341 
342 		/*
343 		 * 64-bit support: check for overflow for 32-bit lock ops
344 		 */
345 		if ((error = flock_check(vp, &bf, offset, maxoffset)) != 0)
346 			break;
347 
348 		/*
349 		 * Not all of the filesystems understand F_O_GETLK, and
350 		 * there's no need for them to know.  Map it to F_GETLK.
351 		 */
352 		if ((error = VOP_FRLOCK(vp, (cmd == F_O_GETLK) ? F_GETLK : cmd,
353 		    &bf, flag, offset, NULL, fp->f_cred, NULL)) != 0)
354 			break;
355 
356 		/*
357 		 * If command is GETLK and no lock is found, only
358 		 * the type field is changed.
359 		 */
360 		if ((cmd == F_O_GETLK || cmd == F_GETLK) &&
361 		    bf.l_type == F_UNLCK) {
362 			/* l_type always first entry, always a short */
363 			if (copyout(&bf.l_type, &((struct flock *)arg)->l_type,
364 			    sizeof (bf.l_type)))
365 				error = EFAULT;
366 			break;
367 		}
368 
369 		if (cmd == F_O_GETLK) {
370 			/*
371 			 * Return an SVR3 flock structure to the user.
372 			 */
373 			obf.l_type = (int16_t)bf.l_type;
374 			obf.l_whence = (int16_t)bf.l_whence;
375 			obf.l_start = (int32_t)bf.l_start;
376 			obf.l_len = (int32_t)bf.l_len;
377 			if (bf.l_sysid > SHRT_MAX || bf.l_pid > SHRT_MAX) {
378 				/*
379 				 * One or both values for the above fields
380 				 * is too large to store in an SVR3 flock
381 				 * structure.
382 				 */
383 				error = EOVERFLOW;
384 				break;
385 			}
386 			obf.l_sysid = (int16_t)bf.l_sysid;
387 			obf.l_pid = (int16_t)bf.l_pid;
388 			if (copyout(&obf, (void *)arg, sizeof (obf)))
389 				error = EFAULT;
390 		} else if (cmd == F_GETLK) {
391 			/*
392 			 * Copy out SVR4 flock.
393 			 */
394 			int i;
395 
396 			if (bf.l_start > maxoffset || bf.l_len > maxoffset) {
397 				error = EOVERFLOW;
398 				break;
399 			}
400 
401 			if (datamodel == DATAMODEL_NATIVE) {
402 				for (i = 0; i < 4; i++)
403 					sbf.l_pad[i] = 0;
404 				/*
405 				 * XXX	In an LP64 kernel with an LP64
406 				 *	application there's no need to do a
407 				 *	structure copy here as currently
408 				 *	struct flock == struct flock64.
409 				 *	We did it this way to avoid more
410 				 *	conditional compilation.
411 				 */
412 				sbf.l_type = bf.l_type;
413 				sbf.l_whence = bf.l_whence;
414 				sbf.l_start = (off_t)bf.l_start;
415 				sbf.l_len = (off_t)bf.l_len;
416 				sbf.l_sysid = bf.l_sysid;
417 				sbf.l_pid = bf.l_pid;
418 				if (copyout(&sbf, (void *)arg, sizeof (sbf)))
419 					error = EFAULT;
420 			}
421 #if defined(_SYSCALL32_IMPL)
422 			else {
423 				struct flock32 sbf32;
424 				if (bf.l_start > MAXOFF32_T ||
425 				    bf.l_len > MAXOFF32_T) {
426 					error = EOVERFLOW;
427 					break;
428 				}
429 				for (i = 0; i < 4; i++)
430 					sbf32.l_pad[i] = 0;
431 				sbf32.l_type = (int16_t)bf.l_type;
432 				sbf32.l_whence = (int16_t)bf.l_whence;
433 				sbf32.l_start = (off32_t)bf.l_start;
434 				sbf32.l_len = (off32_t)bf.l_len;
435 				sbf32.l_sysid = (int32_t)bf.l_sysid;
436 				sbf32.l_pid = (pid32_t)bf.l_pid;
437 				if (copyout(&sbf32,
438 				    (void *)arg, sizeof (sbf32)))
439 					error = EFAULT;
440 			}
441 #endif
442 		}
443 		break;
444 
445 	case F_CHKFL:
446 		/*
447 		 * This is for internal use only, to allow the vnode layer
448 		 * to validate a flags setting before applying it.  User
449 		 * programs can't issue it.
450 		 */
451 		error = EINVAL;
452 		break;
453 
454 	case F_ALLOCSP:
455 	case F_FREESP:
456 	case F_ALLOCSP64:
457 	case F_FREESP64:
458 		/*
459 		 * Test for not-a-regular-file (and returning EINVAL)
460 		 * before testing for open-for-writing (and returning EBADF).
461 		 * This is relied upon by posix_fallocate() in libc.
462 		 */
463 		if (vp->v_type != VREG) {
464 			error = EINVAL;
465 			break;
466 		}
467 
468 		if ((flag & FWRITE) == 0) {
469 			error = EBADF;
470 			break;
471 		}
472 
473 		if (datamodel != DATAMODEL_ILP32 &&
474 		    (cmd == F_ALLOCSP64 || cmd == F_FREESP64)) {
475 			error = EINVAL;
476 			break;
477 		}
478 
479 #if defined(_ILP32) || defined(_SYSCALL32_IMPL)
480 		if (datamodel == DATAMODEL_ILP32 &&
481 		    (cmd == F_ALLOCSP || cmd == F_FREESP)) {
482 			struct flock32 sbf32;
483 			/*
484 			 * For compatibility we overlay an SVR3 flock on an SVR4
485 			 * flock.  This works because the input field offsets
486 			 * in "struct flock" were preserved.
487 			 */
488 			if (copyin((void *)arg, &sbf32, sizeof (sbf32))) {
489 				error = EFAULT;
490 				break;
491 			} else {
492 				bf.l_type = sbf32.l_type;
493 				bf.l_whence = sbf32.l_whence;
494 				bf.l_start = (off64_t)sbf32.l_start;
495 				bf.l_len = (off64_t)sbf32.l_len;
496 				bf.l_sysid = sbf32.l_sysid;
497 				bf.l_pid = sbf32.l_pid;
498 			}
499 		}
500 #endif /* _ILP32 || _SYSCALL32_IMPL */
501 
502 #if defined(_LP64)
503 		if (datamodel == DATAMODEL_LP64 &&
504 		    (cmd == F_ALLOCSP || cmd == F_FREESP)) {
505 			if (copyin((void *)arg, &bf, sizeof (bf))) {
506 				error = EFAULT;
507 				break;
508 			}
509 		}
510 #endif /* defined(_LP64) */
511 
512 #if !defined(_LP64) || defined(_SYSCALL32_IMPL)
513 		if (datamodel == DATAMODEL_ILP32 &&
514 		    (cmd == F_ALLOCSP64 || cmd == F_FREESP64)) {
515 			if (copyin((void *)arg, &bf64_32, sizeof (bf64_32))) {
516 				error = EFAULT;
517 				break;
518 			} else {
519 				/*
520 				 * Note that the size of flock64 is different in
521 				 * the ILP32 and LP64 models, due to the l_pad
522 				 * field. We do not want to assume that the
523 				 * flock64 structure is laid out the same in
524 				 * ILP32 and LP64 environments, so we will
525 				 * copy in the ILP32 version of flock64
526 				 * explicitly and copy it to the native
527 				 * flock64 structure.
528 				 */
529 				bf.l_type = (short)bf64_32.l_type;
530 				bf.l_whence = (short)bf64_32.l_whence;
531 				bf.l_start = bf64_32.l_start;
532 				bf.l_len = bf64_32.l_len;
533 				bf.l_sysid = (int)bf64_32.l_sysid;
534 				bf.l_pid = (pid_t)bf64_32.l_pid;
535 			}
536 		}
537 #endif /* !defined(_LP64) || defined(_SYSCALL32_IMPL) */
538 
539 		if (cmd == F_ALLOCSP || cmd == F_FREESP)
540 			error = flock_check(vp, &bf, offset, maxoffset);
541 		else if (cmd == F_ALLOCSP64 || cmd == F_FREESP64)
542 			error = flock_check(vp, &bf, offset, MAXOFFSET_T);
543 		if (error)
544 			break;
545 
546 		if (vp->v_type == VREG && bf.l_len == 0 &&
547 		    bf.l_start > OFFSET_MAX(fp)) {
548 			error = EFBIG;
549 			break;
550 		}
551 
552 		/*
553 		 * Make sure that there are no conflicting non-blocking
554 		 * mandatory locks in the region being manipulated. If
555 		 * there are such locks then return EACCES.
556 		 */
557 		if ((error = flock_get_start(vp, &bf, offset, &start)) != 0)
558 			break;
559 
560 		if (nbl_need_check(vp)) {
561 			u_offset_t	begin;
562 			ssize_t		length;
563 
564 			nbl_start_crit(vp, RW_READER);
565 			in_crit = 1;
566 			vattr.va_mask = AT_SIZE;
567 			if ((error = VOP_GETATTR(vp, &vattr, 0, CRED(), NULL))
568 			    != 0)
569 				break;
570 			begin = start > vattr.va_size ? vattr.va_size : start;
571 			length = vattr.va_size > start ? vattr.va_size - start :
572 			    start - vattr.va_size;
573 			if (nbl_conflict(vp, NBL_WRITE, begin, length, 0,
574 			    NULL)) {
575 				error = EACCES;
576 				break;
577 			}
578 		}
579 
580 		if (cmd == F_ALLOCSP64)
581 			cmd = F_ALLOCSP;
582 		else if (cmd == F_FREESP64)
583 			cmd = F_FREESP;
584 
585 		error = VOP_SPACE(vp, cmd, &bf, flag, offset, fp->f_cred, NULL);
586 
587 		break;
588 
589 #if !defined(_LP64) || defined(_SYSCALL32_IMPL)
590 	case F_GETLK64:
591 	case F_SETLK64:
592 	case F_SETLKW64:
593 	case F_SETLK64_NBMAND:
594 		/*
595 		 * Large Files: Here we set cmd as *LK and send it to
596 		 * lower layers. *LK64 is only for the user land.
597 		 * Most of the comments described above for F_SETLK
598 		 * applies here too.
599 		 * Large File support is only needed for ILP32 apps!
600 		 */
601 		if (datamodel != DATAMODEL_ILP32) {
602 			error = EINVAL;
603 			break;
604 		}
605 
606 		if (cmd == F_GETLK64)
607 			cmd = F_GETLK;
608 		else if (cmd == F_SETLK64)
609 			cmd = F_SETLK;
610 		else if (cmd == F_SETLKW64)
611 			cmd = F_SETLKW;
612 		else if (cmd == F_SETLK64_NBMAND)
613 			cmd = F_SETLK_NBMAND;
614 
615 		/*
616 		 * Note that the size of flock64 is different in the ILP32
617 		 * and LP64 models, due to the sucking l_pad field.
618 		 * We do not want to assume that the flock64 structure is
619 		 * laid out in the same in ILP32 and LP64 environments, so
620 		 * we will copy in the ILP32 version of flock64 explicitly
621 		 * and copy it to the native flock64 structure.
622 		 */
623 
624 		if (copyin((void *)arg, &bf64_32, sizeof (bf64_32))) {
625 			error = EFAULT;
626 			break;
627 		}
628 
629 		bf.l_type = (short)bf64_32.l_type;
630 		bf.l_whence = (short)bf64_32.l_whence;
631 		bf.l_start = bf64_32.l_start;
632 		bf.l_len = bf64_32.l_len;
633 		bf.l_sysid = (int)bf64_32.l_sysid;
634 		bf.l_pid = (pid_t)bf64_32.l_pid;
635 
636 		if ((error = flock_check(vp, &bf, offset, MAXOFFSET_T)) != 0)
637 			break;
638 
639 		if ((error = VOP_FRLOCK(vp, cmd, &bf, flag, offset,
640 		    NULL, fp->f_cred, NULL)) != 0)
641 			break;
642 
643 		if ((cmd == F_GETLK) && bf.l_type == F_UNLCK) {
644 			if (copyout(&bf.l_type, &((struct flock *)arg)->l_type,
645 			    sizeof (bf.l_type)))
646 				error = EFAULT;
647 			break;
648 		}
649 
650 		if (cmd == F_GETLK) {
651 			int i;
652 
653 			/*
654 			 * We do not want to assume that the flock64 structure
655 			 * is laid out in the same in ILP32 and LP64
656 			 * environments, so we will copy out the ILP32 version
657 			 * of flock64 explicitly after copying the native
658 			 * flock64 structure to it.
659 			 */
660 			for (i = 0; i < 4; i++)
661 				bf64_32.l_pad[i] = 0;
662 			bf64_32.l_type = (int16_t)bf.l_type;
663 			bf64_32.l_whence = (int16_t)bf.l_whence;
664 			bf64_32.l_start = bf.l_start;
665 			bf64_32.l_len = bf.l_len;
666 			bf64_32.l_sysid = (int32_t)bf.l_sysid;
667 			bf64_32.l_pid = (pid32_t)bf.l_pid;
668 			if (copyout(&bf64_32, (void *)arg, sizeof (bf64_32)))
669 				error = EFAULT;
670 		}
671 		break;
672 #endif /* !defined(_LP64) || defined(_SYSCALL32_IMPL) */
673 
674 	case F_SHARE:
675 	case F_SHARE_NBMAND:
676 	case F_UNSHARE:
677 
678 		/*
679 		 * Copy in input fields only.
680 		 */
681 		if (copyin((void *)arg, &fsh, sizeof (fsh))) {
682 			error = EFAULT;
683 			break;
684 		}
685 
686 		/*
687 		 * Local share reservations always have this simple form
688 		 */
689 		shr.s_access = fsh.f_access;
690 		shr.s_deny = fsh.f_deny;
691 		shr.s_sysid = 0;
692 		shr.s_pid = ttoproc(curthread)->p_pid;
693 		shr_own.sl_pid = shr.s_pid;
694 		shr_own.sl_id = fsh.f_id;
695 		shr.s_own_len = sizeof (shr_own);
696 		shr.s_owner = (caddr_t)&shr_own;
697 		error = VOP_SHRLOCK(vp, cmd, &shr, flag, fp->f_cred, NULL);
698 		break;
699 
700 	default:
701 		error = EINVAL;
702 		break;
703 	}
704 
705 	if (in_crit)
706 		nbl_end_crit(vp);
707 
708 done:
709 	releasef(fdes);
710 out:
711 	if (error)
712 		return (set_errno(error));
713 	return (retval);
714 }
715 
716 int
717 flock_check(vnode_t *vp, flock64_t *flp, offset_t offset, offset_t max)
718 {
719 	struct vattr	vattr;
720 	int	error;
721 	u_offset_t start, end;
722 
723 	/*
724 	 * Determine the starting point of the request
725 	 */
726 	switch (flp->l_whence) {
727 	case 0:		/* SEEK_SET */
728 		start = (u_offset_t)flp->l_start;
729 		if (start > max)
730 			return (EINVAL);
731 		break;
732 	case 1:		/* SEEK_CUR */
733 		if (flp->l_start > (max - offset))
734 			return (EOVERFLOW);
735 		start = (u_offset_t)(flp->l_start + offset);
736 		if (start > max)
737 			return (EINVAL);
738 		break;
739 	case 2:		/* SEEK_END */
740 		vattr.va_mask = AT_SIZE;
741 		if (error = VOP_GETATTR(vp, &vattr, 0, CRED(), NULL))
742 			return (error);
743 		if (flp->l_start > (max - (offset_t)vattr.va_size))
744 			return (EOVERFLOW);
745 		start = (u_offset_t)(flp->l_start + (offset_t)vattr.va_size);
746 		if (start > max)
747 			return (EINVAL);
748 		break;
749 	default:
750 		return (EINVAL);
751 	}
752 
753 	/*
754 	 * Determine the range covered by the request.
755 	 */
756 	if (flp->l_len == 0)
757 		end = MAXEND;
758 	else if ((offset_t)flp->l_len > 0) {
759 		if (flp->l_len > (max - start + 1))
760 			return (EOVERFLOW);
761 		end = (u_offset_t)(start + (flp->l_len - 1));
762 		ASSERT(end <= max);
763 	} else {
764 		/*
765 		 * Negative length; why do we even allow this ?
766 		 * Because this allows easy specification of
767 		 * the last n bytes of the file.
768 		 */
769 		end = start;
770 		start += (u_offset_t)flp->l_len;
771 		(start)++;
772 		if (start > max)
773 			return (EINVAL);
774 		ASSERT(end <= max);
775 	}
776 	ASSERT(start <= max);
777 	if (flp->l_type == F_UNLCK && flp->l_len > 0 &&
778 	    end == (offset_t)max) {
779 		flp->l_len = 0;
780 	}
781 	if (start  > end)
782 		return (EINVAL);
783 	return (0);
784 }
785 
786 static int
787 flock_get_start(vnode_t *vp, flock64_t *flp, offset_t offset, u_offset_t *start)
788 {
789 	struct vattr	vattr;
790 	int	error;
791 
792 	/*
793 	 * Determine the starting point of the request. Assume that it is
794 	 * a valid starting point.
795 	 */
796 	switch (flp->l_whence) {
797 	case 0:		/* SEEK_SET */
798 		*start = (u_offset_t)flp->l_start;
799 		break;
800 	case 1:		/* SEEK_CUR */
801 		*start = (u_offset_t)(flp->l_start + offset);
802 		break;
803 	case 2:		/* SEEK_END */
804 		vattr.va_mask = AT_SIZE;
805 		if (error = VOP_GETATTR(vp, &vattr, 0, CRED(), NULL))
806 			return (error);
807 		*start = (u_offset_t)(flp->l_start + (offset_t)vattr.va_size);
808 		break;
809 	default:
810 		return (EINVAL);
811 	}
812 
813 	return (0);
814 }
815 
816 /*
817  * Take rctl action when the requested file descriptor is too big.
818  */
819 static void
820 fd_too_big(proc_t *p)
821 {
822 	mutex_enter(&p->p_lock);
823 	(void) rctl_action(rctlproc_legacy[RLIMIT_NOFILE],
824 	    p->p_rctls, p, RCA_SAFE);
825 	mutex_exit(&p->p_lock);
826 }
827