xref: /titanic_44/usr/src/common/acl/acl_common.c (revision f05d7fc81533be643136e12ce92516d1d4292921)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 /*
22  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
23  * Copyright 2011 Nexenta Systems, Inc.  All rights reserved.
24  */
25 
26 #include <sys/types.h>
27 #include <sys/stat.h>
28 #include <sys/avl.h>
29 #if defined(_KERNEL)
30 #include <sys/systm.h>
31 #include <sys/sysmacros.h>
32 #include <acl/acl_common.h>
33 #else
34 #include <errno.h>
35 #include <stdlib.h>
36 #include <stddef.h>
37 #include <strings.h>
38 #include <unistd.h>
39 #include <assert.h>
40 #include <grp.h>
41 #include <pwd.h>
42 #include <acl_common.h>
43 #define	ASSERT	assert
44 #endif
45 
46 #define	ACE_POSIX_SUPPORTED_BITS (ACE_READ_DATA | \
47     ACE_WRITE_DATA | ACE_APPEND_DATA | ACE_EXECUTE | \
48     ACE_READ_ATTRIBUTES | ACE_READ_ACL | ACE_WRITE_ACL)
49 
50 
51 #define	ACL_SYNCHRONIZE_SET_DENY		0x0000001
52 #define	ACL_SYNCHRONIZE_SET_ALLOW		0x0000002
53 #define	ACL_SYNCHRONIZE_ERR_DENY		0x0000004
54 #define	ACL_SYNCHRONIZE_ERR_ALLOW		0x0000008
55 
56 #define	ACL_WRITE_OWNER_SET_DENY		0x0000010
57 #define	ACL_WRITE_OWNER_SET_ALLOW		0x0000020
58 #define	ACL_WRITE_OWNER_ERR_DENY		0x0000040
59 #define	ACL_WRITE_OWNER_ERR_ALLOW		0x0000080
60 
61 #define	ACL_DELETE_SET_DENY			0x0000100
62 #define	ACL_DELETE_SET_ALLOW			0x0000200
63 #define	ACL_DELETE_ERR_DENY			0x0000400
64 #define	ACL_DELETE_ERR_ALLOW			0x0000800
65 
66 #define	ACL_WRITE_ATTRS_OWNER_SET_DENY		0x0001000
67 #define	ACL_WRITE_ATTRS_OWNER_SET_ALLOW		0x0002000
68 #define	ACL_WRITE_ATTRS_OWNER_ERR_DENY		0x0004000
69 #define	ACL_WRITE_ATTRS_OWNER_ERR_ALLOW		0x0008000
70 
71 #define	ACL_WRITE_ATTRS_WRITER_SET_DENY		0x0010000
72 #define	ACL_WRITE_ATTRS_WRITER_SET_ALLOW	0x0020000
73 #define	ACL_WRITE_ATTRS_WRITER_ERR_DENY		0x0040000
74 #define	ACL_WRITE_ATTRS_WRITER_ERR_ALLOW	0x0080000
75 
76 #define	ACL_WRITE_NAMED_WRITER_SET_DENY		0x0100000
77 #define	ACL_WRITE_NAMED_WRITER_SET_ALLOW	0x0200000
78 #define	ACL_WRITE_NAMED_WRITER_ERR_DENY		0x0400000
79 #define	ACL_WRITE_NAMED_WRITER_ERR_ALLOW	0x0800000
80 
81 #define	ACL_READ_NAMED_READER_SET_DENY		0x1000000
82 #define	ACL_READ_NAMED_READER_SET_ALLOW		0x2000000
83 #define	ACL_READ_NAMED_READER_ERR_DENY		0x4000000
84 #define	ACL_READ_NAMED_READER_ERR_ALLOW		0x8000000
85 
86 
87 #define	ACE_VALID_MASK_BITS (\
88     ACE_READ_DATA | \
89     ACE_LIST_DIRECTORY | \
90     ACE_WRITE_DATA | \
91     ACE_ADD_FILE | \
92     ACE_APPEND_DATA | \
93     ACE_ADD_SUBDIRECTORY | \
94     ACE_READ_NAMED_ATTRS | \
95     ACE_WRITE_NAMED_ATTRS | \
96     ACE_EXECUTE | \
97     ACE_DELETE_CHILD | \
98     ACE_READ_ATTRIBUTES | \
99     ACE_WRITE_ATTRIBUTES | \
100     ACE_DELETE | \
101     ACE_READ_ACL | \
102     ACE_WRITE_ACL | \
103     ACE_WRITE_OWNER | \
104     ACE_SYNCHRONIZE)
105 
106 #define	ACE_MASK_UNDEFINED			0x80000000
107 
108 #define	ACE_VALID_FLAG_BITS (ACE_FILE_INHERIT_ACE | \
109     ACE_DIRECTORY_INHERIT_ACE | \
110     ACE_NO_PROPAGATE_INHERIT_ACE | ACE_INHERIT_ONLY_ACE | \
111     ACE_SUCCESSFUL_ACCESS_ACE_FLAG | ACE_FAILED_ACCESS_ACE_FLAG | \
112     ACE_IDENTIFIER_GROUP | ACE_OWNER | ACE_GROUP | ACE_EVERYONE)
113 
114 /*
115  * ACL conversion helpers
116  */
117 
118 typedef enum {
119 	ace_unused,
120 	ace_user_obj,
121 	ace_user,
122 	ace_group, /* includes GROUP and GROUP_OBJ */
123 	ace_other_obj
124 } ace_to_aent_state_t;
125 
126 typedef struct acevals {
127 	uid_t key;
128 	avl_node_t avl;
129 	uint32_t mask;
130 	uint32_t allowed;
131 	uint32_t denied;
132 	int aent_type;
133 } acevals_t;
134 
135 typedef struct ace_list {
136 	acevals_t user_obj;
137 	avl_tree_t user;
138 	int numusers;
139 	acevals_t group_obj;
140 	avl_tree_t group;
141 	int numgroups;
142 	acevals_t other_obj;
143 	uint32_t acl_mask;
144 	int hasmask;
145 	int dfacl_flag;
146 	ace_to_aent_state_t state;
147 	int seen; /* bitmask of all aclent_t a_type values seen */
148 } ace_list_t;
149 
150 /*
151  * Generic shellsort, from K&R (1st ed, p 58.), somewhat modified.
152  * v = Ptr to array/vector of objs
153  * n = # objs in the array
154  * s = size of each obj (must be multiples of a word size)
155  * f = ptr to function to compare two objs
156  *	returns (-1 = less than, 0 = equal, 1 = greater than
157  */
158 void
159 ksort(caddr_t v, int n, int s, int (*f)())
160 {
161 	int g, i, j, ii;
162 	unsigned int *p1, *p2;
163 	unsigned int tmp;
164 
165 	/* No work to do */
166 	if (v == NULL || n <= 1)
167 		return;
168 
169 	/* Sanity check on arguments */
170 	ASSERT(((uintptr_t)v & 0x3) == 0 && (s & 0x3) == 0);
171 	ASSERT(s > 0);
172 	for (g = n / 2; g > 0; g /= 2) {
173 		for (i = g; i < n; i++) {
174 			for (j = i - g; j >= 0 &&
175 			    (*f)(v + j * s, v + (j + g) * s) == 1;
176 			    j -= g) {
177 				p1 = (void *)(v + j * s);
178 				p2 = (void *)(v + (j + g) * s);
179 				for (ii = 0; ii < s / 4; ii++) {
180 					tmp = *p1;
181 					*p1++ = *p2;
182 					*p2++ = tmp;
183 				}
184 			}
185 		}
186 	}
187 }
188 
189 /*
190  * Compare two acls, all fields.  Returns:
191  * -1 (less than)
192  *  0 (equal)
193  * +1 (greater than)
194  */
195 int
196 cmp2acls(void *a, void *b)
197 {
198 	aclent_t *x = (aclent_t *)a;
199 	aclent_t *y = (aclent_t *)b;
200 
201 	/* Compare types */
202 	if (x->a_type < y->a_type)
203 		return (-1);
204 	if (x->a_type > y->a_type)
205 		return (1);
206 	/* Equal types; compare id's */
207 	if (x->a_id < y->a_id)
208 		return (-1);
209 	if (x->a_id > y->a_id)
210 		return (1);
211 	/* Equal ids; compare perms */
212 	if (x->a_perm < y->a_perm)
213 		return (-1);
214 	if (x->a_perm > y->a_perm)
215 		return (1);
216 	/* Totally equal */
217 	return (0);
218 }
219 
220 /*ARGSUSED*/
221 static void *
222 cacl_realloc(void *ptr, size_t size, size_t new_size)
223 {
224 #if defined(_KERNEL)
225 	void *tmp;
226 
227 	tmp = kmem_alloc(new_size, KM_SLEEP);
228 	(void) memcpy(tmp, ptr, (size < new_size) ? size : new_size);
229 	kmem_free(ptr, size);
230 	return (tmp);
231 #else
232 	return (realloc(ptr, new_size));
233 #endif
234 }
235 
236 static int
237 cacl_malloc(void **ptr, size_t size)
238 {
239 #if defined(_KERNEL)
240 	*ptr = kmem_zalloc(size, KM_SLEEP);
241 	return (0);
242 #else
243 	*ptr = calloc(1, size);
244 	if (*ptr == NULL)
245 		return (errno);
246 
247 	return (0);
248 #endif
249 }
250 
251 /*ARGSUSED*/
252 static void
253 cacl_free(void *ptr, size_t size)
254 {
255 #if defined(_KERNEL)
256 	kmem_free(ptr, size);
257 #else
258 	free(ptr);
259 #endif
260 }
261 
262 acl_t *
263 acl_alloc(enum acl_type type)
264 {
265 	acl_t *aclp;
266 
267 	if (cacl_malloc((void **)&aclp, sizeof (acl_t)) != 0)
268 		return (NULL);
269 
270 	aclp->acl_aclp = NULL;
271 	aclp->acl_cnt = 0;
272 
273 	switch (type) {
274 	case ACE_T:
275 		aclp->acl_type = ACE_T;
276 		aclp->acl_entry_size = sizeof (ace_t);
277 		break;
278 	case ACLENT_T:
279 		aclp->acl_type = ACLENT_T;
280 		aclp->acl_entry_size = sizeof (aclent_t);
281 		break;
282 	default:
283 		acl_free(aclp);
284 		aclp = NULL;
285 	}
286 	return (aclp);
287 }
288 
289 /*
290  * Free acl_t structure
291  */
292 void
293 acl_free(acl_t *aclp)
294 {
295 	int acl_size;
296 
297 	if (aclp == NULL)
298 		return;
299 
300 	if (aclp->acl_aclp) {
301 		acl_size = aclp->acl_cnt * aclp->acl_entry_size;
302 		cacl_free(aclp->acl_aclp, acl_size);
303 	}
304 
305 	cacl_free(aclp, sizeof (acl_t));
306 }
307 
308 static uint32_t
309 access_mask_set(int haswriteperm, int hasreadperm, int isowner, int isallow)
310 {
311 	uint32_t access_mask = 0;
312 	int acl_produce;
313 	int synchronize_set = 0, write_owner_set = 0;
314 	int delete_set = 0, write_attrs_set = 0;
315 	int read_named_set = 0, write_named_set = 0;
316 
317 	acl_produce = (ACL_SYNCHRONIZE_SET_ALLOW |
318 	    ACL_WRITE_ATTRS_OWNER_SET_ALLOW |
319 	    ACL_WRITE_ATTRS_WRITER_SET_DENY);
320 
321 	if (isallow) {
322 		synchronize_set = ACL_SYNCHRONIZE_SET_ALLOW;
323 		write_owner_set = ACL_WRITE_OWNER_SET_ALLOW;
324 		delete_set = ACL_DELETE_SET_ALLOW;
325 		if (hasreadperm)
326 			read_named_set = ACL_READ_NAMED_READER_SET_ALLOW;
327 		if (haswriteperm)
328 			write_named_set = ACL_WRITE_NAMED_WRITER_SET_ALLOW;
329 		if (isowner)
330 			write_attrs_set = ACL_WRITE_ATTRS_OWNER_SET_ALLOW;
331 		else if (haswriteperm)
332 			write_attrs_set = ACL_WRITE_ATTRS_WRITER_SET_ALLOW;
333 	} else {
334 
335 		synchronize_set = ACL_SYNCHRONIZE_SET_DENY;
336 		write_owner_set = ACL_WRITE_OWNER_SET_DENY;
337 		delete_set = ACL_DELETE_SET_DENY;
338 		if (hasreadperm)
339 			read_named_set = ACL_READ_NAMED_READER_SET_DENY;
340 		if (haswriteperm)
341 			write_named_set = ACL_WRITE_NAMED_WRITER_SET_DENY;
342 		if (isowner)
343 			write_attrs_set = ACL_WRITE_ATTRS_OWNER_SET_DENY;
344 		else if (haswriteperm)
345 			write_attrs_set = ACL_WRITE_ATTRS_WRITER_SET_DENY;
346 		else
347 			/*
348 			 * If the entity is not the owner and does not
349 			 * have write permissions ACE_WRITE_ATTRIBUTES will
350 			 * always go in the DENY ACE.
351 			 */
352 			access_mask |= ACE_WRITE_ATTRIBUTES;
353 	}
354 
355 	if (acl_produce & synchronize_set)
356 		access_mask |= ACE_SYNCHRONIZE;
357 	if (acl_produce & write_owner_set)
358 		access_mask |= ACE_WRITE_OWNER;
359 	if (acl_produce & delete_set)
360 		access_mask |= ACE_DELETE;
361 	if (acl_produce & write_attrs_set)
362 		access_mask |= ACE_WRITE_ATTRIBUTES;
363 	if (acl_produce & read_named_set)
364 		access_mask |= ACE_READ_NAMED_ATTRS;
365 	if (acl_produce & write_named_set)
366 		access_mask |= ACE_WRITE_NAMED_ATTRS;
367 
368 	return (access_mask);
369 }
370 
371 /*
372  * Given an mode_t, convert it into an access_mask as used
373  * by nfsace, assuming aclent_t -> nfsace semantics.
374  */
375 static uint32_t
376 mode_to_ace_access(mode_t mode, boolean_t isdir, int isowner, int isallow)
377 {
378 	uint32_t access = 0;
379 	int haswriteperm = 0;
380 	int hasreadperm = 0;
381 
382 	if (isallow) {
383 		haswriteperm = (mode & S_IWOTH);
384 		hasreadperm = (mode & S_IROTH);
385 	} else {
386 		haswriteperm = !(mode & S_IWOTH);
387 		hasreadperm = !(mode & S_IROTH);
388 	}
389 
390 	/*
391 	 * The following call takes care of correctly setting the following
392 	 * mask bits in the access_mask:
393 	 * ACE_SYNCHRONIZE, ACE_WRITE_OWNER, ACE_DELETE,
394 	 * ACE_WRITE_ATTRIBUTES, ACE_WRITE_NAMED_ATTRS, ACE_READ_NAMED_ATTRS
395 	 */
396 	access = access_mask_set(haswriteperm, hasreadperm, isowner, isallow);
397 
398 	if (isallow) {
399 		access |= ACE_READ_ACL | ACE_READ_ATTRIBUTES;
400 		if (isowner)
401 			access |= ACE_WRITE_ACL;
402 	} else {
403 		if (! isowner)
404 			access |= ACE_WRITE_ACL;
405 	}
406 
407 	/* read */
408 	if (mode & S_IROTH) {
409 		access |= ACE_READ_DATA;
410 	}
411 	/* write */
412 	if (mode & S_IWOTH) {
413 		access |= ACE_WRITE_DATA |
414 		    ACE_APPEND_DATA;
415 		if (isdir)
416 			access |= ACE_DELETE_CHILD;
417 	}
418 	/* exec */
419 	if (mode & S_IXOTH) {
420 		access |= ACE_EXECUTE;
421 	}
422 
423 	return (access);
424 }
425 
426 /*
427  * Given an nfsace (presumably an ALLOW entry), make a
428  * corresponding DENY entry at the address given.
429  */
430 static void
431 ace_make_deny(ace_t *allow, ace_t *deny, int isdir, int isowner)
432 {
433 	(void) memcpy(deny, allow, sizeof (ace_t));
434 
435 	deny->a_who = allow->a_who;
436 
437 	deny->a_type = ACE_ACCESS_DENIED_ACE_TYPE;
438 	deny->a_access_mask ^= ACE_POSIX_SUPPORTED_BITS;
439 	if (isdir)
440 		deny->a_access_mask ^= ACE_DELETE_CHILD;
441 
442 	deny->a_access_mask &= ~(ACE_SYNCHRONIZE | ACE_WRITE_OWNER |
443 	    ACE_DELETE | ACE_WRITE_ATTRIBUTES | ACE_READ_NAMED_ATTRS |
444 	    ACE_WRITE_NAMED_ATTRS);
445 	deny->a_access_mask |= access_mask_set((allow->a_access_mask &
446 	    ACE_WRITE_DATA), (allow->a_access_mask & ACE_READ_DATA), isowner,
447 	    B_FALSE);
448 }
449 /*
450  * Make an initial pass over an array of aclent_t's.  Gather
451  * information such as an ACL_MASK (if any), number of users,
452  * number of groups, and whether the array needs to be sorted.
453  */
454 static int
455 ln_aent_preprocess(aclent_t *aclent, int n,
456     int *hasmask, mode_t *mask,
457     int *numuser, int *numgroup, int *needsort)
458 {
459 	int error = 0;
460 	int i;
461 	int curtype = 0;
462 
463 	*hasmask = 0;
464 	*mask = 07;
465 	*needsort = 0;
466 	*numuser = 0;
467 	*numgroup = 0;
468 
469 	for (i = 0; i < n; i++) {
470 		if (aclent[i].a_type < curtype)
471 			*needsort = 1;
472 		else if (aclent[i].a_type > curtype)
473 			curtype = aclent[i].a_type;
474 		if (aclent[i].a_type & USER)
475 			(*numuser)++;
476 		if (aclent[i].a_type & (GROUP | GROUP_OBJ))
477 			(*numgroup)++;
478 		if (aclent[i].a_type & CLASS_OBJ) {
479 			if (*hasmask) {
480 				error = EINVAL;
481 				goto out;
482 			} else {
483 				*hasmask = 1;
484 				*mask = aclent[i].a_perm;
485 			}
486 		}
487 	}
488 
489 	if ((! *hasmask) && (*numuser + *numgroup > 1)) {
490 		error = EINVAL;
491 		goto out;
492 	}
493 
494 out:
495 	return (error);
496 }
497 
498 /*
499  * Convert an array of aclent_t into an array of nfsace entries,
500  * following POSIX draft -> nfsv4 conversion semantics as outlined in
501  * the IETF draft.
502  */
503 static int
504 ln_aent_to_ace(aclent_t *aclent, int n, ace_t **acepp, int *rescount, int isdir)
505 {
506 	int error = 0;
507 	mode_t mask;
508 	int numuser, numgroup, needsort;
509 	int resultsize = 0;
510 	int i, groupi = 0, skip;
511 	ace_t *acep, *result = NULL;
512 	int hasmask;
513 
514 	error = ln_aent_preprocess(aclent, n, &hasmask, &mask,
515 	    &numuser, &numgroup, &needsort);
516 	if (error != 0)
517 		goto out;
518 
519 	/* allow + deny for each aclent */
520 	resultsize = n * 2;
521 	if (hasmask) {
522 		/*
523 		 * stick extra deny on the group_obj and on each
524 		 * user|group for the mask (the group_obj was added
525 		 * into the count for numgroup)
526 		 */
527 		resultsize += numuser + numgroup;
528 		/* ... and don't count the mask itself */
529 		resultsize -= 2;
530 	}
531 
532 	/* sort the source if necessary */
533 	if (needsort)
534 		ksort((caddr_t)aclent, n, sizeof (aclent_t), cmp2acls);
535 
536 	if (cacl_malloc((void **)&result, resultsize * sizeof (ace_t)) != 0)
537 		goto out;
538 
539 	acep = result;
540 
541 	for (i = 0; i < n; i++) {
542 		/*
543 		 * don't process CLASS_OBJ (mask); mask was grabbed in
544 		 * ln_aent_preprocess()
545 		 */
546 		if (aclent[i].a_type & CLASS_OBJ)
547 			continue;
548 
549 		/* If we need an ACL_MASK emulator, prepend it now */
550 		if ((hasmask) &&
551 		    (aclent[i].a_type & (USER | GROUP | GROUP_OBJ))) {
552 			acep->a_type = ACE_ACCESS_DENIED_ACE_TYPE;
553 			acep->a_flags = 0;
554 			if (aclent[i].a_type & GROUP_OBJ) {
555 				acep->a_who = (uid_t)-1;
556 				acep->a_flags |=
557 				    (ACE_IDENTIFIER_GROUP|ACE_GROUP);
558 			} else if (aclent[i].a_type & USER) {
559 				acep->a_who = aclent[i].a_id;
560 			} else {
561 				acep->a_who = aclent[i].a_id;
562 				acep->a_flags |= ACE_IDENTIFIER_GROUP;
563 			}
564 			if (aclent[i].a_type & ACL_DEFAULT) {
565 				acep->a_flags |= ACE_INHERIT_ONLY_ACE |
566 				    ACE_FILE_INHERIT_ACE |
567 				    ACE_DIRECTORY_INHERIT_ACE;
568 			}
569 			/*
570 			 * Set the access mask for the prepended deny
571 			 * ace.  To do this, we invert the mask (found
572 			 * in ln_aent_preprocess()) then convert it to an
573 			 * DENY ace access_mask.
574 			 */
575 			acep->a_access_mask = mode_to_ace_access((mask ^ 07),
576 			    isdir, 0, 0);
577 			acep += 1;
578 		}
579 
580 		/* handle a_perm -> access_mask */
581 		acep->a_access_mask = mode_to_ace_access(aclent[i].a_perm,
582 		    isdir, aclent[i].a_type & USER_OBJ, 1);
583 
584 		/* emulate a default aclent */
585 		if (aclent[i].a_type & ACL_DEFAULT) {
586 			acep->a_flags |= ACE_INHERIT_ONLY_ACE |
587 			    ACE_FILE_INHERIT_ACE |
588 			    ACE_DIRECTORY_INHERIT_ACE;
589 		}
590 
591 		/*
592 		 * handle a_perm and a_id
593 		 *
594 		 * this must be done last, since it involves the
595 		 * corresponding deny aces, which are handled
596 		 * differently for each different a_type.
597 		 */
598 		if (aclent[i].a_type & USER_OBJ) {
599 			acep->a_who = (uid_t)-1;
600 			acep->a_flags |= ACE_OWNER;
601 			ace_make_deny(acep, acep + 1, isdir, B_TRUE);
602 			acep += 2;
603 		} else if (aclent[i].a_type & USER) {
604 			acep->a_who = aclent[i].a_id;
605 			ace_make_deny(acep, acep + 1, isdir, B_FALSE);
606 			acep += 2;
607 		} else if (aclent[i].a_type & (GROUP_OBJ | GROUP)) {
608 			if (aclent[i].a_type & GROUP_OBJ) {
609 				acep->a_who = (uid_t)-1;
610 				acep->a_flags |= ACE_GROUP;
611 			} else {
612 				acep->a_who = aclent[i].a_id;
613 			}
614 			acep->a_flags |= ACE_IDENTIFIER_GROUP;
615 			/*
616 			 * Set the corresponding deny for the group ace.
617 			 *
618 			 * The deny aces go after all of the groups, unlike
619 			 * everything else, where they immediately follow
620 			 * the allow ace.
621 			 *
622 			 * We calculate "skip", the number of slots to
623 			 * skip ahead for the deny ace, here.
624 			 *
625 			 * The pattern is:
626 			 * MD1 A1 MD2 A2 MD3 A3 D1 D2 D3
627 			 * thus, skip is
628 			 * (2 * numgroup) - 1 - groupi
629 			 * (2 * numgroup) to account for MD + A
630 			 * - 1 to account for the fact that we're on the
631 			 * access (A), not the mask (MD)
632 			 * - groupi to account for the fact that we have
633 			 * passed up groupi number of MD's.
634 			 */
635 			skip = (2 * numgroup) - 1 - groupi;
636 			ace_make_deny(acep, acep + skip, isdir, B_FALSE);
637 			/*
638 			 * If we just did the last group, skip acep past
639 			 * all of the denies; else, just move ahead one.
640 			 */
641 			if (++groupi >= numgroup)
642 				acep += numgroup + 1;
643 			else
644 				acep += 1;
645 		} else if (aclent[i].a_type & OTHER_OBJ) {
646 			acep->a_who = (uid_t)-1;
647 			acep->a_flags |= ACE_EVERYONE;
648 			ace_make_deny(acep, acep + 1, isdir, B_FALSE);
649 			acep += 2;
650 		} else {
651 			error = EINVAL;
652 			goto out;
653 		}
654 	}
655 
656 	*acepp = result;
657 	*rescount = resultsize;
658 
659 out:
660 	if (error != 0) {
661 		if ((result != NULL) && (resultsize > 0)) {
662 			cacl_free(result, resultsize * sizeof (ace_t));
663 		}
664 	}
665 
666 	return (error);
667 }
668 
669 static int
670 convert_aent_to_ace(aclent_t *aclentp, int aclcnt, boolean_t isdir,
671     ace_t **retacep, int *retacecnt)
672 {
673 	ace_t *acep;
674 	ace_t *dfacep;
675 	int acecnt = 0;
676 	int dfacecnt = 0;
677 	int dfaclstart = 0;
678 	int dfaclcnt = 0;
679 	aclent_t *aclp;
680 	int i;
681 	int error;
682 	int acesz, dfacesz;
683 
684 	ksort((caddr_t)aclentp, aclcnt, sizeof (aclent_t), cmp2acls);
685 
686 	for (i = 0, aclp = aclentp; i < aclcnt; aclp++, i++) {
687 		if (aclp->a_type & ACL_DEFAULT)
688 			break;
689 	}
690 
691 	if (i < aclcnt) {
692 		dfaclstart = i;
693 		dfaclcnt = aclcnt - i;
694 	}
695 
696 	if (dfaclcnt && !isdir) {
697 		return (EINVAL);
698 	}
699 
700 	error = ln_aent_to_ace(aclentp, i,  &acep, &acecnt, isdir);
701 	if (error)
702 		return (error);
703 
704 	if (dfaclcnt) {
705 		error = ln_aent_to_ace(&aclentp[dfaclstart], dfaclcnt,
706 		    &dfacep, &dfacecnt, isdir);
707 		if (error) {
708 			if (acep) {
709 				cacl_free(acep, acecnt * sizeof (ace_t));
710 			}
711 			return (error);
712 		}
713 	}
714 
715 	if (dfacecnt != 0) {
716 		acesz = sizeof (ace_t) * acecnt;
717 		dfacesz = sizeof (ace_t) * dfacecnt;
718 		acep = cacl_realloc(acep, acesz, acesz + dfacesz);
719 		if (acep == NULL)
720 			return (ENOMEM);
721 		if (dfaclcnt) {
722 			(void) memcpy(acep + acecnt, dfacep, dfacesz);
723 		}
724 	}
725 	if (dfaclcnt)
726 		cacl_free(dfacep, dfacecnt * sizeof (ace_t));
727 
728 	*retacecnt = acecnt + dfacecnt;
729 	*retacep = acep;
730 	return (0);
731 }
732 
733 static int
734 ace_mask_to_mode(uint32_t  mask, o_mode_t *modep, boolean_t isdir)
735 {
736 	int error = 0;
737 	o_mode_t mode = 0;
738 	uint32_t bits, wantbits;
739 
740 	/* read */
741 	if (mask & ACE_READ_DATA)
742 		mode |= S_IROTH;
743 
744 	/* write */
745 	wantbits = (ACE_WRITE_DATA | ACE_APPEND_DATA);
746 	if (isdir)
747 		wantbits |= ACE_DELETE_CHILD;
748 	bits = mask & wantbits;
749 	if (bits != 0) {
750 		if (bits != wantbits) {
751 			error = ENOTSUP;
752 			goto out;
753 		}
754 		mode |= S_IWOTH;
755 	}
756 
757 	/* exec */
758 	if (mask & ACE_EXECUTE) {
759 		mode |= S_IXOTH;
760 	}
761 
762 	*modep = mode;
763 
764 out:
765 	return (error);
766 }
767 
768 static void
769 acevals_init(acevals_t *vals, uid_t key)
770 {
771 	bzero(vals, sizeof (*vals));
772 	vals->allowed = ACE_MASK_UNDEFINED;
773 	vals->denied = ACE_MASK_UNDEFINED;
774 	vals->mask = ACE_MASK_UNDEFINED;
775 	vals->key = key;
776 }
777 
778 static void
779 ace_list_init(ace_list_t *al, int dfacl_flag)
780 {
781 	acevals_init(&al->user_obj, NULL);
782 	acevals_init(&al->group_obj, NULL);
783 	acevals_init(&al->other_obj, NULL);
784 	al->numusers = 0;
785 	al->numgroups = 0;
786 	al->acl_mask = 0;
787 	al->hasmask = 0;
788 	al->state = ace_unused;
789 	al->seen = 0;
790 	al->dfacl_flag = dfacl_flag;
791 }
792 
793 /*
794  * Find or create an acevals holder for a given id and avl tree.
795  *
796  * Note that only one thread will ever touch these avl trees, so
797  * there is no need for locking.
798  */
799 static acevals_t *
800 acevals_find(ace_t *ace, avl_tree_t *avl, int *num)
801 {
802 	acevals_t key, *rc;
803 	avl_index_t where;
804 
805 	key.key = ace->a_who;
806 	rc = avl_find(avl, &key, &where);
807 	if (rc != NULL)
808 		return (rc);
809 
810 	/* this memory is freed by ln_ace_to_aent()->ace_list_free() */
811 	if (cacl_malloc((void **)&rc, sizeof (acevals_t)) != 0)
812 		return (NULL);
813 
814 	acevals_init(rc, ace->a_who);
815 	avl_insert(avl, rc, where);
816 	(*num)++;
817 
818 	return (rc);
819 }
820 
821 static int
822 access_mask_check(ace_t *acep, int mask_bit, int isowner)
823 {
824 	int set_deny, err_deny;
825 	int set_allow, err_allow;
826 	int acl_consume;
827 	int haswriteperm, hasreadperm;
828 
829 	if (acep->a_type == ACE_ACCESS_DENIED_ACE_TYPE) {
830 		haswriteperm = (acep->a_access_mask & ACE_WRITE_DATA) ? 0 : 1;
831 		hasreadperm = (acep->a_access_mask & ACE_READ_DATA) ? 0 : 1;
832 	} else {
833 		haswriteperm = (acep->a_access_mask & ACE_WRITE_DATA) ? 1 : 0;
834 		hasreadperm = (acep->a_access_mask & ACE_READ_DATA) ? 1 : 0;
835 	}
836 
837 	acl_consume = (ACL_SYNCHRONIZE_ERR_DENY |
838 	    ACL_DELETE_ERR_DENY |
839 	    ACL_WRITE_OWNER_ERR_DENY |
840 	    ACL_WRITE_OWNER_ERR_ALLOW |
841 	    ACL_WRITE_ATTRS_OWNER_SET_ALLOW |
842 	    ACL_WRITE_ATTRS_OWNER_ERR_DENY |
843 	    ACL_WRITE_ATTRS_WRITER_SET_DENY |
844 	    ACL_WRITE_ATTRS_WRITER_ERR_ALLOW |
845 	    ACL_WRITE_NAMED_WRITER_ERR_DENY |
846 	    ACL_READ_NAMED_READER_ERR_DENY);
847 
848 	if (mask_bit == ACE_SYNCHRONIZE) {
849 		set_deny = ACL_SYNCHRONIZE_SET_DENY;
850 		err_deny =  ACL_SYNCHRONIZE_ERR_DENY;
851 		set_allow = ACL_SYNCHRONIZE_SET_ALLOW;
852 		err_allow = ACL_SYNCHRONIZE_ERR_ALLOW;
853 	} else if (mask_bit == ACE_WRITE_OWNER) {
854 		set_deny = ACL_WRITE_OWNER_SET_DENY;
855 		err_deny =  ACL_WRITE_OWNER_ERR_DENY;
856 		set_allow = ACL_WRITE_OWNER_SET_ALLOW;
857 		err_allow = ACL_WRITE_OWNER_ERR_ALLOW;
858 	} else if (mask_bit == ACE_DELETE) {
859 		set_deny = ACL_DELETE_SET_DENY;
860 		err_deny =  ACL_DELETE_ERR_DENY;
861 		set_allow = ACL_DELETE_SET_ALLOW;
862 		err_allow = ACL_DELETE_ERR_ALLOW;
863 	} else if (mask_bit == ACE_WRITE_ATTRIBUTES) {
864 		if (isowner) {
865 			set_deny = ACL_WRITE_ATTRS_OWNER_SET_DENY;
866 			err_deny =  ACL_WRITE_ATTRS_OWNER_ERR_DENY;
867 			set_allow = ACL_WRITE_ATTRS_OWNER_SET_ALLOW;
868 			err_allow = ACL_WRITE_ATTRS_OWNER_ERR_ALLOW;
869 		} else if (haswriteperm) {
870 			set_deny = ACL_WRITE_ATTRS_WRITER_SET_DENY;
871 			err_deny =  ACL_WRITE_ATTRS_WRITER_ERR_DENY;
872 			set_allow = ACL_WRITE_ATTRS_WRITER_SET_ALLOW;
873 			err_allow = ACL_WRITE_ATTRS_WRITER_ERR_ALLOW;
874 		} else {
875 			if ((acep->a_access_mask & mask_bit) &&
876 			    (acep->a_type & ACE_ACCESS_ALLOWED_ACE_TYPE)) {
877 				return (ENOTSUP);
878 			}
879 			return (0);
880 		}
881 	} else if (mask_bit == ACE_READ_NAMED_ATTRS) {
882 		if (!hasreadperm)
883 			return (0);
884 
885 		set_deny = ACL_READ_NAMED_READER_SET_DENY;
886 		err_deny = ACL_READ_NAMED_READER_ERR_DENY;
887 		set_allow = ACL_READ_NAMED_READER_SET_ALLOW;
888 		err_allow = ACL_READ_NAMED_READER_ERR_ALLOW;
889 	} else if (mask_bit == ACE_WRITE_NAMED_ATTRS) {
890 		if (!haswriteperm)
891 			return (0);
892 
893 		set_deny = ACL_WRITE_NAMED_WRITER_SET_DENY;
894 		err_deny = ACL_WRITE_NAMED_WRITER_ERR_DENY;
895 		set_allow = ACL_WRITE_NAMED_WRITER_SET_ALLOW;
896 		err_allow = ACL_WRITE_NAMED_WRITER_ERR_ALLOW;
897 	} else {
898 		return (EINVAL);
899 	}
900 
901 	if (acep->a_type == ACE_ACCESS_DENIED_ACE_TYPE) {
902 		if (acl_consume & set_deny) {
903 			if (!(acep->a_access_mask & mask_bit)) {
904 				return (ENOTSUP);
905 			}
906 		} else if (acl_consume & err_deny) {
907 			if (acep->a_access_mask & mask_bit) {
908 				return (ENOTSUP);
909 			}
910 		}
911 	} else {
912 		/* ACE_ACCESS_ALLOWED_ACE_TYPE */
913 		if (acl_consume & set_allow) {
914 			if (!(acep->a_access_mask & mask_bit)) {
915 				return (ENOTSUP);
916 			}
917 		} else if (acl_consume & err_allow) {
918 			if (acep->a_access_mask & mask_bit) {
919 				return (ENOTSUP);
920 			}
921 		}
922 	}
923 	return (0);
924 }
925 
926 static int
927 ace_to_aent_legal(ace_t *acep)
928 {
929 	int error = 0;
930 	int isowner;
931 
932 	/* only ALLOW or DENY */
933 	if ((acep->a_type != ACE_ACCESS_ALLOWED_ACE_TYPE) &&
934 	    (acep->a_type != ACE_ACCESS_DENIED_ACE_TYPE)) {
935 		error = ENOTSUP;
936 		goto out;
937 	}
938 
939 	/* check for invalid flags */
940 	if (acep->a_flags & ~(ACE_VALID_FLAG_BITS)) {
941 		error = EINVAL;
942 		goto out;
943 	}
944 
945 	/* some flags are illegal */
946 	if (acep->a_flags & (ACE_SUCCESSFUL_ACCESS_ACE_FLAG |
947 	    ACE_FAILED_ACCESS_ACE_FLAG |
948 	    ACE_NO_PROPAGATE_INHERIT_ACE)) {
949 		error = ENOTSUP;
950 		goto out;
951 	}
952 
953 	/* check for invalid masks */
954 	if (acep->a_access_mask & ~(ACE_VALID_MASK_BITS)) {
955 		error = EINVAL;
956 		goto out;
957 	}
958 
959 	if ((acep->a_flags & ACE_OWNER)) {
960 		isowner = 1;
961 	} else {
962 		isowner = 0;
963 	}
964 
965 	error = access_mask_check(acep, ACE_SYNCHRONIZE, isowner);
966 	if (error)
967 		goto out;
968 
969 	error = access_mask_check(acep, ACE_WRITE_OWNER, isowner);
970 	if (error)
971 		goto out;
972 
973 	error = access_mask_check(acep, ACE_DELETE, isowner);
974 	if (error)
975 		goto out;
976 
977 	error = access_mask_check(acep, ACE_WRITE_ATTRIBUTES, isowner);
978 	if (error)
979 		goto out;
980 
981 	error = access_mask_check(acep, ACE_READ_NAMED_ATTRS, isowner);
982 	if (error)
983 		goto out;
984 
985 	error = access_mask_check(acep, ACE_WRITE_NAMED_ATTRS, isowner);
986 	if (error)
987 		goto out;
988 
989 	/* more detailed checking of masks */
990 	if (acep->a_type == ACE_ACCESS_ALLOWED_ACE_TYPE) {
991 		if (! (acep->a_access_mask & ACE_READ_ATTRIBUTES)) {
992 			error = ENOTSUP;
993 			goto out;
994 		}
995 		if ((acep->a_access_mask & ACE_WRITE_DATA) &&
996 		    (! (acep->a_access_mask & ACE_APPEND_DATA))) {
997 			error = ENOTSUP;
998 			goto out;
999 		}
1000 		if ((! (acep->a_access_mask & ACE_WRITE_DATA)) &&
1001 		    (acep->a_access_mask & ACE_APPEND_DATA)) {
1002 			error = ENOTSUP;
1003 			goto out;
1004 		}
1005 	}
1006 
1007 	/* ACL enforcement */
1008 	if ((acep->a_access_mask & ACE_READ_ACL) &&
1009 	    (acep->a_type != ACE_ACCESS_ALLOWED_ACE_TYPE)) {
1010 		error = ENOTSUP;
1011 		goto out;
1012 	}
1013 	if (acep->a_access_mask & ACE_WRITE_ACL) {
1014 		if ((acep->a_type == ACE_ACCESS_DENIED_ACE_TYPE) &&
1015 		    (isowner)) {
1016 			error = ENOTSUP;
1017 			goto out;
1018 		}
1019 		if ((acep->a_type == ACE_ACCESS_ALLOWED_ACE_TYPE) &&
1020 		    (! isowner)) {
1021 			error = ENOTSUP;
1022 			goto out;
1023 		}
1024 	}
1025 
1026 out:
1027 	return (error);
1028 }
1029 
1030 static int
1031 ace_allow_to_mode(uint32_t mask, o_mode_t *modep, boolean_t isdir)
1032 {
1033 	/* ACE_READ_ACL and ACE_READ_ATTRIBUTES must both be set */
1034 	if ((mask & (ACE_READ_ACL | ACE_READ_ATTRIBUTES)) !=
1035 	    (ACE_READ_ACL | ACE_READ_ATTRIBUTES)) {
1036 		return (ENOTSUP);
1037 	}
1038 
1039 	return (ace_mask_to_mode(mask, modep, isdir));
1040 }
1041 
1042 static int
1043 acevals_to_aent(acevals_t *vals, aclent_t *dest, ace_list_t *list,
1044     uid_t owner, gid_t group, boolean_t isdir)
1045 {
1046 	int error;
1047 	uint32_t  flips = ACE_POSIX_SUPPORTED_BITS;
1048 
1049 	if (isdir)
1050 		flips |= ACE_DELETE_CHILD;
1051 	if (vals->allowed != (vals->denied ^ flips)) {
1052 		error = ENOTSUP;
1053 		goto out;
1054 	}
1055 	if ((list->hasmask) && (list->acl_mask != vals->mask) &&
1056 	    (vals->aent_type & (USER | GROUP | GROUP_OBJ))) {
1057 		error = ENOTSUP;
1058 		goto out;
1059 	}
1060 	error = ace_allow_to_mode(vals->allowed, &dest->a_perm, isdir);
1061 	if (error != 0)
1062 		goto out;
1063 	dest->a_type = vals->aent_type;
1064 	if (dest->a_type & (USER | GROUP)) {
1065 		dest->a_id = vals->key;
1066 	} else if (dest->a_type & USER_OBJ) {
1067 		dest->a_id = owner;
1068 	} else if (dest->a_type & GROUP_OBJ) {
1069 		dest->a_id = group;
1070 	} else if (dest->a_type & OTHER_OBJ) {
1071 		dest->a_id = 0;
1072 	} else {
1073 		error = EINVAL;
1074 		goto out;
1075 	}
1076 
1077 out:
1078 	return (error);
1079 }
1080 
1081 
1082 static int
1083 ace_list_to_aent(ace_list_t *list, aclent_t **aclentp, int *aclcnt,
1084     uid_t owner, gid_t group, boolean_t isdir)
1085 {
1086 	int error = 0;
1087 	aclent_t *aent, *result = NULL;
1088 	acevals_t *vals;
1089 	int resultcount;
1090 
1091 	if ((list->seen & (USER_OBJ | GROUP_OBJ | OTHER_OBJ)) !=
1092 	    (USER_OBJ | GROUP_OBJ | OTHER_OBJ)) {
1093 		error = ENOTSUP;
1094 		goto out;
1095 	}
1096 	if ((! list->hasmask) && (list->numusers + list->numgroups > 0)) {
1097 		error = ENOTSUP;
1098 		goto out;
1099 	}
1100 
1101 	resultcount = 3 + list->numusers + list->numgroups;
1102 	/*
1103 	 * This must be the same condition as below, when we add the CLASS_OBJ
1104 	 * (aka ACL mask)
1105 	 */
1106 	if ((list->hasmask) || (! list->dfacl_flag))
1107 		resultcount += 1;
1108 
1109 	if (cacl_malloc((void **)&result,
1110 	    resultcount * sizeof (aclent_t)) != 0) {
1111 		error = ENOMEM;
1112 		goto out;
1113 	}
1114 	aent = result;
1115 
1116 	/* USER_OBJ */
1117 	if (!(list->user_obj.aent_type & USER_OBJ)) {
1118 		error = EINVAL;
1119 		goto out;
1120 	}
1121 
1122 	error = acevals_to_aent(&list->user_obj, aent, list, owner, group,
1123 	    isdir);
1124 
1125 	if (error != 0)
1126 		goto out;
1127 	++aent;
1128 	/* USER */
1129 	vals = NULL;
1130 	for (vals = avl_first(&list->user); vals != NULL;
1131 	    vals = AVL_NEXT(&list->user, vals)) {
1132 		if (!(vals->aent_type & USER)) {
1133 			error = EINVAL;
1134 			goto out;
1135 		}
1136 		error = acevals_to_aent(vals, aent, list, owner, group,
1137 		    isdir);
1138 		if (error != 0)
1139 			goto out;
1140 		++aent;
1141 	}
1142 	/* GROUP_OBJ */
1143 	if (!(list->group_obj.aent_type & GROUP_OBJ)) {
1144 		error = EINVAL;
1145 		goto out;
1146 	}
1147 	error = acevals_to_aent(&list->group_obj, aent, list, owner, group,
1148 	    isdir);
1149 	if (error != 0)
1150 		goto out;
1151 	++aent;
1152 	/* GROUP */
1153 	vals = NULL;
1154 	for (vals = avl_first(&list->group); vals != NULL;
1155 	    vals = AVL_NEXT(&list->group, vals)) {
1156 		if (!(vals->aent_type & GROUP)) {
1157 			error = EINVAL;
1158 			goto out;
1159 		}
1160 		error = acevals_to_aent(vals, aent, list, owner, group,
1161 		    isdir);
1162 		if (error != 0)
1163 			goto out;
1164 		++aent;
1165 	}
1166 	/*
1167 	 * CLASS_OBJ (aka ACL_MASK)
1168 	 *
1169 	 * An ACL_MASK is not fabricated if the ACL is a default ACL.
1170 	 * This is to follow UFS's behavior.
1171 	 */
1172 	if ((list->hasmask) || (! list->dfacl_flag)) {
1173 		if (list->hasmask) {
1174 			uint32_t flips = ACE_POSIX_SUPPORTED_BITS;
1175 			if (isdir)
1176 				flips |= ACE_DELETE_CHILD;
1177 			error = ace_mask_to_mode(list->acl_mask ^ flips,
1178 			    &aent->a_perm, isdir);
1179 			if (error != 0)
1180 				goto out;
1181 		} else {
1182 			/* fabricate the ACL_MASK from the group permissions */
1183 			error = ace_mask_to_mode(list->group_obj.allowed,
1184 			    &aent->a_perm, isdir);
1185 			if (error != 0)
1186 				goto out;
1187 		}
1188 		aent->a_id = 0;
1189 		aent->a_type = CLASS_OBJ | list->dfacl_flag;
1190 		++aent;
1191 	}
1192 	/* OTHER_OBJ */
1193 	if (!(list->other_obj.aent_type & OTHER_OBJ)) {
1194 		error = EINVAL;
1195 		goto out;
1196 	}
1197 	error = acevals_to_aent(&list->other_obj, aent, list, owner, group,
1198 	    isdir);
1199 	if (error != 0)
1200 		goto out;
1201 	++aent;
1202 
1203 	*aclentp = result;
1204 	*aclcnt = resultcount;
1205 
1206 out:
1207 	if (error != 0) {
1208 		if (result != NULL)
1209 			cacl_free(result, resultcount * sizeof (aclent_t));
1210 	}
1211 
1212 	return (error);
1213 }
1214 
1215 
1216 /*
1217  * free all data associated with an ace_list
1218  */
1219 static void
1220 ace_list_free(ace_list_t *al)
1221 {
1222 	acevals_t *node;
1223 	void *cookie;
1224 
1225 	if (al == NULL)
1226 		return;
1227 
1228 	cookie = NULL;
1229 	while ((node = avl_destroy_nodes(&al->user, &cookie)) != NULL)
1230 		cacl_free(node, sizeof (acevals_t));
1231 	cookie = NULL;
1232 	while ((node = avl_destroy_nodes(&al->group, &cookie)) != NULL)
1233 		cacl_free(node, sizeof (acevals_t));
1234 
1235 	avl_destroy(&al->user);
1236 	avl_destroy(&al->group);
1237 
1238 	/* free the container itself */
1239 	cacl_free(al, sizeof (ace_list_t));
1240 }
1241 
1242 static int
1243 acevals_compare(const void *va, const void *vb)
1244 {
1245 	const acevals_t *a = va, *b = vb;
1246 
1247 	if (a->key == b->key)
1248 		return (0);
1249 
1250 	if (a->key > b->key)
1251 		return (1);
1252 
1253 	else
1254 		return (-1);
1255 }
1256 
1257 /*
1258  * Convert a list of ace_t entries to equivalent regular and default
1259  * aclent_t lists.  Return error (ENOTSUP) when conversion is not possible.
1260  */
1261 static int
1262 ln_ace_to_aent(ace_t *ace, int n, uid_t owner, gid_t group,
1263     aclent_t **aclentp, int *aclcnt, aclent_t **dfaclentp, int *dfaclcnt,
1264     boolean_t isdir)
1265 {
1266 	int error = 0;
1267 	ace_t *acep;
1268 	uint32_t bits;
1269 	int i;
1270 	ace_list_t *normacl = NULL, *dfacl = NULL, *acl;
1271 	acevals_t *vals;
1272 
1273 	*aclentp = NULL;
1274 	*aclcnt = 0;
1275 	*dfaclentp = NULL;
1276 	*dfaclcnt = 0;
1277 
1278 	/* we need at least user_obj, group_obj, and other_obj */
1279 	if (n < 6) {
1280 		error = ENOTSUP;
1281 		goto out;
1282 	}
1283 	if (ace == NULL) {
1284 		error = EINVAL;
1285 		goto out;
1286 	}
1287 
1288 	error = cacl_malloc((void **)&normacl, sizeof (ace_list_t));
1289 	if (error != 0)
1290 		goto out;
1291 
1292 	avl_create(&normacl->user, acevals_compare, sizeof (acevals_t),
1293 	    offsetof(acevals_t, avl));
1294 	avl_create(&normacl->group, acevals_compare, sizeof (acevals_t),
1295 	    offsetof(acevals_t, avl));
1296 
1297 	ace_list_init(normacl, 0);
1298 
1299 	error = cacl_malloc((void **)&dfacl, sizeof (ace_list_t));
1300 	if (error != 0)
1301 		goto out;
1302 
1303 	avl_create(&dfacl->user, acevals_compare, sizeof (acevals_t),
1304 	    offsetof(acevals_t, avl));
1305 	avl_create(&dfacl->group, acevals_compare, sizeof (acevals_t),
1306 	    offsetof(acevals_t, avl));
1307 	ace_list_init(dfacl, ACL_DEFAULT);
1308 
1309 	/* process every ace_t... */
1310 	for (i = 0; i < n; i++) {
1311 		acep = &ace[i];
1312 
1313 		/* rule out certain cases quickly */
1314 		error = ace_to_aent_legal(acep);
1315 		if (error != 0)
1316 			goto out;
1317 
1318 		/*
1319 		 * Turn off these bits in order to not have to worry about
1320 		 * them when doing the checks for compliments.
1321 		 */
1322 		acep->a_access_mask &= ~(ACE_WRITE_OWNER | ACE_DELETE |
1323 		    ACE_SYNCHRONIZE | ACE_WRITE_ATTRIBUTES |
1324 		    ACE_READ_NAMED_ATTRS | ACE_WRITE_NAMED_ATTRS);
1325 
1326 		/* see if this should be a regular or default acl */
1327 		bits = acep->a_flags &
1328 		    (ACE_INHERIT_ONLY_ACE |
1329 		    ACE_FILE_INHERIT_ACE |
1330 		    ACE_DIRECTORY_INHERIT_ACE);
1331 		if (bits != 0) {
1332 			/* all or nothing on these inherit bits */
1333 			if (bits != (ACE_INHERIT_ONLY_ACE |
1334 			    ACE_FILE_INHERIT_ACE |
1335 			    ACE_DIRECTORY_INHERIT_ACE)) {
1336 				error = ENOTSUP;
1337 				goto out;
1338 			}
1339 			acl = dfacl;
1340 		} else {
1341 			acl = normacl;
1342 		}
1343 
1344 		if ((acep->a_flags & ACE_OWNER)) {
1345 			if (acl->state > ace_user_obj) {
1346 				error = ENOTSUP;
1347 				goto out;
1348 			}
1349 			acl->state = ace_user_obj;
1350 			acl->seen |= USER_OBJ;
1351 			vals = &acl->user_obj;
1352 			vals->aent_type = USER_OBJ | acl->dfacl_flag;
1353 		} else if ((acep->a_flags & ACE_EVERYONE)) {
1354 			acl->state = ace_other_obj;
1355 			acl->seen |= OTHER_OBJ;
1356 			vals = &acl->other_obj;
1357 			vals->aent_type = OTHER_OBJ | acl->dfacl_flag;
1358 		} else if (acep->a_flags & ACE_IDENTIFIER_GROUP) {
1359 			if (acl->state > ace_group) {
1360 				error = ENOTSUP;
1361 				goto out;
1362 			}
1363 			if ((acep->a_flags & ACE_GROUP)) {
1364 				acl->seen |= GROUP_OBJ;
1365 				vals = &acl->group_obj;
1366 				vals->aent_type = GROUP_OBJ | acl->dfacl_flag;
1367 			} else {
1368 				acl->seen |= GROUP;
1369 				vals = acevals_find(acep, &acl->group,
1370 				    &acl->numgroups);
1371 				if (vals == NULL) {
1372 					error = ENOMEM;
1373 					goto out;
1374 				}
1375 				vals->aent_type = GROUP | acl->dfacl_flag;
1376 			}
1377 			acl->state = ace_group;
1378 		} else {
1379 			if (acl->state > ace_user) {
1380 				error = ENOTSUP;
1381 				goto out;
1382 			}
1383 			acl->state = ace_user;
1384 			acl->seen |= USER;
1385 			vals = acevals_find(acep, &acl->user,
1386 			    &acl->numusers);
1387 			if (vals == NULL) {
1388 				error = ENOMEM;
1389 				goto out;
1390 			}
1391 			vals->aent_type = USER | acl->dfacl_flag;
1392 		}
1393 
1394 		if (!(acl->state > ace_unused)) {
1395 			error = EINVAL;
1396 			goto out;
1397 		}
1398 
1399 		if (acep->a_type == ACE_ACCESS_ALLOWED_ACE_TYPE) {
1400 			/* no more than one allowed per aclent_t */
1401 			if (vals->allowed != ACE_MASK_UNDEFINED) {
1402 				error = ENOTSUP;
1403 				goto out;
1404 			}
1405 			vals->allowed = acep->a_access_mask;
1406 		} else {
1407 			/*
1408 			 * it's a DENY; if there was a previous DENY, it
1409 			 * must have been an ACL_MASK.
1410 			 */
1411 			if (vals->denied != ACE_MASK_UNDEFINED) {
1412 				/* ACL_MASK is for USER and GROUP only */
1413 				if ((acl->state != ace_user) &&
1414 				    (acl->state != ace_group)) {
1415 					error = ENOTSUP;
1416 					goto out;
1417 				}
1418 
1419 				if (! acl->hasmask) {
1420 					acl->hasmask = 1;
1421 					acl->acl_mask = vals->denied;
1422 				/* check for mismatched ACL_MASK emulations */
1423 				} else if (acl->acl_mask != vals->denied) {
1424 					error = ENOTSUP;
1425 					goto out;
1426 				}
1427 				vals->mask = vals->denied;
1428 			}
1429 			vals->denied = acep->a_access_mask;
1430 		}
1431 	}
1432 
1433 	/* done collating; produce the aclent_t lists */
1434 	if (normacl->state != ace_unused) {
1435 		error = ace_list_to_aent(normacl, aclentp, aclcnt,
1436 		    owner, group, isdir);
1437 		if (error != 0) {
1438 			goto out;
1439 		}
1440 	}
1441 	if (dfacl->state != ace_unused) {
1442 		error = ace_list_to_aent(dfacl, dfaclentp, dfaclcnt,
1443 		    owner, group, isdir);
1444 		if (error != 0) {
1445 			goto out;
1446 		}
1447 	}
1448 
1449 out:
1450 	if (normacl != NULL)
1451 		ace_list_free(normacl);
1452 	if (dfacl != NULL)
1453 		ace_list_free(dfacl);
1454 
1455 	return (error);
1456 }
1457 
1458 static int
1459 convert_ace_to_aent(ace_t *acebufp, int acecnt, boolean_t isdir,
1460     uid_t owner, gid_t group, aclent_t **retaclentp, int *retaclcnt)
1461 {
1462 	int error = 0;
1463 	aclent_t *aclentp, *dfaclentp;
1464 	int aclcnt, dfaclcnt;
1465 	int aclsz, dfaclsz;
1466 
1467 	error = ln_ace_to_aent(acebufp, acecnt, owner, group,
1468 	    &aclentp, &aclcnt, &dfaclentp, &dfaclcnt, isdir);
1469 
1470 	if (error)
1471 		return (error);
1472 
1473 
1474 	if (dfaclcnt != 0) {
1475 		/*
1476 		 * Slap aclentp and dfaclentp into a single array.
1477 		 */
1478 		aclsz = sizeof (aclent_t) * aclcnt;
1479 		dfaclsz = sizeof (aclent_t) * dfaclcnt;
1480 		aclentp = cacl_realloc(aclentp, aclsz, aclsz + dfaclsz);
1481 		if (aclentp != NULL) {
1482 			(void) memcpy(aclentp + aclcnt, dfaclentp, dfaclsz);
1483 		} else {
1484 			error = ENOMEM;
1485 		}
1486 	}
1487 
1488 	if (aclentp) {
1489 		*retaclentp = aclentp;
1490 		*retaclcnt = aclcnt + dfaclcnt;
1491 	}
1492 
1493 	if (dfaclentp)
1494 		cacl_free(dfaclentp, dfaclsz);
1495 
1496 	return (error);
1497 }
1498 
1499 
1500 int
1501 acl_translate(acl_t *aclp, int target_flavor, boolean_t isdir, uid_t owner,
1502     gid_t group)
1503 {
1504 	int aclcnt;
1505 	void *acldata;
1506 	int error;
1507 
1508 	/*
1509 	 * See if we need to translate
1510 	 */
1511 	if ((target_flavor == _ACL_ACE_ENABLED && aclp->acl_type == ACE_T) ||
1512 	    (target_flavor == _ACL_ACLENT_ENABLED &&
1513 	    aclp->acl_type == ACLENT_T))
1514 		return (0);
1515 
1516 	if (target_flavor == -1) {
1517 		error = EINVAL;
1518 		goto out;
1519 	}
1520 
1521 	if (target_flavor ==  _ACL_ACE_ENABLED &&
1522 	    aclp->acl_type == ACLENT_T) {
1523 		error = convert_aent_to_ace(aclp->acl_aclp,
1524 		    aclp->acl_cnt, isdir, (ace_t **)&acldata, &aclcnt);
1525 		if (error)
1526 			goto out;
1527 
1528 	} else if (target_flavor == _ACL_ACLENT_ENABLED &&
1529 	    aclp->acl_type == ACE_T) {
1530 		error = convert_ace_to_aent(aclp->acl_aclp, aclp->acl_cnt,
1531 		    isdir, owner, group, (aclent_t **)&acldata, &aclcnt);
1532 		if (error)
1533 			goto out;
1534 	} else {
1535 		error = ENOTSUP;
1536 		goto out;
1537 	}
1538 
1539 	/*
1540 	 * replace old acl with newly translated acl
1541 	 */
1542 	cacl_free(aclp->acl_aclp, aclp->acl_cnt * aclp->acl_entry_size);
1543 	aclp->acl_aclp = acldata;
1544 	aclp->acl_cnt = aclcnt;
1545 	if (target_flavor == _ACL_ACE_ENABLED) {
1546 		aclp->acl_type = ACE_T;
1547 		aclp->acl_entry_size = sizeof (ace_t);
1548 	} else {
1549 		aclp->acl_type = ACLENT_T;
1550 		aclp->acl_entry_size = sizeof (aclent_t);
1551 	}
1552 	return (0);
1553 
1554 out:
1555 
1556 #if !defined(_KERNEL)
1557 	errno = error;
1558 	return (-1);
1559 #else
1560 	return (error);
1561 #endif
1562 }
1563 
1564 #define	SET_ACE(acl, index, who, mask, type, flags) { \
1565 	acl[0][index].a_who = (uint32_t)who; \
1566 	acl[0][index].a_type = type; \
1567 	acl[0][index].a_flags = flags; \
1568 	acl[0][index++].a_access_mask = mask; \
1569 }
1570 
1571 void
1572 acl_trivial_access_masks(mode_t mode, boolean_t isdir, trivial_acl_t *masks)
1573 {
1574 	uint32_t read_mask = ACE_READ_DATA;
1575 	uint32_t write_mask = ACE_WRITE_DATA|ACE_APPEND_DATA;
1576 	uint32_t execute_mask = ACE_EXECUTE;
1577 
1578 	(void) isdir;	/* will need this later */
1579 
1580 	masks->deny1 = 0;
1581 	if (!(mode & S_IRUSR) && (mode & (S_IRGRP|S_IROTH)))
1582 		masks->deny1 |= read_mask;
1583 	if (!(mode & S_IWUSR) && (mode & (S_IWGRP|S_IWOTH)))
1584 		masks->deny1 |= write_mask;
1585 	if (!(mode & S_IXUSR) && (mode & (S_IXGRP|S_IXOTH)))
1586 		masks->deny1 |= execute_mask;
1587 
1588 	masks->deny2 = 0;
1589 	if (!(mode & S_IRGRP) && (mode & S_IROTH))
1590 		masks->deny2 |= read_mask;
1591 	if (!(mode & S_IWGRP) && (mode & S_IWOTH))
1592 		masks->deny2 |= write_mask;
1593 	if (!(mode & S_IXGRP) && (mode & S_IXOTH))
1594 		masks->deny2 |= execute_mask;
1595 
1596 	masks->allow0 = 0;
1597 	if ((mode & S_IRUSR) && (!(mode & S_IRGRP) && (mode & S_IROTH)))
1598 		masks->allow0 |= read_mask;
1599 	if ((mode & S_IWUSR) && (!(mode & S_IWGRP) && (mode & S_IWOTH)))
1600 		masks->allow0 |= write_mask;
1601 	if ((mode & S_IXUSR) && (!(mode & S_IXGRP) && (mode & S_IXOTH)))
1602 		masks->allow0 |= execute_mask;
1603 
1604 	masks->owner = ACE_WRITE_ATTRIBUTES|ACE_WRITE_OWNER|ACE_WRITE_ACL|
1605 	    ACE_WRITE_NAMED_ATTRS|ACE_READ_ACL|ACE_READ_ATTRIBUTES|
1606 	    ACE_READ_NAMED_ATTRS|ACE_SYNCHRONIZE;
1607 	if (mode & S_IRUSR)
1608 		masks->owner |= read_mask;
1609 	if (mode & S_IWUSR)
1610 		masks->owner |= write_mask;
1611 	if (mode & S_IXUSR)
1612 		masks->owner |= execute_mask;
1613 
1614 	masks->group = ACE_READ_ACL|ACE_READ_ATTRIBUTES|ACE_READ_NAMED_ATTRS|
1615 	    ACE_SYNCHRONIZE;
1616 	if (mode & S_IRGRP)
1617 		masks->group |= read_mask;
1618 	if (mode & S_IWGRP)
1619 		masks->group |= write_mask;
1620 	if (mode & S_IXGRP)
1621 		masks->group |= execute_mask;
1622 
1623 	masks->everyone = ACE_READ_ACL|ACE_READ_ATTRIBUTES|ACE_READ_NAMED_ATTRS|
1624 	    ACE_SYNCHRONIZE;
1625 	if (mode & S_IROTH)
1626 		masks->everyone |= read_mask;
1627 	if (mode & S_IWOTH)
1628 		masks->everyone |= write_mask;
1629 	if (mode & S_IXOTH)
1630 		masks->everyone |= execute_mask;
1631 }
1632 
1633 int
1634 acl_trivial_create(mode_t mode, boolean_t isdir, ace_t **acl, int *count)
1635 {
1636 	int 		index = 0;
1637 	int		error;
1638 	trivial_acl_t	masks;
1639 
1640 	*count = 3;
1641 	acl_trivial_access_masks(mode, isdir, &masks);
1642 
1643 	if (masks.allow0)
1644 		(*count)++;
1645 	if (masks.deny1)
1646 		(*count)++;
1647 	if (masks.deny2)
1648 		(*count)++;
1649 
1650 	if ((error = cacl_malloc((void **)acl, *count * sizeof (ace_t))) != 0)
1651 		return (error);
1652 
1653 	if (masks.allow0) {
1654 		SET_ACE(acl, index, -1, masks.allow0,
1655 		    ACE_ACCESS_ALLOWED_ACE_TYPE, ACE_OWNER);
1656 	}
1657 	if (masks.deny1) {
1658 		SET_ACE(acl, index, -1, masks.deny1,
1659 		    ACE_ACCESS_DENIED_ACE_TYPE, ACE_OWNER);
1660 	}
1661 	if (masks.deny2) {
1662 		SET_ACE(acl, index, -1, masks.deny2,
1663 		    ACE_ACCESS_DENIED_ACE_TYPE, ACE_GROUP|ACE_IDENTIFIER_GROUP);
1664 	}
1665 
1666 	SET_ACE(acl, index, -1, masks.owner, ACE_ACCESS_ALLOWED_ACE_TYPE,
1667 	    ACE_OWNER);
1668 	SET_ACE(acl, index, -1, masks.group, ACE_ACCESS_ALLOWED_ACE_TYPE,
1669 	    ACE_IDENTIFIER_GROUP|ACE_GROUP);
1670 	SET_ACE(acl, index, -1, masks.everyone, ACE_ACCESS_ALLOWED_ACE_TYPE,
1671 	    ACE_EVERYONE);
1672 
1673 	return (0);
1674 }
1675 
1676 /*
1677  * ace_trivial:
1678  * determine whether an ace_t acl is trivial
1679  *
1680  * Trivialness implies that the acl is composed of only
1681  * owner, group, everyone entries.  ACL can't
1682  * have read_acl denied, and write_owner/write_acl/write_attributes
1683  * can only be owner@ entry.
1684  */
1685 int
1686 ace_trivial_common(void *acep, int aclcnt,
1687     uint64_t (*walk)(void *, uint64_t, int aclcnt,
1688     uint16_t *, uint16_t *, uint32_t *))
1689 {
1690 	uint16_t flags;
1691 	uint32_t mask;
1692 	uint16_t type;
1693 	uint64_t cookie = 0;
1694 
1695 	while (cookie = walk(acep, cookie, aclcnt, &flags, &type, &mask)) {
1696 		switch (flags & ACE_TYPE_FLAGS) {
1697 		case ACE_OWNER:
1698 		case ACE_GROUP|ACE_IDENTIFIER_GROUP:
1699 		case ACE_EVERYONE:
1700 			break;
1701 		default:
1702 			return (1);
1703 
1704 		}
1705 
1706 		if (flags & (ACE_FILE_INHERIT_ACE|
1707 		    ACE_DIRECTORY_INHERIT_ACE|ACE_NO_PROPAGATE_INHERIT_ACE|
1708 		    ACE_INHERIT_ONLY_ACE))
1709 			return (1);
1710 
1711 		/*
1712 		 * Special check for some special bits
1713 		 *
1714 		 * Don't allow anybody to deny reading basic
1715 		 * attributes or a files ACL.
1716 		 */
1717 		if ((mask & (ACE_READ_ACL|ACE_READ_ATTRIBUTES)) &&
1718 		    (type == ACE_ACCESS_DENIED_ACE_TYPE))
1719 			return (1);
1720 
1721 		/*
1722 		 * Delete permissions are never set by default
1723 		 */
1724 		if (mask & (ACE_DELETE|ACE_DELETE_CHILD))
1725 			return (1);
1726 		/*
1727 		 * only allow owner@ to have
1728 		 * write_acl/write_owner/write_attributes/write_xattr/
1729 		 */
1730 		if (type == ACE_ACCESS_ALLOWED_ACE_TYPE &&
1731 		    (!(flags & ACE_OWNER) && (mask &
1732 		    (ACE_WRITE_OWNER|ACE_WRITE_ACL| ACE_WRITE_ATTRIBUTES|
1733 		    ACE_WRITE_NAMED_ATTRS))))
1734 			return (1);
1735 
1736 	}
1737 	return (0);
1738 }
1739 
1740 uint64_t
1741 ace_walk(void *datap, uint64_t cookie, int aclcnt, uint16_t *flags,
1742     uint16_t *type, uint32_t *mask)
1743 {
1744 	ace_t *acep = datap;
1745 
1746 	if (cookie >= aclcnt)
1747 		return (0);
1748 
1749 	*flags = acep[cookie].a_flags;
1750 	*type = acep[cookie].a_type;
1751 	*mask = acep[cookie++].a_access_mask;
1752 
1753 	return (cookie);
1754 }
1755 
1756 int
1757 ace_trivial(ace_t *acep, int aclcnt)
1758 {
1759 	return (ace_trivial_common(acep, aclcnt, ace_walk));
1760 }
1761