xref: /titanic_41/usr/src/common/acl/acl_common.c (revision dfbb3a42fe4d6d7c50b9f750f514f8c7b66e4ed9)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 /*
22  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
23  */
24 
25 #include <sys/types.h>
26 #include <sys/stat.h>
27 #include <sys/avl.h>
28 #if defined(_KERNEL)
29 #include <sys/systm.h>
30 #include <sys/sysmacros.h>
31 #include <acl/acl_common.h>
32 #else
33 #include <errno.h>
34 #include <stdlib.h>
35 #include <stddef.h>
36 #include <strings.h>
37 #include <unistd.h>
38 #include <assert.h>
39 #include <grp.h>
40 #include <pwd.h>
41 #include <acl_common.h>
42 #define	ASSERT	assert
43 #endif
44 
45 #define	ACE_POSIX_SUPPORTED_BITS (ACE_READ_DATA | \
46     ACE_WRITE_DATA | ACE_APPEND_DATA | ACE_EXECUTE | \
47     ACE_READ_ATTRIBUTES | ACE_READ_ACL | ACE_WRITE_ACL)
48 
49 
50 #define	ACL_SYNCHRONIZE_SET_DENY		0x0000001
51 #define	ACL_SYNCHRONIZE_SET_ALLOW		0x0000002
52 #define	ACL_SYNCHRONIZE_ERR_DENY		0x0000004
53 #define	ACL_SYNCHRONIZE_ERR_ALLOW		0x0000008
54 
55 #define	ACL_WRITE_OWNER_SET_DENY		0x0000010
56 #define	ACL_WRITE_OWNER_SET_ALLOW		0x0000020
57 #define	ACL_WRITE_OWNER_ERR_DENY		0x0000040
58 #define	ACL_WRITE_OWNER_ERR_ALLOW		0x0000080
59 
60 #define	ACL_DELETE_SET_DENY			0x0000100
61 #define	ACL_DELETE_SET_ALLOW			0x0000200
62 #define	ACL_DELETE_ERR_DENY			0x0000400
63 #define	ACL_DELETE_ERR_ALLOW			0x0000800
64 
65 #define	ACL_WRITE_ATTRS_OWNER_SET_DENY		0x0001000
66 #define	ACL_WRITE_ATTRS_OWNER_SET_ALLOW		0x0002000
67 #define	ACL_WRITE_ATTRS_OWNER_ERR_DENY		0x0004000
68 #define	ACL_WRITE_ATTRS_OWNER_ERR_ALLOW		0x0008000
69 
70 #define	ACL_WRITE_ATTRS_WRITER_SET_DENY		0x0010000
71 #define	ACL_WRITE_ATTRS_WRITER_SET_ALLOW	0x0020000
72 #define	ACL_WRITE_ATTRS_WRITER_ERR_DENY		0x0040000
73 #define	ACL_WRITE_ATTRS_WRITER_ERR_ALLOW	0x0080000
74 
75 #define	ACL_WRITE_NAMED_WRITER_SET_DENY		0x0100000
76 #define	ACL_WRITE_NAMED_WRITER_SET_ALLOW	0x0200000
77 #define	ACL_WRITE_NAMED_WRITER_ERR_DENY		0x0400000
78 #define	ACL_WRITE_NAMED_WRITER_ERR_ALLOW	0x0800000
79 
80 #define	ACL_READ_NAMED_READER_SET_DENY		0x1000000
81 #define	ACL_READ_NAMED_READER_SET_ALLOW		0x2000000
82 #define	ACL_READ_NAMED_READER_ERR_DENY		0x4000000
83 #define	ACL_READ_NAMED_READER_ERR_ALLOW		0x8000000
84 
85 
86 #define	ACE_VALID_MASK_BITS (\
87     ACE_READ_DATA | \
88     ACE_LIST_DIRECTORY | \
89     ACE_WRITE_DATA | \
90     ACE_ADD_FILE | \
91     ACE_APPEND_DATA | \
92     ACE_ADD_SUBDIRECTORY | \
93     ACE_READ_NAMED_ATTRS | \
94     ACE_WRITE_NAMED_ATTRS | \
95     ACE_EXECUTE | \
96     ACE_DELETE_CHILD | \
97     ACE_READ_ATTRIBUTES | \
98     ACE_WRITE_ATTRIBUTES | \
99     ACE_DELETE | \
100     ACE_READ_ACL | \
101     ACE_WRITE_ACL | \
102     ACE_WRITE_OWNER | \
103     ACE_SYNCHRONIZE)
104 
105 #define	ACE_MASK_UNDEFINED			0x80000000
106 
107 #define	ACE_VALID_FLAG_BITS (ACE_FILE_INHERIT_ACE | \
108     ACE_DIRECTORY_INHERIT_ACE | \
109     ACE_NO_PROPAGATE_INHERIT_ACE | ACE_INHERIT_ONLY_ACE | \
110     ACE_SUCCESSFUL_ACCESS_ACE_FLAG | ACE_FAILED_ACCESS_ACE_FLAG | \
111     ACE_IDENTIFIER_GROUP | ACE_OWNER | ACE_GROUP | ACE_EVERYONE)
112 
113 /*
114  * ACL conversion helpers
115  */
116 
117 typedef enum {
118 	ace_unused,
119 	ace_user_obj,
120 	ace_user,
121 	ace_group, /* includes GROUP and GROUP_OBJ */
122 	ace_other_obj
123 } ace_to_aent_state_t;
124 
125 typedef struct acevals {
126 	uid_t key;
127 	avl_node_t avl;
128 	uint32_t mask;
129 	uint32_t allowed;
130 	uint32_t denied;
131 	int aent_type;
132 } acevals_t;
133 
134 typedef struct ace_list {
135 	acevals_t user_obj;
136 	avl_tree_t user;
137 	int numusers;
138 	acevals_t group_obj;
139 	avl_tree_t group;
140 	int numgroups;
141 	acevals_t other_obj;
142 	uint32_t acl_mask;
143 	int hasmask;
144 	int dfacl_flag;
145 	ace_to_aent_state_t state;
146 	int seen; /* bitmask of all aclent_t a_type values seen */
147 } ace_list_t;
148 
149 /*
150  * Generic shellsort, from K&R (1st ed, p 58.), somewhat modified.
151  * v = Ptr to array/vector of objs
152  * n = # objs in the array
153  * s = size of each obj (must be multiples of a word size)
154  * f = ptr to function to compare two objs
155  *	returns (-1 = less than, 0 = equal, 1 = greater than
156  */
157 void
158 ksort(caddr_t v, int n, int s, int (*f)())
159 {
160 	int g, i, j, ii;
161 	unsigned int *p1, *p2;
162 	unsigned int tmp;
163 
164 	/* No work to do */
165 	if (v == NULL || n <= 1)
166 		return;
167 
168 	/* Sanity check on arguments */
169 	ASSERT(((uintptr_t)v & 0x3) == 0 && (s & 0x3) == 0);
170 	ASSERT(s > 0);
171 	for (g = n / 2; g > 0; g /= 2) {
172 		for (i = g; i < n; i++) {
173 			for (j = i - g; j >= 0 &&
174 			    (*f)(v + j * s, v + (j + g) * s) == 1;
175 			    j -= g) {
176 				p1 = (void *)(v + j * s);
177 				p2 = (void *)(v + (j + g) * s);
178 				for (ii = 0; ii < s / 4; ii++) {
179 					tmp = *p1;
180 					*p1++ = *p2;
181 					*p2++ = tmp;
182 				}
183 			}
184 		}
185 	}
186 }
187 
188 /*
189  * Compare two acls, all fields.  Returns:
190  * -1 (less than)
191  *  0 (equal)
192  * +1 (greater than)
193  */
194 int
195 cmp2acls(void *a, void *b)
196 {
197 	aclent_t *x = (aclent_t *)a;
198 	aclent_t *y = (aclent_t *)b;
199 
200 	/* Compare types */
201 	if (x->a_type < y->a_type)
202 		return (-1);
203 	if (x->a_type > y->a_type)
204 		return (1);
205 	/* Equal types; compare id's */
206 	if (x->a_id < y->a_id)
207 		return (-1);
208 	if (x->a_id > y->a_id)
209 		return (1);
210 	/* Equal ids; compare perms */
211 	if (x->a_perm < y->a_perm)
212 		return (-1);
213 	if (x->a_perm > y->a_perm)
214 		return (1);
215 	/* Totally equal */
216 	return (0);
217 }
218 
219 /*ARGSUSED*/
220 static void *
221 cacl_realloc(void *ptr, size_t size, size_t new_size)
222 {
223 #if defined(_KERNEL)
224 	void *tmp;
225 
226 	tmp = kmem_alloc(new_size, KM_SLEEP);
227 	(void) memcpy(tmp, ptr, (size < new_size) ? size : new_size);
228 	kmem_free(ptr, size);
229 	return (tmp);
230 #else
231 	return (realloc(ptr, new_size));
232 #endif
233 }
234 
235 static int
236 cacl_malloc(void **ptr, size_t size)
237 {
238 #if defined(_KERNEL)
239 	*ptr = kmem_zalloc(size, KM_SLEEP);
240 	return (0);
241 #else
242 	*ptr = calloc(1, size);
243 	if (*ptr == NULL)
244 		return (errno);
245 
246 	return (0);
247 #endif
248 }
249 
250 /*ARGSUSED*/
251 static void
252 cacl_free(void *ptr, size_t size)
253 {
254 #if defined(_KERNEL)
255 	kmem_free(ptr, size);
256 #else
257 	free(ptr);
258 #endif
259 }
260 
261 acl_t *
262 acl_alloc(enum acl_type type)
263 {
264 	acl_t *aclp;
265 
266 	if (cacl_malloc((void **)&aclp, sizeof (acl_t)) != 0)
267 		return (NULL);
268 
269 	aclp->acl_aclp = NULL;
270 	aclp->acl_cnt = 0;
271 
272 	switch (type) {
273 	case ACE_T:
274 		aclp->acl_type = ACE_T;
275 		aclp->acl_entry_size = sizeof (ace_t);
276 		break;
277 	case ACLENT_T:
278 		aclp->acl_type = ACLENT_T;
279 		aclp->acl_entry_size = sizeof (aclent_t);
280 		break;
281 	default:
282 		acl_free(aclp);
283 		aclp = NULL;
284 	}
285 	return (aclp);
286 }
287 
288 /*
289  * Free acl_t structure
290  */
291 void
292 acl_free(acl_t *aclp)
293 {
294 	int acl_size;
295 
296 	if (aclp == NULL)
297 		return;
298 
299 	if (aclp->acl_aclp) {
300 		acl_size = aclp->acl_cnt * aclp->acl_entry_size;
301 		cacl_free(aclp->acl_aclp, acl_size);
302 	}
303 
304 	cacl_free(aclp, sizeof (acl_t));
305 }
306 
307 static uint32_t
308 access_mask_set(int haswriteperm, int hasreadperm, int isowner, int isallow)
309 {
310 	uint32_t access_mask = 0;
311 	int acl_produce;
312 	int synchronize_set = 0, write_owner_set = 0;
313 	int delete_set = 0, write_attrs_set = 0;
314 	int read_named_set = 0, write_named_set = 0;
315 
316 	acl_produce = (ACL_SYNCHRONIZE_SET_ALLOW |
317 	    ACL_WRITE_ATTRS_OWNER_SET_ALLOW |
318 	    ACL_WRITE_ATTRS_WRITER_SET_DENY);
319 
320 	if (isallow) {
321 		synchronize_set = ACL_SYNCHRONIZE_SET_ALLOW;
322 		write_owner_set = ACL_WRITE_OWNER_SET_ALLOW;
323 		delete_set = ACL_DELETE_SET_ALLOW;
324 		if (hasreadperm)
325 			read_named_set = ACL_READ_NAMED_READER_SET_ALLOW;
326 		if (haswriteperm)
327 			write_named_set = ACL_WRITE_NAMED_WRITER_SET_ALLOW;
328 		if (isowner)
329 			write_attrs_set = ACL_WRITE_ATTRS_OWNER_SET_ALLOW;
330 		else if (haswriteperm)
331 			write_attrs_set = ACL_WRITE_ATTRS_WRITER_SET_ALLOW;
332 	} else {
333 
334 		synchronize_set = ACL_SYNCHRONIZE_SET_DENY;
335 		write_owner_set = ACL_WRITE_OWNER_SET_DENY;
336 		delete_set = ACL_DELETE_SET_DENY;
337 		if (hasreadperm)
338 			read_named_set = ACL_READ_NAMED_READER_SET_DENY;
339 		if (haswriteperm)
340 			write_named_set = ACL_WRITE_NAMED_WRITER_SET_DENY;
341 		if (isowner)
342 			write_attrs_set = ACL_WRITE_ATTRS_OWNER_SET_DENY;
343 		else if (haswriteperm)
344 			write_attrs_set = ACL_WRITE_ATTRS_WRITER_SET_DENY;
345 		else
346 			/*
347 			 * If the entity is not the owner and does not
348 			 * have write permissions ACE_WRITE_ATTRIBUTES will
349 			 * always go in the DENY ACE.
350 			 */
351 			access_mask |= ACE_WRITE_ATTRIBUTES;
352 	}
353 
354 	if (acl_produce & synchronize_set)
355 		access_mask |= ACE_SYNCHRONIZE;
356 	if (acl_produce & write_owner_set)
357 		access_mask |= ACE_WRITE_OWNER;
358 	if (acl_produce & delete_set)
359 		access_mask |= ACE_DELETE;
360 	if (acl_produce & write_attrs_set)
361 		access_mask |= ACE_WRITE_ATTRIBUTES;
362 	if (acl_produce & read_named_set)
363 		access_mask |= ACE_READ_NAMED_ATTRS;
364 	if (acl_produce & write_named_set)
365 		access_mask |= ACE_WRITE_NAMED_ATTRS;
366 
367 	return (access_mask);
368 }
369 
370 /*
371  * Given an mode_t, convert it into an access_mask as used
372  * by nfsace, assuming aclent_t -> nfsace semantics.
373  */
374 static uint32_t
375 mode_to_ace_access(mode_t mode, int isdir, int isowner, int isallow)
376 {
377 	uint32_t access = 0;
378 	int haswriteperm = 0;
379 	int hasreadperm = 0;
380 
381 	if (isallow) {
382 		haswriteperm = (mode & S_IWOTH);
383 		hasreadperm = (mode & S_IROTH);
384 	} else {
385 		haswriteperm = !(mode & S_IWOTH);
386 		hasreadperm = !(mode & S_IROTH);
387 	}
388 
389 	/*
390 	 * The following call takes care of correctly setting the following
391 	 * mask bits in the access_mask:
392 	 * ACE_SYNCHRONIZE, ACE_WRITE_OWNER, ACE_DELETE,
393 	 * ACE_WRITE_ATTRIBUTES, ACE_WRITE_NAMED_ATTRS, ACE_READ_NAMED_ATTRS
394 	 */
395 	access = access_mask_set(haswriteperm, hasreadperm, isowner, isallow);
396 
397 	if (isallow) {
398 		access |= ACE_READ_ACL | ACE_READ_ATTRIBUTES;
399 		if (isowner)
400 			access |= ACE_WRITE_ACL;
401 	} else {
402 		if (! isowner)
403 			access |= ACE_WRITE_ACL;
404 	}
405 
406 	/* read */
407 	if (mode & S_IROTH) {
408 		access |= ACE_READ_DATA;
409 	}
410 	/* write */
411 	if (mode & S_IWOTH) {
412 		access |= ACE_WRITE_DATA |
413 		    ACE_APPEND_DATA;
414 		if (isdir)
415 			access |= ACE_DELETE_CHILD;
416 	}
417 	/* exec */
418 	if (mode & 01) {
419 		access |= ACE_EXECUTE;
420 	}
421 
422 	return (access);
423 }
424 
425 /*
426  * Given an nfsace (presumably an ALLOW entry), make a
427  * corresponding DENY entry at the address given.
428  */
429 static void
430 ace_make_deny(ace_t *allow, ace_t *deny, int isdir, int isowner)
431 {
432 	(void) memcpy(deny, allow, sizeof (ace_t));
433 
434 	deny->a_who = allow->a_who;
435 
436 	deny->a_type = ACE_ACCESS_DENIED_ACE_TYPE;
437 	deny->a_access_mask ^= ACE_POSIX_SUPPORTED_BITS;
438 	if (isdir)
439 		deny->a_access_mask ^= ACE_DELETE_CHILD;
440 
441 	deny->a_access_mask &= ~(ACE_SYNCHRONIZE | ACE_WRITE_OWNER |
442 	    ACE_DELETE | ACE_WRITE_ATTRIBUTES | ACE_READ_NAMED_ATTRS |
443 	    ACE_WRITE_NAMED_ATTRS);
444 	deny->a_access_mask |= access_mask_set((allow->a_access_mask &
445 	    ACE_WRITE_DATA), (allow->a_access_mask & ACE_READ_DATA), isowner,
446 	    B_FALSE);
447 }
448 /*
449  * Make an initial pass over an array of aclent_t's.  Gather
450  * information such as an ACL_MASK (if any), number of users,
451  * number of groups, and whether the array needs to be sorted.
452  */
453 static int
454 ln_aent_preprocess(aclent_t *aclent, int n,
455     int *hasmask, mode_t *mask,
456     int *numuser, int *numgroup, int *needsort)
457 {
458 	int error = 0;
459 	int i;
460 	int curtype = 0;
461 
462 	*hasmask = 0;
463 	*mask = 07;
464 	*needsort = 0;
465 	*numuser = 0;
466 	*numgroup = 0;
467 
468 	for (i = 0; i < n; i++) {
469 		if (aclent[i].a_type < curtype)
470 			*needsort = 1;
471 		else if (aclent[i].a_type > curtype)
472 			curtype = aclent[i].a_type;
473 		if (aclent[i].a_type & USER)
474 			(*numuser)++;
475 		if (aclent[i].a_type & (GROUP | GROUP_OBJ))
476 			(*numgroup)++;
477 		if (aclent[i].a_type & CLASS_OBJ) {
478 			if (*hasmask) {
479 				error = EINVAL;
480 				goto out;
481 			} else {
482 				*hasmask = 1;
483 				*mask = aclent[i].a_perm;
484 			}
485 		}
486 	}
487 
488 	if ((! *hasmask) && (*numuser + *numgroup > 1)) {
489 		error = EINVAL;
490 		goto out;
491 	}
492 
493 out:
494 	return (error);
495 }
496 
497 /*
498  * Convert an array of aclent_t into an array of nfsace entries,
499  * following POSIX draft -> nfsv4 conversion semantics as outlined in
500  * the IETF draft.
501  */
502 static int
503 ln_aent_to_ace(aclent_t *aclent, int n, ace_t **acepp, int *rescount, int isdir)
504 {
505 	int error = 0;
506 	mode_t mask;
507 	int numuser, numgroup, needsort;
508 	int resultsize = 0;
509 	int i, groupi = 0, skip;
510 	ace_t *acep, *result = NULL;
511 	int hasmask;
512 
513 	error = ln_aent_preprocess(aclent, n, &hasmask, &mask,
514 	    &numuser, &numgroup, &needsort);
515 	if (error != 0)
516 		goto out;
517 
518 	/* allow + deny for each aclent */
519 	resultsize = n * 2;
520 	if (hasmask) {
521 		/*
522 		 * stick extra deny on the group_obj and on each
523 		 * user|group for the mask (the group_obj was added
524 		 * into the count for numgroup)
525 		 */
526 		resultsize += numuser + numgroup;
527 		/* ... and don't count the mask itself */
528 		resultsize -= 2;
529 	}
530 
531 	/* sort the source if necessary */
532 	if (needsort)
533 		ksort((caddr_t)aclent, n, sizeof (aclent_t), cmp2acls);
534 
535 	if (cacl_malloc((void **)&result, resultsize * sizeof (ace_t)) != 0)
536 		goto out;
537 
538 	acep = result;
539 
540 	for (i = 0; i < n; i++) {
541 		/*
542 		 * don't process CLASS_OBJ (mask); mask was grabbed in
543 		 * ln_aent_preprocess()
544 		 */
545 		if (aclent[i].a_type & CLASS_OBJ)
546 			continue;
547 
548 		/* If we need an ACL_MASK emulator, prepend it now */
549 		if ((hasmask) &&
550 		    (aclent[i].a_type & (USER | GROUP | GROUP_OBJ))) {
551 			acep->a_type = ACE_ACCESS_DENIED_ACE_TYPE;
552 			acep->a_flags = 0;
553 			if (aclent[i].a_type & GROUP_OBJ) {
554 				acep->a_who = (uid_t)-1;
555 				acep->a_flags |=
556 				    (ACE_IDENTIFIER_GROUP|ACE_GROUP);
557 			} else if (aclent[i].a_type & USER) {
558 				acep->a_who = aclent[i].a_id;
559 			} else {
560 				acep->a_who = aclent[i].a_id;
561 				acep->a_flags |= ACE_IDENTIFIER_GROUP;
562 			}
563 			if (aclent[i].a_type & ACL_DEFAULT) {
564 				acep->a_flags |= ACE_INHERIT_ONLY_ACE |
565 				    ACE_FILE_INHERIT_ACE |
566 				    ACE_DIRECTORY_INHERIT_ACE;
567 			}
568 			/*
569 			 * Set the access mask for the prepended deny
570 			 * ace.  To do this, we invert the mask (found
571 			 * in ln_aent_preprocess()) then convert it to an
572 			 * DENY ace access_mask.
573 			 */
574 			acep->a_access_mask = mode_to_ace_access((mask ^ 07),
575 			    isdir, 0, 0);
576 			acep += 1;
577 		}
578 
579 		/* handle a_perm -> access_mask */
580 		acep->a_access_mask = mode_to_ace_access(aclent[i].a_perm,
581 		    isdir, aclent[i].a_type & USER_OBJ, 1);
582 
583 		/* emulate a default aclent */
584 		if (aclent[i].a_type & ACL_DEFAULT) {
585 			acep->a_flags |= ACE_INHERIT_ONLY_ACE |
586 			    ACE_FILE_INHERIT_ACE |
587 			    ACE_DIRECTORY_INHERIT_ACE;
588 		}
589 
590 		/*
591 		 * handle a_perm and a_id
592 		 *
593 		 * this must be done last, since it involves the
594 		 * corresponding deny aces, which are handled
595 		 * differently for each different a_type.
596 		 */
597 		if (aclent[i].a_type & USER_OBJ) {
598 			acep->a_who = (uid_t)-1;
599 			acep->a_flags |= ACE_OWNER;
600 			ace_make_deny(acep, acep + 1, isdir, B_TRUE);
601 			acep += 2;
602 		} else if (aclent[i].a_type & USER) {
603 			acep->a_who = aclent[i].a_id;
604 			ace_make_deny(acep, acep + 1, isdir, B_FALSE);
605 			acep += 2;
606 		} else if (aclent[i].a_type & (GROUP_OBJ | GROUP)) {
607 			if (aclent[i].a_type & GROUP_OBJ) {
608 				acep->a_who = (uid_t)-1;
609 				acep->a_flags |= ACE_GROUP;
610 			} else {
611 				acep->a_who = aclent[i].a_id;
612 			}
613 			acep->a_flags |= ACE_IDENTIFIER_GROUP;
614 			/*
615 			 * Set the corresponding deny for the group ace.
616 			 *
617 			 * The deny aces go after all of the groups, unlike
618 			 * everything else, where they immediately follow
619 			 * the allow ace.
620 			 *
621 			 * We calculate "skip", the number of slots to
622 			 * skip ahead for the deny ace, here.
623 			 *
624 			 * The pattern is:
625 			 * MD1 A1 MD2 A2 MD3 A3 D1 D2 D3
626 			 * thus, skip is
627 			 * (2 * numgroup) - 1 - groupi
628 			 * (2 * numgroup) to account for MD + A
629 			 * - 1 to account for the fact that we're on the
630 			 * access (A), not the mask (MD)
631 			 * - groupi to account for the fact that we have
632 			 * passed up groupi number of MD's.
633 			 */
634 			skip = (2 * numgroup) - 1 - groupi;
635 			ace_make_deny(acep, acep + skip, isdir, B_FALSE);
636 			/*
637 			 * If we just did the last group, skip acep past
638 			 * all of the denies; else, just move ahead one.
639 			 */
640 			if (++groupi >= numgroup)
641 				acep += numgroup + 1;
642 			else
643 				acep += 1;
644 		} else if (aclent[i].a_type & OTHER_OBJ) {
645 			acep->a_who = (uid_t)-1;
646 			acep->a_flags |= ACE_EVERYONE;
647 			ace_make_deny(acep, acep + 1, isdir, B_FALSE);
648 			acep += 2;
649 		} else {
650 			error = EINVAL;
651 			goto out;
652 		}
653 	}
654 
655 	*acepp = result;
656 	*rescount = resultsize;
657 
658 out:
659 	if (error != 0) {
660 		if ((result != NULL) && (resultsize > 0)) {
661 			cacl_free(result, resultsize * sizeof (ace_t));
662 		}
663 	}
664 
665 	return (error);
666 }
667 
668 static int
669 convert_aent_to_ace(aclent_t *aclentp, int aclcnt, int isdir,
670     ace_t **retacep, int *retacecnt)
671 {
672 	ace_t *acep;
673 	ace_t *dfacep;
674 	int acecnt = 0;
675 	int dfacecnt = 0;
676 	int dfaclstart = 0;
677 	int dfaclcnt = 0;
678 	aclent_t *aclp;
679 	int i;
680 	int error;
681 	int acesz, dfacesz;
682 
683 	ksort((caddr_t)aclentp, aclcnt, sizeof (aclent_t), cmp2acls);
684 
685 	for (i = 0, aclp = aclentp; i < aclcnt; aclp++, i++) {
686 		if (aclp->a_type & ACL_DEFAULT)
687 			break;
688 	}
689 
690 	if (i < aclcnt) {
691 		dfaclstart = i;
692 		dfaclcnt = aclcnt - i;
693 	}
694 
695 	if (dfaclcnt && isdir == 0) {
696 		return (EINVAL);
697 	}
698 
699 	error = ln_aent_to_ace(aclentp, i,  &acep, &acecnt, isdir);
700 	if (error)
701 		return (error);
702 
703 	if (dfaclcnt) {
704 		error = ln_aent_to_ace(&aclentp[dfaclstart], dfaclcnt,
705 		    &dfacep, &dfacecnt, isdir);
706 		if (error) {
707 			if (acep) {
708 				cacl_free(acep, acecnt * sizeof (ace_t));
709 			}
710 			return (error);
711 		}
712 	}
713 
714 	if (dfacecnt != 0) {
715 		acesz = sizeof (ace_t) * acecnt;
716 		dfacesz = sizeof (ace_t) * dfacecnt;
717 		acep = cacl_realloc(acep, acesz, acesz + dfacesz);
718 		if (acep == NULL)
719 			return (ENOMEM);
720 		if (dfaclcnt) {
721 			(void) memcpy(acep + acecnt, dfacep, dfacesz);
722 		}
723 	}
724 	if (dfaclcnt)
725 		cacl_free(dfacep, dfacecnt * sizeof (ace_t));
726 
727 	*retacecnt = acecnt + dfacecnt;
728 	*retacep = acep;
729 	return (0);
730 }
731 
732 static int
733 ace_mask_to_mode(uint32_t  mask, o_mode_t *modep, int isdir)
734 {
735 	int error = 0;
736 	o_mode_t mode = 0;
737 	uint32_t bits, wantbits;
738 
739 	/* read */
740 	if (mask & ACE_READ_DATA)
741 		mode |= S_IROTH;
742 
743 	/* write */
744 	wantbits = (ACE_WRITE_DATA | ACE_APPEND_DATA);
745 	if (isdir)
746 		wantbits |= ACE_DELETE_CHILD;
747 	bits = mask & wantbits;
748 	if (bits != 0) {
749 		if (bits != wantbits) {
750 			error = ENOTSUP;
751 			goto out;
752 		}
753 		mode |= S_IWOTH;
754 	}
755 
756 	/* exec */
757 	if (mask & ACE_EXECUTE) {
758 		mode |= S_IXOTH;
759 	}
760 
761 	*modep = mode;
762 
763 out:
764 	return (error);
765 }
766 
767 static void
768 acevals_init(acevals_t *vals, uid_t key)
769 {
770 	bzero(vals, sizeof (*vals));
771 	vals->allowed = ACE_MASK_UNDEFINED;
772 	vals->denied = ACE_MASK_UNDEFINED;
773 	vals->mask = ACE_MASK_UNDEFINED;
774 	vals->key = key;
775 }
776 
777 static void
778 ace_list_init(ace_list_t *al, int dfacl_flag)
779 {
780 	acevals_init(&al->user_obj, NULL);
781 	acevals_init(&al->group_obj, NULL);
782 	acevals_init(&al->other_obj, NULL);
783 	al->numusers = 0;
784 	al->numgroups = 0;
785 	al->acl_mask = 0;
786 	al->hasmask = 0;
787 	al->state = ace_unused;
788 	al->seen = 0;
789 	al->dfacl_flag = dfacl_flag;
790 }
791 
792 /*
793  * Find or create an acevals holder for a given id and avl tree.
794  *
795  * Note that only one thread will ever touch these avl trees, so
796  * there is no need for locking.
797  */
798 static acevals_t *
799 acevals_find(ace_t *ace, avl_tree_t *avl, int *num)
800 {
801 	acevals_t key, *rc;
802 	avl_index_t where;
803 
804 	key.key = ace->a_who;
805 	rc = avl_find(avl, &key, &where);
806 	if (rc != NULL)
807 		return (rc);
808 
809 	/* this memory is freed by ln_ace_to_aent()->ace_list_free() */
810 	if (cacl_malloc((void **)&rc, sizeof (acevals_t)) != 0)
811 		return (NULL);
812 
813 	acevals_init(rc, ace->a_who);
814 	avl_insert(avl, rc, where);
815 	(*num)++;
816 
817 	return (rc);
818 }
819 
820 static int
821 access_mask_check(ace_t *acep, int mask_bit, int isowner)
822 {
823 	int set_deny, err_deny;
824 	int set_allow, err_allow;
825 	int acl_consume;
826 	int haswriteperm, hasreadperm;
827 
828 	if (acep->a_type == ACE_ACCESS_DENIED_ACE_TYPE) {
829 		haswriteperm = (acep->a_access_mask & ACE_WRITE_DATA) ? 0 : 1;
830 		hasreadperm = (acep->a_access_mask & ACE_READ_DATA) ? 0 : 1;
831 	} else {
832 		haswriteperm = (acep->a_access_mask & ACE_WRITE_DATA) ? 1 : 0;
833 		hasreadperm = (acep->a_access_mask & ACE_READ_DATA) ? 1 : 0;
834 	}
835 
836 	acl_consume = (ACL_SYNCHRONIZE_ERR_DENY |
837 	    ACL_DELETE_ERR_DENY |
838 	    ACL_WRITE_OWNER_ERR_DENY |
839 	    ACL_WRITE_OWNER_ERR_ALLOW |
840 	    ACL_WRITE_ATTRS_OWNER_SET_ALLOW |
841 	    ACL_WRITE_ATTRS_OWNER_ERR_DENY |
842 	    ACL_WRITE_ATTRS_WRITER_SET_DENY |
843 	    ACL_WRITE_ATTRS_WRITER_ERR_ALLOW |
844 	    ACL_WRITE_NAMED_WRITER_ERR_DENY |
845 	    ACL_READ_NAMED_READER_ERR_DENY);
846 
847 	if (mask_bit == ACE_SYNCHRONIZE) {
848 		set_deny = ACL_SYNCHRONIZE_SET_DENY;
849 		err_deny =  ACL_SYNCHRONIZE_ERR_DENY;
850 		set_allow = ACL_SYNCHRONIZE_SET_ALLOW;
851 		err_allow = ACL_SYNCHRONIZE_ERR_ALLOW;
852 	} else if (mask_bit == ACE_WRITE_OWNER) {
853 		set_deny = ACL_WRITE_OWNER_SET_DENY;
854 		err_deny =  ACL_WRITE_OWNER_ERR_DENY;
855 		set_allow = ACL_WRITE_OWNER_SET_ALLOW;
856 		err_allow = ACL_WRITE_OWNER_ERR_ALLOW;
857 	} else if (mask_bit == ACE_DELETE) {
858 		set_deny = ACL_DELETE_SET_DENY;
859 		err_deny =  ACL_DELETE_ERR_DENY;
860 		set_allow = ACL_DELETE_SET_ALLOW;
861 		err_allow = ACL_DELETE_ERR_ALLOW;
862 	} else if (mask_bit == ACE_WRITE_ATTRIBUTES) {
863 		if (isowner) {
864 			set_deny = ACL_WRITE_ATTRS_OWNER_SET_DENY;
865 			err_deny =  ACL_WRITE_ATTRS_OWNER_ERR_DENY;
866 			set_allow = ACL_WRITE_ATTRS_OWNER_SET_ALLOW;
867 			err_allow = ACL_WRITE_ATTRS_OWNER_ERR_ALLOW;
868 		} else if (haswriteperm) {
869 			set_deny = ACL_WRITE_ATTRS_WRITER_SET_DENY;
870 			err_deny =  ACL_WRITE_ATTRS_WRITER_ERR_DENY;
871 			set_allow = ACL_WRITE_ATTRS_WRITER_SET_ALLOW;
872 			err_allow = ACL_WRITE_ATTRS_WRITER_ERR_ALLOW;
873 		} else {
874 			if ((acep->a_access_mask & mask_bit) &&
875 			    (acep->a_type & ACE_ACCESS_ALLOWED_ACE_TYPE)) {
876 				return (ENOTSUP);
877 			}
878 			return (0);
879 		}
880 	} else if (mask_bit == ACE_READ_NAMED_ATTRS) {
881 		if (!hasreadperm)
882 			return (0);
883 
884 		set_deny = ACL_READ_NAMED_READER_SET_DENY;
885 		err_deny = ACL_READ_NAMED_READER_ERR_DENY;
886 		set_allow = ACL_READ_NAMED_READER_SET_ALLOW;
887 		err_allow = ACL_READ_NAMED_READER_ERR_ALLOW;
888 	} else if (mask_bit == ACE_WRITE_NAMED_ATTRS) {
889 		if (!haswriteperm)
890 			return (0);
891 
892 		set_deny = ACL_WRITE_NAMED_WRITER_SET_DENY;
893 		err_deny = ACL_WRITE_NAMED_WRITER_ERR_DENY;
894 		set_allow = ACL_WRITE_NAMED_WRITER_SET_ALLOW;
895 		err_allow = ACL_WRITE_NAMED_WRITER_ERR_ALLOW;
896 	} else {
897 		return (EINVAL);
898 	}
899 
900 	if (acep->a_type == ACE_ACCESS_DENIED_ACE_TYPE) {
901 		if (acl_consume & set_deny) {
902 			if (!(acep->a_access_mask & mask_bit)) {
903 				return (ENOTSUP);
904 			}
905 		} else if (acl_consume & err_deny) {
906 			if (acep->a_access_mask & mask_bit) {
907 				return (ENOTSUP);
908 			}
909 		}
910 	} else {
911 		/* ACE_ACCESS_ALLOWED_ACE_TYPE */
912 		if (acl_consume & set_allow) {
913 			if (!(acep->a_access_mask & mask_bit)) {
914 				return (ENOTSUP);
915 			}
916 		} else if (acl_consume & err_allow) {
917 			if (acep->a_access_mask & mask_bit) {
918 				return (ENOTSUP);
919 			}
920 		}
921 	}
922 	return (0);
923 }
924 
925 static int
926 ace_to_aent_legal(ace_t *acep)
927 {
928 	int error = 0;
929 	int isowner;
930 
931 	/* only ALLOW or DENY */
932 	if ((acep->a_type != ACE_ACCESS_ALLOWED_ACE_TYPE) &&
933 	    (acep->a_type != ACE_ACCESS_DENIED_ACE_TYPE)) {
934 		error = ENOTSUP;
935 		goto out;
936 	}
937 
938 	/* check for invalid flags */
939 	if (acep->a_flags & ~(ACE_VALID_FLAG_BITS)) {
940 		error = EINVAL;
941 		goto out;
942 	}
943 
944 	/* some flags are illegal */
945 	if (acep->a_flags & (ACE_SUCCESSFUL_ACCESS_ACE_FLAG |
946 	    ACE_FAILED_ACCESS_ACE_FLAG |
947 	    ACE_NO_PROPAGATE_INHERIT_ACE)) {
948 		error = ENOTSUP;
949 		goto out;
950 	}
951 
952 	/* check for invalid masks */
953 	if (acep->a_access_mask & ~(ACE_VALID_MASK_BITS)) {
954 		error = EINVAL;
955 		goto out;
956 	}
957 
958 	if ((acep->a_flags & ACE_OWNER)) {
959 		isowner = 1;
960 	} else {
961 		isowner = 0;
962 	}
963 
964 	error = access_mask_check(acep, ACE_SYNCHRONIZE, isowner);
965 	if (error)
966 		goto out;
967 
968 	error = access_mask_check(acep, ACE_WRITE_OWNER, isowner);
969 	if (error)
970 		goto out;
971 
972 	error = access_mask_check(acep, ACE_DELETE, isowner);
973 	if (error)
974 		goto out;
975 
976 	error = access_mask_check(acep, ACE_WRITE_ATTRIBUTES, isowner);
977 	if (error)
978 		goto out;
979 
980 	error = access_mask_check(acep, ACE_READ_NAMED_ATTRS, isowner);
981 	if (error)
982 		goto out;
983 
984 	error = access_mask_check(acep, ACE_WRITE_NAMED_ATTRS, isowner);
985 	if (error)
986 		goto out;
987 
988 	/* more detailed checking of masks */
989 	if (acep->a_type == ACE_ACCESS_ALLOWED_ACE_TYPE) {
990 		if (! (acep->a_access_mask & ACE_READ_ATTRIBUTES)) {
991 			error = ENOTSUP;
992 			goto out;
993 		}
994 		if ((acep->a_access_mask & ACE_WRITE_DATA) &&
995 		    (! (acep->a_access_mask & ACE_APPEND_DATA))) {
996 			error = ENOTSUP;
997 			goto out;
998 		}
999 		if ((! (acep->a_access_mask & ACE_WRITE_DATA)) &&
1000 		    (acep->a_access_mask & ACE_APPEND_DATA)) {
1001 			error = ENOTSUP;
1002 			goto out;
1003 		}
1004 	}
1005 
1006 	/* ACL enforcement */
1007 	if ((acep->a_access_mask & ACE_READ_ACL) &&
1008 	    (acep->a_type != ACE_ACCESS_ALLOWED_ACE_TYPE)) {
1009 		error = ENOTSUP;
1010 		goto out;
1011 	}
1012 	if (acep->a_access_mask & ACE_WRITE_ACL) {
1013 		if ((acep->a_type == ACE_ACCESS_DENIED_ACE_TYPE) &&
1014 		    (isowner)) {
1015 			error = ENOTSUP;
1016 			goto out;
1017 		}
1018 		if ((acep->a_type == ACE_ACCESS_ALLOWED_ACE_TYPE) &&
1019 		    (! isowner)) {
1020 			error = ENOTSUP;
1021 			goto out;
1022 		}
1023 	}
1024 
1025 out:
1026 	return (error);
1027 }
1028 
1029 static int
1030 ace_allow_to_mode(uint32_t mask, o_mode_t *modep, int isdir)
1031 {
1032 	/* ACE_READ_ACL and ACE_READ_ATTRIBUTES must both be set */
1033 	if ((mask & (ACE_READ_ACL | ACE_READ_ATTRIBUTES)) !=
1034 	    (ACE_READ_ACL | ACE_READ_ATTRIBUTES)) {
1035 		return (ENOTSUP);
1036 	}
1037 
1038 	return (ace_mask_to_mode(mask, modep, isdir));
1039 }
1040 
1041 static int
1042 acevals_to_aent(acevals_t *vals, aclent_t *dest, ace_list_t *list,
1043     uid_t owner, gid_t group, int isdir)
1044 {
1045 	int error;
1046 	uint32_t  flips = ACE_POSIX_SUPPORTED_BITS;
1047 
1048 	if (isdir)
1049 		flips |= ACE_DELETE_CHILD;
1050 	if (vals->allowed != (vals->denied ^ flips)) {
1051 		error = ENOTSUP;
1052 		goto out;
1053 	}
1054 	if ((list->hasmask) && (list->acl_mask != vals->mask) &&
1055 	    (vals->aent_type & (USER | GROUP | GROUP_OBJ))) {
1056 		error = ENOTSUP;
1057 		goto out;
1058 	}
1059 	error = ace_allow_to_mode(vals->allowed, &dest->a_perm, isdir);
1060 	if (error != 0)
1061 		goto out;
1062 	dest->a_type = vals->aent_type;
1063 	if (dest->a_type & (USER | GROUP)) {
1064 		dest->a_id = vals->key;
1065 	} else if (dest->a_type & USER_OBJ) {
1066 		dest->a_id = owner;
1067 	} else if (dest->a_type & GROUP_OBJ) {
1068 		dest->a_id = group;
1069 	} else if (dest->a_type & OTHER_OBJ) {
1070 		dest->a_id = 0;
1071 	} else {
1072 		error = EINVAL;
1073 		goto out;
1074 	}
1075 
1076 out:
1077 	return (error);
1078 }
1079 
1080 
1081 static int
1082 ace_list_to_aent(ace_list_t *list, aclent_t **aclentp, int *aclcnt,
1083     uid_t owner, gid_t group, int isdir)
1084 {
1085 	int error = 0;
1086 	aclent_t *aent, *result = NULL;
1087 	acevals_t *vals;
1088 	int resultcount;
1089 
1090 	if ((list->seen & (USER_OBJ | GROUP_OBJ | OTHER_OBJ)) !=
1091 	    (USER_OBJ | GROUP_OBJ | OTHER_OBJ)) {
1092 		error = ENOTSUP;
1093 		goto out;
1094 	}
1095 	if ((! list->hasmask) && (list->numusers + list->numgroups > 0)) {
1096 		error = ENOTSUP;
1097 		goto out;
1098 	}
1099 
1100 	resultcount = 3 + list->numusers + list->numgroups;
1101 	/*
1102 	 * This must be the same condition as below, when we add the CLASS_OBJ
1103 	 * (aka ACL mask)
1104 	 */
1105 	if ((list->hasmask) || (! list->dfacl_flag))
1106 		resultcount += 1;
1107 
1108 	if (cacl_malloc((void **)&result,
1109 	    resultcount * sizeof (aclent_t)) != 0) {
1110 		error = ENOMEM;
1111 		goto out;
1112 	}
1113 	aent = result;
1114 
1115 	/* USER_OBJ */
1116 	if (!(list->user_obj.aent_type & USER_OBJ)) {
1117 		error = EINVAL;
1118 		goto out;
1119 	}
1120 
1121 	error = acevals_to_aent(&list->user_obj, aent, list, owner, group,
1122 	    isdir);
1123 
1124 	if (error != 0)
1125 		goto out;
1126 	++aent;
1127 	/* USER */
1128 	vals = NULL;
1129 	for (vals = avl_first(&list->user); vals != NULL;
1130 	    vals = AVL_NEXT(&list->user, vals)) {
1131 		if (!(vals->aent_type & USER)) {
1132 			error = EINVAL;
1133 			goto out;
1134 		}
1135 		error = acevals_to_aent(vals, aent, list, owner, group,
1136 		    isdir);
1137 		if (error != 0)
1138 			goto out;
1139 		++aent;
1140 	}
1141 	/* GROUP_OBJ */
1142 	if (!(list->group_obj.aent_type & GROUP_OBJ)) {
1143 		error = EINVAL;
1144 		goto out;
1145 	}
1146 	error = acevals_to_aent(&list->group_obj, aent, list, owner, group,
1147 	    isdir);
1148 	if (error != 0)
1149 		goto out;
1150 	++aent;
1151 	/* GROUP */
1152 	vals = NULL;
1153 	for (vals = avl_first(&list->group); vals != NULL;
1154 	    vals = AVL_NEXT(&list->group, vals)) {
1155 		if (!(vals->aent_type & GROUP)) {
1156 			error = EINVAL;
1157 			goto out;
1158 		}
1159 		error = acevals_to_aent(vals, aent, list, owner, group,
1160 		    isdir);
1161 		if (error != 0)
1162 			goto out;
1163 		++aent;
1164 	}
1165 	/*
1166 	 * CLASS_OBJ (aka ACL_MASK)
1167 	 *
1168 	 * An ACL_MASK is not fabricated if the ACL is a default ACL.
1169 	 * This is to follow UFS's behavior.
1170 	 */
1171 	if ((list->hasmask) || (! list->dfacl_flag)) {
1172 		if (list->hasmask) {
1173 			uint32_t flips = ACE_POSIX_SUPPORTED_BITS;
1174 			if (isdir)
1175 				flips |= ACE_DELETE_CHILD;
1176 			error = ace_mask_to_mode(list->acl_mask ^ flips,
1177 			    &aent->a_perm, isdir);
1178 			if (error != 0)
1179 				goto out;
1180 		} else {
1181 			/* fabricate the ACL_MASK from the group permissions */
1182 			error = ace_mask_to_mode(list->group_obj.allowed,
1183 			    &aent->a_perm, isdir);
1184 			if (error != 0)
1185 				goto out;
1186 		}
1187 		aent->a_id = 0;
1188 		aent->a_type = CLASS_OBJ | list->dfacl_flag;
1189 		++aent;
1190 	}
1191 	/* OTHER_OBJ */
1192 	if (!(list->other_obj.aent_type & OTHER_OBJ)) {
1193 		error = EINVAL;
1194 		goto out;
1195 	}
1196 	error = acevals_to_aent(&list->other_obj, aent, list, owner, group,
1197 	    isdir);
1198 	if (error != 0)
1199 		goto out;
1200 	++aent;
1201 
1202 	*aclentp = result;
1203 	*aclcnt = resultcount;
1204 
1205 out:
1206 	if (error != 0) {
1207 		if (result != NULL)
1208 			cacl_free(result, resultcount * sizeof (aclent_t));
1209 	}
1210 
1211 	return (error);
1212 }
1213 
1214 
1215 /*
1216  * free all data associated with an ace_list
1217  */
1218 static void
1219 ace_list_free(ace_list_t *al)
1220 {
1221 	acevals_t *node;
1222 	void *cookie;
1223 
1224 	if (al == NULL)
1225 		return;
1226 
1227 	cookie = NULL;
1228 	while ((node = avl_destroy_nodes(&al->user, &cookie)) != NULL)
1229 		cacl_free(node, sizeof (acevals_t));
1230 	cookie = NULL;
1231 	while ((node = avl_destroy_nodes(&al->group, &cookie)) != NULL)
1232 		cacl_free(node, sizeof (acevals_t));
1233 
1234 	avl_destroy(&al->user);
1235 	avl_destroy(&al->group);
1236 
1237 	/* free the container itself */
1238 	cacl_free(al, sizeof (ace_list_t));
1239 }
1240 
1241 static int
1242 acevals_compare(const void *va, const void *vb)
1243 {
1244 	const acevals_t *a = va, *b = vb;
1245 
1246 	if (a->key == b->key)
1247 		return (0);
1248 
1249 	if (a->key > b->key)
1250 		return (1);
1251 
1252 	else
1253 		return (-1);
1254 }
1255 
1256 /*
1257  * Convert a list of ace_t entries to equivalent regular and default
1258  * aclent_t lists.  Return error (ENOTSUP) when conversion is not possible.
1259  */
1260 static int
1261 ln_ace_to_aent(ace_t *ace, int n, uid_t owner, gid_t group,
1262     aclent_t **aclentp, int *aclcnt, aclent_t **dfaclentp, int *dfaclcnt,
1263     int isdir)
1264 {
1265 	int error = 0;
1266 	ace_t *acep;
1267 	uint32_t bits;
1268 	int i;
1269 	ace_list_t *normacl = NULL, *dfacl = NULL, *acl;
1270 	acevals_t *vals;
1271 
1272 	*aclentp = NULL;
1273 	*aclcnt = 0;
1274 	*dfaclentp = NULL;
1275 	*dfaclcnt = 0;
1276 
1277 	/* we need at least user_obj, group_obj, and other_obj */
1278 	if (n < 6) {
1279 		error = ENOTSUP;
1280 		goto out;
1281 	}
1282 	if (ace == NULL) {
1283 		error = EINVAL;
1284 		goto out;
1285 	}
1286 
1287 	error = cacl_malloc((void **)&normacl, sizeof (ace_list_t));
1288 	if (error != 0)
1289 		goto out;
1290 
1291 	avl_create(&normacl->user, acevals_compare, sizeof (acevals_t),
1292 	    offsetof(acevals_t, avl));
1293 	avl_create(&normacl->group, acevals_compare, sizeof (acevals_t),
1294 	    offsetof(acevals_t, avl));
1295 
1296 	ace_list_init(normacl, 0);
1297 
1298 	error = cacl_malloc((void **)&dfacl, sizeof (ace_list_t));
1299 	if (error != 0)
1300 		goto out;
1301 
1302 	avl_create(&dfacl->user, acevals_compare, sizeof (acevals_t),
1303 	    offsetof(acevals_t, avl));
1304 	avl_create(&dfacl->group, acevals_compare, sizeof (acevals_t),
1305 	    offsetof(acevals_t, avl));
1306 	ace_list_init(dfacl, ACL_DEFAULT);
1307 
1308 	/* process every ace_t... */
1309 	for (i = 0; i < n; i++) {
1310 		acep = &ace[i];
1311 
1312 		/* rule out certain cases quickly */
1313 		error = ace_to_aent_legal(acep);
1314 		if (error != 0)
1315 			goto out;
1316 
1317 		/*
1318 		 * Turn off these bits in order to not have to worry about
1319 		 * them when doing the checks for compliments.
1320 		 */
1321 		acep->a_access_mask &= ~(ACE_WRITE_OWNER | ACE_DELETE |
1322 		    ACE_SYNCHRONIZE | ACE_WRITE_ATTRIBUTES |
1323 		    ACE_READ_NAMED_ATTRS | ACE_WRITE_NAMED_ATTRS);
1324 
1325 		/* see if this should be a regular or default acl */
1326 		bits = acep->a_flags &
1327 		    (ACE_INHERIT_ONLY_ACE |
1328 		    ACE_FILE_INHERIT_ACE |
1329 		    ACE_DIRECTORY_INHERIT_ACE);
1330 		if (bits != 0) {
1331 			/* all or nothing on these inherit bits */
1332 			if (bits != (ACE_INHERIT_ONLY_ACE |
1333 			    ACE_FILE_INHERIT_ACE |
1334 			    ACE_DIRECTORY_INHERIT_ACE)) {
1335 				error = ENOTSUP;
1336 				goto out;
1337 			}
1338 			acl = dfacl;
1339 		} else {
1340 			acl = normacl;
1341 		}
1342 
1343 		if ((acep->a_flags & ACE_OWNER)) {
1344 			if (acl->state > ace_user_obj) {
1345 				error = ENOTSUP;
1346 				goto out;
1347 			}
1348 			acl->state = ace_user_obj;
1349 			acl->seen |= USER_OBJ;
1350 			vals = &acl->user_obj;
1351 			vals->aent_type = USER_OBJ | acl->dfacl_flag;
1352 		} else if ((acep->a_flags & ACE_EVERYONE)) {
1353 			acl->state = ace_other_obj;
1354 			acl->seen |= OTHER_OBJ;
1355 			vals = &acl->other_obj;
1356 			vals->aent_type = OTHER_OBJ | acl->dfacl_flag;
1357 		} else if (acep->a_flags & ACE_IDENTIFIER_GROUP) {
1358 			if (acl->state > ace_group) {
1359 				error = ENOTSUP;
1360 				goto out;
1361 			}
1362 			if ((acep->a_flags & ACE_GROUP)) {
1363 				acl->seen |= GROUP_OBJ;
1364 				vals = &acl->group_obj;
1365 				vals->aent_type = GROUP_OBJ | acl->dfacl_flag;
1366 			} else {
1367 				acl->seen |= GROUP;
1368 				vals = acevals_find(acep, &acl->group,
1369 				    &acl->numgroups);
1370 				if (vals == NULL) {
1371 					error = ENOMEM;
1372 					goto out;
1373 				}
1374 				vals->aent_type = GROUP | acl->dfacl_flag;
1375 			}
1376 			acl->state = ace_group;
1377 		} else {
1378 			if (acl->state > ace_user) {
1379 				error = ENOTSUP;
1380 				goto out;
1381 			}
1382 			acl->state = ace_user;
1383 			acl->seen |= USER;
1384 			vals = acevals_find(acep, &acl->user,
1385 			    &acl->numusers);
1386 			if (vals == NULL) {
1387 				error = ENOMEM;
1388 				goto out;
1389 			}
1390 			vals->aent_type = USER | acl->dfacl_flag;
1391 		}
1392 
1393 		if (!(acl->state > ace_unused)) {
1394 			error = EINVAL;
1395 			goto out;
1396 		}
1397 
1398 		if (acep->a_type == ACE_ACCESS_ALLOWED_ACE_TYPE) {
1399 			/* no more than one allowed per aclent_t */
1400 			if (vals->allowed != ACE_MASK_UNDEFINED) {
1401 				error = ENOTSUP;
1402 				goto out;
1403 			}
1404 			vals->allowed = acep->a_access_mask;
1405 		} else {
1406 			/*
1407 			 * it's a DENY; if there was a previous DENY, it
1408 			 * must have been an ACL_MASK.
1409 			 */
1410 			if (vals->denied != ACE_MASK_UNDEFINED) {
1411 				/* ACL_MASK is for USER and GROUP only */
1412 				if ((acl->state != ace_user) &&
1413 				    (acl->state != ace_group)) {
1414 					error = ENOTSUP;
1415 					goto out;
1416 				}
1417 
1418 				if (! acl->hasmask) {
1419 					acl->hasmask = 1;
1420 					acl->acl_mask = vals->denied;
1421 				/* check for mismatched ACL_MASK emulations */
1422 				} else if (acl->acl_mask != vals->denied) {
1423 					error = ENOTSUP;
1424 					goto out;
1425 				}
1426 				vals->mask = vals->denied;
1427 			}
1428 			vals->denied = acep->a_access_mask;
1429 		}
1430 	}
1431 
1432 	/* done collating; produce the aclent_t lists */
1433 	if (normacl->state != ace_unused) {
1434 		error = ace_list_to_aent(normacl, aclentp, aclcnt,
1435 		    owner, group, isdir);
1436 		if (error != 0) {
1437 			goto out;
1438 		}
1439 	}
1440 	if (dfacl->state != ace_unused) {
1441 		error = ace_list_to_aent(dfacl, dfaclentp, dfaclcnt,
1442 		    owner, group, isdir);
1443 		if (error != 0) {
1444 			goto out;
1445 		}
1446 	}
1447 
1448 out:
1449 	if (normacl != NULL)
1450 		ace_list_free(normacl);
1451 	if (dfacl != NULL)
1452 		ace_list_free(dfacl);
1453 
1454 	return (error);
1455 }
1456 
1457 static int
1458 convert_ace_to_aent(ace_t *acebufp, int acecnt, int isdir,
1459     uid_t owner, gid_t group, aclent_t **retaclentp, int *retaclcnt)
1460 {
1461 	int error = 0;
1462 	aclent_t *aclentp, *dfaclentp;
1463 	int aclcnt, dfaclcnt;
1464 	int aclsz, dfaclsz;
1465 
1466 	error = ln_ace_to_aent(acebufp, acecnt, owner, group,
1467 	    &aclentp, &aclcnt, &dfaclentp, &dfaclcnt, isdir);
1468 
1469 	if (error)
1470 		return (error);
1471 
1472 
1473 	if (dfaclcnt != 0) {
1474 		/*
1475 		 * Slap aclentp and dfaclentp into a single array.
1476 		 */
1477 		aclsz = sizeof (aclent_t) * aclcnt;
1478 		dfaclsz = sizeof (aclent_t) * dfaclcnt;
1479 		aclentp = cacl_realloc(aclentp, aclsz, aclsz + dfaclsz);
1480 		if (aclentp != NULL) {
1481 			(void) memcpy(aclentp + aclcnt, dfaclentp, dfaclsz);
1482 		} else {
1483 			error = ENOMEM;
1484 		}
1485 	}
1486 
1487 	if (aclentp) {
1488 		*retaclentp = aclentp;
1489 		*retaclcnt = aclcnt + dfaclcnt;
1490 	}
1491 
1492 	if (dfaclentp)
1493 		cacl_free(dfaclentp, dfaclsz);
1494 
1495 	return (error);
1496 }
1497 
1498 
1499 int
1500 acl_translate(acl_t *aclp, int target_flavor, int isdir, uid_t owner,
1501     gid_t group)
1502 {
1503 	int aclcnt;
1504 	void *acldata;
1505 	int error;
1506 
1507 	/*
1508 	 * See if we need to translate
1509 	 */
1510 	if ((target_flavor == _ACL_ACE_ENABLED && aclp->acl_type == ACE_T) ||
1511 	    (target_flavor == _ACL_ACLENT_ENABLED &&
1512 	    aclp->acl_type == ACLENT_T))
1513 		return (0);
1514 
1515 	if (target_flavor == -1) {
1516 		error = EINVAL;
1517 		goto out;
1518 	}
1519 
1520 	if (target_flavor ==  _ACL_ACE_ENABLED &&
1521 	    aclp->acl_type == ACLENT_T) {
1522 		error = convert_aent_to_ace(aclp->acl_aclp,
1523 		    aclp->acl_cnt, isdir, (ace_t **)&acldata, &aclcnt);
1524 		if (error)
1525 			goto out;
1526 
1527 	} else if (target_flavor == _ACL_ACLENT_ENABLED &&
1528 	    aclp->acl_type == ACE_T) {
1529 		error = convert_ace_to_aent(aclp->acl_aclp, aclp->acl_cnt,
1530 		    isdir, owner, group, (aclent_t **)&acldata, &aclcnt);
1531 		if (error)
1532 			goto out;
1533 	} else {
1534 		error = ENOTSUP;
1535 		goto out;
1536 	}
1537 
1538 	/*
1539 	 * replace old acl with newly translated acl
1540 	 */
1541 	cacl_free(aclp->acl_aclp, aclp->acl_cnt * aclp->acl_entry_size);
1542 	aclp->acl_aclp = acldata;
1543 	aclp->acl_cnt = aclcnt;
1544 	if (target_flavor == _ACL_ACE_ENABLED) {
1545 		aclp->acl_type = ACE_T;
1546 		aclp->acl_entry_size = sizeof (ace_t);
1547 	} else {
1548 		aclp->acl_type = ACLENT_T;
1549 		aclp->acl_entry_size = sizeof (aclent_t);
1550 	}
1551 	return (0);
1552 
1553 out:
1554 
1555 #if !defined(_KERNEL)
1556 	errno = error;
1557 	return (-1);
1558 #else
1559 	return (error);
1560 #endif
1561 }
1562 
1563 #define	SET_ACE(acl, index, who, mask, type, flags) { \
1564 	acl[0][index].a_who = (uint32_t)who; \
1565 	acl[0][index].a_type = type; \
1566 	acl[0][index].a_flags = flags; \
1567 	acl[0][index++].a_access_mask = mask; \
1568 }
1569 
1570 void
1571 acl_trivial_access_masks(mode_t mode, uint32_t *allow0, uint32_t *deny1,
1572     uint32_t *deny2, uint32_t *owner, uint32_t *group, uint32_t *everyone)
1573 {
1574 	*deny1 = *deny2 = *allow0 = *group = 0;
1575 
1576 	if (!(mode & S_IRUSR) && (mode & (S_IRGRP|S_IROTH)))
1577 		*deny1 |= ACE_READ_DATA;
1578 	if (!(mode & S_IWUSR) && (mode & (S_IWGRP|S_IWOTH)))
1579 		*deny1 |= ACE_WRITE_DATA;
1580 	if (!(mode & S_IXUSR) && (mode & (S_IXGRP|S_IXOTH)))
1581 		*deny1 |= ACE_EXECUTE;
1582 
1583 	if (!(mode & S_IRGRP) && (mode & S_IROTH))
1584 		*deny2 = ACE_READ_DATA;
1585 	if (!(mode & S_IWGRP) && (mode & S_IWOTH))
1586 		*deny2 |= ACE_WRITE_DATA;
1587 	if (!(mode & S_IXGRP) && (mode & S_IXOTH))
1588 		*deny2 |= ACE_EXECUTE;
1589 
1590 	if ((mode & S_IRUSR) && (!(mode & S_IRGRP) && (mode & S_IROTH)))
1591 		*allow0 |= ACE_READ_DATA;
1592 	if ((mode & S_IWUSR) && (!(mode & S_IWGRP) && (mode & S_IWOTH)))
1593 		*allow0 |= ACE_WRITE_DATA;
1594 	if ((mode & S_IXUSR) && (!(mode & S_IXGRP) && (mode & S_IXOTH)))
1595 		*allow0 |= ACE_EXECUTE;
1596 
1597 	*owner = ACE_WRITE_ATTRIBUTES|ACE_WRITE_OWNER|ACE_WRITE_ACL|
1598 	    ACE_WRITE_NAMED_ATTRS|ACE_READ_ACL|ACE_READ_ATTRIBUTES|
1599 	    ACE_READ_NAMED_ATTRS|ACE_SYNCHRONIZE;
1600 	if (mode & S_IRUSR)
1601 		*owner |= ACE_READ_DATA;
1602 	if (mode & S_IWUSR)
1603 		*owner |= ACE_WRITE_DATA|ACE_APPEND_DATA;
1604 	if (mode & S_IXUSR)
1605 		*owner |= ACE_EXECUTE;
1606 
1607 	*group = ACE_READ_ACL|ACE_READ_ATTRIBUTES| ACE_READ_NAMED_ATTRS|
1608 	    ACE_SYNCHRONIZE;
1609 	if (mode & S_IRGRP)
1610 		*group |= ACE_READ_DATA;
1611 	if (mode & S_IWGRP)
1612 		*group |= ACE_WRITE_DATA|ACE_APPEND_DATA;
1613 	if (mode & S_IXGRP)
1614 		*group |= ACE_EXECUTE;
1615 
1616 	*everyone = ACE_READ_ACL|ACE_READ_ATTRIBUTES| ACE_READ_NAMED_ATTRS|
1617 	    ACE_SYNCHRONIZE;
1618 	if (mode & S_IROTH)
1619 		*everyone |= ACE_READ_DATA;
1620 	if (mode & S_IWOTH)
1621 		*everyone |= ACE_WRITE_DATA|ACE_APPEND_DATA;
1622 	if (mode & S_IXOTH)
1623 		*everyone |= ACE_EXECUTE;
1624 }
1625 
1626 int
1627 acl_trivial_create(mode_t mode, ace_t **acl, int *count)
1628 {
1629 	uint32_t	deny1, deny2;
1630 	uint32_t	allow0;
1631 	uint32_t	owner, group, everyone;
1632 	int 		index = 0;
1633 	int		error;
1634 
1635 	*count = 3;
1636 	acl_trivial_access_masks(mode, &allow0, &deny1, &deny2, &owner, &group,
1637 	    &everyone);
1638 
1639 	if (allow0)
1640 		(*count)++;
1641 	if (deny1)
1642 		(*count)++;
1643 	if (deny2)
1644 		(*count)++;
1645 
1646 	if ((error = cacl_malloc((void **)acl, *count * sizeof (ace_t))) != 0)
1647 		return (error);
1648 
1649 	if (allow0) {
1650 		SET_ACE(acl, index, -1, allow0, ACE_ACCESS_ALLOWED_ACE_TYPE,
1651 		    ACE_OWNER);
1652 	}
1653 	if (deny1) {
1654 		SET_ACE(acl, index, -1, deny1, ACE_ACCESS_DENIED_ACE_TYPE,
1655 		    ACE_OWNER);
1656 	}
1657 	if (deny2) {
1658 		SET_ACE(acl, index, -1, deny2, ACE_ACCESS_DENIED_ACE_TYPE,
1659 		    ACE_GROUP|ACE_IDENTIFIER_GROUP);
1660 	}
1661 
1662 	SET_ACE(acl, index, -1, owner, ACE_ACCESS_ALLOWED_ACE_TYPE, ACE_OWNER);
1663 	SET_ACE(acl, index, -1, group, ACE_ACCESS_ALLOWED_ACE_TYPE,
1664 	    ACE_IDENTIFIER_GROUP|ACE_GROUP);
1665 	SET_ACE(acl, index, -1, everyone, ACE_ACCESS_ALLOWED_ACE_TYPE,
1666 	    ACE_EVERYONE);
1667 
1668 	return (0);
1669 }
1670 
1671 /*
1672  * ace_trivial:
1673  * determine whether an ace_t acl is trivial
1674  *
1675  * Trivialness implies that the acl is composed of only
1676  * owner, group, everyone entries.  ACL can't
1677  * have read_acl denied, and write_owner/write_acl/write_attributes
1678  * can only be owner@ entry.
1679  */
1680 int
1681 ace_trivial_common(void *acep, int aclcnt,
1682     uint64_t (*walk)(void *, uint64_t, int aclcnt,
1683     uint16_t *, uint16_t *, uint32_t *))
1684 {
1685 	uint16_t flags;
1686 	uint32_t mask;
1687 	uint16_t type;
1688 	uint64_t cookie = 0;
1689 
1690 	while (cookie = walk(acep, cookie, aclcnt, &flags, &type, &mask)) {
1691 		switch (flags & ACE_TYPE_FLAGS) {
1692 		case ACE_OWNER:
1693 		case ACE_GROUP|ACE_IDENTIFIER_GROUP:
1694 		case ACE_EVERYONE:
1695 			break;
1696 		default:
1697 			return (1);
1698 
1699 		}
1700 
1701 		if (flags & (ACE_FILE_INHERIT_ACE|
1702 		    ACE_DIRECTORY_INHERIT_ACE|ACE_NO_PROPAGATE_INHERIT_ACE|
1703 		    ACE_INHERIT_ONLY_ACE))
1704 			return (1);
1705 
1706 		/*
1707 		 * Special check for some special bits
1708 		 *
1709 		 * Don't allow anybody to deny reading basic
1710 		 * attributes or a files ACL.
1711 		 */
1712 		if ((mask & (ACE_READ_ACL|ACE_READ_ATTRIBUTES)) &&
1713 		    (type == ACE_ACCESS_DENIED_ACE_TYPE))
1714 			return (1);
1715 
1716 		/*
1717 		 * Delete permissions are never set by default
1718 		 */
1719 		if (mask & (ACE_DELETE|ACE_DELETE_CHILD))
1720 			return (1);
1721 		/*
1722 		 * only allow owner@ to have
1723 		 * write_acl/write_owner/write_attributes/write_xattr/
1724 		 */
1725 		if (type == ACE_ACCESS_ALLOWED_ACE_TYPE &&
1726 		    (!(flags & ACE_OWNER) && (mask &
1727 		    (ACE_WRITE_OWNER|ACE_WRITE_ACL| ACE_WRITE_ATTRIBUTES|
1728 		    ACE_WRITE_NAMED_ATTRS))))
1729 			return (1);
1730 
1731 	}
1732 	return (0);
1733 }
1734 
1735 uint64_t
1736 ace_walk(void *datap, uint64_t cookie, int aclcnt, uint16_t *flags,
1737     uint16_t *type, uint32_t *mask)
1738 {
1739 	ace_t *acep = datap;
1740 
1741 	if (cookie >= aclcnt)
1742 		return (0);
1743 
1744 	*flags = acep[cookie].a_flags;
1745 	*type = acep[cookie].a_type;
1746 	*mask = acep[cookie++].a_access_mask;
1747 
1748 	return (cookie);
1749 }
1750 
1751 int
1752 ace_trivial(ace_t *acep, int aclcnt)
1753 {
1754 	return (ace_trivial_common(acep, aclcnt, ace_walk));
1755 }
1756