xref: /titanic_44/usr/src/common/acl/acl_common.c (revision 49b225e1cfa7bbf7738d4df0a03f18e3283426eb)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 /*
22  * Copyright 2008 Sun Microsystems, Inc.  All rights reserved.
23  * Use is subject to license terms.
24  */
25 
26 #pragma ident	"%Z%%M%	%I%	%E% SMI"
27 
28 #include <sys/types.h>
29 #include <sys/stat.h>
30 #include <sys/avl.h>
31 #if defined(_KERNEL)
32 #include <sys/systm.h>
33 #include <sys/sysmacros.h>
34 #include <acl/acl_common.h>
35 #else
36 #include <errno.h>
37 #include <stdlib.h>
38 #include <stddef.h>
39 #include <strings.h>
40 #include <unistd.h>
41 #include <assert.h>
42 #include <grp.h>
43 #include <pwd.h>
44 #include <acl_common.h>
45 #define	ASSERT	assert
46 #endif
47 
48 #define	ACE_POSIX_SUPPORTED_BITS (ACE_READ_DATA | \
49     ACE_WRITE_DATA | ACE_APPEND_DATA | ACE_EXECUTE | \
50     ACE_READ_ATTRIBUTES | ACE_READ_ACL | ACE_WRITE_ACL)
51 
52 
53 #define	ACL_SYNCHRONIZE_SET_DENY		0x0000001
54 #define	ACL_SYNCHRONIZE_SET_ALLOW		0x0000002
55 #define	ACL_SYNCHRONIZE_ERR_DENY		0x0000004
56 #define	ACL_SYNCHRONIZE_ERR_ALLOW		0x0000008
57 
58 #define	ACL_WRITE_OWNER_SET_DENY		0x0000010
59 #define	ACL_WRITE_OWNER_SET_ALLOW		0x0000020
60 #define	ACL_WRITE_OWNER_ERR_DENY		0x0000040
61 #define	ACL_WRITE_OWNER_ERR_ALLOW		0x0000080
62 
63 #define	ACL_DELETE_SET_DENY			0x0000100
64 #define	ACL_DELETE_SET_ALLOW			0x0000200
65 #define	ACL_DELETE_ERR_DENY			0x0000400
66 #define	ACL_DELETE_ERR_ALLOW			0x0000800
67 
68 #define	ACL_WRITE_ATTRS_OWNER_SET_DENY		0x0001000
69 #define	ACL_WRITE_ATTRS_OWNER_SET_ALLOW		0x0002000
70 #define	ACL_WRITE_ATTRS_OWNER_ERR_DENY		0x0004000
71 #define	ACL_WRITE_ATTRS_OWNER_ERR_ALLOW		0x0008000
72 
73 #define	ACL_WRITE_ATTRS_WRITER_SET_DENY		0x0010000
74 #define	ACL_WRITE_ATTRS_WRITER_SET_ALLOW	0x0020000
75 #define	ACL_WRITE_ATTRS_WRITER_ERR_DENY		0x0040000
76 #define	ACL_WRITE_ATTRS_WRITER_ERR_ALLOW	0x0080000
77 
78 #define	ACL_WRITE_NAMED_WRITER_SET_DENY		0x0100000
79 #define	ACL_WRITE_NAMED_WRITER_SET_ALLOW	0x0200000
80 #define	ACL_WRITE_NAMED_WRITER_ERR_DENY		0x0400000
81 #define	ACL_WRITE_NAMED_WRITER_ERR_ALLOW	0x0800000
82 
83 #define	ACL_READ_NAMED_READER_SET_DENY		0x1000000
84 #define	ACL_READ_NAMED_READER_SET_ALLOW		0x2000000
85 #define	ACL_READ_NAMED_READER_ERR_DENY		0x4000000
86 #define	ACL_READ_NAMED_READER_ERR_ALLOW		0x8000000
87 
88 
89 #define	ACE_VALID_MASK_BITS (\
90     ACE_READ_DATA | \
91     ACE_LIST_DIRECTORY | \
92     ACE_WRITE_DATA | \
93     ACE_ADD_FILE | \
94     ACE_APPEND_DATA | \
95     ACE_ADD_SUBDIRECTORY | \
96     ACE_READ_NAMED_ATTRS | \
97     ACE_WRITE_NAMED_ATTRS | \
98     ACE_EXECUTE | \
99     ACE_DELETE_CHILD | \
100     ACE_READ_ATTRIBUTES | \
101     ACE_WRITE_ATTRIBUTES | \
102     ACE_DELETE | \
103     ACE_READ_ACL | \
104     ACE_WRITE_ACL | \
105     ACE_WRITE_OWNER | \
106     ACE_SYNCHRONIZE)
107 
108 #define	ACE_MASK_UNDEFINED			0x80000000
109 
110 #define	ACE_VALID_FLAG_BITS (ACE_FILE_INHERIT_ACE | \
111     ACE_DIRECTORY_INHERIT_ACE | \
112     ACE_NO_PROPAGATE_INHERIT_ACE | ACE_INHERIT_ONLY_ACE | \
113     ACE_SUCCESSFUL_ACCESS_ACE_FLAG | ACE_FAILED_ACCESS_ACE_FLAG | \
114     ACE_IDENTIFIER_GROUP | ACE_OWNER | ACE_GROUP | ACE_EVERYONE)
115 
116 /*
117  * ACL conversion helpers
118  */
119 
120 typedef enum {
121 	ace_unused,
122 	ace_user_obj,
123 	ace_user,
124 	ace_group, /* includes GROUP and GROUP_OBJ */
125 	ace_other_obj
126 } ace_to_aent_state_t;
127 
128 typedef struct acevals {
129 	uid_t key;
130 	avl_node_t avl;
131 	uint32_t mask;
132 	uint32_t allowed;
133 	uint32_t denied;
134 	int aent_type;
135 } acevals_t;
136 
137 typedef struct ace_list {
138 	acevals_t user_obj;
139 	avl_tree_t user;
140 	int numusers;
141 	acevals_t group_obj;
142 	avl_tree_t group;
143 	int numgroups;
144 	acevals_t other_obj;
145 	uint32_t acl_mask;
146 	int hasmask;
147 	int dfacl_flag;
148 	ace_to_aent_state_t state;
149 	int seen; /* bitmask of all aclent_t a_type values seen */
150 } ace_list_t;
151 
152 ace_t trivial_acl[] = {
153 	{(uid_t)-1, 0, ACE_OWNER, ACE_ACCESS_DENIED_ACE_TYPE},
154 	{(uid_t)-1, ACE_WRITE_ACL|ACE_WRITE_OWNER|ACE_WRITE_ATTRIBUTES|
155 	    ACE_WRITE_NAMED_ATTRS, ACE_OWNER, ACE_ACCESS_ALLOWED_ACE_TYPE},
156 	{(uid_t)-1, 0, ACE_GROUP|ACE_IDENTIFIER_GROUP,
157 	    ACE_ACCESS_DENIED_ACE_TYPE},
158 	{(uid_t)-1, 0, ACE_GROUP|ACE_IDENTIFIER_GROUP,
159 	    ACE_ACCESS_ALLOWED_ACE_TYPE},
160 	{(uid_t)-1, ACE_WRITE_ACL|ACE_WRITE_OWNER| ACE_WRITE_ATTRIBUTES|
161 	    ACE_WRITE_NAMED_ATTRS, ACE_EVERYONE, ACE_ACCESS_DENIED_ACE_TYPE},
162 	{(uid_t)-1, ACE_READ_ACL|ACE_READ_ATTRIBUTES|ACE_READ_NAMED_ATTRS|
163 	    ACE_SYNCHRONIZE, ACE_EVERYONE, ACE_ACCESS_ALLOWED_ACE_TYPE}
164 };
165 
166 
167 void
168 adjust_ace_pair_common(void *pair, size_t access_off,
169     size_t pairsize, mode_t mode)
170 {
171 	char *datap = (char *)pair;
172 	uint32_t *amask0 = (uint32_t *)(uintptr_t)(datap + access_off);
173 	uint32_t *amask1 = (uint32_t *)(uintptr_t)(datap + pairsize +
174 	    access_off);
175 	if (mode & S_IROTH)
176 		*amask1 |= ACE_READ_DATA;
177 	else
178 		*amask0 |= ACE_READ_DATA;
179 	if (mode & S_IWOTH)
180 		*amask1 |= ACE_WRITE_DATA|ACE_APPEND_DATA;
181 	else
182 		*amask0 |= ACE_WRITE_DATA|ACE_APPEND_DATA;
183 	if (mode & S_IXOTH)
184 		*amask1 |= ACE_EXECUTE;
185 	else
186 		*amask0 |= ACE_EXECUTE;
187 }
188 
189 void
190 adjust_ace_pair(ace_t *pair, mode_t mode)
191 {
192 	adjust_ace_pair_common(pair, offsetof(ace_t, a_access_mask),
193 	    sizeof (ace_t), mode);
194 }
195 
196 static void
197 ace_allow_deny_helper(uint16_t type, boolean_t *allow, boolean_t *deny)
198 {
199 	if (type == ACE_ACCESS_ALLOWED_ACE_TYPE)
200 		*allow = B_TRUE;
201 	else if (type == ACE_ACCESS_DENIED_ACE_TYPE)
202 		*deny = B_TRUE;
203 }
204 
205 /*
206  * ace_trivial:
207  * determine whether an ace_t acl is trivial
208  *
209  * Trivialness implies that the acl is composed of only
210  * owner, group, everyone entries.  ACL can't
211  * have read_acl denied, and write_owner/write_acl/write_attributes
212  * can only be owner@ entry.
213  */
214 int
215 ace_trivial_common(void *acep, int aclcnt,
216     uint64_t (*walk)(void *, uint64_t, int aclcnt,
217     uint16_t *, uint16_t *, uint32_t *))
218 {
219 	boolean_t owner_allow = B_FALSE;
220 	boolean_t group_allow = B_FALSE;
221 	boolean_t everyone_allow = B_FALSE;
222 	boolean_t owner_deny = B_FALSE;
223 	boolean_t group_deny = B_FALSE;
224 	boolean_t everyone_deny = B_FALSE;
225 	uint16_t flags;
226 	uint32_t mask;
227 	uint16_t type;
228 	uint64_t cookie = 0;
229 
230 	while (cookie = walk(acep, cookie, aclcnt, &flags, &type, &mask)) {
231 		switch (flags & ACE_TYPE_FLAGS) {
232 		case ACE_OWNER:
233 			if (group_allow || group_deny || everyone_allow ||
234 			    everyone_deny)
235 				return (1);
236 			ace_allow_deny_helper(type, &owner_allow, &owner_deny);
237 			break;
238 		case ACE_GROUP|ACE_IDENTIFIER_GROUP:
239 			if (everyone_allow || everyone_deny &&
240 			    (!owner_allow && !owner_deny))
241 				return (1);
242 			ace_allow_deny_helper(type, &group_allow, &group_deny);
243 			break;
244 
245 		case ACE_EVERYONE:
246 			if (!owner_allow && !owner_deny &&
247 			    !group_allow && !group_deny)
248 				return (1);
249 			ace_allow_deny_helper(type,
250 			    &everyone_allow, &everyone_deny);
251 			break;
252 		default:
253 			return (1);
254 
255 		}
256 
257 		if (flags & (ACE_FILE_INHERIT_ACE|
258 		    ACE_DIRECTORY_INHERIT_ACE|ACE_NO_PROPAGATE_INHERIT_ACE|
259 		    ACE_INHERIT_ONLY_ACE))
260 			return (1);
261 
262 		/*
263 		 * Special check for some special bits
264 		 *
265 		 * Don't allow anybody to deny reading basic
266 		 * attributes or a files ACL.
267 		 */
268 		if ((mask & (ACE_READ_ACL|ACE_READ_ATTRIBUTES)) &&
269 		    (type == ACE_ACCESS_DENIED_ACE_TYPE))
270 			return (1);
271 
272 		/*
273 		 * Allow on owner@ to allow
274 		 * write_acl/write_owner/write_attributes
275 		 */
276 		if (type == ACE_ACCESS_ALLOWED_ACE_TYPE &&
277 		    (!(flags & ACE_OWNER) && (mask &
278 		    (ACE_WRITE_OWNER|ACE_WRITE_ACL|ACE_WRITE_ATTRIBUTES))))
279 			return (1);
280 
281 	}
282 
283 	if (!owner_allow || !owner_deny || !group_allow || !group_deny ||
284 	    !everyone_allow || !everyone_deny)
285 		return (1);
286 
287 	return (0);
288 }
289 
290 uint64_t
291 ace_walk(void *datap, uint64_t cookie, int aclcnt, uint16_t *flags,
292     uint16_t *type, uint32_t *mask)
293 {
294 	ace_t *acep = datap;
295 
296 	if (cookie >= aclcnt)
297 		return (0);
298 
299 	*flags = acep[cookie].a_flags;
300 	*type = acep[cookie].a_type;
301 	*mask = acep[cookie++].a_access_mask;
302 
303 	return (cookie);
304 }
305 
306 int
307 ace_trivial(ace_t *acep, int aclcnt)
308 {
309 	return (ace_trivial_common(acep, aclcnt, ace_walk));
310 }
311 
312 /*
313  * Generic shellsort, from K&R (1st ed, p 58.), somewhat modified.
314  * v = Ptr to array/vector of objs
315  * n = # objs in the array
316  * s = size of each obj (must be multiples of a word size)
317  * f = ptr to function to compare two objs
318  *	returns (-1 = less than, 0 = equal, 1 = greater than
319  */
320 void
321 ksort(caddr_t v, int n, int s, int (*f)())
322 {
323 	int g, i, j, ii;
324 	unsigned int *p1, *p2;
325 	unsigned int tmp;
326 
327 	/* No work to do */
328 	if (v == NULL || n <= 1)
329 		return;
330 
331 	/* Sanity check on arguments */
332 	ASSERT(((uintptr_t)v & 0x3) == 0 && (s & 0x3) == 0);
333 	ASSERT(s > 0);
334 	for (g = n / 2; g > 0; g /= 2) {
335 		for (i = g; i < n; i++) {
336 			for (j = i - g; j >= 0 &&
337 			    (*f)(v + j * s, v + (j + g) * s) == 1;
338 			    j -= g) {
339 				p1 = (void *)(v + j * s);
340 				p2 = (void *)(v + (j + g) * s);
341 				for (ii = 0; ii < s / 4; ii++) {
342 					tmp = *p1;
343 					*p1++ = *p2;
344 					*p2++ = tmp;
345 				}
346 			}
347 		}
348 	}
349 }
350 
351 /*
352  * Compare two acls, all fields.  Returns:
353  * -1 (less than)
354  *  0 (equal)
355  * +1 (greater than)
356  */
357 int
358 cmp2acls(void *a, void *b)
359 {
360 	aclent_t *x = (aclent_t *)a;
361 	aclent_t *y = (aclent_t *)b;
362 
363 	/* Compare types */
364 	if (x->a_type < y->a_type)
365 		return (-1);
366 	if (x->a_type > y->a_type)
367 		return (1);
368 	/* Equal types; compare id's */
369 	if (x->a_id < y->a_id)
370 		return (-1);
371 	if (x->a_id > y->a_id)
372 		return (1);
373 	/* Equal ids; compare perms */
374 	if (x->a_perm < y->a_perm)
375 		return (-1);
376 	if (x->a_perm > y->a_perm)
377 		return (1);
378 	/* Totally equal */
379 	return (0);
380 }
381 
382 /*ARGSUSED*/
383 static void *
384 cacl_realloc(void *ptr, size_t size, size_t new_size)
385 {
386 #if defined(_KERNEL)
387 	void *tmp;
388 
389 	tmp = kmem_alloc(new_size, KM_SLEEP);
390 	(void) memcpy(tmp, ptr, (size < new_size) ? size : new_size);
391 	kmem_free(ptr, size);
392 	return (tmp);
393 #else
394 	return (realloc(ptr, new_size));
395 #endif
396 }
397 
398 static int
399 cacl_malloc(void **ptr, size_t size)
400 {
401 #if defined(_KERNEL)
402 	*ptr = kmem_zalloc(size, KM_SLEEP);
403 	return (0);
404 #else
405 	*ptr = calloc(1, size);
406 	if (*ptr == NULL)
407 		return (errno);
408 
409 	return (0);
410 #endif
411 }
412 
413 /*ARGSUSED*/
414 static void
415 cacl_free(void *ptr, size_t size)
416 {
417 #if defined(_KERNEL)
418 	kmem_free(ptr, size);
419 #else
420 	free(ptr);
421 #endif
422 }
423 
424 acl_t *
425 acl_alloc(enum acl_type type)
426 {
427 	acl_t *aclp;
428 
429 	if (cacl_malloc((void **)&aclp, sizeof (acl_t)) != 0)
430 		return (NULL);
431 
432 	aclp->acl_aclp = NULL;
433 	aclp->acl_cnt = 0;
434 
435 	switch (type) {
436 	case ACE_T:
437 		aclp->acl_type = ACE_T;
438 		aclp->acl_entry_size = sizeof (ace_t);
439 		break;
440 	case ACLENT_T:
441 		aclp->acl_type = ACLENT_T;
442 		aclp->acl_entry_size = sizeof (aclent_t);
443 		break;
444 	default:
445 		acl_free(aclp);
446 		aclp = NULL;
447 	}
448 	return (aclp);
449 }
450 
451 /*
452  * Free acl_t structure
453  */
454 void
455 acl_free(acl_t *aclp)
456 {
457 	int acl_size;
458 
459 	if (aclp == NULL)
460 		return;
461 
462 	if (aclp->acl_aclp) {
463 		acl_size = aclp->acl_cnt * aclp->acl_entry_size;
464 		cacl_free(aclp->acl_aclp, acl_size);
465 	}
466 
467 	cacl_free(aclp, sizeof (acl_t));
468 }
469 
470 static uint32_t
471 access_mask_set(int haswriteperm, int hasreadperm, int isowner, int isallow)
472 {
473 	uint32_t access_mask = 0;
474 	int acl_produce;
475 	int synchronize_set = 0, write_owner_set = 0;
476 	int delete_set = 0, write_attrs_set = 0;
477 	int read_named_set = 0, write_named_set = 0;
478 
479 	acl_produce = (ACL_SYNCHRONIZE_SET_ALLOW |
480 	    ACL_WRITE_ATTRS_OWNER_SET_ALLOW |
481 	    ACL_WRITE_ATTRS_WRITER_SET_DENY);
482 
483 	if (isallow) {
484 		synchronize_set = ACL_SYNCHRONIZE_SET_ALLOW;
485 		write_owner_set = ACL_WRITE_OWNER_SET_ALLOW;
486 		delete_set = ACL_DELETE_SET_ALLOW;
487 		if (hasreadperm)
488 			read_named_set = ACL_READ_NAMED_READER_SET_ALLOW;
489 		if (haswriteperm)
490 			write_named_set = ACL_WRITE_NAMED_WRITER_SET_ALLOW;
491 		if (isowner)
492 			write_attrs_set = ACL_WRITE_ATTRS_OWNER_SET_ALLOW;
493 		else if (haswriteperm)
494 			write_attrs_set = ACL_WRITE_ATTRS_WRITER_SET_ALLOW;
495 	} else {
496 
497 		synchronize_set = ACL_SYNCHRONIZE_SET_DENY;
498 		write_owner_set = ACL_WRITE_OWNER_SET_DENY;
499 		delete_set = ACL_DELETE_SET_DENY;
500 		if (hasreadperm)
501 			read_named_set = ACL_READ_NAMED_READER_SET_DENY;
502 		if (haswriteperm)
503 			write_named_set = ACL_WRITE_NAMED_WRITER_SET_DENY;
504 		if (isowner)
505 			write_attrs_set = ACL_WRITE_ATTRS_OWNER_SET_DENY;
506 		else if (haswriteperm)
507 			write_attrs_set = ACL_WRITE_ATTRS_WRITER_SET_DENY;
508 		else
509 			/*
510 			 * If the entity is not the owner and does not
511 			 * have write permissions ACE_WRITE_ATTRIBUTES will
512 			 * always go in the DENY ACE.
513 			 */
514 			access_mask |= ACE_WRITE_ATTRIBUTES;
515 	}
516 
517 	if (acl_produce & synchronize_set)
518 		access_mask |= ACE_SYNCHRONIZE;
519 	if (acl_produce & write_owner_set)
520 		access_mask |= ACE_WRITE_OWNER;
521 	if (acl_produce & delete_set)
522 		access_mask |= ACE_DELETE;
523 	if (acl_produce & write_attrs_set)
524 		access_mask |= ACE_WRITE_ATTRIBUTES;
525 	if (acl_produce & read_named_set)
526 		access_mask |= ACE_READ_NAMED_ATTRS;
527 	if (acl_produce & write_named_set)
528 		access_mask |= ACE_WRITE_NAMED_ATTRS;
529 
530 	return (access_mask);
531 }
532 
533 /*
534  * Given an mode_t, convert it into an access_mask as used
535  * by nfsace, assuming aclent_t -> nfsace semantics.
536  */
537 static uint32_t
538 mode_to_ace_access(mode_t mode, int isdir, int isowner, int isallow)
539 {
540 	uint32_t access = 0;
541 	int haswriteperm = 0;
542 	int hasreadperm = 0;
543 
544 	if (isallow) {
545 		haswriteperm = (mode & S_IWOTH);
546 		hasreadperm = (mode & S_IROTH);
547 	} else {
548 		haswriteperm = !(mode & S_IWOTH);
549 		hasreadperm = !(mode & S_IROTH);
550 	}
551 
552 	/*
553 	 * The following call takes care of correctly setting the following
554 	 * mask bits in the access_mask:
555 	 * ACE_SYNCHRONIZE, ACE_WRITE_OWNER, ACE_DELETE,
556 	 * ACE_WRITE_ATTRIBUTES, ACE_WRITE_NAMED_ATTRS, ACE_READ_NAMED_ATTRS
557 	 */
558 	access = access_mask_set(haswriteperm, hasreadperm, isowner, isallow);
559 
560 	if (isallow) {
561 		access |= ACE_READ_ACL | ACE_READ_ATTRIBUTES;
562 		if (isowner)
563 			access |= ACE_WRITE_ACL;
564 	} else {
565 		if (! isowner)
566 			access |= ACE_WRITE_ACL;
567 	}
568 
569 	/* read */
570 	if (mode & S_IROTH) {
571 		access |= ACE_READ_DATA;
572 	}
573 	/* write */
574 	if (mode & S_IWOTH) {
575 		access |= ACE_WRITE_DATA |
576 		    ACE_APPEND_DATA;
577 		if (isdir)
578 			access |= ACE_DELETE_CHILD;
579 	}
580 	/* exec */
581 	if (mode & 01) {
582 		access |= ACE_EXECUTE;
583 	}
584 
585 	return (access);
586 }
587 
588 /*
589  * Given an nfsace (presumably an ALLOW entry), make a
590  * corresponding DENY entry at the address given.
591  */
592 static void
593 ace_make_deny(ace_t *allow, ace_t *deny, int isdir, int isowner)
594 {
595 	(void) memcpy(deny, allow, sizeof (ace_t));
596 
597 	deny->a_who = allow->a_who;
598 
599 	deny->a_type = ACE_ACCESS_DENIED_ACE_TYPE;
600 	deny->a_access_mask ^= ACE_POSIX_SUPPORTED_BITS;
601 	if (isdir)
602 		deny->a_access_mask ^= ACE_DELETE_CHILD;
603 
604 	deny->a_access_mask &= ~(ACE_SYNCHRONIZE | ACE_WRITE_OWNER |
605 	    ACE_DELETE | ACE_WRITE_ATTRIBUTES | ACE_READ_NAMED_ATTRS |
606 	    ACE_WRITE_NAMED_ATTRS);
607 	deny->a_access_mask |= access_mask_set((allow->a_access_mask &
608 	    ACE_WRITE_DATA), (allow->a_access_mask & ACE_READ_DATA), isowner,
609 	    B_FALSE);
610 }
611 /*
612  * Make an initial pass over an array of aclent_t's.  Gather
613  * information such as an ACL_MASK (if any), number of users,
614  * number of groups, and whether the array needs to be sorted.
615  */
616 static int
617 ln_aent_preprocess(aclent_t *aclent, int n,
618     int *hasmask, mode_t *mask,
619     int *numuser, int *numgroup, int *needsort)
620 {
621 	int error = 0;
622 	int i;
623 	int curtype = 0;
624 
625 	*hasmask = 0;
626 	*mask = 07;
627 	*needsort = 0;
628 	*numuser = 0;
629 	*numgroup = 0;
630 
631 	for (i = 0; i < n; i++) {
632 		if (aclent[i].a_type < curtype)
633 			*needsort = 1;
634 		else if (aclent[i].a_type > curtype)
635 			curtype = aclent[i].a_type;
636 		if (aclent[i].a_type & USER)
637 			(*numuser)++;
638 		if (aclent[i].a_type & (GROUP | GROUP_OBJ))
639 			(*numgroup)++;
640 		if (aclent[i].a_type & CLASS_OBJ) {
641 			if (*hasmask) {
642 				error = EINVAL;
643 				goto out;
644 			} else {
645 				*hasmask = 1;
646 				*mask = aclent[i].a_perm;
647 			}
648 		}
649 	}
650 
651 	if ((! *hasmask) && (*numuser + *numgroup > 1)) {
652 		error = EINVAL;
653 		goto out;
654 	}
655 
656 out:
657 	return (error);
658 }
659 
660 /*
661  * Convert an array of aclent_t into an array of nfsace entries,
662  * following POSIX draft -> nfsv4 conversion semantics as outlined in
663  * the IETF draft.
664  */
665 static int
666 ln_aent_to_ace(aclent_t *aclent, int n, ace_t **acepp, int *rescount, int isdir)
667 {
668 	int error = 0;
669 	mode_t mask;
670 	int numuser, numgroup, needsort;
671 	int resultsize = 0;
672 	int i, groupi = 0, skip;
673 	ace_t *acep, *result = NULL;
674 	int hasmask;
675 
676 	error = ln_aent_preprocess(aclent, n, &hasmask, &mask,
677 	    &numuser, &numgroup, &needsort);
678 	if (error != 0)
679 		goto out;
680 
681 	/* allow + deny for each aclent */
682 	resultsize = n * 2;
683 	if (hasmask) {
684 		/*
685 		 * stick extra deny on the group_obj and on each
686 		 * user|group for the mask (the group_obj was added
687 		 * into the count for numgroup)
688 		 */
689 		resultsize += numuser + numgroup;
690 		/* ... and don't count the mask itself */
691 		resultsize -= 2;
692 	}
693 
694 	/* sort the source if necessary */
695 	if (needsort)
696 		ksort((caddr_t)aclent, n, sizeof (aclent_t), cmp2acls);
697 
698 	if (cacl_malloc((void **)&result, resultsize * sizeof (ace_t)) != 0)
699 		goto out;
700 
701 	acep = result;
702 
703 	for (i = 0; i < n; i++) {
704 		/*
705 		 * don't process CLASS_OBJ (mask); mask was grabbed in
706 		 * ln_aent_preprocess()
707 		 */
708 		if (aclent[i].a_type & CLASS_OBJ)
709 			continue;
710 
711 		/* If we need an ACL_MASK emulator, prepend it now */
712 		if ((hasmask) &&
713 		    (aclent[i].a_type & (USER | GROUP | GROUP_OBJ))) {
714 			acep->a_type = ACE_ACCESS_DENIED_ACE_TYPE;
715 			acep->a_flags = 0;
716 			if (aclent[i].a_type & GROUP_OBJ) {
717 				acep->a_who = (uid_t)-1;
718 				acep->a_flags |=
719 				    (ACE_IDENTIFIER_GROUP|ACE_GROUP);
720 			} else if (aclent[i].a_type & USER) {
721 				acep->a_who = aclent[i].a_id;
722 			} else {
723 				acep->a_who = aclent[i].a_id;
724 				acep->a_flags |= ACE_IDENTIFIER_GROUP;
725 			}
726 			if (aclent[i].a_type & ACL_DEFAULT) {
727 				acep->a_flags |= ACE_INHERIT_ONLY_ACE |
728 				    ACE_FILE_INHERIT_ACE |
729 				    ACE_DIRECTORY_INHERIT_ACE;
730 			}
731 			/*
732 			 * Set the access mask for the prepended deny
733 			 * ace.  To do this, we invert the mask (found
734 			 * in ln_aent_preprocess()) then convert it to an
735 			 * DENY ace access_mask.
736 			 */
737 			acep->a_access_mask = mode_to_ace_access((mask ^ 07),
738 			    isdir, 0, 0);
739 			acep += 1;
740 		}
741 
742 		/* handle a_perm -> access_mask */
743 		acep->a_access_mask = mode_to_ace_access(aclent[i].a_perm,
744 		    isdir, aclent[i].a_type & USER_OBJ, 1);
745 
746 		/* emulate a default aclent */
747 		if (aclent[i].a_type & ACL_DEFAULT) {
748 			acep->a_flags |= ACE_INHERIT_ONLY_ACE |
749 			    ACE_FILE_INHERIT_ACE |
750 			    ACE_DIRECTORY_INHERIT_ACE;
751 		}
752 
753 		/*
754 		 * handle a_perm and a_id
755 		 *
756 		 * this must be done last, since it involves the
757 		 * corresponding deny aces, which are handled
758 		 * differently for each different a_type.
759 		 */
760 		if (aclent[i].a_type & USER_OBJ) {
761 			acep->a_who = (uid_t)-1;
762 			acep->a_flags |= ACE_OWNER;
763 			ace_make_deny(acep, acep + 1, isdir, B_TRUE);
764 			acep += 2;
765 		} else if (aclent[i].a_type & USER) {
766 			acep->a_who = aclent[i].a_id;
767 			ace_make_deny(acep, acep + 1, isdir, B_FALSE);
768 			acep += 2;
769 		} else if (aclent[i].a_type & (GROUP_OBJ | GROUP)) {
770 			if (aclent[i].a_type & GROUP_OBJ) {
771 				acep->a_who = (uid_t)-1;
772 				acep->a_flags |= ACE_GROUP;
773 			} else {
774 				acep->a_who = aclent[i].a_id;
775 			}
776 			acep->a_flags |= ACE_IDENTIFIER_GROUP;
777 			/*
778 			 * Set the corresponding deny for the group ace.
779 			 *
780 			 * The deny aces go after all of the groups, unlike
781 			 * everything else, where they immediately follow
782 			 * the allow ace.
783 			 *
784 			 * We calculate "skip", the number of slots to
785 			 * skip ahead for the deny ace, here.
786 			 *
787 			 * The pattern is:
788 			 * MD1 A1 MD2 A2 MD3 A3 D1 D2 D3
789 			 * thus, skip is
790 			 * (2 * numgroup) - 1 - groupi
791 			 * (2 * numgroup) to account for MD + A
792 			 * - 1 to account for the fact that we're on the
793 			 * access (A), not the mask (MD)
794 			 * - groupi to account for the fact that we have
795 			 * passed up groupi number of MD's.
796 			 */
797 			skip = (2 * numgroup) - 1 - groupi;
798 			ace_make_deny(acep, acep + skip, isdir, B_FALSE);
799 			/*
800 			 * If we just did the last group, skip acep past
801 			 * all of the denies; else, just move ahead one.
802 			 */
803 			if (++groupi >= numgroup)
804 				acep += numgroup + 1;
805 			else
806 				acep += 1;
807 		} else if (aclent[i].a_type & OTHER_OBJ) {
808 			acep->a_who = (uid_t)-1;
809 			acep->a_flags |= ACE_EVERYONE;
810 			ace_make_deny(acep, acep + 1, isdir, B_FALSE);
811 			acep += 2;
812 		} else {
813 			error = EINVAL;
814 			goto out;
815 		}
816 	}
817 
818 	*acepp = result;
819 	*rescount = resultsize;
820 
821 out:
822 	if (error != 0) {
823 		if ((result != NULL) && (resultsize > 0)) {
824 			cacl_free(result, resultsize * sizeof (ace_t));
825 		}
826 	}
827 
828 	return (error);
829 }
830 
831 static int
832 convert_aent_to_ace(aclent_t *aclentp, int aclcnt, int isdir,
833     ace_t **retacep, int *retacecnt)
834 {
835 	ace_t *acep;
836 	ace_t *dfacep;
837 	int acecnt = 0;
838 	int dfacecnt = 0;
839 	int dfaclstart = 0;
840 	int dfaclcnt = 0;
841 	aclent_t *aclp;
842 	int i;
843 	int error;
844 	int acesz, dfacesz;
845 
846 	ksort((caddr_t)aclentp, aclcnt, sizeof (aclent_t), cmp2acls);
847 
848 	for (i = 0, aclp = aclentp; i < aclcnt; aclp++, i++) {
849 		if (aclp->a_type & ACL_DEFAULT)
850 			break;
851 	}
852 
853 	if (i < aclcnt) {
854 		dfaclstart = i;
855 		dfaclcnt = aclcnt - i;
856 	}
857 
858 	if (dfaclcnt && isdir == 0) {
859 		return (EINVAL);
860 	}
861 
862 	error = ln_aent_to_ace(aclentp, i,  &acep, &acecnt, isdir);
863 	if (error)
864 		return (error);
865 
866 	if (dfaclcnt) {
867 		error = ln_aent_to_ace(&aclentp[dfaclstart], dfaclcnt,
868 		    &dfacep, &dfacecnt, isdir);
869 		if (error) {
870 			if (acep) {
871 				cacl_free(acep, acecnt * sizeof (ace_t));
872 			}
873 			return (error);
874 		}
875 	}
876 
877 	if (dfacecnt != 0) {
878 		acesz = sizeof (ace_t) * acecnt;
879 		dfacesz = sizeof (ace_t) * dfacecnt;
880 		acep = cacl_realloc(acep, acesz, acesz + dfacesz);
881 		if (acep == NULL)
882 			return (ENOMEM);
883 		if (dfaclcnt) {
884 			(void) memcpy(acep + acecnt, dfacep, dfacesz);
885 		}
886 	}
887 	if (dfaclcnt)
888 		cacl_free(dfacep, dfacecnt * sizeof (ace_t));
889 
890 	*retacecnt = acecnt + dfacecnt;
891 	*retacep = acep;
892 	return (0);
893 }
894 
895 static int
896 ace_mask_to_mode(uint32_t  mask, o_mode_t *modep, int isdir)
897 {
898 	int error = 0;
899 	o_mode_t mode = 0;
900 	uint32_t bits, wantbits;
901 
902 	/* read */
903 	if (mask & ACE_READ_DATA)
904 		mode |= S_IROTH;
905 
906 	/* write */
907 	wantbits = (ACE_WRITE_DATA | ACE_APPEND_DATA);
908 	if (isdir)
909 		wantbits |= ACE_DELETE_CHILD;
910 	bits = mask & wantbits;
911 	if (bits != 0) {
912 		if (bits != wantbits) {
913 			error = ENOTSUP;
914 			goto out;
915 		}
916 		mode |= S_IWOTH;
917 	}
918 
919 	/* exec */
920 	if (mask & ACE_EXECUTE) {
921 		mode |= S_IXOTH;
922 	}
923 
924 	*modep = mode;
925 
926 out:
927 	return (error);
928 }
929 
930 static void
931 acevals_init(acevals_t *vals, uid_t key)
932 {
933 	bzero(vals, sizeof (*vals));
934 	vals->allowed = ACE_MASK_UNDEFINED;
935 	vals->denied = ACE_MASK_UNDEFINED;
936 	vals->mask = ACE_MASK_UNDEFINED;
937 	vals->key = key;
938 }
939 
940 static void
941 ace_list_init(ace_list_t *al, int dfacl_flag)
942 {
943 	acevals_init(&al->user_obj, NULL);
944 	acevals_init(&al->group_obj, NULL);
945 	acevals_init(&al->other_obj, NULL);
946 	al->numusers = 0;
947 	al->numgroups = 0;
948 	al->acl_mask = 0;
949 	al->hasmask = 0;
950 	al->state = ace_unused;
951 	al->seen = 0;
952 	al->dfacl_flag = dfacl_flag;
953 }
954 
955 /*
956  * Find or create an acevals holder for a given id and avl tree.
957  *
958  * Note that only one thread will ever touch these avl trees, so
959  * there is no need for locking.
960  */
961 static acevals_t *
962 acevals_find(ace_t *ace, avl_tree_t *avl, int *num)
963 {
964 	acevals_t key, *rc;
965 	avl_index_t where;
966 
967 	key.key = ace->a_who;
968 	rc = avl_find(avl, &key, &where);
969 	if (rc != NULL)
970 		return (rc);
971 
972 	/* this memory is freed by ln_ace_to_aent()->ace_list_free() */
973 	if (cacl_malloc((void **)&rc, sizeof (acevals_t)) != 0)
974 		return (NULL);
975 
976 	acevals_init(rc, ace->a_who);
977 	avl_insert(avl, rc, where);
978 	(*num)++;
979 
980 	return (rc);
981 }
982 
983 static int
984 access_mask_check(ace_t *acep, int mask_bit, int isowner)
985 {
986 	int set_deny, err_deny;
987 	int set_allow, err_allow;
988 	int acl_consume;
989 	int haswriteperm, hasreadperm;
990 
991 	if (acep->a_type == ACE_ACCESS_DENIED_ACE_TYPE) {
992 		haswriteperm = (acep->a_access_mask & ACE_WRITE_DATA) ? 0 : 1;
993 		hasreadperm = (acep->a_access_mask & ACE_READ_DATA) ? 0 : 1;
994 	} else {
995 		haswriteperm = (acep->a_access_mask & ACE_WRITE_DATA) ? 1 : 0;
996 		hasreadperm = (acep->a_access_mask & ACE_READ_DATA) ? 1 : 0;
997 	}
998 
999 	acl_consume = (ACL_SYNCHRONIZE_ERR_DENY |
1000 	    ACL_DELETE_ERR_DENY |
1001 	    ACL_WRITE_OWNER_ERR_DENY |
1002 	    ACL_WRITE_OWNER_ERR_ALLOW |
1003 	    ACL_WRITE_ATTRS_OWNER_SET_ALLOW |
1004 	    ACL_WRITE_ATTRS_OWNER_ERR_DENY |
1005 	    ACL_WRITE_ATTRS_WRITER_SET_DENY |
1006 	    ACL_WRITE_ATTRS_WRITER_ERR_ALLOW |
1007 	    ACL_WRITE_NAMED_WRITER_ERR_DENY |
1008 	    ACL_READ_NAMED_READER_ERR_DENY);
1009 
1010 	if (mask_bit == ACE_SYNCHRONIZE) {
1011 		set_deny = ACL_SYNCHRONIZE_SET_DENY;
1012 		err_deny =  ACL_SYNCHRONIZE_ERR_DENY;
1013 		set_allow = ACL_SYNCHRONIZE_SET_ALLOW;
1014 		err_allow = ACL_SYNCHRONIZE_ERR_ALLOW;
1015 	} else if (mask_bit == ACE_WRITE_OWNER) {
1016 		set_deny = ACL_WRITE_OWNER_SET_DENY;
1017 		err_deny =  ACL_WRITE_OWNER_ERR_DENY;
1018 		set_allow = ACL_WRITE_OWNER_SET_ALLOW;
1019 		err_allow = ACL_WRITE_OWNER_ERR_ALLOW;
1020 	} else if (mask_bit == ACE_DELETE) {
1021 		set_deny = ACL_DELETE_SET_DENY;
1022 		err_deny =  ACL_DELETE_ERR_DENY;
1023 		set_allow = ACL_DELETE_SET_ALLOW;
1024 		err_allow = ACL_DELETE_ERR_ALLOW;
1025 	} else if (mask_bit == ACE_WRITE_ATTRIBUTES) {
1026 		if (isowner) {
1027 			set_deny = ACL_WRITE_ATTRS_OWNER_SET_DENY;
1028 			err_deny =  ACL_WRITE_ATTRS_OWNER_ERR_DENY;
1029 			set_allow = ACL_WRITE_ATTRS_OWNER_SET_ALLOW;
1030 			err_allow = ACL_WRITE_ATTRS_OWNER_ERR_ALLOW;
1031 		} else if (haswriteperm) {
1032 			set_deny = ACL_WRITE_ATTRS_WRITER_SET_DENY;
1033 			err_deny =  ACL_WRITE_ATTRS_WRITER_ERR_DENY;
1034 			set_allow = ACL_WRITE_ATTRS_WRITER_SET_ALLOW;
1035 			err_allow = ACL_WRITE_ATTRS_WRITER_ERR_ALLOW;
1036 		} else {
1037 			if ((acep->a_access_mask & mask_bit) &&
1038 			    (acep->a_type & ACE_ACCESS_ALLOWED_ACE_TYPE)) {
1039 				return (ENOTSUP);
1040 			}
1041 			return (0);
1042 		}
1043 	} else if (mask_bit == ACE_READ_NAMED_ATTRS) {
1044 		if (!hasreadperm)
1045 			return (0);
1046 
1047 		set_deny = ACL_READ_NAMED_READER_SET_DENY;
1048 		err_deny = ACL_READ_NAMED_READER_ERR_DENY;
1049 		set_allow = ACL_READ_NAMED_READER_SET_ALLOW;
1050 		err_allow = ACL_READ_NAMED_READER_ERR_ALLOW;
1051 	} else if (mask_bit == ACE_WRITE_NAMED_ATTRS) {
1052 		if (!haswriteperm)
1053 			return (0);
1054 
1055 		set_deny = ACL_WRITE_NAMED_WRITER_SET_DENY;
1056 		err_deny = ACL_WRITE_NAMED_WRITER_ERR_DENY;
1057 		set_allow = ACL_WRITE_NAMED_WRITER_SET_ALLOW;
1058 		err_allow = ACL_WRITE_NAMED_WRITER_ERR_ALLOW;
1059 	} else {
1060 		return (EINVAL);
1061 	}
1062 
1063 	if (acep->a_type == ACE_ACCESS_DENIED_ACE_TYPE) {
1064 		if (acl_consume & set_deny) {
1065 			if (!(acep->a_access_mask & mask_bit)) {
1066 				return (ENOTSUP);
1067 			}
1068 		} else if (acl_consume & err_deny) {
1069 			if (acep->a_access_mask & mask_bit) {
1070 				return (ENOTSUP);
1071 			}
1072 		}
1073 	} else {
1074 		/* ACE_ACCESS_ALLOWED_ACE_TYPE */
1075 		if (acl_consume & set_allow) {
1076 			if (!(acep->a_access_mask & mask_bit)) {
1077 				return (ENOTSUP);
1078 			}
1079 		} else if (acl_consume & err_allow) {
1080 			if (acep->a_access_mask & mask_bit) {
1081 				return (ENOTSUP);
1082 			}
1083 		}
1084 	}
1085 	return (0);
1086 }
1087 
1088 static int
1089 ace_to_aent_legal(ace_t *acep)
1090 {
1091 	int error = 0;
1092 	int isowner;
1093 
1094 	/* only ALLOW or DENY */
1095 	if ((acep->a_type != ACE_ACCESS_ALLOWED_ACE_TYPE) &&
1096 	    (acep->a_type != ACE_ACCESS_DENIED_ACE_TYPE)) {
1097 		error = ENOTSUP;
1098 		goto out;
1099 	}
1100 
1101 	/* check for invalid flags */
1102 	if (acep->a_flags & ~(ACE_VALID_FLAG_BITS)) {
1103 		error = EINVAL;
1104 		goto out;
1105 	}
1106 
1107 	/* some flags are illegal */
1108 	if (acep->a_flags & (ACE_SUCCESSFUL_ACCESS_ACE_FLAG |
1109 	    ACE_FAILED_ACCESS_ACE_FLAG |
1110 	    ACE_NO_PROPAGATE_INHERIT_ACE)) {
1111 		error = ENOTSUP;
1112 		goto out;
1113 	}
1114 
1115 	/* check for invalid masks */
1116 	if (acep->a_access_mask & ~(ACE_VALID_MASK_BITS)) {
1117 		error = EINVAL;
1118 		goto out;
1119 	}
1120 
1121 	if ((acep->a_flags & ACE_OWNER)) {
1122 		isowner = 1;
1123 	} else {
1124 		isowner = 0;
1125 	}
1126 
1127 	error = access_mask_check(acep, ACE_SYNCHRONIZE, isowner);
1128 	if (error)
1129 		goto out;
1130 
1131 	error = access_mask_check(acep, ACE_WRITE_OWNER, isowner);
1132 	if (error)
1133 		goto out;
1134 
1135 	error = access_mask_check(acep, ACE_DELETE, isowner);
1136 	if (error)
1137 		goto out;
1138 
1139 	error = access_mask_check(acep, ACE_WRITE_ATTRIBUTES, isowner);
1140 	if (error)
1141 		goto out;
1142 
1143 	error = access_mask_check(acep, ACE_READ_NAMED_ATTRS, isowner);
1144 	if (error)
1145 		goto out;
1146 
1147 	error = access_mask_check(acep, ACE_WRITE_NAMED_ATTRS, isowner);
1148 	if (error)
1149 		goto out;
1150 
1151 	/* more detailed checking of masks */
1152 	if (acep->a_type == ACE_ACCESS_ALLOWED_ACE_TYPE) {
1153 		if (! (acep->a_access_mask & ACE_READ_ATTRIBUTES)) {
1154 			error = ENOTSUP;
1155 			goto out;
1156 		}
1157 		if ((acep->a_access_mask & ACE_WRITE_DATA) &&
1158 		    (! (acep->a_access_mask & ACE_APPEND_DATA))) {
1159 			error = ENOTSUP;
1160 			goto out;
1161 		}
1162 		if ((! (acep->a_access_mask & ACE_WRITE_DATA)) &&
1163 		    (acep->a_access_mask & ACE_APPEND_DATA)) {
1164 			error = ENOTSUP;
1165 			goto out;
1166 		}
1167 	}
1168 
1169 	/* ACL enforcement */
1170 	if ((acep->a_access_mask & ACE_READ_ACL) &&
1171 	    (acep->a_type != ACE_ACCESS_ALLOWED_ACE_TYPE)) {
1172 		error = ENOTSUP;
1173 		goto out;
1174 	}
1175 	if (acep->a_access_mask & ACE_WRITE_ACL) {
1176 		if ((acep->a_type == ACE_ACCESS_DENIED_ACE_TYPE) &&
1177 		    (isowner)) {
1178 			error = ENOTSUP;
1179 			goto out;
1180 		}
1181 		if ((acep->a_type == ACE_ACCESS_ALLOWED_ACE_TYPE) &&
1182 		    (! isowner)) {
1183 			error = ENOTSUP;
1184 			goto out;
1185 		}
1186 	}
1187 
1188 out:
1189 	return (error);
1190 }
1191 
1192 static int
1193 ace_allow_to_mode(uint32_t mask, o_mode_t *modep, int isdir)
1194 {
1195 	/* ACE_READ_ACL and ACE_READ_ATTRIBUTES must both be set */
1196 	if ((mask & (ACE_READ_ACL | ACE_READ_ATTRIBUTES)) !=
1197 	    (ACE_READ_ACL | ACE_READ_ATTRIBUTES)) {
1198 		return (ENOTSUP);
1199 	}
1200 
1201 	return (ace_mask_to_mode(mask, modep, isdir));
1202 }
1203 
1204 static int
1205 acevals_to_aent(acevals_t *vals, aclent_t *dest, ace_list_t *list,
1206     uid_t owner, gid_t group, int isdir)
1207 {
1208 	int error;
1209 	uint32_t  flips = ACE_POSIX_SUPPORTED_BITS;
1210 
1211 	if (isdir)
1212 		flips |= ACE_DELETE_CHILD;
1213 	if (vals->allowed != (vals->denied ^ flips)) {
1214 		error = ENOTSUP;
1215 		goto out;
1216 	}
1217 	if ((list->hasmask) && (list->acl_mask != vals->mask) &&
1218 	    (vals->aent_type & (USER | GROUP | GROUP_OBJ))) {
1219 		error = ENOTSUP;
1220 		goto out;
1221 	}
1222 	error = ace_allow_to_mode(vals->allowed, &dest->a_perm, isdir);
1223 	if (error != 0)
1224 		goto out;
1225 	dest->a_type = vals->aent_type;
1226 	if (dest->a_type & (USER | GROUP)) {
1227 		dest->a_id = vals->key;
1228 	} else if (dest->a_type & USER_OBJ) {
1229 		dest->a_id = owner;
1230 	} else if (dest->a_type & GROUP_OBJ) {
1231 		dest->a_id = group;
1232 	} else if (dest->a_type & OTHER_OBJ) {
1233 		dest->a_id = 0;
1234 	} else {
1235 		error = EINVAL;
1236 		goto out;
1237 	}
1238 
1239 out:
1240 	return (error);
1241 }
1242 
1243 
1244 static int
1245 ace_list_to_aent(ace_list_t *list, aclent_t **aclentp, int *aclcnt,
1246     uid_t owner, gid_t group, int isdir)
1247 {
1248 	int error = 0;
1249 	aclent_t *aent, *result = NULL;
1250 	acevals_t *vals;
1251 	int resultcount;
1252 
1253 	if ((list->seen & (USER_OBJ | GROUP_OBJ | OTHER_OBJ)) !=
1254 	    (USER_OBJ | GROUP_OBJ | OTHER_OBJ)) {
1255 		error = ENOTSUP;
1256 		goto out;
1257 	}
1258 	if ((! list->hasmask) && (list->numusers + list->numgroups > 0)) {
1259 		error = ENOTSUP;
1260 		goto out;
1261 	}
1262 
1263 	resultcount = 3 + list->numusers + list->numgroups;
1264 	/*
1265 	 * This must be the same condition as below, when we add the CLASS_OBJ
1266 	 * (aka ACL mask)
1267 	 */
1268 	if ((list->hasmask) || (! list->dfacl_flag))
1269 		resultcount += 1;
1270 
1271 	if (cacl_malloc((void **)&result,
1272 	    resultcount * sizeof (aclent_t)) != 0) {
1273 		error = ENOMEM;
1274 		goto out;
1275 	}
1276 	aent = result;
1277 
1278 	/* USER_OBJ */
1279 	if (!(list->user_obj.aent_type & USER_OBJ)) {
1280 		error = EINVAL;
1281 		goto out;
1282 	}
1283 
1284 	error = acevals_to_aent(&list->user_obj, aent, list, owner, group,
1285 	    isdir);
1286 
1287 	if (error != 0)
1288 		goto out;
1289 	++aent;
1290 	/* USER */
1291 	vals = NULL;
1292 	for (vals = avl_first(&list->user); vals != NULL;
1293 	    vals = AVL_NEXT(&list->user, vals)) {
1294 		if (!(vals->aent_type & USER)) {
1295 			error = EINVAL;
1296 			goto out;
1297 		}
1298 		error = acevals_to_aent(vals, aent, list, owner, group,
1299 		    isdir);
1300 		if (error != 0)
1301 			goto out;
1302 		++aent;
1303 	}
1304 	/* GROUP_OBJ */
1305 	if (!(list->group_obj.aent_type & GROUP_OBJ)) {
1306 		error = EINVAL;
1307 		goto out;
1308 	}
1309 	error = acevals_to_aent(&list->group_obj, aent, list, owner, group,
1310 	    isdir);
1311 	if (error != 0)
1312 		goto out;
1313 	++aent;
1314 	/* GROUP */
1315 	vals = NULL;
1316 	for (vals = avl_first(&list->group); vals != NULL;
1317 	    vals = AVL_NEXT(&list->group, vals)) {
1318 		if (!(vals->aent_type & GROUP)) {
1319 			error = EINVAL;
1320 			goto out;
1321 		}
1322 		error = acevals_to_aent(vals, aent, list, owner, group,
1323 		    isdir);
1324 		if (error != 0)
1325 			goto out;
1326 		++aent;
1327 	}
1328 	/*
1329 	 * CLASS_OBJ (aka ACL_MASK)
1330 	 *
1331 	 * An ACL_MASK is not fabricated if the ACL is a default ACL.
1332 	 * This is to follow UFS's behavior.
1333 	 */
1334 	if ((list->hasmask) || (! list->dfacl_flag)) {
1335 		if (list->hasmask) {
1336 			uint32_t flips = ACE_POSIX_SUPPORTED_BITS;
1337 			if (isdir)
1338 				flips |= ACE_DELETE_CHILD;
1339 			error = ace_mask_to_mode(list->acl_mask ^ flips,
1340 			    &aent->a_perm, isdir);
1341 			if (error != 0)
1342 				goto out;
1343 		} else {
1344 			/* fabricate the ACL_MASK from the group permissions */
1345 			error = ace_mask_to_mode(list->group_obj.allowed,
1346 			    &aent->a_perm, isdir);
1347 			if (error != 0)
1348 				goto out;
1349 		}
1350 		aent->a_id = 0;
1351 		aent->a_type = CLASS_OBJ | list->dfacl_flag;
1352 		++aent;
1353 	}
1354 	/* OTHER_OBJ */
1355 	if (!(list->other_obj.aent_type & OTHER_OBJ)) {
1356 		error = EINVAL;
1357 		goto out;
1358 	}
1359 	error = acevals_to_aent(&list->other_obj, aent, list, owner, group,
1360 	    isdir);
1361 	if (error != 0)
1362 		goto out;
1363 	++aent;
1364 
1365 	*aclentp = result;
1366 	*aclcnt = resultcount;
1367 
1368 out:
1369 	if (error != 0) {
1370 		if (result != NULL)
1371 			cacl_free(result, resultcount * sizeof (aclent_t));
1372 	}
1373 
1374 	return (error);
1375 }
1376 
1377 
1378 /*
1379  * free all data associated with an ace_list
1380  */
1381 static void
1382 ace_list_free(ace_list_t *al)
1383 {
1384 	acevals_t *node;
1385 	void *cookie;
1386 
1387 	if (al == NULL)
1388 		return;
1389 
1390 	cookie = NULL;
1391 	while ((node = avl_destroy_nodes(&al->user, &cookie)) != NULL)
1392 		cacl_free(node, sizeof (acevals_t));
1393 	cookie = NULL;
1394 	while ((node = avl_destroy_nodes(&al->group, &cookie)) != NULL)
1395 		cacl_free(node, sizeof (acevals_t));
1396 
1397 	avl_destroy(&al->user);
1398 	avl_destroy(&al->group);
1399 
1400 	/* free the container itself */
1401 	cacl_free(al, sizeof (ace_list_t));
1402 }
1403 
1404 static int
1405 acevals_compare(const void *va, const void *vb)
1406 {
1407 	const acevals_t *a = va, *b = vb;
1408 
1409 	if (a->key == b->key)
1410 		return (0);
1411 
1412 	if (a->key > b->key)
1413 		return (1);
1414 
1415 	else
1416 		return (-1);
1417 }
1418 
1419 /*
1420  * Convert a list of ace_t entries to equivalent regular and default
1421  * aclent_t lists.  Return error (ENOTSUP) when conversion is not possible.
1422  */
1423 static int
1424 ln_ace_to_aent(ace_t *ace, int n, uid_t owner, gid_t group,
1425     aclent_t **aclentp, int *aclcnt, aclent_t **dfaclentp, int *dfaclcnt,
1426     int isdir)
1427 {
1428 	int error = 0;
1429 	ace_t *acep;
1430 	uint32_t bits;
1431 	int i;
1432 	ace_list_t *normacl = NULL, *dfacl = NULL, *acl;
1433 	acevals_t *vals;
1434 
1435 	*aclentp = NULL;
1436 	*aclcnt = 0;
1437 	*dfaclentp = NULL;
1438 	*dfaclcnt = 0;
1439 
1440 	/* we need at least user_obj, group_obj, and other_obj */
1441 	if (n < 6) {
1442 		error = ENOTSUP;
1443 		goto out;
1444 	}
1445 	if (ace == NULL) {
1446 		error = EINVAL;
1447 		goto out;
1448 	}
1449 
1450 	error = cacl_malloc((void **)&normacl, sizeof (ace_list_t));
1451 	if (error != 0)
1452 		goto out;
1453 
1454 	avl_create(&normacl->user, acevals_compare, sizeof (acevals_t),
1455 	    offsetof(acevals_t, avl));
1456 	avl_create(&normacl->group, acevals_compare, sizeof (acevals_t),
1457 	    offsetof(acevals_t, avl));
1458 
1459 	ace_list_init(normacl, 0);
1460 
1461 	error = cacl_malloc((void **)&dfacl, sizeof (ace_list_t));
1462 	if (error != 0)
1463 		goto out;
1464 
1465 	avl_create(&dfacl->user, acevals_compare, sizeof (acevals_t),
1466 	    offsetof(acevals_t, avl));
1467 	avl_create(&dfacl->group, acevals_compare, sizeof (acevals_t),
1468 	    offsetof(acevals_t, avl));
1469 	ace_list_init(dfacl, ACL_DEFAULT);
1470 
1471 	/* process every ace_t... */
1472 	for (i = 0; i < n; i++) {
1473 		acep = &ace[i];
1474 
1475 		/* rule out certain cases quickly */
1476 		error = ace_to_aent_legal(acep);
1477 		if (error != 0)
1478 			goto out;
1479 
1480 		/*
1481 		 * Turn off these bits in order to not have to worry about
1482 		 * them when doing the checks for compliments.
1483 		 */
1484 		acep->a_access_mask &= ~(ACE_WRITE_OWNER | ACE_DELETE |
1485 		    ACE_SYNCHRONIZE | ACE_WRITE_ATTRIBUTES |
1486 		    ACE_READ_NAMED_ATTRS | ACE_WRITE_NAMED_ATTRS);
1487 
1488 		/* see if this should be a regular or default acl */
1489 		bits = acep->a_flags &
1490 		    (ACE_INHERIT_ONLY_ACE |
1491 		    ACE_FILE_INHERIT_ACE |
1492 		    ACE_DIRECTORY_INHERIT_ACE);
1493 		if (bits != 0) {
1494 			/* all or nothing on these inherit bits */
1495 			if (bits != (ACE_INHERIT_ONLY_ACE |
1496 			    ACE_FILE_INHERIT_ACE |
1497 			    ACE_DIRECTORY_INHERIT_ACE)) {
1498 				error = ENOTSUP;
1499 				goto out;
1500 			}
1501 			acl = dfacl;
1502 		} else {
1503 			acl = normacl;
1504 		}
1505 
1506 		if ((acep->a_flags & ACE_OWNER)) {
1507 			if (acl->state > ace_user_obj) {
1508 				error = ENOTSUP;
1509 				goto out;
1510 			}
1511 			acl->state = ace_user_obj;
1512 			acl->seen |= USER_OBJ;
1513 			vals = &acl->user_obj;
1514 			vals->aent_type = USER_OBJ | acl->dfacl_flag;
1515 		} else if ((acep->a_flags & ACE_EVERYONE)) {
1516 			acl->state = ace_other_obj;
1517 			acl->seen |= OTHER_OBJ;
1518 			vals = &acl->other_obj;
1519 			vals->aent_type = OTHER_OBJ | acl->dfacl_flag;
1520 		} else if (acep->a_flags & ACE_IDENTIFIER_GROUP) {
1521 			if (acl->state > ace_group) {
1522 				error = ENOTSUP;
1523 				goto out;
1524 			}
1525 			if ((acep->a_flags & ACE_GROUP)) {
1526 				acl->seen |= GROUP_OBJ;
1527 				vals = &acl->group_obj;
1528 				vals->aent_type = GROUP_OBJ | acl->dfacl_flag;
1529 			} else {
1530 				acl->seen |= GROUP;
1531 				vals = acevals_find(acep, &acl->group,
1532 				    &acl->numgroups);
1533 				if (vals == NULL) {
1534 					error = ENOMEM;
1535 					goto out;
1536 				}
1537 				vals->aent_type = GROUP | acl->dfacl_flag;
1538 			}
1539 			acl->state = ace_group;
1540 		} else {
1541 			if (acl->state > ace_user) {
1542 				error = ENOTSUP;
1543 				goto out;
1544 			}
1545 			acl->state = ace_user;
1546 			acl->seen |= USER;
1547 			vals = acevals_find(acep, &acl->user,
1548 			    &acl->numusers);
1549 			if (vals == NULL) {
1550 				error = ENOMEM;
1551 				goto out;
1552 			}
1553 			vals->aent_type = USER | acl->dfacl_flag;
1554 		}
1555 
1556 		if (!(acl->state > ace_unused)) {
1557 			error = EINVAL;
1558 			goto out;
1559 		}
1560 
1561 		if (acep->a_type == ACE_ACCESS_ALLOWED_ACE_TYPE) {
1562 			/* no more than one allowed per aclent_t */
1563 			if (vals->allowed != ACE_MASK_UNDEFINED) {
1564 				error = ENOTSUP;
1565 				goto out;
1566 			}
1567 			vals->allowed = acep->a_access_mask;
1568 		} else {
1569 			/*
1570 			 * it's a DENY; if there was a previous DENY, it
1571 			 * must have been an ACL_MASK.
1572 			 */
1573 			if (vals->denied != ACE_MASK_UNDEFINED) {
1574 				/* ACL_MASK is for USER and GROUP only */
1575 				if ((acl->state != ace_user) &&
1576 				    (acl->state != ace_group)) {
1577 					error = ENOTSUP;
1578 					goto out;
1579 				}
1580 
1581 				if (! acl->hasmask) {
1582 					acl->hasmask = 1;
1583 					acl->acl_mask = vals->denied;
1584 				/* check for mismatched ACL_MASK emulations */
1585 				} else if (acl->acl_mask != vals->denied) {
1586 					error = ENOTSUP;
1587 					goto out;
1588 				}
1589 				vals->mask = vals->denied;
1590 			}
1591 			vals->denied = acep->a_access_mask;
1592 		}
1593 	}
1594 
1595 	/* done collating; produce the aclent_t lists */
1596 	if (normacl->state != ace_unused) {
1597 		error = ace_list_to_aent(normacl, aclentp, aclcnt,
1598 		    owner, group, isdir);
1599 		if (error != 0) {
1600 			goto out;
1601 		}
1602 	}
1603 	if (dfacl->state != ace_unused) {
1604 		error = ace_list_to_aent(dfacl, dfaclentp, dfaclcnt,
1605 		    owner, group, isdir);
1606 		if (error != 0) {
1607 			goto out;
1608 		}
1609 	}
1610 
1611 out:
1612 	if (normacl != NULL)
1613 		ace_list_free(normacl);
1614 	if (dfacl != NULL)
1615 		ace_list_free(dfacl);
1616 
1617 	return (error);
1618 }
1619 
1620 static int
1621 convert_ace_to_aent(ace_t *acebufp, int acecnt, int isdir,
1622     uid_t owner, gid_t group, aclent_t **retaclentp, int *retaclcnt)
1623 {
1624 	int error = 0;
1625 	aclent_t *aclentp, *dfaclentp;
1626 	int aclcnt, dfaclcnt;
1627 	int aclsz, dfaclsz;
1628 
1629 	error = ln_ace_to_aent(acebufp, acecnt, owner, group,
1630 	    &aclentp, &aclcnt, &dfaclentp, &dfaclcnt, isdir);
1631 
1632 	if (error)
1633 		return (error);
1634 
1635 
1636 	if (dfaclcnt != 0) {
1637 		/*
1638 		 * Slap aclentp and dfaclentp into a single array.
1639 		 */
1640 		aclsz = sizeof (aclent_t) * aclcnt;
1641 		dfaclsz = sizeof (aclent_t) * dfaclcnt;
1642 		aclentp = cacl_realloc(aclentp, aclsz, aclsz + dfaclsz);
1643 		if (aclentp != NULL) {
1644 			(void) memcpy(aclentp + aclcnt, dfaclentp, dfaclsz);
1645 		} else {
1646 			error = ENOMEM;
1647 		}
1648 	}
1649 
1650 	if (aclentp) {
1651 		*retaclentp = aclentp;
1652 		*retaclcnt = aclcnt + dfaclcnt;
1653 	}
1654 
1655 	if (dfaclentp)
1656 		cacl_free(dfaclentp, dfaclsz);
1657 
1658 	return (error);
1659 }
1660 
1661 
1662 int
1663 acl_translate(acl_t *aclp, int target_flavor, int isdir, uid_t owner,
1664     gid_t group)
1665 {
1666 	int aclcnt;
1667 	void *acldata;
1668 	int error;
1669 
1670 	/*
1671 	 * See if we need to translate
1672 	 */
1673 	if ((target_flavor == _ACL_ACE_ENABLED && aclp->acl_type == ACE_T) ||
1674 	    (target_flavor == _ACL_ACLENT_ENABLED &&
1675 	    aclp->acl_type == ACLENT_T))
1676 		return (0);
1677 
1678 	if (target_flavor == -1) {
1679 		error = EINVAL;
1680 		goto out;
1681 	}
1682 
1683 	if (target_flavor ==  _ACL_ACE_ENABLED &&
1684 	    aclp->acl_type == ACLENT_T) {
1685 		error = convert_aent_to_ace(aclp->acl_aclp,
1686 		    aclp->acl_cnt, isdir, (ace_t **)&acldata, &aclcnt);
1687 		if (error)
1688 			goto out;
1689 
1690 	} else if (target_flavor == _ACL_ACLENT_ENABLED &&
1691 	    aclp->acl_type == ACE_T) {
1692 		error = convert_ace_to_aent(aclp->acl_aclp, aclp->acl_cnt,
1693 		    isdir, owner, group, (aclent_t **)&acldata, &aclcnt);
1694 		if (error)
1695 			goto out;
1696 	} else {
1697 		error = ENOTSUP;
1698 		goto out;
1699 	}
1700 
1701 	/*
1702 	 * replace old acl with newly translated acl
1703 	 */
1704 	cacl_free(aclp->acl_aclp, aclp->acl_cnt * aclp->acl_entry_size);
1705 	aclp->acl_aclp = acldata;
1706 	aclp->acl_cnt = aclcnt;
1707 	if (target_flavor == _ACL_ACE_ENABLED) {
1708 		aclp->acl_type = ACE_T;
1709 		aclp->acl_entry_size = sizeof (ace_t);
1710 	} else {
1711 		aclp->acl_type = ACLENT_T;
1712 		aclp->acl_entry_size = sizeof (aclent_t);
1713 	}
1714 	return (0);
1715 
1716 out:
1717 
1718 #if !defined(_KERNEL)
1719 	errno = error;
1720 	return (-1);
1721 #else
1722 	return (error);
1723 #endif
1724 }
1725