xref: /illumos-gate/usr/src/uts/common/syscall/lgrpsys.c (revision 13b136d3061155363c62c9f6568d25b8b27da8f6)

/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */

/*
 * Copyright 2006 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 * Copyright 2015 Joyent, Inc.
 */

/*
 * lgroup system calls
 */

#include <sys/types.h>
#include <sys/errno.h>
#include <sys/sunddi.h>
#include <sys/systm.h>
#include <sys/mman.h>
#include <sys/cpupart.h>
#include <sys/lgrp.h>
#include <sys/lgrp_user.h>
#include <sys/promif.h>		/* for prom_printf() */
#include <sys/sysmacros.h>
#include <sys/policy.h>

#include <vm/as.h>


/* definitions for mi_validity */
#define	VALID_ADDR	1
#define	VALID_REQ	2

/*
 * run through the given number of addresses and requests and return the
 * corresponding memory information for each address
 */
static int
meminfo(int addr_count, struct meminfo *mip)
{
	size_t		in_size, out_size, req_size, val_size;
	struct as	*as;
	struct hat	*hat;
	int		i, j, out_idx, info_count;
	lgrp_t		*lgrp;
	pfn_t		pfn;
	ssize_t		pgsz;
	int		*req_array, *val_array;
	uint64_t	*in_array, *out_array;
	uint64_t	addr, paddr;
	uintptr_t	vaddr;
	int		ret = 0;
	struct meminfo minfo;
#if defined(_SYSCALL32_IMPL)
	struct meminfo32 minfo32;
#endif

	/*
	 * Make sure that there is at least one address to translate and
	 * limit how many virtual addresses the kernel can do per call
	 */
	if (addr_count < 1)
		return (set_errno(EINVAL));
	else if (addr_count > MAX_MEMINFO_CNT)
		addr_count = MAX_MEMINFO_CNT;

	if (get_udatamodel() == DATAMODEL_NATIVE) {
		if (copyin(mip, &minfo, sizeof (struct meminfo)))
			return (set_errno(EFAULT));
	}
#if defined(_SYSCALL32_IMPL)
	else {
		bzero(&minfo, sizeof (minfo));
		if (copyin(mip, &minfo32, sizeof (struct meminfo32)))
			return (set_errno(EFAULT));
		minfo.mi_inaddr = (const uint64_t *)(uintptr_t)
		    minfo32.mi_inaddr;
		minfo.mi_info_req = (const uint_t *)(uintptr_t)
		    minfo32.mi_info_req;
		minfo.mi_info_count = minfo32.mi_info_count;
		minfo.mi_outdata = (uint64_t *)(uintptr_t)
		    minfo32.mi_outdata;
		minfo.mi_validity = (uint_t *)(uintptr_t)
		    minfo32.mi_validity;
	}
#endif
	/*
	 * all the input parameters have been copied in:-
	 * addr_count - number of input addresses
	 * minfo.mi_inaddr - array of input addresses
	 * minfo.mi_info_req - array of types of information requested
	 * minfo.mi_info_count - no. of pieces of info requested for each addr
	 * minfo.mi_outdata - array into which the results are placed
	 * minfo.mi_validity -  array containing bitwise result codes; 0th bit
	 *			evaluates validity of corresponding input
	 *			address, 1st bit validity of response to first
	 *			member of info_req, etc.
	 */

	/* make sure mi_info_count is within limit */
	info_count = minfo.mi_info_count;
	if (info_count < 1 || info_count > MAX_MEMINFO_REQ)
		return (set_errno(EINVAL));

	/*
	 * allocate buffer in_array for the input addresses and copy them in
	 */
	in_size = sizeof (uint64_t) * addr_count;
	in_array = kmem_alloc(in_size, KM_SLEEP);
	if (copyin(minfo.mi_inaddr, in_array, in_size)) {
		kmem_free(in_array, in_size);
		return (set_errno(EFAULT));
	}

	/*
	 * allocate buffer req_array for the input info_reqs and copy them in
	 */
	req_size = sizeof (uint_t) * info_count;
	req_array = kmem_alloc(req_size, KM_SLEEP);
	if (copyin(minfo.mi_info_req, req_array, req_size)) {
		kmem_free(req_array, req_size);
		kmem_free(in_array, in_size);
		return (set_errno(EFAULT));
	}

	/*
	 * Validate privs for each req.
	 */
	for (i = 0; i < info_count; i++) {
		switch (req_array[i] & MEMINFO_MASK) {
		case MEMINFO_VLGRP:
		case MEMINFO_VPAGESIZE:
			break;
		default:
			if (secpolicy_meminfo(CRED()) != 0) {
				kmem_free(req_array, req_size);
				kmem_free(in_array, in_size);
				return (set_errno(EPERM));
			}
			break;
		}
	}

	/*
	 * allocate buffer out_array which holds the results and will have
	 * to be copied out later
	 */
	out_size = sizeof (uint64_t) * addr_count * info_count;
	out_array = kmem_alloc(out_size, KM_SLEEP);

	/*
	 * allocate buffer val_array which holds the validity bits and will
	 * have to be copied out later
	 */
	val_size = sizeof (uint_t) * addr_count;
	val_array = kmem_alloc(val_size, KM_SLEEP);

	if ((req_array[0] & MEMINFO_MASK) == MEMINFO_PLGRP) {
		/* find the corresponding lgroup for each physical address */
		for (i = 0; i < addr_count; i++) {
			paddr = in_array[i];
			pfn = btop(paddr);
			lgrp = lgrp_pfn_to_lgrp(pfn);
			if (lgrp) {
				out_array[i] = lgrp->lgrp_id;
				val_array[i] = VALID_ADDR | VALID_REQ;
			} else {
				out_array[i] = 0;
				val_array[i] = 0;
			}
		}
	} else {
		/* get the corresponding memory info for each virtual address */
		as = curproc->p_as;

		AS_LOCK_ENTER(as, RW_READER);
		hat = as->a_hat;
		for (i = out_idx = 0; i < addr_count; i++, out_idx +=
		    info_count) {
			addr = in_array[i];
			vaddr = (uintptr_t)(addr & ~PAGEOFFSET);
			if (!as_segat(as, (caddr_t)vaddr)) {
				val_array[i] = 0;
				continue;
			}
			val_array[i] = VALID_ADDR;
			pfn = hat_getpfnum(hat, (caddr_t)vaddr);
			if (pfn != PFN_INVALID) {
				paddr = (uint64_t)((pfn << PAGESHIFT) |
				    (addr & PAGEOFFSET));
				for (j = 0; j < info_count; j++) {
					switch (req_array[j] & MEMINFO_MASK) {
					case MEMINFO_VPHYSICAL:
						/*
						 * return the physical address
						 * corresponding to the input
						 * virtual address
						 */
						out_array[out_idx + j] = paddr;
						val_array[i] |= VALID_REQ << j;
						break;
					case MEMINFO_VLGRP:
						/*
						 * return the lgroup of physical
						 * page corresponding to the
						 * input virtual address
						 */
						lgrp = lgrp_pfn_to_lgrp(pfn);
						if (lgrp) {
							out_array[out_idx + j] =
							    lgrp->lgrp_id;
							val_array[i] |=
							    VALID_REQ << j;
						}
						break;
					case MEMINFO_VPAGESIZE:
						/*
						 * return the size of physical
						 * page corresponding to the
						 * input virtual address
						 */
						pgsz = hat_getpagesize(hat,
						    (caddr_t)vaddr);
						if (pgsz != -1) {
							out_array[out_idx + j] =
							    pgsz;
							val_array[i] |=
							    VALID_REQ << j;
						}
						break;
					case MEMINFO_VREPLCNT:
						/*
						 * for future use:-
						 * return the no. replicated
						 * physical pages corresponding
						 * to the input virtual address,
						 * so it is always 0 at the
						 * moment
						 */
						out_array[out_idx + j] = 0;
						val_array[i] |= VALID_REQ << j;
						break;
					case MEMINFO_VREPL:
						/*
						 * for future use:-
						 * return the nth physical
						 * replica of the specified
						 * virtual address
						 */
						break;
					case MEMINFO_VREPL_LGRP:
						/*
						 * for future use:-
						 * return the lgroup of nth
						 * physical replica of the
						 * specified virtual address
						 */
						break;
					case MEMINFO_PLGRP:
						/*
						 * this is for physical address
						 * only, shouldn't mix with
						 * virtual address
						 */
						break;
					default:
						break;
					}
				}
			}
		}
		AS_LOCK_EXIT(as);
	}

	/* copy out the results and validity bits and free the buffers */
	if ((copyout(out_array, minfo.mi_outdata, out_size) != 0) ||
	    (copyout(val_array, minfo.mi_validity, val_size) != 0))
		ret = set_errno(EFAULT);

	kmem_free(in_array, in_size);
	kmem_free(out_array, out_size);
	kmem_free(req_array, req_size);
	kmem_free(val_array, val_size);

	return (ret);
}


/*
 * Initialize lgroup affinities for thread
 */
void
lgrp_affinity_init(lgrp_affinity_t **bufaddr)
{
	if (bufaddr)
		*bufaddr = NULL;
}


/*
 * Free lgroup affinities for thread and set to NULL
 * just in case thread gets recycled
 */
void
lgrp_affinity_free(lgrp_affinity_t **bufaddr)
{
	if (bufaddr && *bufaddr) {
		kmem_free(*bufaddr, nlgrpsmax * sizeof (lgrp_affinity_t));
		*bufaddr = NULL;
	}
}


#define	P_ANY	-2	/* cookie specifying any ID */


/*
 * Find LWP with given ID in specified process and get its affinity for
 * specified lgroup
 */
lgrp_affinity_t
lgrp_affinity_get_thread(proc_t *p, id_t lwpid, lgrp_id_t lgrp)
{
	lgrp_affinity_t aff;
	int		found;
	kthread_t	*t;

	ASSERT(MUTEX_HELD(&p->p_lock));

	aff = LGRP_AFF_NONE;
	found = 0;
	t = p->p_tlist;
	/*
	 * The process may be executing in proc_exit() and its p->p_list may be
	 * already NULL.
	 */
	if (t == NULL)
		return (set_errno(ESRCH));

	do {
		if (t->t_tid == lwpid || lwpid == P_ANY) {
			thread_lock(t);
			/*
			 * Check to see whether caller has permission to set
			 * affinity for LWP
			 */
			if (t->t_cid == 0 || !hasprocperm(t->t_cred, CRED())) {
				thread_unlock(t);
				return (set_errno(EPERM));
			}

			if (t->t_lgrp_affinity)
				aff = t->t_lgrp_affinity[lgrp];
			thread_unlock(t);
			found = 1;
			break;
		}
	} while ((t = t->t_forw) != p->p_tlist);
	if (!found)
		aff = set_errno(ESRCH);

	return (aff);
}


/*
 * Get lgroup affinity for given LWP
 */
lgrp_affinity_t
lgrp_affinity_get(lgrp_affinity_args_t *ap)
{
	lgrp_affinity_t		aff;
	lgrp_affinity_args_t	args;
	id_t			id;
	idtype_t		idtype;
	lgrp_id_t		lgrp;
	proc_t			*p;
	kthread_t		*t;

	/*
	 * Copyin arguments
	 */
	if (copyin(ap, &args, sizeof (lgrp_affinity_args_t)) != 0)
		return (set_errno(EFAULT));

	id = args.id;
	idtype = args.idtype;
	lgrp = args.lgrp;

	/*
	 * Check for invalid lgroup
	 */
	if (lgrp < 0 || lgrp == LGRP_NONE)
		return (set_errno(EINVAL));

	/*
	 * Check for existing lgroup
	 */
	if (lgrp > lgrp_alloc_max)
		return (set_errno(ESRCH));

	/*
	 * Get lgroup affinity for given LWP or process
	 */
	switch (idtype) {

	case P_LWPID:
		/*
		 * LWP in current process
		 */
		p = curproc;
		mutex_enter(&p->p_lock);
		if (id != P_MYID)	/* different thread */
			aff = lgrp_affinity_get_thread(p, id, lgrp);
		else {			/* current thread */
			aff = LGRP_AFF_NONE;
			t = curthread;
			thread_lock(t);
			if (t->t_lgrp_affinity)
				aff = t->t_lgrp_affinity[lgrp];
			thread_unlock(t);
		}
		mutex_exit(&p->p_lock);
		break;

	case P_PID:
		/*
		 * Process
		 */
		mutex_enter(&pidlock);

		if (id == P_MYID)
			p = curproc;
		else {
			p = prfind(id);
			if (p == NULL) {
				mutex_exit(&pidlock);
				return (set_errno(ESRCH));
			}
		}

		mutex_enter(&p->p_lock);
		aff = lgrp_affinity_get_thread(p, P_ANY, lgrp);
		mutex_exit(&p->p_lock);

		mutex_exit(&pidlock);
		break;

	default:
		aff = set_errno(EINVAL);
		break;
	}

	return (aff);
}


/*
 * Find lgroup for which this thread has most affinity in specified partition
 * starting from home lgroup unless specified starting lgroup is preferred
 */
lpl_t *
lgrp_affinity_best(kthread_t *t, struct cpupart *cpupart, lgrp_id_t start,
    boolean_t prefer_start)
{
	lgrp_affinity_t	*affs;
	lgrp_affinity_t	best_aff;
	lpl_t		*best_lpl;
	lgrp_id_t	finish;
	lgrp_id_t	home;
	lgrp_id_t	lgrpid;
	lpl_t		*lpl;

	ASSERT(t != NULL);
	ASSERT((MUTEX_HELD(&cpu_lock) || curthread->t_preempt > 0) ||
	    (MUTEX_HELD(&ttoproc(t)->p_lock) && THREAD_LOCK_HELD(t)));
	ASSERT(cpupart != NULL);

	if (t->t_lgrp_affinity == NULL)
		return (NULL);

	affs = t->t_lgrp_affinity;

	/*
	 * Thread bound to CPU
	 */
	if (t->t_bind_cpu != PBIND_NONE) {
		cpu_t	*cp;

		/*
		 * Find which lpl has most affinity among leaf lpl directly
		 * containing CPU and its ancestor lpls
		 */
		cp = cpu[t->t_bind_cpu];

		best_lpl = lpl = cp->cpu_lpl;
		best_aff = affs[best_lpl->lpl_lgrpid];
		while (lpl->lpl_parent != NULL) {
			lpl = lpl->lpl_parent;
			lgrpid = lpl->lpl_lgrpid;
			if (affs[lgrpid] > best_aff) {
				best_lpl = lpl;
				best_aff = affs[lgrpid];
			}
		}
		return (best_lpl);
	}

	/*
	 * Start searching from home lgroup unless given starting lgroup is
	 * preferred or home lgroup isn't in given pset.  Use root lgroup as
	 * starting point if both home and starting lgroups aren't in given
	 * pset.
	 */
	ASSERT(start >= 0 && start <= lgrp_alloc_max);
	home = t->t_lpl->lpl_lgrpid;
	if (!prefer_start && LGRP_CPUS_IN_PART(home, cpupart))
		lgrpid = home;
	else if (start != LGRP_NONE && LGRP_CPUS_IN_PART(start, cpupart))
		lgrpid = start;
	else
		lgrpid = LGRP_ROOTID;

	best_lpl = &cpupart->cp_lgrploads[lgrpid];
	best_aff = affs[lgrpid];
	finish = lgrpid;
	do {
		/*
		 * Skip any lgroups that don't have CPU resources
		 * in this processor set.
		 */
		if (!LGRP_CPUS_IN_PART(lgrpid, cpupart)) {
			if (++lgrpid > lgrp_alloc_max)
				lgrpid = 0;	/* wrap the search */
			continue;
		}

		/*
		 * Find lgroup with most affinity
		 */
		lpl = &cpupart->cp_lgrploads[lgrpid];
		if (affs[lgrpid] > best_aff) {
			best_aff = affs[lgrpid];
			best_lpl = lpl;
		}

		if (++lgrpid > lgrp_alloc_max)
			lgrpid = 0;	/* wrap the search */

	} while (lgrpid != finish);

	/*
	 * No lgroup (in this pset) with any affinity
	 */
	if (best_aff == LGRP_AFF_NONE)
		return (NULL);

	lgrpid = best_lpl->lpl_lgrpid;
	ASSERT(LGRP_CPUS_IN_PART(lgrpid, cpupart) && best_lpl->lpl_ncpu > 0);

	return (best_lpl);
}


/*
 * Set thread's affinity for given lgroup
 */
int
lgrp_affinity_set_thread(kthread_t *t, lgrp_id_t lgrp, lgrp_affinity_t aff,
    lgrp_affinity_t **aff_buf)
{
	lgrp_affinity_t	*affs;
	lgrp_id_t	best;
	lpl_t		*best_lpl;
	lgrp_id_t	home;
	int		retval;

	ASSERT(t != NULL);
	ASSERT(MUTEX_HELD(&ttoproc(t)->p_lock));

	retval = 0;

	thread_lock(t);

	/*
	 * Check to see whether caller has permission to set affinity for
	 * thread
	 */
	if (t->t_cid == 0 || !hasprocperm(t->t_cred, CRED())) {
		thread_unlock(t);
		return (set_errno(EPERM));
	}

	if (t->t_lgrp_affinity == NULL) {
		if (aff == LGRP_AFF_NONE) {
			thread_unlock(t);
			return (0);
		}
		ASSERT(aff_buf != NULL && *aff_buf != NULL);
		t->t_lgrp_affinity = *aff_buf;
		*aff_buf = NULL;
	}

	affs = t->t_lgrp_affinity;
	affs[lgrp] = aff;

	/*
	 * Find lgroup for which thread has most affinity,
	 * starting with lgroup for which affinity being set
	 */
	best_lpl = lgrp_affinity_best(t, t->t_cpupart, lgrp, B_TRUE);

	/*
	 * Rehome if found lgroup with more affinity than home or lgroup for
	 * which affinity is being set has same affinity as home
	 */
	home = t->t_lpl->lpl_lgrpid;
	if (best_lpl != NULL && best_lpl != t->t_lpl) {
		best = best_lpl->lpl_lgrpid;
		if (affs[best] > affs[home] || (affs[best] == affs[home] &&
		    best == lgrp))
			lgrp_move_thread(t, best_lpl, 1);
	}

	thread_unlock(t);

	return (retval);
}


/*
 * Set process' affinity for specified lgroup
 */
int
lgrp_affinity_set_proc(proc_t *p, lgrp_id_t lgrp, lgrp_affinity_t aff,
    lgrp_affinity_t **aff_buf_array)
{
	lgrp_affinity_t	*buf;
	int		err = 0;
	int		i;
	int		retval;
	kthread_t	*t;

	ASSERT(MUTEX_HELD(&pidlock) && MUTEX_HELD(&p->p_lock));
	ASSERT(aff_buf_array != NULL);

	i = 0;
	t = p->p_tlist;
	if (t != NULL) {
		do {
			/*
			 * Set lgroup affinity for thread
			 */
			buf = aff_buf_array[i];
			retval = lgrp_affinity_set_thread(t, lgrp, aff, &buf);

			if (err == 0 && retval != 0)
				err = retval;

			/*
			 * Advance pointer to next buffer
			 */
			if (buf == NULL) {
				ASSERT(i < p->p_lwpcnt);
				aff_buf_array[i] = NULL;
				i++;
			}

		} while ((t = t->t_forw) != p->p_tlist);
	}
	return (err);
}


/*
 * Set LWP's or process' affinity for specified lgroup
 *
 * When setting affinities, pidlock, process p_lock, and thread_lock()
 * need to be held in that order to protect target thread's pset, process,
 * process contents, and thread contents.  thread_lock() does splhigh(),
 * so it ends up having similiar effect as kpreempt_disable(), so it will
 * protect calls to lgrp_move_thread() and lgrp_choose() from pset changes.
 */
int
lgrp_affinity_set(lgrp_affinity_args_t *ap)
{
	lgrp_affinity_t		aff;
	lgrp_affinity_t		*aff_buf;
	lgrp_affinity_args_t	args;
	id_t			id;
	idtype_t		idtype;
	lgrp_id_t		lgrp;
	int			nthreads;
	proc_t			*p;
	int			retval;

	/*
	 * Copyin arguments
	 */
	if (copyin(ap, &args, sizeof (lgrp_affinity_args_t)) != 0)
		return (set_errno(EFAULT));

	idtype = args.idtype;
	id = args.id;
	lgrp = args.lgrp;
	aff = args.aff;

	/*
	 * Check for invalid lgroup
	 */
	if (lgrp < 0 || lgrp == LGRP_NONE)
		return (set_errno(EINVAL));

	/*
	 * Check for existing lgroup
	 */
	if (lgrp > lgrp_alloc_max)
		return (set_errno(ESRCH));

	/*
	 * Check for legal affinity
	 */
	if (aff != LGRP_AFF_NONE && aff != LGRP_AFF_WEAK &&
	    aff != LGRP_AFF_STRONG)
		return (set_errno(EINVAL));

	/*
	 * Must be process or LWP ID
	 */
	if (idtype != P_LWPID && idtype != P_PID)
		return (set_errno(EINVAL));

	retval = EINVAL;
	/*
	 * Set given LWP's or process' affinity for specified lgroup
	 */
	switch (idtype) {

	case P_LWPID:
		/*
		 * Allocate memory for thread's lgroup affinities
		 * ahead of time w/o holding locks
		 */
		aff_buf = kmem_zalloc(nlgrpsmax * sizeof (lgrp_affinity_t),
		    KM_SLEEP);

		p = curproc;

		/*
		 * Set affinity for thread
		 */
		mutex_enter(&p->p_lock);
		if (id == P_MYID) {		/* current thread */
			retval = lgrp_affinity_set_thread(curthread, lgrp, aff,
			    &aff_buf);
		} else if (p->p_tlist == NULL) {
			retval = set_errno(ESRCH);
		} else {			/* other thread */
			int		found = 0;
			kthread_t	*t;

			t = p->p_tlist;
			do {
				if (t->t_tid == id) {
					retval = lgrp_affinity_set_thread(t,
					    lgrp, aff, &aff_buf);
					found = 1;
					break;
				}
			} while ((t = t->t_forw) != p->p_tlist);
			if (!found)
				retval = set_errno(ESRCH);
		}
		mutex_exit(&p->p_lock);

		/*
		 * Free memory for lgroup affinities,
		 * since thread didn't need it
		 */
		if (aff_buf)
			kmem_free(aff_buf,
			    nlgrpsmax * sizeof (lgrp_affinity_t));

		break;

	case P_PID:

		do {
			lgrp_affinity_t	**aff_buf_array;
			int		i;
			size_t		size;

			/*
			 * Get process
			 */
			mutex_enter(&pidlock);

			if (id == P_MYID)
				p = curproc;
			else
				p = prfind(id);

			if (p == NULL) {
				mutex_exit(&pidlock);
				return (set_errno(ESRCH));
			}

			/*
			 * Get number of threads in process
			 *
			 * NOTE: Only care about user processes,
			 *	 so p_lwpcnt should be number of threads.
			 */
			mutex_enter(&p->p_lock);
			nthreads = p->p_lwpcnt;
			mutex_exit(&p->p_lock);

			mutex_exit(&pidlock);

			if (nthreads < 1)
				return (set_errno(ESRCH));

			/*
			 * Preallocate memory for lgroup affinities for
			 * each thread in process now to avoid holding
			 * any locks.  Allocate an array to hold a buffer
			 * for each thread.
			 */
			aff_buf_array = kmem_zalloc(nthreads *
			    sizeof (lgrp_affinity_t *), KM_SLEEP);

			size = nlgrpsmax * sizeof (lgrp_affinity_t);
			for (i = 0; i < nthreads; i++)
				aff_buf_array[i] = kmem_zalloc(size, KM_SLEEP);

			mutex_enter(&pidlock);

			/*
			 * Get process again since dropped locks to allocate
			 * memory (except current process)
			 */
			if (id != P_MYID)
				p = prfind(id);

			/*
			 * Process went away after we dropped locks and before
			 * reacquiring them, so drop locks, free memory, and
			 * return.
			 */
			if (p == NULL) {
				mutex_exit(&pidlock);
				for (i = 0; i < nthreads; i++)
					kmem_free(aff_buf_array[i], size);
				kmem_free(aff_buf_array,
				    nthreads * sizeof (lgrp_affinity_t *));
				return (set_errno(ESRCH));
			}

			mutex_enter(&p->p_lock);

			/*
			 * See whether number of threads is same
			 * If not, drop locks, free memory, and try again
			 */
			if (nthreads != p->p_lwpcnt) {
				mutex_exit(&p->p_lock);
				mutex_exit(&pidlock);
				for (i = 0; i < nthreads; i++)
					kmem_free(aff_buf_array[i], size);
				kmem_free(aff_buf_array,
				    nthreads * sizeof (lgrp_affinity_t *));
				continue;
			}

			/*
			 * Set lgroup affinity for threads in process
			 */
			retval = lgrp_affinity_set_proc(p, lgrp, aff,
			    aff_buf_array);

			mutex_exit(&p->p_lock);
			mutex_exit(&pidlock);

			/*
			 * Free any leftover memory, since some threads may
			 * have already allocated memory and set lgroup
			 * affinities before
			 */
			for (i = 0; i < nthreads; i++)
				if (aff_buf_array[i] != NULL)
					kmem_free(aff_buf_array[i], size);
			kmem_free(aff_buf_array,
			    nthreads * sizeof (lgrp_affinity_t *));

			break;

		} while (nthreads != p->p_lwpcnt);

		break;

	default:
		retval = set_errno(EINVAL);
		break;
	}

	return (retval);
}


/*
 * Return the latest generation number for the lgroup hierarchy
 * with the given view
 */
lgrp_gen_t
lgrp_generation(lgrp_view_t view)
{
	cpupart_t	*cpupart;
	uint_t		gen;

	kpreempt_disable();

	/*
	 * Determine generation number for given view
	 */
	if (view == LGRP_VIEW_OS)
		/*
		 * Return generation number of lgroup hierarchy for OS view
		 */
		gen = lgrp_gen;
	else {
		/*
		 * For caller's view, use generation numbers for lgroup
		 * hierarchy and caller's pset
		 * NOTE: Caller needs to check for change in pset ID
		 */
		cpupart = curthread->t_cpupart;
		ASSERT(cpupart);
		gen = lgrp_gen + cpupart->cp_gen;
	}

	kpreempt_enable();

	return (gen);
}


lgrp_id_t
lgrp_home_thread(kthread_t *t)
{
	lgrp_id_t	home;

	ASSERT(t != NULL);
	ASSERT(MUTEX_HELD(&ttoproc(t)->p_lock));

	thread_lock(t);

	/*
	 * Check to see whether caller has permission to set affinity for
	 * thread
	 */
	if (t->t_cid == 0 || !hasprocperm(t->t_cred, CRED())) {
		thread_unlock(t);
		return (set_errno(EPERM));
	}

	home = lgrp_home_id(t);

	thread_unlock(t);
	return (home);
}


/*
 * Get home lgroup of given process or thread
 */
lgrp_id_t
lgrp_home_get(idtype_t idtype, id_t id)
{
	proc_t		*p;
	lgrp_id_t	retval;
	kthread_t	*t;

	/*
	 * Get home lgroup of given LWP or process
	 */
	switch (idtype) {

	case P_LWPID:
		p = curproc;

		/*
		 * Set affinity for thread
		 */
		mutex_enter(&p->p_lock);
		if (id == P_MYID) {		/* current thread */
			retval = lgrp_home_thread(curthread);
		} else if (p->p_tlist == NULL) {
			retval = set_errno(ESRCH);
		} else {			/* other thread */
			int	found = 0;

			t = p->p_tlist;
			do {
				if (t->t_tid == id) {
					retval = lgrp_home_thread(t);
					found = 1;
					break;
				}
			} while ((t = t->t_forw) != p->p_tlist);
			if (!found)
				retval = set_errno(ESRCH);
		}
		mutex_exit(&p->p_lock);
		break;

	case P_PID:
		/*
		 * Get process
		 */
		mutex_enter(&pidlock);

		if (id == P_MYID)
			p = curproc;
		else
			p = prfind(id);

		if (p == NULL) {
			mutex_exit(&pidlock);
			return (set_errno(ESRCH));
		}

		mutex_enter(&p->p_lock);
		t = p->p_tlist;
		if (t == NULL)
			retval = set_errno(ESRCH);
		else
			retval = lgrp_home_thread(t);
		mutex_exit(&p->p_lock);

		mutex_exit(&pidlock);

		break;

	default:
		retval = set_errno(EINVAL);
		break;
	}

	return (retval);
}


/*
 * Return latency between "from" and "to" lgroups
 *
 * This latency number can only be used for relative comparison
 * between lgroups on the running system, cannot be used across platforms,
 * and may not reflect the actual latency.  It is platform and implementation
 * specific, so platform gets to decide its value.  It would be nice if the
 * number was at least proportional to make comparisons more meaningful though.
 */
int
lgrp_latency(lgrp_id_t from, lgrp_id_t to)
{
	lgrp_t		*from_lgrp;
	int		i;
	int		latency;
	int		latency_max;
	lgrp_t		*to_lgrp;

	ASSERT(MUTEX_HELD(&cpu_lock));

	if (from < 0 || to < 0)
		return (set_errno(EINVAL));

	if (from > lgrp_alloc_max || to > lgrp_alloc_max)
		return (set_errno(ESRCH));

	from_lgrp = lgrp_table[from];
	to_lgrp = lgrp_table[to];

	if (!LGRP_EXISTS(from_lgrp) || !LGRP_EXISTS(to_lgrp)) {
		return (set_errno(ESRCH));
	}

	/*
	 * Get latency for same lgroup
	 */
	if (from == to) {
		latency = from_lgrp->lgrp_latency;
		return (latency);
	}

	/*
	 * Get latency between leaf lgroups
	 */
	if (from_lgrp->lgrp_childcnt == 0 && to_lgrp->lgrp_childcnt == 0)
		return (lgrp_plat_latency(from_lgrp->lgrp_plathand,
		    to_lgrp->lgrp_plathand));

	/*
	 * Determine max latency between resources in two lgroups
	 */
	latency_max = 0;
	for (i = 0; i <= lgrp_alloc_max; i++) {
		lgrp_t	*from_rsrc;
		int	j;
		lgrp_t	*to_rsrc;

		from_rsrc = lgrp_table[i];
		if (!LGRP_EXISTS(from_rsrc) ||
		    !klgrpset_ismember(from_lgrp->lgrp_set[LGRP_RSRC_CPU], i))
			continue;

		for (j = 0; j <= lgrp_alloc_max; j++) {
			to_rsrc = lgrp_table[j];
			if (!LGRP_EXISTS(to_rsrc) ||
			    klgrpset_ismember(to_lgrp->lgrp_set[LGRP_RSRC_MEM],
			    j) == 0)
				continue;
			latency = lgrp_plat_latency(from_rsrc->lgrp_plathand,
			    to_rsrc->lgrp_plathand);
			if (latency > latency_max)
				latency_max = latency;
		}
	}
	return (latency_max);
}


/*
 * Return lgroup interface version number
 * 0 - none
 * 1 - original
 * 2 - lgrp_latency_cookie() and lgrp_resources() added
 */
int
lgrp_version(int version)
{
	/*
	 * Return LGRP_VER_NONE when requested version isn't supported
	 */
	if (version < LGRP_VER_NONE || version > LGRP_VER_CURRENT)
		return (LGRP_VER_NONE);

	/*
	 * Return current version when LGRP_VER_NONE passed in
	 */
	if (version == LGRP_VER_NONE)
		return (LGRP_VER_CURRENT);

	/*
	 * Otherwise, return supported version.
	 */
	return (version);
}


/*
 * Snapshot of lgroup hieararchy
 *
 * One snapshot is kept and is based on the kernel's native data model, so
 * a 32-bit snapshot is kept for the 32-bit kernel and a 64-bit one for the
 * 64-bit kernel.  If a 32-bit user wants a snapshot from the 64-bit kernel,
 * the kernel generates a 32-bit snapshot from the data in its 64-bit snapshot.
 *
 * The format is defined by lgroup snapshot header and the layout of
 * the snapshot in memory is as follows:
 * 1) lgroup snapshot header
 *    - specifies format of snapshot
 *    - defined by lgrp_snapshot_header_t
 * 2) lgroup info array
 *    - contains information about each lgroup
 *    - one element for each lgroup
 *    - each element is defined by lgrp_info_t
 * 3) lgroup CPU ID array
 *    - contains list (array) of CPU IDs for each lgroup
 *    - lgrp_info_t points into array and specifies how many CPUs belong to
 *      given lgroup
 * 4) lgroup parents array
 *    - contains lgroup bitmask of parents for each lgroup
 *    - bitmask is an array of unsigned longs and its size depends on nlgrpsmax
 * 5) lgroup children array
 *    - contains lgroup bitmask of children for each lgroup
 *    - bitmask is an array of unsigned longs and its size depends on nlgrpsmax
 * 6) lgroup resources array
 *    - contains lgroup bitmask of resources for each lgroup
 *    - bitmask is an array of unsigned longs and its size depends on nlgrpsmax
 * 7) lgroup latency table
 *    - contains latency from each lgroup to each of other lgroups
 *
 * NOTE:  Must use nlgrpsmax for per lgroup data structures because lgroups
 *	  may be sparsely allocated.
 */
lgrp_snapshot_header_t	*lgrp_snap = NULL;	/* lgroup snapshot */
static kmutex_t		lgrp_snap_lock;		/* snapshot lock */


/*
 * Take a snapshot of lgroup hierarchy and return size of buffer
 * needed to hold snapshot
 */
static int
lgrp_snapshot(void)
{
	size_t		bitmask_size;
	size_t		bitmasks_size;
	size_t		bufsize;
	int		cpu_index;
	size_t		cpuids_size;
	int		i;
	int		j;
	size_t		info_size;
	size_t		lats_size;
	ulong_t		*lgrp_children;
	processorid_t	*lgrp_cpuids;
	lgrp_info_t	*lgrp_info;
	int		**lgrp_lats;
	ulong_t		*lgrp_parents;
	ulong_t		*lgrp_rsets;
	ulong_t		*lgrpset;
	int		snap_ncpus;
	int		snap_nlgrps;
	int		snap_nlgrpsmax;
	size_t		snap_hdr_size;
#ifdef	_SYSCALL32_IMPL
	model_t		model = DATAMODEL_NATIVE;

	/*
	 * Have up-to-date snapshot, so check to see whether caller is 32-bit
	 * program and need to return size of 32-bit snapshot now.
	 */
	model = get_udatamodel();
	if (model == DATAMODEL_ILP32 && lgrp_snap &&
	    lgrp_snap->ss_gen == lgrp_gen) {

		snap_nlgrpsmax = lgrp_snap->ss_nlgrps_max;

		/*
		 * Calculate size of buffer needed for 32-bit snapshot,
		 * rounding up size of each object to allow for alignment
		 * of next object in buffer.
		 */
		snap_hdr_size = P2ROUNDUP(sizeof (lgrp_snapshot_header32_t),
		    sizeof (caddr32_t));
		info_size =
		    P2ROUNDUP(snap_nlgrpsmax * sizeof (lgrp_info32_t),
		    sizeof (processorid_t));
		cpuids_size =
		    P2ROUNDUP(lgrp_snap->ss_ncpus * sizeof (processorid_t),
		    sizeof (ulong_t));

		/*
		 * lgroup bitmasks needed for parents, children, and resources
		 * for each lgroup and pset lgroup set
		 */
		bitmask_size = BT_SIZEOFMAP(snap_nlgrpsmax);
		bitmasks_size = (((2 + LGRP_RSRC_COUNT) *
		    snap_nlgrpsmax) + 1) * bitmask_size;

		/*
		 * Size of latency table and buffer
		 */
		lats_size = snap_nlgrpsmax * sizeof (caddr32_t) +
		    snap_nlgrpsmax * snap_nlgrpsmax * sizeof (int);

		bufsize = snap_hdr_size + info_size + cpuids_size +
		    bitmasks_size + lats_size;
		return (bufsize);
	}
#endif	/* _SYSCALL32_IMPL */

	/*
	 * Check whether snapshot is up-to-date
	 * Free it and take another one if not
	 */
	if (lgrp_snap) {
		if (lgrp_snap->ss_gen == lgrp_gen)
			return (lgrp_snap->ss_size);

		kmem_free(lgrp_snap, lgrp_snap->ss_size);
		lgrp_snap = NULL;
	}

	/*
	 * Allocate memory for snapshot
	 * w/o holding cpu_lock while waiting for memory
	 */
	while (lgrp_snap == NULL) {
		int	old_generation;

		/*
		 * Take snapshot of lgroup generation number
		 * and configuration size dependent information
		 * NOTE: Only count number of online CPUs,
		 * since only online CPUs appear in lgroups.
		 */
		mutex_enter(&cpu_lock);
		old_generation = lgrp_gen;
		snap_ncpus = ncpus_online;
		snap_nlgrps = nlgrps;
		snap_nlgrpsmax = nlgrpsmax;
		mutex_exit(&cpu_lock);

		/*
		 * Calculate size of buffer needed for snapshot,
		 * rounding up size of each object to allow for alignment
		 * of next object in buffer.
		 */
		snap_hdr_size = P2ROUNDUP(sizeof (lgrp_snapshot_header_t),
		    sizeof (void *));
		info_size = P2ROUNDUP(snap_nlgrpsmax * sizeof (lgrp_info_t),
		    sizeof (processorid_t));
		cpuids_size = P2ROUNDUP(snap_ncpus * sizeof (processorid_t),
		    sizeof (ulong_t));
		/*
		 * lgroup bitmasks needed for pset lgroup set and  parents,
		 * children, and resource sets for each lgroup
		 */
		bitmask_size = BT_SIZEOFMAP(snap_nlgrpsmax);
		bitmasks_size = (((2 + LGRP_RSRC_COUNT) *
		    snap_nlgrpsmax) + 1) * bitmask_size;

		/*
		 * Size of latency table and buffer
		 */
		lats_size = snap_nlgrpsmax * sizeof (int *) +
		    snap_nlgrpsmax * snap_nlgrpsmax * sizeof (int);

		bufsize = snap_hdr_size + info_size + cpuids_size +
		    bitmasks_size + lats_size;

		/*
		 * Allocate memory for buffer
		 */
		lgrp_snap = kmem_zalloc(bufsize, KM_NOSLEEP);
		if (lgrp_snap == NULL)
			return (set_errno(ENOMEM));

		/*
		 * Check whether generation number has changed
		 */
		mutex_enter(&cpu_lock);
		if (lgrp_gen == old_generation)
			break;		/* hasn't change, so done. */

		/*
		 * Generation number changed, so free memory and try again.
		 */
		mutex_exit(&cpu_lock);
		kmem_free(lgrp_snap, bufsize);
		lgrp_snap = NULL;
	}

	/*
	 * Fill in lgroup snapshot header
	 * (including pointers to tables of lgroup info, CPU IDs, and parents
	 * and children)
	 */
	lgrp_snap->ss_version = LGRP_VER_CURRENT;

	/*
	 * XXX For now, liblgrp only needs to know whether the hierarchy
	 * XXX only has one level or not
	 */
	if (snap_nlgrps == 1)
		lgrp_snap->ss_levels = 1;
	else
		lgrp_snap->ss_levels = 2;

	lgrp_snap->ss_root = LGRP_ROOTID;

	lgrp_snap->ss_nlgrps = lgrp_snap->ss_nlgrps_os = snap_nlgrps;
	lgrp_snap->ss_nlgrps_max = snap_nlgrpsmax;
	lgrp_snap->ss_ncpus = snap_ncpus;
	lgrp_snap->ss_gen = lgrp_gen;
	lgrp_snap->ss_view = LGRP_VIEW_OS;
	lgrp_snap->ss_pset = 0;		/* NOTE: caller should set if needed */
	lgrp_snap->ss_size = bufsize;
	lgrp_snap->ss_magic = (uintptr_t)lgrp_snap;

	lgrp_snap->ss_info = lgrp_info =
	    (lgrp_info_t *)((uintptr_t)lgrp_snap + snap_hdr_size);

	lgrp_snap->ss_cpuids = lgrp_cpuids =
	    (processorid_t *)((uintptr_t)lgrp_info + info_size);

	lgrp_snap->ss_lgrpset = lgrpset =
	    (ulong_t *)((uintptr_t)lgrp_cpuids + cpuids_size);

	lgrp_snap->ss_parents = lgrp_parents =
	    (ulong_t *)((uintptr_t)lgrpset + bitmask_size);

	lgrp_snap->ss_children = lgrp_children =
	    (ulong_t *)((uintptr_t)lgrp_parents + (snap_nlgrpsmax *
	    bitmask_size));

	lgrp_snap->ss_rsets = lgrp_rsets =
	    (ulong_t *)((uintptr_t)lgrp_children + (snap_nlgrpsmax *
	    bitmask_size));

	lgrp_snap->ss_latencies = lgrp_lats =
	    (int **)((uintptr_t)lgrp_rsets + (LGRP_RSRC_COUNT *
	    snap_nlgrpsmax * bitmask_size));

	/*
	 * Fill in lgroup information
	 */
	cpu_index = 0;
	for (i = 0; i < snap_nlgrpsmax; i++) {
		struct cpu	*cp;
		int		cpu_count;
		struct cpu	*head;
		int		k;
		lgrp_t		*lgrp;

		lgrp = lgrp_table[i];
		if (!LGRP_EXISTS(lgrp)) {
			bzero(&lgrp_info[i], sizeof (lgrp_info[i]));
			lgrp_info[i].info_lgrpid = LGRP_NONE;
			continue;
		}

		lgrp_info[i].info_lgrpid = i;
		lgrp_info[i].info_latency = lgrp->lgrp_latency;

		/*
		 * Fill in parents, children, and lgroup resources
		 */
		lgrp_info[i].info_parents =
		    (ulong_t *)((uintptr_t)lgrp_parents + (i * bitmask_size));

		if (lgrp->lgrp_parent)
			BT_SET(lgrp_info[i].info_parents,
			    lgrp->lgrp_parent->lgrp_id);

		lgrp_info[i].info_children =
		    (ulong_t *)((uintptr_t)lgrp_children + (i * bitmask_size));

		for (j = 0; j < snap_nlgrpsmax; j++)
			if (klgrpset_ismember(lgrp->lgrp_children, j))
				BT_SET(lgrp_info[i].info_children, j);

		lgrp_info[i].info_rset =
		    (ulong_t *)((uintptr_t)lgrp_rsets +
		    (i * LGRP_RSRC_COUNT * bitmask_size));

		for (j = 0; j < LGRP_RSRC_COUNT; j++) {
			ulong_t	*rset;

			rset = (ulong_t *)((uintptr_t)lgrp_info[i].info_rset +
			    (j * bitmask_size));
			for (k = 0; k < snap_nlgrpsmax; k++)
				if (klgrpset_ismember(lgrp->lgrp_set[j], k))
					BT_SET(rset, k);
		}

		/*
		 * Fill in CPU IDs
		 */
		cpu_count = 0;
		lgrp_info[i].info_cpuids = NULL;
		cp = head = lgrp->lgrp_cpu;
		if (head != NULL) {
			lgrp_info[i].info_cpuids = &lgrp_cpuids[cpu_index];
			do {
				lgrp_cpuids[cpu_index] = cp->cpu_id;
				cpu_index++;
				cpu_count++;
				cp = cp->cpu_next_lgrp;
			} while (cp != head);
		}
		ASSERT(cpu_count == lgrp->lgrp_cpucnt);
		lgrp_info[i].info_ncpus = cpu_count;

		/*
		 * Fill in memory sizes for lgroups that directly contain
		 * memory
		 */
		if (klgrpset_ismember(lgrp->lgrp_set[LGRP_RSRC_MEM], i)) {
			lgrp_info[i].info_mem_free =
			    lgrp_mem_size(i, LGRP_MEM_SIZE_FREE);
			lgrp_info[i].info_mem_install =
			    lgrp_mem_size(i, LGRP_MEM_SIZE_INSTALL);
		}

		/*
		 * Fill in latency table and buffer
		 */
		lgrp_lats[i] = (int *)((uintptr_t)lgrp_lats + snap_nlgrpsmax *
		    sizeof (int *) + i * snap_nlgrpsmax * sizeof (int));
		for (j = 0; j < snap_nlgrpsmax; j++) {
			lgrp_t	*to;

			to = lgrp_table[j];
			if (!LGRP_EXISTS(to))
				continue;
			lgrp_lats[i][j] = lgrp_latency(lgrp->lgrp_id,
			    to->lgrp_id);
		}
	}
	ASSERT(cpu_index == snap_ncpus);


	mutex_exit(&cpu_lock);

#ifdef	_SYSCALL32_IMPL
	/*
	 * Check to see whether caller is 32-bit program and need to return
	 * size of 32-bit snapshot now that snapshot has been taken/updated.
	 * May not have been able to do this earlier if snapshot was out of
	 * date or didn't exist yet.
	 */
	if (model == DATAMODEL_ILP32) {

		snap_nlgrpsmax = lgrp_snap->ss_nlgrps_max;

		/*
		 * Calculate size of buffer needed for 32-bit snapshot,
		 * rounding up size of each object to allow for alignment
		 * of next object in buffer.
		 */
		snap_hdr_size = P2ROUNDUP(sizeof (lgrp_snapshot_header32_t),
		    sizeof (caddr32_t));
		info_size =
		    P2ROUNDUP(snap_nlgrpsmax * sizeof (lgrp_info32_t),
		    sizeof (processorid_t));
		cpuids_size =
		    P2ROUNDUP(lgrp_snap->ss_ncpus * sizeof (processorid_t),
		    sizeof (ulong_t));

		bitmask_size = BT_SIZEOFMAP(snap_nlgrpsmax);
		bitmasks_size = (((2 + LGRP_RSRC_COUNT) * snap_nlgrpsmax) +
		    1) * bitmask_size;


		/*
		 * Size of latency table and buffer
		 */
		lats_size = (snap_nlgrpsmax * sizeof (caddr32_t)) +
		    (snap_nlgrpsmax * snap_nlgrpsmax * sizeof (int));

		bufsize = snap_hdr_size + info_size + cpuids_size +
		    bitmasks_size + lats_size;
		return (bufsize);
	}
#endif	/* _SYSCALL32_IMPL */

	return (lgrp_snap->ss_size);
}


/*
 * Copy snapshot into given user buffer, fix up any pointers in buffer to point
 * into user instead of kernel address space, and return size of buffer
 * needed to hold snapshot
 */
static int
lgrp_snapshot_copy(char *buf, size_t bufsize)
{
	size_t			bitmask_size;
	int			cpu_index;
	size_t			cpuids_size;
	int			i;
	size_t			info_size;
	lgrp_info_t		*lgrp_info;
	int			retval;
	size_t			snap_hdr_size;
	int			snap_ncpus;
	int			snap_nlgrpsmax;
	lgrp_snapshot_header_t	*user_snap;
	lgrp_info_t		*user_info;
	lgrp_info_t		*user_info_buffer;
	processorid_t		*user_cpuids;
	ulong_t			*user_lgrpset;
	ulong_t			*user_parents;
	ulong_t			*user_children;
	int			**user_lats;
	int			**user_lats_buffer;
	ulong_t			*user_rsets;

	if (lgrp_snap == NULL)
		return (0);

	if (buf == NULL || bufsize <= 0)
		return (lgrp_snap->ss_size);

	/*
	 * User needs to try getting size of buffer again
	 * because given buffer size is too small.
	 * The lgroup hierarchy may have changed after they asked for the size
	 * but before the snapshot was taken.
	 */
	if (bufsize < lgrp_snap->ss_size)
		return (set_errno(EAGAIN));

	snap_ncpus = lgrp_snap->ss_ncpus;
	snap_nlgrpsmax = lgrp_snap->ss_nlgrps_max;

	/*
	 * Fill in lgrpset now because caller may have change psets
	 */
	kpreempt_disable();
	for (i = 0; i < snap_nlgrpsmax; i++) {
		if (klgrpset_ismember(curthread->t_cpupart->cp_lgrpset,
		    i)) {
			BT_SET(lgrp_snap->ss_lgrpset, i);
		}
	}
	kpreempt_enable();

	/*
	 * Copy lgroup snapshot (snapshot header, lgroup info, and CPU IDs)
	 * into user buffer all at once
	 */
	if (copyout(lgrp_snap, buf, lgrp_snap->ss_size) != 0)
		return (set_errno(EFAULT));

	/*
	 * Round up sizes of lgroup snapshot header and info for alignment
	 */
	snap_hdr_size = P2ROUNDUP(sizeof (lgrp_snapshot_header_t),
	    sizeof (void *));
	info_size = P2ROUNDUP(snap_nlgrpsmax * sizeof (lgrp_info_t),
	    sizeof (processorid_t));
	cpuids_size = P2ROUNDUP(snap_ncpus * sizeof (processorid_t),
	    sizeof (ulong_t));

	bitmask_size = BT_SIZEOFMAP(snap_nlgrpsmax);

	/*
	 * Calculate pointers into user buffer for lgroup snapshot header,
	 * info, and CPU IDs
	 */
	user_snap = (lgrp_snapshot_header_t *)buf;
	user_info = (lgrp_info_t *)((uintptr_t)user_snap + snap_hdr_size);
	user_cpuids = (processorid_t *)((uintptr_t)user_info + info_size);
	user_lgrpset = (ulong_t *)((uintptr_t)user_cpuids + cpuids_size);
	user_parents = (ulong_t *)((uintptr_t)user_lgrpset + bitmask_size);
	user_children = (ulong_t *)((uintptr_t)user_parents +
	    (snap_nlgrpsmax * bitmask_size));
	user_rsets = (ulong_t *)((uintptr_t)user_children +
	    (snap_nlgrpsmax * bitmask_size));
	user_lats = (int **)((uintptr_t)user_rsets +
	    (LGRP_RSRC_COUNT * snap_nlgrpsmax * bitmask_size));

	/*
	 * Copyout magic number (ie. pointer to beginning of buffer)
	 */
	if (copyout(&buf, &user_snap->ss_magic, sizeof (buf)) != 0)
		return (set_errno(EFAULT));

	/*
	 * Fix up pointers in user buffer to point into user buffer
	 * not kernel snapshot
	 */
	if (copyout(&user_info, &user_snap->ss_info, sizeof (user_info)) != 0)
		return (set_errno(EFAULT));

	if (copyout(&user_cpuids, &user_snap->ss_cpuids,
	    sizeof (user_cpuids)) != 0)
		return (set_errno(EFAULT));

	if (copyout(&user_lgrpset, &user_snap->ss_lgrpset,
	    sizeof (user_lgrpset)) != 0)
		return (set_errno(EFAULT));

	if (copyout(&user_parents, &user_snap->ss_parents,
	    sizeof (user_parents)) != 0)
		return (set_errno(EFAULT));

	if (copyout(&user_children, &user_snap->ss_children,
	    sizeof (user_children)) != 0)
		return (set_errno(EFAULT));

	if (copyout(&user_rsets, &user_snap->ss_rsets,
	    sizeof (user_rsets)) != 0)
		return (set_errno(EFAULT));

	if (copyout(&user_lats, &user_snap->ss_latencies,
	    sizeof (user_lats)) != 0)
		return (set_errno(EFAULT));

	/*
	 * Make copies of lgroup info and latency table, fix up pointers,
	 * and then copy them into user buffer
	 */
	user_info_buffer = kmem_zalloc(info_size, KM_NOSLEEP);
	if (user_info_buffer == NULL)
		return (set_errno(ENOMEM));

	user_lats_buffer = kmem_zalloc(snap_nlgrpsmax * sizeof (int *),
	    KM_NOSLEEP);
	if (user_lats_buffer == NULL) {
		kmem_free(user_info_buffer, info_size);
		return (set_errno(ENOMEM));
	}

	lgrp_info = (lgrp_info_t *)((uintptr_t)lgrp_snap + snap_hdr_size);
	bcopy(lgrp_info, user_info_buffer, info_size);

	cpu_index = 0;
	for (i = 0; i < snap_nlgrpsmax; i++) {
		ulong_t	*snap_rset;

		/*
		 * Skip non-existent lgroups
		 */
		if (user_info_buffer[i].info_lgrpid == LGRP_NONE)
			continue;

		/*
		 * Update free memory size since it changes frequently
		 * Only do so for lgroups directly containing memory
		 *
		 * NOTE: This must be done before changing the pointers to
		 *	 point into user space since we need to dereference
		 *	 lgroup resource set
		 */
		snap_rset = &lgrp_info[i].info_rset[LGRP_RSRC_MEM *
		    BT_BITOUL(snap_nlgrpsmax)];
		if (BT_TEST(snap_rset, i))
			user_info_buffer[i].info_mem_free =
			    lgrp_mem_size(i, LGRP_MEM_SIZE_FREE);

		/*
		 * Fix up pointers to parents, children, resources, and
		 * latencies
		 */
		user_info_buffer[i].info_parents =
		    (ulong_t *)((uintptr_t)user_parents + (i * bitmask_size));
		user_info_buffer[i].info_children =
		    (ulong_t *)((uintptr_t)user_children + (i * bitmask_size));
		user_info_buffer[i].info_rset =
		    (ulong_t *)((uintptr_t)user_rsets +
		    (i * LGRP_RSRC_COUNT * bitmask_size));
		user_lats_buffer[i] = (int *)((uintptr_t)user_lats +
		    (snap_nlgrpsmax * sizeof (int *)) + (i * snap_nlgrpsmax *
		    sizeof (int)));

		/*
		 * Fix up pointer to CPU IDs
		 */
		if (user_info_buffer[i].info_ncpus == 0) {
			user_info_buffer[i].info_cpuids = NULL;
			continue;
		}
		user_info_buffer[i].info_cpuids = &user_cpuids[cpu_index];
		cpu_index += user_info_buffer[i].info_ncpus;
	}
	ASSERT(cpu_index == snap_ncpus);

	/*
	 * Copy lgroup info and latency table with pointers fixed up to point
	 * into user buffer out to user buffer now
	 */
	retval = lgrp_snap->ss_size;
	if (copyout(user_info_buffer, user_info, info_size) != 0)
		retval = set_errno(EFAULT);
	kmem_free(user_info_buffer, info_size);

	if (copyout(user_lats_buffer, user_lats, snap_nlgrpsmax *
	    sizeof (int *)) != 0)
		retval = set_errno(EFAULT);
	kmem_free(user_lats_buffer, snap_nlgrpsmax * sizeof (int *));

	return (retval);
}


#ifdef	_SYSCALL32_IMPL
/*
 * Make 32-bit copy of snapshot, fix up any pointers in buffer to point
 * into user instead of kernel address space, copy 32-bit snapshot into
 * given user buffer, and return size of buffer needed to hold snapshot
 */
static int
lgrp_snapshot_copy32(caddr32_t buf, size32_t bufsize)
{
	size32_t			bitmask_size;
	size32_t			bitmasks_size;
	size32_t			children_size;
	int				cpu_index;
	size32_t			cpuids_size;
	int				i;
	int				j;
	size32_t			info_size;
	size32_t			lats_size;
	lgrp_info_t			*lgrp_info;
	lgrp_snapshot_header32_t	*lgrp_snap32;
	lgrp_info32_t			*lgrp_info32;
	processorid_t			*lgrp_cpuids32;
	caddr32_t			*lgrp_lats32;
	int				**lgrp_lats32_kernel;
	uint_t				*lgrp_set32;
	uint_t				*lgrp_parents32;
	uint_t				*lgrp_children32;
	uint_t				*lgrp_rsets32;
	size32_t			parents_size;
	size32_t			rsets_size;
	size32_t			set_size;
	size32_t			snap_hdr_size;
	int				snap_ncpus;
	int				snap_nlgrpsmax;
	size32_t			snap_size;

	if (lgrp_snap == NULL)
		return (0);

	snap_ncpus = lgrp_snap->ss_ncpus;
	snap_nlgrpsmax = lgrp_snap->ss_nlgrps_max;

	/*
	 * Calculate size of buffer needed for 32-bit snapshot,
	 * rounding up size of each object to allow for alignment
	 * of next object in buffer.
	 */
	snap_hdr_size = P2ROUNDUP(sizeof (lgrp_snapshot_header32_t),
	    sizeof (caddr32_t));
	info_size = P2ROUNDUP(snap_nlgrpsmax * sizeof (lgrp_info32_t),
	    sizeof (processorid_t));
	cpuids_size = P2ROUNDUP(snap_ncpus * sizeof (processorid_t),
	    sizeof (ulong_t));

	bitmask_size = BT_SIZEOFMAP32(snap_nlgrpsmax);

	set_size = bitmask_size;
	parents_size = snap_nlgrpsmax * bitmask_size;
	children_size = snap_nlgrpsmax * bitmask_size;
	rsets_size = P2ROUNDUP(LGRP_RSRC_COUNT * snap_nlgrpsmax *
	    (int)bitmask_size, sizeof (caddr32_t));

	bitmasks_size = set_size + parents_size + children_size + rsets_size;

	/*
	 * Size of latency table and buffer
	 */
	lats_size = (snap_nlgrpsmax * sizeof (caddr32_t)) +
	    (snap_nlgrpsmax * snap_nlgrpsmax * sizeof (int));

	snap_size = snap_hdr_size + info_size + cpuids_size + bitmasks_size +
	    lats_size;

	if (buf == 0 || bufsize <= 0) {
		return (snap_size);
	}

	/*
	 * User needs to try getting size of buffer again
	 * because given buffer size is too small.
	 * The lgroup hierarchy may have changed after they asked for the size
	 * but before the snapshot was taken.
	 */
	if (bufsize < snap_size)
		return (set_errno(EAGAIN));

	/*
	 * Make 32-bit copy of snapshot, fix up pointers to point into user
	 * buffer not kernel, and then copy whole thing into user buffer
	 */
	lgrp_snap32 = kmem_zalloc(snap_size, KM_NOSLEEP);
	if (lgrp_snap32 == NULL)
		return (set_errno(ENOMEM));

	/*
	 * Calculate pointers into 32-bit copy of snapshot
	 * for lgroup info, CPU IDs, pset lgroup bitmask, parents, children,
	 * resources, and latency table and buffer
	 */
	lgrp_info32 = (lgrp_info32_t *)((uintptr_t)lgrp_snap32 +
	    snap_hdr_size);
	lgrp_cpuids32 = (processorid_t *)((uintptr_t)lgrp_info32 + info_size);
	lgrp_set32 = (uint_t *)((uintptr_t)lgrp_cpuids32 + cpuids_size);
	lgrp_parents32 = (uint_t *)((uintptr_t)lgrp_set32 + set_size);
	lgrp_children32 = (uint_t *)((uintptr_t)lgrp_parents32 + parents_size);
	lgrp_rsets32 = (uint_t *)((uintptr_t)lgrp_children32 + children_size);
	lgrp_lats32 = (caddr32_t *)((uintptr_t)lgrp_rsets32 + rsets_size);

	/*
	 * Make temporary lgroup latency table of pointers for kernel to use
	 * to fill in rows of table with latencies from each lgroup
	 */
	lgrp_lats32_kernel =  kmem_zalloc(snap_nlgrpsmax * sizeof (int *),
	    KM_NOSLEEP);
	if (lgrp_lats32_kernel == NULL) {
		kmem_free(lgrp_snap32, snap_size);
		return (set_errno(ENOMEM));
	}

	/*
	 * Fill in 32-bit lgroup snapshot header
	 * (with pointers into user's buffer for lgroup info, CPU IDs,
	 * bit masks, and latencies)
	 */
	lgrp_snap32->ss_version = lgrp_snap->ss_version;
	lgrp_snap32->ss_levels = lgrp_snap->ss_levels;
	lgrp_snap32->ss_nlgrps = lgrp_snap32->ss_nlgrps_os =
	    lgrp_snap->ss_nlgrps;
	lgrp_snap32->ss_nlgrps_max = snap_nlgrpsmax;
	lgrp_snap32->ss_root = lgrp_snap->ss_root;
	lgrp_snap32->ss_ncpus = lgrp_snap->ss_ncpus;
	lgrp_snap32->ss_gen = lgrp_snap->ss_gen;
	lgrp_snap32->ss_view = LGRP_VIEW_OS;
	lgrp_snap32->ss_size = snap_size;
	lgrp_snap32->ss_magic = buf;
	lgrp_snap32->ss_info = buf + snap_hdr_size;
	lgrp_snap32->ss_cpuids = lgrp_snap32->ss_info + info_size;
	lgrp_snap32->ss_lgrpset = lgrp_snap32->ss_cpuids + cpuids_size;
	lgrp_snap32->ss_parents = lgrp_snap32->ss_lgrpset + bitmask_size;
	lgrp_snap32->ss_children = lgrp_snap32->ss_parents +
	    (snap_nlgrpsmax * bitmask_size);
	lgrp_snap32->ss_rsets = lgrp_snap32->ss_children +
	    (snap_nlgrpsmax * bitmask_size);
	lgrp_snap32->ss_latencies = lgrp_snap32->ss_rsets +
	    (LGRP_RSRC_COUNT * snap_nlgrpsmax * bitmask_size);

	/*
	 * Fill in lgrpset now because caller may have change psets
	 */
	kpreempt_disable();
	for (i = 0; i < snap_nlgrpsmax; i++) {
		if (klgrpset_ismember(curthread->t_cpupart->cp_lgrpset,
		    i)) {
			BT_SET32(lgrp_set32, i);
		}
	}
	kpreempt_enable();

	/*
	 * Fill in 32-bit copy of lgroup info and fix up pointers
	 * to point into user's buffer instead of kernel's
	 */
	cpu_index = 0;
	lgrp_info = lgrp_snap->ss_info;
	for (i = 0; i < snap_nlgrpsmax; i++) {
		uint_t	*children;
		uint_t	*lgrp_rset;
		uint_t	*parents;
		ulong_t	*snap_rset;

		/*
		 * Skip non-existent lgroups
		 */
		if (lgrp_info[i].info_lgrpid == LGRP_NONE) {
			bzero(&lgrp_info32[i], sizeof (lgrp_info32[i]));
			lgrp_info32[i].info_lgrpid = LGRP_NONE;
			continue;
		}

		/*
		 * Fill in parents, children, lgroup resource set, and
		 * latencies from snapshot
		 */
		parents = (uint_t *)((uintptr_t)lgrp_parents32 +
		    i * bitmask_size);
		children = (uint_t *)((uintptr_t)lgrp_children32 +
		    i * bitmask_size);
		snap_rset = (ulong_t *)((uintptr_t)lgrp_snap->ss_rsets +
		    (i * LGRP_RSRC_COUNT * BT_SIZEOFMAP(snap_nlgrpsmax)));
		lgrp_rset = (uint_t *)((uintptr_t)lgrp_rsets32 +
		    (i * LGRP_RSRC_COUNT * bitmask_size));
		lgrp_lats32_kernel[i] = (int *)((uintptr_t)lgrp_lats32 +
		    snap_nlgrpsmax * sizeof (caddr32_t) + i * snap_nlgrpsmax *
		    sizeof (int));
		for (j = 0; j < snap_nlgrpsmax; j++) {
			int	k;
			uint_t	*rset;

			if (BT_TEST(&lgrp_snap->ss_parents[i], j))
				BT_SET32(parents, j);

			if (BT_TEST(&lgrp_snap->ss_children[i], j))
				BT_SET32(children, j);

			for (k = 0; k < LGRP_RSRC_COUNT; k++) {
				rset = (uint_t *)((uintptr_t)lgrp_rset +
				    k * bitmask_size);
				if (BT_TEST(&snap_rset[k], j))
					BT_SET32(rset, j);
			}

			lgrp_lats32_kernel[i][j] =
			    lgrp_snap->ss_latencies[i][j];
		}

		/*
		 * Fix up pointer to latency buffer
		 */
		lgrp_lats32[i] = lgrp_snap32->ss_latencies +
		    snap_nlgrpsmax * sizeof (caddr32_t) + i * snap_nlgrpsmax *
		    sizeof (int);

		/*
		 * Fix up pointers for parents, children, and resources
		 */
		lgrp_info32[i].info_parents = lgrp_snap32->ss_parents +
		    (i * bitmask_size);
		lgrp_info32[i].info_children = lgrp_snap32->ss_children +
		    (i * bitmask_size);
		lgrp_info32[i].info_rset = lgrp_snap32->ss_rsets +
		    (i * LGRP_RSRC_COUNT * bitmask_size);

		/*
		 * Fill in memory and CPU info
		 * Only fill in memory for lgroups directly containing memory
		 */
		snap_rset = &lgrp_info[i].info_rset[LGRP_RSRC_MEM *
		    BT_BITOUL(snap_nlgrpsmax)];
		if (BT_TEST(snap_rset, i)) {
			lgrp_info32[i].info_mem_free = lgrp_mem_size(i,
			    LGRP_MEM_SIZE_FREE);
			lgrp_info32[i].info_mem_install =
			    lgrp_info[i].info_mem_install;
		}

		lgrp_info32[i].info_ncpus = lgrp_info[i].info_ncpus;

		lgrp_info32[i].info_lgrpid = lgrp_info[i].info_lgrpid;
		lgrp_info32[i].info_latency = lgrp_info[i].info_latency;

		if (lgrp_info32[i].info_ncpus == 0) {
			lgrp_info32[i].info_cpuids = 0;
			continue;
		}

		/*
		 * Fix up pointer for CPU IDs
		 */
		lgrp_info32[i].info_cpuids = lgrp_snap32->ss_cpuids +
		    (cpu_index * sizeof (processorid_t));
		cpu_index += lgrp_info32[i].info_ncpus;
	}
	ASSERT(cpu_index == snap_ncpus);

	/*
	 * Copy lgroup CPU IDs into 32-bit snapshot
	 * before copying it out into user's buffer
	 */
	bcopy(lgrp_snap->ss_cpuids, lgrp_cpuids32, cpuids_size);

	/*
	 * Copy 32-bit lgroup snapshot into user's buffer all at once
	 */
	if (copyout(lgrp_snap32, (void *)(uintptr_t)buf, snap_size) != 0) {
		kmem_free(lgrp_snap32, snap_size);
		kmem_free(lgrp_lats32_kernel, snap_nlgrpsmax * sizeof (int *));
		return (set_errno(EFAULT));
	}

	kmem_free(lgrp_snap32, snap_size);
	kmem_free(lgrp_lats32_kernel, snap_nlgrpsmax * sizeof (int *));

	return (snap_size);
}
#endif	/* _SYSCALL32_IMPL */


int
lgrpsys(int subcode, long ia, void *ap)
{
	size_t	bufsize;
	int	latency;

	switch (subcode) {

	case LGRP_SYS_AFFINITY_GET:
		return (lgrp_affinity_get((lgrp_affinity_args_t *)ap));

	case LGRP_SYS_AFFINITY_SET:
		return (lgrp_affinity_set((lgrp_affinity_args_t *)ap));

	case LGRP_SYS_GENERATION:
		return (lgrp_generation(ia));

	case LGRP_SYS_HOME:
		return (lgrp_home_get((idtype_t)ia, (id_t)(uintptr_t)ap));

	case LGRP_SYS_LATENCY:
		mutex_enter(&cpu_lock);
		latency = lgrp_latency(ia, (lgrp_id_t)(uintptr_t)ap);
		mutex_exit(&cpu_lock);
		return (latency);

	case LGRP_SYS_MEMINFO:
		return (meminfo(ia, (struct meminfo *)ap));

	case LGRP_SYS_VERSION:
		return (lgrp_version(ia));

	case LGRP_SYS_SNAPSHOT:
		mutex_enter(&lgrp_snap_lock);
		bufsize = lgrp_snapshot();
		if (ap && ia > 0) {
			if (get_udatamodel() == DATAMODEL_NATIVE)
				bufsize = lgrp_snapshot_copy(ap, ia);
#ifdef	_SYSCALL32_IMPL
			else
				bufsize = lgrp_snapshot_copy32(
				    (caddr32_t)(uintptr_t)ap, ia);
#endif	/* _SYSCALL32_IMPL */
		}
		mutex_exit(&lgrp_snap_lock);
		return (bufsize);

	default:
		break;

	}

	return (set_errno(EINVAL));
}