common/vm/vm_usage.c

/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */

/*
 * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 * Copyright 2026 Oxide Computer Company
 */

/*
 * vm_usage
 *
 * This file implements the getvmusage() private system call.
 * getvmusage() counts the amount of resident memory pages and swap
 * reserved by the specified process collective. A "process collective" is
 * the set of processes owned by a particular, zone, project, task, or user.
 *
 * rss and swap are counted so that for a given process collective, a page is
 * only counted once.  For example, this means that if multiple processes in
 * the same project map the same page, then the project will only be charged
 * once for that page.  On the other hand, if two processes in different
 * projects map the same page, then both projects will be charged
 * for the page.
 *
 * The vm_getusage() calculation is implemented so that the first thread
 * performs the rss/swap counting. Other callers will wait for that thread to
 * finish, copying the results.  This enables multiple rcapds and prstats to
 * consume data from the same calculation.  The results are also cached so that
 * a caller interested in recent results can just copy them instead of starting
 * a new calculation. The caller passes the maximium age (in seconds) of the
 * data.  If the cached data is young enough, the cache is copied, otherwise,
 * a new calculation is executed and the cache is replaced with the new
 * data.
 *
 * The rss calculation for each process collective is as follows:
 *
 *   - Inspect flags, determine if counting rss for zones, projects, tasks,
 *     and/or users.
 *   - For each proc:
 *	- Figure out proc's collectives (zone, project, task, and/or user).
 *	- For each seg in proc's address space:
 *		- If seg is private:
 *			- Lookup anons in the amp.
 *			- For incore pages not previously visited each of the
 *			  proc's collectives, add incore pagesize to each.
 *			  collective.
 *			  Anon's with a refcnt of 1 can be assummed to be not
 *			  previously visited.
 *			- For address ranges without anons in the amp:
 *				- Lookup pages in underlying vnode.
 *				- For incore pages not previously visiting for
 *				  each of the proc's collectives, add incore
 *				  pagesize to each collective.
 *		- If seg is shared:
 *			- Lookup pages in the shared amp or vnode.
 *			- For incore pages not previously visited for each of
 *			  the proc's collectives, add incore pagesize to each
 *			  collective.
 *
 * Swap is reserved by private segments, and shared anonymous segments.
 * The only shared anon segments which do not reserve swap are ISM segments
 * and schedctl segments, both of which can be identified by having
 * amp->swresv == 0.
 *
 * The swap calculation for each collective is as follows:
 *
 *   - Inspect flags, determine if counting rss for zones, projects, tasks,
 *     and/or users.
 *   - For each proc:
 *	- Figure out proc's collectives (zone, project, task, and/or user).
 *	- For each seg in proc's address space:
 *		- If seg is private:
 *			- Add svd->swresv pages to swap count for each of the
 *			  proc's collectives.
 *		- If seg is anon, shared, and amp->swresv != 0
 *			- For address ranges in amp not previously visited for
 *			  each of the proc's collectives, add size of address
 *			  range to the swap count for each collective.
 *
 * These two calculations are done simultaneously, with most of the work
 * being done in vmu_calculate_seg().  The results of the calculation are
 * copied into "vmu_data.vmu_cache_results".
 *
 * To perform the calculation, various things are tracked and cached:
 *
 *    - incore/not-incore page ranges for all vnodes.
 *	(vmu_data.vmu_all_vnodes_hash)
 *	This eliminates looking up the same page more than once.
 *
 *    - incore/not-incore page ranges for all shared amps.
 *	(vmu_data.vmu_all_amps_hash)
 *	This eliminates looking up the same page more than once.
 *
 *    - visited page ranges for each collective.
 *	   - per vnode (entity->vme_vnode_hash)
 *	   - per shared amp (entity->vme_amp_hash)
 *	For accurate counting of map-shared and COW-shared pages.
 *
 *    - visited private anons (refcnt > 1) for each collective.
 *	(entity->vme_anon_hash)
 *	For accurate counting of COW-shared pages.
 *
 * The common accounting structure is the vmu_entity_t, which represents
 * collectives:
 *
 *    - A zone.
 *    - A project, task, or user within a zone.
 *    - The entire system (vmu_data.vmu_system).
 *    - Each collapsed (col) project and user.  This means a given projid or
 *	uid, regardless of which zone the process is in.  For instance,
 *      project 0 in the global zone and project 0 in a non global zone are
 *	the same collapsed project.
 *
 *  Each entity structure tracks which pages have been already visited for
 *  that entity (via previously inspected processes) so that these pages are
 *  not double counted.
 */

#include <sys/errno.h>
#include <sys/types.h>
#include <sys/zone.h>
#include <sys/proc.h>
#include <sys/project.h>
#include <sys/task.h>
#include <sys/thread.h>
#include <sys/time.h>
#include <sys/mman.h>
#include <sys/modhash.h>
#include <sys/modhash_impl.h>
#include <sys/shm.h>
#include <sys/swap.h>
#include <sys/synch.h>
#include <sys/systm.h>
#include <sys/uio.h>
#include <sys/var.h>
#include <sys/vm_usage.h>
#include <sys/zone.h>
#include <sys/sunddi.h>
#include <sys/avl.h>
#include <vm/anon.h>
#include <vm/as.h>
#include <vm/seg_vn.h>
#include <vm/seg_spt.h>

#define	VMUSAGE_HASH_SIZE		512

#define	VMUSAGE_TYPE_VNODE		1
#define	VMUSAGE_TYPE_AMP		2
#define	VMUSAGE_TYPE_ANON		3

#define	VMUSAGE_BOUND_UNKNOWN		0
#define	VMUSAGE_BOUND_INCORE		1
#define	VMUSAGE_BOUND_NOT_INCORE	2

#define	ISWITHIN(node, addr)	((node)->vmb_start <= addr && \
				    (node)->vmb_end >= addr ? 1 : 0)

/*
 * bounds for vnodes and shared amps
 * Each bound is either entirely incore, entirely not in core, or
 * entirely unknown.  bounds are stored in an avl tree sorted by start member
 * when in use, otherwise (free or temporary lists) they're strung
 * together off of vmb_next.
 */
typedef struct vmu_bound {
	avl_node_t vmb_node;
	struct vmu_bound *vmb_next; /* NULL in tree else on free or temp list */
	pgcnt_t vmb_start;  /* page offset in vnode/amp on which bound starts */
	pgcnt_t	vmb_end;    /* page offset in vnode/amp on which bound ends */
	char	vmb_type;   /* One of VMUSAGE_BOUND_* */
} vmu_bound_t;

/*
 * hash of visited objects (vnodes or shared amps)
 * key is address of vnode or amp.  Bounds lists known incore/non-incore
 * bounds for vnode/amp.
 */
typedef struct vmu_object {
	struct vmu_object	*vmo_next;	/* free list */
	caddr_t		vmo_key;
	short		vmo_type;
	avl_tree_t	vmo_bounds;
} vmu_object_t;

/*
 * Entity by which to count results.
 *
 * The entity structure keeps the current rss/swap counts for each entity
 * (zone, project, etc), and hashes of vm structures that have already
 * been visited for the entity.
 *
 * vme_next:	links the list of all entities currently being counted by
 *		vmu_calculate().
 *
 * vme_next_calc: links the list of entities related to the current process
 *		 being counted by vmu_calculate_proc().
 *
 * vmu_calculate_proc() walks all processes.  For each process, it makes a
 * list of the entities related to that process using vme_next_calc.  This
 * list changes each time vmu_calculate_proc() is called.
 *
 */
typedef struct vmu_entity {
	struct vmu_entity *vme_next;
	struct vmu_entity *vme_next_calc;
	mod_hash_t	*vme_vnode_hash; /* vnodes visited for entity */
	mod_hash_t	*vme_amp_hash;	 /* shared amps visited for entity */
	mod_hash_t	*vme_anon_hash;	 /* COW anons visited for entity */
	vmusage_t	vme_result;	 /* identifies entity and results */
} vmu_entity_t;

/*
 * Hash of entities visited within a zone, and an entity for the zone
 * itself.
 */
typedef struct vmu_zone {
	struct vmu_zone	*vmz_next;	/* free list */
	id_t		vmz_id;
	vmu_entity_t	*vmz_zone;
	mod_hash_t	*vmz_projects_hash;
	mod_hash_t	*vmz_tasks_hash;
	mod_hash_t	*vmz_rusers_hash;
	mod_hash_t	*vmz_eusers_hash;
} vmu_zone_t;

/*
 * Cache of results from last calculation
 */
typedef struct vmu_cache {
	vmusage_t	*vmc_results;	/* Results from last call to */
					/* vm_getusage(). */
	uint64_t	vmc_nresults;	/* Count of cached results */
	uint64_t	vmc_refcnt;	/* refcnt for free */
	uint_t		vmc_flags;	/* Flags for vm_getusage() */
	hrtime_t	vmc_timestamp;	/* when cache was created */
} vmu_cache_t;

/*
 * top level rss info for the system
 */
typedef struct vmu_data {
	kmutex_t	vmu_lock;		/* Protects vmu_data */
	kcondvar_t	vmu_cv;			/* Used to signal threads */
						/* Waiting for */
						/* Rss_calc_thread to finish */
	vmu_entity_t	*vmu_system;		/* Entity for tracking */
						/* rss/swap for all processes */
						/* in all zones */
	mod_hash_t	*vmu_zones_hash;	/* Zones visited */
	mod_hash_t	*vmu_projects_col_hash; /* These *_col_hash hashes */
	mod_hash_t	*vmu_rusers_col_hash;	/* keep track of entities, */
	mod_hash_t	*vmu_eusers_col_hash;	/* ignoring zoneid, in order */
						/* to implement VMUSAGE_COL_* */
						/* flags, which aggregate by */
						/* project or user regardless */
						/* of zoneid. */
	mod_hash_t	*vmu_all_vnodes_hash;	/* System wide visited vnodes */
						/* to track incore/not-incore */
	mod_hash_t	*vmu_all_amps_hash;	/* System wide visited shared */
						/* amps to track incore/not- */
						/* incore */
	vmu_entity_t	*vmu_entities;		/* Linked list of entities */
	size_t		vmu_nentities;		/* Count of entities in list */
	vmu_cache_t	*vmu_cache;		/* Cached results */
	kthread_t	*vmu_calc_thread;	/* NULL, or thread running */
						/* vmu_calculate() */
	uint_t		vmu_calc_flags;		/* Flags being using by */
						/* currently running calc */
						/* thread */
	uint_t		vmu_pending_flags;	/* Flags of vm_getusage() */
						/* threads waiting for */
						/* calc thread to finish */
	uint_t		vmu_pending_waiters;	/* Number of threads waiting */
						/* for calc thread */
	vmu_bound_t	*vmu_free_bounds;
	vmu_object_t	*vmu_free_objects;
	vmu_entity_t	*vmu_free_entities;
	vmu_zone_t	*vmu_free_zones;
} vmu_data_t;

extern struct as kas;
extern proc_t *practive;
extern zone_t *global_zone;
extern struct seg_ops segvn_ops;
extern struct seg_ops segspt_shmops;

static vmu_data_t vmu_data;
static kmem_cache_t *vmu_bound_cache;
static kmem_cache_t *vmu_object_cache;

/*
 * Comparison routine for AVL tree. We base our comparison on vmb_start.
 */
static int
bounds_cmp(const void *bnd1, const void *bnd2)
{
	const vmu_bound_t *bound1 = bnd1;
	const vmu_bound_t *bound2 = bnd2;

	if (bound1->vmb_start == bound2->vmb_start) {
		return (0);
	}
	if (bound1->vmb_start < bound2->vmb_start) {
		return (-1);
	}

	return (1);
}

/*
 * Save a bound on the free list.
 */
static void
vmu_free_bound(vmu_bound_t *bound)
{
	bound->vmb_next = vmu_data.vmu_free_bounds;
	bound->vmb_start = 0;
	bound->vmb_end = 0;
	bound->vmb_type = 0;
	vmu_data.vmu_free_bounds = bound;
}

/*
 * Free an object, and all visited bound info.
 */
static void
vmu_free_object(mod_hash_val_t val)
{
	vmu_object_t *obj = (vmu_object_t *)val;
	avl_tree_t *tree = &(obj->vmo_bounds);
	vmu_bound_t *bound;
	void *cookie = NULL;

	while ((bound = avl_destroy_nodes(tree, &cookie)) != NULL)
		vmu_free_bound(bound);
	avl_destroy(tree);

	obj->vmo_type = 0;
	obj->vmo_next = vmu_data.vmu_free_objects;
	vmu_data.vmu_free_objects = obj;
}

/*
 * Free an entity, and hashes of visited objects for that entity.
 */
static void
vmu_free_entity(mod_hash_val_t val)
{
	vmu_entity_t *entity = (vmu_entity_t *)val;

	if (entity->vme_vnode_hash != NULL)
		i_mod_hash_clear_nosync(entity->vme_vnode_hash);
	if (entity->vme_amp_hash != NULL)
		i_mod_hash_clear_nosync(entity->vme_amp_hash);
	if (entity->vme_anon_hash != NULL)
		i_mod_hash_clear_nosync(entity->vme_anon_hash);

	entity->vme_next = vmu_data.vmu_free_entities;
	vmu_data.vmu_free_entities = entity;
}

/*
 * Free zone entity, and all hashes of entities inside that zone,
 * which are projects, tasks, and users.
 */
static void
vmu_free_zone(mod_hash_val_t val)
{
	vmu_zone_t *zone = (vmu_zone_t *)val;

	if (zone->vmz_zone != NULL) {
		vmu_free_entity((mod_hash_val_t)zone->vmz_zone);
		zone->vmz_zone = NULL;
	}
	if (zone->vmz_projects_hash != NULL)
		i_mod_hash_clear_nosync(zone->vmz_projects_hash);
	if (zone->vmz_tasks_hash != NULL)
		i_mod_hash_clear_nosync(zone->vmz_tasks_hash);
	if (zone->vmz_rusers_hash != NULL)
		i_mod_hash_clear_nosync(zone->vmz_rusers_hash);
	if (zone->vmz_eusers_hash != NULL)
		i_mod_hash_clear_nosync(zone->vmz_eusers_hash);
	zone->vmz_next = vmu_data.vmu_free_zones;
	vmu_data.vmu_free_zones = zone;
}

/*
 * Initialize synchronization primitives and hashes for system-wide tracking
 * of visited vnodes and shared amps.  Initialize results cache.
 */
void
vm_usage_init()
{
	mutex_init(&vmu_data.vmu_lock, NULL, MUTEX_DEFAULT, NULL);
	cv_init(&vmu_data.vmu_cv, NULL, CV_DEFAULT, NULL);

	vmu_data.vmu_system = NULL;
	vmu_data.vmu_zones_hash = NULL;
	vmu_data.vmu_projects_col_hash = NULL;
	vmu_data.vmu_rusers_col_hash = NULL;
	vmu_data.vmu_eusers_col_hash = NULL;

	vmu_data.vmu_free_bounds = NULL;
	vmu_data.vmu_free_objects = NULL;
	vmu_data.vmu_free_entities = NULL;
	vmu_data.vmu_free_zones = NULL;

	vmu_data.vmu_all_vnodes_hash = mod_hash_create_ptrhash(
	    "vmusage vnode hash", VMUSAGE_HASH_SIZE, vmu_free_object,
	    sizeof (vnode_t));
	vmu_data.vmu_all_amps_hash = mod_hash_create_ptrhash(
	    "vmusage amp hash", VMUSAGE_HASH_SIZE, vmu_free_object,
	    sizeof (struct anon_map));
	vmu_data.vmu_projects_col_hash = mod_hash_create_idhash(
	    "vmusage collapsed project hash", VMUSAGE_HASH_SIZE,
	    vmu_free_entity);
	vmu_data.vmu_rusers_col_hash = mod_hash_create_idhash(
	    "vmusage collapsed ruser hash", VMUSAGE_HASH_SIZE,
	    vmu_free_entity);
	vmu_data.vmu_eusers_col_hash = mod_hash_create_idhash(
	    "vmusage collpased euser hash", VMUSAGE_HASH_SIZE,
	    vmu_free_entity);
	vmu_data.vmu_zones_hash = mod_hash_create_idhash(
	    "vmusage zone hash", VMUSAGE_HASH_SIZE, vmu_free_zone);

	vmu_bound_cache = kmem_cache_create("vmu_bound_cache",
	    sizeof (vmu_bound_t), 0, NULL, NULL, NULL, NULL, NULL, 0);
	vmu_object_cache = kmem_cache_create("vmu_object_cache",
	    sizeof (vmu_object_t), 0, NULL, NULL, NULL, NULL, NULL, 0);

	vmu_data.vmu_entities = NULL;
	vmu_data.vmu_nentities = 0;

	vmu_data.vmu_cache = NULL;
	vmu_data.vmu_calc_thread = NULL;
	vmu_data.vmu_calc_flags = 0;
	vmu_data.vmu_pending_flags = 0;
	vmu_data.vmu_pending_waiters = 0;
}

/*
 * Allocate hashes for tracking vm objects visited for an entity.
 * Update list of entities.
 */
static vmu_entity_t *
vmu_alloc_entity(id_t id, int type, id_t zoneid)
{
	vmu_entity_t *entity;

	if (vmu_data.vmu_free_entities != NULL) {
		entity = vmu_data.vmu_free_entities;
		vmu_data.vmu_free_entities =
		    vmu_data.vmu_free_entities->vme_next;
		bzero(&entity->vme_result, sizeof (vmusage_t));
	} else {
		entity = kmem_zalloc(sizeof (vmu_entity_t), KM_SLEEP);
	}
	entity->vme_result.vmu_id = id;
	entity->vme_result.vmu_zoneid = zoneid;
	entity->vme_result.vmu_type = type;

	if (entity->vme_vnode_hash == NULL)
		entity->vme_vnode_hash = mod_hash_create_ptrhash(
		    "vmusage vnode hash", VMUSAGE_HASH_SIZE, vmu_free_object,
		    sizeof (vnode_t));

	if (entity->vme_amp_hash == NULL)
		entity->vme_amp_hash = mod_hash_create_ptrhash(
		    "vmusage amp hash", VMUSAGE_HASH_SIZE, vmu_free_object,
		    sizeof (struct anon_map));

	if (entity->vme_anon_hash == NULL)
		entity->vme_anon_hash = mod_hash_create_ptrhash(
		    "vmusage anon hash", VMUSAGE_HASH_SIZE,
		    mod_hash_null_valdtor, sizeof (struct anon));

	entity->vme_next = vmu_data.vmu_entities;
	vmu_data.vmu_entities = entity;
	vmu_data.vmu_nentities++;

	return (entity);
}

/*
 * Allocate a zone entity, and hashes for tracking visited vm objects
 * for projects, tasks, and users within that zone.
 */
static vmu_zone_t *
vmu_alloc_zone(id_t id)
{
	vmu_zone_t *zone;

	if (vmu_data.vmu_free_zones != NULL) {
		zone = vmu_data.vmu_free_zones;
		vmu_data.vmu_free_zones =
		    vmu_data.vmu_free_zones->vmz_next;
		zone->vmz_next = NULL;
		zone->vmz_zone = NULL;
	} else {
		zone = kmem_zalloc(sizeof (vmu_zone_t), KM_SLEEP);
	}

	zone->vmz_id = id;

	if ((vmu_data.vmu_calc_flags & (VMUSAGE_ZONE | VMUSAGE_ALL_ZONES)) != 0)
		zone->vmz_zone = vmu_alloc_entity(id, VMUSAGE_ZONE, id);

	if ((vmu_data.vmu_calc_flags & (VMUSAGE_PROJECTS |
	    VMUSAGE_ALL_PROJECTS)) != 0 && zone->vmz_projects_hash == NULL)
		zone->vmz_projects_hash = mod_hash_create_idhash(
		    "vmusage project hash", VMUSAGE_HASH_SIZE, vmu_free_entity);

	if ((vmu_data.vmu_calc_flags & (VMUSAGE_TASKS | VMUSAGE_ALL_TASKS))
	    != 0 && zone->vmz_tasks_hash == NULL)
		zone->vmz_tasks_hash = mod_hash_create_idhash(
		    "vmusage task hash", VMUSAGE_HASH_SIZE, vmu_free_entity);

	if ((vmu_data.vmu_calc_flags & (VMUSAGE_RUSERS | VMUSAGE_ALL_RUSERS))
	    != 0 && zone->vmz_rusers_hash == NULL)
		zone->vmz_rusers_hash = mod_hash_create_idhash(
		    "vmusage ruser hash", VMUSAGE_HASH_SIZE, vmu_free_entity);

	if ((vmu_data.vmu_calc_flags & (VMUSAGE_EUSERS | VMUSAGE_ALL_EUSERS))
	    != 0 && zone->vmz_eusers_hash == NULL)
		zone->vmz_eusers_hash = mod_hash_create_idhash(
		    "vmusage euser hash", VMUSAGE_HASH_SIZE, vmu_free_entity);

	return (zone);
}

/*
 * Allocate a structure for tracking visited bounds for a vm object.
 */
static vmu_object_t *
vmu_alloc_object(caddr_t key, int type)
{
	vmu_object_t *object;

	if (vmu_data.vmu_free_objects != NULL) {
		object = vmu_data.vmu_free_objects;
		vmu_data.vmu_free_objects =
		    vmu_data.vmu_free_objects->vmo_next;
	} else {
		object = kmem_cache_alloc(vmu_object_cache, KM_SLEEP);
	}

	object->vmo_next = NULL;
	object->vmo_key = key;
	object->vmo_type = type;
	avl_create(&(object->vmo_bounds), bounds_cmp, sizeof (vmu_bound_t), 0);

	return (object);
}

/*
 * Allocate and return a bound structure.
 */
static vmu_bound_t *
vmu_alloc_bound()
{
	vmu_bound_t *bound;

	if (vmu_data.vmu_free_bounds != NULL) {
		bound = vmu_data.vmu_free_bounds;
		vmu_data.vmu_free_bounds =
		    vmu_data.vmu_free_bounds->vmb_next;
	} else {
		bound = kmem_cache_alloc(vmu_bound_cache, KM_SLEEP);
	}

	bound->vmb_next = NULL;
	bound->vmb_start = 0;
	bound->vmb_end = 0;
	bound->vmb_type = 0;
	return (bound);
}

/*
 * vmu_find_insert_* functions implement hash lookup or allocate and
 * insert operations.
 */
static vmu_object_t *
vmu_find_insert_object(mod_hash_t *hash, caddr_t key, uint_t type)
{
	int ret;
	vmu_object_t *object;

	ret = i_mod_hash_find_nosync(hash, (mod_hash_key_t)key,
	    (mod_hash_val_t *)&object);
	if (ret != 0) {
		object = vmu_alloc_object(key, type);
		ret = i_mod_hash_insert_nosync(hash, (mod_hash_key_t)key,
		    (mod_hash_val_t)object, (mod_hash_hndl_t)0);
		ASSERT(ret == 0);
	}
	return (object);
}

static int
vmu_find_insert_anon(mod_hash_t *hash, caddr_t key)
{
	int ret;
	caddr_t val;

	ret = i_mod_hash_find_nosync(hash, (mod_hash_key_t)key,
	    (mod_hash_val_t *)&val);

	if (ret == 0)
		return (0);

	ret = i_mod_hash_insert_nosync(hash, (mod_hash_key_t)key,
	    (mod_hash_val_t)key, (mod_hash_hndl_t)0);

	ASSERT(ret == 0);

	return (1);
}

static vmu_entity_t *
vmu_find_insert_entity(mod_hash_t *hash, id_t id, uint_t type, id_t zoneid)
{
	int ret;
	vmu_entity_t *entity;

	ret = i_mod_hash_find_nosync(hash, (mod_hash_key_t)(uintptr_t)id,
	    (mod_hash_val_t *)&entity);
	if (ret != 0) {
		entity = vmu_alloc_entity(id, type, zoneid);
		ret = i_mod_hash_insert_nosync(hash,
		    (mod_hash_key_t)(uintptr_t)id, (mod_hash_val_t)entity,
		    (mod_hash_hndl_t)0);
		ASSERT(ret == 0);
	}
	return (entity);
}


/*
 * Returns list of object bounds between start and end.  New bounds inserted
 * by this call are given type.
 *
 * Returns the number of pages covered if new bounds are created.  Returns 0
 * if region between start/end consists of all existing bounds.
 */
static pgcnt_t
vmu_insert_lookup_object_bounds(vmu_object_t *ro, pgcnt_t start, pgcnt_t
    end, char type, vmu_bound_t **first, vmu_bound_t **last)
{
	avl_tree_t	*tree = &(ro->vmo_bounds);
	avl_index_t	where;
	vmu_bound_t	*walker, *tmp;
	pgcnt_t		ret = 0;

	ASSERT(start <= end);

	*first = *last = NULL;

	tmp = vmu_alloc_bound();
	tmp->vmb_start = start;
	tmp->vmb_type = type;

	/* Hopelessly optimistic case. */
	if (walker = avl_find(tree, tmp, &where)) {
		/* We got lucky. */
		vmu_free_bound(tmp);
		*first = walker;
	}

	if (walker == NULL) {
		/* Is start in the previous node? */
		walker = avl_nearest(tree, where, AVL_BEFORE);
		if (walker != NULL) {
			if (ISWITHIN(walker, start)) {
				/* We found start. */
				vmu_free_bound(tmp);
				*first = walker;
			}
		}
	}

	/*
	 * At this point, if *first is still NULL, then we
	 * didn't get a direct hit and start isn't covered
	 * by the previous node. We know that the next node
	 * must have a greater start value than we require
	 * because avl_find tells us where the AVL routines would
	 * insert our new node. We have some gap between the
	 * start we want and the next node.
	 */
	if (*first == NULL) {
		walker = avl_nearest(tree, where, AVL_AFTER);
		if (walker != NULL && walker->vmb_start <= end) {
			/* Fill the gap. */
			tmp->vmb_end = walker->vmb_start - 1;
			*first = tmp;
		} else {
			/* We have a gap over [start, end]. */
			tmp->vmb_end = end;
			*first = *last = tmp;
		}
		ret += tmp->vmb_end - tmp->vmb_start + 1;
		avl_insert(tree, tmp, where);
	}

	ASSERT(*first != NULL);

	if (*last != NULL) {
		/* We're done. */
		return (ret);
	}

	/*
	 * If we are here we still need to set *last and
	 * that may involve filling in some gaps.
	 */
	*last = *first;
	for (;;) {
		if (ISWITHIN(*last, end)) {
			/* We're done. */
			break;
		}
		walker = AVL_NEXT(tree, *last);
		if (walker == NULL || walker->vmb_start > end) {
			/* Bottom or mid tree with gap. */
			tmp = vmu_alloc_bound();
			tmp->vmb_start = (*last)->vmb_end + 1;
			tmp->vmb_end = end;
			tmp->vmb_type = type;
			ret += tmp->vmb_end - tmp->vmb_start + 1;
			avl_insert_here(tree, tmp, *last, AVL_AFTER);
			*last = tmp;
			break;
		} else {
			if ((*last)->vmb_end + 1 != walker->vmb_start) {
				/* Non-contiguous. */
				tmp = vmu_alloc_bound();
				tmp->vmb_start = (*last)->vmb_end + 1;
				tmp->vmb_end = walker->vmb_start - 1;
				tmp->vmb_type = type;
				ret += tmp->vmb_end - tmp->vmb_start + 1;
				avl_insert_here(tree, tmp, *last, AVL_AFTER);
				*last = tmp;
			} else {
				*last = walker;
			}
		}
	}

	return (ret);
}

/*
 * vmu_update_bounds()
 *
 * tree: avl_tree in which first and last hang.
 *
 * first, last:	list of continuous bounds, of which zero or more are of
 *		type VMUSAGE_BOUND_UNKNOWN.
 *
 * new_tree: avl_tree in which new_first and new_last hang.
 *
 * new_first, new_last:	list of continuous bounds, of which none are of
 *			type VMUSAGE_BOUND_UNKNOWN.  These bounds are used to
 *			update the types of bounds in (first,last) with
 *			type VMUSAGE_BOUND_UNKNOWN.
 *
 * For the list of bounds (first,last), this function updates any bounds
 * with type VMUSAGE_BOUND_UNKNOWN using the type of the corresponding bound in
 * the list (new_first, new_last).
 *
 * If a bound of type VMUSAGE_BOUND_UNKNOWN spans multiple bounds in the list
 * (new_first, new_last), it will be split into multiple bounds.
 *
 * Return value:
 *	The number of pages in the list of bounds (first,last) that were of
 *	type VMUSAGE_BOUND_UNKNOWN, which have been updated to be of type
 *	VMUSAGE_BOUND_INCORE.
 *
 */
static pgcnt_t
vmu_update_bounds(avl_tree_t *tree, vmu_bound_t **first, vmu_bound_t **last,
    avl_tree_t *new_tree, vmu_bound_t *new_first, vmu_bound_t *new_last)
{
	vmu_bound_t *next, *new_next, *tmp;
	pgcnt_t rss = 0;

	next = *first;
	new_next = new_first;

	/*
	 * Verify first and last bound are covered by new bounds if they
	 * have unknown type.
	 */
	ASSERT((*first)->vmb_type != VMUSAGE_BOUND_UNKNOWN ||
	    (*first)->vmb_start >= new_first->vmb_start);
	ASSERT((*last)->vmb_type != VMUSAGE_BOUND_UNKNOWN ||
	    (*last)->vmb_end <= new_last->vmb_end);
	for (;;) {
		/* If bound already has type, proceed to next bound. */
		if (next->vmb_type != VMUSAGE_BOUND_UNKNOWN) {
			if (next == *last)
				break;
			next = AVL_NEXT(tree, next);
			continue;
		}
		while (new_next->vmb_end < next->vmb_start)
			new_next = AVL_NEXT(new_tree, new_next);
		ASSERT(new_next->vmb_type != VMUSAGE_BOUND_UNKNOWN);
		next->vmb_type = new_next->vmb_type;
		if (new_next->vmb_end < next->vmb_end) {
			/* need to split bound */
			tmp = vmu_alloc_bound();
			tmp->vmb_type = VMUSAGE_BOUND_UNKNOWN;
			tmp->vmb_start = new_next->vmb_end + 1;
			tmp->vmb_end = next->vmb_end;
			avl_insert_here(tree, tmp, next, AVL_AFTER);
			next->vmb_end = new_next->vmb_end;
			if (*last == next)
				*last = tmp;
			if (next->vmb_type == VMUSAGE_BOUND_INCORE)
				rss += next->vmb_end - next->vmb_start + 1;
			next = tmp;
		} else {
			if (next->vmb_type == VMUSAGE_BOUND_INCORE)
				rss += next->vmb_end - next->vmb_start + 1;
			if (next == *last)
				break;
			next = AVL_NEXT(tree, next);
		}
	}
	return (rss);
}

/*
 * Merges adjacent bounds with same type between first and last bound.
 * After merge, last pointer may point to a different bound, as (incoming)
 * last bound may have been merged away.
 */
static void
vmu_merge_bounds(avl_tree_t *tree, vmu_bound_t **first, vmu_bound_t **last)
{
	vmu_bound_t *current;
	vmu_bound_t *next;

	ASSERT(tree != NULL);
	ASSERT(*first != NULL);
	ASSERT(*last != NULL);

	current = *first;
	while (current != *last) {
		next = AVL_NEXT(tree, current);
		if ((current->vmb_end + 1) == next->vmb_start &&
		    current->vmb_type == next->vmb_type) {
			current->vmb_end = next->vmb_end;
			avl_remove(tree, next);
			vmu_free_bound(next);
			if (next == *last) {
				*last = current;
			}
		} else {
			current = AVL_NEXT(tree, current);
		}
	}
}

/*
 * Given an amp and a list of bounds, updates each bound's type with
 * VMUSAGE_BOUND_INCORE or VMUSAGE_BOUND_NOT_INCORE.
 *
 * If a bound is partially incore, it will be split into two bounds.
 * first and last may be modified, as bounds may be split into multiple
 * bounds if they are partially incore/not-incore.
 *
 * Set incore to non-zero if bounds are already known to be incore.
 *
 */
static void
vmu_amp_update_incore_bounds(avl_tree_t *tree, struct anon_map *amp,
    vmu_bound_t **first, vmu_bound_t **last, boolean_t incore)
{
	vmu_bound_t *next;
	vmu_bound_t *tmp;
	pgcnt_t index;
	short bound_type;
	short page_type;
	vnode_t *vn;
	anoff_t off;
	struct anon *ap;

	next = *first;
	/* Shared anon slots don't change once set. */
	ANON_LOCK_ENTER(&amp->a_rwlock, RW_READER);
	for (;;) {
		if (incore == B_TRUE)
			next->vmb_type = VMUSAGE_BOUND_INCORE;

		if (next->vmb_type != VMUSAGE_BOUND_UNKNOWN) {
			if (next == *last)
				break;
			next = AVL_NEXT(tree, next);
			continue;
		}
		bound_type = next->vmb_type;
		index = next->vmb_start;
		while (index <= next->vmb_end) {

			/*
			 * These are used to determine how much to increment
			 * index when a large page is found.
			 */
			page_t *page;
			pgcnt_t pgcnt = 1;
			uint_t pgshft;
			pgcnt_t pgmsk;

			ap = anon_get_ptr(amp->ahp, index);
			if (ap != NULL)
				swap_xlate(ap, &vn, &off);

			if (ap != NULL && vn != NULL && vn->v_pages != NULL &&
			    (page = page_exists(vn, off)) != NULL) {
				page_type = VMUSAGE_BOUND_INCORE;
				if (page->p_szc > 0) {
					pgcnt = page_get_pagecnt(page->p_szc);
					pgshft = page_get_shift(page->p_szc);
					pgmsk = (0x1 << (pgshft - PAGESHIFT))
					    - 1;
				}
			} else {
				page_type = VMUSAGE_BOUND_NOT_INCORE;
			}
			if (bound_type == VMUSAGE_BOUND_UNKNOWN) {
				next->vmb_type = page_type;
			} else if (next->vmb_type != page_type) {
				/*
				 * If current bound type does not match page
				 * type, need to split off new bound.
				 */
				tmp = vmu_alloc_bound();
				tmp->vmb_type = page_type;
				tmp->vmb_start = index;
				tmp->vmb_end = next->vmb_end;
				avl_insert_here(tree, tmp, next, AVL_AFTER);
				next->vmb_end = index - 1;
				if (*last == next)
					*last = tmp;
				next = tmp;
			}
			if (pgcnt > 1) {
				/*
				 * If inside large page, jump to next large
				 * page
				 */
				index = (index & ~pgmsk) + pgcnt;
			} else {
				index++;
			}
		}
		if (next == *last) {
			ASSERT(next->vmb_type != VMUSAGE_BOUND_UNKNOWN);
			break;
		} else
			next = AVL_NEXT(tree, next);
	}
	ANON_LOCK_EXIT(&amp->a_rwlock);
}

/*
 * Same as vmu_amp_update_incore_bounds(), except for tracking
 * incore-/not-incore for vnodes.
 */
static void
vmu_vnode_update_incore_bounds(avl_tree_t *tree, vnode_t *vnode,
    vmu_bound_t **first, vmu_bound_t **last)
{
	vmu_bound_t *next;
	vmu_bound_t *tmp;
	pgcnt_t index;
	short bound_type;
	short page_type;

	next = *first;
	for (;;) {
		if (vnode->v_pages == NULL)
			next->vmb_type = VMUSAGE_BOUND_NOT_INCORE;

		if (next->vmb_type != VMUSAGE_BOUND_UNKNOWN) {
			if (next == *last)
				break;
			next = AVL_NEXT(tree, next);
			continue;
		}

		bound_type = next->vmb_type;
		index = next->vmb_start;
		while (index <= next->vmb_end) {

			/*
			 * These are used to determine how much to increment
			 * index when a large page is found.
			 */
			page_t *page;
			pgcnt_t pgcnt = 1;
			uint_t pgshft;
			pgcnt_t pgmsk;

			if (vnode->v_pages != NULL &&
			    (page = page_exists(vnode, ptob(index))) != NULL) {
				page_type = VMUSAGE_BOUND_INCORE;
				if (page->p_szc > 0) {
					pgcnt = page_get_pagecnt(page->p_szc);
					pgshft = page_get_shift(page->p_szc);
					pgmsk = (0x1 << (pgshft - PAGESHIFT))
					    - 1;
				}
			} else {
				page_type = VMUSAGE_BOUND_NOT_INCORE;
			}
			if (bound_type == VMUSAGE_BOUND_UNKNOWN) {
				next->vmb_type = page_type;
			} else if (next->vmb_type != page_type) {
				/*
				 * If current bound type does not match page
				 * type, need to split off new bound.
				 */
				tmp = vmu_alloc_bound();
				tmp->vmb_type = page_type;
				tmp->vmb_start = index;
				tmp->vmb_end = next->vmb_end;
				avl_insert_here(tree, tmp, next, AVL_AFTER);
				next->vmb_end = index - 1;
				if (*last == next)
					*last = tmp;
				next = tmp;
			}
			if (pgcnt > 1) {
				/*
				 * If inside large page, jump to next large
				 * page
				 */
				index = (index & ~pgmsk) + pgcnt;
			} else {
				index++;
			}
		}
		if (next == *last) {
			ASSERT(next->vmb_type != VMUSAGE_BOUND_UNKNOWN);
			break;
		} else
			next = AVL_NEXT(tree, next);
	}
}

/*
 * Calculate the rss and swap consumed by a segment.  vmu_entities is the
 * list of entities to visit.  For shared segments, the vnode or amp
 * is looked up in each entity to see if it has been already counted.  Private
 * anon pages are checked per entity to ensure that COW pages are not
 * double counted.
 *
 * For private mapped files, first the amp is checked for private pages.
 * Bounds not backed by the amp are looked up in the vnode for each entity
 * to avoid double counting of private COW vnode pages.
 */
static void
vmu_calculate_seg(vmu_entity_t *vmu_entities, struct seg *seg)
{
	struct segvn_data *svd;
	struct shm_data *shmd;
	struct spt_data *sptd;
	vmu_object_t *shared_object = NULL;
	vmu_object_t *entity_object = NULL;
	vmu_entity_t *entity;
	vmusage_t *result;
	vmu_bound_t *first = NULL;
	vmu_bound_t *last = NULL;
	vmu_bound_t *cur = NULL;
	vmu_bound_t *e_first = NULL;
	vmu_bound_t *e_last = NULL;
	vmu_bound_t *tmp;
	pgcnt_t p_index, s_index, p_start, p_end, s_start, s_end, rss, virt;
	struct anon_map *private_amp = NULL;
	boolean_t incore = B_FALSE;
	boolean_t shared = B_FALSE;
	int file = 0;
	pgcnt_t swresv = 0;
	pgcnt_t panon = 0;

	s_start = 0;
	p_end = 0;
	/* Can zero-length segments exist?  Not sure, so paranoia. */
	if (seg->s_size <= 0)
		return;

	/*
	 * Figure out if there is a shared object (such as a named vnode or
	 * a shared amp, then figure out if there is a private amp, which
	 * identifies private pages.
	 */
	if (seg->s_ops == &segvn_ops) {
		svd = (struct segvn_data *)seg->s_data;
		if (svd->type == MAP_SHARED) {
			shared = B_TRUE;
		} else {
			swresv = svd->swresv;

			if (SEGVN_LOCK_TRYENTER(seg->s_as, &svd->lock,
			    RW_READER) != 0) {
				/*
				 * Text replication anon maps can be shared
				 * across all zones. Space used for text
				 * replication is typically capped as a small %
				 * of memory.  To keep it simple for now we
				 * don't account for swap and memory space used
				 * for text replication.
				 */
				if (svd->tr_state == SEGVN_TR_OFF &&
				    svd->amp != NULL) {
					private_amp = svd->amp;
					p_start = svd->anon_index;
					p_end = svd->anon_index +
					    btop(seg->s_size) - 1;
				}
				SEGVN_LOCK_EXIT(seg->s_as, &svd->lock);
			}
		}
		if (svd->vp != NULL) {
			file = 1;
			shared_object = vmu_find_insert_object(
			    vmu_data.vmu_all_vnodes_hash, (caddr_t)svd->vp,
			    VMUSAGE_TYPE_VNODE);
			s_start = btop(svd->offset);
			s_end = btop(svd->offset + seg->s_size) - 1;
		}
		if (svd->amp != NULL && svd->type == MAP_SHARED) {
			ASSERT(shared_object == NULL);
			shared_object = vmu_find_insert_object(
			    vmu_data.vmu_all_amps_hash, (caddr_t)svd->amp,
			    VMUSAGE_TYPE_AMP);
			s_start = svd->anon_index;
			s_end = svd->anon_index + btop(seg->s_size) - 1;
			/* schedctl mappings are always in core */
			if (svd->amp->swresv == 0)
				incore = B_TRUE;
		}
	} else if (seg->s_ops == &segspt_shmops) {
		shared = B_TRUE;
		shmd = (struct shm_data *)seg->s_data;
		shared_object = vmu_find_insert_object(
		    vmu_data.vmu_all_amps_hash, (caddr_t)shmd->shm_amp,
		    VMUSAGE_TYPE_AMP);
		s_start = 0;
		s_end = btop(seg->s_size) - 1;
		sptd = shmd->shm_sptseg->s_data;

		/* ism segments are always incore and do not reserve swap */
		if (sptd->spt_flags & SHM_SHARE_MMU)
			incore = B_TRUE;

	} else {
		return;
	}

	/*
	 * If there is a private amp, count anon pages that exist.  If an
	 * anon has a refcnt > 1 (COW sharing), then save the anon in a
	 * hash so that it is not double counted.
	 *
	 * If there is also a shared object, then figure out the bounds
	 * which are not mapped by the private amp.
	 */
	if (private_amp != NULL) {

		/* Enter as writer to prevent COW anons from being freed */
		ANON_LOCK_ENTER(&private_amp->a_rwlock, RW_WRITER);

		p_index = p_start;
		s_index = s_start;

		while (p_index <= p_end) {

			pgcnt_t p_index_next;
			pgcnt_t p_bound_size;
			int cnt;
			anoff_t off;
			struct vnode *vn;
			struct anon *ap;
			page_t *page;		/* For handling of large */
			pgcnt_t pgcnt = 1;	/* pages */
			pgcnt_t pgstart;
			pgcnt_t pgend;
			uint_t pgshft;
			pgcnt_t pgmsk;

			p_index_next = p_index;
			ap = anon_get_next_ptr(private_amp->ahp,
			    &p_index_next);

			/*
			 * If next anon is past end of mapping, simulate
			 * end of anon so loop terminates.
			 */
			if (p_index_next > p_end) {
				p_index_next = p_end + 1;
				ap = NULL;
			}
			/*
			 * For COW segments, keep track of bounds not
			 * backed by private amp so they can be looked
			 * up in the backing vnode
			 */
			if (p_index_next != p_index) {

				/*
				 * Compute index difference between anon and
				 * previous anon.
				 */
				p_bound_size = p_index_next - p_index - 1;

				if (shared_object != NULL) {
					cur = vmu_alloc_bound();
					cur->vmb_start = s_index;
					cur->vmb_end = s_index + p_bound_size;
					cur->vmb_type = VMUSAGE_BOUND_UNKNOWN;
					if (first == NULL) {
						first = cur;
						last = cur;
					} else {
						last->vmb_next = cur;
						last = cur;
					}
				}
				p_index = p_index + p_bound_size + 1;
				s_index = s_index + p_bound_size + 1;
			}

			/* Detect end of anons in amp */
			if (ap == NULL)
				break;

			cnt = ap->an_refcnt;
			swap_xlate(ap, &vn, &off);

			if (vn == NULL || vn->v_pages == NULL ||
			    (page = page_exists(vn, off)) == NULL) {
				p_index++;
				s_index++;
				continue;
			}

			/*
			 * If large page is found, compute portion of large
			 * page in mapping, and increment indicies to the next
			 * large page.
			 */
			if (page->p_szc > 0) {

				pgcnt = page_get_pagecnt(page->p_szc);
				pgshft = page_get_shift(page->p_szc);
				pgmsk = (0x1 << (pgshft - PAGESHIFT)) - 1;

				/* First page in large page */
				pgstart = p_index & ~pgmsk;
				/* Last page in large page */
				pgend = pgstart + pgcnt - 1;
				/*
				 * Artifically end page if page extends past
				 * end of mapping.
				 */
				if (pgend > p_end)
					pgend = p_end;

				/*
				 * Compute number of pages from large page
				 * which are mapped.
				 */
				pgcnt = pgend - p_index + 1;

				/*
				 * Point indicies at page after large page,
				 * or at page after end of mapping.
				 */
				p_index += pgcnt;
				s_index += pgcnt;
			} else {
				p_index++;
				s_index++;
			}

			/*
			 * Assume anon structs with a refcnt
			 * of 1 are not COW shared, so there
			 * is no reason to track them per entity.
			 */
			if (cnt == 1) {
				panon += pgcnt;
				continue;
			}
			for (entity = vmu_entities; entity != NULL;
			    entity = entity->vme_next_calc) {

				result = &entity->vme_result;
				/*
				 * Track COW anons per entity so
				 * they are not double counted.
				 */
				if (vmu_find_insert_anon(entity->vme_anon_hash,
				    (caddr_t)ap) == 0)
					continue;

				result->vmu_rss_all += (pgcnt << PAGESHIFT);
				result->vmu_rss_private +=
				    (pgcnt << PAGESHIFT);
			}
		}
		ANON_LOCK_EXIT(&private_amp->a_rwlock);
	}

	/* Add up resident anon and swap reserved for private mappings */
	if (swresv > 0 || panon > 0) {
		for (entity = vmu_entities; entity != NULL;
		    entity = entity->vme_next_calc) {
			result = &entity->vme_result;
			result->vmu_swap_all += swresv;
			result->vmu_swap_private += swresv;
			result->vmu_rss_all += (panon << PAGESHIFT);
			result->vmu_rss_private += (panon << PAGESHIFT);
		}
	}

	/* Compute resident pages backing shared amp or named vnode */
	if (shared_object != NULL) {
		avl_tree_t *tree = &(shared_object->vmo_bounds);

		if (first == NULL) {
			/*
			 * No private amp, or private amp has no anon
			 * structs.  This means entire segment is backed by
			 * the shared object.
			 */
			first = vmu_alloc_bound();
			first->vmb_start = s_start;
			first->vmb_end = s_end;
			first->vmb_type = VMUSAGE_BOUND_UNKNOWN;
		}
		/*
		 * Iterate bounds not backed by private amp, and compute
		 * resident pages.
		 */
		cur = first;
		while (cur != NULL) {

			if (vmu_insert_lookup_object_bounds(shared_object,
			    cur->vmb_start, cur->vmb_end, VMUSAGE_BOUND_UNKNOWN,
			    &first, &last) > 0) {
				/* new bounds, find incore/not-incore */
				if (shared_object->vmo_type ==
				    VMUSAGE_TYPE_VNODE) {
					vmu_vnode_update_incore_bounds(
					    tree,
					    (vnode_t *)
					    shared_object->vmo_key, &first,
					    &last);
				} else {
					vmu_amp_update_incore_bounds(
					    tree,
					    (struct anon_map *)
					    shared_object->vmo_key, &first,
					    &last, incore);
				}
				vmu_merge_bounds(tree, &first, &last);
			}
			for (entity = vmu_entities; entity != NULL;
			    entity = entity->vme_next_calc) {
				avl_tree_t *e_tree;

				result = &entity->vme_result;

				entity_object = vmu_find_insert_object(
				    shared_object->vmo_type ==
				    VMUSAGE_TYPE_VNODE ? entity->vme_vnode_hash:
				    entity->vme_amp_hash,
				    shared_object->vmo_key,
				    shared_object->vmo_type);

				virt = vmu_insert_lookup_object_bounds(
				    entity_object, cur->vmb_start, cur->vmb_end,
				    VMUSAGE_BOUND_UNKNOWN, &e_first, &e_last);

				if (virt == 0)
					continue;
				/*
				 * Range visited for this entity
				 */
				e_tree = &(entity_object->vmo_bounds);
				rss = vmu_update_bounds(e_tree, &e_first,
				    &e_last, tree, first, last);
				result->vmu_rss_all += (rss << PAGESHIFT);
				if (shared == B_TRUE && file == B_FALSE) {
					/* shared anon mapping */
					result->vmu_swap_all +=
					    (virt << PAGESHIFT);
					result->vmu_swap_shared +=
					    (virt << PAGESHIFT);
					result->vmu_rss_shared +=
					    (rss << PAGESHIFT);
				} else if (shared == B_TRUE && file == B_TRUE) {
					/* shared file mapping */
					result->vmu_rss_shared +=
					    (rss << PAGESHIFT);
				} else if (shared == B_FALSE &&
				    file == B_TRUE) {
					/* private file mapping */
					result->vmu_rss_private +=
					    (rss << PAGESHIFT);
				}
				vmu_merge_bounds(e_tree, &e_first, &e_last);
			}
			tmp = cur;
			cur = cur->vmb_next;
			vmu_free_bound(tmp);
		}
	}
}

/*
 * Based on the current calculation flags, find the relevant entities
 * which are relative to the process.  Then calculate each segment
 * in the process'es address space for each relevant entity.
 */
static void
vmu_calculate_proc(proc_t *p)
{
	vmu_entity_t *entities = NULL;
	vmu_zone_t *zone;
	vmu_entity_t *tmp;
	struct as *as;
	struct seg *seg;
	int ret;

	/* Figure out which entities are being computed */
	if ((vmu_data.vmu_system) != NULL) {
		tmp = vmu_data.vmu_system;
		tmp->vme_next_calc = entities;
		entities = tmp;
	}
	if (vmu_data.vmu_calc_flags &
	    (VMUSAGE_ZONE | VMUSAGE_ALL_ZONES | VMUSAGE_PROJECTS |
	    VMUSAGE_ALL_PROJECTS | VMUSAGE_TASKS | VMUSAGE_ALL_TASKS |
	    VMUSAGE_RUSERS | VMUSAGE_ALL_RUSERS | VMUSAGE_EUSERS |
	    VMUSAGE_ALL_EUSERS)) {
		ret = i_mod_hash_find_nosync(vmu_data.vmu_zones_hash,
		    (mod_hash_key_t)(uintptr_t)p->p_zone->zone_id,
		    (mod_hash_val_t *)&zone);
		if (ret != 0) {
			zone = vmu_alloc_zone(p->p_zone->zone_id);
			ret = i_mod_hash_insert_nosync(vmu_data.vmu_zones_hash,
			    (mod_hash_key_t)(uintptr_t)p->p_zone->zone_id,
			    (mod_hash_val_t)zone, (mod_hash_hndl_t)0);
			ASSERT(ret == 0);
		}
		if (zone->vmz_zone != NULL) {
			tmp = zone->vmz_zone;
			tmp->vme_next_calc = entities;
			entities = tmp;
		}
		if (vmu_data.vmu_calc_flags &
		    (VMUSAGE_PROJECTS | VMUSAGE_ALL_PROJECTS)) {
			tmp = vmu_find_insert_entity(zone->vmz_projects_hash,
			    p->p_task->tk_proj->kpj_id, VMUSAGE_PROJECTS,
			    zone->vmz_id);
			tmp->vme_next_calc = entities;
			entities = tmp;
		}
		if (vmu_data.vmu_calc_flags &
		    (VMUSAGE_TASKS | VMUSAGE_ALL_TASKS)) {
			tmp = vmu_find_insert_entity(zone->vmz_tasks_hash,
			    p->p_task->tk_tkid, VMUSAGE_TASKS, zone->vmz_id);
			tmp->vme_next_calc = entities;
			entities = tmp;
		}
		if (vmu_data.vmu_calc_flags &
		    (VMUSAGE_RUSERS | VMUSAGE_ALL_RUSERS)) {
			tmp = vmu_find_insert_entity(zone->vmz_rusers_hash,
			    crgetruid(p->p_cred), VMUSAGE_RUSERS, zone->vmz_id);
			tmp->vme_next_calc = entities;
			entities = tmp;
		}
		if (vmu_data.vmu_calc_flags &
		    (VMUSAGE_EUSERS | VMUSAGE_ALL_EUSERS)) {
			tmp = vmu_find_insert_entity(zone->vmz_eusers_hash,
			    crgetuid(p->p_cred), VMUSAGE_EUSERS, zone->vmz_id);
			tmp->vme_next_calc = entities;
			entities = tmp;
		}
	}
	/* Entities which collapse projects and users for all zones */
	if (vmu_data.vmu_calc_flags & VMUSAGE_COL_PROJECTS) {
		tmp = vmu_find_insert_entity(vmu_data.vmu_projects_col_hash,
		    p->p_task->tk_proj->kpj_id, VMUSAGE_PROJECTS, ALL_ZONES);
		tmp->vme_next_calc = entities;
		entities = tmp;
	}
	if (vmu_data.vmu_calc_flags & VMUSAGE_COL_RUSERS) {
		tmp = vmu_find_insert_entity(vmu_data.vmu_rusers_col_hash,
		    crgetruid(p->p_cred), VMUSAGE_RUSERS, ALL_ZONES);
		tmp->vme_next_calc = entities;
		entities = tmp;
	}
	if (vmu_data.vmu_calc_flags & VMUSAGE_COL_EUSERS) {
		tmp = vmu_find_insert_entity(vmu_data.vmu_eusers_col_hash,
		    crgetuid(p->p_cred), VMUSAGE_EUSERS, ALL_ZONES);
		tmp->vme_next_calc = entities;
		entities = tmp;
	}

	ASSERT(entities != NULL);
	/* process all segs in process's address space */
	as = p->p_as;
	AS_LOCK_ENTER(as, RW_READER);
	for (seg = AS_SEGFIRST(as); seg != NULL;
	    seg = AS_SEGNEXT(as, seg)) {
		vmu_calculate_seg(entities, seg);
	}
	AS_LOCK_EXIT(as);
}

/*
 * Free data created by previous call to vmu_calculate().
 */
static void
vmu_clear_calc()
{
	if (vmu_data.vmu_system != NULL) {
		vmu_free_entity(vmu_data.vmu_system);
		vmu_data.vmu_system = NULL;
	}
	if (vmu_data.vmu_zones_hash != NULL)
		i_mod_hash_clear_nosync(vmu_data.vmu_zones_hash);
	if (vmu_data.vmu_projects_col_hash != NULL)
		i_mod_hash_clear_nosync(vmu_data.vmu_projects_col_hash);
	if (vmu_data.vmu_rusers_col_hash != NULL)
		i_mod_hash_clear_nosync(vmu_data.vmu_rusers_col_hash);
	if (vmu_data.vmu_eusers_col_hash != NULL)
		i_mod_hash_clear_nosync(vmu_data.vmu_eusers_col_hash);

	i_mod_hash_clear_nosync(vmu_data.vmu_all_vnodes_hash);
	i_mod_hash_clear_nosync(vmu_data.vmu_all_amps_hash);
}

/*
 * Free unused data structures.  These can result if the system workload
 * decreases between calculations.
 */
static void
vmu_free_extra()
{
	vmu_bound_t *tb;
	vmu_object_t *to;
	vmu_entity_t *te;
	vmu_zone_t *tz;

	while (vmu_data.vmu_free_bounds != NULL) {
		tb = vmu_data.vmu_free_bounds;
		vmu_data.vmu_free_bounds = vmu_data.vmu_free_bounds->vmb_next;
		kmem_cache_free(vmu_bound_cache, tb);
	}
	while (vmu_data.vmu_free_objects != NULL) {
		to = vmu_data.vmu_free_objects;
		vmu_data.vmu_free_objects =
		    vmu_data.vmu_free_objects->vmo_next;
		kmem_cache_free(vmu_object_cache, to);
	}
	while (vmu_data.vmu_free_entities != NULL) {
		te = vmu_data.vmu_free_entities;
		vmu_data.vmu_free_entities =
		    vmu_data.vmu_free_entities->vme_next;
		if (te->vme_vnode_hash != NULL)
			mod_hash_destroy_hash(te->vme_vnode_hash);
		if (te->vme_amp_hash != NULL)
			mod_hash_destroy_hash(te->vme_amp_hash);
		if (te->vme_anon_hash != NULL)
			mod_hash_destroy_hash(te->vme_anon_hash);
		kmem_free(te, sizeof (vmu_entity_t));
	}
	while (vmu_data.vmu_free_zones != NULL) {
		tz = vmu_data.vmu_free_zones;
		vmu_data.vmu_free_zones =
		    vmu_data.vmu_free_zones->vmz_next;
		if (tz->vmz_projects_hash != NULL)
			mod_hash_destroy_hash(tz->vmz_projects_hash);
		if (tz->vmz_tasks_hash != NULL)
			mod_hash_destroy_hash(tz->vmz_tasks_hash);
		if (tz->vmz_rusers_hash != NULL)
			mod_hash_destroy_hash(tz->vmz_rusers_hash);
		if (tz->vmz_eusers_hash != NULL)
			mod_hash_destroy_hash(tz->vmz_eusers_hash);
		kmem_free(tz, sizeof (vmu_zone_t));
	}
}

extern kcondvar_t *pr_pid_cv;

/*
 * Determine which entity types are relevant and allocate the hashes to
 * track them.  Then walk the process table and count rss and swap
 * for each process'es address space.  Address space object such as
 * vnodes, amps and anons are tracked per entity, so that they are
 * not double counted in the results.
 *
 */
static void
vmu_calculate()
{
	int i = 0;
	int ret;
	proc_t *p;

	vmu_clear_calc();

	if (vmu_data.vmu_calc_flags & VMUSAGE_SYSTEM)
		vmu_data.vmu_system = vmu_alloc_entity(0, VMUSAGE_SYSTEM,
		    ALL_ZONES);

	/*
	 * Walk process table and calculate rss of each proc.
	 *
	 * Pidlock and p_lock cannot be held while doing the rss calculation.
	 * This is because:
	 *	1.  The calculation allocates using KM_SLEEP.
	 *	2.  The calculation grabs a_lock, which cannot be grabbed
	 *	    after p_lock.
	 *
	 * Since pidlock must be dropped, we cannot simply just walk the
	 * practive list.  Instead, we walk the process table, and sprlock
	 * each process to ensure that it does not exit during the
	 * calculation.
	 */

	mutex_enter(&pidlock);
	for (i = 0; i < v.v_proc; i++) {
again:
		p = pid_entry(i);
		if (p == NULL)
			continue;

		mutex_enter(&p->p_lock);
		mutex_exit(&pidlock);

		if (panicstr) {
			mutex_exit(&p->p_lock);
			return;
		}

		/* Try to set P_PR_LOCK */
		ret = sprtrylock_proc(p);
		if (ret == -1) {
			/* Process in invalid state */
			mutex_exit(&p->p_lock);
			mutex_enter(&pidlock);
			continue;
		} else if (ret == 1) {
			/*
			 * P_PR_LOCK is already set.  Wait and try again.
			 * This also drops p_lock.
			 */
			sprwaitlock_proc(p);
			mutex_enter(&pidlock);
			goto again;
		}
		mutex_exit(&p->p_lock);

		vmu_calculate_proc(p);

		mutex_enter(&p->p_lock);
		sprunlock(p);
		mutex_enter(&pidlock);
	}
	mutex_exit(&pidlock);

	vmu_free_extra();
}

/*
 * allocate a new cache for N results satisfying flags
 */
vmu_cache_t *
vmu_cache_alloc(size_t nres, uint_t flags)
{
	vmu_cache_t *cache;

	cache = kmem_zalloc(sizeof (vmu_cache_t), KM_SLEEP);
	cache->vmc_results = kmem_zalloc(sizeof (vmusage_t) * nres, KM_SLEEP);
	cache->vmc_nresults = nres;
	cache->vmc_flags = flags;
	cache->vmc_refcnt = 1;
	return (cache);
}

/*
 * Make sure cached results are not freed
 */
static void
vmu_cache_hold(vmu_cache_t *cache)
{
	ASSERT(MUTEX_HELD(&vmu_data.vmu_lock));
	cache->vmc_refcnt++;
}

/*
 * free cache data
 */
static void
vmu_cache_rele(vmu_cache_t *cache)
{
	ASSERT(MUTEX_HELD(&vmu_data.vmu_lock));
	ASSERT(cache->vmc_refcnt > 0);
	cache->vmc_refcnt--;
	if (cache->vmc_refcnt == 0) {
		kmem_free(cache->vmc_results, sizeof (vmusage_t) *
		    cache->vmc_nresults);
		kmem_free(cache, sizeof (vmu_cache_t));
	}
}

/*
 * Copy out the cached results to a caller.  Inspect the callers flags
 * and zone to determine which cached results should be copied.
 */
static int
vmu_copyout_results(vmu_cache_t *cache, vmusage_t *buf, size_t *nres,
    uint_t flags, uio_seg_t seg)
{
	vmusage_t *result, *out_result;
	vmusage_t dummy;
	size_t i, count = 0;
	size_t bufsize;
	int ret = 0;
	uint_t types = 0;

	if (nres != NULL) {
		if (uio_copyin(nres, &bufsize, sizeof (size_t), seg) != 0)
			return (set_errno(EFAULT));
	} else {
		bufsize = 0;
	}

	/* figure out what results the caller is interested in. */
	if ((flags & VMUSAGE_SYSTEM) && curproc->p_zone == global_zone)
		types |= VMUSAGE_SYSTEM;
	if (flags & (VMUSAGE_ZONE | VMUSAGE_ALL_ZONES))
		types |= VMUSAGE_ZONE;
	if (flags & (VMUSAGE_PROJECTS | VMUSAGE_ALL_PROJECTS |
	    VMUSAGE_COL_PROJECTS))
		types |= VMUSAGE_PROJECTS;
	if (flags & (VMUSAGE_TASKS | VMUSAGE_ALL_TASKS))
		types |= VMUSAGE_TASKS;
	if (flags & (VMUSAGE_RUSERS | VMUSAGE_ALL_RUSERS | VMUSAGE_COL_RUSERS))
		types |= VMUSAGE_RUSERS;
	if (flags & (VMUSAGE_EUSERS | VMUSAGE_ALL_EUSERS | VMUSAGE_COL_EUSERS))
		types |= VMUSAGE_EUSERS;

	/* count results for current zone */
	out_result = buf;
	for (result = cache->vmc_results, i = 0;
	    i < cache->vmc_nresults; result++, i++) {

		/* Do not return "other-zone" results to non-global zones */
		if (curproc->p_zone != global_zone &&
		    curproc->p_zone->zone_id != result->vmu_zoneid)
			continue;

		/*
		 * If non-global zone requests VMUSAGE_SYSTEM, fake
		 * up VMUSAGE_ZONE result as VMUSAGE_SYSTEM result.
		 */
		if (curproc->p_zone != global_zone &&
		    (flags & VMUSAGE_SYSTEM) != 0 &&
		    result->vmu_type == VMUSAGE_ZONE) {
			count++;
			if (out_result != NULL) {
				if (bufsize < count) {
					ret = set_errno(EOVERFLOW);
				} else {
					dummy = *result;
					dummy.vmu_zoneid = ALL_ZONES;
					dummy.vmu_id = 0;
					dummy.vmu_type = VMUSAGE_SYSTEM;
					if (uio_copyout(&dummy, out_result,
					    sizeof (vmusage_t), seg) != 0)
						return (set_errno(EFAULT));
					out_result++;
				}
			}
		}

		/* Skip results that do not match requested type */
		if ((result->vmu_type & types) == 0)
			continue;

		/* Skip collated results if not requested */
		if (result->vmu_zoneid == ALL_ZONES) {
			if (result->vmu_type == VMUSAGE_PROJECTS &&
			    (flags & VMUSAGE_COL_PROJECTS) == 0)
				continue;
			if (result->vmu_type == VMUSAGE_EUSERS &&
			    (flags & VMUSAGE_COL_EUSERS) == 0)
				continue;
			if (result->vmu_type == VMUSAGE_RUSERS &&
			    (flags & VMUSAGE_COL_RUSERS) == 0)
				continue;
		}

		/* Skip "other zone" results if not requested */
		if (result->vmu_zoneid != curproc->p_zone->zone_id) {
			if (result->vmu_type == VMUSAGE_ZONE &&
			    (flags & VMUSAGE_ALL_ZONES) == 0)
				continue;
			if (result->vmu_type == VMUSAGE_PROJECTS &&
			    (flags & (VMUSAGE_ALL_PROJECTS |
			    VMUSAGE_COL_PROJECTS)) == 0)
				continue;
			if (result->vmu_type == VMUSAGE_TASKS &&
			    (flags & VMUSAGE_ALL_TASKS) == 0)
				continue;
			if (result->vmu_type == VMUSAGE_RUSERS &&
			    (flags & (VMUSAGE_ALL_RUSERS |
			    VMUSAGE_COL_RUSERS)) == 0)
				continue;
			if (result->vmu_type == VMUSAGE_EUSERS &&
			    (flags & (VMUSAGE_ALL_EUSERS |
			    VMUSAGE_COL_EUSERS)) == 0)
				continue;
		}
		count++;
		if (out_result != NULL) {
			if (bufsize < count) {
				ret = set_errno(EOVERFLOW);
			} else {
				if (uio_copyout(result, out_result,
				    sizeof (vmusage_t), seg) != 0)
					return (set_errno(EFAULT));
				out_result++;
			}
		}
	}
	if (nres != NULL)
		if (uio_copyout(&count, nres, sizeof (size_t), seg) != 0)
			return (set_errno(EFAULT));

	return (ret);
}

/*
 * vm_getusage()
 *
 * Counts rss and swap by zone, project, task, and/or user.  The flags argument
 * determines the type of results structures returned.  Flags requesting
 * results from more than one zone are "flattened" to the local zone if the
 * caller is not the global zone.
 *
 * args:
 *	flags:	bitmap consisting of one or more of VMUSAGE_*.
 *	age:	maximum allowable age (time since counting was done) in
 *		seconds of the results.  Results from previous callers are
 *		cached in kernel.
 *	buf:	pointer to buffer array of vmusage_t.  If NULL, then only nres
 *		set on success.
 *	nres:	Set to number of vmusage_t structures pointed to by buf
 *		before calling vm_getusage().
 *		On return 0 (success) or ENOSPC, is set to the number of result
 *		structures returned or attempted to return.
 *	seg:	address space in which buf and nres reside, either
 *		UIO_USERSPACE or UIO_SYSSPACE.
 *
 * returns 0 on success, -1 on failure:
 *	EINTR (interrupted)
 *	ENOSPC (nres to small for results, nres set to needed value for success)
 *	EINVAL (flags invalid)
 *	EFAULT (bad address for buf or nres)
 */
int
vm_getusage(uint_t flags, time_t age, vmusage_t *buf, size_t *nres,
    uio_seg_t seg)
{
	vmu_entity_t *entity;
	vmusage_t *result;
	int ret = 0;
	int cacherecent = 0;
	hrtime_t now;
	uint_t flags_orig;

	/*
	 * Non-global zones cannot request system wide and/or collated
	 * results, or the system result, so munge the flags accordingly.
	 */
	flags_orig = flags;
	if (curproc->p_zone != global_zone) {
		if (flags & (VMUSAGE_ALL_PROJECTS | VMUSAGE_COL_PROJECTS)) {
			flags &= ~(VMUSAGE_ALL_PROJECTS | VMUSAGE_COL_PROJECTS);
			flags |= VMUSAGE_PROJECTS;
		}
		if (flags & (VMUSAGE_ALL_RUSERS | VMUSAGE_COL_RUSERS)) {
			flags &= ~(VMUSAGE_ALL_RUSERS | VMUSAGE_COL_RUSERS);
			flags |= VMUSAGE_RUSERS;
		}
		if (flags & (VMUSAGE_ALL_EUSERS | VMUSAGE_COL_EUSERS)) {
			flags &= ~(VMUSAGE_ALL_EUSERS | VMUSAGE_COL_EUSERS);
			flags |= VMUSAGE_EUSERS;
		}
		if (flags & VMUSAGE_SYSTEM) {
			flags &= ~VMUSAGE_SYSTEM;
			flags |= VMUSAGE_ZONE;
		}
	}

	/* Check for unknown flags */
	if ((flags & (~VMUSAGE_MASK)) != 0)
		return (set_errno(EINVAL));

	/* Check for no flags */
	if ((flags & VMUSAGE_MASK) == 0)
		return (set_errno(EINVAL));

	mutex_enter(&vmu_data.vmu_lock);
	now = gethrtime();

start:
	if (vmu_data.vmu_cache != NULL) {

		vmu_cache_t *cache;

		if ((vmu_data.vmu_cache->vmc_timestamp +
		    ((hrtime_t)age * NANOSEC)) > now)
			cacherecent = 1;

		if ((vmu_data.vmu_cache->vmc_flags & flags) == flags &&
		    cacherecent == 1) {
			cache = vmu_data.vmu_cache;
			vmu_cache_hold(cache);
			mutex_exit(&vmu_data.vmu_lock);

			ret = vmu_copyout_results(cache, buf, nres, flags_orig,
			    seg);
			mutex_enter(&vmu_data.vmu_lock);
			vmu_cache_rele(cache);
			if (vmu_data.vmu_pending_waiters > 0)
				cv_broadcast(&vmu_data.vmu_cv);
			mutex_exit(&vmu_data.vmu_lock);
			return (ret);
		}
		/*
		 * If the cache is recent, it is likely that there are other
		 * consumers of vm_getusage running, so add their flags to the
		 * desired flags for the calculation.
		 */
		if (cacherecent == 1)
			flags = vmu_data.vmu_cache->vmc_flags | flags;
	}
	if (vmu_data.vmu_calc_thread == NULL) {

		vmu_cache_t *cache;

		vmu_data.vmu_calc_thread = curthread;
		vmu_data.vmu_calc_flags = flags;
		vmu_data.vmu_entities = NULL;
		vmu_data.vmu_nentities = 0;
		if (vmu_data.vmu_pending_waiters > 0)
			vmu_data.vmu_calc_flags |=
			    vmu_data.vmu_pending_flags;

		vmu_data.vmu_pending_flags = 0;
		mutex_exit(&vmu_data.vmu_lock);
		vmu_calculate();
		mutex_enter(&vmu_data.vmu_lock);
		/* copy results to cache */
		if (vmu_data.vmu_cache != NULL)
			vmu_cache_rele(vmu_data.vmu_cache);
		cache = vmu_data.vmu_cache =
		    vmu_cache_alloc(vmu_data.vmu_nentities,
		    vmu_data.vmu_calc_flags);

		result = cache->vmc_results;
		for (entity = vmu_data.vmu_entities; entity != NULL;
		    entity = entity->vme_next) {
			*result = entity->vme_result;
			result++;
		}
		cache->vmc_timestamp = gethrtime();
		vmu_cache_hold(cache);

		vmu_data.vmu_calc_flags = 0;
		vmu_data.vmu_calc_thread = NULL;

		if (vmu_data.vmu_pending_waiters > 0)
			cv_broadcast(&vmu_data.vmu_cv);

		mutex_exit(&vmu_data.vmu_lock);

		/* copy cache */
		ret = vmu_copyout_results(cache, buf, nres, flags_orig, seg);
		mutex_enter(&vmu_data.vmu_lock);
		vmu_cache_rele(cache);
		mutex_exit(&vmu_data.vmu_lock);

		return (ret);
	}
	vmu_data.vmu_pending_flags |= flags;
	vmu_data.vmu_pending_waiters++;
	while (vmu_data.vmu_calc_thread != NULL) {
		if (cv_wait_sig(&vmu_data.vmu_cv,
		    &vmu_data.vmu_lock) == 0) {
			vmu_data.vmu_pending_waiters--;
			mutex_exit(&vmu_data.vmu_lock);
			return (set_errno(EINTR));
		}
	}
	vmu_data.vmu_pending_waiters--;
	goto start;
}