common/os/pghw.c

/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */
/*
 * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 */

#include <sys/systm.h>
#include <sys/types.h>
#include <sys/param.h>
#include <sys/thread.h>
#include <sys/cpuvar.h>
#include <sys/kmem.h>
#include <sys/cmn_err.h>
#include <sys/group.h>
#include <sys/pg.h>
#include <sys/pghw.h>
#include <sys/cpu_pm.h>
#include <sys/cap_util.h>

/*
 * Processor Groups: Hardware sharing relationship layer
 *
 * This file implements an extension to Processor Groups to capture
 * hardware sharing relationships existing between logical CPUs. Examples of
 * hardware sharing relationships include shared caches on some CMT
 * procesoor architectures, or shared local memory controllers on NUMA
 * based system architectures.
 *
 * The pghw_t structure represents the extended PG. The first member
 * of the structure is the generic pg_t with the pghw specific members
 * following. The generic pg_t *must* remain the first member of the
 * structure as the code uses casting of structure references to access
 * the generic pg_t structure elements.
 *
 * In addition to the generic CPU grouping, physical PGs have a hardware
 * sharing relationship enumerated "type", and an instance id. The enumerated
 * type is defined by the pghw_type_t enumeration, while the instance id
 * uniquely identifies the sharing instance from among others of the same
 * hardware sharing type.
 *
 * The physical PGs are organized into an overall hierarchy, and are tracked
 * in a number of different per CPU, and per pghw_type_t type groups.
 * As an example:
 *
 * -------------
 * | pg_hw     |
 * | (group_t) |
 * -------------
 *  ||                          ============================
 *  ||\\-----------------------//       \\                 \\
 *  ||  | hwset (PGC_HW_CHIP) |        -------------      -------------
 *  ||  | (group_t)           |        | pghw_t    |      | pghw_t    |
 *  ||  -----------------------        | chip 0    |      | chip 1    |
 *  ||                                 -------------      -------------
 *  ||                                 \\  \\  \\  \\     \\  \\  \\  \\
 *  ||                                  cpu cpu cpu cpu    cpu cpu cpu cpu
 *  ||
 *  ||                          ============================
 *  ||\\-----------------------//       \\                 \\
 *  ||  | hwset (PGC_HW_IPIPE)|        -------------      -------------
 *  ||  | (group_t)           |        | pghw_t    |      | pghw_t    |
 *  ||  -----------------------        | ipipe 0   |      | ipipe 1   |
 *  ||                                 -------------      -------------
 *  ||                                 \\  \\             \\  \\
 *  ||                                  cpu cpu            cpu cpu
 *  ...
 *
 *
 * The top level pg_hw is a group of "hwset" groups. Each hwset holds of group
 * of physical PGs of the same hardware sharing type. Within each hwset, the
 * PG's instance id uniquely identifies the grouping relationshsip among other
 * groupings of the same sharing type. The instance id for a grouping is
 * platform defined, and in some cases may be used by platform code as a handle
 * to search for a particular relationship instance.
 *
 * Each physical PG (by virtue of the embedded pg_t) contains a group of CPUs
 * that participate in the sharing relationship. Each CPU also has associated
 * with it a grouping tracking the PGs in which the CPU belongs. This can be
 * used to iterate over the various relationships in which the CPU participates
 * (the CPU's chip, cache, lgroup, etc.).
 *
 * The hwsets are created dynamically as new hardware sharing relationship types
 * are instantiated. They are never destroyed, as once a given relationship
 * type appears in the system, it is quite likely that at least one instance of
 * that relationship will always persist as long as the system is running.
 */

static group_t		*pg_hw;		/* top level pg hw group */

/*
 * Physical PG kstats
 */
struct pghw_kstat {
	kstat_named_t	pg_id;
	kstat_named_t	pg_class;
	kstat_named_t	pg_ncpus;
	kstat_named_t	pg_instance_id;
	kstat_named_t	pg_hw;
	kstat_named_t	pg_policy;
} pghw_kstat = {
	{ "id",			KSTAT_DATA_UINT32 },
	{ "pg_class",		KSTAT_DATA_STRING },
	{ "ncpus",		KSTAT_DATA_UINT32 },
	{ "instance_id",	KSTAT_DATA_UINT32 },
	{ "hardware",		KSTAT_DATA_STRING },
	{ "policy",		KSTAT_DATA_STRING },
};

kmutex_t		pghw_kstat_lock;

/*
 * Capacity and Utilization PG kstats
 *
 * These kstats are updated one at a time, so we can have a single scratch space
 * to fill the data.
 *
 * kstat fields:
 *
 *   pgid		PG ID for PG described by this kstat
 *
 *   pg_ncpus		Number of CPUs within this PG
 *
 *   pg_cpus		String describing CPUs within this PG
 *
 *   pg_sharing		Name of sharing relationship for this PG
 *
 *   pg_generation	Generation value that increases whenever any CPU leaves
 *			  or joins PG. Two kstat snapshots for the same
 *			  CPU may only be compared if they have the same
 *			  generation
 *
 *   pg_hw_util		Running value of PG utilization for the sharing
 *			  relationship
 *
 *   pg_hw_util_time_running
 *			Total time spent collecting CU data. The time may be
 *			less than wall time if CU counters were stopped for
 *			some time.
 *
 *   pg_hw_util_time_stopped Total time the CU counters were stopped.
 *
 *   pg_hw_util_rate	Utilization rate, expressed in operations per second.
 *
 *   pg_hw_util_rate_max Maximum observed value of utilization rate.
 */
struct pghw_cu_kstat {
	kstat_named_t	pg_id;
	kstat_named_t	pg_ncpus;
	kstat_named_t	pg_generation;
	kstat_named_t	pg_hw_util;
	kstat_named_t	pg_hw_util_time_running;
	kstat_named_t	pg_hw_util_time_stopped;
	kstat_named_t	pg_hw_util_rate;
	kstat_named_t	pg_hw_util_rate_max;
	kstat_named_t	pg_cpus;
	kstat_named_t	pg_sharing;
} pghw_cu_kstat = {
	{ "id",			KSTAT_DATA_UINT32 },
	{ "ncpus",		KSTAT_DATA_UINT32 },
	{ "generation",		KSTAT_DATA_UINT32   },
	{ "hw_util",		KSTAT_DATA_UINT64   },
	{ "hw_util_time_running",	KSTAT_DATA_UINT64   },
	{ "hw_util_time_stopped",	KSTAT_DATA_UINT64   },
	{ "hw_util_rate",	KSTAT_DATA_UINT64   },
	{ "hw_util_rate_max",	KSTAT_DATA_UINT64   },
	{ "cpus",		KSTAT_DATA_STRING   },
	{ "sharing_relation",	KSTAT_DATA_STRING   },
};

/*
 * Calculate the string size to represent NCPUS. Allow 5 digits for each CPU ID
 * plus one space per CPU plus NUL byte in the end. This is only an estimate,
 * since we try to compress CPU ranges as x-y. In the worst case the string
 * representation of CPUs may be truncated.
 */
#define	CPUSTR_LEN(ncpus) ((ncpus) * 6)

/*
 * Maximum length of the string that represents list of CPUs
 */
static int pg_cpulist_maxlen = 0;

static void		pghw_kstat_create(pghw_t *);
static int		pghw_kstat_update(kstat_t *, int);
static int		pghw_cu_kstat_update(kstat_t *, int);
static int		cpu2id(void *);

/*
 * hwset operations
 */
static group_t		*pghw_set_create(pghw_type_t);
static void		pghw_set_add(group_t *, pghw_t *);
static void		pghw_set_remove(group_t *, pghw_t *);

static void		pghw_cpulist_alloc(pghw_t *);
static int		cpu2id(void *);

/*
 * Initialize the physical portion of a hardware PG
 */
void
pghw_init(pghw_t *pg, cpu_t *cp, pghw_type_t hw)
{
	group_t		*hwset;

	if ((hwset = pghw_set_lookup(hw)) == NULL) {
		/*
		 * Haven't seen this hardware type yet
		 */
		hwset = pghw_set_create(hw);
	}

	pghw_set_add(hwset, pg);
	pg->pghw_hw = hw;
	pg->pghw_generation = 0;
	pg->pghw_instance =
	    pg_plat_hw_instance_id(cp, hw);
	pghw_kstat_create(pg);

	/*
	 * Hardware sharing relationship specific initialization
	 */
	switch (pg->pghw_hw) {
	case PGHW_POW_ACTIVE:
		pg->pghw_handle =
		    (pghw_handle_t)cpupm_domain_init(cp, CPUPM_DTYPE_ACTIVE);
		break;
	case PGHW_POW_IDLE:
		pg->pghw_handle =
		    (pghw_handle_t)cpupm_domain_init(cp, CPUPM_DTYPE_IDLE);
		break;
	default:
		pg->pghw_handle = (pghw_handle_t)NULL;
	}
}

/*
 * Teardown the physical portion of a physical PG
 */
void
pghw_fini(pghw_t *pg)
{
	group_t		*hwset;

	hwset = pghw_set_lookup(pg->pghw_hw);
	ASSERT(hwset != NULL);

	pghw_set_remove(hwset, pg);
	pg->pghw_instance = (id_t)PGHW_INSTANCE_ANON;
	pg->pghw_hw = (pghw_type_t)-1;

	if (pg->pghw_kstat != NULL)
		kstat_delete(pg->pghw_kstat);

	/*
	 * Destroy string representation of CPUs
	 */
	if (pg->pghw_cpulist != NULL) {
		kmem_free(pg->pghw_cpulist,
		    pg->pghw_cpulist_len);
		pg->pghw_cpulist = NULL;
	}

	if (pg->pghw_cu_kstat != NULL)
		kstat_delete(pg->pghw_cu_kstat);
}

/*
 * Find an existing physical PG in which to place
 * the given CPU for the specified hardware sharing
 * relationship
 */
pghw_t *
pghw_place_cpu(cpu_t *cp, pghw_type_t hw)
{
	group_t		*hwset;

	if ((hwset = pghw_set_lookup(hw)) == NULL) {
		return (NULL);
	}

	return ((pghw_t *)pg_cpu_find_pg(cp, hwset));
}

/*
 * Find the pg representing the hw sharing relationship in which
 * cp belongs
 */
pghw_t *
pghw_find_pg(cpu_t *cp, pghw_type_t hw)
{
	group_iter_t	i;
	pghw_t	*pg;

	group_iter_init(&i);
	while ((pg = group_iterate(&cp->cpu_pg->pgs, &i)) != NULL) {
		if (pg->pghw_hw == hw)
			return (pg);
	}
	return (NULL);
}

/*
 * Find the PG of the given hardware sharing relationship
 * type with the given instance id
 */
pghw_t *
pghw_find_by_instance(id_t id, pghw_type_t hw)
{
	group_iter_t	i;
	group_t		*set;
	pghw_t		*pg;

	set = pghw_set_lookup(hw);
	if (!set)
		return (NULL);

	group_iter_init(&i);
	while ((pg = group_iterate(set, &i)) != NULL) {
		if (pg->pghw_instance == id)
			return (pg);
	}
	return (NULL);
}

/*
 * CPUs physical ID cache creation / destruction
 * The cache's elements are initialized to the CPU's id
 */
void
pghw_physid_create(cpu_t *cp)
{
	int	i;

	cp->cpu_physid = kmem_alloc(sizeof (cpu_physid_t), KM_SLEEP);

	for (i = 0; i < (sizeof (cpu_physid_t) / sizeof (id_t)); i++) {
		((id_t *)cp->cpu_physid)[i] = cp->cpu_id;
	}
}

void
pghw_physid_destroy(cpu_t *cp)
{
	if (cp->cpu_physid) {
		kmem_free(cp->cpu_physid, sizeof (cpu_physid_t));
		cp->cpu_physid = NULL;
	}
}

/*
 * Create a new, empty hwset.
 * This routine may block, and must not be called from any
 * paused CPU context.
 */
static group_t	*
pghw_set_create(pghw_type_t hw)
{
	group_t	*g;
	int	ret;

	/*
	 * Create the top level PG hw group if it doesn't already exist
	 * This is a "set" of hardware sets, that is ordered (and indexed)
	 * by the pghw_type_t enum.
	 */
	if (pg_hw == NULL) {
		pg_hw = kmem_alloc(sizeof (group_t), KM_SLEEP);
		group_create(pg_hw);
		group_expand(pg_hw, (uint_t)PGHW_NUM_COMPONENTS);
	}

	/*
	 * Create the new hwset
	 * Add it to the top level pg_hw group.
	 */
	g = kmem_alloc(sizeof (group_t), KM_SLEEP);
	group_create(g);

	ret = group_add_at(pg_hw, g, (uint_t)hw);
	ASSERT(ret == 0);

	return (g);
}

/*
 * Find the hwset associated with the given hardware sharing type
 */
group_t *
pghw_set_lookup(pghw_type_t hw)
{
	group_t	*hwset;

	if (pg_hw == NULL)
		return (NULL);

	hwset = GROUP_ACCESS(pg_hw, (uint_t)hw);
	return (hwset);
}

/*
 * Add a PG to a hwset
 */
static void
pghw_set_add(group_t *hwset, pghw_t *pg)
{
	(void) group_add(hwset, pg, GRP_RESIZE);
}

/*
 * Remove a PG from a hwset
 */
static void
pghw_set_remove(group_t *hwset, pghw_t *pg)
{
	int result;

	result = group_remove(hwset, pg, GRP_RESIZE);
	ASSERT(result == 0);
}

/*
 * Return a string name given a pg_hw sharing type
 */
char *
pghw_type_string(pghw_type_t hw)
{
	switch (hw) {
	case PGHW_IPIPE:
		return ("Integer Pipeline");
	case PGHW_CACHE:
		return ("Cache");
	case PGHW_FPU:
		return ("Floating Point Unit");
	case PGHW_MPIPE:
		return ("Data Pipe to memory");
	case PGHW_CHIP:
		return ("Socket");
	case PGHW_MEMORY:
		return ("Memory");
	case PGHW_POW_ACTIVE:
		return ("CPU PM Active Power Domain");
	case PGHW_POW_IDLE:
		return ("CPU PM Idle Power Domain");
	default:
		return ("unknown");
	}
}

/*
 * Return a short string name given a pg_hw sharing type
 */
char *
pghw_type_shortstring(pghw_type_t hw)
{
	switch (hw) {
	case PGHW_IPIPE:
		return ("instr_pipeline");
	case PGHW_CACHE:
		return ("Cache");
	case PGHW_FPU:
		return ("FPU");
	case PGHW_MPIPE:
		return ("memory_pipeline");
	case PGHW_CHIP:
		return ("Socket");
	case PGHW_MEMORY:
		return ("Memory");
	case PGHW_POW_ACTIVE:
		return ("CPU_PM_Active");
	case PGHW_POW_IDLE:
		return ("CPU_PM_Idle");
	default:
		return ("unknown");
	}
}

/*
 * Create / Update routines for PG hw kstats
 *
 * It is the intention of these kstats to provide some level
 * of informational / debugging observability into the types
 * and nature of the system's detected hardware sharing relationships
 */
void
pghw_kstat_create(pghw_t *pg)
{
	char *class = pghw_type_string(pg->pghw_hw);

	/*
	 * Create a physical pg kstat
	 */
	if ((pg->pghw_kstat = kstat_create("pg", ((pg_t *)pg)->pg_id,
	    "pg", "pg",
	    KSTAT_TYPE_NAMED,
	    sizeof (pghw_kstat) / sizeof (kstat_named_t),
	    KSTAT_FLAG_VIRTUAL)) != NULL) {
		/* Class string, hw string, and policy string */
		pg->pghw_kstat->ks_data_size += PG_CLASS_NAME_MAX;
		pg->pghw_kstat->ks_data_size += PGHW_KSTAT_STR_LEN_MAX;
		pg->pghw_kstat->ks_data_size += PGHW_KSTAT_STR_LEN_MAX;
		pg->pghw_kstat->ks_lock = &pghw_kstat_lock;
		pg->pghw_kstat->ks_data = &pghw_kstat;
		pg->pghw_kstat->ks_update = pghw_kstat_update;
		pg->pghw_kstat->ks_private = pg;
		kstat_install(pg->pghw_kstat);
	}

	if (pg_cpulist_maxlen == 0)
		pg_cpulist_maxlen = CPUSTR_LEN(max_ncpus);

	/*
	 * Create a physical pg kstat
	 */
	if ((pg->pghw_cu_kstat = kstat_create("pg", ((pg_t *)pg)->pg_id,
	    "hardware", class,
	    KSTAT_TYPE_NAMED,
	    sizeof (pghw_cu_kstat) / sizeof (kstat_named_t),
	    KSTAT_FLAG_VIRTUAL)) != NULL) {
		pg->pghw_cu_kstat->ks_lock = &pghw_kstat_lock;
		pg->pghw_cu_kstat->ks_data = &pghw_cu_kstat;
		pg->pghw_cu_kstat->ks_update = pghw_cu_kstat_update;
		pg->pghw_cu_kstat->ks_private = pg;
		pg->pghw_cu_kstat->ks_data_size += strlen(class) + 1;
		/* Allow space for CPU strings */
		pg->pghw_cu_kstat->ks_data_size += PGHW_KSTAT_STR_LEN_MAX;
		pg->pghw_cu_kstat->ks_data_size += pg_cpulist_maxlen;
		kstat_install(pg->pghw_cu_kstat);
	}
}

int
pghw_kstat_update(kstat_t *ksp, int rw)
{
	struct pghw_kstat	*pgsp = &pghw_kstat;
	pghw_t			*pg = ksp->ks_private;

	if (rw == KSTAT_WRITE)
		return (EACCES);

	pgsp->pg_id.value.ui32 = ((pg_t *)pg)->pg_id;
	pgsp->pg_ncpus.value.ui32 = GROUP_SIZE(&((pg_t *)pg)->pg_cpus);
	pgsp->pg_instance_id.value.ui32 = pg->pghw_instance;
	kstat_named_setstr(&pgsp->pg_class, ((pg_t *)pg)->pg_class->pgc_name);
	kstat_named_setstr(&pgsp->pg_hw, pghw_type_string(pg->pghw_hw));
	kstat_named_setstr(&pgsp->pg_policy, pg_policy_name((pg_t *)pg));
	return (0);
}

int
pghw_cu_kstat_update(kstat_t *ksp, int rw)
{
	struct pghw_cu_kstat	*pgsp = &pghw_cu_kstat;
	pghw_t			*pg = ksp->ks_private;
	pghw_util_t		*hw_util = &pg->pghw_stats;

	if (rw == KSTAT_WRITE)
		return (EACCES);

	pgsp->pg_id.value.ui32 = ((pg_t *)pg)->pg_id;
	pgsp->pg_ncpus.value.ui32 = GROUP_SIZE(&((pg_t *)pg)->pg_cpus);

	/*
	 * Allocate memory for the string representing the list of CPUs in PG.
	 * This memory should persist past the call to pghw_cu_kstat_update()
	 * since the kstat snapshot routine will reference this memory.
	 */
	pghw_cpulist_alloc(pg);

	if (pg->pghw_kstat_gen != pg->pghw_generation) {
		/*
		 * PG kstat generation number is out of sync with PG's
		 * generation mumber. It means that some CPUs could have joined
		 * or left PG and it is not possible to compare the numbers
		 * obtained before and after the generation change.
		 *
		 * Reset the maximum utilization rate and start computing it
		 * from scratch.
		 */
		hw_util->pghw_util = 0;
		hw_util->pghw_rate_max = 0;
		pg->pghw_kstat_gen = pg->pghw_generation;
	}

	/*
	 * We can't block on CPU lock because when PG is destroyed (under
	 * cpu_lock) it tries to delete this kstat and it will wait for us to
	 * complete which will never happen since we are waiting for cpu_lock to
	 * drop. Deadlocks are fun!
	 */
	if (mutex_tryenter(&cpu_lock)) {
		if (pg->pghw_cpulist != NULL &&
		    *(pg->pghw_cpulist) == '\0') {
			(void) group2intlist(&(((pg_t *)pg)->pg_cpus),
			    pg->pghw_cpulist, pg->pghw_cpulist_len, cpu2id);
		}
		cu_pg_update(pg);
		mutex_exit(&cpu_lock);
	}

	pgsp->pg_generation.value.ui32 = pg->pghw_kstat_gen;
	pgsp->pg_hw_util.value.ui64 = hw_util->pghw_util;
	pgsp->pg_hw_util_time_running.value.ui64 = hw_util->pghw_time_running;
	pgsp->pg_hw_util_time_stopped.value.ui64 = hw_util->pghw_time_stopped;
	pgsp->pg_hw_util_rate.value.ui64 = hw_util->pghw_rate;
	pgsp->pg_hw_util_rate_max.value.ui64 = hw_util->pghw_rate_max;
	if (pg->pghw_cpulist != NULL)
		kstat_named_setstr(&pgsp->pg_cpus, pg->pghw_cpulist);
	else
		kstat_named_setstr(&pgsp->pg_cpus, "");

	kstat_named_setstr(&pgsp->pg_sharing, pghw_type_string(pg->pghw_hw));

	return (0);
}

/*
 * Update the string representation of CPUs in PG (pg->pghw_cpulist).
 * The string representation is used for kstats.
 *
 * The string is allocated if it has not already been or if it is already
 * allocated and PG has more CPUs now. If PG has smaller or equal number of
 * CPUs, but the actual CPUs may have changed, the string is reset to the empty
 * string causes the string representation to be recreated. The pghw_generation
 * field is used to detect whether CPUs within the pg may have changed.
 */
static void
pghw_cpulist_alloc(pghw_t *pg)
{
	uint_t	ncpus = GROUP_SIZE(&((pg_t *)pg)->pg_cpus);
	size_t	len = CPUSTR_LEN(ncpus);

	/*
	 * If the pghw_cpulist string is already allocated we need to make sure
	 * that it has sufficient length. Also if the set of CPUs may have
	 * changed, we need to re-generate the string.
	 */
	if (pg->pghw_cpulist != NULL &&
	    pg->pghw_kstat_gen != pg->pghw_generation) {
		if (len <= pg->pghw_cpulist_len) {
			/*
			 * There is sufficient space in the pghw_cpulist for
			 * the new set of CPUs. Just clear the string to trigger
			 * re-generation of list of CPUs
			 */
			*(pg->pghw_cpulist) = '\0';
		} else {
			/*
			 * There is, potentially, insufficient space in
			 * pghw_cpulist, so reallocate the string.
			 */
			ASSERT(strlen(pg->pghw_cpulist) < pg->pghw_cpulist_len);
			kmem_free(pg->pghw_cpulist, pg->pghw_cpulist_len);
			pg->pghw_cpulist = NULL;
			pg->pghw_cpulist_len = 0;
		}
	}

	if (pg->pghw_cpulist == NULL) {
		/*
		 * Allocate space to hold cpulist.
		 *
		 * Length can not be bigger that the maximum space we have
		 * allowed for the kstat buffer
		 */
		if (len > pg_cpulist_maxlen)
			len = pg_cpulist_maxlen;
		if (len > 0) {
			pg->pghw_cpulist = kmem_zalloc(len, KM_NOSLEEP);
			if (pg->pghw_cpulist != NULL)
				pg->pghw_cpulist_len = len;
		}
	}
}

static int
cpu2id(void *v)
{
	cpu_t *cp = (cpu_t *)v;

	ASSERT(v != NULL);

	return (cp->cpu_id);
}