common/dtrace/dcpc.c

/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */

/*
 * Copyright 2010 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 */

#include <sys/errno.h>
#include <sys/cpuvar.h>
#include <sys/stat.h>
#include <sys/modctl.h>
#include <sys/cmn_err.h>
#include <sys/ddi.h>
#include <sys/sunddi.h>
#include <sys/ksynch.h>
#include <sys/conf.h>
#include <sys/kmem.h>
#include <sys/kcpc.h>
#include <sys/cap_util.h>
#include <sys/cpc_pcbe.h>
#include <sys/cpc_impl.h>
#include <sys/dtrace_impl.h>

/*
 * DTrace CPU Performance Counter Provider
 * ---------------------------------------
 *
 * The DTrace cpc provider allows DTrace consumers to access the CPU
 * performance counter overflow mechanism of a CPU. The configuration
 * presented in a probe specification is programmed into the performance
 * counter hardware of all available CPUs on a system. Programming the
 * hardware causes a counter on each CPU to begin counting events of the
 * given type. When the specified number of events have occurred, an overflow
 * interrupt will be generated and the probe is fired.
 *
 * The required configuration for the performance counter is encoded into
 * the probe specification and this includes the performance counter event
 * name, processor mode, overflow rate and an optional unit mask.
 *
 * Most processors provide several counters (PICs) which can count all or a
 * subset of the events available for a given CPU. However, when overflow
 * profiling is being used, not all CPUs can detect which counter generated the
 * overflow interrupt. In this case we cannot reliably determine which counter
 * overflowed and we therefore only allow such CPUs to configure one event at
 * a time. Processors that can determine the counter which overflowed are
 * allowed to program as many events at one time as possible (in theory up to
 * the number of instrumentation counters supported by that platform).
 * Therefore, multiple consumers can enable multiple probes at the same time
 * on such platforms. Platforms which cannot determine the source of an
 * overflow interrupt are only allowed to program a single event at one time.
 *
 * The performance counter hardware is made available to consumers on a
 * first-come, first-served basis. Only a finite amount of hardware resource
 * is available and, while we make every attempt to accomodate requests from
 * consumers, we must deny requests when hardware resources have been exhausted.
 * A consumer will fail to enable probes when resources are currently in use.
 *
 * The cpc provider contends for shared hardware resources along with other
 * consumers of the kernel CPU performance counter subsystem (e.g. cpustat(1M)).
 * Only one such consumer can use the performance counters at any one time and
 * counters are made available on a first-come, first-served basis. As with
 * cpustat, the cpc provider has priority over per-LWP libcpc usage (e.g.
 * cputrack(1)). Invoking the cpc provider will cause all existing per-LWP
 * counter contexts to be invalidated.
 */

typedef struct dcpc_probe {
	char		dcpc_event_name[CPC_MAX_EVENT_LEN];
	int		dcpc_flag;	/* flags (USER/SYS) */
	uint32_t	dcpc_ovfval;	/* overflow value */
	int64_t		dcpc_umask;	/* umask/emask for this event */
	int		dcpc_picno;	/* pic this event is programmed in */
	int		dcpc_enabled;	/* probe is actually enabled? */
	int		dcpc_disabling;	/* probe is currently being disabled */
	dtrace_id_t	dcpc_id;	/* probeid this request is enabling */
	int		dcpc_actv_req_idx;	/* idx into dcpc_actv_reqs[] */
} dcpc_probe_t;

static dev_info_t			*dcpc_devi;
static dtrace_provider_id_t		dcpc_pid;
static dcpc_probe_t			**dcpc_actv_reqs;
static uint32_t				dcpc_enablings = 0;
static int				dcpc_ovf_mask = 0;
static int				dcpc_mult_ovf_cap = 0;
static int				dcpc_mask_type = 0;

/*
 * When the dcpc provider is loaded, dcpc_min_overflow is set to either
 * DCPC_MIN_OVF_DEFAULT or the value that dcpc-min-overflow is set to in
 * the dcpc.conf file. Decrease this value to set probes with smaller
 * overflow values. Remember that very small values could render a system
 * unusable with frequently occurring events.
 */
#define	DCPC_MIN_OVF_DEFAULT		5000
static uint32_t				dcpc_min_overflow;

static int dcpc_aframes = 0;	/* override for artificial frame setting */
#if defined(__x86)
#define	DCPC_ARTIFICIAL_FRAMES	8
#elif defined(__sparc)
#define	DCPC_ARTIFICIAL_FRAMES	2
#endif

/*
 * Called from the platform overflow interrupt handler. 'bitmap' is a mask
 * which contains the pic(s) that have overflowed.
 */
static void
dcpc_fire(uint64_t bitmap)
{
	int i;

	/*
	 * No counter was marked as overflowing. Shout about it and get out.
	 */
	if ((bitmap & dcpc_ovf_mask) == 0) {
		cmn_err(CE_NOTE, "dcpc_fire: no counter overflow found\n");
		return;
	}

	/*
	 * This is the common case of a processor that doesn't support
	 * multiple overflow events. Such systems are only allowed a single
	 * enabling and therefore we just look for the first entry in
	 * the active request array.
	 */
	if (!dcpc_mult_ovf_cap) {
		for (i = 0; i < cpc_ncounters; i++) {
			if (dcpc_actv_reqs[i] != NULL) {
				dtrace_probe(dcpc_actv_reqs[i]->dcpc_id,
				    CPU->cpu_cpcprofile_pc,
				    CPU->cpu_cpcprofile_upc, 0, 0, 0);
				return;
			}
		}
		return;
	}

	/*
	 * This is a processor capable of handling multiple overflow events.
	 * Iterate over the array of active requests and locate the counters
	 * that overflowed (note: it is possible for more than one counter to
	 * have overflowed at the same time).
	 */
	for (i = 0; i < cpc_ncounters; i++) {
		if (dcpc_actv_reqs[i] != NULL &&
		    (bitmap & (1ULL << dcpc_actv_reqs[i]->dcpc_picno))) {
			dtrace_probe(dcpc_actv_reqs[i]->dcpc_id,
			    CPU->cpu_cpcprofile_pc,
			    CPU->cpu_cpcprofile_upc, 0, 0, 0);
		}
	}
}

static void
dcpc_create_probe(dtrace_provider_id_t id, const char *probename,
    char *eventname, int64_t umask, uint32_t ovfval, char flag)
{
	dcpc_probe_t *pp;
	int nr_frames = DCPC_ARTIFICIAL_FRAMES + dtrace_mach_aframes();

	if (dcpc_aframes)
		nr_frames = dcpc_aframes;

	if (dtrace_probe_lookup(id, NULL, NULL, probename) != 0)
		return;

	pp = kmem_zalloc(sizeof (dcpc_probe_t), KM_SLEEP);
	(void) strncpy(pp->dcpc_event_name, eventname,
	    sizeof (pp->dcpc_event_name) - 1);
	pp->dcpc_event_name[sizeof (pp->dcpc_event_name) - 1] = '\0';
	pp->dcpc_flag = flag | CPC_OVF_NOTIFY_EMT;
	pp->dcpc_ovfval = ovfval;
	pp->dcpc_umask = umask;
	pp->dcpc_actv_req_idx = pp->dcpc_picno = pp->dcpc_disabling = -1;

	pp->dcpc_id = dtrace_probe_create(id, NULL, NULL, probename,
	    nr_frames, pp);
}

/*ARGSUSED*/
static void
dcpc_provide(void *arg, const dtrace_probedesc_t *desc)
{
	/*
	 * The format of a probe is:
	 *
	 *	event_name-mode-{optional_umask}-overflow_rate
	 * e.g.
	 *	DC_refill_from_system-user-0x1e-50000, or,
	 *	DC_refill_from_system-all-10000
	 *
	 */
	char *str, *end, *p;
	int i, flag = 0;
	char event[CPC_MAX_EVENT_LEN];
	long umask = -1, val = 0;
	size_t evlen, len;

	/*
	 * The 'cpc' provider offers no probes by default.
	 */
	if (desc == NULL)
		return;

	len = strlen(desc->dtpd_name);
	p = str = kmem_alloc(len + 1, KM_SLEEP);
	(void) strcpy(str, desc->dtpd_name);

	/*
	 * We have a poor man's strtok() going on here. Replace any hyphens
	 * in the the probe name with NULL characters in order to make it
	 * easy to parse the string with regular string functions.
	 */
	for (i = 0; i < len; i++) {
		if (str[i] == '-')
			str[i] = '\0';
	}

	/*
	 * The first part of the string must be either a platform event
	 * name or a generic event name.
	 */
	evlen = strlen(p);
	(void) strncpy(event, p, CPC_MAX_EVENT_LEN - 1);
	event[CPC_MAX_EVENT_LEN - 1] = '\0';

	/*
	 * The next part of the name is the mode specification. Valid
	 * settings are "user", "kernel" or "all".
	 */
	p += evlen + 1;

	if (strcmp(p, "user") == 0)
		flag |= CPC_COUNT_USER;
	else if (strcmp(p, "kernel") == 0)
		flag |= CPC_COUNT_SYSTEM;
	else if (strcmp(p, "all") == 0)
		flag |= CPC_COUNT_USER | CPC_COUNT_SYSTEM;
	else
		goto err;

	/*
	 * Next we either have a mask specification followed by an overflow
	 * rate or just an overflow rate on its own.
	 */
	p += strlen(p) + 1;
	if (p[0] == '0' && (p[1] == 'x' || p[1] == 'X')) {
		/*
		 * A unit mask can only be specified if:
		 * 1) this performance counter back end supports masks.
		 * 2) the specified event is platform specific.
		 * 3) a valid hex number is converted.
		 * 4) no extraneous characters follow the mask specification.
		 */
		if (dcpc_mask_type != 0 && strncmp(event, "PAPI", 4) != 0 &&
		    ddi_strtol(p, &end, 16, &umask) == 0 &&
		    end == p + strlen(p)) {
			p += strlen(p) + 1;
		} else {
			goto err;
		}
	}

	/*
	 * This final part must be an overflow value which has to be greater
	 * than the minimum permissible overflow rate.
	 */
	if ((ddi_strtol(p, &end, 10, &val) != 0) || end != p + strlen(p) ||
	    val < dcpc_min_overflow)
		goto err;

	/*
	 * Validate the event and create the probe.
	 */
	for (i = 0; i < cpc_ncounters; i++) {
		char *events, *cp, *p, *end;
		int found = 0, j;
		size_t llen;

		if ((events = kcpc_list_events(i)) == NULL)
			goto err;

		llen = strlen(events);
		p = cp = ddi_strdup(events, KM_NOSLEEP);
		end = cp + llen;

		for (j = 0; j < llen; j++) {
			if (cp[j] == ',')
				cp[j] = '\0';
		}

		while (p < end && found == 0) {
			if (strcmp(p, event) == 0) {
				dcpc_create_probe(dcpc_pid, desc->dtpd_name,
				    event, umask, (uint32_t)val, flag);
				found = 1;
			}
			p += strlen(p) + 1;
		}
		kmem_free(cp, llen + 1);

		if (found)
			break;
	}

err:
	kmem_free(str, len + 1);
}

/*ARGSUSED*/
static void
dcpc_destroy(void *arg, dtrace_id_t id, void *parg)
{
	dcpc_probe_t *pp = parg;

	ASSERT(pp->dcpc_enabled == 0);
	kmem_free(pp, sizeof (dcpc_probe_t));
}

/*ARGSUSED*/
static int
dcpc_mode(void *arg, dtrace_id_t id, void *parg)
{
	if (CPU->cpu_cpcprofile_pc == 0) {
		return (DTRACE_MODE_NOPRIV_DROP | DTRACE_MODE_USER);
	} else {
		return (DTRACE_MODE_NOPRIV_DROP | DTRACE_MODE_KERNEL);
	}
}

static void
dcpc_populate_set(cpu_t *c, dcpc_probe_t *pp, kcpc_set_t *set, int reqno)
{
	kcpc_set_t *oset;
	int i;

	(void) strncpy(set->ks_req[reqno].kr_event, pp->dcpc_event_name,
	    CPC_MAX_EVENT_LEN);
	set->ks_req[reqno].kr_config = NULL;
	set->ks_req[reqno].kr_index = reqno;
	set->ks_req[reqno].kr_picnum = -1;
	set->ks_req[reqno].kr_flags =  pp->dcpc_flag;

	/*
	 * If a unit mask has been specified then detect which attribute
	 * the platform needs. For now, it's either "umask" or "emask".
	 */
	if (pp->dcpc_umask >= 0) {
		set->ks_req[reqno].kr_attr =
		    kmem_zalloc(sizeof (kcpc_attr_t), KM_SLEEP);
		set->ks_req[reqno].kr_nattrs = 1;
		if (dcpc_mask_type & DCPC_UMASK)
			(void) strncpy(set->ks_req[reqno].kr_attr->ka_name,
			    "umask", 5);
		else
			(void) strncpy(set->ks_req[reqno].kr_attr->ka_name,
			    "emask", 5);
		set->ks_req[reqno].kr_attr->ka_val = pp->dcpc_umask;
	} else {
		set->ks_req[reqno].kr_attr = NULL;
		set->ks_req[reqno].kr_nattrs = 0;
	}

	/*
	 * If this probe is enabled, obtain its current countdown value
	 * and use that. The CPUs cpc context might not exist yet if we
	 * are dealing with a CPU that is just coming online.
	 */
	if (pp->dcpc_enabled && (c->cpu_cpc_ctx != NULL)) {
		oset = c->cpu_cpc_ctx->kc_set;

		for (i = 0; i < oset->ks_nreqs; i++) {
			if (strcmp(oset->ks_req[i].kr_event,
			    set->ks_req[reqno].kr_event) == 0) {
				set->ks_req[reqno].kr_preset =
				    *(oset->ks_req[i].kr_data);
			}
		}
	} else {
		set->ks_req[reqno].kr_preset = UINT64_MAX - pp->dcpc_ovfval;
	}

	set->ks_nreqs++;
}


/*
 * Create a fresh request set for the enablings represented in the
 * 'dcpc_actv_reqs' array which contains the probes we want to be
 * in the set. This can be called for several reasons:
 *
 * 1)	We are on a single or multi overflow platform and we have no
 *	current events so we can just create the set and initialize it.
 * 2)	We are on a multi-overflow platform and we already have one or
 *	more existing events and we are adding a new enabling. Create a
 *	new set and copy old requests in and then add the new request.
 * 3)	We are on a multi-overflow platform and we have just removed an
 *	enabling but we still have enablings whch are valid. Create a new
 *	set and copy in still valid requests.
 */
static kcpc_set_t *
dcpc_create_set(cpu_t *c)
{
	int i, reqno = 0;
	int active_requests = 0;
	kcpc_set_t *set;

	/*
	 * First get a count of the number of currently active requests.
	 * Note that dcpc_actv_reqs[] should always reflect which requests
	 * we want to be in the set that is to be created. It is the
	 * responsibility of the caller of dcpc_create_set() to adjust that
	 * array accordingly beforehand.
	 */
	for (i = 0; i < cpc_ncounters; i++) {
		if (dcpc_actv_reqs[i] != NULL)
			active_requests++;
	}

	set = kmem_zalloc(sizeof (kcpc_set_t), KM_SLEEP);

	set->ks_req =
	    kmem_zalloc(sizeof (kcpc_request_t) * active_requests, KM_SLEEP);

	set->ks_data =
	    kmem_zalloc(active_requests * sizeof (uint64_t), KM_SLEEP);

	/*
	 * Look for valid entries in the active requests array and populate
	 * the request set for any entries found.
	 */
	for (i = 0; i < cpc_ncounters; i++) {
		if (dcpc_actv_reqs[i] != NULL) {
			dcpc_populate_set(c, dcpc_actv_reqs[i], set, reqno);
			reqno++;
		}
	}

	return (set);
}

static int
dcpc_program_cpu_event(cpu_t *c)
{
	int i, j, subcode;
	kcpc_ctx_t *ctx, *octx;
	kcpc_set_t *set;

	set = dcpc_create_set(c);

	set->ks_ctx = ctx = kcpc_ctx_alloc(KM_SLEEP);
	ctx->kc_set = set;
	ctx->kc_cpuid = c->cpu_id;

	if (kcpc_assign_reqs(set, ctx) != 0)
		goto err;

	if (kcpc_configure_reqs(ctx, set, &subcode) != 0)
		goto err;

	for (i = 0; i < set->ks_nreqs; i++) {
		for (j = 0; j < cpc_ncounters; j++) {
			if (dcpc_actv_reqs[j] != NULL &&
			    strcmp(set->ks_req[i].kr_event,
			    dcpc_actv_reqs[j]->dcpc_event_name) == 0) {
				dcpc_actv_reqs[j]->dcpc_picno =
				    set->ks_req[i].kr_picnum;
			}
		}
	}

	/*
	 * If we already have an active enabling then save the current cpc
	 * context away.
	 */
	octx = c->cpu_cpc_ctx;

	kcpc_cpu_program(c, ctx);

	if (octx != NULL) {
		kcpc_set_t *oset = octx->kc_set;
		kmem_free(oset->ks_data, oset->ks_nreqs * sizeof (uint64_t));
		kcpc_free_configs(oset);
		kcpc_free_set(oset);
		kcpc_ctx_free(octx);
	}

	return (0);

err:
	/*
	 * We failed to configure this request up so free things up and
	 * get out.
	 */
	kcpc_free_configs(set);
	kmem_free(set->ks_data, set->ks_nreqs * sizeof (uint64_t));
	kcpc_free_set(set);
	kcpc_ctx_free(ctx);

	return (-1);
}

static void
dcpc_disable_cpu(cpu_t *c)
{
	kcpc_ctx_t *ctx;
	kcpc_set_t *set;

	/*
	 * Leave this CPU alone if it's already offline.
	 */
	if (c->cpu_flags & CPU_OFFLINE)
		return;

	/*
	 * Grab CPUs CPC context before kcpc_cpu_stop() stops counters and
	 * changes it.
	 */
	ctx = c->cpu_cpc_ctx;

	kcpc_cpu_stop(c, B_FALSE);

	set = ctx->kc_set;

	kcpc_free_configs(set);
	kmem_free(set->ks_data, set->ks_nreqs * sizeof (uint64_t));
	kcpc_free_set(set);
	kcpc_ctx_free(ctx);
}

/*
 * The dcpc_*_interrupts() routines are responsible for manipulating the
 * per-CPU dcpc interrupt state byte. The purpose of the state byte is to
 * synchronize processing of hardware overflow interrupts wth configuration
 * changes made to the CPU performance counter subsystem by the dcpc provider.
 *
 * The dcpc provider claims ownership of the overflow interrupt mechanism
 * by transitioning the state byte from DCPC_INTR_INACTIVE (indicating the
 * dcpc provider is not in use) to DCPC_INTR_FREE (the dcpc provider owns the
 * overflow mechanism and interrupts may be processed). Before modifying
 * a CPUs configuration state the state byte is transitioned from
 * DCPC_INTR_FREE to DCPC_INTR_CONFIG ("configuration in process" state).
 * The hardware overflow handler, kcpc_hw_overflow_intr(), will only process
 * an interrupt when a configuration is not in process (i.e. the state is
 * marked as free). During interrupt processing the state is set to
 * DCPC_INTR_PROCESSING by the overflow handler. When the last dcpc based
 * enabling is removed, the state byte is set to DCPC_INTR_INACTIVE to indicate
 * the dcpc provider is no longer interested in overflow interrupts.
 */
static void
dcpc_block_interrupts(void)
{
	cpu_t *c = cpu_list;
	uint8_t *state;

	ASSERT(cpu_core[c->cpu_id].cpuc_dcpc_intr_state != DCPC_INTR_INACTIVE);

	do {
		state = &cpu_core[c->cpu_id].cpuc_dcpc_intr_state;

		while (atomic_cas_8(state, DCPC_INTR_FREE,
		    DCPC_INTR_CONFIG) != DCPC_INTR_FREE)
			continue;

	} while ((c = c->cpu_next) != cpu_list);
}

/*
 * Set all CPUs dcpc interrupt state to DCPC_INTR_FREE to indicate that
 * overflow interrupts can be processed safely.
 */
static void
dcpc_release_interrupts(void)
{
	cpu_t *c = cpu_list;

	ASSERT(cpu_core[c->cpu_id].cpuc_dcpc_intr_state != DCPC_INTR_INACTIVE);

	do {
		cpu_core[c->cpu_id].cpuc_dcpc_intr_state = DCPC_INTR_FREE;
		membar_producer();
	} while ((c = c->cpu_next) != cpu_list);
}

/*
 * Transition all CPUs dcpc interrupt state from DCPC_INTR_INACTIVE to
 * to DCPC_INTR_FREE. This indicates that the dcpc provider is now
 * responsible for handling all overflow interrupt activity. Should only be
 * called before enabling the first dcpc based probe.
 */
static void
dcpc_claim_interrupts(void)
{
	cpu_t *c = cpu_list;

	ASSERT(cpu_core[c->cpu_id].cpuc_dcpc_intr_state == DCPC_INTR_INACTIVE);

	do {
		cpu_core[c->cpu_id].cpuc_dcpc_intr_state = DCPC_INTR_FREE;
		membar_producer();
	} while ((c = c->cpu_next) != cpu_list);
}

/*
 * Set all CPUs dcpc interrupt state to DCPC_INTR_INACTIVE to indicate that
 * the dcpc provider is no longer processing overflow interrupts. Only called
 * during removal of the last dcpc based enabling.
 */
static void
dcpc_surrender_interrupts(void)
{
	cpu_t *c = cpu_list;

	ASSERT(cpu_core[c->cpu_id].cpuc_dcpc_intr_state != DCPC_INTR_INACTIVE);

	do {
		cpu_core[c->cpu_id].cpuc_dcpc_intr_state = DCPC_INTR_INACTIVE;
		membar_producer();
	} while ((c = c->cpu_next) != cpu_list);
}

/*
 * dcpc_program_event() can be called owing to a new enabling or if a multi
 * overflow platform has disabled a request but needs to  program the requests
 * that are still valid.
 *
 * Every invocation of dcpc_program_event() will create a new kcpc_ctx_t
 * and a new request set which contains the new enabling and any old enablings
 * which are still valid (possible with multi-overflow platforms).
 */
static int
dcpc_program_event(dcpc_probe_t *pp)
{
	cpu_t *c;
	int ret = 0;

	ASSERT(MUTEX_HELD(&cpu_lock));

	kpreempt_disable();

	dcpc_block_interrupts();

	c = cpu_list;

	do {
		/*
		 * Skip CPUs that are currently offline.
		 */
		if (c->cpu_flags & CPU_OFFLINE)
			continue;

		/*
		 * Stop counters but preserve existing DTrace CPC context
		 * if there is one.
		 *
		 * If we come here when the first event is programmed for a CPU,
		 * there should be no DTrace CPC context installed. In this
		 * case, kcpc_cpu_stop() will ensure that there is no other
		 * context on the CPU.
		 *
		 * If we add new enabling to the original one, the CPU should
		 * have the old DTrace CPC context which we need to keep around
		 * since dcpc_program_event() will add to it.
		 */
		if (c->cpu_cpc_ctx != NULL)
			kcpc_cpu_stop(c, B_TRUE);
	} while ((c = c->cpu_next) != cpu_list);

	dcpc_release_interrupts();

	/*
	 * If this enabling is being removed (in the case of a multi event
	 * capable system with more than one active enabling), we can now
	 * update the active request array to reflect the enablings that need
	 * to be reprogrammed.
	 */
	if (pp->dcpc_disabling == 1)
		dcpc_actv_reqs[pp->dcpc_actv_req_idx] = NULL;

	do {
		/*
		 * Skip CPUs that are currently offline.
		 */
		if (c->cpu_flags & CPU_OFFLINE)
			continue;

		ret = dcpc_program_cpu_event(c);
	} while ((c = c->cpu_next) != cpu_list && ret == 0);

	/*
	 * If dcpc_program_cpu_event() fails then it is because we couldn't
	 * configure the requests in the set for the CPU and not because of
	 * an error programming the hardware. If we have a failure here then
	 * we assume no CPUs have been programmed in the above step as they
	 * are all configured identically.
	 */
	if (ret != 0) {
		pp->dcpc_enabled = 0;
		kpreempt_enable();
		return (-1);
	}

	if (pp->dcpc_disabling != 1)
		pp->dcpc_enabled = 1;

	kpreempt_enable();

	return (0);
}

/*ARGSUSED*/
static int
dcpc_enable(void *arg, dtrace_id_t id, void *parg)
{
	dcpc_probe_t *pp = parg;
	int i, found = 0;
	cpu_t *c;

	ASSERT(MUTEX_HELD(&cpu_lock));

	/*
	 * Bail out if the counters are being used by a libcpc consumer.
	 */
	rw_enter(&kcpc_cpuctx_lock, RW_READER);
	if (kcpc_cpuctx > 0) {
		rw_exit(&kcpc_cpuctx_lock);
		return (-1);
	}

	dtrace_cpc_in_use++;
	rw_exit(&kcpc_cpuctx_lock);

	/*
	 * Locate this enabling in the first free entry of the active
	 * request array.
	 */
	for (i = 0; i < cpc_ncounters; i++) {
		if (dcpc_actv_reqs[i] == NULL) {
			dcpc_actv_reqs[i] = pp;
			pp->dcpc_actv_req_idx = i;
			found = 1;
			break;
		}
	}

	/*
	 * If we couldn't find a slot for this probe then there is no
	 * room at the inn.
	 */
	if (!found) {
		dtrace_cpc_in_use--;
		return (-1);
	}

	ASSERT(pp->dcpc_actv_req_idx >= 0);

	/*
	 * DTrace is taking over CPC contexts, so stop collecting
	 * capacity/utilization data for all CPUs.
	 */
	if (dtrace_cpc_in_use == 1)
		cu_disable();

	/*
	 * The following must hold true if we are to (attempt to) enable
	 * this request:
	 *
	 * 1) No enablings currently exist. We allow all platforms to
	 * proceed if this is true.
	 *
	 * OR
	 *
	 * 2) If the platform is multi overflow capable and there are
	 * less valid enablings than there are counters. There is no
	 * guarantee that a platform can accommodate as many events as
	 * it has counters for but we will at least try to program
	 * up to that many requests.
	 *
	 * The 'dcpc_enablings' variable is implictly protected by locking
	 * provided by the DTrace framework and the cpu management framework.
	 */
	if (dcpc_enablings == 0 || (dcpc_mult_ovf_cap &&
	    dcpc_enablings < cpc_ncounters)) {
		/*
		 * Before attempting to program the first enabling we need to
		 * invalidate any lwp-based contexts and lay claim to the
		 * overflow interrupt mechanism.
		 */
		if (dcpc_enablings == 0) {
			kcpc_invalidate_all();
			dcpc_claim_interrupts();
		}

		if (dcpc_program_event(pp) == 0) {
			dcpc_enablings++;
			return (0);
		}
	}

	/*
	 * If active enablings existed before we failed to enable this probe
	 * on a multi event capable platform then we need to restart counters
	 * as they will have been stopped in the attempted configuration. The
	 * context should now just contain the request prior to this failed
	 * enabling.
	 */
	if (dcpc_enablings > 0 && dcpc_mult_ovf_cap) {
		c = cpu_list;

		ASSERT(dcpc_mult_ovf_cap == 1);
		do {
			/*
			 * Skip CPUs that are currently offline.
			 */
			if (c->cpu_flags & CPU_OFFLINE)
				continue;

			kcpc_cpu_program(c, c->cpu_cpc_ctx);
		} while ((c = c->cpu_next) != cpu_list);
	}

	/*
	 * Give up any claim to the overflow interrupt mechanism if no
	 * dcpc based enablings exist.
	 */
	if (dcpc_enablings == 0)
		dcpc_surrender_interrupts();

	dtrace_cpc_in_use--;
	dcpc_actv_reqs[pp->dcpc_actv_req_idx] = NULL;
	pp->dcpc_actv_req_idx = pp->dcpc_picno = -1;

	/*
	 * If all probes are removed, enable capacity/utilization data
	 * collection for every CPU.
	 */
	if (dtrace_cpc_in_use == 0)
		cu_enable();

	return (-1);
}

/*
 * If only one enabling is active then remove the context and free
 * everything up. If there are multiple enablings active then remove this
 * one, its associated meta-data and re-program the hardware.
 */
/*ARGSUSED*/
static void
dcpc_disable(void *arg, dtrace_id_t id, void *parg)
{
	cpu_t *c;
	dcpc_probe_t *pp = parg;

	ASSERT(MUTEX_HELD(&cpu_lock));

	kpreempt_disable();

	/*
	 * This probe didn't actually make it as far as being fully enabled
	 * so we needn't do anything with it.
	 */
	if (pp->dcpc_enabled == 0) {
		/*
		 * If we actually allocated this request a slot in the
		 * request array but failed to enabled it then remove the
		 * entry in the array.
		 */
		if (pp->dcpc_actv_req_idx >= 0) {
			dcpc_actv_reqs[pp->dcpc_actv_req_idx] = NULL;
			pp->dcpc_actv_req_idx = pp->dcpc_picno =
			    pp->dcpc_disabling = -1;
		}

		kpreempt_enable();
		return;
	}

	/*
	 * If this is the only enabling then stop all the counters and
	 * free up the meta-data.
	 */
	if (dcpc_enablings == 1) {
		ASSERT(dtrace_cpc_in_use == 1);

		dcpc_block_interrupts();

		c = cpu_list;

		do {
			dcpc_disable_cpu(c);
		} while ((c = c->cpu_next) != cpu_list);

		dcpc_actv_reqs[pp->dcpc_actv_req_idx] = NULL;
		dcpc_surrender_interrupts();
	} else {
		/*
		 * This platform can support multiple overflow events and
		 * the enabling being disabled is not the last one. Remove this
		 * enabling and re-program the hardware with the new config.
		 */
		ASSERT(dcpc_mult_ovf_cap);
		ASSERT(dcpc_enablings > 1);

		pp->dcpc_disabling = 1;
		(void) dcpc_program_event(pp);
	}

	kpreempt_enable();

	dcpc_enablings--;
	dtrace_cpc_in_use--;
	pp->dcpc_enabled = 0;
	pp->dcpc_actv_req_idx = pp->dcpc_picno = pp->dcpc_disabling = -1;

	/*
	 * If all probes are removed, enable capacity/utilization data
	 * collection for every CPU
	 */
	if (dtrace_cpc_in_use == 0)
		cu_enable();
}

/*ARGSUSED*/
static int
dcpc_cpu_setup(cpu_setup_t what, processorid_t cpu, void *arg)
{
	cpu_t *c;
	uint8_t *state;

	ASSERT(MUTEX_HELD(&cpu_lock));

	switch (what) {
	case CPU_OFF:
		/*
		 * Offline CPUs are not allowed to take part so remove this
		 * CPU if we are actively tracing.
		 */
		if (dtrace_cpc_in_use) {
			c = cpu_get(cpu);
			state = &cpu_core[c->cpu_id].cpuc_dcpc_intr_state;

			/*
			 * Indicate that a configuration is in process in
			 * order to stop overflow interrupts being processed
			 * on this CPU while we disable it.
			 */
			while (atomic_cas_8(state, DCPC_INTR_FREE,
			    DCPC_INTR_CONFIG) != DCPC_INTR_FREE)
				continue;

			dcpc_disable_cpu(c);

			/*
			 * Reset this CPUs interrupt state as the configuration
			 * has ended.
			 */
			cpu_core[c->cpu_id].cpuc_dcpc_intr_state =
			    DCPC_INTR_FREE;
			membar_producer();
		}
		break;

	case CPU_ON:
	case CPU_SETUP:
		/*
		 * This CPU is being initialized or brought online so program
		 * it with the current request set if we are actively tracing.
		 */
		if (dtrace_cpc_in_use) {
			c = cpu_get(cpu);
			(void) dcpc_program_cpu_event(c);
		}
		break;

	default:
		break;
	}

	return (0);
}

static dtrace_pattr_t dcpc_attr = {
{ DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_COMMON },
{ DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN },
{ DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN },
{ DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_CPU },
{ DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_COMMON },
};

static dtrace_pops_t dcpc_pops = {
    dcpc_provide,
    NULL,
    dcpc_enable,
    dcpc_disable,
    NULL,
    NULL,
    NULL,
    NULL,
    dcpc_mode,
    dcpc_destroy
};

/*ARGSUSED*/
static int
dcpc_open(dev_t *devp, int flag, int otyp, cred_t *cred_p)
{
	return (0);
}

/*ARGSUSED*/
static int
dcpc_info(dev_info_t *dip, ddi_info_cmd_t infocmd, void *arg, void **result)
{
	int error;

	switch (infocmd) {
	case DDI_INFO_DEVT2DEVINFO:
		*result = (void *)dcpc_devi;
		error = DDI_SUCCESS;
		break;
	case DDI_INFO_DEVT2INSTANCE:
		*result = (void *)0;
		error = DDI_SUCCESS;
		break;
	default:
		error = DDI_FAILURE;
	}
	return (error);
}

static int
dcpc_detach(dev_info_t *devi, ddi_detach_cmd_t cmd)
{
	switch (cmd) {
	case DDI_DETACH:
		break;
	case DDI_SUSPEND:
		return (DDI_SUCCESS);
	default:
		return (DDI_FAILURE);
	}

	if (dtrace_unregister(dcpc_pid) != 0)
		return (DDI_FAILURE);

	ddi_remove_minor_node(devi, NULL);

	mutex_enter(&cpu_lock);
	unregister_cpu_setup_func(dcpc_cpu_setup, NULL);
	mutex_exit(&cpu_lock);

	kmem_free(dcpc_actv_reqs, cpc_ncounters * sizeof (dcpc_probe_t *));

	kcpc_unregister_dcpc();

	return (DDI_SUCCESS);
}

static int
dcpc_attach(dev_info_t *devi, ddi_attach_cmd_t cmd)
{
	uint_t caps;
	char *attrs;

	switch (cmd) {
	case DDI_ATTACH:
		break;
	case DDI_RESUME:
		return (DDI_SUCCESS);
	default:
		return (DDI_FAILURE);
	}

	if (kcpc_pcbe_loaded() == -1)
		return (DDI_FAILURE);

	caps = kcpc_pcbe_capabilities();

	if (!(caps & CPC_CAP_OVERFLOW_INTERRUPT)) {
		cmn_err(CE_NOTE, "!dcpc: Counter Overflow not supported"\
		    " on this processor");
		return (DDI_FAILURE);
	}

	if (ddi_create_minor_node(devi, "dcpc", S_IFCHR, 0,
	    DDI_PSEUDO, NULL) == DDI_FAILURE ||
	    dtrace_register("cpc", &dcpc_attr, DTRACE_PRIV_KERNEL,
	    NULL, &dcpc_pops, NULL, &dcpc_pid) != 0) {
		ddi_remove_minor_node(devi, NULL);
		return (DDI_FAILURE);
	}

	mutex_enter(&cpu_lock);
	register_cpu_setup_func(dcpc_cpu_setup, NULL);
	mutex_exit(&cpu_lock);

	dcpc_ovf_mask = (1 << cpc_ncounters) - 1;
	ASSERT(dcpc_ovf_mask != 0);

	if (caps & CPC_CAP_OVERFLOW_PRECISE)
		dcpc_mult_ovf_cap = 1;

	/*
	 * Determine which, if any, mask attribute the back-end can use.
	 */
	attrs = kcpc_list_attrs();
	if (strstr(attrs, "umask") != NULL)
		dcpc_mask_type |= DCPC_UMASK;
	else if (strstr(attrs, "emask") != NULL)
		dcpc_mask_type |= DCPC_EMASK;

	/*
	 * The dcpc_actv_reqs array is used to store the requests that
	 * we currently have programmed. The order of requests in this
	 * array is not necessarily the order that the event appears in
	 * the kcpc_request_t array. Once entered into a slot in the array
	 * the entry is not moved until it's removed.
	 */
	dcpc_actv_reqs =
	    kmem_zalloc(cpc_ncounters * sizeof (dcpc_probe_t *), KM_SLEEP);

	dcpc_min_overflow = ddi_prop_get_int(DDI_DEV_T_ANY, devi,
	    DDI_PROP_DONTPASS, "dcpc-min-overflow", DCPC_MIN_OVF_DEFAULT);

	kcpc_register_dcpc(dcpc_fire);

	ddi_report_dev(devi);
	dcpc_devi = devi;

	return (DDI_SUCCESS);
}

static struct cb_ops dcpc_cb_ops = {
	dcpc_open,		/* open */
	nodev,			/* close */
	nulldev,		/* strategy */
	nulldev,		/* print */
	nodev,			/* dump */
	nodev,			/* read */
	nodev,			/* write */
	nodev,			/* ioctl */
	nodev,			/* devmap */
	nodev,			/* mmap */
	nodev,			/* segmap */
	nochpoll,		/* poll */
	ddi_prop_op,		/* cb_prop_op */
	0,			/* streamtab  */
	D_NEW | D_MP		/* Driver compatibility flag */
};

static struct dev_ops dcpc_ops = {
	DEVO_REV,		/* devo_rev, */
	0,			/* refcnt  */
	dcpc_info,		/* get_dev_info */
	nulldev,		/* identify */
	nulldev,		/* probe */
	dcpc_attach,		/* attach */
	dcpc_detach,		/* detach */
	nodev,			/* reset */
	&dcpc_cb_ops,		/* driver operations */
	NULL,			/* bus operations */
	nodev,			/* dev power */
	ddi_quiesce_not_needed	/* quiesce */
};

/*
 * Module linkage information for the kernel.
 */
static struct modldrv modldrv = {
	&mod_driverops,		/* module type */
	"DTrace CPC Module",	/* name of module */
	&dcpc_ops,		/* driver ops */
};

static struct modlinkage modlinkage = {
	MODREV_1,
	(void *)&modldrv,
	NULL
};

int
_init(void)
{
	return (mod_install(&modlinkage));
}

int
_info(struct modinfo *modinfop)
{
	return (mod_info(&modlinkage, modinfop));
}

int
_fini(void)
{
	return (mod_remove(&modlinkage));
}