62  * These are set when a PCBE module is loaded.
84 static int kcpc_tryassign(kcpc_set_t *set, int starting_req, int *scratch);
85 static kcpc_set_t *kcpc_dup_set(kcpc_set_t *set);
181 kcpc_bind_cpu(kcpc_set_t *set, processorid_t cpuid, int *subcode)  in kcpc_bind_cpu()  argument
190 	if (kcpc_assign_reqs(set, ctx) != 0) {  in kcpc_bind_cpu()
199 	set->ks_data = kmem_zalloc(set->ks_nreqs * sizeof (uint64_t), KM_SLEEP);  in kcpc_bind_cpu()
201 	if ((error = kcpc_configure_reqs(ctx, set, subcode)) != 0) {  in kcpc_bind_cpu()
202 		kmem_free(set->ks_data, set->ks_nreqs * sizeof (uint64_t));  in kcpc_bind_cpu()
207 	set->ks_ctx = ctx;  in kcpc_bind_cpu()
208 	ctx->kc_set = set;  in kcpc_bind_cpu()
220 		 * The CPU could have been DRd out while we were getting set up.  in kcpc_bind_cpu()
235 		 * If this CPU already has a bound set, return an error.  in kcpc_bind_cpu()
258 	mutex_enter(&set->ks_lock);  in kcpc_bind_cpu()
259 	set->ks_state |= KCPC_SET_BOUND;  in kcpc_bind_cpu()
260 	cv_signal(&set->ks_condv);  in kcpc_bind_cpu()
261 	mutex_exit(&set->ks_lock);  in kcpc_bind_cpu()
267 	set->ks_ctx = NULL;  in kcpc_bind_cpu()
268 	kmem_free(set->ks_data, set->ks_nreqs * sizeof (uint64_t));  in kcpc_bind_cpu()
274 kcpc_bind_thread(kcpc_set_t *set, kthread_t *t, int *subcode)  in kcpc_bind_thread()  argument
280 	 * Only one set is allowed per context, so ensure there is no  in kcpc_bind_thread()
297 	if (kcpc_assign_reqs(set, ctx) != 0) {  in kcpc_bind_thread()
304 	if (set->ks_flags & CPC_BIND_LWP_INHERIT)  in kcpc_bind_thread()
314 	 * Create the data store for this set.  in kcpc_bind_thread()
316 	set->ks_data = kmem_alloc(set->ks_nreqs * sizeof (uint64_t), KM_SLEEP);  in kcpc_bind_thread()
318 	if ((error = kcpc_configure_reqs(ctx, set, subcode)) != 0) {  in kcpc_bind_thread()
319 		kmem_free(set->ks_data, set->ks_nreqs * sizeof (uint64_t));  in kcpc_bind_thread()
325 	set->ks_ctx = ctx;  in kcpc_bind_thread()
326 	ctx->kc_set = set;  in kcpc_bind_thread()
356 	mutex_enter(&set->ks_lock);  in kcpc_bind_thread()
357 	set->ks_state |= KCPC_SET_BOUND;  in kcpc_bind_thread()
358 	cv_signal(&set->ks_condv);  in kcpc_bind_thread()
359 	mutex_exit(&set->ks_lock);  in kcpc_bind_thread()
365  * Walk through each request in the set and ask the PCBE to configure a
369 kcpc_configure_reqs(kcpc_ctx_t *ctx, kcpc_set_t *set, int *subcode)  in kcpc_configure_reqs()  argument
375 	for (i = 0; i < set->ks_nreqs; i++) {  in kcpc_configure_reqs()
377 		rp = &set->ks_req[i];  in kcpc_configure_reqs()
403 			kcpc_free_configs(set);  in kcpc_configure_reqs()
416 		rp->kr_data = set->ks_data + rp->kr_index;  in kcpc_configure_reqs()
424 kcpc_free_configs(kcpc_set_t *set)  in kcpc_free_configs()  argument
428 	for (i = 0; i < set->ks_nreqs; i++)  in kcpc_free_configs()
429 		if (set->ks_req[i].kr_config != NULL)  in kcpc_free_configs()
430 			pcbe_ops->pcbe_free(set->ks_req[i].kr_config);  in kcpc_free_configs()
438 kcpc_sample(kcpc_set_t *set, uint64_t *buf, hrtime_t *hrtime, uint64_t *tick)  in kcpc_sample()  argument
440 	kcpc_ctx_t	*ctx = set->ks_ctx;  in kcpc_sample()
443 	mutex_enter(&set->ks_lock);  in kcpc_sample()
444 	if ((set->ks_state & KCPC_SET_BOUND) == 0) {  in kcpc_sample()
445 		mutex_exit(&set->ks_lock);  in kcpc_sample()
448 	mutex_exit(&set->ks_lock);  in kcpc_sample()
497 	if (copyout(set->ks_data, buf,  in kcpc_sample()
498 	    set->ks_nreqs * sizeof (uint64_t)) == -1)  in kcpc_sample()
531 kcpc_unbind(kcpc_set_t *set)  in kcpc_unbind()  argument
538 	 * binds the set; we must wait for the set to finish binding  in kcpc_unbind()
541 	mutex_enter(&set->ks_lock);  in kcpc_unbind()
542 	while ((set->ks_state & KCPC_SET_BOUND) == 0)  in kcpc_unbind()
543 		cv_wait(&set->ks_condv, &set->ks_lock);  in kcpc_unbind()
544 	mutex_exit(&set->ks_lock);  in kcpc_unbind()
546 	ctx = set->ks_ctx;  in kcpc_unbind()
577 		 * If we are unbinding a CPU-bound set from a remote CPU, the  in kcpc_unbind()
611 kcpc_preset(kcpc_set_t *set, int index, uint64_t preset)  in kcpc_preset()  argument
615 	ASSERT(set != NULL);  in kcpc_preset()
616 	ASSERT(set->ks_state & KCPC_SET_BOUND);  in kcpc_preset()
617 	ASSERT(set->ks_ctx->kc_thread == curthread);  in kcpc_preset()
618 	ASSERT(set->ks_ctx->kc_cpuid == -1);  in kcpc_preset()
620 	if (index < 0 || index >= set->ks_nreqs)  in kcpc_preset()
623 	for (i = 0; i < set->ks_nreqs; i++)  in kcpc_preset()
624 		if (set->ks_req[i].kr_index == index)  in kcpc_preset()
626 	ASSERT(i != set->ks_nreqs);  in kcpc_preset()
628 	set->ks_req[i].kr_preset = preset;  in kcpc_preset()
633 kcpc_restart(kcpc_set_t *set)  in kcpc_restart()  argument
635 	kcpc_ctx_t	*ctx = set->ks_ctx;  in kcpc_restart()
639 	ASSERT(set->ks_state & KCPC_SET_BOUND);  in kcpc_restart()
643 	for (i = 0; i < set->ks_nreqs; i++) {  in kcpc_restart()
644 		*(set->ks_req[i].kr_data) = set->ks_req[i].kr_preset;  in kcpc_restart()
645 		pcbe_ops->pcbe_configure(0, NULL, set->ks_req[i].kr_preset,  in kcpc_restart()
646 		    0, 0, NULL, &set->ks_req[i].kr_config, NULL);  in kcpc_restart()
653 	 * If the user is doing this on a running set, make sure the counters  in kcpc_restart()
678 	kcpc_set_t	*set = t->t_cpc_set;  in kcpc_enable()  local
688 		 * This thread has a set but no context; it must be a  in kcpc_enable()
689 		 * CPU-bound set.  in kcpc_enable()
713 		 * Strategy for usr/sys: stop counters and update set's presets  in kcpc_enable()
726 		for (i = 0; i < set->ks_nreqs; i++) {  in kcpc_enable()
727 			set->ks_req[i].kr_preset = *(set->ks_req[i].kr_data);  in kcpc_enable()
729 				set->ks_req[i].kr_flags |= flag;  in kcpc_enable()
731 				set->ks_req[i].kr_flags &= ~flag;  in kcpc_enable()
733 		newset = kcpc_dup_set(set);  in kcpc_enable()
734 		if (kcpc_unbind(set) != 0)  in kcpc_enable()
837  * Copy set from ctx to the child context, cctx, if it has CPC_BIND_LWP_INHERIT
946 		 * Note that t_lwp is always set to point at the underlying  in kcpc_overflow_intr()
990 		 * set when we enter the handler. This variable is unset after  in kcpc_overflow_intr()
1079 		 * Set the per-CPU state bit to indicate that we are currently  in kcpc_hw_overflow_intr()
1117 			 * We've finished processing the interrupt so set  in kcpc_hw_overflow_intr()
1292 	 * Set kc_flags to show that a kcpc_restore() is in progress to avoid  in kcpc_restore()
1293 	 * ctx & set related memory objects being freed without us knowing.  in kcpc_restore()
1324  * If kcpc_counts_include_idle is set to 0 by the sys admin, we add the the
1429 		 * and SIGOVF flags set. In addition, all counters should be  in kcpc_lwp_create()
1430 		 * set to UINT64_MAX, and their pic's overflow flag turned on  in kcpc_lwp_create()
1458  * 4) A bound set is unbound.
1488 	kcpc_set_t	*set = ctx->kc_set;  in kcpc_free()  local
1490 	ASSERT(set != NULL);  in kcpc_free()
1554 	 * Walk through each request in this context's set and free the PCBE's  in kcpc_free()
1557 	for (i = 0; i < set->ks_nreqs; i++) {  in kcpc_free()
1558 		if (set->ks_req[i].kr_config != NULL)  in kcpc_free()
1559 			pcbe_ops->pcbe_free(set->ks_req[i].kr_config);  in kcpc_free()
1562 	kmem_free(set->ks_data, set->ks_nreqs * sizeof (uint64_t));  in kcpc_free()
1564 	kcpc_free_set(set);  in kcpc_free()
1574  * Free the memory associated with a request set.
1577 kcpc_free_set(kcpc_set_t *set)  in kcpc_free_set()  argument
1582 	ASSERT(set->ks_req != NULL);  in kcpc_free_set()
1584 	for (i = 0; i < set->ks_nreqs; i++) {  in kcpc_free_set()
1585 		req = &set->ks_req[i];  in kcpc_free_set()
1593 	kmem_free(set->ks_req, sizeof (kcpc_request_t) * set->ks_nreqs);  in kcpc_free_set()
1594 	cv_destroy(&set->ks_condv);  in kcpc_free_set()
1595 	mutex_destroy(&set->ks_lock);  in kcpc_free_set()
1596 	kmem_free(set, sizeof (kcpc_set_t));  in kcpc_free_set()
1637 	kcpc_set_t *set = curthread->t_cpc_set;  in kcpc_passivate()  local
1640 	if (set == NULL)  in kcpc_passivate()
1645 		 * This thread has a set but no context; it must be a CPU-bound  in kcpc_passivate()
1646 		 * set. The hardware will be stopped via kcpc_unbind() when the  in kcpc_passivate()
1649 		 * state; the set will be freed with the unbind().  in kcpc_passivate()
1651 		(void) kcpc_unbind(set);  in kcpc_passivate()
1653 		 * Unbinding a set belonging to the current thread should clear  in kcpc_passivate()
1654 		 * its set pointer.  in kcpc_passivate()
1679 	 * CPC pointers left behind. The context and set will be freed by  in kcpc_passivate()
1689  * Assign the requests in the given set to the PICs in the context.
1694 kcpc_assign_reqs(kcpc_set_t *set, kcpc_ctx_t *ctx)  in kcpc_assign_reqs()  argument
1699 	ASSERT(set->ks_nreqs <= cpc_ncounters);  in kcpc_assign_reqs()
1705 	picnum_save = kmem_alloc(set->ks_nreqs * sizeof (int), KM_SLEEP);  in kcpc_assign_reqs()
1707 	 * kcpc_tryassign() blindly walks through each request in the set,  in kcpc_assign_reqs()
1715 	for (i = 0; i < set->ks_nreqs; i++)  in kcpc_assign_reqs()
1716 		if (kcpc_tryassign(set, i, picnum_save) == 0)  in kcpc_assign_reqs()
1719 	kmem_free(picnum_save, set->ks_nreqs * sizeof (int));  in kcpc_assign_reqs()
1720 	if (i == set->ks_nreqs)  in kcpc_assign_reqs()
1726 kcpc_tryassign(kcpc_set_t *set, int starting_req, int *scratch)  in kcpc_tryassign()  argument
1743 	for (i = 0; i < set->ks_nreqs; i++) {  in kcpc_tryassign()
1744 		scratch[i] = set->ks_req[i].kr_picnum;  in kcpc_tryassign()
1745 		if (set->ks_req[i].kr_picnum != -1)  in kcpc_tryassign()
1746 			resmap |= (1 << set->ks_req[i].kr_picnum);  in kcpc_tryassign()
1755 		if (set->ks_req[i].kr_picnum != -1) {  in kcpc_tryassign()
1756 			ASSERT((bitmap & (1 << set->ks_req[i].kr_picnum)) == 0);  in kcpc_tryassign()
1757 			bitmap |= (1 << set->ks_req[i].kr_picnum);  in kcpc_tryassign()
1758 			if (++i == set->ks_nreqs)  in kcpc_tryassign()
1763 		ctrmap = pcbe_ops->pcbe_event_coverage(set->ks_req[i].kr_event);  in kcpc_tryassign()
1779 			for (i = 0; i < set->ks_nreqs; i++)  in kcpc_tryassign()
1780 				set->ks_req[i].kr_picnum = scratch[i];  in kcpc_tryassign()
1783 		set->ks_req[i].kr_picnum = j;  in kcpc_tryassign()
1785 		if (++i == set->ks_nreqs)  in kcpc_tryassign()
1793 kcpc_dup_set(kcpc_set_t *set)  in kcpc_dup_set()  argument
1801 	new->ks_flags = set->ks_flags;  in kcpc_dup_set()
1802 	new->ks_nreqs = set->ks_nreqs;  in kcpc_dup_set()
1803 	new->ks_req = kmem_alloc(set->ks_nreqs * sizeof (kcpc_request_t),  in kcpc_dup_set()
1810 		new->ks_req[i].kr_index = set->ks_req[i].kr_index;  in kcpc_dup_set()
1811 		new->ks_req[i].kr_picnum = set->ks_req[i].kr_picnum;  in kcpc_dup_set()
1814 		(void) strncpy(new->ks_req[i].kr_event, set->ks_req[i].kr_event,  in kcpc_dup_set()
1816 		new->ks_req[i].kr_preset = set->ks_req[i].kr_preset;  in kcpc_dup_set()
1817 		new->ks_req[i].kr_flags = set->ks_req[i].kr_flags;  in kcpc_dup_set()
1818 		new->ks_req[i].kr_nattrs = set->ks_req[i].kr_nattrs;  in kcpc_dup_set()
1823 			    set->ks_req[i].kr_attr[j].ka_val;  in kcpc_dup_set()
1825 			    set->ks_req[i].kr_attr[j].ka_name,  in kcpc_dup_set()
1909 	 * Allocate number of sets assuming that each set contains one and only  in kcpc_cpu_ctx_create()
1925 		kcpc_set_t	*set;  in kcpc_cpu_ctx_create()  local
1929 		 * Allocate CPC context and set for requested counter events  in kcpc_cpu_ctx_create()
1932 		set = kcpc_set_create(reqs, nreqs, 0, kmem_flags);  in kcpc_cpu_ctx_create()
1933 		if (set == NULL) {  in kcpc_cpu_ctx_create()
1942 		if (kcpc_assign_reqs(set, ctx) != 0) {  in kcpc_cpu_ctx_create()
1947 			 * set of counter requests when this happens since at  in kcpc_cpu_ctx_create()
1951 			kcpc_free_set(set);  in kcpc_cpu_ctx_create()
1952 			set = kcpc_set_create(reqs, 1, 0, kmem_flags);  in kcpc_cpu_ctx_create()
1953 			if (set == NULL) {  in kcpc_cpu_ctx_create()
1957 			if (kcpc_assign_reqs(set, ctx) != 0) {  in kcpc_cpu_ctx_create()
1961 				    set->ks_req->kr_event);  in kcpc_cpu_ctx_create()
1963 				kcpc_free_set(set);  in kcpc_cpu_ctx_create()
1974 		set->ks_data = kmem_zalloc(set->ks_nreqs * sizeof (uint64_t),  in kcpc_cpu_ctx_create()
1976 		if (set->ks_data == NULL) {  in kcpc_cpu_ctx_create()
1977 			kcpc_free_set(set);  in kcpc_cpu_ctx_create()
1985 		if (kcpc_configure_reqs(ctx, set, &subcode) != 0) {  in kcpc_cpu_ctx_create()
1989 			    "set of counter event requests!\n");  in kcpc_cpu_ctx_create()
1991 			reqs += set->ks_nreqs;  in kcpc_cpu_ctx_create()
1992 			nreqs -= set->ks_nreqs;  in kcpc_cpu_ctx_create()
1993 			kmem_free(set->ks_data,  in kcpc_cpu_ctx_create()
1994 			    set->ks_nreqs * sizeof (uint64_t));  in kcpc_cpu_ctx_create()
1995 			kcpc_free_set(set);  in kcpc_cpu_ctx_create()
2001 		 * Point set of counter event requests at this context and fill  in kcpc_cpu_ctx_create()
2004 		set->ks_ctx = ctx;  in kcpc_cpu_ctx_create()
2005 		ctx->kc_set = set;  in kcpc_cpu_ctx_create()
2014 		reqs += set->ks_nreqs;  in kcpc_cpu_ctx_create()
2015 		nreqs -= set->ks_nreqs;  in kcpc_cpu_ctx_create()
2119 		kcpc_set_t *set = ctx->kc_set;  in kcpc_program()  local
2122 		ASSERT(set != NULL);  in kcpc_program()
2127 		 * set, but it is not very reliable, so we start again from the  in kcpc_program()
2130 		for (i = 0; i < set->ks_nreqs; i++) {  in kcpc_program()
2134 			*(set->ks_req[i].kr_data) = set->ks_req[i].kr_preset;  in kcpc_program()
2140 			    set->ks_req[i].kr_preset,  in kcpc_program()
2141 			    0, 0, NULL, &set->ks_req[i].kr_config, NULL);  in kcpc_program()
2224 	kcpc_set_t	*set;  in kcpc_read()  local
2244 	set = ctx->kc_set;  in kcpc_read()
2245 	if (set == NULL || set->ks_req == NULL) {  in kcpc_read()
2254 	req = set->ks_req;  in kcpc_read()
2256 	for (i = 0; i < set->ks_nreqs; i++) {  in kcpc_read()
2338 	 * to be set when request is assigned to a set.  in kcpc_reqs_add()
2343 	req->kr_index = -1;	/* set when assigning request to set */  in kcpc_reqs_add()
2344 	req->kr_data = NULL;	/* set when configuring request */  in kcpc_reqs_add()
2362  * Reset list of CPC event requests so its space can be used for another set
2397  * Create set of given counter event requests
2403 	kcpc_set_t	*set;  in kcpc_set_create()  local
2406 	 * Allocate set and assign number of requests in set and flags  in kcpc_set_create()
2408 	set = kmem_zalloc(sizeof (kcpc_set_t), kmem_flags);  in kcpc_set_create()
2409 	if (set == NULL)  in kcpc_set_create()
2413 		set->ks_nreqs = nreqs;  in kcpc_set_create()
2415 		set->ks_nreqs = cpc_ncounters;  in kcpc_set_create()
2417 	set->ks_flags = set_flags;  in kcpc_set_create()
2420 	 * Allocate requests needed, copy requests into set, and set index into  in kcpc_set_create()
2424 	set->ks_req = (kcpc_request_t *)kmem_zalloc(sizeof (kcpc_request_t) *  in kcpc_set_create()
2425 	    set->ks_nreqs, kmem_flags);  in kcpc_set_create()
2426 	if (set->ks_req == NULL) {  in kcpc_set_create()
2427 		kmem_free(set, sizeof (kcpc_set_t));  in kcpc_set_create()
2431 	bcopy(reqs, set->ks_req, sizeof (kcpc_request_t) * set->ks_nreqs);  in kcpc_set_create()
2433 	for (i = 0; i < set->ks_nreqs; i++)  in kcpc_set_create()
2434 		set->ks_req[i].kr_index = i;  in kcpc_set_create()
2436 	return (set);  in kcpc_set_create()
2447  * to be preserved, so it is set to NULL.
2479  * Stop counters on given CPU and set its CPC context to NULL unless