81 static int kcpc_tryassign(kcpc_set_t *set, int starting_req, int *scratch);
82 static kcpc_set_t *kcpc_dup_set(kcpc_set_t *set);
170 kcpc_bind_cpu(kcpc_set_t *set, processorid_t cpuid, int *subcode)  in kcpc_bind_cpu()  argument
179 	if (kcpc_assign_reqs(set, ctx) != 0) {  in kcpc_bind_cpu()
188 	set->ks_data = kmem_zalloc(set->ks_nreqs * sizeof (uint64_t), KM_SLEEP);  in kcpc_bind_cpu()
190 	if ((error = kcpc_configure_reqs(ctx, set, subcode)) != 0) {  in kcpc_bind_cpu()
191 		kmem_free(set->ks_data, set->ks_nreqs * sizeof (uint64_t));  in kcpc_bind_cpu()
196 	set->ks_ctx = ctx;  in kcpc_bind_cpu()
197 	ctx->kc_set = set;  in kcpc_bind_cpu()
247 	mutex_enter(&set->ks_lock);  in kcpc_bind_cpu()
248 	set->ks_state |= KCPC_SET_BOUND;  in kcpc_bind_cpu()
249 	cv_signal(&set->ks_condv);  in kcpc_bind_cpu()
250 	mutex_exit(&set->ks_lock);  in kcpc_bind_cpu()
256 	set->ks_ctx = NULL;  in kcpc_bind_cpu()
257 	kmem_free(set->ks_data, set->ks_nreqs * sizeof (uint64_t));  in kcpc_bind_cpu()
263 kcpc_bind_thread(kcpc_set_t *set, kthread_t *t, int *subcode)  in kcpc_bind_thread()  argument
286 	if (kcpc_assign_reqs(set, ctx) != 0) {  in kcpc_bind_thread()
293 	if (set->ks_flags & CPC_BIND_LWP_INHERIT)  in kcpc_bind_thread()
305 	set->ks_data = kmem_alloc(set->ks_nreqs * sizeof (uint64_t), KM_SLEEP);  in kcpc_bind_thread()
307 	if ((error = kcpc_configure_reqs(ctx, set, subcode)) != 0) {  in kcpc_bind_thread()
308 		kmem_free(set->ks_data, set->ks_nreqs * sizeof (uint64_t));  in kcpc_bind_thread()
314 	set->ks_ctx = ctx;  in kcpc_bind_thread()
315 	ctx->kc_set = set;  in kcpc_bind_thread()
346 	mutex_enter(&set->ks_lock);  in kcpc_bind_thread()
347 	set->ks_state |= KCPC_SET_BOUND;  in kcpc_bind_thread()
348 	cv_signal(&set->ks_condv);  in kcpc_bind_thread()
349 	mutex_exit(&set->ks_lock);  in kcpc_bind_thread()
359 kcpc_configure_reqs(kcpc_ctx_t *ctx, kcpc_set_t *set, int *subcode)  in kcpc_configure_reqs()  argument
365 	for (i = 0; i < set->ks_nreqs; i++) {  in kcpc_configure_reqs()
367 		rp = &set->ks_req[i];  in kcpc_configure_reqs()
393 			kcpc_free_configs(set);  in kcpc_configure_reqs()
406 		rp->kr_data = set->ks_data + rp->kr_index;  in kcpc_configure_reqs()
414 kcpc_free_configs(kcpc_set_t *set)  in kcpc_free_configs()  argument
418 	for (i = 0; i < set->ks_nreqs; i++)  in kcpc_free_configs()
419 		if (set->ks_req[i].kr_config != NULL)  in kcpc_free_configs()
420 			pcbe_ops->pcbe_free(set->ks_req[i].kr_config);  in kcpc_free_configs()
428 kcpc_sample(kcpc_set_t *set, uint64_t *buf, hrtime_t *hrtime, uint64_t *tick)  in kcpc_sample()  argument
430 	kcpc_ctx_t	*ctx = set->ks_ctx;  in kcpc_sample()
433 	mutex_enter(&set->ks_lock);  in kcpc_sample()
434 	if ((set->ks_state & KCPC_SET_BOUND) == 0) {  in kcpc_sample()
435 		mutex_exit(&set->ks_lock);  in kcpc_sample()
438 	mutex_exit(&set->ks_lock);  in kcpc_sample()
487 	if (copyout(set->ks_data, buf,  in kcpc_sample()
488 	    set->ks_nreqs * sizeof (uint64_t)) == -1)  in kcpc_sample()
521 kcpc_unbind(kcpc_set_t *set)  in kcpc_unbind()  argument
531 	mutex_enter(&set->ks_lock);  in kcpc_unbind()
532 	while ((set->ks_state & KCPC_SET_BOUND) == 0)  in kcpc_unbind()
533 		cv_wait(&set->ks_condv, &set->ks_lock);  in kcpc_unbind()
534 	mutex_exit(&set->ks_lock);  in kcpc_unbind()
536 	ctx = set->ks_ctx;  in kcpc_unbind()
609 kcpc_preset(kcpc_set_t *set, int index, uint64_t preset)  in kcpc_preset()  argument
613 	ASSERT(set != NULL);  in kcpc_preset()
614 	ASSERT(set->ks_state & KCPC_SET_BOUND);  in kcpc_preset()
615 	ASSERT(set->ks_ctx->kc_thread == curthread);  in kcpc_preset()
616 	ASSERT(set->ks_ctx->kc_cpuid == -1);  in kcpc_preset()
618 	if (index < 0 || index >= set->ks_nreqs)  in kcpc_preset()
621 	for (i = 0; i < set->ks_nreqs; i++)  in kcpc_preset()
622 		if (set->ks_req[i].kr_index == index)  in kcpc_preset()
624 	ASSERT(i != set->ks_nreqs);  in kcpc_preset()
626 	set->ks_req[i].kr_preset = preset;  in kcpc_preset()
631 kcpc_restart(kcpc_set_t *set)  in kcpc_restart()  argument
633 	kcpc_ctx_t	*ctx = set->ks_ctx;  in kcpc_restart()
637 	ASSERT(set->ks_state & KCPC_SET_BOUND);  in kcpc_restart()
641 	for (i = 0; i < set->ks_nreqs; i++) {  in kcpc_restart()
642 		*(set->ks_req[i].kr_data) = set->ks_req[i].kr_preset;  in kcpc_restart()
643 		pcbe_ops->pcbe_configure(0, NULL, set->ks_req[i].kr_preset,  in kcpc_restart()
644 		    0, 0, NULL, &set->ks_req[i].kr_config, NULL);  in kcpc_restart()
676 	kcpc_set_t	*set = t->t_cpc_set;  in kcpc_enable()  local
724 		for (i = 0; i < set->ks_nreqs; i++) {  in kcpc_enable()
725 			set->ks_req[i].kr_preset = *(set->ks_req[i].kr_data);  in kcpc_enable()
727 				set->ks_req[i].kr_flags |= flag;  in kcpc_enable()
729 				set->ks_req[i].kr_flags &= ~flag;  in kcpc_enable()
731 		newset = kcpc_dup_set(set);  in kcpc_enable()
732 		if (kcpc_unbind(set) != 0)  in kcpc_enable()
1467 	kcpc_set_t	*set = ctx->kc_set;  in kcpc_free()  local
1469 	ASSERT(set != NULL);  in kcpc_free()
1536 	for (i = 0; i < set->ks_nreqs; i++) {  in kcpc_free()
1537 		if (set->ks_req[i].kr_config != NULL)  in kcpc_free()
1538 			pcbe_ops->pcbe_free(set->ks_req[i].kr_config);  in kcpc_free()
1541 	kmem_free(set->ks_data, set->ks_nreqs * sizeof (uint64_t));  in kcpc_free()
1543 	kcpc_free_set(set);  in kcpc_free()
1550 kcpc_free_set(kcpc_set_t *set)  in kcpc_free_set()  argument
1555 	ASSERT(set->ks_req != NULL);  in kcpc_free_set()
1557 	for (i = 0; i < set->ks_nreqs; i++) {  in kcpc_free_set()
1558 		req = &set->ks_req[i];  in kcpc_free_set()
1566 	kmem_free(set->ks_req, sizeof (kcpc_request_t) * set->ks_nreqs);  in kcpc_free_set()
1567 	cv_destroy(&set->ks_condv);  in kcpc_free_set()
1568 	mutex_destroy(&set->ks_lock);  in kcpc_free_set()
1569 	kmem_free(set, sizeof (kcpc_set_t));  in kcpc_free_set()
1610 	kcpc_set_t *set = curthread->t_cpc_set;  in kcpc_passivate()  local
1613 	if (set == NULL)  in kcpc_passivate()
1624 		(void) kcpc_unbind(set);  in kcpc_passivate()
1667 kcpc_assign_reqs(kcpc_set_t *set, kcpc_ctx_t *ctx)  in kcpc_assign_reqs()  argument
1672 	ASSERT(set->ks_nreqs <= cpc_ncounters);  in kcpc_assign_reqs()
1678 	picnum_save = kmem_alloc(set->ks_nreqs * sizeof (int), KM_SLEEP);  in kcpc_assign_reqs()
1688 	for (i = 0; i < set->ks_nreqs; i++)  in kcpc_assign_reqs()
1689 		if (kcpc_tryassign(set, i, picnum_save) == 0)  in kcpc_assign_reqs()
1692 	kmem_free(picnum_save, set->ks_nreqs * sizeof (int));  in kcpc_assign_reqs()
1693 	if (i == set->ks_nreqs)  in kcpc_assign_reqs()
1699 kcpc_tryassign(kcpc_set_t *set, int starting_req, int *scratch)  in kcpc_tryassign()  argument
1716 	for (i = 0; i < set->ks_nreqs; i++) {  in kcpc_tryassign()
1717 		scratch[i] = set->ks_req[i].kr_picnum;  in kcpc_tryassign()
1718 		if (set->ks_req[i].kr_picnum != -1)  in kcpc_tryassign()
1719 			resmap |= (1 << set->ks_req[i].kr_picnum);  in kcpc_tryassign()
1728 		if (set->ks_req[i].kr_picnum != -1) {  in kcpc_tryassign()
1729 			ASSERT((bitmap & (1 << set->ks_req[i].kr_picnum)) == 0);  in kcpc_tryassign()
1730 			bitmap |= (1 << set->ks_req[i].kr_picnum);  in kcpc_tryassign()
1731 			if (++i == set->ks_nreqs)  in kcpc_tryassign()
1736 		ctrmap = pcbe_ops->pcbe_event_coverage(set->ks_req[i].kr_event);  in kcpc_tryassign()
1752 			for (i = 0; i < set->ks_nreqs; i++)  in kcpc_tryassign()
1753 				set->ks_req[i].kr_picnum = scratch[i];  in kcpc_tryassign()
1756 		set->ks_req[i].kr_picnum = j;  in kcpc_tryassign()
1758 		if (++i == set->ks_nreqs)  in kcpc_tryassign()
1766 kcpc_dup_set(kcpc_set_t *set)  in kcpc_dup_set()  argument
1774 	new->ks_flags = set->ks_flags;  in kcpc_dup_set()
1775 	new->ks_nreqs = set->ks_nreqs;  in kcpc_dup_set()
1776 	new->ks_req = kmem_alloc(set->ks_nreqs * sizeof (kcpc_request_t),  in kcpc_dup_set()
1783 		new->ks_req[i].kr_index = set->ks_req[i].kr_index;  in kcpc_dup_set()
1784 		new->ks_req[i].kr_picnum = set->ks_req[i].kr_picnum;  in kcpc_dup_set()
1787 		(void) strncpy(new->ks_req[i].kr_event, set->ks_req[i].kr_event,  in kcpc_dup_set()
1789 		new->ks_req[i].kr_preset = set->ks_req[i].kr_preset;  in kcpc_dup_set()
1790 		new->ks_req[i].kr_flags = set->ks_req[i].kr_flags;  in kcpc_dup_set()
1791 		new->ks_req[i].kr_nattrs = set->ks_req[i].kr_nattrs;  in kcpc_dup_set()
1796 			    set->ks_req[i].kr_attr[j].ka_val;  in kcpc_dup_set()
1798 			    set->ks_req[i].kr_attr[j].ka_name,  in kcpc_dup_set()
1898 		kcpc_set_t	*set;  in kcpc_cpu_ctx_create()  local
1905 		set = kcpc_set_create(reqs, nreqs, 0, kmem_flags);  in kcpc_cpu_ctx_create()
1906 		if (set == NULL) {  in kcpc_cpu_ctx_create()
1915 		if (kcpc_assign_reqs(set, ctx) != 0) {  in kcpc_cpu_ctx_create()
1924 			kcpc_free_set(set);  in kcpc_cpu_ctx_create()
1925 			set = kcpc_set_create(reqs, 1, 0, kmem_flags);  in kcpc_cpu_ctx_create()
1926 			if (set == NULL) {  in kcpc_cpu_ctx_create()
1930 			if (kcpc_assign_reqs(set, ctx) != 0) {  in kcpc_cpu_ctx_create()
1934 				    set->ks_req->kr_event);  in kcpc_cpu_ctx_create()
1936 				kcpc_free_set(set);  in kcpc_cpu_ctx_create()
1947 		set->ks_data = kmem_zalloc(set->ks_nreqs * sizeof (uint64_t),  in kcpc_cpu_ctx_create()
1949 		if (set->ks_data == NULL) {  in kcpc_cpu_ctx_create()
1950 			kcpc_free_set(set);  in kcpc_cpu_ctx_create()
1958 		if (kcpc_configure_reqs(ctx, set, &subcode) != 0) {  in kcpc_cpu_ctx_create()
1964 			reqs += set->ks_nreqs;  in kcpc_cpu_ctx_create()
1965 			nreqs -= set->ks_nreqs;  in kcpc_cpu_ctx_create()
1966 			kmem_free(set->ks_data,  in kcpc_cpu_ctx_create()
1967 			    set->ks_nreqs * sizeof (uint64_t));  in kcpc_cpu_ctx_create()
1968 			kcpc_free_set(set);  in kcpc_cpu_ctx_create()
1977 		set->ks_ctx = ctx;  in kcpc_cpu_ctx_create()
1978 		ctx->kc_set = set;  in kcpc_cpu_ctx_create()
1987 		reqs += set->ks_nreqs;  in kcpc_cpu_ctx_create()
1988 		nreqs -= set->ks_nreqs;  in kcpc_cpu_ctx_create()
2092 		kcpc_set_t *set = ctx->kc_set;  in kcpc_program()  local
2095 		ASSERT(set != NULL);  in kcpc_program()
2103 		for (i = 0; i < set->ks_nreqs; i++) {  in kcpc_program()
2107 			*(set->ks_req[i].kr_data) = set->ks_req[i].kr_preset;  in kcpc_program()
2113 			    set->ks_req[i].kr_preset,  in kcpc_program()
2114 			    0, 0, NULL, &set->ks_req[i].kr_config, NULL);  in kcpc_program()
2197 	kcpc_set_t	*set;  in kcpc_read()  local
2217 	set = ctx->kc_set;  in kcpc_read()
2218 	if (set == NULL || set->ks_req == NULL) {  in kcpc_read()
2227 	req = set->ks_req;  in kcpc_read()
2229 	for (i = 0; i < set->ks_nreqs; i++) {  in kcpc_read()
2376 	kcpc_set_t	*set;  in kcpc_set_create()  local
2381 	set = kmem_zalloc(sizeof (kcpc_set_t), kmem_flags);  in kcpc_set_create()
2382 	if (set == NULL)  in kcpc_set_create()
2386 		set->ks_nreqs = nreqs;  in kcpc_set_create()
2388 		set->ks_nreqs = cpc_ncounters;  in kcpc_set_create()
2390 	set->ks_flags = set_flags;  in kcpc_set_create()
2397 	set->ks_req = (kcpc_request_t *)kmem_zalloc(sizeof (kcpc_request_t) *  in kcpc_set_create()
2398 	    set->ks_nreqs, kmem_flags);  in kcpc_set_create()
2399 	if (set->ks_req == NULL) {  in kcpc_set_create()
2400 		kmem_free(set, sizeof (kcpc_set_t));  in kcpc_set_create()
2404 	bcopy(reqs, set->ks_req, sizeof (kcpc_request_t) * set->ks_nreqs);  in kcpc_set_create()
2406 	for (i = 0; i < set->ks_nreqs; i++)  in kcpc_set_create()
2407 		set->ks_req[i].kr_index = i;  in kcpc_set_create()
2409 	return (set);  in kcpc_set_create()