122 #define	P4_THIS_USR	0x1 /* HTT: Measure usr events on this logical CPU */
123 #define	P4_THIS_SYS	0x2 /* HTT: Measure os events on this logical CPU */
124 #define	P4_SIBLING_USR	0x4 /* HTT: Measure os events on other logical CPU */
125 #define	P4_SIBLING_SYS	0x8 /* HTT: Measure usr events on other logical CPU */
126 #define	P4_PMI		0x10 /* HTT: Set PMI bit for local logical CPU */
130 	uint8_t		p4_picno;	/* From 0 to 18 */
142 	uint32_t	pe_map; /* bitmap of counters; bit 1 means ctr 0 */
145 #define	MASK40			UINT64_C(0xffffffffff)
164 #define	CCCR_ACTV_THR_MASK	0x3
165 #define	CCCR_THRESHOLD_MAX	0xF
179 #define	ESCR_TAG_VALUE_MAX	0xF
193 #define	ESCR_T1_USR_SHIFT	0
203 #define	BPU0_map	(0x1 | 0x2)		/* Counters 0 and 1 */
204 #define	BPU2_map	(0x4 | 0x8)		/* Counters 2 and 3 */
205 #define	MS0_map		(0x10 | 0x20)		/* Counters 4 and 5 */
206 #define	MS2_map		(0x40 | 0x80)		/* Counters 6 and 7 */
207 #define	FLAME0_map	(0x100 | 0x200)		/* Counters 8 and 9 */
208 #define	FLAME2_map	(0x400 | 0x800)		/* Counters 10 and 11 */
209 #define	IQ0_map		(0x1000 | 0x2000 | 0x10000) /* Counters 12, 13, 16 */
210 #define	IQ2_map		(0x4000 | 0x8000 | 0x20000) /* Counters 14, 15, 17 */
217 	{ 0, 0x3B2, BPU0_map },		/* 0 */
219 	{ 1, 0x3B4, BPU0_map },		/* 1 */
221 	{ 2, 0x3AA, BPU0_map },		/* 2 */
223 	{ 3, 0x3B6, BPU0_map },		/* 3 */
225 	{ 4, 0x3AC, BPU0_map },		/* 4 */
227 	{ 5, 0x3C8, BPU0_map },		/* 5 */
229 	{ 6, 0x3A2, BPU0_map },		/* 6 */
231 	{ 7, 0x3A0, BPU0_map },		/* 7 */
233 	{ 0, 0x3B3, BPU2_map },		/* 8 */
235 	{ 1, 0x3B5, BPU2_map },		/* 9 */
237 	{ 2, 0x3AB, BPU2_map },		/* 10 */
239 	{ 3, 0x3B7, BPU2_map },		/* 11 */
241 	{ 4, 0x3AD, BPU2_map },		/* 12 */
243 	{ 5, 0x3C9, BPU2_map },		/* 13 */
245 	{ 6, 0x3A3, BPU2_map },		/* 14 */
247 	{ 7, 0x3A1, BPU2_map },		/* 15 */
249 	{ 0, 0x3C0, MS0_map },		/* 16 */
251 	{ 1, 0x3C4, MS0_map },		/* 17 */
253 	{ 2, 0x3C2, MS0_map },		/* 18 */
255 	{ 0, 0x3C1, MS2_map },		/* 19 */
257 	{ 1, 0x3C5, MS2_map },		/* 20 */
259 	{ 2, 0x3C3, MS2_map },		/* 21 */
261 	{ 0, 0x3A6, FLAME0_map },	/* 22 */
263 	{ 1, 0x3A4, FLAME0_map },	/* 23 */
265 	{ 2, 0x3AE, FLAME0_map },	/* 24 */
267 	{ 3, 0x3B0, FLAME0_map },	/* 25 */
269 	{ 5, 0x3A8, FLAME0_map },	/* 26 */
271 	{ 0, 0x3A7, FLAME2_map },	/* 27 */
273 	{ 1, 0x3A5, FLAME2_map },	/* 28 */
275 	{ 2, 0x3AF, FLAME2_map },	/* 29 */
277 	{ 3, 0x3B1, FLAME2_map },	/* 30 */
279 	{ 5, 0x3A9, FLAME2_map },	/* 31 */
281 	{ 0, 0x3BA, IQ0_map },		/* 32 */
283 	{ 1, 0x3CA, IQ0_map },		/* 33 */
285 	{ 2, 0x3BC, IQ0_map },		/* 34 */
287 	{ 3, 0x3BE, IQ0_map },		/* 35 */
289 	{ 4, 0x3B8, IQ0_map },		/* 36 */
291 	{ 5, 0x3CC, IQ0_map },		/* 37 */
293 	{ 6, 0x3E0, IQ0_map },		/* 38 */
295 	{ 0, 0x3BB, IQ2_map },		/* 39 */
297 	{ 1, 0x3CB, IQ2_map },		/* 40 */
299 	{ 2, 0x3BD, IQ2_map },		/* 41 */
301 	{ 4, 0x3B9, IQ2_map },		/* 42 */
303 	{ 5, 0x3CD, IQ2_map },		/* 43 */
305 	{ 6, 0x3E1, IQ2_map }		/* 44 */
317 { /* BPU_COUNTER0 */ 0x300, 0x360, BSU0|FSB0|MOB0|PMH0|BPU0|IS0|ITLB0|IX0},
318 { /* BPU_COUNTER1 */ 0x301, 0x361, BSU0|FSB0|MOB0|PMH0|BPU0|IS0|ITLB0|IX0},
319 { /* BPU_COUNTER2 */ 0x302, 0x362, BSU1|FSB1|MOB1|PMH1|BPU1|IS1|ITLB1|IX1},
320 { /* BPU_COUNTER3 */ 0x303, 0x363, BSU1|FSB1|MOB1|PMH1|BPU1|IS1|ITLB1|IX1},
321 { /* MS_COUNTER0 */  0x304, 0x364, MS0|TBPU0|TC0 },
322 { /* MS_COUNTER1 */  0x305, 0x365, MS0|TBPU0|TC0 },
323 { /* MS_COUNTER2 */  0x306, 0x366, MS1|TBPU1|TC1 },
324 { /* MS_COUNTER3 */  0x307, 0x367, MS1|TBPU1|TC1 },
325 { /* FLAME_COUNTER0 */ 0x308, 0x368, FIRM0|FLAME0|DAC0|SAAT0|U2L0 },
326 { /* FLAME_COUNTER1 */ 0x309, 0x369, FIRM0|FLAME0|DAC0|SAAT0|U2L0 },
327 { /* FLAME_COUNTER2 */ 0x30A, 0x36A, FIRM1|FLAME1|DAC1|SAAT1|U2L1 },
328 { /* FLAME_COUNTER3 */ 0x30B, 0x36B, FIRM1|FLAME1|DAC1|SAAT1|U2L1 },
329 { /* IQ_COUNTER0 */  0x30C, 0x36C, CRU0|CRU2|CRU4|IQ0|RAT0|SSU0|ALF0 },
330 { /* IQ_COUNTER1 */  0x30D, 0x36D, CRU0|CRU2|CRU4|IQ0|RAT0|SSU0|ALF0 },
331 { /* IQ_COUNTER2 */  0x30E, 0x36E, CRU1|CRU3|CRU5|IQ1|RAT1|ALF1 },
332 { /* IQ_COUNTER3 */  0x30F, 0x36F, CRU1|CRU3|CRU5|IQ1|RAT1|ALF1 },
333 { /* IQ_COUNTER4 */  0x310, 0x370, CRU0|CRU2|CRU4|IQ0|RAT0|SSU0|ALF0 },
334 { /* IQ_COUNTER5 */  0x311, 0x371, CRU1|CRU3|CRU5|IQ1|RAT1|ALF1 }
354 #define	GEN_EVT_END { NULL, NULL, 0x0, 0x0 }
357 { "branch_retired", CRU2|CRU3, 0xF, 0x6, 0x5, C(12)|C(13)|C(14)|C(15)|C(16) },
358 { "mispred_branch_retired", CRU0|CRU1, 0x1, 0x3, 0x4,
360 { "TC_deliver_mode", TC0|TC1, 0xFF, 0x1, 0x1, C(4)|C(5)|C(6)|C(7) },
361 { "BPU_fetch_request", BPU0|BPU1, 0x1, 0x3, 0x0, C(0)|C(1)|C(2)|C(3) },
362 { "ITLB_reference", ITLB0|ITLB1, 0x7, 0x18, 0x3, C(0)|C(1)|C(2)|C(3) },
363 { "memory_cancel", DAC0|DAC1, 0x6, 0x2, 0x5, C(8)|C(9)|C(10)|C(11) },
364 { "memory_complete", SAAT0|SAAT1, 0x3, 0x8, 0x2, C(8)|C(9)|C(10)|C(11) },
365 { "load_port_replay", SAAT0|SAAT1, 0x1, 0x4, 0x2, C(8)|C(9)|C(10)|C(11) },
366 { "store_port_replay", SAAT0|SAAT1, 0x1, 0x5, 0x2, C(8)|C(9)|C(10)|C(11) },
367 { "MOB_load_replay", MOB0|MOB1, 0x35, 0x3, 0x2, C(0)|C(1)|C(2)|C(3) },
368 { "page_walk_type", PMH0|PMH1, 0x3, 0x1, 0x4, C(0)|C(1)|C(2)|C(3) },
369 { "BSQ_cache_reference", BSU0|BSU1, 0x73F, 0xC, 0x7, C(0)|C(1)|C(2)|C(3) },
370 { "IOQ_allocation", FSB0, 0xEFFF, 0x3, 0x6, C(0)|C(1) },
371 { "IOQ_active_entries", FSB1, 0xEFFF, 0x1A, 0x6, C(2)|C(3) },
372 { "FSB_data_activity", FSB0|FSB1, 0x3F, 0x17, 0x6, C(0)|C(1)|C(2)|C(3) },
373 { "BSQ_allocation", BSU0, 0x3FEF, 0x5, 0x7, C(0)|C(1) },
374 { "bsq_active_entries", BSU1, 0x3FEF, 0x6, 0x7, C(2)|C(3) },
375 { "x87_assist", CRU2|CRU3, 0x1F, 0x3, 0x5, C(12)|C(13)|C(14)|C(15)|C(16)|C(17)},
376 { "SSE_input_assist", FIRM0|FIRM1, 0x8000, 0x34, 0x1, C(8)|C(9)|C(10)|C(11) },
377 { "packed_SP_uop", FIRM0|FIRM1, 0x8000, 0x8, 0x1, C(8)|C(9)|C(10)|C(11) },
378 { "packed_DP_uop", FIRM0|FIRM1, 0x8000, 0xC, 0x1, C(8)|C(9)|C(10)|C(11) },
379 { "scalar_SP_uop", FIRM0|FIRM1, 0x8000, 0xA, 0x1, C(8)|C(9)|C(10)|C(11) },
380 { "scalar_DP_uop", FIRM0|FIRM1, 0x8000, 0xE, 0x1, C(8)|C(9)|C(10)|C(11) },
381 { "64bit_MMX_uop", FIRM0|FIRM1, 0x8000, 0x2, 0x1, C(8)|C(9)|C(10)|C(11) },
382 { "128bit_MMX_uop", FIRM0|FIRM1, 0x8000, 0x1A, 0x1, C(8)|C(9)|C(10)|C(11) },
383 { "x87_FP_uop", FIRM0|FIRM1, 0x8000, 0x4, 0x1, C(8)|C(9)|C(10)|C(11) },
384 { "x87_SIMD_moves_uop", FIRM0|FIRM1, 0x18, 0x2E, 0x1, C(8)|C(9)|C(10)|C(11) },
385 { "machine_clear", CRU2|CRU3, 0xD, 0x2, 0x5,
387 { "global_power_events", FSB0|FSB1, 0x1, 0x13, 0x6, C(0)|C(1)|C(2)|C(3) },
388 { "tc_ms_xfer", MS0|MS1, 0x1, 0x5, 0x0, C(4)|C(5)|C(6)|C(7) },
389 { "uop_queue_writes", MS0|MS1, 0x7, 0x9, 0x0, C(4)|C(5)|C(6)|C(7) },
390 { "front_end_event", CRU2|CRU3, 0x3, 0x8, 0x5,
392 { "execution_event", CRU2|CRU3, 0xFF, 0xC, 0x5,
394 { "replay_event", CRU2|CRU3, 0x3, 0x9, 0x5,
396 { "instr_retired", CRU0|CRU1, 0xF, 0x2, 0x4,
398 { "uops_retired", CRU0|CRU1, 0x3, 0x1, 0x4,
400 { "uop_type", RAT0|RAT1, 0x3, 0x2, 0x2, C(12)|C(13)|C(14)|C(15)|C(16)|C(17)},
401 { "retired_mispred_branch_type", TBPU0|TBPU1, 0x1F, 0x5, 0x2,
403 { "retired_branch_type", TBPU0|TBPU1, 0x1F, 0x4, 0x2, C(4)|C(5)|C(6)|C(7) },
404 { NULL, 0, 0, 0, 0 }
408 { "PAPI_br_msp", "branch_retired", 0xa, C(12)|C(13)|C(14)|C(15)|C(16) },
409 { "PAPI_br_ins", "branch_retired", 0xf, C(12)|C(13)|C(14)|C(15)|C(16) },
410 { "PAPI_br_tkn", "branch_retired", 0xc, C(12)|C(13)|C(14)|C(15)|C(16) },
411 { "PAPI_br_ntk", "branch_retired", 0x3, C(12)|C(13)|C(14)|C(15)|C(16) },
412 { "PAPI_br_prc", "branch_retired", 0x5, C(12)|C(13)|C(14)|C(15)|C(16) },
413 { "PAPI_tot_ins", "instr_retired", 0x3, C(12)|C(13)|C(14)|C(15)|C(16)|C(17) },
414 { "PAPI_tot_cyc", "global_power_events", 0x1, C(0)|C(1)|C(2)|C(3) },
415 { "PAPI_tlb_dm", "page_walk_type", 0x1, C(0)|C(1)|C(2)|C(3) },
416 { "PAPI_tlb_im", "page_walk_type", 0x2, C(0)|C(1)|C(2)|C(3) },
417 { "PAPI_tlb_tm", "page_walk_type", 0x3, C(0)|C(1)|C(2)|C(3) },
418 { "PAPI_l1_icm", "BPU_fetch_request", 0x1, C(0)|C(1)|C(2)|C(3) },
419 { "PAPI_l2_ldm", "BSQ_cache_reference", 0x100, C(0)|C(1)|C(2)|C(3) },
420 { "PAPI_l2_stm", "BSQ_cache_reference", 0x400, C(0)|C(1)|C(2)|C(3) },
421 { "PAPI_l2_tcm", "BSQ_cache_reference", 0x500, C(0)|C(1)|C(2)|C(3) },
428 static int p4_rdpmc_avail = 0;
430 static const uint64_t p4_cccrstop = 0;
437 static int p4_htt = 0;
439 #define	P4_FAMILY	0xF
462 	for (i = 0; i < 18; i++) {  in p4_pcbe_init()
463 		size = 0;  in p4_pcbe_init()
480 		*p4_eventlist[i] = '\0';  in p4_pcbe_init()
499 		p4_eventlist[i][size - 1] = '\0';  in p4_pcbe_init()
511 	return (0);  in p4_pcbe_init()
539 	ASSERT(picnum >= 0 && picnum < 18);  in p4_pcbe_list_events()
561 		if (strcmp(name, gevp->name) == 0)  in find_generic_event()
573 		if (strcmp(name, evp->pe_name) == 0)  in find_event()
589 			return (0);  in p4_pcbe_event_coverage()
599 	uint64_t	ret = 0;  in p4_pcbe_overflow_bitmap()
607 	for (i = 0; i < 18; i++) {  in p4_pcbe_overflow_bitmap()
627 	for (i = 0; i < 18; i++) {  in p4_escr_inuse()
634 	return (0);  in p4_escr_inuse()
679 	uint16_t		emask = 0;  in p4_pcbe_configure()
681 	int			use_tag = 0;  in p4_pcbe_configure()
682 	int			active_thread = 0x3; /* default is "any" */  in p4_pcbe_configure()
683 	int			compare = 0;  in p4_pcbe_configure()
684 	int			complement = 0;  in p4_pcbe_configure()
685 	int			threshold = 0;  in p4_pcbe_configure()
686 	int			edge = 0;  in p4_pcbe_configure()
687 	int			sibling_usr = 0; /* count usr on other cpu */  in p4_pcbe_configure()
688 	int			sibling_sys = 0; /* count sys on other cpu */  in p4_pcbe_configure()
689 	int			invalid_attr = 0;  in p4_pcbe_configure()
698 		return (0);  in p4_pcbe_configure()
701 	if (picnum < 0 || picnum >= 18)  in p4_pcbe_configure()
715 				for (i = 0; i < nattrs; i++)  in p4_pcbe_configure()
721 			    (!p4_htt && nattrs > 0))  in p4_pcbe_configure()
735 	for (escr_ndx = 0; escr_ndx < ESCR_MAX_INDEX; escr_ndx++) {  in p4_pcbe_configure()
737 		    p4_escr_inuse(cfgs, escr_ndx) == 0)  in p4_pcbe_configure()
754 	for (i = 0; i < nattrs; i++) {  in p4_pcbe_configure()
755 		if (strcmp("emask", attrs[i].ka_name) == 0) {  in p4_pcbe_configure()
761 		} else if (strcmp("tag", attrs[i].ka_name) == 0) {  in p4_pcbe_configure()
767 		} else if (strcmp("compare", attrs[i].ka_name) == 0) {  in p4_pcbe_configure()
768 			if (attrs[i].ka_val != 0)  in p4_pcbe_configure()
771 		} else if (strcmp("complement", attrs[i].ka_name) == 0) {  in p4_pcbe_configure()
772 			if (attrs[i].ka_val != 0)  in p4_pcbe_configure()
775 		} else if (strcmp("threshold", attrs[i].ka_name) == 0) {  in p4_pcbe_configure()
780 		} else if (strcmp("edge", attrs[i].ka_name) == 0) {  in p4_pcbe_configure()
781 			if (attrs[i].ka_val != 0)  in p4_pcbe_configure()
790 		if (p4_htt == 0)  in p4_pcbe_configure()
793 		if (secpolicy_cpc_cpu(crgetcred()) != 0)  in p4_pcbe_configure()
796 		if (strcmp("active_thread", attrs[i].ka_name) == 0) {  in p4_pcbe_configure()
801 		} else if (strcmp("count_sibling_usr", attrs[i].ka_name) == 0) {  in p4_pcbe_configure()
802 			if (attrs[i].ka_val != 0)  in p4_pcbe_configure()
804 		} else if (strcmp("count_sibling_sys", attrs[i].ka_name) == 0) {  in p4_pcbe_configure()
805 			if (attrs[i].ka_val != 0)  in p4_pcbe_configure()
814 	if ((ev->pe_ctr_mask & C(picnum)) == 0)  in p4_pcbe_configure()
820 	for (escr_ndx = 0; escr_ndx < ESCR_MAX_INDEX; escr_ndx++) {  in p4_pcbe_configure()
823 		    p4_escr_inuse(cfgs, escr_ndx) == 0)  in p4_pcbe_configure()
832 	cfg->p4_flags = 0;  in p4_pcbe_configure()
905 	return (0);  in p4_pcbe_configure()
937 		for (i = 0; i < 18; i++) {  in p4_pcbe_program()
945 				escr |= (lid == 0) ? ESCR_T0_USR : ESCR_T1_USR;  in p4_pcbe_program()
947 				escr |= (lid == 0) ? ESCR_T0_OS : ESCR_T1_OS;  in p4_pcbe_program()
949 				escr |= (lid == 0) ? ESCR_T1_USR : ESCR_T0_USR;  in p4_pcbe_program()
951 				escr |= (lid == 0) ? ESCR_T1_OS : ESCR_T0_OS;  in p4_pcbe_program()
957 		for (i = 0; i < 18; i++) {  in p4_pcbe_program()
966 				cccr |= (lid == 0) ?  in p4_pcbe_program()
971 		for (i = 0; i < 18; i++) {  in p4_pcbe_program()
979 		for (i = 0; i < 18; i++) {  in p4_pcbe_program()
993 	for (i = 0; i < 18; i++)  in p4_pcbe_allstop()
994 		wrmsr(p4_ctrs[i].pc_ctladdr, 0ULL);  in p4_pcbe_allstop()
1009 	for (i = 0; i < 18; i++)  in p4_pcbe_sample()
1014 	for (i = 0; i < 18; i++) {  in p4_pcbe_sample()
1018 		if (diff < 0)  in p4_pcbe_sample()
1047 	if (p4_pcbe_init() != 0)  in _init()