xref: /titanic_50/usr/src/uts/sun4v/cpu/niagara_perfctr.c (revision 6b1abd46bd2c746d42947df930a0a3a632f81256)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 /*
22  * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
23  * Use is subject to license terms.
24  */
25 
26 #include <sys/types.h>
27 #include <sys/async.h>
28 #include <sys/sunddi.h>
29 #include <sys/sunndi.h>
30 #include <sys/ddi_impldefs.h>
31 #include <sys/machsystm.h>
32 #include <sys/hypervisor_api.h>
33 #include <sys/kstat.h>
34 #if defined(NIAGARA_IMPL)
35 #include <sys/niagararegs.h>
36 #elif defined(NIAGARA2_IMPL) || defined(VFALLS_IMPL) || defined(KT_IMPL)
37 #include <sys/niagara2regs.h>
38 #endif
39 
40 extern char cpu_module_name[];
41 
42 /*
43  * Data structure used to build array of event-names and pcr-mask values
44  */
45 typedef struct ni_kev_mask {
46 	char		*event_name;
47 	uint64_t	pcr_mask;
48 } ni_kev_mask_t;
49 
50 /*
51  * Kstat data structure for DRAM and JBUS performance counters
52  *
53  * Note that these performance counters are only 31 bits wide. Since
54  * the "busstat" command assumes a 32-bit counter, we emulate a 32-bit
55  * counter by detecting overflow on read of these performance counters
56  * and using the least significant bit of the overflow count as the
57  * most significant bit (i.e. bit# 31) of the DRAM and JBUS performance
58  * counters.
59  */
60 #define	NUM_OF_PICS	2
61 #define	NUM_OF_PIC_REGS	1
62 
63 typedef struct ni_ksinfo {
64 	uint8_t		pic_no_evs;			/* number of events */
65 	uint8_t		pic_sel_shift[NUM_OF_PICS];
66 	uint8_t		pic_shift[NUM_OF_PICS];
67 	uint64_t	pic_mask[NUM_OF_PICS];
68 	kstat_t		*pic_name_ksp[NUM_OF_PICS];
69 	kstat_t		*cntr_ksp;
70 	uint32_t	pic_reg[NUM_OF_PIC_REGS];
71 	uint32_t	pcr_reg;
72 	uint32_t	pic_overflow[NUM_OF_PICS];	/* overflow count */
73 	uint32_t	pic_last_val[NUM_OF_PICS];	/* last PIC value */
74 } ni_ksinfo_t;
75 
76 static ni_ksinfo_t	*ni_dram_kstats[DRAM_BANKS];
77 
78 #if defined(NIAGARA_IMPL)
79 static ni_ksinfo_t	*ni_jbus_kstat;
80 #endif
81 
82 typedef struct ni_perf_regs {
83 	uint32_t	pcr_reg;
84 	uint32_t	pic_reg[NUM_OF_PIC_REGS];
85 } ni_perf_regs_t;
86 
87 static ni_perf_regs_t dram_perf_regs[] = {
88 #if defined(NIAGARA_IMPL) || defined(NIAGARA2_IMPL)
89 	{HV_DRAM_CTL0, HV_DRAM_COUNT0},
90 	{HV_DRAM_CTL1, HV_DRAM_COUNT1},
91 	{HV_DRAM_CTL2, HV_DRAM_COUNT2},
92 	{HV_DRAM_CTL3, HV_DRAM_COUNT3},
93 #elif defined(VFALLS_IMPL) || defined(KT_IMPL)
94 	{HV_DRAM_CTL0, HV_DRAM_COUNT0},
95 	{HV_DRAM_CTL1, HV_DRAM_COUNT1},
96 	{HV_DRAM_CTL2, HV_DRAM_COUNT2},
97 	{HV_DRAM_CTL3, HV_DRAM_COUNT3},
98 	{HV_DRAM_CTL4, HV_DRAM_COUNT4},
99 	{HV_DRAM_CTL5, HV_DRAM_COUNT5},
100 	{HV_DRAM_CTL6, HV_DRAM_COUNT6},
101 	{HV_DRAM_CTL7, HV_DRAM_COUNT7}
102 #endif
103 };
104 
105 #ifdef VFALLS_IMPL
106 /*
107  * Kstat data structure for Zambezi performance counters
108  * These performance counters are 64 bits wide.
109  */
110 static ni_ksinfo_t	*zam_lpu_kstats[ZAMBEZI_LPU_COUNTERS];
111 static ni_ksinfo_t	*zam_gpd_kstats[ZAMBEZI_GPD_COUNTERS];
112 static ni_ksinfo_t	*zam_asu_kstats[ZAMBEZI_ASU_COUNTERS];
113 
114 typedef struct zam_perf_regs {
115 	uint32_t	pcr_reg;
116 	uint32_t	pic_reg[NUM_OF_PICS];
117 } zam_perf_regs_t;
118 
119 static zam_perf_regs_t lpu_perf_regs[] = {
120 	{HV_ZAM0_LPU_A_PCR, HV_ZAM0_LPU_A_PIC0, HV_ZAM0_LPU_A_PIC1},
121 	{HV_ZAM0_LPU_B_PCR, HV_ZAM0_LPU_B_PIC0, HV_ZAM0_LPU_B_PIC1},
122 	{HV_ZAM0_LPU_C_PCR, HV_ZAM0_LPU_C_PIC0, HV_ZAM0_LPU_C_PIC1},
123 	{HV_ZAM0_LPU_D_PCR, HV_ZAM0_LPU_D_PIC0, HV_ZAM0_LPU_D_PIC1},
124 
125 	{HV_ZAM1_LPU_A_PCR, HV_ZAM1_LPU_A_PIC0, HV_ZAM1_LPU_A_PIC1},
126 	{HV_ZAM1_LPU_B_PCR, HV_ZAM1_LPU_B_PIC0, HV_ZAM1_LPU_B_PIC1},
127 	{HV_ZAM1_LPU_C_PCR, HV_ZAM1_LPU_C_PIC0, HV_ZAM1_LPU_C_PIC1},
128 	{HV_ZAM1_LPU_D_PCR, HV_ZAM1_LPU_D_PIC0, HV_ZAM1_LPU_D_PIC1},
129 
130 	{HV_ZAM2_LPU_A_PCR, HV_ZAM2_LPU_A_PIC0, HV_ZAM2_LPU_A_PIC1},
131 	{HV_ZAM2_LPU_B_PCR, HV_ZAM2_LPU_B_PIC0, HV_ZAM2_LPU_B_PIC1},
132 	{HV_ZAM2_LPU_C_PCR, HV_ZAM2_LPU_C_PIC0, HV_ZAM2_LPU_C_PIC1},
133 	{HV_ZAM2_LPU_D_PCR, HV_ZAM2_LPU_D_PIC0, HV_ZAM2_LPU_D_PIC1},
134 
135 	{HV_ZAM3_LPU_A_PCR, HV_ZAM3_LPU_A_PIC0, HV_ZAM3_LPU_A_PIC1},
136 	{HV_ZAM3_LPU_B_PCR, HV_ZAM3_LPU_B_PIC0, HV_ZAM3_LPU_B_PIC1},
137 	{HV_ZAM3_LPU_C_PCR, HV_ZAM3_LPU_C_PIC0, HV_ZAM3_LPU_C_PIC1},
138 	{HV_ZAM3_LPU_D_PCR, HV_ZAM3_LPU_D_PIC0, HV_ZAM3_LPU_D_PIC1}
139 };
140 
141 static zam_perf_regs_t gpd_perf_regs[] = {
142 	{HV_ZAM0_GPD_PCR, HV_ZAM0_GPD_PIC0, HV_ZAM0_GPD_PIC1},
143 	{HV_ZAM1_GPD_PCR, HV_ZAM1_GPD_PIC0, HV_ZAM1_GPD_PIC1},
144 	{HV_ZAM2_GPD_PCR, HV_ZAM2_GPD_PIC0, HV_ZAM2_GPD_PIC1},
145 	{HV_ZAM3_GPD_PCR, HV_ZAM3_GPD_PIC0, HV_ZAM3_GPD_PIC1}
146 };
147 
148 static zam_perf_regs_t asu_perf_regs[] = {
149 	{HV_ZAM0_ASU_PCR, HV_ZAM0_ASU_PIC0, HV_ZAM0_ASU_PIC1},
150 	{HV_ZAM1_ASU_PCR, HV_ZAM1_ASU_PIC0, HV_ZAM1_ASU_PIC1},
151 	{HV_ZAM2_ASU_PCR, HV_ZAM2_ASU_PIC0, HV_ZAM2_ASU_PIC1},
152 	{HV_ZAM3_ASU_PCR, HV_ZAM3_ASU_PIC0, HV_ZAM3_ASU_PIC1}
153 };
154 
155 static int zam_cntr_kstat_update(kstat_t *, int);
156 #endif
157 
158 static void ni_create_name_kstat(char *, ni_ksinfo_t *, ni_kev_mask_t *);
159 static void ni_delete_name_kstat(ni_ksinfo_t *);
160 
161 static kstat_t *ni_create_cntr_kstat(char *, int,
162 	int (*update)(kstat_t *, int), void *);
163 
164 static int ni_cntr_kstat_update(kstat_t *, int);
165 
166 static kstat_t *ni_create_picN_kstat(char *, int, int, int,
167 	ni_kev_mask_t *);
168 
169 #ifdef DEBUG
170 static int	ni_perf_debug;
171 #endif
172 
173 /*
174  * Niagara, Niagara2 and VFalls DRAM Performance Events
175  */
176 static ni_kev_mask_t
177 niagara_dram_events[] = {
178 #if defined(NIAGARA_IMPL) || defined(NIAGARA2_IMPL) || defined(VFALLS_IMPL)
179 	{"mem_reads",		0x0},
180 	{"mem_writes",		0x1},
181 	{"mem_read_write",	0x2},
182 #elif defined(KT_IMPL)
183 	{"remote_reads",	0x0},
184 	{"remote_writes",	0x1},
185 	{"remote_read_write",	0x2},
186 #endif
187 #if defined(NIAGARA_IMPL) || defined(KT_IMPL)
188 	{"bank_busy_stalls",	0x3},
189 #endif
190 	{"rd_queue_latency",	0x4},
191 	{"wr_queue_latency",	0x5},
192 	{"rw_queue_latency",	0x6},
193 #if defined(NIAGARA_IMPL) || defined(NIAGARA2_IMPL) || defined(VFALLS_IMPL)
194 	{"wb_buf_hits",		0x7},
195 #elif defined(KT_IMPL)
196 	{"write_queue_drain",	0x7},
197 	{"read_all_channels",	0x8},
198 	{"write_starved",	0x9},
199 	{"write_all_channels",	0xa},
200 	{"read_write_channel0",	0xb},
201 	{"read_write_channel1",	0xc},
202 #endif
203 	{"clear_pic",		0xf}
204 };
205 
206 #if defined(VFALLS_IMPL)
207 /*
208  * Zambezi Performance Events
209  */
210 static ni_kev_mask_t
211 zam_lpu_perf_events[] = {
212 	{"none",		0x0},
213 	{"clock_cycles",	0x1},
214 	{"cycles_c2c_portX",	0x2},
215 	{"cycles_mem_portX",	0x3},
216 	{"cycles_WB_portX",	0x4},
217 	{"cycles_NC_portX",	0x5},
218 	{"cycles_c2c_portY",	0x6},
219 	{"cycles_mem_portY",	0x7},
220 	{"cycles_WB_portY",	0x8},
221 	{"cycles_NC_portY",	0x9},
222 	{"cycles_c2c_portZ",	0xa},
223 	{"cycles_mem_portZ",	0xb},
224 	{"cycles_WB_portZ",	0xc},
225 	{"cycles_NC_portZ",	0xd},
226 	{"cycles_TID_WB",	0xe},
227 	{"cycles_TID_INV",	0xf},
228 	{"cycles_TID_RTD",	0x10},
229 	{"cycles_TID_RTO",	0x11},
230 	{"cycles_TID_RTS",	0x12},
231 	{"cycles_IO_WRM",	0x13},
232 	{"cycles_IO_RD",	0x14},
233 	{"cycles_WB_egress",	0x15},
234 	{"cycles_INV_egress",	0x16},
235 	{"cycles_RTO_egress",	0x17},
236 	{"cycles_RTD_egress",	0x18},
237 	{"cycles_RTS_egress",	0x19},
238 	{"cycles_no_WB",	0x1a},
239 	{"cycles_no_read/inv",	0x1b},
240 	{"cycles_HIT_M",	0x1c},
241 	{"cycles_HIT_O",	0x1d},
242 	{"cycles_HIT_S",	0x1e},
243 	{"cycles_WB_HIT",	0x1f},
244 	{"cycles_MISS",		0x20},
245 	{"cycles_READ_or_INV",	0x21},
246 	{"cycles_WB",		0x22},
247 	{"cycles_NDR",		0x23},
248 	{"cycles_cache_miss",	0x24},
249 	{"cycles_cache_hit",	0x25},
250 	{"cycles_CRC_errors",	0x26},
251 	{"cycles_replys_sent",	0x27},
252 	{"cycles_replys_recev",	0x28},
253 	{"cycles_link_retrain",	0x29},
254 	{"clear_pic",		0xff}
255 };
256 
257 static ni_kev_mask_t
258 zam_gpd_perf_events[] = {
259 	{"none",		0x0},
260 	{"clock_cycles",	0x1},
261 	{"clear_pic",		0xf}
262 };
263 
264 static ni_kev_mask_t
265 zam_asu_perf_events[] = {
266 	{"none",		0x0},
267 	{"clock_cycles",	0x1},
268 	{"asu_in_pck",		0x2},
269 	{"asu_out_pck",		0x3},
270 	{"asu_CAM_hit",		0x4},
271 	{"asu_wakeup",		0x5},
272 	{"clear_pic",		0xf}
273 };
274 #endif
275 
276 #if defined(NIAGARA_IMPL)
277 /*
278  * Niagara JBUS Performance Events
279  */
280 static ni_kev_mask_t
281 niagara_jbus_events[] = {
282 	{"jbus_cycles",		0x1},
283 	{"dma_reads",		0x2},
284 	{"dma_read_latency",	0x3},
285 	{"dma_writes",		0x4},
286 	{"dma_write8",		0x5},
287 	{"ordering_waits",	0x6},
288 	{"pio_reads",		0x8},
289 	{"pio_read_latency",	0x9},
290 	{"aok_dok_off_cycles",	0xc},
291 	{"aok_off_cycles",	0xd},
292 	{"dok_off_cycles",	0xe},
293 	{"clear_pic",		0xf}
294 };
295 #endif
296 
297 /*
298  * Create the picN kstats for DRAM, JBUS and Zambezi events
299  */
300 void
301 niagara_kstat_init()
302 {
303 	int i;
304 	ni_ksinfo_t *ksinfop;
305 #if defined(VFALLS_IMPL) || defined(KT_IMPL)
306 	uint64_t stat, pcr;
307 #endif
308 
309 #ifdef DEBUG
310 	if (ni_perf_debug)
311 		printf("ni_kstat_init called\n");
312 #endif
313 
314 	/*
315 	 * Create DRAM perf events kstat
316 	 */
317 	for (i = 0; i < DRAM_BANKS; i++) {
318 #if defined(VFALLS_IMPL) || defined(KT_IMPL)
319 		/* check if this dram instance is enabled in the HW */
320 		stat = hv_niagara_getperf(dram_perf_regs[i].pcr_reg, &pcr);
321 		if ((stat != H_EINVAL) && (stat != H_ENOTSUPPORTED)) {
322 #endif
323 			ksinfop = (ni_ksinfo_t *)kmem_zalloc(
324 			    sizeof (ni_ksinfo_t), KM_NOSLEEP);
325 
326 			if (ksinfop == NULL) {
327 				cmn_err(CE_WARN,
328 				    "%s: no space for dram kstat\n",
329 				    cpu_module_name);
330 				break;
331 			}
332 			ksinfop->pic_no_evs =
333 			    sizeof (niagara_dram_events) /
334 			    sizeof (ni_kev_mask_t);
335 			ksinfop->pic_sel_shift[0] = DRAM_PIC0_SEL_SHIFT;
336 			ksinfop->pic_shift[0] = DRAM_PIC0_SHIFT;
337 			ksinfop->pic_mask[0] = DRAM_PIC0_MASK;
338 			ksinfop->pic_sel_shift[1] = DRAM_PIC1_SEL_SHIFT;
339 			ksinfop->pic_shift[1] = DRAM_PIC1_SHIFT;
340 			ksinfop->pic_mask[1] = DRAM_PIC1_MASK;
341 			ksinfop->pic_reg[0] = dram_perf_regs[i].pic_reg[0];
342 			ksinfop->pcr_reg = dram_perf_regs[i].pcr_reg;
343 			ni_dram_kstats[i] = ksinfop;
344 
345 			/* create basic pic event/mask pair (only once) */
346 			if (i == 0)
347 				ni_create_name_kstat("dram", ksinfop,
348 				    niagara_dram_events);
349 
350 			/* create counter kstats */
351 			ni_dram_kstats[i]->cntr_ksp = ni_create_cntr_kstat(
352 			    "dram", i, ni_cntr_kstat_update, ksinfop);
353 #if defined(VFALLS_IMPL) || defined(KT_IMPL)
354 		}
355 #endif
356 	}
357 
358 #ifdef VFALLS_IMPL
359 	/*
360 	 * Create Zambezi LPU perf events kstat
361 	 */
362 	for (i = 0; i < ZAMBEZI_LPU_COUNTERS; i++) {
363 		/* check if this Zambezi LPU instance is enabled in the HW */
364 		stat = hv_niagara_getperf(lpu_perf_regs[i].pcr_reg, &pcr);
365 		if ((stat != H_EINVAL) && (stat != H_ENOTSUPPORTED)) {
366 			ksinfop = (ni_ksinfo_t *)kmem_zalloc(
367 			    sizeof (ni_ksinfo_t), KM_NOSLEEP);
368 
369 			if (ksinfop == NULL) {
370 				cmn_err(CE_WARN,
371 				    "%s: no space for zambezi lpu kstat\n",
372 				    cpu_module_name);
373 				break;
374 			}
375 			ksinfop->pic_no_evs =
376 			    sizeof (zam_lpu_perf_events) /
377 			    sizeof (ni_kev_mask_t);
378 			ksinfop->pic_sel_shift[0] = ZAMBEZI_PIC0_SEL_SHIFT;
379 			ksinfop->pic_reg[0] = lpu_perf_regs[i].pic_reg[0];
380 			ksinfop->pic_sel_shift[1] = ZAMBEZI_PIC1_SEL_SHIFT;
381 			ksinfop->pic_reg[1] = lpu_perf_regs[i].pic_reg[1];
382 			ksinfop->pcr_reg = lpu_perf_regs[i].pcr_reg;
383 			zam_lpu_kstats[i] = ksinfop;
384 
385 			/* create basic pic event/mask pair (only once) */
386 			if (i == 0)
387 				ni_create_name_kstat("lpu", ksinfop,
388 				    zam_lpu_perf_events);
389 
390 			/* create counter kstats */
391 			zam_lpu_kstats[i]->cntr_ksp = ni_create_cntr_kstat(
392 			    "lpu", i, zam_cntr_kstat_update, ksinfop);
393 		}
394 	}
395 	/*
396 	 * Create Zambezi GPD perf events kstat
397 	 */
398 	for (i = 0; i < ZAMBEZI_GPD_COUNTERS; i++) {
399 		/* check if this Zambezi GPD instance is enabled in the HW */
400 		stat = hv_niagara_getperf(gpd_perf_regs[i].pcr_reg, &pcr);
401 		if ((stat != H_EINVAL) && (stat != H_ENOTSUPPORTED)) {
402 			ksinfop = (ni_ksinfo_t *)kmem_zalloc(
403 			    sizeof (ni_ksinfo_t), KM_NOSLEEP);
404 
405 			if (ksinfop == NULL) {
406 				cmn_err(CE_WARN,
407 				    "%s: no space for zambezi gpd kstat\n",
408 				    cpu_module_name);
409 				break;
410 			}
411 			ksinfop->pic_no_evs =
412 			    sizeof (zam_gpd_perf_events) /
413 			    sizeof (ni_kev_mask_t);
414 			ksinfop->pic_sel_shift[0] = ZAMBEZI_PIC0_SEL_SHIFT;
415 			ksinfop->pic_reg[0] = gpd_perf_regs[i].pic_reg[0];
416 			ksinfop->pic_sel_shift[1] = ZAMBEZI_PIC1_SEL_SHIFT;
417 			ksinfop->pic_reg[1] = gpd_perf_regs[i].pic_reg[1];
418 			ksinfop->pcr_reg = gpd_perf_regs[i].pcr_reg;
419 			zam_gpd_kstats[i] = ksinfop;
420 
421 			/* create basic pic event/mask pair (only once) */
422 			if (i == 0)
423 				ni_create_name_kstat("gpd", ksinfop,
424 				    zam_gpd_perf_events);
425 
426 			/* create counter kstats */
427 			zam_lpu_kstats[i]->cntr_ksp = ni_create_cntr_kstat(
428 			    "gpd", i, zam_cntr_kstat_update, ksinfop);
429 		}
430 	}
431 	/*
432 	 * Create Zambezi ASU perf events kstat
433 	 */
434 	for (i = 0; i < ZAMBEZI_ASU_COUNTERS; i++) {
435 		/* check if this Zambezi ASU instance is enabled in the HW */
436 		stat = hv_niagara_getperf(asu_perf_regs[i].pcr_reg, &pcr);
437 		if ((stat != H_EINVAL) && (stat != H_ENOTSUPPORTED)) {
438 			ksinfop = (ni_ksinfo_t *)kmem_zalloc(
439 			    sizeof (ni_ksinfo_t), KM_NOSLEEP);
440 
441 			if (ksinfop == NULL) {
442 				cmn_err(CE_WARN,
443 				    "%s: no space for zambezi asu kstat\n",
444 				    cpu_module_name);
445 				break;
446 			}
447 			ksinfop->pic_no_evs =
448 			    sizeof (zam_asu_perf_events) /
449 			    sizeof (ni_kev_mask_t);
450 			ksinfop->pic_sel_shift[0] = ZAMBEZI_PIC0_SEL_SHIFT;
451 			ksinfop->pic_reg[0] = asu_perf_regs[i].pic_reg[0];
452 			ksinfop->pic_sel_shift[1] = ZAMBEZI_PIC1_SEL_SHIFT;
453 			ksinfop->pic_reg[1] = asu_perf_regs[i].pic_reg[1];
454 			ksinfop->pcr_reg = asu_perf_regs[i].pcr_reg;
455 			zam_asu_kstats[i] = ksinfop;
456 
457 			/* create basic pic event/mask pair (only once) */
458 			if (i == 0)
459 				ni_create_name_kstat("asu", ksinfop,
460 				    zam_asu_perf_events);
461 
462 			/* create counter kstats */
463 			zam_lpu_kstats[i]->cntr_ksp = ni_create_cntr_kstat(
464 			    "asu", i, zam_cntr_kstat_update, ksinfop);
465 		}
466 	}
467 #endif
468 
469 #if defined(NIAGARA_IMPL)
470 	/*
471 	 * Create JBUS perf events kstat
472 	 */
473 	ni_jbus_kstat = (ni_ksinfo_t *)kmem_alloc(sizeof (ni_ksinfo_t),
474 	    KM_NOSLEEP);
475 
476 	if (ni_jbus_kstat == NULL) {
477 		cmn_err(CE_WARN, "%s: no space for niagara jbus kstat\n",
478 		    cpu_module_name);
479 	} else {
480 		ni_jbus_kstat->pic_no_evs =
481 		    sizeof (niagara_jbus_events) / sizeof (ni_kev_mask_t);
482 		ni_jbus_kstat->pic_sel_shift[0] = NIAGARA_JBUS_PIC0_SEL_SHIFT;
483 		ni_jbus_kstat->pic_shift[0] = NIAGARA_JBUS_PIC0_SHIFT;
484 		ni_jbus_kstat->pic_mask[0] = NIAGARA_JBUS_PIC0_MASK;
485 		ni_jbus_kstat->pic_sel_shift[1] = NIAGARA_JBUS_PIC1_SEL_SHIFT;
486 		ni_jbus_kstat->pic_shift[1] = NIAGARA_JBUS_PIC1_SHIFT;
487 		ni_jbus_kstat->pic_mask[1] = NIAGARA_JBUS_PIC1_MASK;
488 		ni_jbus_kstat->pic_reg[0] = HV_NIAGARA_JBUS_COUNT;
489 		ni_jbus_kstat->pcr_reg = HV_NIAGARA_JBUS_CTL;
490 		ni_create_name_kstat("jbus", ni_jbus_kstat,
491 		    niagara_jbus_events);
492 		ni_jbus_kstat->cntr_ksp = ni_create_cntr_kstat("jbus", 0,
493 		    ni_cntr_kstat_update, ni_jbus_kstat);
494 	}
495 #endif
496 }
497 
498 void
499 niagara_kstat_fini()
500 {
501 	int i;
502 
503 #ifdef DEBUG
504 	if (ni_perf_debug)
505 		printf("ni_kstat_fini called\n");
506 #endif
507 
508 	for (i = 0; i < DRAM_BANKS; i++) {
509 		if (ni_dram_kstats[i] != NULL) {
510 			ni_delete_name_kstat(ni_dram_kstats[i]);
511 			if (ni_dram_kstats[i]->cntr_ksp != NULL)
512 				kstat_delete(ni_dram_kstats[i]->cntr_ksp);
513 			kmem_free(ni_dram_kstats[i], sizeof (ni_ksinfo_t));
514 			ni_dram_kstats[i] = NULL;
515 		}
516 	}
517 
518 #if defined(VFALLS_IMPL)
519 	for (i = 0; i < ZAMBEZI_LPU_COUNTERS; i++) {
520 		if (zam_lpu_kstats[i] != NULL) {
521 			ni_delete_name_kstat(zam_lpu_kstats[i]);
522 			if (zam_lpu_kstats[i]->cntr_ksp != NULL)
523 				kstat_delete(zam_lpu_kstats[i]->cntr_ksp);
524 			kmem_free(zam_lpu_kstats[i], sizeof (ni_ksinfo_t));
525 			zam_lpu_kstats[i] = NULL;
526 		}
527 	}
528 
529 	for (i = 0; i < ZAMBEZI_GPD_COUNTERS; i++) {
530 		if (zam_gpd_kstats[i] != NULL) {
531 			ni_delete_name_kstat(zam_gpd_kstats[i]);
532 			if (zam_gpd_kstats[i]->cntr_ksp != NULL)
533 				kstat_delete(zam_gpd_kstats[i]->cntr_ksp);
534 			kmem_free(zam_gpd_kstats[i], sizeof (ni_ksinfo_t));
535 			zam_gpd_kstats[i] = NULL;
536 		}
537 	}
538 
539 	for (i = 0; i < ZAMBEZI_ASU_COUNTERS; i++) {
540 		if (zam_asu_kstats[i] != NULL) {
541 			ni_delete_name_kstat(zam_asu_kstats[i]);
542 			if (zam_asu_kstats[i]->cntr_ksp != NULL)
543 				kstat_delete(zam_asu_kstats[i]->cntr_ksp);
544 			kmem_free(zam_asu_kstats[i], sizeof (ni_ksinfo_t));
545 			zam_asu_kstats[i] = NULL;
546 		}
547 	}
548 #endif
549 
550 #if defined(NIAGARA_IMPL)
551 	if (ni_jbus_kstat != NULL) {
552 		ni_delete_name_kstat(ni_jbus_kstat);
553 		if (ni_jbus_kstat->cntr_ksp != NULL)
554 			kstat_delete(ni_jbus_kstat->cntr_ksp);
555 		kmem_free(ni_jbus_kstat, sizeof (ni_ksinfo_t));
556 		ni_jbus_kstat = NULL;
557 	}
558 #endif
559 }
560 
561 static void
562 ni_create_name_kstat(char *name, ni_ksinfo_t *pp, ni_kev_mask_t *ev)
563 {
564 	int	i;
565 
566 #ifdef DEBUG
567 	if (ni_perf_debug > 1)
568 		printf("ni_create_name_kstat: name: %s\n", name);
569 #endif
570 	for (i = 0; i < NUM_OF_PICS; i++) {
571 		pp->pic_name_ksp[i] = ni_create_picN_kstat(name,
572 		    i, pp->pic_sel_shift[i], pp->pic_no_evs, ev);
573 
574 		if (pp->pic_name_ksp[i] == NULL) {
575 			cmn_err(CE_WARN, "%s: unable to create name kstat",
576 			    cpu_module_name);
577 		}
578 	}
579 }
580 
581 static void
582 ni_delete_name_kstat(ni_ksinfo_t *pp)
583 {
584 	int	i;
585 
586 	if (pp != NULL) {
587 		for (i = 0; i < NUM_OF_PICS; i++) {
588 			if (pp->pic_name_ksp[i] != NULL)
589 				kstat_delete(pp->pic_name_ksp[i]);
590 		}
591 	}
592 }
593 
594 /*
595  * Create the picN kstat. Returns a pointer to the
596  * kstat which the driver must store to allow it
597  * to be deleted when necessary.
598  */
599 static kstat_t *
600 ni_create_picN_kstat(char *mod_name, int pic, int pic_sel_shift,
601 	int num_ev, ni_kev_mask_t *ev_array)
602 {
603 	struct kstat_named *pic_named_data;
604 	int	inst = 0;
605 	int	event;
606 	char	pic_name[30];
607 	kstat_t	*picN_ksp = NULL;
608 
609 	(void) sprintf(pic_name, "pic%d", pic);
610 	if ((picN_ksp = kstat_create(mod_name, inst, pic_name,
611 	    "bus", KSTAT_TYPE_NAMED, num_ev, NULL)) == NULL) {
612 		cmn_err(CE_WARN, "%s %s : kstat create failed",
613 		    mod_name, pic_name);
614 
615 		/*
616 		 * It is up to the calling function to delete any kstats
617 		 * that may have been created already. We just
618 		 * return NULL to indicate an error has occured.
619 		 */
620 		return (NULL);
621 	}
622 
623 	pic_named_data = (struct kstat_named *)
624 	    picN_ksp->ks_data;
625 
626 	/*
627 	 * Write event names and their associated pcr masks. The
628 	 * last entry in the array (clear_pic) is added seperately
629 	 * below as the pic value must be inverted.
630 	 */
631 	for (event = 0; event < num_ev - 1; event++) {
632 		pic_named_data[event].value.ui64 =
633 		    (ev_array[event].pcr_mask << pic_sel_shift);
634 
635 		kstat_named_init(&pic_named_data[event],
636 		    ev_array[event].event_name,
637 		    KSTAT_DATA_UINT64);
638 	}
639 
640 	/*
641 	 * add the clear_pic entry.
642 	 */
643 	pic_named_data[event].value.ui64 =
644 	    (uint64_t)~(ev_array[event].pcr_mask << pic_sel_shift);
645 
646 	kstat_named_init(&pic_named_data[event], ev_array[event].event_name,
647 	    KSTAT_DATA_UINT64);
648 
649 	kstat_install(picN_ksp);
650 
651 	return (picN_ksp);
652 }
653 
654 /*
655  * Create the "counters" kstat.
656  */
657 static kstat_t *
658 ni_create_cntr_kstat(char *name, int instance, int (*update)(kstat_t *, int),
659 	void *ksinfop)
660 {
661 	struct kstat	*counters_ksp;
662 	struct kstat_named	*counters_named_data;
663 	char		pic_str[10];
664 	int		i;
665 	int		num_pics = NUM_OF_PICS;
666 
667 #ifdef DEBUG
668 	if (ni_perf_debug > 1)
669 		printf("ni_create_cntr_kstat: name: %s instance: %d\n",
670 		    name, instance);
671 #endif
672 
673 	/*
674 	 * Size of kstat is num_pics + 1 as it
675 	 * also contains the %pcr
676 	 */
677 	if ((counters_ksp = kstat_create(name, instance, "counters", "bus",
678 	    KSTAT_TYPE_NAMED, num_pics + 1, KSTAT_FLAG_WRITABLE)) == NULL) {
679 		cmn_err(CE_WARN,
680 		    "%s: kstat_create for %s%d failed", cpu_module_name,
681 		    name, instance);
682 		return (NULL);
683 	}
684 
685 	counters_named_data = (struct kstat_named *)(counters_ksp->ks_data);
686 
687 	/*
688 	 * Iinitialize the named kstats
689 	 */
690 	kstat_named_init(&counters_named_data[0], "pcr", KSTAT_DATA_UINT64);
691 
692 	for (i = 0; i < num_pics; i++) {
693 		(void) sprintf(pic_str, "pic%d", i);
694 
695 		kstat_named_init(&counters_named_data[i+1], pic_str,
696 		    KSTAT_DATA_UINT64);
697 	}
698 
699 	/*
700 	 * Store the register offset's in the kstat's
701 	 * private field so that they are available
702 	 * to the update function.
703 	 */
704 	counters_ksp->ks_private = (void *)ksinfop;
705 	counters_ksp->ks_update = update;
706 
707 	kstat_install(counters_ksp);
708 
709 	return (counters_ksp);
710 }
711 
712 #if defined(VFALLS_IMPL)
713 /*
714  * zambezi kstat update function. Handles reads/writes
715  * from/to kstat.
716  */
717 static int
718 zam_cntr_kstat_update(kstat_t *ksp, int rw)
719 {
720 	struct kstat_named	*data_p;
721 	ni_ksinfo_t	*ksinfop = ksp->ks_private;
722 	uint64_t	pic0, pic1, pcr;
723 	int		stat = 0;
724 	uint64_t	pic0_stat = 0, pic1_stat = 0, pcr_stat = 0;
725 
726 	data_p = (struct kstat_named *)ksp->ks_data;
727 
728 	if (rw == KSTAT_WRITE) {
729 #ifdef DEBUG
730 		if (ni_perf_debug)
731 			printf("zam_cntr_kstat_update: wr pcr-%d: %lx\n",
732 			    ksinfop->pcr_reg, data_p[0].value.ui64);
733 #endif
734 		if (hv_niagara_setperf(ksinfop->pcr_reg, data_p[0].value.ui64))
735 			stat = EACCES;
736 	} else {
737 		do {
738 			pic0_stat = hv_niagara_getperf(ksinfop->pic_reg[0],
739 			    &pic0);
740 		} while (pic0_stat == H_EWOULDBLOCK);
741 		do {
742 			pic1_stat = hv_niagara_getperf(ksinfop->pic_reg[1],
743 			    &pic1);
744 		} while (pic1_stat == H_EWOULDBLOCK);
745 		do {
746 			pcr_stat = hv_niagara_getperf(ksinfop->pcr_reg,
747 			    &pcr);
748 		} while (pcr_stat == H_EWOULDBLOCK);
749 		if (pic0_stat != 0 || pic1_stat != 0 || pcr_stat != 0)
750 			stat = EACCES;
751 		else {
752 			data_p[0].value.ui64 = pcr;
753 			data_p[1].value.ui64 = pic0;
754 			data_p[2].value.ui64 = pic1;
755 		}
756 #ifdef DEBUG
757 		if (ni_perf_debug)
758 			printf("zam_cntr_kstat_update: rd pcr%d: %lx  "
759 			    "pic0: %16lx pic1: %16lx\n",
760 			    ksinfop->pcr_reg, pcr,
761 			    data_p[1].value.ui64, data_p[2].value.ui64);
762 #endif
763 	}
764 
765 	return (stat);
766 }
767 #endif
768 
769 /*
770  * kstat update function. Handles reads/writes
771  * from/to kstat.
772  */
773 static int
774 ni_cntr_kstat_update(kstat_t *ksp, int rw)
775 {
776 	struct kstat_named	*data_p;
777 	ni_ksinfo_t	*ksinfop = ksp->ks_private;
778 	uint64_t	pic, pcr;
779 	int		stat = 0;
780 	uint32_t	pic0, pic1;
781 
782 	data_p = (struct kstat_named *)ksp->ks_data;
783 
784 	if (rw == KSTAT_WRITE) {
785 #ifdef DEBUG
786 		if (ni_perf_debug)
787 			printf("ni_cntr_kstat_update: wr pcr-%d: %lx\n",
788 			    ksinfop->pcr_reg, data_p[0].value.ui64);
789 #endif
790 		if (hv_niagara_setperf(ksinfop->pcr_reg, data_p[0].value.ui64))
791 			stat = EACCES;
792 	} else {
793 		if (hv_niagara_getperf(ksinfop->pic_reg[0], &pic) != 0 ||
794 		    hv_niagara_getperf(ksinfop->pcr_reg, &pcr) != 0)
795 			stat = EACCES;
796 		else {
797 			data_p[0].value.ui64 = pcr;
798 
799 			/*
800 			 * Generate a 32-bit PIC0 value by detecting overflow
801 			 */
802 			pic0 = (uint32_t)((pic >> ksinfop->pic_shift[0]) &
803 			    ksinfop->pic_mask[0]);
804 			if (pic0 < ksinfop->pic_last_val[0])
805 				ksinfop->pic_overflow[0]++;
806 			ksinfop->pic_last_val[0] = pic0;
807 			pic0 += (ksinfop->pic_overflow[0] & 1) << 31;
808 			data_p[1].value.ui64 = (uint64_t)pic0;
809 
810 			/*
811 			 * Generate a 32-bit PIC1 value by detecting overflow
812 			 */
813 			pic1 = (uint32_t)((pic >> ksinfop->pic_shift[1]) &
814 			    ksinfop->pic_mask[1]);
815 			if (pic1 < ksinfop->pic_last_val[1])
816 				ksinfop->pic_overflow[1]++;
817 			ksinfop->pic_last_val[1] = pic1;
818 			pic1 += (ksinfop->pic_overflow[1] & 1) << 31;
819 			data_p[2].value.ui64 = (uint64_t)pic1;
820 		}
821 #ifdef DEBUG
822 		if (ni_perf_debug)
823 			printf("ni_cntr_kstat_update: rd pcr%d: %lx  "
824 			    "pic%d: %16lx pic0: %8lx pic1: %8lx\n",
825 			    ksinfop->pcr_reg, pcr, ksinfop->pic_reg[0], pic,
826 			    data_p[1].value.ui64, data_p[2].value.ui64);
827 #endif
828 	}
829 	return (stat);
830 }
831