1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21 /*
22 * Copyright 2009 Sun Microsystems, Inc. All rights reserved.
23 * Use is subject to license terms.
24 */
25
26 /*
27 * PCI Interrupt Block (RISCx) implementation
28 * initialization
29 * interrupt enable/disable/clear and mapping register manipulation
30 */
31
32 #include <sys/types.h>
33 #include <sys/kmem.h>
34 #include <sys/async.h>
35 #include <sys/systm.h> /* panicstr */
36 #include <sys/spl.h>
37 #include <sys/sunddi.h>
38 #include <sys/machsystm.h> /* intr_dist_add */
39 #include <sys/ddi_impldefs.h>
40 #include <sys/clock.h>
41 #include <sys/cpuvar.h>
42 #include <sys/pci/pci_obj.h>
43
44 #ifdef _STARFIRE
45 #include <sys/starfire.h>
46 #endif /* _STARFIRE */
47
48 /*LINTLIBRARY*/
49 static uint_t ib_intr_reset(void *arg);
50
51 void
ib_create(pci_t * pci_p)52 ib_create(pci_t *pci_p)
53 {
54 dev_info_t *dip = pci_p->pci_dip;
55 ib_t *ib_p;
56 uintptr_t a;
57 int i;
58
59 /*
60 * Allocate interrupt block state structure and link it to
61 * the pci state structure.
62 */
63 ib_p = kmem_zalloc(sizeof (ib_t), KM_SLEEP);
64 pci_p->pci_ib_p = ib_p;
65 ib_p->ib_pci_p = pci_p;
66
67 a = pci_ib_setup(ib_p);
68
69 /*
70 * Determine virtual addresses of interrupt mapping, clear and diag
71 * registers that have common offsets.
72 */
73 ib_p->ib_slot_clear_intr_regs =
74 a + COMMON_IB_SLOT_CLEAR_INTR_REG_OFFSET;
75 ib_p->ib_intr_retry_timer_reg =
76 (uint64_t *)(a + COMMON_IB_INTR_RETRY_TIMER_OFFSET);
77 ib_p->ib_slot_intr_state_diag_reg =
78 (uint64_t *)(a + COMMON_IB_SLOT_INTR_STATE_DIAG_REG);
79 ib_p->ib_obio_intr_state_diag_reg =
80 (uint64_t *)(a + COMMON_IB_OBIO_INTR_STATE_DIAG_REG);
81
82 if (CHIP_TYPE(pci_p) != PCI_CHIP_XMITS) {
83 ib_p->ib_upa_imr[0] = (volatile uint64_t *)
84 (a + COMMON_IB_UPA0_INTR_MAP_REG_OFFSET);
85 ib_p->ib_upa_imr[1] = (volatile uint64_t *)
86 (a + COMMON_IB_UPA1_INTR_MAP_REG_OFFSET);
87 }
88
89 DEBUG2(DBG_ATTACH, dip, "ib_create: slot_imr=%x, slot_cir=%x\n",
90 ib_p->ib_slot_intr_map_regs, ib_p->ib_obio_intr_map_regs);
91 DEBUG2(DBG_ATTACH, dip, "ib_create: obio_imr=%x, obio_cir=%x\n",
92 ib_p->ib_slot_clear_intr_regs, ib_p->ib_obio_clear_intr_regs);
93 DEBUG2(DBG_ATTACH, dip, "ib_create: upa0_imr=%x, upa1_imr=%x\n",
94 ib_p->ib_upa_imr[0], ib_p->ib_upa_imr[1]);
95 DEBUG3(DBG_ATTACH, dip,
96 "ib_create: retry_timer=%x, obio_diag=%x slot_diag=%x\n",
97 ib_p->ib_intr_retry_timer_reg,
98 ib_p->ib_obio_intr_state_diag_reg,
99 ib_p->ib_slot_intr_state_diag_reg);
100
101 ib_p->ib_ino_lst = (ib_ino_info_t *)NULL;
102 mutex_init(&ib_p->ib_intr_lock, NULL, MUTEX_DRIVER, NULL);
103 mutex_init(&ib_p->ib_ino_lst_mutex, NULL, MUTEX_DRIVER, NULL);
104
105 DEBUG1(DBG_ATTACH, dip, "ib_create: numproxy=%x\n",
106 pci_p->pci_numproxy);
107 for (i = 1; i <= pci_p->pci_numproxy; i++) {
108 set_intr_mapping_reg(pci_p->pci_id,
109 (uint64_t *)ib_p->ib_upa_imr[i - 1], i);
110 }
111
112 ib_configure(ib_p);
113 bus_func_register(BF_TYPE_RESINTR, ib_intr_reset, ib_p);
114 }
115
116 void
ib_destroy(pci_t * pci_p)117 ib_destroy(pci_t *pci_p)
118 {
119 ib_t *ib_p = pci_p->pci_ib_p;
120 dev_info_t *dip = pci_p->pci_dip;
121
122 DEBUG0(DBG_IB, dip, "ib_destroy\n");
123 bus_func_unregister(BF_TYPE_RESINTR, ib_intr_reset, ib_p);
124
125 intr_dist_rem_weighted(ib_intr_dist_all, ib_p);
126 mutex_destroy(&ib_p->ib_ino_lst_mutex);
127 mutex_destroy(&ib_p->ib_intr_lock);
128
129 ib_free_ino_all(ib_p);
130
131 kmem_free(ib_p, sizeof (ib_t));
132 pci_p->pci_ib_p = NULL;
133 }
134
135 void
ib_configure(ib_t * ib_p)136 ib_configure(ib_t *ib_p)
137 {
138 /* XXX could be different between psycho and schizo */
139 *ib_p->ib_intr_retry_timer_reg = pci_intr_retry_intv;
140 }
141
142 /*
143 * can only used for psycho internal interrupts thermal, power,
144 * ue, ce, pbm
145 */
146 void
ib_intr_enable(pci_t * pci_p,ib_ino_t ino)147 ib_intr_enable(pci_t *pci_p, ib_ino_t ino)
148 {
149 ib_t *ib_p = pci_p->pci_ib_p;
150 ib_mondo_t mondo = IB_INO_TO_MONDO(ib_p, ino);
151 volatile uint64_t *imr_p = ib_intr_map_reg_addr(ib_p, ino);
152 uint_t cpu_id;
153
154 /*
155 * Determine the cpu for the interrupt.
156 */
157 mutex_enter(&ib_p->ib_intr_lock);
158 cpu_id = intr_dist_cpuid();
159 #ifdef _STARFIRE
160 cpu_id = pc_translate_tgtid(IB2CB(ib_p)->cb_ittrans_cookie, cpu_id,
161 IB_GET_MAPREG_INO(ino));
162 #endif /* _STARFIRE */
163 DEBUG2(DBG_IB, pci_p->pci_dip,
164 "ib_intr_enable: ino=%x cpu_id=%x\n", ino, cpu_id);
165
166 *imr_p = ib_get_map_reg(mondo, cpu_id);
167 IB_INO_INTR_CLEAR(ib_clear_intr_reg_addr(ib_p, ino));
168 mutex_exit(&ib_p->ib_intr_lock);
169 }
170
171 /*
172 * Disable the interrupt via its interrupt mapping register.
173 * Can only be used for internal interrupts: thermal, power, ue, ce, pbm.
174 * If called under interrupt context, wait should be set to 0
175 */
176 void
ib_intr_disable(ib_t * ib_p,ib_ino_t ino,int wait)177 ib_intr_disable(ib_t *ib_p, ib_ino_t ino, int wait)
178 {
179 volatile uint64_t *imr_p = ib_intr_map_reg_addr(ib_p, ino);
180 volatile uint64_t *state_reg_p = IB_INO_INTR_STATE_REG(ib_p, ino);
181 hrtime_t start_time;
182
183 /* disable the interrupt */
184 mutex_enter(&ib_p->ib_intr_lock);
185 IB_INO_INTR_OFF(imr_p);
186 *imr_p; /* flush previous write */
187 mutex_exit(&ib_p->ib_intr_lock);
188
189 if (!wait)
190 goto wait_done;
191
192 start_time = gethrtime();
193 /* busy wait if there is interrupt being processed */
194 while (IB_INO_INTR_PENDING(state_reg_p, ino) && !panicstr) {
195 if (gethrtime() - start_time > pci_intrpend_timeout) {
196 pbm_t *pbm_p = ib_p->ib_pci_p->pci_pbm_p;
197 cmn_err(CE_WARN, "%s:%s: ib_intr_disable timeout %x",
198 pbm_p->pbm_nameinst_str,
199 pbm_p->pbm_nameaddr_str, ino);
200 break;
201 }
202 }
203 wait_done:
204 IB_INO_INTR_PEND(ib_clear_intr_reg_addr(ib_p, ino));
205 #ifdef _STARFIRE
206 pc_ittrans_cleanup(IB2CB(ib_p)->cb_ittrans_cookie,
207 (volatile uint64_t *)(uintptr_t)ino);
208 #endif /* _STARFIRE */
209 }
210
211 /* can only used for psycho internal interrupts thermal, power, ue, ce, pbm */
212 void
ib_nintr_clear(ib_t * ib_p,ib_ino_t ino)213 ib_nintr_clear(ib_t *ib_p, ib_ino_t ino)
214 {
215 uint64_t *clr_reg = ib_clear_intr_reg_addr(ib_p, ino);
216 IB_INO_INTR_CLEAR(clr_reg);
217 }
218
219 /*
220 * distribute PBM and UPA interrupts. ino is set to 0 by caller if we
221 * are dealing with UPA interrupts (without inos).
222 */
223 void
ib_intr_dist_nintr(ib_t * ib_p,ib_ino_t ino,volatile uint64_t * imr_p)224 ib_intr_dist_nintr(ib_t *ib_p, ib_ino_t ino, volatile uint64_t *imr_p)
225 {
226 volatile uint64_t imr = *imr_p;
227 uint32_t cpu_id;
228
229 if (!IB_INO_INTR_ISON(imr))
230 return;
231
232 cpu_id = intr_dist_cpuid();
233
234 #ifdef _STARFIRE
235 if (ino) {
236 cpu_id = pc_translate_tgtid(IB2CB(ib_p)->cb_ittrans_cookie,
237 cpu_id, IB_GET_MAPREG_INO(ino));
238 }
239 #else /* _STARFIRE */
240 if (ib_map_reg_get_cpu(*imr_p) == cpu_id)
241 return;
242 #endif /* _STARFIRE */
243
244 *imr_p = ib_get_map_reg(IB_IMR2MONDO(imr), cpu_id);
245 imr = *imr_p; /* flush previous write */
246 }
247
248 /*
249 * Converts into nsec, ticks logged with a given CPU. Adds nsec to ih.
250 */
251 /*ARGSUSED*/
252 void
ib_cpu_ticks_to_ih_nsec(ib_t * ib_p,ih_t * ih_p,uint32_t cpu_id)253 ib_cpu_ticks_to_ih_nsec(ib_t *ib_p, ih_t *ih_p, uint32_t cpu_id)
254 {
255 extern kmutex_t pciintr_ks_template_lock;
256 hrtime_t ticks;
257
258 /*
259 * Because we are updating two fields in ih_t we must lock
260 * pciintr_ks_template_lock to prevent someone from reading the
261 * kstats after we set ih_ticks to 0 and before we increment
262 * ih_nsec to compensate.
263 *
264 * We must also protect against the interrupt arriving and incrementing
265 * ih_ticks between the time we read it and when we reset it to 0.
266 * To do this we use atomic_swap.
267 */
268
269 ASSERT(MUTEX_HELD(&ib_p->ib_ino_lst_mutex));
270
271 mutex_enter(&pciintr_ks_template_lock);
272 ticks = atomic_swap_64(&ih_p->ih_ticks, 0);
273 ih_p->ih_nsec += (uint64_t)tick2ns(ticks, cpu_id);
274 mutex_exit(&pciintr_ks_template_lock);
275 }
276
277 static void
ib_intr_dist(ib_t * ib_p,ib_ino_info_t * ino_p)278 ib_intr_dist(ib_t *ib_p, ib_ino_info_t *ino_p)
279 {
280 uint32_t cpu_id = ino_p->ino_cpuid;
281 ib_ino_t ino = ino_p->ino_ino;
282 volatile uint64_t imr, *imr_p, *state_reg;
283 hrtime_t start_time;
284
285 ASSERT(MUTEX_HELD(&ib_p->ib_ino_lst_mutex));
286 imr_p = ib_intr_map_reg_addr(ib_p, ino);
287 state_reg = IB_INO_INTR_STATE_REG(ib_p, ino);
288
289 #ifdef _STARFIRE
290 /*
291 * For Starfire it is a pain to check the current target for
292 * the mondo since we have to read the PC asics ITTR slot
293 * assigned to this mondo. It will be much easier to assume
294 * the current target is always different and do the target
295 * reprogram all the time.
296 */
297 cpu_id = pc_translate_tgtid(IB2CB(ib_p)->cb_ittrans_cookie, cpu_id,
298 IB_GET_MAPREG_INO(ino));
299 #else
300 if (ib_map_reg_get_cpu(*imr_p) == cpu_id) /* same cpu, no reprog */
301 return;
302 #endif /* _STARFIRE */
303
304 /* disable interrupt, this could disrupt devices sharing our slot */
305 IB_INO_INTR_OFF(imr_p);
306 imr = *imr_p; /* flush previous write */
307
308 /* busy wait if there is interrupt being processed */
309 start_time = gethrtime();
310 while (IB_INO_INTR_PENDING(state_reg, ino) && !panicstr) {
311 if (gethrtime() - start_time > pci_intrpend_timeout) {
312 pbm_t *pbm_p = ib_p->ib_pci_p->pci_pbm_p;
313 cmn_err(CE_WARN, "%s:%s: ib_intr_dist(%p,%x) timeout",
314 pbm_p->pbm_nameinst_str,
315 pbm_p->pbm_nameaddr_str,
316 imr_p, IB_INO_TO_MONDO(ib_p, ino));
317 break;
318 }
319 }
320 *imr_p = ib_get_map_reg(IB_IMR2MONDO(imr), cpu_id);
321 imr = *imr_p; /* flush previous write */
322 }
323
324 /*
325 * Redistribute interrupts of the specified weight. The first call has a weight
326 * of weight_max, which can be used to trigger initialization for
327 * redistribution. The inos with weight [weight_max, inf.) should be processed
328 * on the "weight == weight_max" call. This first call is followed by calls
329 * of decreasing weights, inos of that weight should be processed. The final
330 * call specifies a weight of zero, this can be used to trigger processing of
331 * stragglers.
332 */
333 void
ib_intr_dist_all(void * arg,int32_t weight_max,int32_t weight)334 ib_intr_dist_all(void *arg, int32_t weight_max, int32_t weight)
335 {
336 ib_t *ib_p = (ib_t *)arg;
337 pci_t *pci_p = ib_p->ib_pci_p;
338 ib_ino_info_t *ino_p;
339 ib_ino_pil_t *ipil_p;
340 ih_t *ih_lst;
341 int32_t dweight;
342 int i;
343
344 if (weight == 0) {
345 mutex_enter(&ib_p->ib_intr_lock);
346 if (CHIP_TYPE(pci_p) != PCI_CHIP_XMITS) {
347 for (i = 0; i < 2; i++)
348 ib_intr_dist_nintr(ib_p, 0,
349 ib_p->ib_upa_imr[i]);
350 }
351 mutex_exit(&ib_p->ib_intr_lock);
352 }
353
354 mutex_enter(&ib_p->ib_ino_lst_mutex);
355
356 /* Perform special processing for first call of a redistribution. */
357 if (weight == weight_max) {
358 for (ino_p = ib_p->ib_ino_lst; ino_p;
359 ino_p = ino_p->ino_next_p) {
360
361 /*
362 * Clear ino_established of each ino on first call.
363 * The ino_established field may be used by a pci
364 * nexus driver's pci_intr_dist_cpuid implementation
365 * when detection of established pci slot-cpu binding
366 * for multi function pci cards.
367 */
368 ino_p->ino_established = 0;
369
370 /*
371 * recompute the ino_intr_weight based on the device
372 * weight of all devinfo nodes sharing the ino (this
373 * will allow us to pick up new weights established by
374 * i_ddi_set_intr_weight()).
375 */
376 ino_p->ino_intr_weight = 0;
377
378 for (ipil_p = ino_p->ino_ipil_p; ipil_p;
379 ipil_p = ipil_p->ipil_next_p) {
380 for (i = 0, ih_lst = ipil_p->ipil_ih_head;
381 i < ipil_p->ipil_ih_size; i++,
382 ih_lst = ih_lst->ih_next) {
383 dweight = i_ddi_get_intr_weight
384 (ih_lst->ih_dip);
385 if (dweight > 0)
386 ino_p->ino_intr_weight +=
387 dweight;
388 }
389 }
390 }
391 }
392
393 for (ino_p = ib_p->ib_ino_lst; ino_p; ino_p = ino_p->ino_next_p) {
394 uint32_t orig_cpuid;
395
396 /*
397 * Get the weight of the ino and determine if we are going to
398 * process call. We wait until an ib_intr_dist_all call of
399 * the proper weight occurs to support redistribution of all
400 * heavy weighted interrupts first (across all nexus driver
401 * instances). This is done to ensure optimal
402 * INTR_WEIGHTED_DIST behavior.
403 */
404 if ((weight == ino_p->ino_intr_weight) ||
405 ((weight >= weight_max) &&
406 (ino_p->ino_intr_weight >= weight_max))) {
407 /* select cpuid to target and mark ino established */
408 orig_cpuid = ino_p->ino_cpuid;
409 if (cpu[orig_cpuid] == NULL)
410 orig_cpuid = CPU->cpu_id;
411 ino_p->ino_cpuid = pci_intr_dist_cpuid(ib_p, ino_p);
412 ino_p->ino_established = 1;
413
414 /* Add device weight of ino devinfos to targeted cpu. */
415 for (ipil_p = ino_p->ino_ipil_p; ipil_p;
416 ipil_p = ipil_p->ipil_next_p) {
417 for (i = 0, ih_lst = ipil_p->ipil_ih_head;
418 i < ipil_p->ipil_ih_size; i++,
419 ih_lst = ih_lst->ih_next) {
420
421 dweight = i_ddi_get_intr_weight(
422 ih_lst->ih_dip);
423 intr_dist_cpuid_add_device_weight(
424 ino_p->ino_cpuid, ih_lst->ih_dip,
425 dweight);
426
427 /*
428 * Different cpus may have different
429 * clock speeds. to account for this,
430 * whenever an interrupt is moved to a
431 * new CPU, we convert the accumulated
432 * ticks into nsec, based upon the clock
433 * rate of the prior CPU.
434 *
435 * It is possible that the prior CPU no
436 * longer exists. In this case, fall
437 * back to using this CPU's clock rate.
438 *
439 * Note that the value in ih_ticks has
440 * already been corrected for any power
441 * savings mode which might have been
442 * in effect.
443 */
444 ib_cpu_ticks_to_ih_nsec(ib_p, ih_lst,
445 orig_cpuid);
446 }
447 }
448
449 /* program the hardware */
450 ib_intr_dist(ib_p, ino_p);
451 }
452 }
453 mutex_exit(&ib_p->ib_ino_lst_mutex);
454 }
455
456 /*
457 * Reset interrupts to IDLE. This function is called during
458 * panic handling after redistributing interrupts; it's needed to
459 * support dumping to network devices after 'sync' from OBP.
460 *
461 * N.B. This routine runs in a context where all other threads
462 * are permanently suspended.
463 */
464 static uint_t
ib_intr_reset(void * arg)465 ib_intr_reset(void *arg)
466 {
467 ib_t *ib_p = (ib_t *)arg;
468 ib_ino_t ino;
469 uint64_t *clr_reg;
470
471 /*
472 * Note that we only actually care about interrupts that are
473 * potentially from network devices.
474 */
475 for (ino = 0; ino <= ib_p->ib_max_ino; ino++) {
476 clr_reg = ib_clear_intr_reg_addr(ib_p, ino);
477 IB_INO_INTR_CLEAR(clr_reg);
478 }
479
480 return (BF_NONE);
481 }
482
483 void
ib_suspend(ib_t * ib_p)484 ib_suspend(ib_t *ib_p)
485 {
486 ib_ino_info_t *ip;
487 pci_t *pci_p = ib_p->ib_pci_p;
488
489 /* save ino_lst interrupts' mapping registers content */
490 mutex_enter(&ib_p->ib_ino_lst_mutex);
491 for (ip = ib_p->ib_ino_lst; ip; ip = ip->ino_next_p)
492 ip->ino_map_reg_save = *ip->ino_map_reg;
493 mutex_exit(&ib_p->ib_ino_lst_mutex);
494
495 if (CHIP_TYPE(pci_p) != PCI_CHIP_XMITS) {
496 ib_p->ib_upa_imr_state[0] = *ib_p->ib_upa_imr[0];
497 ib_p->ib_upa_imr_state[1] = *ib_p->ib_upa_imr[1];
498 }
499 }
500
501 void
ib_resume(ib_t * ib_p)502 ib_resume(ib_t *ib_p)
503 {
504 ib_ino_info_t *ip;
505 pci_t *pci_p = ib_p->ib_pci_p;
506
507 /* restore ino_lst interrupts' mapping registers content */
508 mutex_enter(&ib_p->ib_ino_lst_mutex);
509 for (ip = ib_p->ib_ino_lst; ip; ip = ip->ino_next_p) {
510 IB_INO_INTR_CLEAR(ip->ino_clr_reg); /* set intr to idle */
511 *ip->ino_map_reg = ip->ino_map_reg_save; /* restore IMR */
512 }
513 mutex_exit(&ib_p->ib_ino_lst_mutex);
514
515 if (CHIP_TYPE(pci_p) != PCI_CHIP_XMITS) {
516 *ib_p->ib_upa_imr[0] = ib_p->ib_upa_imr_state[0];
517 *ib_p->ib_upa_imr[1] = ib_p->ib_upa_imr_state[1];
518 }
519 }
520
521 /*
522 * locate ino_info structure on ib_p->ib_ino_lst according to ino#
523 * returns NULL if not found.
524 */
525 ib_ino_info_t *
ib_locate_ino(ib_t * ib_p,ib_ino_t ino_num)526 ib_locate_ino(ib_t *ib_p, ib_ino_t ino_num)
527 {
528 ib_ino_info_t *ino_p = ib_p->ib_ino_lst;
529 ASSERT(MUTEX_HELD(&ib_p->ib_ino_lst_mutex));
530
531 for (; ino_p && ino_p->ino_ino != ino_num; ino_p = ino_p->ino_next_p)
532 ;
533 return (ino_p);
534 }
535
536 #define IB_INO_TO_SLOT(ino) (IB_IS_OBIO_INO(ino) ? 0xff : ((ino) & 0x1f) >> 2)
537
538 ib_ino_pil_t *
ib_new_ino_pil(ib_t * ib_p,ib_ino_t ino_num,uint_t pil,ih_t * ih_p)539 ib_new_ino_pil(ib_t *ib_p, ib_ino_t ino_num, uint_t pil, ih_t *ih_p)
540 {
541 ib_ino_pil_t *ipil_p = kmem_zalloc(sizeof (ib_ino_pil_t), KM_SLEEP);
542 ib_ino_info_t *ino_p;
543
544 if ((ino_p = ib_locate_ino(ib_p, ino_num)) == NULL) {
545 ino_p = kmem_zalloc(sizeof (ib_ino_info_t), KM_SLEEP);
546
547 ino_p->ino_next_p = ib_p->ib_ino_lst;
548 ib_p->ib_ino_lst = ino_p;
549
550 ino_p->ino_ino = ino_num;
551 ino_p->ino_slot_no = IB_INO_TO_SLOT(ino_num);
552 ino_p->ino_ib_p = ib_p;
553 ino_p->ino_clr_reg = ib_clear_intr_reg_addr(ib_p, ino_num);
554 ino_p->ino_map_reg = ib_intr_map_reg_addr(ib_p, ino_num);
555 ino_p->ino_unclaimed_intrs = 0;
556 ino_p->ino_lopil = pil;
557 }
558
559 ih_p->ih_next = ih_p;
560 ipil_p->ipil_pil = pil;
561 ipil_p->ipil_ih_head = ih_p;
562 ipil_p->ipil_ih_tail = ih_p;
563 ipil_p->ipil_ih_start = ih_p;
564 ipil_p->ipil_ih_size = 1;
565 ipil_p->ipil_ino_p = ino_p;
566
567 ipil_p->ipil_next_p = ino_p->ino_ipil_p;
568 ino_p->ino_ipil_p = ipil_p;
569 ino_p->ino_ipil_size++;
570
571 if (ino_p->ino_lopil > pil)
572 ino_p->ino_lopil = pil;
573
574 return (ipil_p);
575 }
576
577 void
ib_delete_ino_pil(ib_t * ib_p,ib_ino_pil_t * ipil_p)578 ib_delete_ino_pil(ib_t *ib_p, ib_ino_pil_t *ipil_p)
579 {
580 ib_ino_info_t *ino_p = ipil_p->ipil_ino_p;
581 ib_ino_pil_t *prev, *next;
582 ushort_t pil = ipil_p->ipil_pil;
583
584 ASSERT(MUTEX_HELD(&ib_p->ib_ino_lst_mutex));
585
586 if (ino_p->ino_ipil_p == ipil_p)
587 ino_p->ino_ipil_p = ipil_p->ipil_next_p;
588 else {
589 for (prev = next = ino_p->ino_ipil_p; next != ipil_p;
590 prev = next, next = next->ipil_next_p)
591 ;
592
593 if (prev)
594 prev->ipil_next_p = ipil_p->ipil_next_p;
595 }
596
597 kmem_free(ipil_p, sizeof (ib_ino_pil_t));
598
599 if ((--ino_p->ino_ipil_size) && (ino_p->ino_lopil == pil)) {
600 for (next = ino_p->ino_ipil_p, pil = next->ipil_pil;
601 next; next = next->ipil_next_p) {
602
603 if (pil > next->ipil_pil)
604 pil = next->ipil_pil;
605 }
606 /*
607 * Value stored in pil should be the lowest pil.
608 */
609 ino_p->ino_lopil = pil;
610 }
611
612 if (ino_p->ino_ipil_size)
613 return;
614
615 if (ib_p->ib_ino_lst == ino_p)
616 ib_p->ib_ino_lst = ino_p->ino_next_p;
617 else {
618 ib_ino_info_t *list = ib_p->ib_ino_lst;
619
620 for (; list->ino_next_p != ino_p; list = list->ino_next_p)
621 ;
622 list->ino_next_p = ino_p->ino_next_p;
623 }
624 }
625
626 /* free all ino when we are detaching */
627 void
ib_free_ino_all(ib_t * ib_p)628 ib_free_ino_all(ib_t *ib_p)
629 {
630 ib_ino_info_t *ino_p = ib_p->ib_ino_lst;
631 ib_ino_info_t *next = NULL;
632
633 while (ino_p) {
634 next = ino_p->ino_next_p;
635 kmem_free(ino_p, sizeof (ib_ino_info_t));
636 ino_p = next;
637 }
638 }
639
640 /*
641 * Locate ib_ino_pil_t structure on ino_p->ino_ipil_p according to ino#
642 * returns NULL if not found.
643 */
644 ib_ino_pil_t *
ib_ino_locate_ipil(ib_ino_info_t * ino_p,uint_t pil)645 ib_ino_locate_ipil(ib_ino_info_t *ino_p, uint_t pil)
646 {
647 ib_ino_pil_t *ipil_p = ino_p->ino_ipil_p;
648
649 for (; ipil_p && ipil_p->ipil_pil != pil; ipil_p = ipil_p->ipil_next_p)
650 ;
651
652 return (ipil_p);
653 }
654
655 void
ib_ino_add_intr(pci_t * pci_p,ib_ino_pil_t * ipil_p,ih_t * ih_p)656 ib_ino_add_intr(pci_t *pci_p, ib_ino_pil_t *ipil_p, ih_t *ih_p)
657 {
658 ib_ino_info_t *ino_p = ipil_p->ipil_ino_p;
659 ib_ino_t ino = ino_p->ino_ino;
660 ib_t *ib_p = ino_p->ino_ib_p;
661 volatile uint64_t *state_reg = IB_INO_INTR_STATE_REG(ib_p, ino);
662 hrtime_t start_time;
663
664 ASSERT(ib_p == pci_p->pci_ib_p);
665 ASSERT(MUTEX_HELD(&ib_p->ib_ino_lst_mutex));
666
667 /* disable interrupt, this could disrupt devices sharing our slot */
668 IB_INO_INTR_OFF(ino_p->ino_map_reg);
669 *ino_p->ino_map_reg;
670
671 /* do NOT modify the link list until after the busy wait */
672
673 /*
674 * busy wait if there is interrupt being processed.
675 * either the pending state will be cleared by the interrupt wrapper
676 * or the interrupt will be marked as blocked indicating that it was
677 * jabbering.
678 */
679 start_time = gethrtime();
680 while ((ino_p->ino_unclaimed_intrs <= pci_unclaimed_intr_max) &&
681 IB_INO_INTR_PENDING(state_reg, ino) && !panicstr) {
682 if (gethrtime() - start_time > pci_intrpend_timeout) {
683 pbm_t *pbm_p = pci_p->pci_pbm_p;
684 cmn_err(CE_WARN, "%s:%s: ib_ino_add_intr %x timeout",
685 pbm_p->pbm_nameinst_str,
686 pbm_p->pbm_nameaddr_str, ino);
687 break;
688 }
689 }
690
691 /* link up ih_t */
692 ih_p->ih_next = ipil_p->ipil_ih_head;
693 ipil_p->ipil_ih_tail->ih_next = ih_p;
694 ipil_p->ipil_ih_tail = ih_p;
695
696 ipil_p->ipil_ih_start = ipil_p->ipil_ih_head;
697 ipil_p->ipil_ih_size++;
698
699 /*
700 * if the interrupt was previously blocked (left in pending state)
701 * because of jabber we need to clear the pending state in case the
702 * jabber has gone away.
703 */
704 if (ino_p->ino_unclaimed_intrs > pci_unclaimed_intr_max) {
705 cmn_err(CE_WARN,
706 "%s%d: ib_ino_add_intr: ino 0x%x has been unblocked",
707 ddi_driver_name(pci_p->pci_dip),
708 ddi_get_instance(pci_p->pci_dip),
709 ino_p->ino_ino);
710 ino_p->ino_unclaimed_intrs = 0;
711 IB_INO_INTR_CLEAR(ino_p->ino_clr_reg);
712 }
713
714 /* re-enable interrupt */
715 IB_INO_INTR_ON(ino_p->ino_map_reg);
716 *ino_p->ino_map_reg;
717 }
718
719 /*
720 * removes pci_ispec_t from the ino's link list.
721 * uses hardware mutex to lock out interrupt threads.
722 * Side effects: interrupt belongs to that ino is turned off on return.
723 * if we are sharing PCI slot with other inos, the caller needs
724 * to turn it back on.
725 */
726 void
ib_ino_rem_intr(pci_t * pci_p,ib_ino_pil_t * ipil_p,ih_t * ih_p)727 ib_ino_rem_intr(pci_t *pci_p, ib_ino_pil_t *ipil_p, ih_t *ih_p)
728 {
729 ib_ino_info_t *ino_p = ipil_p->ipil_ino_p;
730 int i;
731 ib_ino_t ino = ino_p->ino_ino;
732 ih_t *ih_lst = ipil_p->ipil_ih_head;
733 volatile uint64_t *state_reg =
734 IB_INO_INTR_STATE_REG(ino_p->ino_ib_p, ino);
735 hrtime_t start_time;
736
737 ASSERT(MUTEX_HELD(&ino_p->ino_ib_p->ib_ino_lst_mutex));
738 /* disable interrupt, this could disrupt devices sharing our slot */
739 IB_INO_INTR_OFF(ino_p->ino_map_reg);
740 *ino_p->ino_map_reg;
741
742 /* do NOT modify the link list until after the busy wait */
743
744 /*
745 * busy wait if there is interrupt being processed.
746 * either the pending state will be cleared by the interrupt wrapper
747 * or the interrupt will be marked as blocked indicating that it was
748 * jabbering.
749 */
750 start_time = gethrtime();
751 while ((ino_p->ino_unclaimed_intrs <= pci_unclaimed_intr_max) &&
752 IB_INO_INTR_PENDING(state_reg, ino) && !panicstr) {
753 if (gethrtime() - start_time > pci_intrpend_timeout) {
754 pbm_t *pbm_p = pci_p->pci_pbm_p;
755 cmn_err(CE_WARN, "%s:%s: ib_ino_rem_intr %x timeout",
756 pbm_p->pbm_nameinst_str,
757 pbm_p->pbm_nameaddr_str, ino);
758 break;
759 }
760 }
761
762 if (ipil_p->ipil_ih_size == 1) {
763 if (ih_lst != ih_p)
764 goto not_found;
765 /* no need to set head/tail as ino_p will be freed */
766 goto reset;
767 }
768
769 /*
770 * if the interrupt was previously blocked (left in pending state)
771 * because of jabber we need to clear the pending state in case the
772 * jabber has gone away.
773 */
774 if (ino_p->ino_unclaimed_intrs > pci_unclaimed_intr_max) {
775 cmn_err(CE_WARN,
776 "%s%d: ib_ino_rem_intr: ino 0x%x has been unblocked",
777 ddi_driver_name(pci_p->pci_dip),
778 ddi_get_instance(pci_p->pci_dip),
779 ino_p->ino_ino);
780 ino_p->ino_unclaimed_intrs = 0;
781 IB_INO_INTR_CLEAR(ino_p->ino_clr_reg);
782 }
783
784 /* search the link list for ih_p */
785 for (i = 0;
786 (i < ipil_p->ipil_ih_size) && (ih_lst->ih_next != ih_p);
787 i++, ih_lst = ih_lst->ih_next)
788 ;
789 if (ih_lst->ih_next != ih_p)
790 goto not_found;
791
792 /* remove ih_p from the link list and maintain the head/tail */
793 ih_lst->ih_next = ih_p->ih_next;
794 if (ipil_p->ipil_ih_head == ih_p)
795 ipil_p->ipil_ih_head = ih_p->ih_next;
796 if (ipil_p->ipil_ih_tail == ih_p)
797 ipil_p->ipil_ih_tail = ih_lst;
798 ipil_p->ipil_ih_start = ipil_p->ipil_ih_head;
799 reset:
800 if (ih_p->ih_config_handle)
801 pci_config_teardown(&ih_p->ih_config_handle);
802 if (ih_p->ih_ksp != NULL)
803 kstat_delete(ih_p->ih_ksp);
804 kmem_free(ih_p, sizeof (ih_t));
805 ipil_p->ipil_ih_size--;
806
807 return;
808 not_found:
809 DEBUG2(DBG_R_INTX, ino_p->ino_ib_p->ib_pci_p->pci_dip,
810 "ino_p=%x does not have ih_p=%x\n", ino_p, ih_p);
811 }
812
813 ih_t *
ib_intr_locate_ih(ib_ino_pil_t * ipil_p,dev_info_t * rdip,uint32_t inum)814 ib_intr_locate_ih(ib_ino_pil_t *ipil_p, dev_info_t *rdip, uint32_t inum)
815 {
816 ih_t *ih_p = ipil_p->ipil_ih_head;
817 int i;
818
819 for (i = 0; i < ipil_p->ipil_ih_size; i++, ih_p = ih_p->ih_next) {
820 if (ih_p->ih_dip == rdip && ih_p->ih_inum == inum)
821 return (ih_p);
822 }
823
824 return ((ih_t *)NULL);
825 }
826
827 ih_t *
ib_alloc_ih(dev_info_t * rdip,uint32_t inum,uint_t (* int_handler)(caddr_t int_handler_arg1,caddr_t int_handler_arg2),caddr_t int_handler_arg1,caddr_t int_handler_arg2)828 ib_alloc_ih(dev_info_t *rdip, uint32_t inum,
829 uint_t (*int_handler)(caddr_t int_handler_arg1,
830 caddr_t int_handler_arg2),
831 caddr_t int_handler_arg1,
832 caddr_t int_handler_arg2)
833 {
834 ih_t *ih_p;
835
836 ih_p = kmem_alloc(sizeof (ih_t), KM_SLEEP);
837 ih_p->ih_dip = rdip;
838 ih_p->ih_inum = inum;
839 ih_p->ih_intr_state = PCI_INTR_STATE_DISABLE;
840 ih_p->ih_handler = int_handler;
841 ih_p->ih_handler_arg1 = int_handler_arg1;
842 ih_p->ih_handler_arg2 = int_handler_arg2;
843 ih_p->ih_config_handle = NULL;
844 ih_p->ih_nsec = 0;
845 ih_p->ih_ticks = 0;
846 ih_p->ih_ksp = NULL;
847
848 return (ih_p);
849 }
850
851 int
ib_update_intr_state(pci_t * pci_p,dev_info_t * rdip,ddi_intr_handle_impl_t * hdlp,uint_t new_intr_state)852 ib_update_intr_state(pci_t *pci_p, dev_info_t *rdip,
853 ddi_intr_handle_impl_t *hdlp, uint_t new_intr_state)
854 {
855 ib_t *ib_p = pci_p->pci_ib_p;
856 ib_ino_info_t *ino_p;
857 ib_ino_pil_t *ipil_p;
858 ib_mondo_t mondo;
859 ih_t *ih_p;
860 int ret = DDI_FAILURE;
861
862 /*
863 * For PULSE interrupts, pci driver don't allocate
864 * ib_ino_info_t and ih_t data structures and also,
865 * not maintains any interrupt state information.
866 * So, just return success from here.
867 */
868 if (hdlp->ih_vector & PCI_PULSE_INO) {
869 DEBUG0(DBG_IB, ib_p->ib_pci_p->pci_dip,
870 "ib_update_intr_state: PULSE interrupt, return success\n");
871
872 return (DDI_SUCCESS);
873 }
874
875 mutex_enter(&ib_p->ib_ino_lst_mutex);
876
877 if ((mondo = pci_xlate_intr(pci_p->pci_dip, rdip, pci_p->pci_ib_p,
878 IB_MONDO_TO_INO(hdlp->ih_vector))) == 0) {
879 mutex_exit(&ib_p->ib_ino_lst_mutex);
880 return (ret);
881 }
882
883 ino_p = ib_locate_ino(ib_p, IB_MONDO_TO_INO(mondo));
884 if (ino_p && (ipil_p = ib_ino_locate_ipil(ino_p, hdlp->ih_pri))) {
885 if (ih_p = ib_intr_locate_ih(ipil_p, rdip, hdlp->ih_inum)) {
886 ih_p->ih_intr_state = new_intr_state;
887 ret = DDI_SUCCESS;
888 }
889 }
890
891 mutex_exit(&ib_p->ib_ino_lst_mutex);
892 return (ret);
893 }
894
895 /*
896 * Get interrupt CPU for a given ino.
897 * Return info only for inos which are already mapped to devices.
898 */
899 /*ARGSUSED*/
900 int
ib_get_intr_target(pci_t * pci_p,ib_ino_t ino,int * cpu_id_p)901 ib_get_intr_target(pci_t *pci_p, ib_ino_t ino, int *cpu_id_p)
902 {
903 dev_info_t *dip = pci_p->pci_dip;
904 ib_t *ib_p = pci_p->pci_ib_p;
905 volatile uint64_t *imregp;
906 uint64_t imregval;
907
908 DEBUG1(DBG_IB, dip, "ib_get_intr_target: ino %x\n", ino);
909
910 imregp = ib_intr_map_reg_addr(ib_p, ino);
911 imregval = *imregp;
912
913 *cpu_id_p = ib_map_reg_get_cpu(imregval);
914
915 DEBUG1(DBG_IB, dip, "ib_get_intr_target: cpu_id %x\n", *cpu_id_p);
916
917 return (DDI_SUCCESS);
918 }
919
920 /*
921 * Associate a new CPU with a given ino.
922 * Operate only on inos which are already mapped to devices.
923 */
924 int
ib_set_intr_target(pci_t * pci_p,ib_ino_t ino,int cpu_id)925 ib_set_intr_target(pci_t *pci_p, ib_ino_t ino, int cpu_id)
926 {
927 dev_info_t *dip = pci_p->pci_dip;
928 ib_t *ib_p = pci_p->pci_ib_p;
929 int ret = DDI_SUCCESS;
930 uint32_t old_cpu_id;
931 hrtime_t start_time;
932 uint64_t imregval;
933 uint64_t new_imregval;
934 volatile uint64_t *imregp;
935 volatile uint64_t *idregp;
936 extern const int _ncpu;
937 extern cpu_t *cpu[];
938
939 DEBUG2(DBG_IB, dip, "ib_set_intr_target: ino %x cpu_id %x\n",
940 ino, cpu_id);
941
942 imregp = (uint64_t *)ib_intr_map_reg_addr(ib_p, ino);
943 idregp = IB_INO_INTR_STATE_REG(ib_p, ino);
944
945 /* Save original mapreg value. */
946 imregval = *imregp;
947 DEBUG1(DBG_IB, dip, "ib_set_intr_target: orig mapreg value: 0x%llx\n",
948 imregval);
949
950 /* Operate only on inos which are already enabled. */
951 if (!(imregval & COMMON_INTR_MAP_REG_VALID))
952 return (DDI_FAILURE);
953
954 /* Is this request a noop? */
955 if ((old_cpu_id = ib_map_reg_get_cpu(imregval)) == cpu_id)
956 return (DDI_SUCCESS);
957
958 /* Clear the interrupt valid/enable bit for particular ino. */
959 DEBUG0(DBG_IB, dip, "Clearing intr_enabled...\n");
960 *imregp = imregval & ~COMMON_INTR_MAP_REG_VALID;
961
962 /* Wait until there are no more pending interrupts. */
963 start_time = gethrtime();
964
965 DEBUG0(DBG_IB, dip, "About to check for pending interrupts...\n");
966
967 while (IB_INO_INTR_PENDING(idregp, ino)) {
968 DEBUG0(DBG_IB, dip, "Waiting for pending ints to clear\n");
969 if ((gethrtime() - start_time) < pci_intrpend_timeout) {
970 continue;
971 } else { /* Timed out waiting. */
972 DEBUG0(DBG_IB, dip, "Timed out waiting \n");
973 return (DDI_EPENDING);
974 }
975 }
976
977 new_imregval = *imregp;
978
979 DEBUG1(DBG_IB, dip,
980 "after disabling intr, mapreg value: 0x%llx\n", new_imregval);
981
982 /*
983 * Get lock, validate cpu and write new mapreg value.
984 */
985 mutex_enter(&cpu_lock);
986 if ((cpu_id < _ncpu) && (cpu[cpu_id] && cpu_is_online(cpu[cpu_id]))) {
987 /* Prepare new mapreg value with intr enabled and new cpu_id. */
988 new_imregval &=
989 COMMON_INTR_MAP_REG_IGN | COMMON_INTR_MAP_REG_INO;
990 new_imregval = ib_get_map_reg(new_imregval, cpu_id);
991
992 DEBUG1(DBG_IB, dip, "Writing new mapreg value:0x%llx\n",
993 new_imregval);
994
995 *imregp = new_imregval;
996
997 ib_log_new_cpu(ib_p, old_cpu_id, cpu_id, ino);
998 } else { /* Invalid cpu. Restore original register image. */
999 DEBUG0(DBG_IB, dip,
1000 "Invalid cpuid: writing orig mapreg value\n");
1001
1002 *imregp = imregval;
1003 ret = DDI_EINVAL;
1004 }
1005 mutex_exit(&cpu_lock);
1006
1007 return (ret);
1008 }
1009
1010
1011 /*
1012 * Return the dips or number of dips associated with a given interrupt block.
1013 * Size of dips array arg is passed in as dips_ret arg.
1014 * Number of dips returned is returned in dips_ret arg.
1015 * Array of dips gets returned in the dips argument.
1016 * Function returns number of dips existing for the given interrupt block.
1017 *
1018 */
1019 uint8_t
ib_get_ino_devs(ib_t * ib_p,uint32_t ino,uint8_t * devs_ret,pcitool_intr_dev_t * devs)1020 ib_get_ino_devs(
1021 ib_t *ib_p, uint32_t ino, uint8_t *devs_ret, pcitool_intr_dev_t *devs)
1022 {
1023 ib_ino_info_t *ino_p;
1024 ib_ino_pil_t *ipil_p;
1025 ih_t *ih_p;
1026 uint32_t num_devs = 0;
1027 int i, j;
1028
1029 mutex_enter(&ib_p->ib_ino_lst_mutex);
1030 ino_p = ib_locate_ino(ib_p, ino);
1031 if (ino_p != NULL) {
1032 for (j = 0, ipil_p = ino_p->ino_ipil_p; ipil_p;
1033 ipil_p = ipil_p->ipil_next_p) {
1034 num_devs += ipil_p->ipil_ih_size;
1035
1036 for (i = 0, ih_p = ipil_p->ipil_ih_head;
1037 ((i < ipil_p->ipil_ih_size) && (i < *devs_ret));
1038 i++, j++, ih_p = ih_p->ih_next) {
1039 (void) strlcpy(devs[i].driver_name,
1040 ddi_driver_name(ih_p->ih_dip),
1041 MAXMODCONFNAME);
1042 (void) ddi_pathname(ih_p->ih_dip, devs[i].path);
1043 devs[i].dev_inst =
1044 ddi_get_instance(ih_p->ih_dip);
1045 }
1046 }
1047 *devs_ret = j;
1048 }
1049
1050 mutex_exit(&ib_p->ib_ino_lst_mutex);
1051
1052 return (num_devs);
1053 }
1054
ib_log_new_cpu(ib_t * ib_p,uint32_t old_cpu_id,uint32_t new_cpu_id,uint32_t ino)1055 void ib_log_new_cpu(ib_t *ib_p, uint32_t old_cpu_id, uint32_t new_cpu_id,
1056 uint32_t ino)
1057 {
1058 ib_ino_info_t *ino_p;
1059 ib_ino_pil_t *ipil_p;
1060 ih_t *ih_p;
1061 int i;
1062
1063 mutex_enter(&ib_p->ib_ino_lst_mutex);
1064
1065 /* Log in OS data structures the new CPU. */
1066 ino_p = ib_locate_ino(ib_p, ino);
1067 if (ino_p != NULL) {
1068
1069 /* Log in OS data structures the new CPU. */
1070 ino_p->ino_cpuid = new_cpu_id;
1071
1072 for (ipil_p = ino_p->ino_ipil_p; ipil_p;
1073 ipil_p = ipil_p->ipil_next_p) {
1074 for (i = 0, ih_p = ipil_p->ipil_ih_head;
1075 (i < ipil_p->ipil_ih_size);
1076 i++, ih_p = ih_p->ih_next) {
1077 /*
1078 * Account for any residual time
1079 * to be logged for old cpu.
1080 */
1081 ib_cpu_ticks_to_ih_nsec(ib_p,
1082 ipil_p->ipil_ih_head, old_cpu_id);
1083 }
1084 }
1085 }
1086
1087 mutex_exit(&ib_p->ib_ino_lst_mutex);
1088 }
1089