1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21 /*
22 * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
23 */
24
25 /*
26 * PX nexus interrupt handling:
27 * PX device interrupt handler wrapper
28 * PIL lookup routine
29 * PX device interrupt related initchild code
30 */
31
32 #include <sys/types.h>
33 #include <sys/kmem.h>
34 #include <sys/async.h>
35 #include <sys/spl.h>
36 #include <sys/sunddi.h>
37 #include <sys/fm/protocol.h>
38 #include <sys/fm/util.h>
39 #include <sys/machsystm.h> /* e_ddi_nodeid_to_dip() */
40 #include <sys/ddi_impldefs.h>
41 #include <sys/sdt.h>
42 #include <sys/atomic.h>
43 #include "px_obj.h"
44 #include <sys/ontrap.h>
45 #include <sys/membar.h>
46 #include <sys/clock.h>
47
48 /*
49 * interrupt jabber:
50 *
51 * When an interrupt line is jabbering, every time the state machine for the
52 * associated ino is idled, a new mondo will be sent and the ino will go into
53 * the pending state again. The mondo will cause a new call to
54 * px_intr_wrapper() which normally idles the ino's state machine which would
55 * precipitate another trip round the loop.
56 *
57 * The loop can be broken by preventing the ino's state machine from being
58 * idled when an interrupt line is jabbering. See the comment at the
59 * beginning of px_intr_wrapper() explaining how the 'interrupt jabber
60 * protection' code does this.
61 */
62
63 /*LINTLIBRARY*/
64
65 /*
66 * If the unclaimed interrupt count has reached the limit set by
67 * pci_unclaimed_intr_max within the time limit, then all interrupts
68 * on this ino is blocked by not idling the interrupt state machine.
69 */
70 static int
px_spurintr(px_ino_pil_t * ipil_p)71 px_spurintr(px_ino_pil_t *ipil_p)
72 {
73 px_ino_t *ino_p = ipil_p->ipil_ino_p;
74 px_ih_t *ih_p;
75 px_t *px_p = ino_p->ino_ib_p->ib_px_p;
76 char *err_fmt_str;
77 boolean_t blocked = B_FALSE;
78 int i;
79
80 if (ino_p->ino_unclaimed_intrs > px_unclaimed_intr_max)
81 return (DDI_INTR_CLAIMED);
82
83 if (!ino_p->ino_unclaimed_intrs)
84 ino_p->ino_spurintr_begin = ddi_get_lbolt();
85
86 ino_p->ino_unclaimed_intrs++;
87
88 if (ino_p->ino_unclaimed_intrs <= px_unclaimed_intr_max)
89 goto clear;
90
91 if (drv_hztousec(ddi_get_lbolt() - ino_p->ino_spurintr_begin)
92 > px_spurintr_duration) {
93 ino_p->ino_unclaimed_intrs = 0;
94 goto clear;
95 }
96 err_fmt_str = "%s%d: ino 0x%x blocked";
97 blocked = B_TRUE;
98 goto warn;
99 clear:
100 err_fmt_str = "!%s%d: spurious interrupt from ino 0x%x";
101 warn:
102 cmn_err(CE_WARN, err_fmt_str, NAMEINST(px_p->px_dip), ino_p->ino_ino);
103 for (ipil_p = ino_p->ino_ipil_p; ipil_p;
104 ipil_p = ipil_p->ipil_next_p) {
105 for (i = 0, ih_p = ipil_p->ipil_ih_start;
106 i < ipil_p->ipil_ih_size; i++, ih_p = ih_p->ih_next)
107 cmn_err(CE_CONT, "!%s-%d#%x ", NAMEINST(ih_p->ih_dip),
108 ih_p->ih_inum);
109 }
110 cmn_err(CE_CONT, "!\n");
111
112 /* Clear the pending state */
113 if (blocked == B_FALSE) {
114 if (px_lib_intr_setstate(px_p->px_dip, ino_p->ino_sysino,
115 INTR_IDLE_STATE) != DDI_SUCCESS)
116 return (DDI_INTR_UNCLAIMED);
117 }
118
119 return (DDI_INTR_CLAIMED);
120 }
121
122 extern uint64_t intr_get_time(void);
123
124 /*
125 * px_intx_intr (INTx or legacy interrupt handler)
126 *
127 * This routine is used as wrapper around interrupt handlers installed by child
128 * device drivers. This routine invokes the driver interrupt handlers and
129 * examines the return codes.
130 *
131 * There is a count of unclaimed interrupts kept on a per-ino basis. If at
132 * least one handler claims the interrupt then the counter is halved and the
133 * interrupt state machine is idled. If no handler claims the interrupt then
134 * the counter is incremented by one and the state machine is idled.
135 * If the count ever reaches the limit value set by pci_unclaimed_intr_max
136 * then the interrupt state machine is not idled thus preventing any further
137 * interrupts on that ino. The state machine will only be idled again if a
138 * handler is subsequently added or removed.
139 *
140 * return value: DDI_INTR_CLAIMED if any handlers claimed the interrupt,
141 * DDI_INTR_UNCLAIMED otherwise.
142 */
143 uint_t
px_intx_intr(caddr_t arg)144 px_intx_intr(caddr_t arg)
145 {
146 px_ino_pil_t *ipil_p = (px_ino_pil_t *)arg;
147 px_ino_t *ino_p = ipil_p->ipil_ino_p;
148 px_t *px_p = ino_p->ino_ib_p->ib_px_p;
149 px_ih_t *ih_p = ipil_p->ipil_ih_start;
150 ushort_t pil = ipil_p->ipil_pil;
151 uint_t result = 0, r = DDI_INTR_UNCLAIMED;
152 int i;
153
154 DBG(DBG_INTX_INTR, px_p->px_dip, "px_intx_intr:"
155 "ino=%x sysino=%llx pil=%x ih_size=%x ih_lst=%x\n",
156 ino_p->ino_ino, ino_p->ino_sysino, ipil_p->ipil_pil,
157 ipil_p->ipil_ih_size, ipil_p->ipil_ih_head);
158
159 for (i = 0; i < ipil_p->ipil_ih_size; i++, ih_p = ih_p->ih_next) {
160 dev_info_t *dip = ih_p->ih_dip;
161 uint_t (*handler)() = ih_p->ih_handler;
162 caddr_t arg1 = ih_p->ih_handler_arg1;
163 caddr_t arg2 = ih_p->ih_handler_arg2;
164
165 if (ih_p->ih_intr_state == PX_INTR_STATE_DISABLE) {
166 DBG(DBG_INTX_INTR, px_p->px_dip,
167 "px_intx_intr: %s%d interrupt %d is disabled\n",
168 ddi_driver_name(dip), ddi_get_instance(dip),
169 ino_p->ino_ino);
170
171 continue;
172 }
173
174 DBG(DBG_INTX_INTR, px_p->px_dip, "px_intx_intr:"
175 "ino=%x handler=%p arg1 =%p arg2 = %p\n",
176 ino_p->ino_ino, handler, arg1, arg2);
177
178 DTRACE_PROBE4(interrupt__start, dev_info_t, dip,
179 void *, handler, caddr_t, arg1, caddr_t, arg2);
180
181 r = (*handler)(arg1, arg2);
182
183 /*
184 * Account for time used by this interrupt. Protect against
185 * conflicting writes to ih_ticks from ib_intr_dist_all() by
186 * using atomic ops.
187 */
188
189 if (pil <= LOCK_LEVEL)
190 atomic_add_64(&ih_p->ih_ticks, intr_get_time());
191
192 DTRACE_PROBE4(interrupt__complete, dev_info_t, dip,
193 void *, handler, caddr_t, arg1, int, r);
194
195 result += r;
196
197 if (px_check_all_handlers)
198 continue;
199 if (result)
200 break;
201 }
202
203 if (result)
204 ino_p->ino_claimed |= (1 << pil);
205
206 /* Interrupt can only be cleared after all pil levels are handled */
207 if (pil != ino_p->ino_lopil)
208 return (DDI_INTR_CLAIMED);
209
210 if (!ino_p->ino_claimed) {
211 if (px_unclaimed_intr_block)
212 return (px_spurintr(ipil_p));
213 }
214
215 ino_p->ino_unclaimed_intrs = 0;
216 ino_p->ino_claimed = 0;
217
218 /* Clear the pending state */
219 if (px_lib_intr_setstate(px_p->px_dip,
220 ino_p->ino_sysino, INTR_IDLE_STATE) != DDI_SUCCESS)
221 return (DDI_INTR_UNCLAIMED);
222
223 return (DDI_INTR_CLAIMED);
224 }
225
226 /*
227 * px_msiq_intr (MSI/X or PCIe MSG interrupt handler)
228 *
229 * This routine is used as wrapper around interrupt handlers installed by child
230 * device drivers. This routine invokes the driver interrupt handlers and
231 * examines the return codes.
232 *
233 * There is a count of unclaimed interrupts kept on a per-ino basis. If at
234 * least one handler claims the interrupt then the counter is halved and the
235 * interrupt state machine is idled. If no handler claims the interrupt then
236 * the counter is incremented by one and the state machine is idled.
237 * If the count ever reaches the limit value set by pci_unclaimed_intr_max
238 * then the interrupt state machine is not idled thus preventing any further
239 * interrupts on that ino. The state machine will only be idled again if a
240 * handler is subsequently added or removed.
241 *
242 * return value: DDI_INTR_CLAIMED if any handlers claimed the interrupt,
243 * DDI_INTR_UNCLAIMED otherwise.
244 */
245 uint_t
px_msiq_intr(caddr_t arg)246 px_msiq_intr(caddr_t arg)
247 {
248 px_ino_pil_t *ipil_p = (px_ino_pil_t *)arg;
249 px_ino_t *ino_p = ipil_p->ipil_ino_p;
250 px_t *px_p = ino_p->ino_ib_p->ib_px_p;
251 px_msiq_state_t *msiq_state_p = &px_p->px_ib_p->ib_msiq_state;
252 px_msiq_t *msiq_p = ino_p->ino_msiq_p;
253 dev_info_t *dip = px_p->px_dip;
254 ushort_t pil = ipil_p->ipil_pil;
255 msiq_rec_t msiq_rec, *msiq_rec_p = &msiq_rec;
256 msiqhead_t *curr_head_p;
257 msiqtail_t curr_tail_index;
258 msgcode_t msg_code;
259 px_ih_t *ih_p;
260 uint_t ret = DDI_INTR_UNCLAIMED;
261 int i, j;
262
263 DBG(DBG_MSIQ_INTR, dip, "px_msiq_intr: msiq_id =%x ino=%x pil=%x "
264 "ih_size=%x ih_lst=%x\n", msiq_p->msiq_id, ino_p->ino_ino,
265 ipil_p->ipil_pil, ipil_p->ipil_ih_size, ipil_p->ipil_ih_head);
266
267 /*
268 * The px_msiq_intr() handles multiple interrupt priorities and it
269 * will set msiq->msiq_rec2process to the number of MSIQ records to
270 * process while handling the highest priority interrupt. Subsequent
271 * lower priority interrupts will just process any unprocessed MSIQ
272 * records or will just return immediately.
273 */
274 if (msiq_p->msiq_recs2process == 0) {
275 ASSERT(ino_p->ino_ipil_cntr == 0);
276 ino_p->ino_ipil_cntr = ino_p->ino_ipil_size;
277
278 /* Read current MSIQ tail index */
279 px_lib_msiq_gettail(dip, msiq_p->msiq_id, &curr_tail_index);
280 msiq_p->msiq_new_head_index = msiq_p->msiq_curr_head_index;
281
282 if (curr_tail_index < msiq_p->msiq_curr_head_index)
283 curr_tail_index += msiq_state_p->msiq_rec_cnt;
284
285 msiq_p->msiq_recs2process = curr_tail_index -
286 msiq_p->msiq_curr_head_index;
287 }
288
289 DBG(DBG_MSIQ_INTR, dip, "px_msiq_intr: curr_head %x new_head %x "
290 "rec2process %x\n", msiq_p->msiq_curr_head_index,
291 msiq_p->msiq_new_head_index, msiq_p->msiq_recs2process);
292
293 /* If all MSIQ records are already processed, just return immediately */
294 if ((msiq_p->msiq_new_head_index - msiq_p->msiq_curr_head_index)
295 == msiq_p->msiq_recs2process)
296 goto intr_done;
297
298 curr_head_p = (msiqhead_t *)((caddr_t)msiq_p->msiq_base_p +
299 (msiq_p->msiq_curr_head_index * sizeof (msiq_rec_t)));
300
301 /*
302 * Calculate the number of recs to process by taking the difference
303 * between the head and tail pointers. For all records we always
304 * verify that we have a valid record type before we do any processing.
305 * If triggered, we should always have at least one valid record.
306 */
307 for (i = 0; i < msiq_p->msiq_recs2process; i++) {
308 msiq_rec_type_t rec_type;
309
310 /* Read next MSIQ record */
311 px_lib_get_msiq_rec(dip, curr_head_p, msiq_rec_p);
312
313 rec_type = msiq_rec_p->msiq_rec_type;
314
315 DBG(DBG_MSIQ_INTR, dip, "px_msiq_intr: MSIQ RECORD, "
316 "msiq_rec_type 0x%llx msiq_rec_rid 0x%llx\n",
317 rec_type, msiq_rec_p->msiq_rec_rid);
318
319 if (!rec_type)
320 goto next_rec;
321
322 /* Check MSIQ record type */
323 switch (rec_type) {
324 case MSG_REC:
325 msg_code = msiq_rec_p->msiq_rec_data.msg.msg_code;
326 DBG(DBG_MSIQ_INTR, dip, "px_msiq_intr: PCIE MSG "
327 "record, msg type 0x%x\n", msg_code);
328 break;
329 case MSI32_REC:
330 case MSI64_REC:
331 msg_code = msiq_rec_p->msiq_rec_data.msi.msi_data;
332 DBG(DBG_MSIQ_INTR, dip, "px_msiq_intr: MSI record, "
333 "msi 0x%x\n", msg_code);
334 break;
335 default:
336 msg_code = 0;
337 cmn_err(CE_WARN, "%s%d: px_msiq_intr: 0x%x MSIQ "
338 "record type is not supported",
339 ddi_driver_name(dip), ddi_get_instance(dip),
340 rec_type);
341
342 goto next_rec;
343 }
344
345 /*
346 * Scan through px_ih_t linked list, searching for the
347 * right px_ih_t, matching MSIQ record data.
348 */
349 for (j = 0, ih_p = ipil_p->ipil_ih_start;
350 ih_p && (j < ipil_p->ipil_ih_size) &&
351 ((ih_p->ih_msg_code != msg_code) ||
352 (ih_p->ih_rec_type != rec_type));
353 ih_p = ih_p->ih_next, j++)
354 ;
355
356 if ((ih_p->ih_msg_code == msg_code) &&
357 (ih_p->ih_rec_type == rec_type)) {
358 dev_info_t *ih_dip = ih_p->ih_dip;
359 uint_t (*handler)() = ih_p->ih_handler;
360 caddr_t arg1 = ih_p->ih_handler_arg1;
361 caddr_t arg2 = ih_p->ih_handler_arg2;
362
363 DBG(DBG_MSIQ_INTR, ih_dip, "px_msiq_intr: ino=%x "
364 "data=%x handler=%p arg1 =%p arg2=%p\n",
365 ino_p->ino_ino, msg_code, handler, arg1, arg2);
366
367 DTRACE_PROBE4(interrupt__start, dev_info_t, ih_dip,
368 void *, handler, caddr_t, arg1, caddr_t, arg2);
369
370 ih_p->ih_intr_flags = PX_INTR_PENDING;
371
372 /*
373 * Special case for PCIE Error Messages.
374 * The current frame work doesn't fit PCIE Err Msgs
375 * This should be fixed when PCIE MESSAGES as a whole
376 * is architected correctly.
377 */
378 if ((rec_type == MSG_REC) &&
379 ((msg_code == PCIE_MSG_CODE_ERR_COR) ||
380 (msg_code == PCIE_MSG_CODE_ERR_NONFATAL) ||
381 (msg_code == PCIE_MSG_CODE_ERR_FATAL))) {
382 ret = px_err_fabric_intr(px_p, msg_code,
383 msiq_rec_p->msiq_rec_rid);
384 } else {
385 /* Clear MSI state */
386 px_lib_msi_setstate(dip, (msinum_t)msg_code,
387 PCI_MSI_STATE_IDLE);
388
389 ret = (*handler)(arg1, arg2);
390 }
391
392 /*
393 * Account for time used by this interrupt. Protect
394 * against conflicting writes to ih_ticks from
395 * ib_intr_dist_all() by using atomic ops.
396 */
397
398 if (pil <= LOCK_LEVEL)
399 atomic_add_64(&ih_p->ih_ticks, intr_get_time());
400
401 DTRACE_PROBE4(interrupt__complete, dev_info_t, ih_dip,
402 void *, handler, caddr_t, arg1, int, ret);
403
404 /* clear handler status flags */
405 ih_p->ih_intr_flags = PX_INTR_IDLE;
406
407 msiq_p->msiq_new_head_index++;
408 px_lib_clr_msiq_rec(ih_dip, curr_head_p);
409 } else {
410 DBG(DBG_MSIQ_INTR, dip, "px_msiq_intr: "
411 "No matching MSIQ record found\n");
412 }
413 next_rec:
414 /* Get the pointer next EQ record */
415 curr_head_p = (msiqhead_t *)
416 ((caddr_t)curr_head_p + sizeof (msiq_rec_t));
417
418 /* Check for overflow condition */
419 if (curr_head_p >= (msiqhead_t *)((caddr_t)msiq_p->msiq_base_p
420 + (msiq_state_p->msiq_rec_cnt * sizeof (msiq_rec_t))))
421 curr_head_p = (msiqhead_t *)msiq_p->msiq_base_p;
422 }
423
424 DBG(DBG_MSIQ_INTR, dip, "px_msiq_intr: No of MSIQ recs processed %x\n",
425 (msiq_p->msiq_new_head_index - msiq_p->msiq_curr_head_index));
426
427 DBG(DBG_MSIQ_INTR, dip, "px_msiq_intr: curr_head %x new_head %x "
428 "rec2process %x\n", msiq_p->msiq_curr_head_index,
429 msiq_p->msiq_new_head_index, msiq_p->msiq_recs2process);
430
431 /* ino_claimed used just for debugging purpose */
432 if (ret)
433 ino_p->ino_claimed |= (1 << pil);
434
435 intr_done:
436 /* Interrupt can only be cleared after all pil levels are handled */
437 if (--ino_p->ino_ipil_cntr != 0)
438 return (DDI_INTR_CLAIMED);
439
440 if (msiq_p->msiq_new_head_index <= msiq_p->msiq_curr_head_index) {
441 if (px_unclaimed_intr_block)
442 return (px_spurintr(ipil_p));
443 }
444
445 /* Update MSIQ head index with no of MSIQ records processed */
446 if (msiq_p->msiq_new_head_index >= msiq_state_p->msiq_rec_cnt)
447 msiq_p->msiq_new_head_index -= msiq_state_p->msiq_rec_cnt;
448
449 msiq_p->msiq_curr_head_index = msiq_p->msiq_new_head_index;
450 px_lib_msiq_sethead(dip, msiq_p->msiq_id, msiq_p->msiq_new_head_index);
451
452 msiq_p->msiq_new_head_index = 0;
453 msiq_p->msiq_recs2process = 0;
454 ino_p->ino_claimed = 0;
455
456 /* Clear the pending state */
457 if (px_lib_intr_setstate(dip, ino_p->ino_sysino,
458 INTR_IDLE_STATE) != DDI_SUCCESS)
459 return (DDI_INTR_UNCLAIMED);
460
461 return (DDI_INTR_CLAIMED);
462 }
463
464 dev_info_t *
px_get_my_childs_dip(dev_info_t * dip,dev_info_t * rdip)465 px_get_my_childs_dip(dev_info_t *dip, dev_info_t *rdip)
466 {
467 dev_info_t *cdip = rdip;
468
469 for (; ddi_get_parent(cdip) != dip; cdip = ddi_get_parent(cdip))
470 ;
471
472 return (cdip);
473 }
474
475 /* ARGSUSED */
476 int
px_intx_ops(dev_info_t * dip,dev_info_t * rdip,ddi_intr_op_t intr_op,ddi_intr_handle_impl_t * hdlp,void * result)477 px_intx_ops(dev_info_t *dip, dev_info_t *rdip, ddi_intr_op_t intr_op,
478 ddi_intr_handle_impl_t *hdlp, void *result)
479 {
480 px_t *px_p = DIP_TO_STATE(dip);
481 int ret = DDI_SUCCESS;
482
483 DBG(DBG_INTROPS, dip, "px_intx_ops: dip=%x rdip=%x intr_op=%x "
484 "handle=%p\n", dip, rdip, intr_op, hdlp);
485
486 switch (intr_op) {
487 case DDI_INTROP_GETCAP:
488 ret = pci_intx_get_cap(rdip, (int *)result);
489 break;
490 case DDI_INTROP_SETCAP:
491 DBG(DBG_INTROPS, dip, "px_intx_ops: SetCap is not supported\n");
492 ret = DDI_ENOTSUP;
493 break;
494 case DDI_INTROP_ALLOC:
495 *(int *)result = hdlp->ih_scratch1;
496 break;
497 case DDI_INTROP_FREE:
498 break;
499 case DDI_INTROP_GETPRI:
500 *(int *)result = hdlp->ih_pri ?
501 hdlp->ih_pri : pci_class_to_pil(rdip);
502 break;
503 case DDI_INTROP_SETPRI:
504 break;
505 case DDI_INTROP_ADDISR:
506 ret = px_add_intx_intr(dip, rdip, hdlp);
507 break;
508 case DDI_INTROP_REMISR:
509 ret = px_rem_intx_intr(dip, rdip, hdlp);
510 break;
511 case DDI_INTROP_GETTARGET:
512 ret = px_ib_get_intr_target(px_p, hdlp->ih_vector,
513 (cpuid_t *)result);
514 break;
515 case DDI_INTROP_SETTARGET:
516 ret = DDI_ENOTSUP;
517 break;
518 case DDI_INTROP_ENABLE:
519 ret = px_ib_update_intr_state(px_p, rdip, hdlp->ih_inum,
520 hdlp->ih_vector, hdlp->ih_pri, PX_INTR_STATE_ENABLE, 0, 0);
521 break;
522 case DDI_INTROP_DISABLE:
523 ret = px_ib_update_intr_state(px_p, rdip, hdlp->ih_inum,
524 hdlp->ih_vector, hdlp->ih_pri, PX_INTR_STATE_DISABLE, 0, 0);
525 break;
526 case DDI_INTROP_SETMASK:
527 ret = pci_intx_set_mask(rdip);
528 break;
529 case DDI_INTROP_CLRMASK:
530 ret = pci_intx_clr_mask(rdip);
531 break;
532 case DDI_INTROP_GETPENDING:
533 ret = pci_intx_get_pending(rdip, (int *)result);
534 break;
535 case DDI_INTROP_NINTRS:
536 case DDI_INTROP_NAVAIL:
537 *(int *)result = i_ddi_get_intx_nintrs(rdip);
538 break;
539 default:
540 ret = DDI_ENOTSUP;
541 break;
542 }
543
544 return (ret);
545 }
546
547 /* ARGSUSED */
548 int
px_msix_ops(dev_info_t * dip,dev_info_t * rdip,ddi_intr_op_t intr_op,ddi_intr_handle_impl_t * hdlp,void * result)549 px_msix_ops(dev_info_t *dip, dev_info_t *rdip, ddi_intr_op_t intr_op,
550 ddi_intr_handle_impl_t *hdlp, void *result)
551 {
552 px_t *px_p = DIP_TO_STATE(dip);
553 px_msi_state_t *msi_state_p = &px_p->px_ib_p->ib_msi_state;
554 msiq_rec_type_t msiq_rec_type;
555 msi_type_t msi_type;
556 uint64_t msi_addr;
557 msinum_t msi_num;
558 msiqid_t msiq_id;
559 uint_t nintrs;
560 int ret = DDI_SUCCESS;
561
562 DBG(DBG_INTROPS, dip, "px_msix_ops: dip=%x rdip=%x intr_op=%x "
563 "handle=%p\n", dip, rdip, intr_op, hdlp);
564
565 /* Check for MSI64 support */
566 if ((hdlp->ih_cap & DDI_INTR_FLAG_MSI64) && msi_state_p->msi_addr64) {
567 msiq_rec_type = MSI64_REC;
568 msi_type = MSI64_TYPE;
569 msi_addr = msi_state_p->msi_addr64;
570 } else {
571 msiq_rec_type = MSI32_REC;
572 msi_type = MSI32_TYPE;
573 msi_addr = msi_state_p->msi_addr32;
574 }
575
576 (void) px_msi_get_msinum(px_p, hdlp->ih_dip,
577 (hdlp->ih_flags & DDI_INTR_MSIX_DUP) ? hdlp->ih_main->ih_inum :
578 hdlp->ih_inum, &msi_num);
579
580 switch (intr_op) {
581 case DDI_INTROP_GETCAP:
582 ret = pci_msi_get_cap(rdip, hdlp->ih_type, (int *)result);
583 break;
584 case DDI_INTROP_SETCAP:
585 DBG(DBG_INTROPS, dip, "px_msix_ops: SetCap is not supported\n");
586 ret = DDI_ENOTSUP;
587 break;
588 case DDI_INTROP_ALLOC:
589 /*
590 * We need to restrict this allocation in future
591 * based on Resource Management policies.
592 */
593 if ((ret = px_msi_alloc(px_p, rdip, hdlp->ih_type,
594 hdlp->ih_inum, hdlp->ih_scratch1,
595 (uintptr_t)hdlp->ih_scratch2,
596 (int *)result)) != DDI_SUCCESS) {
597 DBG(DBG_INTROPS, dip, "px_msix_ops: allocation "
598 "failed, rdip 0x%p type 0x%d inum 0x%x "
599 "count 0x%x\n", rdip, hdlp->ih_type, hdlp->ih_inum,
600 hdlp->ih_scratch1);
601
602 return (ret);
603 }
604
605 if ((hdlp->ih_type == DDI_INTR_TYPE_MSIX) &&
606 (i_ddi_get_msix(rdip) == NULL)) {
607 ddi_intr_msix_t *msix_p;
608
609 if (msix_p = pci_msix_init(rdip)) {
610 i_ddi_set_msix(rdip, msix_p);
611 break;
612 }
613
614 DBG(DBG_INTROPS, dip, "px_msix_ops: MSI-X allocation "
615 "failed, rdip 0x%p inum 0x%x\n", rdip,
616 hdlp->ih_inum);
617
618 (void) px_msi_free(px_p, rdip, hdlp->ih_inum,
619 hdlp->ih_scratch1);
620
621 return (DDI_FAILURE);
622 }
623
624 break;
625 case DDI_INTROP_FREE:
626 (void) pci_msi_unconfigure(rdip, hdlp->ih_type, hdlp->ih_inum);
627
628 if (hdlp->ih_type == DDI_INTR_TYPE_MSI)
629 goto msi_free;
630
631 if (hdlp->ih_flags & DDI_INTR_MSIX_DUP)
632 break;
633
634 if (((i_ddi_intr_get_current_nintrs(hdlp->ih_dip) - 1) == 0) &&
635 (i_ddi_get_msix(rdip))) {
636 pci_msix_fini(i_ddi_get_msix(rdip));
637 i_ddi_set_msix(rdip, NULL);
638 }
639 msi_free:
640 (void) px_msi_free(px_p, rdip, hdlp->ih_inum,
641 hdlp->ih_scratch1);
642 break;
643 case DDI_INTROP_GETPRI:
644 *(int *)result = hdlp->ih_pri ?
645 hdlp->ih_pri : pci_class_to_pil(rdip);
646 break;
647 case DDI_INTROP_SETPRI:
648 break;
649 case DDI_INTROP_ADDISR:
650 if ((ret = px_add_msiq_intr(dip, rdip, hdlp,
651 msiq_rec_type, msi_num, -1, &msiq_id)) != DDI_SUCCESS) {
652 DBG(DBG_INTROPS, dip, "px_msix_ops: Add MSI handler "
653 "failed, rdip 0x%p msi 0x%x\n", rdip, msi_num);
654 return (ret);
655 }
656
657 DBG(DBG_INTROPS, dip, "px_msix_ops: msiq used 0x%x\n", msiq_id);
658
659 if ((ret = px_lib_msi_setmsiq(dip, msi_num,
660 msiq_id, msi_type)) != DDI_SUCCESS) {
661 (void) px_rem_msiq_intr(dip, rdip,
662 hdlp, msiq_rec_type, msi_num, msiq_id);
663 return (ret);
664 }
665
666 if ((ret = px_lib_msi_setstate(dip, msi_num,
667 PCI_MSI_STATE_IDLE)) != DDI_SUCCESS) {
668 (void) px_rem_msiq_intr(dip, rdip,
669 hdlp, msiq_rec_type, msi_num, msiq_id);
670 return (ret);
671 }
672
673 if ((ret = px_lib_msi_setvalid(dip, msi_num,
674 PCI_MSI_VALID)) != DDI_SUCCESS)
675 return (ret);
676
677 ret = px_ib_update_intr_state(px_p, rdip, hdlp->ih_inum,
678 px_msiqid_to_devino(px_p, msiq_id), hdlp->ih_pri,
679 PX_INTR_STATE_ENABLE, msiq_rec_type, msi_num);
680
681 break;
682 case DDI_INTROP_DUPVEC:
683 DBG(DBG_INTROPS, dip, "px_msix_ops: dupisr - inum: %x, "
684 "new_vector: %x\n", hdlp->ih_inum, hdlp->ih_scratch1);
685
686 ret = pci_msix_dup(hdlp->ih_dip, hdlp->ih_inum,
687 hdlp->ih_scratch1);
688 break;
689 case DDI_INTROP_REMISR:
690 if ((ret = px_lib_msi_getmsiq(dip, msi_num,
691 &msiq_id)) != DDI_SUCCESS)
692 return (ret);
693
694 if ((ret = px_ib_update_intr_state(px_p, rdip,
695 hdlp->ih_inum, px_msiqid_to_devino(px_p, msiq_id),
696 hdlp->ih_pri, PX_INTR_STATE_DISABLE, msiq_rec_type,
697 msi_num)) != DDI_SUCCESS)
698 return (ret);
699
700 if ((ret = px_lib_msi_setvalid(dip, msi_num,
701 PCI_MSI_INVALID)) != DDI_SUCCESS)
702 return (ret);
703
704 if ((ret = px_lib_msi_setstate(dip, msi_num,
705 PCI_MSI_STATE_IDLE)) != DDI_SUCCESS)
706 return (ret);
707
708 ret = px_rem_msiq_intr(dip, rdip,
709 hdlp, msiq_rec_type, msi_num, msiq_id);
710
711 break;
712 case DDI_INTROP_GETTARGET:
713 if ((ret = px_lib_msi_getmsiq(dip, msi_num,
714 &msiq_id)) != DDI_SUCCESS)
715 return (ret);
716
717 ret = px_ib_get_intr_target(px_p,
718 px_msiqid_to_devino(px_p, msiq_id), (cpuid_t *)result);
719 break;
720 case DDI_INTROP_SETTARGET:
721 ret = px_ib_set_msix_target(px_p, hdlp, msi_num,
722 *(cpuid_t *)result);
723 break;
724 case DDI_INTROP_ENABLE:
725 /*
726 * For MSI, just clear the mask bit and return if curr_nenables
727 * is > 1. For MSI-X, program MSI address and data for every
728 * MSI-X vector including dup vectors irrespective of current
729 * curr_nenables value.
730 */
731 if ((pci_is_msi_enabled(rdip, hdlp->ih_type) != DDI_SUCCESS) ||
732 (hdlp->ih_type == DDI_INTR_TYPE_MSIX)) {
733 nintrs = i_ddi_intr_get_current_nintrs(hdlp->ih_dip);
734
735 if ((ret = pci_msi_configure(rdip, hdlp->ih_type,
736 nintrs, hdlp->ih_inum, msi_addr,
737 hdlp->ih_type == DDI_INTR_TYPE_MSIX ?
738 msi_num : msi_num & ~(nintrs - 1))) != DDI_SUCCESS)
739 return (ret);
740
741 if (i_ddi_intr_get_current_nenables(rdip) < 1) {
742 if ((ret = pci_msi_enable_mode(rdip,
743 hdlp->ih_type)) != DDI_SUCCESS)
744 return (ret);
745 }
746 }
747
748 if ((ret = pci_msi_clr_mask(rdip, hdlp->ih_type,
749 hdlp->ih_inum)) != DDI_SUCCESS)
750 return (ret);
751
752 break;
753 case DDI_INTROP_DISABLE:
754 if ((ret = pci_msi_set_mask(rdip, hdlp->ih_type,
755 hdlp->ih_inum)) != DDI_SUCCESS)
756 return (ret);
757
758 /*
759 * curr_nenables will be greater than 1 if rdip is using
760 * MSI-X and also, if it is using DUP interface. If this
761 * curr_enables is > 1, return after setting the mask bit.
762 */
763 if (i_ddi_intr_get_current_nenables(rdip) > 1)
764 return (DDI_SUCCESS);
765
766 if ((ret = pci_msi_disable_mode(rdip, hdlp->ih_type))
767 != DDI_SUCCESS)
768 return (ret);
769
770 break;
771 case DDI_INTROP_BLOCKENABLE:
772 nintrs = i_ddi_intr_get_current_nintrs(hdlp->ih_dip);
773
774 if ((ret = pci_msi_configure(rdip, hdlp->ih_type,
775 nintrs, hdlp->ih_inum, msi_addr,
776 msi_num & ~(nintrs - 1))) != DDI_SUCCESS)
777 return (ret);
778
779 ret = pci_msi_enable_mode(rdip, hdlp->ih_type);
780 break;
781 case DDI_INTROP_BLOCKDISABLE:
782 ret = pci_msi_disable_mode(rdip, hdlp->ih_type);
783 break;
784 case DDI_INTROP_SETMASK:
785 ret = pci_msi_set_mask(rdip, hdlp->ih_type, hdlp->ih_inum);
786 break;
787 case DDI_INTROP_CLRMASK:
788 ret = pci_msi_clr_mask(rdip, hdlp->ih_type, hdlp->ih_inum);
789 break;
790 case DDI_INTROP_GETPENDING:
791 ret = pci_msi_get_pending(rdip, hdlp->ih_type,
792 hdlp->ih_inum, (int *)result);
793 break;
794 case DDI_INTROP_NINTRS:
795 ret = pci_msi_get_nintrs(rdip, hdlp->ih_type, (int *)result);
796 break;
797 case DDI_INTROP_NAVAIL:
798 /* XXX - a new interface may be needed */
799 ret = pci_msi_get_nintrs(rdip, hdlp->ih_type, (int *)result);
800 break;
801 case DDI_INTROP_GETPOOL:
802 if (msi_state_p->msi_pool_p == NULL) {
803 *(ddi_irm_pool_t **)result = NULL;
804 return (DDI_ENOTSUP);
805 }
806 *(ddi_irm_pool_t **)result = msi_state_p->msi_pool_p;
807 ret = DDI_SUCCESS;
808 break;
809 default:
810 ret = DDI_ENOTSUP;
811 break;
812 }
813
814 return (ret);
815 }
816
817 static struct {
818 kstat_named_t pxintr_ks_name;
819 kstat_named_t pxintr_ks_type;
820 kstat_named_t pxintr_ks_cpu;
821 kstat_named_t pxintr_ks_pil;
822 kstat_named_t pxintr_ks_time;
823 kstat_named_t pxintr_ks_ino;
824 kstat_named_t pxintr_ks_cookie;
825 kstat_named_t pxintr_ks_devpath;
826 kstat_named_t pxintr_ks_buspath;
827 } pxintr_ks_template = {
828 { "name", KSTAT_DATA_CHAR },
829 { "type", KSTAT_DATA_CHAR },
830 { "cpu", KSTAT_DATA_UINT64 },
831 { "pil", KSTAT_DATA_UINT64 },
832 { "time", KSTAT_DATA_UINT64 },
833 { "ino", KSTAT_DATA_UINT64 },
834 { "cookie", KSTAT_DATA_UINT64 },
835 { "devpath", KSTAT_DATA_STRING },
836 { "buspath", KSTAT_DATA_STRING },
837 };
838
839 static uint32_t pxintr_ks_instance;
840 static char ih_devpath[MAXPATHLEN];
841 static char ih_buspath[MAXPATHLEN];
842 kmutex_t pxintr_ks_template_lock;
843
844 int
px_ks_update(kstat_t * ksp,int rw)845 px_ks_update(kstat_t *ksp, int rw)
846 {
847 px_ih_t *ih_p = ksp->ks_private;
848 int maxlen = sizeof (pxintr_ks_template.pxintr_ks_name.value.c);
849 px_ino_pil_t *ipil_p = ih_p->ih_ipil_p;
850 px_ino_t *ino_p = ipil_p->ipil_ino_p;
851 px_t *px_p = ino_p->ino_ib_p->ib_px_p;
852 devino_t ino;
853 sysino_t sysino;
854
855 ino = ino_p->ino_ino;
856 if (px_lib_intr_devino_to_sysino(px_p->px_dip, ino, &sysino) !=
857 DDI_SUCCESS) {
858 cmn_err(CE_WARN, "px_ks_update: px_lib_intr_devino_to_sysino "
859 "failed");
860 }
861
862 (void) snprintf(pxintr_ks_template.pxintr_ks_name.value.c, maxlen,
863 "%s%d", ddi_driver_name(ih_p->ih_dip),
864 ddi_get_instance(ih_p->ih_dip));
865
866 (void) ddi_pathname(ih_p->ih_dip, ih_devpath);
867 (void) ddi_pathname(px_p->px_dip, ih_buspath);
868 kstat_named_setstr(&pxintr_ks_template.pxintr_ks_devpath, ih_devpath);
869 kstat_named_setstr(&pxintr_ks_template.pxintr_ks_buspath, ih_buspath);
870
871 if (ih_p->ih_intr_state == PX_INTR_STATE_ENABLE) {
872
873 switch (i_ddi_intr_get_current_type(ih_p->ih_dip)) {
874 case DDI_INTR_TYPE_MSI:
875 (void) strcpy(pxintr_ks_template.pxintr_ks_type.value.c,
876 "msi");
877 break;
878 case DDI_INTR_TYPE_MSIX:
879 (void) strcpy(pxintr_ks_template.pxintr_ks_type.value.c,
880 "msix");
881 break;
882 default:
883 (void) strcpy(pxintr_ks_template.pxintr_ks_type.value.c,
884 "fixed");
885 break;
886 }
887
888 pxintr_ks_template.pxintr_ks_cpu.value.ui64 = ino_p->ino_cpuid;
889 pxintr_ks_template.pxintr_ks_pil.value.ui64 = ipil_p->ipil_pil;
890 pxintr_ks_template.pxintr_ks_time.value.ui64 = ih_p->ih_nsec +
891 (uint64_t)tick2ns((hrtime_t)ih_p->ih_ticks,
892 ino_p->ino_cpuid);
893 pxintr_ks_template.pxintr_ks_ino.value.ui64 = ino;
894 pxintr_ks_template.pxintr_ks_cookie.value.ui64 = sysino;
895 } else {
896 (void) strcpy(pxintr_ks_template.pxintr_ks_type.value.c,
897 "disabled");
898 pxintr_ks_template.pxintr_ks_cpu.value.ui64 = 0;
899 pxintr_ks_template.pxintr_ks_pil.value.ui64 = 0;
900 pxintr_ks_template.pxintr_ks_time.value.ui64 = 0;
901 pxintr_ks_template.pxintr_ks_ino.value.ui64 = 0;
902 pxintr_ks_template.pxintr_ks_cookie.value.ui64 = 0;
903 }
904 return (0);
905 }
906
907 void
px_create_intr_kstats(px_ih_t * ih_p)908 px_create_intr_kstats(px_ih_t *ih_p)
909 {
910 msiq_rec_type_t rec_type = ih_p->ih_rec_type;
911
912 ASSERT(ih_p->ih_ksp == NULL);
913
914 /*
915 * Create pci_intrs::: kstats for all ih types except messages,
916 * which represent unusual conditions and don't need to be tracked.
917 */
918 if (rec_type == 0 || rec_type == MSI32_REC || rec_type == MSI64_REC) {
919 ih_p->ih_ksp = kstat_create("pci_intrs",
920 atomic_inc_32_nv(&pxintr_ks_instance), "config",
921 "interrupts", KSTAT_TYPE_NAMED,
922 sizeof (pxintr_ks_template) / sizeof (kstat_named_t),
923 KSTAT_FLAG_VIRTUAL);
924 }
925 if (ih_p->ih_ksp != NULL) {
926 ih_p->ih_ksp->ks_data_size += MAXPATHLEN * 2;
927 ih_p->ih_ksp->ks_lock = &pxintr_ks_template_lock;
928 ih_p->ih_ksp->ks_data = &pxintr_ks_template;
929 ih_p->ih_ksp->ks_private = ih_p;
930 ih_p->ih_ksp->ks_update = px_ks_update;
931 }
932 }
933
934 /*
935 * px_add_intx_intr:
936 *
937 * This function is called to register INTx and legacy hardware
938 * interrupt pins interrupts.
939 */
940 int
px_add_intx_intr(dev_info_t * dip,dev_info_t * rdip,ddi_intr_handle_impl_t * hdlp)941 px_add_intx_intr(dev_info_t *dip, dev_info_t *rdip,
942 ddi_intr_handle_impl_t *hdlp)
943 {
944 px_t *px_p = INST_TO_STATE(ddi_get_instance(dip));
945 px_ib_t *ib_p = px_p->px_ib_p;
946 devino_t ino;
947 px_ih_t *ih_p;
948 px_ino_t *ino_p;
949 px_ino_pil_t *ipil_p, *ipil_list;
950 int32_t weight;
951 int ret = DDI_SUCCESS;
952 cpuid_t curr_cpu;
953
954 ino = hdlp->ih_vector;
955
956 DBG(DBG_A_INTX, dip, "px_add_intx_intr: rdip=%s%d ino=%x "
957 "handler=%x arg1=%x arg2=%x\n", ddi_driver_name(rdip),
958 ddi_get_instance(rdip), ino, hdlp->ih_cb_func,
959 hdlp->ih_cb_arg1, hdlp->ih_cb_arg2);
960
961 ih_p = px_ib_alloc_ih(rdip, hdlp->ih_inum,
962 hdlp->ih_cb_func, hdlp->ih_cb_arg1, hdlp->ih_cb_arg2, 0, 0);
963
964 mutex_enter(&ib_p->ib_ino_lst_mutex);
965
966 ino_p = px_ib_locate_ino(ib_p, ino);
967 ipil_list = ino_p ? ino_p->ino_ipil_p : NULL;
968
969 if (hdlp->ih_pri == 0)
970 hdlp->ih_pri = pci_class_to_pil(rdip);
971
972 /* Sharing the INO using a PIL that already exists */
973 if (ino_p && (ipil_p = px_ib_ino_locate_ipil(ino_p, hdlp->ih_pri))) {
974 if (px_ib_intr_locate_ih(ipil_p, rdip, hdlp->ih_inum, 0, 0)) {
975 DBG(DBG_A_INTX, dip, "px_add_intx_intr: "
976 "dup intr #%d\n", hdlp->ih_inum);
977
978 ret = DDI_FAILURE;
979 goto fail1;
980 }
981
982 /* Save mondo value in hdlp */
983 hdlp->ih_vector = ino_p->ino_sysino;
984
985 if ((ret = px_ib_ino_add_intr(px_p, ipil_p,
986 ih_p)) != DDI_SUCCESS)
987 goto fail1;
988
989 goto ino_done;
990 }
991
992 /* Sharing the INO using a new PIL */
993 if (ipil_list != NULL) {
994 /*
995 * disable INO to avoid lopil race condition with
996 * px_intx_intr
997 */
998
999 if ((ret = px_lib_intr_gettarget(dip, ino_p->ino_sysino,
1000 &curr_cpu)) != DDI_SUCCESS) {
1001 DBG(DBG_IB, dip,
1002 "px_add_intx_intr px_intr_gettarget() failed\n");
1003
1004 goto fail1;
1005 }
1006
1007 /* Wait on pending interrupt */
1008 if ((ret = px_ib_intr_pend(dip, ino_p->ino_sysino)) !=
1009 DDI_SUCCESS) {
1010 cmn_err(CE_WARN, "%s%d: px_add_intx_intr: "
1011 "pending sysino 0x%lx(ino 0x%x) timeout",
1012 ddi_driver_name(dip), ddi_get_instance(dip),
1013 ino_p->ino_sysino, ino_p->ino_ino);
1014 goto fail1;
1015 }
1016 }
1017
1018 ipil_p = px_ib_new_ino_pil(ib_p, ino, hdlp->ih_pri, ih_p);
1019 ino_p = ipil_p->ipil_ino_p;
1020
1021 /* Save mondo value in hdlp */
1022 hdlp->ih_vector = ino_p->ino_sysino;
1023
1024 DBG(DBG_A_INTX, dip, "px_add_intx_intr: pil=0x%x mondo=0x%x\n",
1025 hdlp->ih_pri, hdlp->ih_vector);
1026
1027 DDI_INTR_ASSIGN_HDLR_N_ARGS(hdlp,
1028 (ddi_intr_handler_t *)px_intx_intr, (caddr_t)ipil_p, NULL);
1029
1030 ret = i_ddi_add_ivintr(hdlp);
1031
1032 /*
1033 * Restore original interrupt handler
1034 * and arguments in interrupt handle.
1035 */
1036 DDI_INTR_ASSIGN_HDLR_N_ARGS(hdlp, ih_p->ih_handler,
1037 ih_p->ih_handler_arg1, ih_p->ih_handler_arg2);
1038
1039 if (ret != DDI_SUCCESS)
1040 goto fail2;
1041
1042 /* Save the pil for this ino */
1043 ipil_p->ipil_pil = hdlp->ih_pri;
1044
1045 /* Select cpu, saving it for sharing and removal */
1046 if (ipil_list == NULL) {
1047 if (ino_p->ino_cpuid == -1)
1048 ino_p->ino_cpuid = intr_dist_cpuid();
1049
1050 /* Enable interrupt */
1051 px_ib_intr_enable(px_p, ino_p->ino_cpuid, ino);
1052 } else {
1053 /* Re-enable interrupt */
1054 PX_INTR_ENABLE(dip, ino_p->ino_sysino, curr_cpu);
1055 }
1056
1057 ino_done:
1058 hdlp->ih_target = ino_p->ino_cpuid;
1059
1060 /* Add weight to the cpu that we are already targeting */
1061 weight = pci_class_to_intr_weight(rdip);
1062 intr_dist_cpuid_add_device_weight(ino_p->ino_cpuid, rdip, weight);
1063
1064 ih_p->ih_ipil_p = ipil_p;
1065 px_create_intr_kstats(ih_p);
1066 if (ih_p->ih_ksp)
1067 kstat_install(ih_p->ih_ksp);
1068 mutex_exit(&ib_p->ib_ino_lst_mutex);
1069
1070 DBG(DBG_A_INTX, dip, "px_add_intx_intr: done! Interrupt 0x%x pil=%x\n",
1071 ino_p->ino_sysino, hdlp->ih_pri);
1072
1073 return (ret);
1074 fail2:
1075 px_ib_delete_ino_pil(ib_p, ipil_p);
1076 fail1:
1077 if (ih_p->ih_config_handle)
1078 pci_config_teardown(&ih_p->ih_config_handle);
1079
1080 mutex_exit(&ib_p->ib_ino_lst_mutex);
1081 kmem_free(ih_p, sizeof (px_ih_t));
1082
1083 DBG(DBG_A_INTX, dip, "px_add_intx_intr: Failed! Interrupt 0x%x "
1084 "pil=%x\n", ino_p->ino_sysino, hdlp->ih_pri);
1085
1086 return (ret);
1087 }
1088
1089 /*
1090 * px_rem_intx_intr:
1091 *
1092 * This function is called to unregister INTx and legacy hardware
1093 * interrupt pins interrupts.
1094 */
1095 int
px_rem_intx_intr(dev_info_t * dip,dev_info_t * rdip,ddi_intr_handle_impl_t * hdlp)1096 px_rem_intx_intr(dev_info_t *dip, dev_info_t *rdip,
1097 ddi_intr_handle_impl_t *hdlp)
1098 {
1099 px_t *px_p = INST_TO_STATE(ddi_get_instance(dip));
1100 px_ib_t *ib_p = px_p->px_ib_p;
1101 devino_t ino;
1102 cpuid_t curr_cpu;
1103 px_ino_t *ino_p;
1104 px_ino_pil_t *ipil_p;
1105 px_ih_t *ih_p;
1106 int ret = DDI_SUCCESS;
1107
1108 ino = hdlp->ih_vector;
1109
1110 DBG(DBG_R_INTX, dip, "px_rem_intx_intr: rdip=%s%d ino=%x\n",
1111 ddi_driver_name(rdip), ddi_get_instance(rdip), ino);
1112
1113 mutex_enter(&ib_p->ib_ino_lst_mutex);
1114
1115 ino_p = px_ib_locate_ino(ib_p, ino);
1116 ipil_p = px_ib_ino_locate_ipil(ino_p, hdlp->ih_pri);
1117 ih_p = px_ib_intr_locate_ih(ipil_p, rdip, hdlp->ih_inum, 0, 0);
1118
1119 /* Get the current cpu */
1120 if ((ret = px_lib_intr_gettarget(px_p->px_dip, ino_p->ino_sysino,
1121 &curr_cpu)) != DDI_SUCCESS)
1122 goto fail;
1123
1124 if ((ret = px_ib_ino_rem_intr(px_p, ipil_p, ih_p)) != DDI_SUCCESS)
1125 goto fail;
1126
1127 intr_dist_cpuid_rem_device_weight(ino_p->ino_cpuid, rdip);
1128
1129 if (ipil_p->ipil_ih_size == 0) {
1130 hdlp->ih_vector = ino_p->ino_sysino;
1131 i_ddi_rem_ivintr(hdlp);
1132
1133 px_ib_delete_ino_pil(ib_p, ipil_p);
1134 }
1135
1136 if (ino_p->ino_ipil_size == 0) {
1137 kmem_free(ino_p, sizeof (px_ino_t));
1138 } else {
1139 /* Re-enable interrupt only if mapping register still shared */
1140 PX_INTR_ENABLE(px_p->px_dip, ino_p->ino_sysino, curr_cpu);
1141 }
1142
1143 fail:
1144 mutex_exit(&ib_p->ib_ino_lst_mutex);
1145 return (ret);
1146 }
1147
1148 /*
1149 * px_add_msiq_intr:
1150 *
1151 * This function is called to register MSI/Xs and PCIe message interrupts.
1152 */
1153 int
px_add_msiq_intr(dev_info_t * dip,dev_info_t * rdip,ddi_intr_handle_impl_t * hdlp,msiq_rec_type_t rec_type,msgcode_t msg_code,cpuid_t cpu_id,msiqid_t * msiq_id_p)1154 px_add_msiq_intr(dev_info_t *dip, dev_info_t *rdip,
1155 ddi_intr_handle_impl_t *hdlp, msiq_rec_type_t rec_type,
1156 msgcode_t msg_code, cpuid_t cpu_id, msiqid_t *msiq_id_p)
1157 {
1158 px_t *px_p = INST_TO_STATE(ddi_get_instance(dip));
1159 px_ib_t *ib_p = px_p->px_ib_p;
1160 px_msiq_state_t *msiq_state_p = &ib_p->ib_msiq_state;
1161 devino_t ino;
1162 px_ih_t *ih_p;
1163 px_ino_t *ino_p;
1164 px_ino_pil_t *ipil_p, *ipil_list;
1165 int32_t weight;
1166 int ret = DDI_SUCCESS;
1167
1168 DBG(DBG_MSIQ, dip, "px_add_msiq_intr: rdip=%s%d handler=0x%x "
1169 "arg1=0x%x arg2=0x%x cpu=0x%x\n", ddi_driver_name(rdip),
1170 ddi_get_instance(rdip), hdlp->ih_cb_func, hdlp->ih_cb_arg1,
1171 hdlp->ih_cb_arg2, cpu_id);
1172
1173 ih_p = px_ib_alloc_ih(rdip, hdlp->ih_inum, hdlp->ih_cb_func,
1174 hdlp->ih_cb_arg1, hdlp->ih_cb_arg2, rec_type, msg_code);
1175
1176 mutex_enter(&ib_p->ib_ino_lst_mutex);
1177
1178 ret = (cpu_id == -1) ? px_msiq_alloc(px_p, rec_type, msg_code,
1179 msiq_id_p) : px_msiq_alloc_based_on_cpuid(px_p, rec_type,
1180 cpu_id, msiq_id_p);
1181
1182 if (ret != DDI_SUCCESS) {
1183 DBG(DBG_MSIQ, dip, "px_add_msiq_intr: "
1184 "msiq allocation failed\n");
1185 goto fail;
1186 }
1187
1188 ino = px_msiqid_to_devino(px_p, *msiq_id_p);
1189
1190 ino_p = px_ib_locate_ino(ib_p, ino);
1191 ipil_list = ino_p ? ino_p->ino_ipil_p : NULL;
1192
1193 if (hdlp->ih_pri == 0)
1194 hdlp->ih_pri = pci_class_to_pil(rdip);
1195
1196 /* Sharing ino */
1197 if (ino_p && (ipil_p = px_ib_ino_locate_ipil(ino_p, hdlp->ih_pri))) {
1198 if (px_ib_intr_locate_ih(ipil_p, rdip,
1199 hdlp->ih_inum, rec_type, msg_code)) {
1200 DBG(DBG_MSIQ, dip, "px_add_msiq_intr: "
1201 "dup intr #%d\n", hdlp->ih_inum);
1202
1203 ret = DDI_FAILURE;
1204 goto fail1;
1205 }
1206
1207 /* Save mondo value in hdlp */
1208 hdlp->ih_vector = ino_p->ino_sysino;
1209
1210 if ((ret = px_ib_ino_add_intr(px_p, ipil_p,
1211 ih_p)) != DDI_SUCCESS)
1212 goto fail1;
1213
1214 goto ino_done;
1215 }
1216
1217 ipil_p = px_ib_new_ino_pil(ib_p, ino, hdlp->ih_pri, ih_p);
1218 ino_p = ipil_p->ipil_ino_p;
1219
1220 ino_p->ino_msiq_p = msiq_state_p->msiq_p +
1221 (*msiq_id_p - msiq_state_p->msiq_1st_msiq_id);
1222
1223 /* Save mondo value in hdlp */
1224 hdlp->ih_vector = ino_p->ino_sysino;
1225
1226 DBG(DBG_MSIQ, dip, "px_add_msiq_intr: pil=0x%x mondo=0x%x\n",
1227 hdlp->ih_pri, hdlp->ih_vector);
1228
1229 DDI_INTR_ASSIGN_HDLR_N_ARGS(hdlp,
1230 (ddi_intr_handler_t *)px_msiq_intr, (caddr_t)ipil_p, NULL);
1231
1232 ret = i_ddi_add_ivintr(hdlp);
1233
1234 /*
1235 * Restore original interrupt handler
1236 * and arguments in interrupt handle.
1237 */
1238 DDI_INTR_ASSIGN_HDLR_N_ARGS(hdlp, ih_p->ih_handler,
1239 ih_p->ih_handler_arg1, ih_p->ih_handler_arg2);
1240
1241 if (ret != DDI_SUCCESS)
1242 goto fail2;
1243
1244 /* Save the pil for this ino */
1245 ipil_p->ipil_pil = hdlp->ih_pri;
1246
1247 /* Select cpu, saving it for sharing and removal */
1248 if (ipil_list == NULL) {
1249 /* Enable MSIQ */
1250 px_lib_msiq_setstate(dip, *msiq_id_p, PCI_MSIQ_STATE_IDLE);
1251 px_lib_msiq_setvalid(dip, *msiq_id_p, PCI_MSIQ_VALID);
1252
1253 if (ino_p->ino_cpuid == -1)
1254 ino_p->ino_cpuid = intr_dist_cpuid();
1255
1256 /* Enable interrupt */
1257 px_ib_intr_enable(px_p, ino_p->ino_cpuid, ino);
1258 }
1259
1260 ino_done:
1261 hdlp->ih_target = ino_p->ino_cpuid;
1262
1263 /* Add weight to the cpu that we are already targeting */
1264 weight = pci_class_to_intr_weight(rdip);
1265 intr_dist_cpuid_add_device_weight(ino_p->ino_cpuid, rdip, weight);
1266
1267 ih_p->ih_ipil_p = ipil_p;
1268 px_create_intr_kstats(ih_p);
1269 if (ih_p->ih_ksp)
1270 kstat_install(ih_p->ih_ksp);
1271 mutex_exit(&ib_p->ib_ino_lst_mutex);
1272
1273 DBG(DBG_MSIQ, dip, "px_add_msiq_intr: done! Interrupt 0x%x pil=%x\n",
1274 ino_p->ino_sysino, hdlp->ih_pri);
1275
1276 return (ret);
1277 fail2:
1278 px_ib_delete_ino_pil(ib_p, ipil_p);
1279 fail1:
1280 (void) px_msiq_free(px_p, *msiq_id_p);
1281 fail:
1282 if (ih_p->ih_config_handle)
1283 pci_config_teardown(&ih_p->ih_config_handle);
1284
1285 mutex_exit(&ib_p->ib_ino_lst_mutex);
1286 kmem_free(ih_p, sizeof (px_ih_t));
1287
1288 DBG(DBG_MSIQ, dip, "px_add_msiq_intr: Failed! Interrupt 0x%x pil=%x\n",
1289 ino_p->ino_sysino, hdlp->ih_pri);
1290
1291 return (ret);
1292 }
1293
1294 /*
1295 * px_rem_msiq_intr:
1296 *
1297 * This function is called to unregister MSI/Xs and PCIe message interrupts.
1298 */
1299 int
px_rem_msiq_intr(dev_info_t * dip,dev_info_t * rdip,ddi_intr_handle_impl_t * hdlp,msiq_rec_type_t rec_type,msgcode_t msg_code,msiqid_t msiq_id)1300 px_rem_msiq_intr(dev_info_t *dip, dev_info_t *rdip,
1301 ddi_intr_handle_impl_t *hdlp, msiq_rec_type_t rec_type,
1302 msgcode_t msg_code, msiqid_t msiq_id)
1303 {
1304 px_t *px_p = INST_TO_STATE(ddi_get_instance(dip));
1305 px_ib_t *ib_p = px_p->px_ib_p;
1306 devino_t ino = px_msiqid_to_devino(px_p, msiq_id);
1307 cpuid_t curr_cpu;
1308 px_ino_t *ino_p;
1309 px_ino_pil_t *ipil_p;
1310 px_ih_t *ih_p;
1311 int ret = DDI_SUCCESS;
1312
1313 DBG(DBG_MSIQ, dip, "px_rem_msiq_intr: rdip=%s%d msiq_id=%x ino=%x\n",
1314 ddi_driver_name(rdip), ddi_get_instance(rdip), msiq_id, ino);
1315
1316 mutex_enter(&ib_p->ib_ino_lst_mutex);
1317
1318 ino_p = px_ib_locate_ino(ib_p, ino);
1319 ipil_p = px_ib_ino_locate_ipil(ino_p, hdlp->ih_pri);
1320 ih_p = px_ib_intr_locate_ih(ipil_p, rdip, hdlp->ih_inum, rec_type,
1321 msg_code);
1322
1323 /* Get the current cpu */
1324 if ((ret = px_lib_intr_gettarget(px_p->px_dip, ino_p->ino_sysino,
1325 &curr_cpu)) != DDI_SUCCESS)
1326 goto fail;
1327
1328 if ((ret = px_ib_ino_rem_intr(px_p, ipil_p, ih_p)) != DDI_SUCCESS)
1329 goto fail;
1330
1331 intr_dist_cpuid_rem_device_weight(ino_p->ino_cpuid, rdip);
1332
1333 if (ipil_p->ipil_ih_size == 0) {
1334 hdlp->ih_vector = ino_p->ino_sysino;
1335 i_ddi_rem_ivintr(hdlp);
1336
1337 px_ib_delete_ino_pil(ib_p, ipil_p);
1338
1339 if (ino_p->ino_ipil_size == 0)
1340 px_lib_msiq_setvalid(dip,
1341 px_devino_to_msiqid(px_p, ino), PCI_MSIQ_INVALID);
1342 }
1343
1344 (void) px_msiq_free(px_p, msiq_id);
1345
1346 if (ino_p->ino_ipil_size) {
1347 /* Re-enable interrupt only if mapping register still shared */
1348 PX_INTR_ENABLE(px_p->px_dip, ino_p->ino_sysino, curr_cpu);
1349 }
1350
1351 fail:
1352 mutex_exit(&ib_p->ib_ino_lst_mutex);
1353 return (ret);
1354 }
1355