1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21
22 /*
23 * Copyright 2005 Sun Microsystems, Inc. All rights reserved.
24 * Use is subject to license terms.
25 */
26
27 /*
28 * tavor_event.c
29 * Tavor Interrupt and Event Processing Routines
30 *
31 * Implements all the routines necessary for allocating, freeing, and
32 * handling all of the various event types that the Tavor hardware can
33 * generate.
34 * These routines include the main Tavor interrupt service routine
35 * (tavor_isr()) as well as all the code necessary to setup and handle
36 * events from each of the many event queues used by the Tavor device.
37 */
38
39 #include <sys/types.h>
40 #include <sys/conf.h>
41 #include <sys/ddi.h>
42 #include <sys/sunddi.h>
43 #include <sys/modctl.h>
44
45 #include <sys/ib/adapters/tavor/tavor.h>
46
47 static void tavor_eq_poll(tavor_state_t *state, tavor_eqhdl_t eq);
48 static void tavor_eq_catastrophic(tavor_state_t *state);
49 static int tavor_eq_alloc(tavor_state_t *state, uint32_t log_eq_size,
50 uint_t intr, tavor_eqhdl_t *eqhdl);
51 static int tavor_eq_free(tavor_state_t *state, tavor_eqhdl_t *eqhdl);
52 static int tavor_eq_handler_init(tavor_state_t *state, tavor_eqhdl_t eq,
53 uint_t evt_type_mask, int (*eqfunc)(tavor_state_t *state,
54 tavor_eqhdl_t eq, tavor_hw_eqe_t *eqe));
55 static int tavor_eq_handler_fini(tavor_state_t *state, tavor_eqhdl_t eq);
56 static void tavor_eqe_sync(tavor_eqhdl_t eq, tavor_hw_eqe_t *eqe, uint_t flag,
57 uint_t force_sync);
58 static int tavor_port_state_change_handler(tavor_state_t *state,
59 tavor_eqhdl_t eq, tavor_hw_eqe_t *eqe);
60 static int tavor_comm_estbl_handler(tavor_state_t *state,
61 tavor_eqhdl_t eq, tavor_hw_eqe_t *eqe);
62 static int tavor_local_wq_cat_err_handler(tavor_state_t *state,
63 tavor_eqhdl_t eq, tavor_hw_eqe_t *eqe);
64 static int tavor_invreq_local_wq_err_handler(tavor_state_t *state,
65 tavor_eqhdl_t eq, tavor_hw_eqe_t *eqe);
66 static int tavor_local_acc_vio_wq_err_handler(tavor_state_t *state,
67 tavor_eqhdl_t eq, tavor_hw_eqe_t *eqe);
68 static int tavor_sendq_drained_handler(tavor_state_t *state,
69 tavor_eqhdl_t eq, tavor_hw_eqe_t *eqe);
70 static int tavor_path_mig_handler(tavor_state_t *state,
71 tavor_eqhdl_t eq, tavor_hw_eqe_t *eqe);
72 static int tavor_path_mig_err_handler(tavor_state_t *state,
73 tavor_eqhdl_t eq, tavor_hw_eqe_t *eqe);
74 static int tavor_srq_catastrophic_handler(tavor_state_t *state,
75 tavor_eqhdl_t eq, tavor_hw_eqe_t *eqe);
76 static int tavor_srq_last_wqe_reached_handler(tavor_state_t *state,
77 tavor_eqhdl_t eq, tavor_hw_eqe_t *eqe);
78 static int tavor_ecc_detection_handler(tavor_state_t *state,
79 tavor_eqhdl_t eq, tavor_hw_eqe_t *eqe);
80 static int tavor_no_eqhandler(tavor_state_t *state, tavor_eqhdl_t eq,
81 tavor_hw_eqe_t *eqe);
82
83
84 /*
85 * tavor_eq_init_all
86 * Context: Only called from attach() path context
87 */
88 int
tavor_eq_init_all(tavor_state_t * state)89 tavor_eq_init_all(tavor_state_t *state)
90 {
91 uint_t log_eq_size, intr_num;
92 uint_t num_eq, num_eq_init, num_eq_unmap;
93 int status, i;
94
95 /*
96 * For now, all Event Queues default to the same size (pulled from
97 * the current configuration profile) and are all assigned to the
98 * same interrupt or MSI. In the future we may support assigning
99 * EQs to specific interrupts or MSIs XXX
100 */
101 log_eq_size = state->ts_cfg_profile->cp_log_default_eq_sz;
102
103 /*
104 * If MSI is to be used, then set intr_num to the MSI number
105 * (currently zero because we're using only one) or'd with the
106 * MSI enable flag. Otherwise, for regular (i.e. 'legacy') interrupt,
107 * use the 'inta_pin' value returned by QUERY_ADAPTER.
108 */
109 if (state->ts_intr_type_chosen == DDI_INTR_TYPE_MSI) {
110 intr_num = TAVOR_EQ_MSI_ENABLE_FLAG | 0;
111 } else {
112 intr_num = state->ts_adapter.inta_pin;
113 }
114
115 /*
116 * Total number of supported EQs is hardcoded. Tavor hardware
117 * supports up to 64 EQs. We are currently using only 45 of them
118 * We will set aside the first 32 for use with Completion Queues (CQ)
119 * and reserve a few of the other 32 for each specific class of event
120 * (see below for more details).
121 */
122 num_eq = TAVOR_NUM_EQ_USED;
123
124 /*
125 * The "num_eq_unmap" variable is used in any possible failure
126 * cleanup (below) to indicate which events queues might require
127 * possible event class unmapping.
128 */
129 num_eq_unmap = 0;
130
131 /*
132 * Allocate and initialize all the Event Queues. If any of these
133 * EQ allocations fail then jump to the end, cleanup what had been
134 * successfully initialized, and return an error.
135 */
136 for (i = 0; i < num_eq; i++) {
137 status = tavor_eq_alloc(state, log_eq_size, intr_num,
138 &state->ts_eqhdl[i]);
139 if (status != DDI_SUCCESS) {
140 num_eq_init = i;
141 goto all_eq_init_fail;
142 }
143 }
144 num_eq_init = num_eq;
145
146 /*
147 * Setup EQ0-EQ31 for use with Completion Queues. Note: We can
148 * cast the return value to void here because, when we use the
149 * TAVOR_EVT_NO_MASK flag, it is not possible for
150 * tavor_eq_handler_init() to return an error.
151 */
152 for (i = 0; i < 32; i++) {
153 (void) tavor_eq_handler_init(state, state->ts_eqhdl[i],
154 TAVOR_EVT_NO_MASK, tavor_cq_handler);
155 }
156 num_eq_unmap = 32;
157
158 /*
159 * Setup EQ32 for handling Completion Queue Error Events.
160 *
161 * These events include things like CQ overflow or CQ access
162 * violation errors. If this setup fails for any reason (which, in
163 * general, it really never should), then jump to the end, cleanup
164 * everything that has been successfully initialized, and return an
165 * error.
166 */
167 status = tavor_eq_handler_init(state, state->ts_eqhdl[32],
168 TAVOR_EVT_MSK_CQ_ERRORS, tavor_cq_err_handler);
169 if (status != DDI_SUCCESS) {
170 goto all_eq_init_fail;
171 }
172 num_eq_unmap = 33;
173
174 /*
175 * Setup EQ33 for handling Port State Change Events
176 *
177 * These events include things like Port Up and Port Down events.
178 * If this setup fails for any reason (which, in general, it really
179 * never should), then undo all previous EQ mapping, jump to the end,
180 * cleanup everything that has been successfully initialized, and
181 * return an error.
182 */
183 status = tavor_eq_handler_init(state, state->ts_eqhdl[33],
184 TAVOR_EVT_MSK_PORT_STATE_CHANGE, tavor_port_state_change_handler);
185 if (status != DDI_SUCCESS) {
186 goto all_eq_init_fail;
187 }
188 num_eq_unmap = 34;
189
190 /*
191 * Setup EQ34 for handling Communication Established Events
192 *
193 * These events correspond to the IB affiliated asynchronous events
194 * that are used for connection management. If this setup fails for
195 * any reason (which, in general, it really never should), then undo
196 * all previous EQ mapping, jump to the end, cleanup everything that
197 * has been successfully initialized, and return an error.
198 */
199 status = tavor_eq_handler_init(state, state->ts_eqhdl[34],
200 TAVOR_EVT_MSK_COMM_ESTABLISHED, tavor_comm_estbl_handler);
201 if (status != DDI_SUCCESS) {
202 goto all_eq_init_fail;
203 }
204 num_eq_unmap = 35;
205
206 /*
207 * Setup EQ35 for handling Command Completion Events
208 *
209 * These events correspond to the Tavor generated events that are used
210 * to indicate Tavor firmware command completion. These events are
211 * only generated when Tavor firmware commands are posted using the
212 * asynchronous completion mechanism. If this setup fails for any
213 * reason (which, in general, it really never should), then undo all
214 * previous EQ mapping, jump to the end, cleanup everything that has
215 * been successfully initialized, and return an error.
216 */
217 status = tavor_eq_handler_init(state, state->ts_eqhdl[35],
218 TAVOR_EVT_MSK_COMMAND_INTF_COMP, tavor_cmd_complete_handler);
219 if (status != DDI_SUCCESS) {
220 goto all_eq_init_fail;
221 }
222 num_eq_unmap = 36;
223
224 /*
225 * Setup EQ36 for handling Local WQ Catastrophic Error Events
226 *
227 * These events correspond to the similarly-named IB affiliated
228 * asynchronous error type. If this setup fails for any reason
229 * (which, in general, it really never should), then undo all previous
230 * EQ mapping, jump to the end, cleanup everything that has been
231 * successfully initialized, and return an error.
232 */
233 status = tavor_eq_handler_init(state, state->ts_eqhdl[36],
234 TAVOR_EVT_MSK_LOCAL_WQ_CAT_ERROR, tavor_local_wq_cat_err_handler);
235 if (status != DDI_SUCCESS) {
236 goto all_eq_init_fail;
237 }
238 num_eq_unmap = 37;
239
240 /*
241 * Setup EQ37 for handling Invalid Req Local WQ Error Events
242 *
243 * These events also correspond to the similarly-named IB affiliated
244 * asynchronous error type. If this setup fails for any reason
245 * (which, in general, it really never should), then undo all previous
246 * EQ mapping, jump to the end, cleanup everything that has been
247 * successfully initialized, and return an error.
248 */
249 status = tavor_eq_handler_init(state, state->ts_eqhdl[37],
250 TAVOR_EVT_MSK_INV_REQ_LOCAL_WQ_ERROR,
251 tavor_invreq_local_wq_err_handler);
252 if (status != DDI_SUCCESS) {
253 goto all_eq_init_fail;
254 }
255 num_eq_unmap = 38;
256
257 /*
258 * Setup EQ38 for handling Local Access Violation WQ Error Events
259 *
260 * These events also correspond to the similarly-named IB affiliated
261 * asynchronous error type. If this setup fails for any reason
262 * (which, in general, it really never should), then undo all previous
263 * EQ mapping, jump to the end, cleanup everything that has been
264 * successfully initialized, and return an error.
265 */
266 status = tavor_eq_handler_init(state, state->ts_eqhdl[38],
267 TAVOR_EVT_MSK_LOCAL_ACC_VIO_WQ_ERROR,
268 tavor_local_acc_vio_wq_err_handler);
269 if (status != DDI_SUCCESS) {
270 goto all_eq_init_fail;
271 }
272 num_eq_unmap = 39;
273
274 /*
275 * Setup EQ39 for handling Send Queue Drained Events
276 *
277 * These events correspond to the IB affiliated asynchronous events
278 * that are used to indicate completion of a Send Queue Drained QP
279 * state transition. If this setup fails for any reason (which, in
280 * general, it really never should), then undo all previous EQ
281 * mapping, jump to the end, cleanup everything that has been
282 * successfully initialized, and return an error.
283 */
284 status = tavor_eq_handler_init(state, state->ts_eqhdl[39],
285 TAVOR_EVT_MSK_SEND_QUEUE_DRAINED, tavor_sendq_drained_handler);
286 if (status != DDI_SUCCESS) {
287 goto all_eq_init_fail;
288 }
289 num_eq_unmap = 40;
290
291 /*
292 * Setup EQ40 for handling Path Migration Succeeded Events
293 *
294 * These events correspond to the IB affiliated asynchronous events
295 * that are used to indicate successful completion of a path
296 * migration. If this setup fails for any reason (which, in general,
297 * it really never should), then undo all previous EQ mapping, jump
298 * to the end, cleanup everything that has been successfully
299 * initialized, and return an error.
300 */
301 status = tavor_eq_handler_init(state, state->ts_eqhdl[40],
302 TAVOR_EVT_MSK_PATH_MIGRATED, tavor_path_mig_handler);
303 if (status != DDI_SUCCESS) {
304 goto all_eq_init_fail;
305 }
306 num_eq_unmap = 41;
307
308 /*
309 * Setup EQ41 for handling Path Migration Failed Events
310 *
311 * These events correspond to the IB affiliated asynchronous events
312 * that are used to indicate that path migration was not successful.
313 * If this setup fails for any reason (which, in general, it really
314 * never should), then undo all previous EQ mapping, jump to the end,
315 * cleanup everything that has been successfully initialized, and
316 * return an error.
317 */
318 status = tavor_eq_handler_init(state, state->ts_eqhdl[41],
319 TAVOR_EVT_MSK_PATH_MIGRATE_FAILED, tavor_path_mig_err_handler);
320 if (status != DDI_SUCCESS) {
321 goto all_eq_init_fail;
322 }
323 num_eq_unmap = 42;
324
325 /*
326 * Setup EQ42 for handling Local Catastrophic Error Events
327 *
328 * These events correspond to the similarly-named IB unaffiliated
329 * asynchronous error type. If this setup fails for any reason
330 * (which, in general, it really never should), then undo all previous
331 * EQ mapping, jump to the end, cleanup everything that has been
332 * successfully initialized, and return an error.
333 *
334 * This error is unique, in that an EQE is not generated if this event
335 * occurs. Instead, an interrupt is called and we must poll the
336 * Catastrophic Error buffer in CR-Space. This mapping is setup simply
337 * to enable this error reporting. We pass in a NULL handler since it
338 * will never be called.
339 */
340 status = tavor_eq_handler_init(state, state->ts_eqhdl[42],
341 TAVOR_EVT_MSK_LOCAL_CAT_ERROR, NULL);
342 if (status != DDI_SUCCESS) {
343 goto all_eq_init_fail;
344 }
345 num_eq_unmap = 43;
346
347 /*
348 * Setup EQ43 for handling SRQ Catastrophic Error Events
349 *
350 * These events correspond to the similarly-named IB affiliated
351 * asynchronous error type. If this setup fails for any reason
352 * (which, in general, it really never should), then undo all previous
353 * EQ mapping, jump to the end, cleanup everything that has been
354 * successfully initialized, and return an error.
355 */
356 status = tavor_eq_handler_init(state, state->ts_eqhdl[43],
357 TAVOR_EVT_MSK_SRQ_CATASTROPHIC_ERROR,
358 tavor_srq_catastrophic_handler);
359 if (status != DDI_SUCCESS) {
360 goto all_eq_init_fail;
361 }
362 num_eq_unmap = 44;
363
364 /*
365 * Setup EQ44 for handling SRQ Last WQE Reached Events
366 *
367 * These events correspond to the similarly-named IB affiliated
368 * asynchronous event type. If this setup fails for any reason
369 * (which, in general, it really never should), then undo all previous
370 * EQ mapping, jump to the end, cleanup everything that has been
371 * successfully initialized, and return an error.
372 */
373 status = tavor_eq_handler_init(state, state->ts_eqhdl[44],
374 TAVOR_EVT_MSK_SRQ_LAST_WQE_REACHED,
375 tavor_srq_last_wqe_reached_handler);
376 if (status != DDI_SUCCESS) {
377 goto all_eq_init_fail;
378 }
379 num_eq_unmap = 45;
380
381 /*
382 * Setup EQ45 for handling ECC error detection events
383 *
384 * These events correspond to the similarly-named IB affiliated
385 * asynchronous event type. If this setup fails for any reason
386 * (which, in general, it really never should), then undo all previous
387 * EQ mapping, jump to the end, cleanup everything that has been
388 * successfully initialized, and return an error.
389 */
390 status = tavor_eq_handler_init(state, state->ts_eqhdl[45],
391 TAVOR_EVT_MSK_ECC_DETECTION,
392 tavor_ecc_detection_handler);
393 if (status != DDI_SUCCESS) {
394 goto all_eq_init_fail;
395 }
396 num_eq_unmap = 46;
397
398 /*
399 * Setup EQ46 to catch all other types of events. Specifically, we
400 * do not catch the "Local EEC Catastrophic Error Event" because we
401 * should have no EEC (the Tavor driver does not support RD). We also
402 * choose not to handle any of the address translation page fault
403 * event types. Since we are not doing any page fault handling (and
404 * since the Tavor firmware does not currently support any such
405 * handling), we allow these events to go to the catch-all handler.
406 */
407 status = tavor_eq_handler_init(state, state->ts_eqhdl[46],
408 TAVOR_EVT_CATCHALL_MASK, tavor_no_eqhandler);
409 if (status != DDI_SUCCESS) {
410 goto all_eq_init_fail;
411 }
412
413 return (DDI_SUCCESS);
414
415 all_eq_init_fail:
416 /* Unmap any of the partially mapped EQs from above */
417 for (i = 0; i < num_eq_unmap; i++) {
418 (void) tavor_eq_handler_fini(state, state->ts_eqhdl[i]);
419 }
420
421 /* Free up any of the partially allocated EQs from above */
422 for (i = 0; i < num_eq_init; i++) {
423 (void) tavor_eq_free(state, &state->ts_eqhdl[i]);
424 }
425 return (status);
426 }
427
428
429 /*
430 * tavor_eq_fini_all
431 * Context: Only called from attach() and/or detach() path contexts
432 */
433 int
tavor_eq_fini_all(tavor_state_t * state)434 tavor_eq_fini_all(tavor_state_t *state)
435 {
436 uint_t num_eq;
437 int status, i;
438
439 /*
440 * Grab the total number of supported EQs again. This is the same
441 * hardcoded value that was used above (during the event queue
442 * initialization.)
443 */
444 num_eq = TAVOR_NUM_EQ_USED;
445
446 /*
447 * For each of the event queues that we initialized and mapped
448 * earlier, attempt to unmap the events from the EQ.
449 */
450 for (i = 0; i < num_eq; i++) {
451 status = tavor_eq_handler_fini(state, state->ts_eqhdl[i]);
452 if (status != DDI_SUCCESS) {
453 return (DDI_FAILURE);
454 }
455 }
456
457 /*
458 * Teardown and free up all the Event Queues that were allocated
459 * earlier.
460 */
461 for (i = 0; i < num_eq; i++) {
462 status = tavor_eq_free(state, &state->ts_eqhdl[i]);
463 if (status != DDI_SUCCESS) {
464 return (DDI_FAILURE);
465 }
466 }
467
468 return (DDI_SUCCESS);
469 }
470
471
472 /*
473 * tavor_eq_arm_all
474 * Context: Only called from attach() and/or detach() path contexts
475 */
476 void
tavor_eq_arm_all(tavor_state_t * state)477 tavor_eq_arm_all(tavor_state_t *state)
478 {
479 uint_t num_eq;
480 int i;
481
482 /*
483 * Grab the total number of supported EQs again. This is the same
484 * hardcoded value that was used above (during the event queue
485 * initialization.)
486 */
487 num_eq = TAVOR_NUM_EQ_USED;
488
489 /*
490 * For each of the event queues that we initialized and mapped
491 * earlier, attempt to arm it for event generation.
492 */
493 for (i = 0; i < num_eq; i++) {
494 tavor_eq_doorbell(state, TAVOR_EQDB_REARM_EQ, i, 0);
495 }
496 }
497
498
499 /*
500 * tavor_isr()
501 * Context: Only called from interrupt context (and during panic)
502 */
503 /* ARGSUSED */
504 uint_t
tavor_isr(caddr_t arg1,caddr_t arg2)505 tavor_isr(caddr_t arg1, caddr_t arg2)
506 {
507 tavor_state_t *state;
508 uint64_t *ecr, *clr_int;
509 uint64_t ecrreg, int_mask;
510 uint_t status;
511 int i;
512
513 /*
514 * Grab the Tavor softstate pointer from the input parameter
515 */
516 state = (tavor_state_t *)arg1;
517
518 /*
519 * Find the pointers to the ECR and clr_INT registers
520 */
521 ecr = state->ts_cmd_regs.ecr;
522 clr_int = state->ts_cmd_regs.clr_int;
523
524 /*
525 * Read the ECR register. Each of the 64 bits in the ECR register
526 * corresponds to an event queue. If a bit is set, then the
527 * corresponding event queue has fired.
528 */
529 ecrreg = ddi_get64(state->ts_reg_cmdhdl, ecr);
530
531 /*
532 * As long as there are bits set (i.e. as long as there are still
533 * EQs in the "fired" state), call tavor_eq_poll() to process each
534 * fired EQ. If no ECR bits are set, do not claim the interrupt.
535 */
536 status = DDI_INTR_UNCLAIMED;
537 do {
538 i = 0;
539 while (ecrreg != 0x0) {
540 if (ecrreg & 0x1) {
541 tavor_eq_poll(state, state->ts_eqhdl[i]);
542 status = DDI_INTR_CLAIMED;
543 }
544 ecrreg = ecrreg >> 1;
545 i++;
546 }
547
548 /*
549 * Clear the interrupt. Note: Depending on the type of
550 * event (interrupt or MSI), we need to use a different
551 * mask to clear the event. In the case of MSI, the bit
552 * to clear corresponds to the MSI number, and for legacy
553 * interrupts the bit corresponds to the value in 'inta_pin'.
554 */
555 if (state->ts_intr_type_chosen == DDI_INTR_TYPE_MSI) {
556 int_mask = ((uint64_t)1 << 0);
557 } else {
558 int_mask = ((uint64_t)1 << state->ts_adapter.inta_pin);
559 }
560 ddi_put64(state->ts_reg_cmdhdl, clr_int, int_mask);
561
562 /* Reread the ECR register */
563 ecrreg = ddi_get64(state->ts_reg_cmdhdl, ecr);
564
565 } while (ecrreg != 0x0);
566
567 return (status);
568 }
569
570
571 /*
572 * tavor_eq_doorbell
573 * Context: Only called from interrupt context
574 */
575 void
tavor_eq_doorbell(tavor_state_t * state,uint32_t eq_cmd,uint32_t eqn,uint32_t eq_param)576 tavor_eq_doorbell(tavor_state_t *state, uint32_t eq_cmd, uint32_t eqn,
577 uint32_t eq_param)
578 {
579 uint64_t doorbell = 0;
580
581 /* Build the doorbell from the parameters */
582 doorbell = ((uint64_t)eq_cmd << TAVOR_EQDB_CMD_SHIFT) |
583 ((uint64_t)eqn << TAVOR_EQDB_EQN_SHIFT) | eq_param;
584
585 /* Write the doorbell to UAR */
586 TAVOR_UAR_DOORBELL(state, (uint64_t *)&state->ts_uar->eq,
587 doorbell);
588 }
589
590 /*
591 * tavor_eq_poll
592 * Context: Only called from interrupt context (and during panic)
593 */
594 static void
tavor_eq_poll(tavor_state_t * state,tavor_eqhdl_t eq)595 tavor_eq_poll(tavor_state_t *state, tavor_eqhdl_t eq)
596 {
597 uint64_t *clr_ecr;
598 tavor_hw_eqe_t *eqe;
599 uint64_t ecr_mask;
600 uint32_t cons_indx, wrap_around_mask;
601 int (*eqfunction)(tavor_state_t *state, tavor_eqhdl_t eq,
602 tavor_hw_eqe_t *eqe);
603
604 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*eq))
605
606 /* Find the pointer to the clr_ECR register */
607 clr_ecr = state->ts_cmd_regs.clr_ecr;
608
609 /*
610 * Check for Local Catastrophic Error If we have this kind of error,
611 * then we don't need to do anything else here, as this kind of
612 * catastrophic error is handled separately. So we call the
613 * catastrophic handler, clear the ECR and then return.
614 */
615 if (eq->eq_evttypemask == TAVOR_EVT_MSK_LOCAL_CAT_ERROR) {
616 /*
617 * Call Catastrophic Error handler
618 */
619 tavor_eq_catastrophic(state);
620
621 /*
622 * Clear the ECR. Specifically, clear the bit corresponding
623 * to the event queue just processed.
624 */
625 ecr_mask = ((uint64_t)1 << eq->eq_eqnum);
626 ddi_put64(state->ts_reg_cmdhdl, clr_ecr, ecr_mask);
627
628 return;
629 }
630
631 /* Get the consumer pointer index */
632 cons_indx = eq->eq_consindx;
633
634 /*
635 * Calculate the wrap around mask. Note: This operation only works
636 * because all Tavor event queues have power-of-2 sizes
637 */
638 wrap_around_mask = (eq->eq_bufsz - 1);
639
640 /* Calculate the pointer to the first EQ entry */
641 eqe = &eq->eq_buf[cons_indx];
642 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*eqe))
643
644 /*
645 * Sync the current EQE to read
646 * We need to force a ddi_dma_sync() here (independent of how the
647 * EQ was mapped) because it is possible for us to receive the
648 * interrupt, do a read of the ECR, and have each of these
649 * operations complete successfully even though the hardware's DMA
650 * to the EQ has not yet completed.
651 */
652 tavor_eqe_sync(eq, eqe, DDI_DMA_SYNC_FORCPU, TAVOR_EQ_SYNC_FORCE);
653
654 /*
655 * Pull the handler function for this EQ from the Tavor Event Queue
656 * handle
657 */
658 eqfunction = eq->eq_func;
659
660 /*
661 * Keep pulling entries from the EQ until we find an entry owner by
662 * the hardware. As long as there the EQE's owned by SW, process
663 * each entry by calling its handler function and updating the EQ
664 * consumer index.
665 */
666 do {
667 while (TAVOR_EQE_OWNER_IS_SW(eq, eqe)) {
668 /*
669 * Call the EQ handler function. But only call if we
670 * are not in polled I/O mode (i.e. not processing
671 * because of a system panic). Note: We don't call
672 * the EQ handling functions from a system panic
673 * because we are primarily concerned only with
674 * ensuring that the event queues do not overflow (or,
675 * more specifically, the event queue associated with
676 * the CQ that is being used in the sync/dump process).
677 * Also, we don't want to make any upcalls (to the
678 * IBTF) because we can't guarantee when/if those
679 * calls would ever return. And, if we're in panic,
680 * then we reached here through a PollCQ() call (from
681 * tavor_cq_poll()), and we need to ensure that we
682 * successfully return any work completions to the
683 * caller.
684 */
685 if (ddi_in_panic() == 0) {
686 eqfunction(state, eq, eqe);
687 }
688
689 /* Reset entry to hardware ownership */
690 TAVOR_EQE_OWNER_SET_HW(eq, eqe);
691
692 /* Sync the current EQE for device */
693 tavor_eqe_sync(eq, eqe, DDI_DMA_SYNC_FORDEV,
694 TAVOR_EQ_SYNC_NORMAL);
695
696 /* Increment the consumer index */
697 cons_indx = (cons_indx + 1) & wrap_around_mask;
698
699 /* Update the pointer to the next EQ entry */
700 eqe = &eq->eq_buf[cons_indx];
701
702 /* Sync the next EQE to read */
703 tavor_eqe_sync(eq, eqe, DDI_DMA_SYNC_FORCPU,
704 TAVOR_EQ_SYNC_NORMAL);
705 }
706
707 /*
708 * Clear the ECR. Specifically, clear the bit corresponding
709 * to the event queue just processed.
710 */
711 ecr_mask = ((uint64_t)1 << eq->eq_eqnum);
712 ddi_put64(state->ts_reg_cmdhdl, clr_ecr, ecr_mask);
713
714 /* Write an EQ doorbell to update the consumer index */
715 eq->eq_consindx = cons_indx;
716 tavor_eq_doorbell(state, TAVOR_EQDB_SET_CONSINDX, eq->eq_eqnum,
717 cons_indx);
718
719 /* Write another EQ doorbell to rearm */
720 tavor_eq_doorbell(state, TAVOR_EQDB_REARM_EQ, eq->eq_eqnum, 0);
721
722 /*
723 * NOTE: Due to the nature of Mellanox hardware, we do not have
724 * to do an explicit PIO read to ensure that the doorbell write
725 * has been flushed to the hardware. There is state encoded in
726 * the doorbell information we write which makes this
727 * unnecessary. We can be assured that if an event needs to be
728 * generated, the hardware will make sure that it is, solving
729 * the possible race condition.
730 */
731
732 /* Sync the next EQE to read */
733 tavor_eqe_sync(eq, eqe, DDI_DMA_SYNC_FORCPU,
734 TAVOR_EQ_SYNC_NORMAL);
735
736 } while (TAVOR_EQE_OWNER_IS_SW(eq, eqe));
737 }
738
739
740 /*
741 * tavor_eq_catastrophic
742 * Context: Only called from interrupt context (and during panic)
743 */
744 static void
tavor_eq_catastrophic(tavor_state_t * state)745 tavor_eq_catastrophic(tavor_state_t *state)
746 {
747 ibt_async_code_t type;
748 ibc_async_event_t event;
749 uint32_t *base_addr;
750 uint32_t buf_size;
751 uint32_t word;
752 uint8_t err_type;
753 uint32_t err_buf;
754 int i;
755
756 bzero(&event, sizeof (ibc_async_event_t));
757
758 base_addr = (uint32_t *)(uintptr_t)(
759 (uintptr_t)state->ts_reg_cmd_baseaddr +
760 state->ts_fw.error_buf_addr);
761 buf_size = state->ts_fw.error_buf_sz;
762
763 word = ddi_get32(state->ts_reg_cmdhdl, base_addr);
764
765 err_type = (word & 0xFF000000) >> 24;
766 type = IBT_ERROR_LOCAL_CATASTROPHIC;
767
768 switch (err_type) {
769 case TAVOR_CATASTROPHIC_INTERNAL_ERROR:
770 cmn_err(CE_WARN, "Catastrophic Internal Error: 0x%02x",
771 err_type);
772
773 break;
774
775 case TAVOR_CATASTROPHIC_UPLINK_BUS_ERROR:
776 cmn_err(CE_WARN, "Catastrophic Uplink Bus Error: 0x%02x",
777 err_type);
778
779 break;
780
781 case TAVOR_CATASTROPHIC_DDR_DATA_ERROR:
782 cmn_err(CE_WARN, "Catastrophic DDR Data Error: 0x%02x",
783 err_type);
784
785 break;
786
787 case TAVOR_CATASTROPHIC_INTERNAL_PARITY_ERROR:
788 cmn_err(CE_WARN, "Catastrophic Internal Parity Error: 0x%02x",
789 err_type);
790
791 break;
792
793 default:
794 /* Unknown type of Catastrophic error */
795 cmn_err(CE_WARN, "Catastrophic Unknown Error: 0x%02x",
796 err_type);
797
798 break;
799 }
800
801 /*
802 * Read in the catastrophic error buffer from the hardware, printing
803 * only to the log file only
804 */
805 for (i = 0; i < buf_size; i += 4) {
806 base_addr = (uint32_t *)((uintptr_t)(state->ts_reg_cmd_baseaddr
807 + state->ts_fw.error_buf_addr + (i * 4)));
808 err_buf = ddi_get32(state->ts_reg_cmdhdl, base_addr);
809 cmn_err(CE_WARN, "catastrophic_error[%02x]: %08X", i, err_buf);
810 }
811
812 /*
813 * We also call the IBTF here to inform it of the catastrophic error.
814 * Note: Since no event information (i.e. QP handles, CQ handles,
815 * etc.) is necessary, we pass a NULL pointer instead of a pointer to
816 * an empty ibc_async_event_t struct.
817 *
818 * But we also check if "ts_ibtfpriv" is NULL. If it is then it
819 * means that we've have either received this event before we
820 * finished attaching to the IBTF or we've received it while we
821 * are in the process of detaching.
822 */
823 if (state->ts_ibtfpriv != NULL) {
824 TAVOR_DO_IBTF_ASYNC_CALLB(state, type, &event);
825 }
826 }
827
828
829 /*
830 * tavor_eq_alloc()
831 * Context: Only called from attach() path context
832 */
833 static int
tavor_eq_alloc(tavor_state_t * state,uint32_t log_eq_size,uint_t intr,tavor_eqhdl_t * eqhdl)834 tavor_eq_alloc(tavor_state_t *state, uint32_t log_eq_size, uint_t intr,
835 tavor_eqhdl_t *eqhdl)
836 {
837 tavor_rsrc_t *eqc, *rsrc;
838 tavor_hw_eqc_t eqc_entry;
839 tavor_eqhdl_t eq;
840 ibt_mr_attr_t mr_attr;
841 tavor_mr_options_t op;
842 tavor_pdhdl_t pd;
843 tavor_mrhdl_t mr;
844 tavor_hw_eqe_t *buf;
845 uint64_t addr;
846 uint32_t lkey;
847 uint_t dma_xfer_mode;
848 int status, i;
849
850 /* Use the internal protection domain (PD) for setting up EQs */
851 pd = state->ts_pdhdl_internal;
852
853 /* Increment the reference count on the protection domain (PD) */
854 tavor_pd_refcnt_inc(pd);
855
856 /*
857 * Allocate an EQ context entry. This will be filled in with all
858 * the necessary parameters to define the Event Queue. And then
859 * ownership will be passed to the hardware in the final step
860 * below. If we fail here, we must undo the protection domain
861 * reference count.
862 */
863 status = tavor_rsrc_alloc(state, TAVOR_EQC, 1, TAVOR_SLEEP, &eqc);
864 if (status != DDI_SUCCESS) {
865 goto eqalloc_fail1;
866 }
867
868 /*
869 * Allocate the software structure for tracking the event queue (i.e.
870 * the Tavor Event Queue handle). If we fail here, we must undo the
871 * protection domain reference count and the previous resource
872 * allocation.
873 */
874 status = tavor_rsrc_alloc(state, TAVOR_EQHDL, 1, TAVOR_SLEEP, &rsrc);
875 if (status != DDI_SUCCESS) {
876 goto eqalloc_fail2;
877 }
878 eq = (tavor_eqhdl_t)rsrc->tr_addr;
879
880 /*
881 * Allocate the memory for Event Queue. Note: Although we use the
882 * common queue allocation routine, we always specify
883 * TAVOR_QUEUE_LOCATION_NORMAL (i.e. EQ located in system memory)
884 * because it would be inefficient to have EQs located in DDR memory.
885 * This is primarily because EQs are read from (by software) more
886 * than they are written to. Also note that, unlike Tavor QP work
887 * queues, event queues do not have the same strict alignment
888 * requirements. It is sufficient for the EQ memory to be both
889 * aligned to and bound to addresses which are a multiple of EQE size.
890 */
891 eq->eq_eqinfo.qa_size = (1 << log_eq_size) * sizeof (tavor_hw_eqe_t);
892 eq->eq_eqinfo.qa_alloc_align = sizeof (tavor_hw_eqe_t);
893 eq->eq_eqinfo.qa_bind_align = sizeof (tavor_hw_eqe_t);
894 eq->eq_eqinfo.qa_location = TAVOR_QUEUE_LOCATION_NORMAL;
895 status = tavor_queue_alloc(state, &eq->eq_eqinfo, TAVOR_SLEEP);
896 if (status != DDI_SUCCESS) {
897 goto eqalloc_fail3;
898 }
899 buf = (tavor_hw_eqe_t *)eq->eq_eqinfo.qa_buf_aligned;
900
901 /*
902 * Initialize each of the Event Queue Entries (EQE) by setting their
903 * ownership to hardware ("owner" bit set to HW). This is in
904 * preparation for the final transfer of ownership (below) of the
905 * EQ context itself.
906 */
907 for (i = 0; i < (1 << log_eq_size); i++) {
908 TAVOR_EQE_OWNER_SET_HW(eq, &buf[i]);
909 }
910
911 /*
912 * Register the memory for the EQ. The memory for the EQ must
913 * be registered in the Tavor TPT tables. This gives us the LKey
914 * to specify in the EQ context below.
915 *
916 * Because we are in the attach path we use NOSLEEP here so that we
917 * SPIN in the HCR since the event queues are not setup yet, and we
918 * cannot NOSPIN at this point in time.
919 */
920 mr_attr.mr_vaddr = (uint64_t)(uintptr_t)buf;
921 mr_attr.mr_len = eq->eq_eqinfo.qa_size;
922 mr_attr.mr_as = NULL;
923 mr_attr.mr_flags = IBT_MR_NOSLEEP | IBT_MR_ENABLE_LOCAL_WRITE;
924 dma_xfer_mode = state->ts_cfg_profile->cp_streaming_consistent;
925 if (dma_xfer_mode == DDI_DMA_STREAMING) {
926 mr_attr.mr_flags |= IBT_MR_NONCOHERENT;
927 }
928 op.mro_bind_type = state->ts_cfg_profile->cp_iommu_bypass;
929 op.mro_bind_dmahdl = eq->eq_eqinfo.qa_dmahdl;
930 op.mro_bind_override_addr = 0;
931 status = tavor_mr_register(state, pd, &mr_attr, &mr, &op);
932 if (status != DDI_SUCCESS) {
933 goto eqalloc_fail4;
934 }
935 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*mr))
936 addr = mr->mr_bindinfo.bi_addr;
937 lkey = mr->mr_lkey;
938
939 /* Determine if later ddi_dma_sync will be necessary */
940 eq->eq_sync = TAVOR_EQ_IS_SYNC_REQ(state, eq->eq_eqinfo);
941
942 /* Sync entire EQ for use by the hardware (if necessary) */
943 if (eq->eq_sync) {
944 (void) ddi_dma_sync(mr->mr_bindinfo.bi_dmahdl, 0,
945 eq->eq_eqinfo.qa_size, DDI_DMA_SYNC_FORDEV);
946 }
947
948 /*
949 * Fill in the EQC entry. This is the final step before passing
950 * ownership of the EQC entry to the Tavor hardware. We use all of
951 * the information collected/calculated above to fill in the
952 * requisite portions of the EQC. Note: We create all EQs in the
953 * "fired" state. We will arm them later (after our interrupt
954 * routine had been registered.)
955 */
956 bzero(&eqc_entry, sizeof (tavor_hw_eqc_t));
957 eqc_entry.owner = TAVOR_HW_OWNER;
958 eqc_entry.xlat = TAVOR_VA2PA_XLAT_ENABLED;
959 eqc_entry.state = TAVOR_EQ_FIRED;
960 eqc_entry.start_addr_h = (addr >> 32);
961 eqc_entry.start_addr_l = (addr & 0xFFFFFFFF);
962 eqc_entry.log_eq_sz = log_eq_size;
963 eqc_entry.usr_page = 0;
964 eqc_entry.pd = pd->pd_pdnum;
965 eqc_entry.intr = intr;
966 eqc_entry.lkey = lkey;
967
968 /*
969 * Write the EQC entry to hardware. Lastly, we pass ownership of
970 * the entry to the hardware (using the Tavor SW2HW_EQ firmware
971 * command). Note: in general, this operation shouldn't fail. But
972 * if it does, we have to undo everything we've done above before
973 * returning error.
974 */
975 status = tavor_cmn_ownership_cmd_post(state, SW2HW_EQ, &eqc_entry,
976 sizeof (tavor_hw_eqc_t), eqc->tr_indx, TAVOR_CMD_NOSLEEP_SPIN);
977 if (status != TAVOR_CMD_SUCCESS) {
978 cmn_err(CE_CONT, "Tavor: SW2HW_EQ command failed: %08x\n",
979 status);
980 goto eqalloc_fail5;
981 }
982
983 /*
984 * Fill in the rest of the Tavor Event Queue handle. Having
985 * successfully transferred ownership of the EQC, we can update the
986 * following fields for use in further operations on the EQ.
987 */
988 eq->eq_eqcrsrcp = eqc;
989 eq->eq_rsrcp = rsrc;
990 eq->eq_consindx = 0;
991 eq->eq_eqnum = eqc->tr_indx;
992 eq->eq_buf = buf;
993 eq->eq_bufsz = (1 << log_eq_size);
994 eq->eq_mrhdl = mr;
995 *eqhdl = eq;
996
997 return (DDI_SUCCESS);
998
999 /*
1000 * The following is cleanup for all possible failure cases in this routine
1001 */
1002 eqalloc_fail5:
1003 if (tavor_mr_deregister(state, &mr, TAVOR_MR_DEREG_ALL,
1004 TAVOR_NOSLEEP) != DDI_SUCCESS) {
1005 TAVOR_WARNING(state, "failed to deregister EQ memory");
1006 }
1007 eqalloc_fail4:
1008 tavor_queue_free(state, &eq->eq_eqinfo);
1009 eqalloc_fail3:
1010 tavor_rsrc_free(state, &rsrc);
1011 eqalloc_fail2:
1012 tavor_rsrc_free(state, &eqc);
1013 eqalloc_fail1:
1014 tavor_pd_refcnt_dec(pd);
1015 eqalloc_fail:
1016 return (status);
1017 }
1018
1019
1020 /*
1021 * tavor_eq_free()
1022 * Context: Only called from attach() and/or detach() path contexts
1023 */
1024 static int
tavor_eq_free(tavor_state_t * state,tavor_eqhdl_t * eqhdl)1025 tavor_eq_free(tavor_state_t *state, tavor_eqhdl_t *eqhdl)
1026 {
1027 tavor_rsrc_t *eqc, *rsrc;
1028 tavor_hw_eqc_t eqc_entry;
1029 tavor_pdhdl_t pd;
1030 tavor_mrhdl_t mr;
1031 tavor_eqhdl_t eq;
1032 uint32_t eqnum;
1033 int status;
1034
1035 /*
1036 * Pull all the necessary information from the Tavor Event Queue
1037 * handle. This is necessary here because the resource for the
1038 * EQ handle is going to be freed up as part of this operation.
1039 */
1040 eq = *eqhdl;
1041 eqc = eq->eq_eqcrsrcp;
1042 rsrc = eq->eq_rsrcp;
1043 pd = state->ts_pdhdl_internal;
1044 mr = eq->eq_mrhdl;
1045 eqnum = eq->eq_eqnum;
1046
1047 /*
1048 * Reclaim EQC entry from hardware (using the Tavor HW2SW_EQ
1049 * firmware command). If the ownership transfer fails for any reason,
1050 * then it is an indication that something (either in HW or SW) has
1051 * gone seriously wrong.
1052 */
1053 status = tavor_cmn_ownership_cmd_post(state, HW2SW_EQ, &eqc_entry,
1054 sizeof (tavor_hw_eqc_t), eqnum, TAVOR_CMD_NOSLEEP_SPIN);
1055 if (status != TAVOR_CMD_SUCCESS) {
1056 TAVOR_WARNING(state, "failed to reclaim EQC ownership");
1057 cmn_err(CE_CONT, "Tavor: HW2SW_EQ command failed: %08x\n",
1058 status);
1059 return (DDI_FAILURE);
1060 }
1061
1062 /*
1063 * Deregister the memory for the Event Queue. If this fails
1064 * for any reason, then it is an indication that something (either
1065 * in HW or SW) has gone seriously wrong. So we print a warning
1066 * message and continue.
1067 */
1068 status = tavor_mr_deregister(state, &mr, TAVOR_MR_DEREG_ALL,
1069 TAVOR_NOSLEEP);
1070 if (status != DDI_SUCCESS) {
1071 TAVOR_WARNING(state, "failed to deregister EQ memory");
1072 }
1073
1074 /* Free the memory for the EQ */
1075 tavor_queue_free(state, &eq->eq_eqinfo);
1076
1077 /* Free the Tavor Event Queue handle */
1078 tavor_rsrc_free(state, &rsrc);
1079
1080 /* Free up the EQC entry resource */
1081 tavor_rsrc_free(state, &eqc);
1082
1083 /* Decrement the reference count on the protection domain (PD) */
1084 tavor_pd_refcnt_dec(pd);
1085
1086 /* Set the eqhdl pointer to NULL and return success */
1087 *eqhdl = NULL;
1088
1089 return (DDI_SUCCESS);
1090 }
1091
1092
1093 /*
1094 * tavor_eq_handler_init
1095 * Context: Only called from attach() path context
1096 */
1097 static int
tavor_eq_handler_init(tavor_state_t * state,tavor_eqhdl_t eq,uint_t evt_type_mask,int (* eq_func)(tavor_state_t * state,tavor_eqhdl_t eq,tavor_hw_eqe_t * eqe))1098 tavor_eq_handler_init(tavor_state_t *state, tavor_eqhdl_t eq,
1099 uint_t evt_type_mask, int (*eq_func)(tavor_state_t *state,
1100 tavor_eqhdl_t eq, tavor_hw_eqe_t *eqe))
1101 {
1102 int status;
1103
1104 /*
1105 * Save away the EQ handler function and the event type mask. These
1106 * will be used later during interrupt and event queue processing.
1107 */
1108 eq->eq_func = eq_func;
1109 eq->eq_evttypemask = evt_type_mask;
1110
1111 /*
1112 * Map the EQ to a specific class of event (or events) depending
1113 * on the mask value passed in. The TAVOR_EVT_NO_MASK means not
1114 * to attempt associating the EQ with any specific class of event.
1115 * This is particularly useful when initializing the events queues
1116 * used for CQ events. The mapping is done using the Tavor MAP_EQ
1117 * firmware command. Note: This command should not, in general, fail.
1118 * If it does, then something (probably HW related) has gone seriously
1119 * wrong.
1120 */
1121 if (evt_type_mask != TAVOR_EVT_NO_MASK) {
1122 status = tavor_map_eq_cmd_post(state,
1123 TAVOR_CMD_MAP_EQ_EVT_MAP, eq->eq_eqnum, evt_type_mask,
1124 TAVOR_CMD_NOSLEEP_SPIN);
1125 if (status != TAVOR_CMD_SUCCESS) {
1126 cmn_err(CE_CONT, "Tavor: MAP_EQ command failed: "
1127 "%08x\n", status);
1128 return (DDI_FAILURE);
1129 }
1130 }
1131
1132 return (DDI_SUCCESS);
1133 }
1134
1135
1136 /*
1137 * tavor_eq_handler_fini
1138 * Context: Only called from attach() and/or detach() path contexts
1139 */
1140 static int
tavor_eq_handler_fini(tavor_state_t * state,tavor_eqhdl_t eq)1141 tavor_eq_handler_fini(tavor_state_t *state, tavor_eqhdl_t eq)
1142 {
1143 int status;
1144
1145 /*
1146 * Unmap the EQ from the event class to which it had been previously
1147 * mapped. The unmapping is done using the Tavor MAP_EQ (in much
1148 * the same way that the initial mapping was done). The difference,
1149 * however, is in the TAVOR_EQ_EVT_UNMAP flag that is passed to the
1150 * MAP_EQ firmware command. The TAVOR_EVT_NO_MASK (which may have
1151 * been passed in at init time) still means that no association has
1152 * been made between the EQ and any specific class of event (and,
1153 * hence, no unmapping is necessary). Note: This command should not,
1154 * in general, fail. If it does, then something (probably HW related)
1155 * has gone seriously wrong.
1156 */
1157 if (eq->eq_evttypemask != TAVOR_EVT_NO_MASK) {
1158 status = tavor_map_eq_cmd_post(state,
1159 TAVOR_CMD_MAP_EQ_EVT_UNMAP, eq->eq_eqnum,
1160 eq->eq_evttypemask, TAVOR_CMD_NOSLEEP_SPIN);
1161 if (status != TAVOR_CMD_SUCCESS) {
1162 cmn_err(CE_CONT, "Tavor: MAP_EQ command failed: "
1163 "%08x\n", status);
1164 return (DDI_FAILURE);
1165 }
1166 }
1167
1168 return (DDI_SUCCESS);
1169 }
1170
1171
1172 /*
1173 * tavor_eqe_sync()
1174 * Context: Can be called from interrupt or base context.
1175 *
1176 * Typically, this routine does nothing unless the EQ memory is
1177 * mapped as DDI_DMA_STREAMING. However, there is a condition where
1178 * ddi_dma_sync() is necessary even if the memory was mapped in
1179 * consistent mode. The "force_sync" parameter is used here to force
1180 * the call to ddi_dma_sync() independent of how the EQ memory was
1181 * mapped.
1182 */
1183 static void
tavor_eqe_sync(tavor_eqhdl_t eq,tavor_hw_eqe_t * eqe,uint_t flag,uint_t force_sync)1184 tavor_eqe_sync(tavor_eqhdl_t eq, tavor_hw_eqe_t *eqe, uint_t flag,
1185 uint_t force_sync)
1186 {
1187 ddi_dma_handle_t dmahdl;
1188 off_t offset;
1189
1190 /* Determine if EQ needs to be synced or not */
1191 if ((eq->eq_sync == 0) && (force_sync == TAVOR_EQ_SYNC_NORMAL)) {
1192 return;
1193 }
1194
1195 /* Get the DMA handle from EQ context */
1196 dmahdl = eq->eq_mrhdl->mr_bindinfo.bi_dmahdl;
1197
1198 /* Calculate offset of next EQE */
1199 offset = (off_t)((uintptr_t)eqe - (uintptr_t)&eq->eq_buf[0]);
1200 (void) ddi_dma_sync(dmahdl, offset, sizeof (tavor_hw_eqe_t), flag);
1201 }
1202
1203
1204 /*
1205 * tavor_port_state_change_handler()
1206 * Context: Only called from interrupt context
1207 */
1208 static int
tavor_port_state_change_handler(tavor_state_t * state,tavor_eqhdl_t eq,tavor_hw_eqe_t * eqe)1209 tavor_port_state_change_handler(tavor_state_t *state, tavor_eqhdl_t eq,
1210 tavor_hw_eqe_t *eqe)
1211 {
1212 ibc_async_event_t event;
1213 ibt_async_code_t type;
1214 uint_t port, subtype;
1215 uint_t eqe_evttype;
1216 char link_msg[24];
1217
1218 eqe_evttype = TAVOR_EQE_EVTTYPE_GET(eq, eqe);
1219
1220 ASSERT(eqe_evttype == TAVOR_EVT_PORT_STATE_CHANGE ||
1221 eqe_evttype == TAVOR_EVT_EQ_OVERFLOW);
1222
1223 if (eqe_evttype == TAVOR_EVT_EQ_OVERFLOW) {
1224 tavor_eq_overflow_handler(state, eq, eqe);
1225
1226 return (DDI_FAILURE);
1227 }
1228
1229 /*
1230 * Depending on the type of Port State Change event, pass the
1231 * appropriate asynch event to the IBTF.
1232 */
1233 port = TAVOR_EQE_PORTNUM_GET(eq, eqe);
1234
1235 /* Check for valid port number in event */
1236 if ((port == 0) || (port > state->ts_cfg_profile->cp_num_ports)) {
1237 TAVOR_WARNING(state, "Unexpected port number in port state "
1238 "change event");
1239 cmn_err(CE_CONT, " Port number: %02x\n", port);
1240 return (DDI_FAILURE);
1241 }
1242
1243 subtype = TAVOR_EQE_EVTSUBTYPE_GET(eq, eqe);
1244 if (subtype == TAVOR_PORT_LINK_ACTIVE) {
1245 event.ev_port = port;
1246 type = IBT_EVENT_PORT_UP;
1247
1248 (void) snprintf(link_msg, 23, "port %d up", port);
1249 ddi_dev_report_fault(state->ts_dip, DDI_SERVICE_RESTORED,
1250 DDI_EXTERNAL_FAULT, link_msg);
1251 } else if (subtype == TAVOR_PORT_LINK_DOWN) {
1252 event.ev_port = port;
1253 type = IBT_ERROR_PORT_DOWN;
1254
1255 (void) snprintf(link_msg, 23, "port %d down", port);
1256 ddi_dev_report_fault(state->ts_dip, DDI_SERVICE_LOST,
1257 DDI_EXTERNAL_FAULT, link_msg);
1258 } else {
1259 TAVOR_WARNING(state, "Unexpected subtype in port state change "
1260 "event");
1261 cmn_err(CE_CONT, " Event type: %02x, subtype: %02x\n",
1262 TAVOR_EQE_EVTTYPE_GET(eq, eqe), subtype);
1263 return (DDI_FAILURE);
1264 }
1265
1266 /*
1267 * Deliver the event to the IBTF. Note: If "ts_ibtfpriv" is NULL,
1268 * then we have either received this event before we finished
1269 * attaching to the IBTF or we've received it while we are in the
1270 * process of detaching.
1271 */
1272 if (state->ts_ibtfpriv != NULL) {
1273 TAVOR_DO_IBTF_ASYNC_CALLB(state, type, &event);
1274 }
1275
1276 return (DDI_SUCCESS);
1277 }
1278
1279
1280 /*
1281 * tavor_comm_estbl_handler()
1282 * Context: Only called from interrupt context
1283 */
1284 static int
tavor_comm_estbl_handler(tavor_state_t * state,tavor_eqhdl_t eq,tavor_hw_eqe_t * eqe)1285 tavor_comm_estbl_handler(tavor_state_t *state, tavor_eqhdl_t eq,
1286 tavor_hw_eqe_t *eqe)
1287 {
1288 tavor_qphdl_t qp;
1289 uint_t qpnum;
1290 ibc_async_event_t event;
1291 ibt_async_code_t type;
1292 uint_t eqe_evttype;
1293
1294 eqe_evttype = TAVOR_EQE_EVTTYPE_GET(eq, eqe);
1295
1296 ASSERT(eqe_evttype == TAVOR_EVT_COMM_ESTABLISHED ||
1297 eqe_evttype == TAVOR_EVT_EQ_OVERFLOW);
1298
1299 if (eqe_evttype == TAVOR_EVT_EQ_OVERFLOW) {
1300 tavor_eq_overflow_handler(state, eq, eqe);
1301
1302 return (DDI_FAILURE);
1303 }
1304
1305 /* Get the QP handle from QP number in event descriptor */
1306 qpnum = TAVOR_EQE_QPNUM_GET(eq, eqe);
1307 qp = tavor_qphdl_from_qpnum(state, qpnum);
1308
1309 /*
1310 * If the QP handle is NULL, this is probably an indication
1311 * that the QP has been freed already. In which case, we
1312 * should not deliver this event.
1313 *
1314 * We also check that the QP number in the handle is the
1315 * same as the QP number in the event queue entry. This
1316 * extra check allows us to handle the case where a QP was
1317 * freed and then allocated again in the time it took to
1318 * handle the event queue processing. By constantly incrementing
1319 * the non-constrained portion of the QP number every time
1320 * a new QP is allocated, we mitigate (somewhat) the chance
1321 * that a stale event could be passed to the client's QP
1322 * handler.
1323 *
1324 * Lastly, we check if "ts_ibtfpriv" is NULL. If it is then it
1325 * means that we've have either received this event before we
1326 * finished attaching to the IBTF or we've received it while we
1327 * are in the process of detaching.
1328 */
1329 if ((qp != NULL) && (qp->qp_qpnum == qpnum) &&
1330 (state->ts_ibtfpriv != NULL)) {
1331 event.ev_qp_hdl = (ibtl_qp_hdl_t)qp->qp_hdlrarg;
1332 type = IBT_EVENT_COM_EST_QP;
1333
1334 TAVOR_DO_IBTF_ASYNC_CALLB(state, type, &event);
1335 }
1336
1337 return (DDI_SUCCESS);
1338 }
1339
1340
1341 /*
1342 * tavor_local_wq_cat_err_handler()
1343 * Context: Only called from interrupt context
1344 */
1345 static int
tavor_local_wq_cat_err_handler(tavor_state_t * state,tavor_eqhdl_t eq,tavor_hw_eqe_t * eqe)1346 tavor_local_wq_cat_err_handler(tavor_state_t *state, tavor_eqhdl_t eq,
1347 tavor_hw_eqe_t *eqe)
1348 {
1349 tavor_qphdl_t qp;
1350 uint_t qpnum;
1351 ibc_async_event_t event;
1352 ibt_async_code_t type;
1353 uint_t eqe_evttype;
1354
1355 eqe_evttype = TAVOR_EQE_EVTTYPE_GET(eq, eqe);
1356
1357 ASSERT(eqe_evttype == TAVOR_EVT_LOCAL_WQ_CAT_ERROR ||
1358 eqe_evttype == TAVOR_EVT_EQ_OVERFLOW);
1359
1360 if (eqe_evttype == TAVOR_EVT_EQ_OVERFLOW) {
1361 tavor_eq_overflow_handler(state, eq, eqe);
1362
1363 return (DDI_FAILURE);
1364 }
1365
1366 /* Get the QP handle from QP number in event descriptor */
1367 qpnum = TAVOR_EQE_QPNUM_GET(eq, eqe);
1368 qp = tavor_qphdl_from_qpnum(state, qpnum);
1369
1370 /*
1371 * If the QP handle is NULL, this is probably an indication
1372 * that the QP has been freed already. In which case, we
1373 * should not deliver this event.
1374 *
1375 * We also check that the QP number in the handle is the
1376 * same as the QP number in the event queue entry. This
1377 * extra check allows us to handle the case where a QP was
1378 * freed and then allocated again in the time it took to
1379 * handle the event queue processing. By constantly incrementing
1380 * the non-constrained portion of the QP number every time
1381 * a new QP is allocated, we mitigate (somewhat) the chance
1382 * that a stale event could be passed to the client's QP
1383 * handler.
1384 *
1385 * Lastly, we check if "ts_ibtfpriv" is NULL. If it is then it
1386 * means that we've have either received this event before we
1387 * finished attaching to the IBTF or we've received it while we
1388 * are in the process of detaching.
1389 */
1390 if ((qp != NULL) && (qp->qp_qpnum == qpnum) &&
1391 (state->ts_ibtfpriv != NULL)) {
1392 event.ev_qp_hdl = (ibtl_qp_hdl_t)qp->qp_hdlrarg;
1393 type = IBT_ERROR_CATASTROPHIC_QP;
1394
1395 TAVOR_DO_IBTF_ASYNC_CALLB(state, type, &event);
1396 }
1397
1398 return (DDI_SUCCESS);
1399 }
1400
1401
1402 /*
1403 * tavor_invreq_local_wq_err_handler()
1404 * Context: Only called from interrupt context
1405 */
1406 static int
tavor_invreq_local_wq_err_handler(tavor_state_t * state,tavor_eqhdl_t eq,tavor_hw_eqe_t * eqe)1407 tavor_invreq_local_wq_err_handler(tavor_state_t *state, tavor_eqhdl_t eq,
1408 tavor_hw_eqe_t *eqe)
1409 {
1410 tavor_qphdl_t qp;
1411 uint_t qpnum;
1412 ibc_async_event_t event;
1413 ibt_async_code_t type;
1414 uint_t eqe_evttype;
1415
1416 eqe_evttype = TAVOR_EQE_EVTTYPE_GET(eq, eqe);
1417
1418 ASSERT(eqe_evttype == TAVOR_EVT_INV_REQ_LOCAL_WQ_ERROR ||
1419 eqe_evttype == TAVOR_EVT_EQ_OVERFLOW);
1420
1421 if (eqe_evttype == TAVOR_EVT_EQ_OVERFLOW) {
1422 tavor_eq_overflow_handler(state, eq, eqe);
1423
1424 return (DDI_FAILURE);
1425 }
1426
1427 /* Get the QP handle from QP number in event descriptor */
1428 qpnum = TAVOR_EQE_QPNUM_GET(eq, eqe);
1429 qp = tavor_qphdl_from_qpnum(state, qpnum);
1430
1431 /*
1432 * If the QP handle is NULL, this is probably an indication
1433 * that the QP has been freed already. In which case, we
1434 * should not deliver this event.
1435 *
1436 * We also check that the QP number in the handle is the
1437 * same as the QP number in the event queue entry. This
1438 * extra check allows us to handle the case where a QP was
1439 * freed and then allocated again in the time it took to
1440 * handle the event queue processing. By constantly incrementing
1441 * the non-constrained portion of the QP number every time
1442 * a new QP is allocated, we mitigate (somewhat) the chance
1443 * that a stale event could be passed to the client's QP
1444 * handler.
1445 *
1446 * Lastly, we check if "ts_ibtfpriv" is NULL. If it is then it
1447 * means that we've have either received this event before we
1448 * finished attaching to the IBTF or we've received it while we
1449 * are in the process of detaching.
1450 */
1451 if ((qp != NULL) && (qp->qp_qpnum == qpnum) &&
1452 (state->ts_ibtfpriv != NULL)) {
1453 event.ev_qp_hdl = (ibtl_qp_hdl_t)qp->qp_hdlrarg;
1454 type = IBT_ERROR_INVALID_REQUEST_QP;
1455
1456 TAVOR_DO_IBTF_ASYNC_CALLB(state, type, &event);
1457 }
1458
1459 return (DDI_SUCCESS);
1460 }
1461
1462
1463 /*
1464 * tavor_local_acc_vio_wq_err_handler()
1465 * Context: Only called from interrupt context
1466 */
1467 static int
tavor_local_acc_vio_wq_err_handler(tavor_state_t * state,tavor_eqhdl_t eq,tavor_hw_eqe_t * eqe)1468 tavor_local_acc_vio_wq_err_handler(tavor_state_t *state, tavor_eqhdl_t eq,
1469 tavor_hw_eqe_t *eqe)
1470 {
1471 tavor_qphdl_t qp;
1472 uint_t qpnum;
1473 ibc_async_event_t event;
1474 ibt_async_code_t type;
1475 uint_t eqe_evttype;
1476
1477 eqe_evttype = TAVOR_EQE_EVTTYPE_GET(eq, eqe);
1478
1479 ASSERT(eqe_evttype == TAVOR_EVT_LOCAL_ACC_VIO_WQ_ERROR ||
1480 eqe_evttype == TAVOR_EVT_EQ_OVERFLOW);
1481
1482 if (eqe_evttype == TAVOR_EVT_EQ_OVERFLOW) {
1483 tavor_eq_overflow_handler(state, eq, eqe);
1484
1485 return (DDI_FAILURE);
1486 }
1487
1488 /* Get the QP handle from QP number in event descriptor */
1489 qpnum = TAVOR_EQE_QPNUM_GET(eq, eqe);
1490 qp = tavor_qphdl_from_qpnum(state, qpnum);
1491
1492 /*
1493 * If the QP handle is NULL, this is probably an indication
1494 * that the QP has been freed already. In which case, we
1495 * should not deliver this event.
1496 *
1497 * We also check that the QP number in the handle is the
1498 * same as the QP number in the event queue entry. This
1499 * extra check allows us to handle the case where a QP was
1500 * freed and then allocated again in the time it took to
1501 * handle the event queue processing. By constantly incrementing
1502 * the non-constrained portion of the QP number every time
1503 * a new QP is allocated, we mitigate (somewhat) the chance
1504 * that a stale event could be passed to the client's QP
1505 * handler.
1506 *
1507 * Lastly, we check if "ts_ibtfpriv" is NULL. If it is then it
1508 * means that we've have either received this event before we
1509 * finished attaching to the IBTF or we've received it while we
1510 * are in the process of detaching.
1511 */
1512 if ((qp != NULL) && (qp->qp_qpnum == qpnum) &&
1513 (state->ts_ibtfpriv != NULL)) {
1514 event.ev_qp_hdl = (ibtl_qp_hdl_t)qp->qp_hdlrarg;
1515 type = IBT_ERROR_ACCESS_VIOLATION_QP;
1516
1517 TAVOR_DO_IBTF_ASYNC_CALLB(state, type, &event);
1518 }
1519
1520 return (DDI_SUCCESS);
1521 }
1522
1523
1524 /*
1525 * tavor_sendq_drained_handler()
1526 * Context: Only called from interrupt context
1527 */
1528 static int
tavor_sendq_drained_handler(tavor_state_t * state,tavor_eqhdl_t eq,tavor_hw_eqe_t * eqe)1529 tavor_sendq_drained_handler(tavor_state_t *state, tavor_eqhdl_t eq,
1530 tavor_hw_eqe_t *eqe)
1531 {
1532 tavor_qphdl_t qp;
1533 uint_t qpnum;
1534 ibc_async_event_t event;
1535 uint_t forward_sqd_event;
1536 ibt_async_code_t type;
1537 uint_t eqe_evttype;
1538
1539 eqe_evttype = TAVOR_EQE_EVTTYPE_GET(eq, eqe);
1540
1541 ASSERT(eqe_evttype == TAVOR_EVT_SEND_QUEUE_DRAINED ||
1542 eqe_evttype == TAVOR_EVT_EQ_OVERFLOW);
1543
1544 if (eqe_evttype == TAVOR_EVT_EQ_OVERFLOW) {
1545 tavor_eq_overflow_handler(state, eq, eqe);
1546
1547 return (DDI_FAILURE);
1548 }
1549
1550 /* Get the QP handle from QP number in event descriptor */
1551 qpnum = TAVOR_EQE_QPNUM_GET(eq, eqe);
1552 qp = tavor_qphdl_from_qpnum(state, qpnum);
1553
1554 /*
1555 * If the QP handle is NULL, this is probably an indication
1556 * that the QP has been freed already. In which case, we
1557 * should not deliver this event.
1558 *
1559 * We also check that the QP number in the handle is the
1560 * same as the QP number in the event queue entry. This
1561 * extra check allows us to handle the case where a QP was
1562 * freed and then allocated again in the time it took to
1563 * handle the event queue processing. By constantly incrementing
1564 * the non-constrained portion of the QP number every time
1565 * a new QP is allocated, we mitigate (somewhat) the chance
1566 * that a stale event could be passed to the client's QP
1567 * handler.
1568 *
1569 * And then we check if "ts_ibtfpriv" is NULL. If it is then it
1570 * means that we've have either received this event before we
1571 * finished attaching to the IBTF or we've received it while we
1572 * are in the process of detaching.
1573 */
1574 if ((qp != NULL) && (qp->qp_qpnum == qpnum) &&
1575 (state->ts_ibtfpriv != NULL)) {
1576 event.ev_qp_hdl = (ibtl_qp_hdl_t)qp->qp_hdlrarg;
1577 type = IBT_EVENT_SQD;
1578
1579 /*
1580 * Grab the QP lock and update the QP state to reflect that
1581 * the Send Queue Drained event has arrived. Also determine
1582 * whether the event is intended to be forwarded on to the
1583 * consumer or not. This information is used below in
1584 * determining whether or not to call the IBTF.
1585 */
1586 mutex_enter(&qp->qp_lock);
1587 forward_sqd_event = qp->qp_forward_sqd_event;
1588 qp->qp_forward_sqd_event = 0;
1589 qp->qp_sqd_still_draining = 0;
1590 mutex_exit(&qp->qp_lock);
1591
1592 if (forward_sqd_event != 0) {
1593 TAVOR_DO_IBTF_ASYNC_CALLB(state, type, &event);
1594 }
1595 }
1596
1597 return (DDI_SUCCESS);
1598 }
1599
1600
1601 /*
1602 * tavor_path_mig_handler()
1603 * Context: Only called from interrupt context
1604 */
1605 static int
tavor_path_mig_handler(tavor_state_t * state,tavor_eqhdl_t eq,tavor_hw_eqe_t * eqe)1606 tavor_path_mig_handler(tavor_state_t *state, tavor_eqhdl_t eq,
1607 tavor_hw_eqe_t *eqe)
1608 {
1609 tavor_qphdl_t qp;
1610 uint_t qpnum;
1611 ibc_async_event_t event;
1612 ibt_async_code_t type;
1613 uint_t eqe_evttype;
1614
1615 eqe_evttype = TAVOR_EQE_EVTTYPE_GET(eq, eqe);
1616
1617 ASSERT(eqe_evttype == TAVOR_EVT_PATH_MIGRATED ||
1618 eqe_evttype == TAVOR_EVT_EQ_OVERFLOW);
1619
1620 if (eqe_evttype == TAVOR_EVT_EQ_OVERFLOW) {
1621 tavor_eq_overflow_handler(state, eq, eqe);
1622
1623 return (DDI_FAILURE);
1624 }
1625
1626 /* Get the QP handle from QP number in event descriptor */
1627 qpnum = TAVOR_EQE_QPNUM_GET(eq, eqe);
1628 qp = tavor_qphdl_from_qpnum(state, qpnum);
1629
1630 /*
1631 * If the QP handle is NULL, this is probably an indication
1632 * that the QP has been freed already. In which case, we
1633 * should not deliver this event.
1634 *
1635 * We also check that the QP number in the handle is the
1636 * same as the QP number in the event queue entry. This
1637 * extra check allows us to handle the case where a QP was
1638 * freed and then allocated again in the time it took to
1639 * handle the event queue processing. By constantly incrementing
1640 * the non-constrained portion of the QP number every time
1641 * a new QP is allocated, we mitigate (somewhat) the chance
1642 * that a stale event could be passed to the client's QP
1643 * handler.
1644 *
1645 * Lastly, we check if "ts_ibtfpriv" is NULL. If it is then it
1646 * means that we've have either received this event before we
1647 * finished attaching to the IBTF or we've received it while we
1648 * are in the process of detaching.
1649 */
1650 if ((qp != NULL) && (qp->qp_qpnum == qpnum) &&
1651 (state->ts_ibtfpriv != NULL)) {
1652 event.ev_qp_hdl = (ibtl_qp_hdl_t)qp->qp_hdlrarg;
1653 type = IBT_EVENT_PATH_MIGRATED_QP;
1654
1655 TAVOR_DO_IBTF_ASYNC_CALLB(state, type, &event);
1656 }
1657
1658 return (DDI_SUCCESS);
1659 }
1660
1661
1662 /*
1663 * tavor_path_mig_err_handler()
1664 * Context: Only called from interrupt context
1665 */
1666 static int
tavor_path_mig_err_handler(tavor_state_t * state,tavor_eqhdl_t eq,tavor_hw_eqe_t * eqe)1667 tavor_path_mig_err_handler(tavor_state_t *state, tavor_eqhdl_t eq,
1668 tavor_hw_eqe_t *eqe)
1669 {
1670 tavor_qphdl_t qp;
1671 uint_t qpnum;
1672 ibc_async_event_t event;
1673 ibt_async_code_t type;
1674 uint_t eqe_evttype;
1675
1676 eqe_evttype = TAVOR_EQE_EVTTYPE_GET(eq, eqe);
1677
1678 ASSERT(eqe_evttype == TAVOR_EVT_PATH_MIGRATE_FAILED ||
1679 eqe_evttype == TAVOR_EVT_EQ_OVERFLOW);
1680
1681 if (eqe_evttype == TAVOR_EVT_EQ_OVERFLOW) {
1682 tavor_eq_overflow_handler(state, eq, eqe);
1683
1684 return (DDI_FAILURE);
1685 }
1686
1687 /* Get the QP handle from QP number in event descriptor */
1688 qpnum = TAVOR_EQE_QPNUM_GET(eq, eqe);
1689 qp = tavor_qphdl_from_qpnum(state, qpnum);
1690
1691 /*
1692 * If the QP handle is NULL, this is probably an indication
1693 * that the QP has been freed already. In which case, we
1694 * should not deliver this event.
1695 *
1696 * We also check that the QP number in the handle is the
1697 * same as the QP number in the event queue entry. This
1698 * extra check allows us to handle the case where a QP was
1699 * freed and then allocated again in the time it took to
1700 * handle the event queue processing. By constantly incrementing
1701 * the non-constrained portion of the QP number every time
1702 * a new QP is allocated, we mitigate (somewhat) the chance
1703 * that a stale event could be passed to the client's QP
1704 * handler.
1705 *
1706 * Lastly, we check if "ts_ibtfpriv" is NULL. If it is then it
1707 * means that we've have either received this event before we
1708 * finished attaching to the IBTF or we've received it while we
1709 * are in the process of detaching.
1710 */
1711 if ((qp != NULL) && (qp->qp_qpnum == qpnum) &&
1712 (state->ts_ibtfpriv != NULL)) {
1713 event.ev_qp_hdl = (ibtl_qp_hdl_t)qp->qp_hdlrarg;
1714 type = IBT_ERROR_PATH_MIGRATE_REQ_QP;
1715
1716 TAVOR_DO_IBTF_ASYNC_CALLB(state, type, &event);
1717 }
1718
1719 return (DDI_SUCCESS);
1720 }
1721
1722
1723 /*
1724 * tavor_srq_catastrophic_handler()
1725 * Context: Only called from interrupt context
1726 */
1727 static int
tavor_srq_catastrophic_handler(tavor_state_t * state,tavor_eqhdl_t eq,tavor_hw_eqe_t * eqe)1728 tavor_srq_catastrophic_handler(tavor_state_t *state, tavor_eqhdl_t eq,
1729 tavor_hw_eqe_t *eqe)
1730 {
1731 tavor_qphdl_t qp;
1732 uint_t qpnum;
1733 ibc_async_event_t event;
1734 ibt_async_code_t type;
1735 uint_t eqe_evttype;
1736
1737 eqe_evttype = TAVOR_EQE_EVTTYPE_GET(eq, eqe);
1738
1739 ASSERT(eqe_evttype == TAVOR_EVT_SRQ_CATASTROPHIC_ERROR ||
1740 eqe_evttype == TAVOR_EVT_EQ_OVERFLOW);
1741
1742 if (eqe_evttype == TAVOR_EVT_EQ_OVERFLOW) {
1743 tavor_eq_overflow_handler(state, eq, eqe);
1744
1745 return (DDI_FAILURE);
1746 }
1747
1748 /* Get the QP handle from QP number in event descriptor */
1749 qpnum = TAVOR_EQE_QPNUM_GET(eq, eqe);
1750 qp = tavor_qphdl_from_qpnum(state, qpnum);
1751
1752 /*
1753 * If the QP handle is NULL, this is probably an indication
1754 * that the QP has been freed already. In which case, we
1755 * should not deliver this event.
1756 *
1757 * We also check that the QP number in the handle is the
1758 * same as the QP number in the event queue entry. This
1759 * extra check allows us to handle the case where a QP was
1760 * freed and then allocated again in the time it took to
1761 * handle the event queue processing. By constantly incrementing
1762 * the non-constrained portion of the QP number every time
1763 * a new QP is allocated, we mitigate (somewhat) the chance
1764 * that a stale event could be passed to the client's QP
1765 * handler.
1766 *
1767 * Lastly, we check if "ts_ibtfpriv" is NULL. If it is then it
1768 * means that we've have either received this event before we
1769 * finished attaching to the IBTF or we've received it while we
1770 * are in the process of detaching.
1771 */
1772 if ((qp != NULL) && (qp->qp_qpnum == qpnum) &&
1773 (state->ts_ibtfpriv != NULL)) {
1774 event.ev_srq_hdl = (ibt_srq_hdl_t)qp->qp_srqhdl->srq_hdlrarg;
1775 type = IBT_ERROR_CATASTROPHIC_SRQ;
1776
1777 mutex_enter(&qp->qp_srqhdl->srq_lock);
1778 qp->qp_srqhdl->srq_state = TAVOR_SRQ_STATE_ERROR;
1779 mutex_exit(&qp->qp_srqhdl->srq_lock);
1780
1781 TAVOR_DO_IBTF_ASYNC_CALLB(state, type, &event);
1782 }
1783
1784 return (DDI_SUCCESS);
1785 }
1786
1787
1788 /*
1789 * tavor_srq_last_wqe_reached_handler()
1790 * Context: Only called from interrupt context
1791 */
1792 static int
tavor_srq_last_wqe_reached_handler(tavor_state_t * state,tavor_eqhdl_t eq,tavor_hw_eqe_t * eqe)1793 tavor_srq_last_wqe_reached_handler(tavor_state_t *state, tavor_eqhdl_t eq,
1794 tavor_hw_eqe_t *eqe)
1795 {
1796 tavor_qphdl_t qp;
1797 uint_t qpnum;
1798 ibc_async_event_t event;
1799 ibt_async_code_t type;
1800 uint_t eqe_evttype;
1801
1802 eqe_evttype = TAVOR_EQE_EVTTYPE_GET(eq, eqe);
1803
1804 ASSERT(eqe_evttype == TAVOR_EVT_SRQ_LAST_WQE_REACHED ||
1805 eqe_evttype == TAVOR_EVT_EQ_OVERFLOW);
1806
1807 if (eqe_evttype == TAVOR_EVT_EQ_OVERFLOW) {
1808 tavor_eq_overflow_handler(state, eq, eqe);
1809
1810 return (DDI_FAILURE);
1811 }
1812
1813 /* Get the QP handle from QP number in event descriptor */
1814 qpnum = TAVOR_EQE_QPNUM_GET(eq, eqe);
1815 qp = tavor_qphdl_from_qpnum(state, qpnum);
1816
1817 /*
1818 * If the QP handle is NULL, this is probably an indication
1819 * that the QP has been freed already. In which case, we
1820 * should not deliver this event.
1821 *
1822 * We also check that the QP number in the handle is the
1823 * same as the QP number in the event queue entry. This
1824 * extra check allows us to handle the case where a QP was
1825 * freed and then allocated again in the time it took to
1826 * handle the event queue processing. By constantly incrementing
1827 * the non-constrained portion of the QP number every time
1828 * a new QP is allocated, we mitigate (somewhat) the chance
1829 * that a stale event could be passed to the client's QP
1830 * handler.
1831 *
1832 * Lastly, we check if "ts_ibtfpriv" is NULL. If it is then it
1833 * means that we've have either received this event before we
1834 * finished attaching to the IBTF or we've received it while we
1835 * are in the process of detaching.
1836 */
1837 if ((qp != NULL) && (qp->qp_qpnum == qpnum) &&
1838 (state->ts_ibtfpriv != NULL)) {
1839 event.ev_qp_hdl = (ibtl_qp_hdl_t)qp->qp_hdlrarg;
1840 type = IBT_EVENT_EMPTY_CHAN;
1841
1842 TAVOR_DO_IBTF_ASYNC_CALLB(state, type, &event);
1843 }
1844
1845 return (DDI_SUCCESS);
1846 }
1847
1848
1849 /*
1850 * tavor_ecc_detection_handler()
1851 * Context: Only called from interrupt context
1852 */
1853 static int
tavor_ecc_detection_handler(tavor_state_t * state,tavor_eqhdl_t eq,tavor_hw_eqe_t * eqe)1854 tavor_ecc_detection_handler(tavor_state_t *state, tavor_eqhdl_t eq,
1855 tavor_hw_eqe_t *eqe)
1856 {
1857 uint_t eqe_evttype;
1858 uint_t data;
1859 int i;
1860
1861 eqe_evttype = TAVOR_EQE_EVTTYPE_GET(eq, eqe);
1862
1863 ASSERT(eqe_evttype == TAVOR_EVT_ECC_DETECTION ||
1864 eqe_evttype == TAVOR_EVT_EQ_OVERFLOW);
1865
1866 if (eqe_evttype == TAVOR_EVT_EQ_OVERFLOW) {
1867 tavor_eq_overflow_handler(state, eq, eqe);
1868
1869 return (DDI_FAILURE);
1870 }
1871
1872 /*
1873 * The "ECC Detection Event" indicates that a correctable single-bit
1874 * has occurred with the attached DDR. The EQE provides some
1875 * additional information about the errored EQ. So we print a warning
1876 * message here along with that additional information.
1877 */
1878 TAVOR_WARNING(state, "ECC Correctable Error Event Detected");
1879 for (i = 0; i < sizeof (tavor_hw_eqe_t) >> 2; i++) {
1880 data = ((uint_t *)eqe)[i];
1881 cmn_err(CE_CONT, "! EQE[%02x]: %08x\n", i, data);
1882 }
1883
1884 return (DDI_SUCCESS);
1885 }
1886
1887
1888 /*
1889 * tavor_eq_overflow_handler()
1890 * Context: Only called from interrupt context
1891 */
1892 void
tavor_eq_overflow_handler(tavor_state_t * state,tavor_eqhdl_t eq,tavor_hw_eqe_t * eqe)1893 tavor_eq_overflow_handler(tavor_state_t *state, tavor_eqhdl_t eq,
1894 tavor_hw_eqe_t *eqe)
1895 {
1896 uint_t error_type, data;
1897
1898 ASSERT(TAVOR_EQE_EVTTYPE_GET(eq, eqe) == TAVOR_EVT_EQ_OVERFLOW);
1899
1900 /*
1901 * The "Event Queue Overflow Event" indicates that something has
1902 * probably gone seriously wrong with some hardware (or, perhaps,
1903 * with the software... though it's unlikely in this case). The EQE
1904 * provides some additional information about the errored EQ. So we
1905 * print a warning message here along with that additional information.
1906 */
1907 error_type = TAVOR_EQE_OPERRTYPE_GET(eq, eqe);
1908 data = TAVOR_EQE_OPERRDATA_GET(eq, eqe);
1909
1910 TAVOR_WARNING(state, "Event Queue overflow");
1911 cmn_err(CE_CONT, " Error type: %02x, data: %08x\n", error_type, data);
1912 }
1913
1914
1915 /*
1916 * tavor_no_eqhandler
1917 * Context: Only called from interrupt context
1918 */
1919 /* ARGSUSED */
1920 static int
tavor_no_eqhandler(tavor_state_t * state,tavor_eqhdl_t eq,tavor_hw_eqe_t * eqe)1921 tavor_no_eqhandler(tavor_state_t *state, tavor_eqhdl_t eq,
1922 tavor_hw_eqe_t *eqe)
1923 {
1924 uint_t data;
1925 int i;
1926
1927 /*
1928 * This "unexpected event" handler (or "catch-all" handler) will
1929 * receive all events for which no other handler has been registered.
1930 * If we end up here, then something has probably gone seriously wrong
1931 * with the Tavor hardware (or, perhaps, with the software... though
1932 * it's unlikely in this case). The EQE provides all the information
1933 * about the event. So we print a warning message here along with
1934 * the contents of the EQE.
1935 */
1936 TAVOR_WARNING(state, "Unexpected Event handler");
1937 cmn_err(CE_CONT, " Event type: %02x, subtype: %02x\n",
1938 TAVOR_EQE_EVTTYPE_GET(eq, eqe), TAVOR_EQE_EVTSUBTYPE_GET(eq, eqe));
1939 for (i = 0; i < sizeof (tavor_hw_eqe_t) >> 2; i++) {
1940 data = ((uint_t *)eqe)[i];
1941 cmn_err(CE_CONT, " EQE[%02x]: %08x\n", i, data);
1942 }
1943
1944 return (DDI_SUCCESS);
1945 }
1946