1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 22 /* 23 * Copyright 2005 Sun Microsystems, Inc. All rights reserved. 24 * Use is subject to license terms. 25 */ 26 27 /* 28 * tavor_event.c 29 * Tavor Interrupt and Event Processing Routines 30 * 31 * Implements all the routines necessary for allocating, freeing, and 32 * handling all of the various event types that the Tavor hardware can 33 * generate. 34 * These routines include the main Tavor interrupt service routine 35 * (tavor_isr()) as well as all the code necessary to setup and handle 36 * events from each of the many event queues used by the Tavor device. 37 */ 38 39 #include <sys/types.h> 40 #include <sys/conf.h> 41 #include <sys/ddi.h> 42 #include <sys/sunddi.h> 43 #include <sys/modctl.h> 44 45 #include <sys/ib/adapters/tavor/tavor.h> 46 47 static void tavor_eq_poll(tavor_state_t *state, tavor_eqhdl_t eq); 48 static void tavor_eq_catastrophic(tavor_state_t *state); 49 static int tavor_eq_alloc(tavor_state_t *state, uint32_t log_eq_size, 50 uint_t intr, tavor_eqhdl_t *eqhdl); 51 static int tavor_eq_free(tavor_state_t *state, tavor_eqhdl_t *eqhdl); 52 static int tavor_eq_handler_init(tavor_state_t *state, tavor_eqhdl_t eq, 53 uint_t evt_type_mask, int (*eqfunc)(tavor_state_t *state, 54 tavor_eqhdl_t eq, tavor_hw_eqe_t *eqe)); 55 static int tavor_eq_handler_fini(tavor_state_t *state, tavor_eqhdl_t eq); 56 static void tavor_eqe_sync(tavor_eqhdl_t eq, tavor_hw_eqe_t *eqe, uint_t flag, 57 uint_t force_sync); 58 static int tavor_port_state_change_handler(tavor_state_t *state, 59 tavor_eqhdl_t eq, tavor_hw_eqe_t *eqe); 60 static int tavor_comm_estbl_handler(tavor_state_t *state, 61 tavor_eqhdl_t eq, tavor_hw_eqe_t *eqe); 62 static int tavor_local_wq_cat_err_handler(tavor_state_t *state, 63 tavor_eqhdl_t eq, tavor_hw_eqe_t *eqe); 64 static int tavor_invreq_local_wq_err_handler(tavor_state_t *state, 65 tavor_eqhdl_t eq, tavor_hw_eqe_t *eqe); 66 static int tavor_local_acc_vio_wq_err_handler(tavor_state_t *state, 67 tavor_eqhdl_t eq, tavor_hw_eqe_t *eqe); 68 static int tavor_sendq_drained_handler(tavor_state_t *state, 69 tavor_eqhdl_t eq, tavor_hw_eqe_t *eqe); 70 static int tavor_path_mig_handler(tavor_state_t *state, 71 tavor_eqhdl_t eq, tavor_hw_eqe_t *eqe); 72 static int tavor_path_mig_err_handler(tavor_state_t *state, 73 tavor_eqhdl_t eq, tavor_hw_eqe_t *eqe); 74 static int tavor_srq_catastrophic_handler(tavor_state_t *state, 75 tavor_eqhdl_t eq, tavor_hw_eqe_t *eqe); 76 static int tavor_srq_last_wqe_reached_handler(tavor_state_t *state, 77 tavor_eqhdl_t eq, tavor_hw_eqe_t *eqe); 78 static int tavor_ecc_detection_handler(tavor_state_t *state, 79 tavor_eqhdl_t eq, tavor_hw_eqe_t *eqe); 80 static int tavor_no_eqhandler(tavor_state_t *state, tavor_eqhdl_t eq, 81 tavor_hw_eqe_t *eqe); 82 83 84 /* 85 * tavor_eq_init_all 86 * Context: Only called from attach() path context 87 */ 88 int 89 tavor_eq_init_all(tavor_state_t *state) 90 { 91 uint_t log_eq_size, intr_num; 92 uint_t num_eq, num_eq_init, num_eq_unmap; 93 int status, i; 94 95 /* 96 * For now, all Event Queues default to the same size (pulled from 97 * the current configuration profile) and are all assigned to the 98 * same interrupt or MSI. In the future we may support assigning 99 * EQs to specific interrupts or MSIs XXX 100 */ 101 log_eq_size = state->ts_cfg_profile->cp_log_default_eq_sz; 102 103 /* 104 * If MSI is to be used, then set intr_num to the MSI number 105 * (currently zero because we're using only one) or'd with the 106 * MSI enable flag. Otherwise, for regular (i.e. 'legacy') interrupt, 107 * use the 'inta_pin' value returned by QUERY_ADAPTER. 108 */ 109 if (state->ts_intr_type_chosen == DDI_INTR_TYPE_MSI) { 110 intr_num = TAVOR_EQ_MSI_ENABLE_FLAG | 0; 111 } else { 112 intr_num = state->ts_adapter.inta_pin; 113 } 114 115 /* 116 * Total number of supported EQs is hardcoded. Tavor hardware 117 * supports up to 64 EQs. We are currently using only 45 of them 118 * We will set aside the first 32 for use with Completion Queues (CQ) 119 * and reserve a few of the other 32 for each specific class of event 120 * (see below for more details). 121 */ 122 num_eq = TAVOR_NUM_EQ_USED; 123 124 /* 125 * The "num_eq_unmap" variable is used in any possible failure 126 * cleanup (below) to indicate which events queues might require 127 * possible event class unmapping. 128 */ 129 num_eq_unmap = 0; 130 131 /* 132 * Allocate and initialize all the Event Queues. If any of these 133 * EQ allocations fail then jump to the end, cleanup what had been 134 * successfully initialized, and return an error. 135 */ 136 for (i = 0; i < num_eq; i++) { 137 status = tavor_eq_alloc(state, log_eq_size, intr_num, 138 &state->ts_eqhdl[i]); 139 if (status != DDI_SUCCESS) { 140 num_eq_init = i; 141 goto all_eq_init_fail; 142 } 143 } 144 num_eq_init = num_eq; 145 146 /* 147 * Setup EQ0-EQ31 for use with Completion Queues. Note: We can 148 * cast the return value to void here because, when we use the 149 * TAVOR_EVT_NO_MASK flag, it is not possible for 150 * tavor_eq_handler_init() to return an error. 151 */ 152 for (i = 0; i < 32; i++) { 153 (void) tavor_eq_handler_init(state, state->ts_eqhdl[i], 154 TAVOR_EVT_NO_MASK, tavor_cq_handler); 155 } 156 num_eq_unmap = 32; 157 158 /* 159 * Setup EQ32 for handling Completion Queue Error Events. 160 * 161 * These events include things like CQ overflow or CQ access 162 * violation errors. If this setup fails for any reason (which, in 163 * general, it really never should), then jump to the end, cleanup 164 * everything that has been successfully initialized, and return an 165 * error. 166 */ 167 status = tavor_eq_handler_init(state, state->ts_eqhdl[32], 168 TAVOR_EVT_MSK_CQ_ERRORS, tavor_cq_err_handler); 169 if (status != DDI_SUCCESS) { 170 goto all_eq_init_fail; 171 } 172 num_eq_unmap = 33; 173 174 /* 175 * Setup EQ33 for handling Port State Change Events 176 * 177 * These events include things like Port Up and Port Down events. 178 * If this setup fails for any reason (which, in general, it really 179 * never should), then undo all previous EQ mapping, jump to the end, 180 * cleanup everything that has been successfully initialized, and 181 * return an error. 182 */ 183 status = tavor_eq_handler_init(state, state->ts_eqhdl[33], 184 TAVOR_EVT_MSK_PORT_STATE_CHANGE, tavor_port_state_change_handler); 185 if (status != DDI_SUCCESS) { 186 goto all_eq_init_fail; 187 } 188 num_eq_unmap = 34; 189 190 /* 191 * Setup EQ34 for handling Communication Established Events 192 * 193 * These events correspond to the IB affiliated asynchronous events 194 * that are used for connection management. If this setup fails for 195 * any reason (which, in general, it really never should), then undo 196 * all previous EQ mapping, jump to the end, cleanup everything that 197 * has been successfully initialized, and return an error. 198 */ 199 status = tavor_eq_handler_init(state, state->ts_eqhdl[34], 200 TAVOR_EVT_MSK_COMM_ESTABLISHED, tavor_comm_estbl_handler); 201 if (status != DDI_SUCCESS) { 202 goto all_eq_init_fail; 203 } 204 num_eq_unmap = 35; 205 206 /* 207 * Setup EQ35 for handling Command Completion Events 208 * 209 * These events correspond to the Tavor generated events that are used 210 * to indicate Tavor firmware command completion. These events are 211 * only generated when Tavor firmware commands are posted using the 212 * asynchronous completion mechanism. If this setup fails for any 213 * reason (which, in general, it really never should), then undo all 214 * previous EQ mapping, jump to the end, cleanup everything that has 215 * been successfully initialized, and return an error. 216 */ 217 status = tavor_eq_handler_init(state, state->ts_eqhdl[35], 218 TAVOR_EVT_MSK_COMMAND_INTF_COMP, tavor_cmd_complete_handler); 219 if (status != DDI_SUCCESS) { 220 goto all_eq_init_fail; 221 } 222 num_eq_unmap = 36; 223 224 /* 225 * Setup EQ36 for handling Local WQ Catastrophic Error Events 226 * 227 * These events correspond to the similarly-named IB affiliated 228 * asynchronous error type. If this setup fails for any reason 229 * (which, in general, it really never should), then undo all previous 230 * EQ mapping, jump to the end, cleanup everything that has been 231 * successfully initialized, and return an error. 232 */ 233 status = tavor_eq_handler_init(state, state->ts_eqhdl[36], 234 TAVOR_EVT_MSK_LOCAL_WQ_CAT_ERROR, tavor_local_wq_cat_err_handler); 235 if (status != DDI_SUCCESS) { 236 goto all_eq_init_fail; 237 } 238 num_eq_unmap = 37; 239 240 /* 241 * Setup EQ37 for handling Invalid Req Local WQ Error Events 242 * 243 * These events also correspond to the similarly-named IB affiliated 244 * asynchronous error type. If this setup fails for any reason 245 * (which, in general, it really never should), then undo all previous 246 * EQ mapping, jump to the end, cleanup everything that has been 247 * successfully initialized, and return an error. 248 */ 249 status = tavor_eq_handler_init(state, state->ts_eqhdl[37], 250 TAVOR_EVT_MSK_INV_REQ_LOCAL_WQ_ERROR, 251 tavor_invreq_local_wq_err_handler); 252 if (status != DDI_SUCCESS) { 253 goto all_eq_init_fail; 254 } 255 num_eq_unmap = 38; 256 257 /* 258 * Setup EQ38 for handling Local Access Violation WQ Error Events 259 * 260 * These events also correspond to the similarly-named IB affiliated 261 * asynchronous error type. If this setup fails for any reason 262 * (which, in general, it really never should), then undo all previous 263 * EQ mapping, jump to the end, cleanup everything that has been 264 * successfully initialized, and return an error. 265 */ 266 status = tavor_eq_handler_init(state, state->ts_eqhdl[38], 267 TAVOR_EVT_MSK_LOCAL_ACC_VIO_WQ_ERROR, 268 tavor_local_acc_vio_wq_err_handler); 269 if (status != DDI_SUCCESS) { 270 goto all_eq_init_fail; 271 } 272 num_eq_unmap = 39; 273 274 /* 275 * Setup EQ39 for handling Send Queue Drained Events 276 * 277 * These events correspond to the IB affiliated asynchronous events 278 * that are used to indicate completion of a Send Queue Drained QP 279 * state transition. If this setup fails for any reason (which, in 280 * general, it really never should), then undo all previous EQ 281 * mapping, jump to the end, cleanup everything that has been 282 * successfully initialized, and return an error. 283 */ 284 status = tavor_eq_handler_init(state, state->ts_eqhdl[39], 285 TAVOR_EVT_MSK_SEND_QUEUE_DRAINED, tavor_sendq_drained_handler); 286 if (status != DDI_SUCCESS) { 287 goto all_eq_init_fail; 288 } 289 num_eq_unmap = 40; 290 291 /* 292 * Setup EQ40 for handling Path Migration Succeeded Events 293 * 294 * These events correspond to the IB affiliated asynchronous events 295 * that are used to indicate successful completion of a path 296 * migration. If this setup fails for any reason (which, in general, 297 * it really never should), then undo all previous EQ mapping, jump 298 * to the end, cleanup everything that has been successfully 299 * initialized, and return an error. 300 */ 301 status = tavor_eq_handler_init(state, state->ts_eqhdl[40], 302 TAVOR_EVT_MSK_PATH_MIGRATED, tavor_path_mig_handler); 303 if (status != DDI_SUCCESS) { 304 goto all_eq_init_fail; 305 } 306 num_eq_unmap = 41; 307 308 /* 309 * Setup EQ41 for handling Path Migration Failed Events 310 * 311 * These events correspond to the IB affiliated asynchronous events 312 * that are used to indicate that path migration was not successful. 313 * If this setup fails for any reason (which, in general, it really 314 * never should), then undo all previous EQ mapping, jump to the end, 315 * cleanup everything that has been successfully initialized, and 316 * return an error. 317 */ 318 status = tavor_eq_handler_init(state, state->ts_eqhdl[41], 319 TAVOR_EVT_MSK_PATH_MIGRATE_FAILED, tavor_path_mig_err_handler); 320 if (status != DDI_SUCCESS) { 321 goto all_eq_init_fail; 322 } 323 num_eq_unmap = 42; 324 325 /* 326 * Setup EQ42 for handling Local Catastrophic Error Events 327 * 328 * These events correspond to the similarly-named IB unaffiliated 329 * asynchronous error type. If this setup fails for any reason 330 * (which, in general, it really never should), then undo all previous 331 * EQ mapping, jump to the end, cleanup everything that has been 332 * successfully initialized, and return an error. 333 * 334 * This error is unique, in that an EQE is not generated if this event 335 * occurs. Instead, an interrupt is called and we must poll the 336 * Catastrophic Error buffer in CR-Space. This mapping is setup simply 337 * to enable this error reporting. We pass in a NULL handler since it 338 * will never be called. 339 */ 340 status = tavor_eq_handler_init(state, state->ts_eqhdl[42], 341 TAVOR_EVT_MSK_LOCAL_CAT_ERROR, NULL); 342 if (status != DDI_SUCCESS) { 343 goto all_eq_init_fail; 344 } 345 num_eq_unmap = 43; 346 347 /* 348 * Setup EQ43 for handling SRQ Catastrophic Error Events 349 * 350 * These events correspond to the similarly-named IB affiliated 351 * asynchronous error type. If this setup fails for any reason 352 * (which, in general, it really never should), then undo all previous 353 * EQ mapping, jump to the end, cleanup everything that has been 354 * successfully initialized, and return an error. 355 */ 356 status = tavor_eq_handler_init(state, state->ts_eqhdl[43], 357 TAVOR_EVT_MSK_SRQ_CATASTROPHIC_ERROR, 358 tavor_srq_catastrophic_handler); 359 if (status != DDI_SUCCESS) { 360 goto all_eq_init_fail; 361 } 362 num_eq_unmap = 44; 363 364 /* 365 * Setup EQ44 for handling SRQ Last WQE Reached Events 366 * 367 * These events correspond to the similarly-named IB affiliated 368 * asynchronous event type. If this setup fails for any reason 369 * (which, in general, it really never should), then undo all previous 370 * EQ mapping, jump to the end, cleanup everything that has been 371 * successfully initialized, and return an error. 372 */ 373 status = tavor_eq_handler_init(state, state->ts_eqhdl[44], 374 TAVOR_EVT_MSK_SRQ_LAST_WQE_REACHED, 375 tavor_srq_last_wqe_reached_handler); 376 if (status != DDI_SUCCESS) { 377 goto all_eq_init_fail; 378 } 379 num_eq_unmap = 45; 380 381 /* 382 * Setup EQ45 for handling ECC error detection events 383 * 384 * These events correspond to the similarly-named IB affiliated 385 * asynchronous event type. If this setup fails for any reason 386 * (which, in general, it really never should), then undo all previous 387 * EQ mapping, jump to the end, cleanup everything that has been 388 * successfully initialized, and return an error. 389 */ 390 status = tavor_eq_handler_init(state, state->ts_eqhdl[45], 391 TAVOR_EVT_MSK_ECC_DETECTION, 392 tavor_ecc_detection_handler); 393 if (status != DDI_SUCCESS) { 394 goto all_eq_init_fail; 395 } 396 num_eq_unmap = 46; 397 398 /* 399 * Setup EQ46 to catch all other types of events. Specifically, we 400 * do not catch the "Local EEC Catastrophic Error Event" because we 401 * should have no EEC (the Tavor driver does not support RD). We also 402 * choose not to handle any of the address translation page fault 403 * event types. Since we are not doing any page fault handling (and 404 * since the Tavor firmware does not currently support any such 405 * handling), we allow these events to go to the catch-all handler. 406 */ 407 status = tavor_eq_handler_init(state, state->ts_eqhdl[46], 408 TAVOR_EVT_CATCHALL_MASK, tavor_no_eqhandler); 409 if (status != DDI_SUCCESS) { 410 goto all_eq_init_fail; 411 } 412 413 return (DDI_SUCCESS); 414 415 all_eq_init_fail: 416 /* Unmap any of the partially mapped EQs from above */ 417 for (i = 0; i < num_eq_unmap; i++) { 418 (void) tavor_eq_handler_fini(state, state->ts_eqhdl[i]); 419 } 420 421 /* Free up any of the partially allocated EQs from above */ 422 for (i = 0; i < num_eq_init; i++) { 423 (void) tavor_eq_free(state, &state->ts_eqhdl[i]); 424 } 425 return (status); 426 } 427 428 429 /* 430 * tavor_eq_fini_all 431 * Context: Only called from attach() and/or detach() path contexts 432 */ 433 int 434 tavor_eq_fini_all(tavor_state_t *state) 435 { 436 uint_t num_eq; 437 int status, i; 438 439 /* 440 * Grab the total number of supported EQs again. This is the same 441 * hardcoded value that was used above (during the event queue 442 * initialization.) 443 */ 444 num_eq = TAVOR_NUM_EQ_USED; 445 446 /* 447 * For each of the event queues that we initialized and mapped 448 * earlier, attempt to unmap the events from the EQ. 449 */ 450 for (i = 0; i < num_eq; i++) { 451 status = tavor_eq_handler_fini(state, state->ts_eqhdl[i]); 452 if (status != DDI_SUCCESS) { 453 return (DDI_FAILURE); 454 } 455 } 456 457 /* 458 * Teardown and free up all the Event Queues that were allocated 459 * earlier. 460 */ 461 for (i = 0; i < num_eq; i++) { 462 status = tavor_eq_free(state, &state->ts_eqhdl[i]); 463 if (status != DDI_SUCCESS) { 464 return (DDI_FAILURE); 465 } 466 } 467 468 return (DDI_SUCCESS); 469 } 470 471 472 /* 473 * tavor_eq_arm_all 474 * Context: Only called from attach() and/or detach() path contexts 475 */ 476 void 477 tavor_eq_arm_all(tavor_state_t *state) 478 { 479 uint_t num_eq; 480 int i; 481 482 /* 483 * Grab the total number of supported EQs again. This is the same 484 * hardcoded value that was used above (during the event queue 485 * initialization.) 486 */ 487 num_eq = TAVOR_NUM_EQ_USED; 488 489 /* 490 * For each of the event queues that we initialized and mapped 491 * earlier, attempt to arm it for event generation. 492 */ 493 for (i = 0; i < num_eq; i++) { 494 tavor_eq_doorbell(state, TAVOR_EQDB_REARM_EQ, i, 0); 495 } 496 } 497 498 499 /* 500 * tavor_isr() 501 * Context: Only called from interrupt context (and during panic) 502 */ 503 /* ARGSUSED */ 504 uint_t 505 tavor_isr(caddr_t arg1, caddr_t arg2) 506 { 507 tavor_state_t *state; 508 uint64_t *ecr, *clr_int; 509 uint64_t ecrreg, int_mask; 510 uint_t status; 511 int i; 512 513 /* 514 * Grab the Tavor softstate pointer from the input parameter 515 */ 516 state = (tavor_state_t *)arg1; 517 518 /* 519 * Find the pointers to the ECR and clr_INT registers 520 */ 521 ecr = state->ts_cmd_regs.ecr; 522 clr_int = state->ts_cmd_regs.clr_int; 523 524 /* 525 * Read the ECR register. Each of the 64 bits in the ECR register 526 * corresponds to an event queue. If a bit is set, then the 527 * corresponding event queue has fired. 528 */ 529 ecrreg = ddi_get64(state->ts_reg_cmdhdl, ecr); 530 531 /* 532 * As long as there are bits set (i.e. as long as there are still 533 * EQs in the "fired" state), call tavor_eq_poll() to process each 534 * fired EQ. If no ECR bits are set, do not claim the interrupt. 535 */ 536 status = DDI_INTR_UNCLAIMED; 537 do { 538 i = 0; 539 while (ecrreg != 0x0) { 540 if (ecrreg & 0x1) { 541 tavor_eq_poll(state, state->ts_eqhdl[i]); 542 status = DDI_INTR_CLAIMED; 543 } 544 ecrreg = ecrreg >> 1; 545 i++; 546 } 547 548 /* 549 * Clear the interrupt. Note: Depending on the type of 550 * event (interrupt or MSI), we need to use a different 551 * mask to clear the event. In the case of MSI, the bit 552 * to clear corresponds to the MSI number, and for legacy 553 * interrupts the bit corresponds to the value in 'inta_pin'. 554 */ 555 if (state->ts_intr_type_chosen == DDI_INTR_TYPE_MSI) { 556 int_mask = ((uint64_t)1 << 0); 557 } else { 558 int_mask = ((uint64_t)1 << state->ts_adapter.inta_pin); 559 } 560 ddi_put64(state->ts_reg_cmdhdl, clr_int, int_mask); 561 562 /* Reread the ECR register */ 563 ecrreg = ddi_get64(state->ts_reg_cmdhdl, ecr); 564 565 } while (ecrreg != 0x0); 566 567 return (status); 568 } 569 570 571 /* 572 * tavor_eq_doorbell 573 * Context: Only called from interrupt context 574 */ 575 void 576 tavor_eq_doorbell(tavor_state_t *state, uint32_t eq_cmd, uint32_t eqn, 577 uint32_t eq_param) 578 { 579 uint64_t doorbell = 0; 580 581 /* Build the doorbell from the parameters */ 582 doorbell = ((uint64_t)eq_cmd << TAVOR_EQDB_CMD_SHIFT) | 583 ((uint64_t)eqn << TAVOR_EQDB_EQN_SHIFT) | eq_param; 584 585 /* Write the doorbell to UAR */ 586 TAVOR_UAR_DOORBELL(state, (uint64_t *)&state->ts_uar->eq, 587 doorbell); 588 } 589 590 /* 591 * tavor_eq_poll 592 * Context: Only called from interrupt context (and during panic) 593 */ 594 static void 595 tavor_eq_poll(tavor_state_t *state, tavor_eqhdl_t eq) 596 { 597 uint64_t *clr_ecr; 598 tavor_hw_eqe_t *eqe; 599 uint64_t ecr_mask; 600 uint32_t cons_indx, wrap_around_mask; 601 int (*eqfunction)(tavor_state_t *state, tavor_eqhdl_t eq, 602 tavor_hw_eqe_t *eqe); 603 604 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*eq)) 605 606 /* Find the pointer to the clr_ECR register */ 607 clr_ecr = state->ts_cmd_regs.clr_ecr; 608 609 /* 610 * Check for Local Catastrophic Error If we have this kind of error, 611 * then we don't need to do anything else here, as this kind of 612 * catastrophic error is handled separately. So we call the 613 * catastrophic handler, clear the ECR and then return. 614 */ 615 if (eq->eq_evttypemask == TAVOR_EVT_MSK_LOCAL_CAT_ERROR) { 616 /* 617 * Call Catastrophic Error handler 618 */ 619 tavor_eq_catastrophic(state); 620 621 /* 622 * Clear the ECR. Specifically, clear the bit corresponding 623 * to the event queue just processed. 624 */ 625 ecr_mask = ((uint64_t)1 << eq->eq_eqnum); 626 ddi_put64(state->ts_reg_cmdhdl, clr_ecr, ecr_mask); 627 628 return; 629 } 630 631 /* Get the consumer pointer index */ 632 cons_indx = eq->eq_consindx; 633 634 /* 635 * Calculate the wrap around mask. Note: This operation only works 636 * because all Tavor event queues have power-of-2 sizes 637 */ 638 wrap_around_mask = (eq->eq_bufsz - 1); 639 640 /* Calculate the pointer to the first EQ entry */ 641 eqe = &eq->eq_buf[cons_indx]; 642 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*eqe)) 643 644 /* 645 * Sync the current EQE to read 646 * We need to force a ddi_dma_sync() here (independent of how the 647 * EQ was mapped) because it is possible for us to receive the 648 * interrupt, do a read of the ECR, and have each of these 649 * operations complete successfully even though the hardware's DMA 650 * to the EQ has not yet completed. 651 */ 652 tavor_eqe_sync(eq, eqe, DDI_DMA_SYNC_FORCPU, TAVOR_EQ_SYNC_FORCE); 653 654 /* 655 * Pull the handler function for this EQ from the Tavor Event Queue 656 * handle 657 */ 658 eqfunction = eq->eq_func; 659 660 /* 661 * Keep pulling entries from the EQ until we find an entry owner by 662 * the hardware. As long as there the EQE's owned by SW, process 663 * each entry by calling its handler function and updating the EQ 664 * consumer index. 665 */ 666 do { 667 while (TAVOR_EQE_OWNER_IS_SW(eq, eqe)) { 668 /* 669 * Call the EQ handler function. But only call if we 670 * are not in polled I/O mode (i.e. not processing 671 * because of a system panic). Note: We don't call 672 * the EQ handling functions from a system panic 673 * because we are primarily concerned only with 674 * ensuring that the event queues do not overflow (or, 675 * more specifically, the event queue associated with 676 * the CQ that is being used in the sync/dump process). 677 * Also, we don't want to make any upcalls (to the 678 * IBTF) because we can't guarantee when/if those 679 * calls would ever return. And, if we're in panic, 680 * then we reached here through a PollCQ() call (from 681 * tavor_cq_poll()), and we need to ensure that we 682 * successfully return any work completions to the 683 * caller. 684 */ 685 if (ddi_in_panic() == 0) { 686 eqfunction(state, eq, eqe); 687 } 688 689 /* Reset entry to hardware ownership */ 690 TAVOR_EQE_OWNER_SET_HW(eq, eqe); 691 692 /* Sync the current EQE for device */ 693 tavor_eqe_sync(eq, eqe, DDI_DMA_SYNC_FORDEV, 694 TAVOR_EQ_SYNC_NORMAL); 695 696 /* Increment the consumer index */ 697 cons_indx = (cons_indx + 1) & wrap_around_mask; 698 699 /* Update the pointer to the next EQ entry */ 700 eqe = &eq->eq_buf[cons_indx]; 701 702 /* Sync the next EQE to read */ 703 tavor_eqe_sync(eq, eqe, DDI_DMA_SYNC_FORCPU, 704 TAVOR_EQ_SYNC_NORMAL); 705 } 706 707 /* 708 * Clear the ECR. Specifically, clear the bit corresponding 709 * to the event queue just processed. 710 */ 711 ecr_mask = ((uint64_t)1 << eq->eq_eqnum); 712 ddi_put64(state->ts_reg_cmdhdl, clr_ecr, ecr_mask); 713 714 /* Write an EQ doorbell to update the consumer index */ 715 eq->eq_consindx = cons_indx; 716 tavor_eq_doorbell(state, TAVOR_EQDB_SET_CONSINDX, eq->eq_eqnum, 717 cons_indx); 718 719 /* Write another EQ doorbell to rearm */ 720 tavor_eq_doorbell(state, TAVOR_EQDB_REARM_EQ, eq->eq_eqnum, 0); 721 722 /* 723 * NOTE: Due to the nature of Mellanox hardware, we do not have 724 * to do an explicit PIO read to ensure that the doorbell write 725 * has been flushed to the hardware. There is state encoded in 726 * the doorbell information we write which makes this 727 * unnecessary. We can be assured that if an event needs to be 728 * generated, the hardware will make sure that it is, solving 729 * the possible race condition. 730 */ 731 732 /* Sync the next EQE to read */ 733 tavor_eqe_sync(eq, eqe, DDI_DMA_SYNC_FORCPU, 734 TAVOR_EQ_SYNC_NORMAL); 735 736 } while (TAVOR_EQE_OWNER_IS_SW(eq, eqe)); 737 } 738 739 740 /* 741 * tavor_eq_catastrophic 742 * Context: Only called from interrupt context (and during panic) 743 */ 744 static void 745 tavor_eq_catastrophic(tavor_state_t *state) 746 { 747 ibt_async_code_t type; 748 ibc_async_event_t event; 749 uint32_t *base_addr; 750 uint32_t buf_size; 751 uint32_t word; 752 uint8_t err_type; 753 uint32_t err_buf; 754 int i; 755 756 bzero(&event, sizeof (ibc_async_event_t)); 757 758 base_addr = (uint32_t *)(uintptr_t)( 759 (uintptr_t)state->ts_reg_cmd_baseaddr + 760 state->ts_fw.error_buf_addr); 761 buf_size = state->ts_fw.error_buf_sz; 762 763 word = ddi_get32(state->ts_reg_cmdhdl, base_addr); 764 765 err_type = (word & 0xFF000000) >> 24; 766 type = IBT_ERROR_LOCAL_CATASTROPHIC; 767 768 switch (err_type) { 769 case TAVOR_CATASTROPHIC_INTERNAL_ERROR: 770 cmn_err(CE_WARN, "Catastrophic Internal Error: 0x%02x", 771 err_type); 772 773 break; 774 775 case TAVOR_CATASTROPHIC_UPLINK_BUS_ERROR: 776 cmn_err(CE_WARN, "Catastrophic Uplink Bus Error: 0x%02x", 777 err_type); 778 779 break; 780 781 case TAVOR_CATASTROPHIC_DDR_DATA_ERROR: 782 cmn_err(CE_WARN, "Catastrophic DDR Data Error: 0x%02x", 783 err_type); 784 785 break; 786 787 case TAVOR_CATASTROPHIC_INTERNAL_PARITY_ERROR: 788 cmn_err(CE_WARN, "Catastrophic Internal Parity Error: 0x%02x", 789 err_type); 790 791 break; 792 793 default: 794 /* Unknown type of Catastrophic error */ 795 cmn_err(CE_WARN, "Catastrophic Unknown Error: 0x%02x", 796 err_type); 797 798 break; 799 } 800 801 /* 802 * Read in the catastrophic error buffer from the hardware, printing 803 * only to the log file only 804 */ 805 for (i = 0; i < buf_size; i += 4) { 806 base_addr = (uint32_t *)((uintptr_t)(state->ts_reg_cmd_baseaddr 807 + state->ts_fw.error_buf_addr + (i * 4))); 808 err_buf = ddi_get32(state->ts_reg_cmdhdl, base_addr); 809 cmn_err(CE_WARN, "catastrophic_error[%02x]: %08X", i, err_buf); 810 } 811 812 /* 813 * We also call the IBTF here to inform it of the catastrophic error. 814 * Note: Since no event information (i.e. QP handles, CQ handles, 815 * etc.) is necessary, we pass a NULL pointer instead of a pointer to 816 * an empty ibc_async_event_t struct. 817 * 818 * But we also check if "ts_ibtfpriv" is NULL. If it is then it 819 * means that we've have either received this event before we 820 * finished attaching to the IBTF or we've received it while we 821 * are in the process of detaching. 822 */ 823 if (state->ts_ibtfpriv != NULL) { 824 TAVOR_DO_IBTF_ASYNC_CALLB(state, type, &event); 825 } 826 } 827 828 829 /* 830 * tavor_eq_alloc() 831 * Context: Only called from attach() path context 832 */ 833 static int 834 tavor_eq_alloc(tavor_state_t *state, uint32_t log_eq_size, uint_t intr, 835 tavor_eqhdl_t *eqhdl) 836 { 837 tavor_rsrc_t *eqc, *rsrc; 838 tavor_hw_eqc_t eqc_entry; 839 tavor_eqhdl_t eq; 840 ibt_mr_attr_t mr_attr; 841 tavor_mr_options_t op; 842 tavor_pdhdl_t pd; 843 tavor_mrhdl_t mr; 844 tavor_hw_eqe_t *buf; 845 uint64_t addr; 846 uint32_t lkey; 847 uint_t dma_xfer_mode; 848 int status, i; 849 850 /* Use the internal protection domain (PD) for setting up EQs */ 851 pd = state->ts_pdhdl_internal; 852 853 /* Increment the reference count on the protection domain (PD) */ 854 tavor_pd_refcnt_inc(pd); 855 856 /* 857 * Allocate an EQ context entry. This will be filled in with all 858 * the necessary parameters to define the Event Queue. And then 859 * ownership will be passed to the hardware in the final step 860 * below. If we fail here, we must undo the protection domain 861 * reference count. 862 */ 863 status = tavor_rsrc_alloc(state, TAVOR_EQC, 1, TAVOR_SLEEP, &eqc); 864 if (status != DDI_SUCCESS) { 865 goto eqalloc_fail1; 866 } 867 868 /* 869 * Allocate the software structure for tracking the event queue (i.e. 870 * the Tavor Event Queue handle). If we fail here, we must undo the 871 * protection domain reference count and the previous resource 872 * allocation. 873 */ 874 status = tavor_rsrc_alloc(state, TAVOR_EQHDL, 1, TAVOR_SLEEP, &rsrc); 875 if (status != DDI_SUCCESS) { 876 goto eqalloc_fail2; 877 } 878 eq = (tavor_eqhdl_t)rsrc->tr_addr; 879 880 /* 881 * Allocate the memory for Event Queue. Note: Although we use the 882 * common queue allocation routine, we always specify 883 * TAVOR_QUEUE_LOCATION_NORMAL (i.e. EQ located in system memory) 884 * because it would be inefficient to have EQs located in DDR memory. 885 * This is primarily because EQs are read from (by software) more 886 * than they are written to. Also note that, unlike Tavor QP work 887 * queues, event queues do not have the same strict alignment 888 * requirements. It is sufficient for the EQ memory to be both 889 * aligned to and bound to addresses which are a multiple of EQE size. 890 */ 891 eq->eq_eqinfo.qa_size = (1 << log_eq_size) * sizeof (tavor_hw_eqe_t); 892 eq->eq_eqinfo.qa_alloc_align = sizeof (tavor_hw_eqe_t); 893 eq->eq_eqinfo.qa_bind_align = sizeof (tavor_hw_eqe_t); 894 eq->eq_eqinfo.qa_location = TAVOR_QUEUE_LOCATION_NORMAL; 895 status = tavor_queue_alloc(state, &eq->eq_eqinfo, TAVOR_SLEEP); 896 if (status != DDI_SUCCESS) { 897 goto eqalloc_fail3; 898 } 899 buf = (tavor_hw_eqe_t *)eq->eq_eqinfo.qa_buf_aligned; 900 901 /* 902 * Initialize each of the Event Queue Entries (EQE) by setting their 903 * ownership to hardware ("owner" bit set to HW). This is in 904 * preparation for the final transfer of ownership (below) of the 905 * EQ context itself. 906 */ 907 for (i = 0; i < (1 << log_eq_size); i++) { 908 TAVOR_EQE_OWNER_SET_HW(eq, &buf[i]); 909 } 910 911 /* 912 * Register the memory for the EQ. The memory for the EQ must 913 * be registered in the Tavor TPT tables. This gives us the LKey 914 * to specify in the EQ context below. 915 * 916 * Because we are in the attach path we use NOSLEEP here so that we 917 * SPIN in the HCR since the event queues are not setup yet, and we 918 * cannot NOSPIN at this point in time. 919 */ 920 mr_attr.mr_vaddr = (uint64_t)(uintptr_t)buf; 921 mr_attr.mr_len = eq->eq_eqinfo.qa_size; 922 mr_attr.mr_as = NULL; 923 mr_attr.mr_flags = IBT_MR_NOSLEEP | IBT_MR_ENABLE_LOCAL_WRITE; 924 dma_xfer_mode = state->ts_cfg_profile->cp_streaming_consistent; 925 if (dma_xfer_mode == DDI_DMA_STREAMING) { 926 mr_attr.mr_flags |= IBT_MR_NONCOHERENT; 927 } 928 op.mro_bind_type = state->ts_cfg_profile->cp_iommu_bypass; 929 op.mro_bind_dmahdl = eq->eq_eqinfo.qa_dmahdl; 930 op.mro_bind_override_addr = 0; 931 status = tavor_mr_register(state, pd, &mr_attr, &mr, &op); 932 if (status != DDI_SUCCESS) { 933 goto eqalloc_fail4; 934 } 935 _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*mr)) 936 addr = mr->mr_bindinfo.bi_addr; 937 lkey = mr->mr_lkey; 938 939 /* Determine if later ddi_dma_sync will be necessary */ 940 eq->eq_sync = TAVOR_EQ_IS_SYNC_REQ(state, eq->eq_eqinfo); 941 942 /* Sync entire EQ for use by the hardware (if necessary) */ 943 if (eq->eq_sync) { 944 (void) ddi_dma_sync(mr->mr_bindinfo.bi_dmahdl, 0, 945 eq->eq_eqinfo.qa_size, DDI_DMA_SYNC_FORDEV); 946 } 947 948 /* 949 * Fill in the EQC entry. This is the final step before passing 950 * ownership of the EQC entry to the Tavor hardware. We use all of 951 * the information collected/calculated above to fill in the 952 * requisite portions of the EQC. Note: We create all EQs in the 953 * "fired" state. We will arm them later (after our interrupt 954 * routine had been registered.) 955 */ 956 bzero(&eqc_entry, sizeof (tavor_hw_eqc_t)); 957 eqc_entry.owner = TAVOR_HW_OWNER; 958 eqc_entry.xlat = TAVOR_VA2PA_XLAT_ENABLED; 959 eqc_entry.state = TAVOR_EQ_FIRED; 960 eqc_entry.start_addr_h = (addr >> 32); 961 eqc_entry.start_addr_l = (addr & 0xFFFFFFFF); 962 eqc_entry.log_eq_sz = log_eq_size; 963 eqc_entry.usr_page = 0; 964 eqc_entry.pd = pd->pd_pdnum; 965 eqc_entry.intr = intr; 966 eqc_entry.lkey = lkey; 967 968 /* 969 * Write the EQC entry to hardware. Lastly, we pass ownership of 970 * the entry to the hardware (using the Tavor SW2HW_EQ firmware 971 * command). Note: in general, this operation shouldn't fail. But 972 * if it does, we have to undo everything we've done above before 973 * returning error. 974 */ 975 status = tavor_cmn_ownership_cmd_post(state, SW2HW_EQ, &eqc_entry, 976 sizeof (tavor_hw_eqc_t), eqc->tr_indx, TAVOR_CMD_NOSLEEP_SPIN); 977 if (status != TAVOR_CMD_SUCCESS) { 978 cmn_err(CE_CONT, "Tavor: SW2HW_EQ command failed: %08x\n", 979 status); 980 goto eqalloc_fail5; 981 } 982 983 /* 984 * Fill in the rest of the Tavor Event Queue handle. Having 985 * successfully transferred ownership of the EQC, we can update the 986 * following fields for use in further operations on the EQ. 987 */ 988 eq->eq_eqcrsrcp = eqc; 989 eq->eq_rsrcp = rsrc; 990 eq->eq_consindx = 0; 991 eq->eq_eqnum = eqc->tr_indx; 992 eq->eq_buf = buf; 993 eq->eq_bufsz = (1 << log_eq_size); 994 eq->eq_mrhdl = mr; 995 *eqhdl = eq; 996 997 return (DDI_SUCCESS); 998 999 /* 1000 * The following is cleanup for all possible failure cases in this routine 1001 */ 1002 eqalloc_fail5: 1003 if (tavor_mr_deregister(state, &mr, TAVOR_MR_DEREG_ALL, 1004 TAVOR_NOSLEEP) != DDI_SUCCESS) { 1005 TAVOR_WARNING(state, "failed to deregister EQ memory"); 1006 } 1007 eqalloc_fail4: 1008 tavor_queue_free(state, &eq->eq_eqinfo); 1009 eqalloc_fail3: 1010 tavor_rsrc_free(state, &rsrc); 1011 eqalloc_fail2: 1012 tavor_rsrc_free(state, &eqc); 1013 eqalloc_fail1: 1014 tavor_pd_refcnt_dec(pd); 1015 eqalloc_fail: 1016 return (status); 1017 } 1018 1019 1020 /* 1021 * tavor_eq_free() 1022 * Context: Only called from attach() and/or detach() path contexts 1023 */ 1024 static int 1025 tavor_eq_free(tavor_state_t *state, tavor_eqhdl_t *eqhdl) 1026 { 1027 tavor_rsrc_t *eqc, *rsrc; 1028 tavor_hw_eqc_t eqc_entry; 1029 tavor_pdhdl_t pd; 1030 tavor_mrhdl_t mr; 1031 tavor_eqhdl_t eq; 1032 uint32_t eqnum; 1033 int status; 1034 1035 /* 1036 * Pull all the necessary information from the Tavor Event Queue 1037 * handle. This is necessary here because the resource for the 1038 * EQ handle is going to be freed up as part of this operation. 1039 */ 1040 eq = *eqhdl; 1041 eqc = eq->eq_eqcrsrcp; 1042 rsrc = eq->eq_rsrcp; 1043 pd = state->ts_pdhdl_internal; 1044 mr = eq->eq_mrhdl; 1045 eqnum = eq->eq_eqnum; 1046 1047 /* 1048 * Reclaim EQC entry from hardware (using the Tavor HW2SW_EQ 1049 * firmware command). If the ownership transfer fails for any reason, 1050 * then it is an indication that something (either in HW or SW) has 1051 * gone seriously wrong. 1052 */ 1053 status = tavor_cmn_ownership_cmd_post(state, HW2SW_EQ, &eqc_entry, 1054 sizeof (tavor_hw_eqc_t), eqnum, TAVOR_CMD_NOSLEEP_SPIN); 1055 if (status != TAVOR_CMD_SUCCESS) { 1056 TAVOR_WARNING(state, "failed to reclaim EQC ownership"); 1057 cmn_err(CE_CONT, "Tavor: HW2SW_EQ command failed: %08x\n", 1058 status); 1059 return (DDI_FAILURE); 1060 } 1061 1062 /* 1063 * Deregister the memory for the Event Queue. If this fails 1064 * for any reason, then it is an indication that something (either 1065 * in HW or SW) has gone seriously wrong. So we print a warning 1066 * message and continue. 1067 */ 1068 status = tavor_mr_deregister(state, &mr, TAVOR_MR_DEREG_ALL, 1069 TAVOR_NOSLEEP); 1070 if (status != DDI_SUCCESS) { 1071 TAVOR_WARNING(state, "failed to deregister EQ memory"); 1072 } 1073 1074 /* Free the memory for the EQ */ 1075 tavor_queue_free(state, &eq->eq_eqinfo); 1076 1077 /* Free the Tavor Event Queue handle */ 1078 tavor_rsrc_free(state, &rsrc); 1079 1080 /* Free up the EQC entry resource */ 1081 tavor_rsrc_free(state, &eqc); 1082 1083 /* Decrement the reference count on the protection domain (PD) */ 1084 tavor_pd_refcnt_dec(pd); 1085 1086 /* Set the eqhdl pointer to NULL and return success */ 1087 *eqhdl = NULL; 1088 1089 return (DDI_SUCCESS); 1090 } 1091 1092 1093 /* 1094 * tavor_eq_handler_init 1095 * Context: Only called from attach() path context 1096 */ 1097 static int 1098 tavor_eq_handler_init(tavor_state_t *state, tavor_eqhdl_t eq, 1099 uint_t evt_type_mask, int (*eq_func)(tavor_state_t *state, 1100 tavor_eqhdl_t eq, tavor_hw_eqe_t *eqe)) 1101 { 1102 int status; 1103 1104 /* 1105 * Save away the EQ handler function and the event type mask. These 1106 * will be used later during interrupt and event queue processing. 1107 */ 1108 eq->eq_func = eq_func; 1109 eq->eq_evttypemask = evt_type_mask; 1110 1111 /* 1112 * Map the EQ to a specific class of event (or events) depending 1113 * on the mask value passed in. The TAVOR_EVT_NO_MASK means not 1114 * to attempt associating the EQ with any specific class of event. 1115 * This is particularly useful when initializing the events queues 1116 * used for CQ events. The mapping is done using the Tavor MAP_EQ 1117 * firmware command. Note: This command should not, in general, fail. 1118 * If it does, then something (probably HW related) has gone seriously 1119 * wrong. 1120 */ 1121 if (evt_type_mask != TAVOR_EVT_NO_MASK) { 1122 status = tavor_map_eq_cmd_post(state, 1123 TAVOR_CMD_MAP_EQ_EVT_MAP, eq->eq_eqnum, evt_type_mask, 1124 TAVOR_CMD_NOSLEEP_SPIN); 1125 if (status != TAVOR_CMD_SUCCESS) { 1126 cmn_err(CE_CONT, "Tavor: MAP_EQ command failed: " 1127 "%08x\n", status); 1128 return (DDI_FAILURE); 1129 } 1130 } 1131 1132 return (DDI_SUCCESS); 1133 } 1134 1135 1136 /* 1137 * tavor_eq_handler_fini 1138 * Context: Only called from attach() and/or detach() path contexts 1139 */ 1140 static int 1141 tavor_eq_handler_fini(tavor_state_t *state, tavor_eqhdl_t eq) 1142 { 1143 int status; 1144 1145 /* 1146 * Unmap the EQ from the event class to which it had been previously 1147 * mapped. The unmapping is done using the Tavor MAP_EQ (in much 1148 * the same way that the initial mapping was done). The difference, 1149 * however, is in the TAVOR_EQ_EVT_UNMAP flag that is passed to the 1150 * MAP_EQ firmware command. The TAVOR_EVT_NO_MASK (which may have 1151 * been passed in at init time) still means that no association has 1152 * been made between the EQ and any specific class of event (and, 1153 * hence, no unmapping is necessary). Note: This command should not, 1154 * in general, fail. If it does, then something (probably HW related) 1155 * has gone seriously wrong. 1156 */ 1157 if (eq->eq_evttypemask != TAVOR_EVT_NO_MASK) { 1158 status = tavor_map_eq_cmd_post(state, 1159 TAVOR_CMD_MAP_EQ_EVT_UNMAP, eq->eq_eqnum, 1160 eq->eq_evttypemask, TAVOR_CMD_NOSLEEP_SPIN); 1161 if (status != TAVOR_CMD_SUCCESS) { 1162 cmn_err(CE_CONT, "Tavor: MAP_EQ command failed: " 1163 "%08x\n", status); 1164 return (DDI_FAILURE); 1165 } 1166 } 1167 1168 return (DDI_SUCCESS); 1169 } 1170 1171 1172 /* 1173 * tavor_eqe_sync() 1174 * Context: Can be called from interrupt or base context. 1175 * 1176 * Typically, this routine does nothing unless the EQ memory is 1177 * mapped as DDI_DMA_STREAMING. However, there is a condition where 1178 * ddi_dma_sync() is necessary even if the memory was mapped in 1179 * consistent mode. The "force_sync" parameter is used here to force 1180 * the call to ddi_dma_sync() independent of how the EQ memory was 1181 * mapped. 1182 */ 1183 static void 1184 tavor_eqe_sync(tavor_eqhdl_t eq, tavor_hw_eqe_t *eqe, uint_t flag, 1185 uint_t force_sync) 1186 { 1187 ddi_dma_handle_t dmahdl; 1188 off_t offset; 1189 1190 /* Determine if EQ needs to be synced or not */ 1191 if ((eq->eq_sync == 0) && (force_sync == TAVOR_EQ_SYNC_NORMAL)) { 1192 return; 1193 } 1194 1195 /* Get the DMA handle from EQ context */ 1196 dmahdl = eq->eq_mrhdl->mr_bindinfo.bi_dmahdl; 1197 1198 /* Calculate offset of next EQE */ 1199 offset = (off_t)((uintptr_t)eqe - (uintptr_t)&eq->eq_buf[0]); 1200 (void) ddi_dma_sync(dmahdl, offset, sizeof (tavor_hw_eqe_t), flag); 1201 } 1202 1203 1204 /* 1205 * tavor_port_state_change_handler() 1206 * Context: Only called from interrupt context 1207 */ 1208 static int 1209 tavor_port_state_change_handler(tavor_state_t *state, tavor_eqhdl_t eq, 1210 tavor_hw_eqe_t *eqe) 1211 { 1212 ibc_async_event_t event; 1213 ibt_async_code_t type; 1214 uint_t port, subtype; 1215 uint_t eqe_evttype; 1216 char link_msg[24]; 1217 1218 eqe_evttype = TAVOR_EQE_EVTTYPE_GET(eq, eqe); 1219 1220 ASSERT(eqe_evttype == TAVOR_EVT_PORT_STATE_CHANGE || 1221 eqe_evttype == TAVOR_EVT_EQ_OVERFLOW); 1222 1223 if (eqe_evttype == TAVOR_EVT_EQ_OVERFLOW) { 1224 tavor_eq_overflow_handler(state, eq, eqe); 1225 1226 return (DDI_FAILURE); 1227 } 1228 1229 /* 1230 * Depending on the type of Port State Change event, pass the 1231 * appropriate asynch event to the IBTF. 1232 */ 1233 port = TAVOR_EQE_PORTNUM_GET(eq, eqe); 1234 1235 /* Check for valid port number in event */ 1236 if ((port == 0) || (port > state->ts_cfg_profile->cp_num_ports)) { 1237 TAVOR_WARNING(state, "Unexpected port number in port state " 1238 "change event"); 1239 cmn_err(CE_CONT, " Port number: %02x\n", port); 1240 return (DDI_FAILURE); 1241 } 1242 1243 subtype = TAVOR_EQE_EVTSUBTYPE_GET(eq, eqe); 1244 if (subtype == TAVOR_PORT_LINK_ACTIVE) { 1245 event.ev_port = port; 1246 type = IBT_EVENT_PORT_UP; 1247 1248 (void) snprintf(link_msg, 23, "port %d up", port); 1249 ddi_dev_report_fault(state->ts_dip, DDI_SERVICE_RESTORED, 1250 DDI_EXTERNAL_FAULT, link_msg); 1251 } else if (subtype == TAVOR_PORT_LINK_DOWN) { 1252 event.ev_port = port; 1253 type = IBT_ERROR_PORT_DOWN; 1254 1255 (void) snprintf(link_msg, 23, "port %d down", port); 1256 ddi_dev_report_fault(state->ts_dip, DDI_SERVICE_LOST, 1257 DDI_EXTERNAL_FAULT, link_msg); 1258 } else { 1259 TAVOR_WARNING(state, "Unexpected subtype in port state change " 1260 "event"); 1261 cmn_err(CE_CONT, " Event type: %02x, subtype: %02x\n", 1262 TAVOR_EQE_EVTTYPE_GET(eq, eqe), subtype); 1263 return (DDI_FAILURE); 1264 } 1265 1266 /* 1267 * Deliver the event to the IBTF. Note: If "ts_ibtfpriv" is NULL, 1268 * then we have either received this event before we finished 1269 * attaching to the IBTF or we've received it while we are in the 1270 * process of detaching. 1271 */ 1272 if (state->ts_ibtfpriv != NULL) { 1273 TAVOR_DO_IBTF_ASYNC_CALLB(state, type, &event); 1274 } 1275 1276 return (DDI_SUCCESS); 1277 } 1278 1279 1280 /* 1281 * tavor_comm_estbl_handler() 1282 * Context: Only called from interrupt context 1283 */ 1284 static int 1285 tavor_comm_estbl_handler(tavor_state_t *state, tavor_eqhdl_t eq, 1286 tavor_hw_eqe_t *eqe) 1287 { 1288 tavor_qphdl_t qp; 1289 uint_t qpnum; 1290 ibc_async_event_t event; 1291 ibt_async_code_t type; 1292 uint_t eqe_evttype; 1293 1294 eqe_evttype = TAVOR_EQE_EVTTYPE_GET(eq, eqe); 1295 1296 ASSERT(eqe_evttype == TAVOR_EVT_COMM_ESTABLISHED || 1297 eqe_evttype == TAVOR_EVT_EQ_OVERFLOW); 1298 1299 if (eqe_evttype == TAVOR_EVT_EQ_OVERFLOW) { 1300 tavor_eq_overflow_handler(state, eq, eqe); 1301 1302 return (DDI_FAILURE); 1303 } 1304 1305 /* Get the QP handle from QP number in event descriptor */ 1306 qpnum = TAVOR_EQE_QPNUM_GET(eq, eqe); 1307 qp = tavor_qphdl_from_qpnum(state, qpnum); 1308 1309 /* 1310 * If the QP handle is NULL, this is probably an indication 1311 * that the QP has been freed already. In which case, we 1312 * should not deliver this event. 1313 * 1314 * We also check that the QP number in the handle is the 1315 * same as the QP number in the event queue entry. This 1316 * extra check allows us to handle the case where a QP was 1317 * freed and then allocated again in the time it took to 1318 * handle the event queue processing. By constantly incrementing 1319 * the non-constrained portion of the QP number every time 1320 * a new QP is allocated, we mitigate (somewhat) the chance 1321 * that a stale event could be passed to the client's QP 1322 * handler. 1323 * 1324 * Lastly, we check if "ts_ibtfpriv" is NULL. If it is then it 1325 * means that we've have either received this event before we 1326 * finished attaching to the IBTF or we've received it while we 1327 * are in the process of detaching. 1328 */ 1329 if ((qp != NULL) && (qp->qp_qpnum == qpnum) && 1330 (state->ts_ibtfpriv != NULL)) { 1331 event.ev_qp_hdl = (ibtl_qp_hdl_t)qp->qp_hdlrarg; 1332 type = IBT_EVENT_COM_EST_QP; 1333 1334 TAVOR_DO_IBTF_ASYNC_CALLB(state, type, &event); 1335 } 1336 1337 return (DDI_SUCCESS); 1338 } 1339 1340 1341 /* 1342 * tavor_local_wq_cat_err_handler() 1343 * Context: Only called from interrupt context 1344 */ 1345 static int 1346 tavor_local_wq_cat_err_handler(tavor_state_t *state, tavor_eqhdl_t eq, 1347 tavor_hw_eqe_t *eqe) 1348 { 1349 tavor_qphdl_t qp; 1350 uint_t qpnum; 1351 ibc_async_event_t event; 1352 ibt_async_code_t type; 1353 uint_t eqe_evttype; 1354 1355 eqe_evttype = TAVOR_EQE_EVTTYPE_GET(eq, eqe); 1356 1357 ASSERT(eqe_evttype == TAVOR_EVT_LOCAL_WQ_CAT_ERROR || 1358 eqe_evttype == TAVOR_EVT_EQ_OVERFLOW); 1359 1360 if (eqe_evttype == TAVOR_EVT_EQ_OVERFLOW) { 1361 tavor_eq_overflow_handler(state, eq, eqe); 1362 1363 return (DDI_FAILURE); 1364 } 1365 1366 /* Get the QP handle from QP number in event descriptor */ 1367 qpnum = TAVOR_EQE_QPNUM_GET(eq, eqe); 1368 qp = tavor_qphdl_from_qpnum(state, qpnum); 1369 1370 /* 1371 * If the QP handle is NULL, this is probably an indication 1372 * that the QP has been freed already. In which case, we 1373 * should not deliver this event. 1374 * 1375 * We also check that the QP number in the handle is the 1376 * same as the QP number in the event queue entry. This 1377 * extra check allows us to handle the case where a QP was 1378 * freed and then allocated again in the time it took to 1379 * handle the event queue processing. By constantly incrementing 1380 * the non-constrained portion of the QP number every time 1381 * a new QP is allocated, we mitigate (somewhat) the chance 1382 * that a stale event could be passed to the client's QP 1383 * handler. 1384 * 1385 * Lastly, we check if "ts_ibtfpriv" is NULL. If it is then it 1386 * means that we've have either received this event before we 1387 * finished attaching to the IBTF or we've received it while we 1388 * are in the process of detaching. 1389 */ 1390 if ((qp != NULL) && (qp->qp_qpnum == qpnum) && 1391 (state->ts_ibtfpriv != NULL)) { 1392 event.ev_qp_hdl = (ibtl_qp_hdl_t)qp->qp_hdlrarg; 1393 type = IBT_ERROR_CATASTROPHIC_QP; 1394 1395 TAVOR_DO_IBTF_ASYNC_CALLB(state, type, &event); 1396 } 1397 1398 return (DDI_SUCCESS); 1399 } 1400 1401 1402 /* 1403 * tavor_invreq_local_wq_err_handler() 1404 * Context: Only called from interrupt context 1405 */ 1406 static int 1407 tavor_invreq_local_wq_err_handler(tavor_state_t *state, tavor_eqhdl_t eq, 1408 tavor_hw_eqe_t *eqe) 1409 { 1410 tavor_qphdl_t qp; 1411 uint_t qpnum; 1412 ibc_async_event_t event; 1413 ibt_async_code_t type; 1414 uint_t eqe_evttype; 1415 1416 eqe_evttype = TAVOR_EQE_EVTTYPE_GET(eq, eqe); 1417 1418 ASSERT(eqe_evttype == TAVOR_EVT_INV_REQ_LOCAL_WQ_ERROR || 1419 eqe_evttype == TAVOR_EVT_EQ_OVERFLOW); 1420 1421 if (eqe_evttype == TAVOR_EVT_EQ_OVERFLOW) { 1422 tavor_eq_overflow_handler(state, eq, eqe); 1423 1424 return (DDI_FAILURE); 1425 } 1426 1427 /* Get the QP handle from QP number in event descriptor */ 1428 qpnum = TAVOR_EQE_QPNUM_GET(eq, eqe); 1429 qp = tavor_qphdl_from_qpnum(state, qpnum); 1430 1431 /* 1432 * If the QP handle is NULL, this is probably an indication 1433 * that the QP has been freed already. In which case, we 1434 * should not deliver this event. 1435 * 1436 * We also check that the QP number in the handle is the 1437 * same as the QP number in the event queue entry. This 1438 * extra check allows us to handle the case where a QP was 1439 * freed and then allocated again in the time it took to 1440 * handle the event queue processing. By constantly incrementing 1441 * the non-constrained portion of the QP number every time 1442 * a new QP is allocated, we mitigate (somewhat) the chance 1443 * that a stale event could be passed to the client's QP 1444 * handler. 1445 * 1446 * Lastly, we check if "ts_ibtfpriv" is NULL. If it is then it 1447 * means that we've have either received this event before we 1448 * finished attaching to the IBTF or we've received it while we 1449 * are in the process of detaching. 1450 */ 1451 if ((qp != NULL) && (qp->qp_qpnum == qpnum) && 1452 (state->ts_ibtfpriv != NULL)) { 1453 event.ev_qp_hdl = (ibtl_qp_hdl_t)qp->qp_hdlrarg; 1454 type = IBT_ERROR_INVALID_REQUEST_QP; 1455 1456 TAVOR_DO_IBTF_ASYNC_CALLB(state, type, &event); 1457 } 1458 1459 return (DDI_SUCCESS); 1460 } 1461 1462 1463 /* 1464 * tavor_local_acc_vio_wq_err_handler() 1465 * Context: Only called from interrupt context 1466 */ 1467 static int 1468 tavor_local_acc_vio_wq_err_handler(tavor_state_t *state, tavor_eqhdl_t eq, 1469 tavor_hw_eqe_t *eqe) 1470 { 1471 tavor_qphdl_t qp; 1472 uint_t qpnum; 1473 ibc_async_event_t event; 1474 ibt_async_code_t type; 1475 uint_t eqe_evttype; 1476 1477 eqe_evttype = TAVOR_EQE_EVTTYPE_GET(eq, eqe); 1478 1479 ASSERT(eqe_evttype == TAVOR_EVT_LOCAL_ACC_VIO_WQ_ERROR || 1480 eqe_evttype == TAVOR_EVT_EQ_OVERFLOW); 1481 1482 if (eqe_evttype == TAVOR_EVT_EQ_OVERFLOW) { 1483 tavor_eq_overflow_handler(state, eq, eqe); 1484 1485 return (DDI_FAILURE); 1486 } 1487 1488 /* Get the QP handle from QP number in event descriptor */ 1489 qpnum = TAVOR_EQE_QPNUM_GET(eq, eqe); 1490 qp = tavor_qphdl_from_qpnum(state, qpnum); 1491 1492 /* 1493 * If the QP handle is NULL, this is probably an indication 1494 * that the QP has been freed already. In which case, we 1495 * should not deliver this event. 1496 * 1497 * We also check that the QP number in the handle is the 1498 * same as the QP number in the event queue entry. This 1499 * extra check allows us to handle the case where a QP was 1500 * freed and then allocated again in the time it took to 1501 * handle the event queue processing. By constantly incrementing 1502 * the non-constrained portion of the QP number every time 1503 * a new QP is allocated, we mitigate (somewhat) the chance 1504 * that a stale event could be passed to the client's QP 1505 * handler. 1506 * 1507 * Lastly, we check if "ts_ibtfpriv" is NULL. If it is then it 1508 * means that we've have either received this event before we 1509 * finished attaching to the IBTF or we've received it while we 1510 * are in the process of detaching. 1511 */ 1512 if ((qp != NULL) && (qp->qp_qpnum == qpnum) && 1513 (state->ts_ibtfpriv != NULL)) { 1514 event.ev_qp_hdl = (ibtl_qp_hdl_t)qp->qp_hdlrarg; 1515 type = IBT_ERROR_ACCESS_VIOLATION_QP; 1516 1517 TAVOR_DO_IBTF_ASYNC_CALLB(state, type, &event); 1518 } 1519 1520 return (DDI_SUCCESS); 1521 } 1522 1523 1524 /* 1525 * tavor_sendq_drained_handler() 1526 * Context: Only called from interrupt context 1527 */ 1528 static int 1529 tavor_sendq_drained_handler(tavor_state_t *state, tavor_eqhdl_t eq, 1530 tavor_hw_eqe_t *eqe) 1531 { 1532 tavor_qphdl_t qp; 1533 uint_t qpnum; 1534 ibc_async_event_t event; 1535 uint_t forward_sqd_event; 1536 ibt_async_code_t type; 1537 uint_t eqe_evttype; 1538 1539 eqe_evttype = TAVOR_EQE_EVTTYPE_GET(eq, eqe); 1540 1541 ASSERT(eqe_evttype == TAVOR_EVT_SEND_QUEUE_DRAINED || 1542 eqe_evttype == TAVOR_EVT_EQ_OVERFLOW); 1543 1544 if (eqe_evttype == TAVOR_EVT_EQ_OVERFLOW) { 1545 tavor_eq_overflow_handler(state, eq, eqe); 1546 1547 return (DDI_FAILURE); 1548 } 1549 1550 /* Get the QP handle from QP number in event descriptor */ 1551 qpnum = TAVOR_EQE_QPNUM_GET(eq, eqe); 1552 qp = tavor_qphdl_from_qpnum(state, qpnum); 1553 1554 /* 1555 * If the QP handle is NULL, this is probably an indication 1556 * that the QP has been freed already. In which case, we 1557 * should not deliver this event. 1558 * 1559 * We also check that the QP number in the handle is the 1560 * same as the QP number in the event queue entry. This 1561 * extra check allows us to handle the case where a QP was 1562 * freed and then allocated again in the time it took to 1563 * handle the event queue processing. By constantly incrementing 1564 * the non-constrained portion of the QP number every time 1565 * a new QP is allocated, we mitigate (somewhat) the chance 1566 * that a stale event could be passed to the client's QP 1567 * handler. 1568 * 1569 * And then we check if "ts_ibtfpriv" is NULL. If it is then it 1570 * means that we've have either received this event before we 1571 * finished attaching to the IBTF or we've received it while we 1572 * are in the process of detaching. 1573 */ 1574 if ((qp != NULL) && (qp->qp_qpnum == qpnum) && 1575 (state->ts_ibtfpriv != NULL)) { 1576 event.ev_qp_hdl = (ibtl_qp_hdl_t)qp->qp_hdlrarg; 1577 type = IBT_EVENT_SQD; 1578 1579 /* 1580 * Grab the QP lock and update the QP state to reflect that 1581 * the Send Queue Drained event has arrived. Also determine 1582 * whether the event is intended to be forwarded on to the 1583 * consumer or not. This information is used below in 1584 * determining whether or not to call the IBTF. 1585 */ 1586 mutex_enter(&qp->qp_lock); 1587 forward_sqd_event = qp->qp_forward_sqd_event; 1588 qp->qp_forward_sqd_event = 0; 1589 qp->qp_sqd_still_draining = 0; 1590 mutex_exit(&qp->qp_lock); 1591 1592 if (forward_sqd_event != 0) { 1593 TAVOR_DO_IBTF_ASYNC_CALLB(state, type, &event); 1594 } 1595 } 1596 1597 return (DDI_SUCCESS); 1598 } 1599 1600 1601 /* 1602 * tavor_path_mig_handler() 1603 * Context: Only called from interrupt context 1604 */ 1605 static int 1606 tavor_path_mig_handler(tavor_state_t *state, tavor_eqhdl_t eq, 1607 tavor_hw_eqe_t *eqe) 1608 { 1609 tavor_qphdl_t qp; 1610 uint_t qpnum; 1611 ibc_async_event_t event; 1612 ibt_async_code_t type; 1613 uint_t eqe_evttype; 1614 1615 eqe_evttype = TAVOR_EQE_EVTTYPE_GET(eq, eqe); 1616 1617 ASSERT(eqe_evttype == TAVOR_EVT_PATH_MIGRATED || 1618 eqe_evttype == TAVOR_EVT_EQ_OVERFLOW); 1619 1620 if (eqe_evttype == TAVOR_EVT_EQ_OVERFLOW) { 1621 tavor_eq_overflow_handler(state, eq, eqe); 1622 1623 return (DDI_FAILURE); 1624 } 1625 1626 /* Get the QP handle from QP number in event descriptor */ 1627 qpnum = TAVOR_EQE_QPNUM_GET(eq, eqe); 1628 qp = tavor_qphdl_from_qpnum(state, qpnum); 1629 1630 /* 1631 * If the QP handle is NULL, this is probably an indication 1632 * that the QP has been freed already. In which case, we 1633 * should not deliver this event. 1634 * 1635 * We also check that the QP number in the handle is the 1636 * same as the QP number in the event queue entry. This 1637 * extra check allows us to handle the case where a QP was 1638 * freed and then allocated again in the time it took to 1639 * handle the event queue processing. By constantly incrementing 1640 * the non-constrained portion of the QP number every time 1641 * a new QP is allocated, we mitigate (somewhat) the chance 1642 * that a stale event could be passed to the client's QP 1643 * handler. 1644 * 1645 * Lastly, we check if "ts_ibtfpriv" is NULL. If it is then it 1646 * means that we've have either received this event before we 1647 * finished attaching to the IBTF or we've received it while we 1648 * are in the process of detaching. 1649 */ 1650 if ((qp != NULL) && (qp->qp_qpnum == qpnum) && 1651 (state->ts_ibtfpriv != NULL)) { 1652 event.ev_qp_hdl = (ibtl_qp_hdl_t)qp->qp_hdlrarg; 1653 type = IBT_EVENT_PATH_MIGRATED_QP; 1654 1655 TAVOR_DO_IBTF_ASYNC_CALLB(state, type, &event); 1656 } 1657 1658 return (DDI_SUCCESS); 1659 } 1660 1661 1662 /* 1663 * tavor_path_mig_err_handler() 1664 * Context: Only called from interrupt context 1665 */ 1666 static int 1667 tavor_path_mig_err_handler(tavor_state_t *state, tavor_eqhdl_t eq, 1668 tavor_hw_eqe_t *eqe) 1669 { 1670 tavor_qphdl_t qp; 1671 uint_t qpnum; 1672 ibc_async_event_t event; 1673 ibt_async_code_t type; 1674 uint_t eqe_evttype; 1675 1676 eqe_evttype = TAVOR_EQE_EVTTYPE_GET(eq, eqe); 1677 1678 ASSERT(eqe_evttype == TAVOR_EVT_PATH_MIGRATE_FAILED || 1679 eqe_evttype == TAVOR_EVT_EQ_OVERFLOW); 1680 1681 if (eqe_evttype == TAVOR_EVT_EQ_OVERFLOW) { 1682 tavor_eq_overflow_handler(state, eq, eqe); 1683 1684 return (DDI_FAILURE); 1685 } 1686 1687 /* Get the QP handle from QP number in event descriptor */ 1688 qpnum = TAVOR_EQE_QPNUM_GET(eq, eqe); 1689 qp = tavor_qphdl_from_qpnum(state, qpnum); 1690 1691 /* 1692 * If the QP handle is NULL, this is probably an indication 1693 * that the QP has been freed already. In which case, we 1694 * should not deliver this event. 1695 * 1696 * We also check that the QP number in the handle is the 1697 * same as the QP number in the event queue entry. This 1698 * extra check allows us to handle the case where a QP was 1699 * freed and then allocated again in the time it took to 1700 * handle the event queue processing. By constantly incrementing 1701 * the non-constrained portion of the QP number every time 1702 * a new QP is allocated, we mitigate (somewhat) the chance 1703 * that a stale event could be passed to the client's QP 1704 * handler. 1705 * 1706 * Lastly, we check if "ts_ibtfpriv" is NULL. If it is then it 1707 * means that we've have either received this event before we 1708 * finished attaching to the IBTF or we've received it while we 1709 * are in the process of detaching. 1710 */ 1711 if ((qp != NULL) && (qp->qp_qpnum == qpnum) && 1712 (state->ts_ibtfpriv != NULL)) { 1713 event.ev_qp_hdl = (ibtl_qp_hdl_t)qp->qp_hdlrarg; 1714 type = IBT_ERROR_PATH_MIGRATE_REQ_QP; 1715 1716 TAVOR_DO_IBTF_ASYNC_CALLB(state, type, &event); 1717 } 1718 1719 return (DDI_SUCCESS); 1720 } 1721 1722 1723 /* 1724 * tavor_srq_catastrophic_handler() 1725 * Context: Only called from interrupt context 1726 */ 1727 static int 1728 tavor_srq_catastrophic_handler(tavor_state_t *state, tavor_eqhdl_t eq, 1729 tavor_hw_eqe_t *eqe) 1730 { 1731 tavor_qphdl_t qp; 1732 uint_t qpnum; 1733 ibc_async_event_t event; 1734 ibt_async_code_t type; 1735 uint_t eqe_evttype; 1736 1737 eqe_evttype = TAVOR_EQE_EVTTYPE_GET(eq, eqe); 1738 1739 ASSERT(eqe_evttype == TAVOR_EVT_SRQ_CATASTROPHIC_ERROR || 1740 eqe_evttype == TAVOR_EVT_EQ_OVERFLOW); 1741 1742 if (eqe_evttype == TAVOR_EVT_EQ_OVERFLOW) { 1743 tavor_eq_overflow_handler(state, eq, eqe); 1744 1745 return (DDI_FAILURE); 1746 } 1747 1748 /* Get the QP handle from QP number in event descriptor */ 1749 qpnum = TAVOR_EQE_QPNUM_GET(eq, eqe); 1750 qp = tavor_qphdl_from_qpnum(state, qpnum); 1751 1752 /* 1753 * If the QP handle is NULL, this is probably an indication 1754 * that the QP has been freed already. In which case, we 1755 * should not deliver this event. 1756 * 1757 * We also check that the QP number in the handle is the 1758 * same as the QP number in the event queue entry. This 1759 * extra check allows us to handle the case where a QP was 1760 * freed and then allocated again in the time it took to 1761 * handle the event queue processing. By constantly incrementing 1762 * the non-constrained portion of the QP number every time 1763 * a new QP is allocated, we mitigate (somewhat) the chance 1764 * that a stale event could be passed to the client's QP 1765 * handler. 1766 * 1767 * Lastly, we check if "ts_ibtfpriv" is NULL. If it is then it 1768 * means that we've have either received this event before we 1769 * finished attaching to the IBTF or we've received it while we 1770 * are in the process of detaching. 1771 */ 1772 if ((qp != NULL) && (qp->qp_qpnum == qpnum) && 1773 (state->ts_ibtfpriv != NULL)) { 1774 event.ev_srq_hdl = (ibt_srq_hdl_t)qp->qp_srqhdl->srq_hdlrarg; 1775 type = IBT_ERROR_CATASTROPHIC_SRQ; 1776 1777 mutex_enter(&qp->qp_srqhdl->srq_lock); 1778 qp->qp_srqhdl->srq_state = TAVOR_SRQ_STATE_ERROR; 1779 mutex_exit(&qp->qp_srqhdl->srq_lock); 1780 1781 TAVOR_DO_IBTF_ASYNC_CALLB(state, type, &event); 1782 } 1783 1784 return (DDI_SUCCESS); 1785 } 1786 1787 1788 /* 1789 * tavor_srq_last_wqe_reached_handler() 1790 * Context: Only called from interrupt context 1791 */ 1792 static int 1793 tavor_srq_last_wqe_reached_handler(tavor_state_t *state, tavor_eqhdl_t eq, 1794 tavor_hw_eqe_t *eqe) 1795 { 1796 tavor_qphdl_t qp; 1797 uint_t qpnum; 1798 ibc_async_event_t event; 1799 ibt_async_code_t type; 1800 uint_t eqe_evttype; 1801 1802 eqe_evttype = TAVOR_EQE_EVTTYPE_GET(eq, eqe); 1803 1804 ASSERT(eqe_evttype == TAVOR_EVT_SRQ_LAST_WQE_REACHED || 1805 eqe_evttype == TAVOR_EVT_EQ_OVERFLOW); 1806 1807 if (eqe_evttype == TAVOR_EVT_EQ_OVERFLOW) { 1808 tavor_eq_overflow_handler(state, eq, eqe); 1809 1810 return (DDI_FAILURE); 1811 } 1812 1813 /* Get the QP handle from QP number in event descriptor */ 1814 qpnum = TAVOR_EQE_QPNUM_GET(eq, eqe); 1815 qp = tavor_qphdl_from_qpnum(state, qpnum); 1816 1817 /* 1818 * If the QP handle is NULL, this is probably an indication 1819 * that the QP has been freed already. In which case, we 1820 * should not deliver this event. 1821 * 1822 * We also check that the QP number in the handle is the 1823 * same as the QP number in the event queue entry. This 1824 * extra check allows us to handle the case where a QP was 1825 * freed and then allocated again in the time it took to 1826 * handle the event queue processing. By constantly incrementing 1827 * the non-constrained portion of the QP number every time 1828 * a new QP is allocated, we mitigate (somewhat) the chance 1829 * that a stale event could be passed to the client's QP 1830 * handler. 1831 * 1832 * Lastly, we check if "ts_ibtfpriv" is NULL. If it is then it 1833 * means that we've have either received this event before we 1834 * finished attaching to the IBTF or we've received it while we 1835 * are in the process of detaching. 1836 */ 1837 if ((qp != NULL) && (qp->qp_qpnum == qpnum) && 1838 (state->ts_ibtfpriv != NULL)) { 1839 event.ev_qp_hdl = (ibtl_qp_hdl_t)qp->qp_hdlrarg; 1840 type = IBT_EVENT_EMPTY_CHAN; 1841 1842 TAVOR_DO_IBTF_ASYNC_CALLB(state, type, &event); 1843 } 1844 1845 return (DDI_SUCCESS); 1846 } 1847 1848 1849 /* 1850 * tavor_ecc_detection_handler() 1851 * Context: Only called from interrupt context 1852 */ 1853 static int 1854 tavor_ecc_detection_handler(tavor_state_t *state, tavor_eqhdl_t eq, 1855 tavor_hw_eqe_t *eqe) 1856 { 1857 uint_t eqe_evttype; 1858 uint_t data; 1859 int i; 1860 1861 eqe_evttype = TAVOR_EQE_EVTTYPE_GET(eq, eqe); 1862 1863 ASSERT(eqe_evttype == TAVOR_EVT_ECC_DETECTION || 1864 eqe_evttype == TAVOR_EVT_EQ_OVERFLOW); 1865 1866 if (eqe_evttype == TAVOR_EVT_EQ_OVERFLOW) { 1867 tavor_eq_overflow_handler(state, eq, eqe); 1868 1869 return (DDI_FAILURE); 1870 } 1871 1872 /* 1873 * The "ECC Detection Event" indicates that a correctable single-bit 1874 * has occurred with the attached DDR. The EQE provides some 1875 * additional information about the errored EQ. So we print a warning 1876 * message here along with that additional information. 1877 */ 1878 TAVOR_WARNING(state, "ECC Correctable Error Event Detected"); 1879 for (i = 0; i < sizeof (tavor_hw_eqe_t) >> 2; i++) { 1880 data = ((uint_t *)eqe)[i]; 1881 cmn_err(CE_CONT, "! EQE[%02x]: %08x\n", i, data); 1882 } 1883 1884 return (DDI_SUCCESS); 1885 } 1886 1887 1888 /* 1889 * tavor_eq_overflow_handler() 1890 * Context: Only called from interrupt context 1891 */ 1892 void 1893 tavor_eq_overflow_handler(tavor_state_t *state, tavor_eqhdl_t eq, 1894 tavor_hw_eqe_t *eqe) 1895 { 1896 uint_t error_type, data; 1897 1898 ASSERT(TAVOR_EQE_EVTTYPE_GET(eq, eqe) == TAVOR_EVT_EQ_OVERFLOW); 1899 1900 /* 1901 * The "Event Queue Overflow Event" indicates that something has 1902 * probably gone seriously wrong with some hardware (or, perhaps, 1903 * with the software... though it's unlikely in this case). The EQE 1904 * provides some additional information about the errored EQ. So we 1905 * print a warning message here along with that additional information. 1906 */ 1907 error_type = TAVOR_EQE_OPERRTYPE_GET(eq, eqe); 1908 data = TAVOR_EQE_OPERRDATA_GET(eq, eqe); 1909 1910 TAVOR_WARNING(state, "Event Queue overflow"); 1911 cmn_err(CE_CONT, " Error type: %02x, data: %08x\n", error_type, data); 1912 } 1913 1914 1915 /* 1916 * tavor_no_eqhandler 1917 * Context: Only called from interrupt context 1918 */ 1919 /* ARGSUSED */ 1920 static int 1921 tavor_no_eqhandler(tavor_state_t *state, tavor_eqhdl_t eq, 1922 tavor_hw_eqe_t *eqe) 1923 { 1924 uint_t data; 1925 int i; 1926 1927 /* 1928 * This "unexpected event" handler (or "catch-all" handler) will 1929 * receive all events for which no other handler has been registered. 1930 * If we end up here, then something has probably gone seriously wrong 1931 * with the Tavor hardware (or, perhaps, with the software... though 1932 * it's unlikely in this case). The EQE provides all the information 1933 * about the event. So we print a warning message here along with 1934 * the contents of the EQE. 1935 */ 1936 TAVOR_WARNING(state, "Unexpected Event handler"); 1937 cmn_err(CE_CONT, " Event type: %02x, subtype: %02x\n", 1938 TAVOR_EQE_EVTTYPE_GET(eq, eqe), TAVOR_EQE_EVTSUBTYPE_GET(eq, eqe)); 1939 for (i = 0; i < sizeof (tavor_hw_eqe_t) >> 2; i++) { 1940 data = ((uint_t *)eqe)[i]; 1941 cmn_err(CE_CONT, " EQE[%02x]: %08x\n", i, data); 1942 } 1943 1944 return (DDI_SUCCESS); 1945 } 1946