1 /*- 2 * Copyright (c) 2010-2015 Solarflare Communications Inc. 3 * All rights reserved. 4 * 5 * This software was developed in part by Philip Paeps under contract for 6 * Solarflare Communications, Inc. 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions are met: 10 * 11 * 1. Redistributions of source code must retain the above copyright notice, 12 * this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright notice, 14 * this list of conditions and the following disclaimer in the documentation 15 * and/or other materials provided with the distribution. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 18 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, 19 * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 20 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR 21 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, 22 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 23 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; 24 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 25 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR 26 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, 27 * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 * 29 * The views and conclusions contained in the software and documentation are 30 * those of the authors and should not be interpreted as representing official 31 * policies, either expressed or implied, of the FreeBSD Project. 32 */ 33 34 #include <sys/cdefs.h> 35 __FBSDID("$FreeBSD$"); 36 37 #include <sys/param.h> 38 #include <sys/kernel.h> 39 #include <sys/malloc.h> 40 #include <sys/param.h> 41 #include <sys/queue.h> 42 #include <sys/systm.h> 43 #include <sys/taskqueue.h> 44 45 #include "common/efx.h" 46 47 #include "sfxge.h" 48 49 static void 50 sfxge_ev_qcomplete(struct sfxge_evq *evq, boolean_t eop) 51 { 52 struct sfxge_softc *sc; 53 unsigned int index; 54 struct sfxge_rxq *rxq; 55 struct sfxge_txq *txq; 56 57 SFXGE_EVQ_LOCK_ASSERT_OWNED(evq); 58 59 sc = evq->sc; 60 index = evq->index; 61 rxq = sc->rxq[index]; 62 63 if ((txq = evq->txq) != NULL) { 64 evq->txq = NULL; 65 evq->txqs = &(evq->txq); 66 67 do { 68 struct sfxge_txq *next; 69 70 next = txq->next; 71 txq->next = NULL; 72 73 KASSERT(txq->evq_index == index, 74 ("txq->evq_index != index")); 75 76 if (txq->pending != txq->completed) 77 sfxge_tx_qcomplete(txq, evq); 78 79 txq = next; 80 } while (txq != NULL); 81 } 82 83 if (rxq->pending != rxq->completed) 84 sfxge_rx_qcomplete(rxq, eop); 85 } 86 87 static struct sfxge_rxq * 88 sfxge_get_rxq_by_label(struct sfxge_evq *evq, uint32_t label) 89 { 90 struct sfxge_rxq *rxq; 91 92 KASSERT(label == 0, ("unexpected rxq label != 0")); 93 94 rxq = evq->sc->rxq[evq->index]; 95 96 KASSERT(rxq != NULL, ("rxq == NULL")); 97 KASSERT(evq->index == rxq->index, ("evq->index != rxq->index")); 98 99 return (rxq); 100 } 101 102 static boolean_t 103 sfxge_ev_rx(void *arg, uint32_t label, uint32_t id, uint32_t size, 104 uint16_t flags) 105 { 106 struct sfxge_evq *evq; 107 struct sfxge_softc *sc; 108 struct sfxge_rxq *rxq; 109 unsigned int stop; 110 unsigned int delta; 111 struct sfxge_rx_sw_desc *rx_desc; 112 113 evq = arg; 114 SFXGE_EVQ_LOCK_ASSERT_OWNED(evq); 115 116 sc = evq->sc; 117 118 if (evq->exception) 119 goto done; 120 121 rxq = sfxge_get_rxq_by_label(evq, label); 122 if (__predict_false(rxq->init_state != SFXGE_RXQ_STARTED)) 123 goto done; 124 125 stop = (id + 1) & rxq->ptr_mask; 126 id = rxq->pending & rxq->ptr_mask; 127 delta = (stop >= id) ? (stop - id) : (rxq->entries - id + stop); 128 rxq->pending += delta; 129 130 if (delta != 1) { 131 if ((!efx_nic_cfg_get(sc->enp)->enc_rx_batching_enabled) || 132 (delta <= 0) || 133 (delta > efx_nic_cfg_get(sc->enp)->enc_rx_batch_max)) { 134 evq->exception = B_TRUE; 135 136 device_printf(sc->dev, "RX completion out of order" 137 " (id=%#x delta=%u flags=%#x); resetting\n", 138 id, delta, flags); 139 sfxge_schedule_reset(sc); 140 141 goto done; 142 } 143 } 144 145 rx_desc = &rxq->queue[id]; 146 147 prefetch_read_many(rx_desc->mbuf); 148 149 for (; id != stop; id = (id + 1) & rxq->ptr_mask) { 150 rx_desc = &rxq->queue[id]; 151 KASSERT(rx_desc->flags == EFX_DISCARD, 152 ("rx_desc->flags != EFX_DISCARD")); 153 rx_desc->flags = flags; 154 155 KASSERT(size < (1 << 16), ("size > (1 << 16)")); 156 rx_desc->size = (uint16_t)size; 157 } 158 159 evq->rx_done++; 160 161 if (rxq->pending - rxq->completed >= SFXGE_RX_BATCH) 162 sfxge_ev_qcomplete(evq, B_FALSE); 163 164 done: 165 return (evq->rx_done >= SFXGE_EV_BATCH); 166 } 167 168 static boolean_t 169 sfxge_ev_exception(void *arg, uint32_t code, uint32_t data) 170 { 171 struct sfxge_evq *evq; 172 struct sfxge_softc *sc; 173 174 evq = (struct sfxge_evq *)arg; 175 SFXGE_EVQ_LOCK_ASSERT_OWNED(evq); 176 177 sc = evq->sc; 178 179 DBGPRINT(sc->dev, "[%d] %s", evq->index, 180 (code == EFX_EXCEPTION_RX_RECOVERY) ? "RX_RECOVERY" : 181 (code == EFX_EXCEPTION_RX_DSC_ERROR) ? "RX_DSC_ERROR" : 182 (code == EFX_EXCEPTION_TX_DSC_ERROR) ? "TX_DSC_ERROR" : 183 (code == EFX_EXCEPTION_UNKNOWN_SENSOREVT) ? "UNKNOWN_SENSOREVT" : 184 (code == EFX_EXCEPTION_FWALERT_SRAM) ? "FWALERT_SRAM" : 185 (code == EFX_EXCEPTION_UNKNOWN_FWALERT) ? "UNKNOWN_FWALERT" : 186 (code == EFX_EXCEPTION_RX_ERROR) ? "RX_ERROR" : 187 (code == EFX_EXCEPTION_TX_ERROR) ? "TX_ERROR" : 188 (code == EFX_EXCEPTION_EV_ERROR) ? "EV_ERROR" : 189 "UNKNOWN"); 190 191 evq->exception = B_TRUE; 192 193 if (code != EFX_EXCEPTION_UNKNOWN_SENSOREVT) { 194 device_printf(sc->dev, 195 "hardware exception (code=%u); resetting\n", 196 code); 197 sfxge_schedule_reset(sc); 198 } 199 200 return (B_FALSE); 201 } 202 203 static boolean_t 204 sfxge_ev_rxq_flush_done(void *arg, uint32_t rxq_index) 205 { 206 struct sfxge_evq *evq; 207 struct sfxge_softc *sc; 208 struct sfxge_rxq *rxq; 209 unsigned int index; 210 unsigned int label; 211 uint16_t magic; 212 213 evq = (struct sfxge_evq *)arg; 214 SFXGE_EVQ_LOCK_ASSERT_OWNED(evq); 215 216 sc = evq->sc; 217 rxq = sc->rxq[rxq_index]; 218 219 KASSERT(rxq != NULL, ("rxq == NULL")); 220 221 /* Resend a software event on the correct queue */ 222 index = rxq->index; 223 if (index == evq->index) { 224 sfxge_rx_qflush_done(rxq); 225 return (B_FALSE); 226 } 227 228 evq = sc->evq[index]; 229 230 label = 0; 231 KASSERT((label & SFXGE_MAGIC_DMAQ_LABEL_MASK) == label, 232 ("(label & SFXGE_MAGIC_DMAQ_LABEL_MASK) != level")); 233 magic = SFXGE_MAGIC_RX_QFLUSH_DONE | label; 234 235 KASSERT(evq->init_state == SFXGE_EVQ_STARTED, 236 ("evq not started")); 237 efx_ev_qpost(evq->common, magic); 238 239 return (B_FALSE); 240 } 241 242 static boolean_t 243 sfxge_ev_rxq_flush_failed(void *arg, uint32_t rxq_index) 244 { 245 struct sfxge_evq *evq; 246 struct sfxge_softc *sc; 247 struct sfxge_rxq *rxq; 248 unsigned int index; 249 unsigned int label; 250 uint16_t magic; 251 252 evq = (struct sfxge_evq *)arg; 253 SFXGE_EVQ_LOCK_ASSERT_OWNED(evq); 254 255 sc = evq->sc; 256 rxq = sc->rxq[rxq_index]; 257 258 KASSERT(rxq != NULL, ("rxq == NULL")); 259 260 /* Resend a software event on the correct queue */ 261 index = rxq->index; 262 evq = sc->evq[index]; 263 264 label = 0; 265 KASSERT((label & SFXGE_MAGIC_DMAQ_LABEL_MASK) == label, 266 ("(label & SFXGE_MAGIC_DMAQ_LABEL_MASK) != label")); 267 magic = SFXGE_MAGIC_RX_QFLUSH_FAILED | label; 268 269 KASSERT(evq->init_state == SFXGE_EVQ_STARTED, 270 ("evq not started")); 271 efx_ev_qpost(evq->common, magic); 272 273 return (B_FALSE); 274 } 275 276 static struct sfxge_txq * 277 sfxge_get_txq_by_label(struct sfxge_evq *evq, enum sfxge_txq_type label) 278 { 279 unsigned int index; 280 281 KASSERT((evq->index == 0 && label < SFXGE_TXQ_NTYPES) || 282 (label == SFXGE_TXQ_IP_TCP_UDP_CKSUM), ("unexpected txq label")); 283 index = (evq->index == 0) ? label : (evq->index - 1 + SFXGE_TXQ_NTYPES); 284 return (evq->sc->txq[index]); 285 } 286 287 static boolean_t 288 sfxge_ev_tx(void *arg, uint32_t label, uint32_t id) 289 { 290 struct sfxge_evq *evq; 291 struct sfxge_txq *txq; 292 unsigned int stop; 293 unsigned int delta; 294 295 evq = (struct sfxge_evq *)arg; 296 SFXGE_EVQ_LOCK_ASSERT_OWNED(evq); 297 298 txq = sfxge_get_txq_by_label(evq, label); 299 300 KASSERT(txq != NULL, ("txq == NULL")); 301 KASSERT(evq->index == txq->evq_index, 302 ("evq->index != txq->evq_index")); 303 304 if (__predict_false(txq->init_state != SFXGE_TXQ_STARTED)) 305 goto done; 306 307 stop = (id + 1) & txq->ptr_mask; 308 id = txq->pending & txq->ptr_mask; 309 310 delta = (stop >= id) ? (stop - id) : (txq->entries - id + stop); 311 txq->pending += delta; 312 313 evq->tx_done++; 314 315 if (txq->next == NULL && 316 evq->txqs != &(txq->next)) { 317 *(evq->txqs) = txq; 318 evq->txqs = &(txq->next); 319 } 320 321 if (txq->pending - txq->completed >= SFXGE_TX_BATCH) 322 sfxge_tx_qcomplete(txq, evq); 323 324 done: 325 return (evq->tx_done >= SFXGE_EV_BATCH); 326 } 327 328 static boolean_t 329 sfxge_ev_txq_flush_done(void *arg, uint32_t txq_index) 330 { 331 struct sfxge_evq *evq; 332 struct sfxge_softc *sc; 333 struct sfxge_txq *txq; 334 unsigned int label; 335 uint16_t magic; 336 337 evq = (struct sfxge_evq *)arg; 338 SFXGE_EVQ_LOCK_ASSERT_OWNED(evq); 339 340 sc = evq->sc; 341 txq = sc->txq[txq_index]; 342 343 KASSERT(txq != NULL, ("txq == NULL")); 344 KASSERT(txq->init_state == SFXGE_TXQ_INITIALIZED, 345 ("txq not initialized")); 346 347 if (txq->evq_index == evq->index) { 348 sfxge_tx_qflush_done(txq); 349 return (B_FALSE); 350 } 351 352 /* Resend a software event on the correct queue */ 353 evq = sc->evq[txq->evq_index]; 354 355 label = txq->type; 356 KASSERT((label & SFXGE_MAGIC_DMAQ_LABEL_MASK) == label, 357 ("(label & SFXGE_MAGIC_DMAQ_LABEL_MASK) != label")); 358 magic = SFXGE_MAGIC_TX_QFLUSH_DONE | label; 359 360 KASSERT(evq->init_state == SFXGE_EVQ_STARTED, 361 ("evq not started")); 362 efx_ev_qpost(evq->common, magic); 363 364 return (B_FALSE); 365 } 366 367 static boolean_t 368 sfxge_ev_software(void *arg, uint16_t magic) 369 { 370 struct sfxge_evq *evq; 371 struct sfxge_softc *sc; 372 unsigned int label; 373 374 evq = (struct sfxge_evq *)arg; 375 SFXGE_EVQ_LOCK_ASSERT_OWNED(evq); 376 377 sc = evq->sc; 378 379 label = magic & SFXGE_MAGIC_DMAQ_LABEL_MASK; 380 magic &= ~SFXGE_MAGIC_DMAQ_LABEL_MASK; 381 382 switch (magic) { 383 case SFXGE_MAGIC_RX_QFLUSH_DONE: 384 sfxge_rx_qflush_done(sfxge_get_rxq_by_label(evq, label)); 385 break; 386 387 case SFXGE_MAGIC_RX_QFLUSH_FAILED: 388 sfxge_rx_qflush_failed(sfxge_get_rxq_by_label(evq, label)); 389 break; 390 391 case SFXGE_MAGIC_RX_QREFILL: 392 sfxge_rx_qrefill(sfxge_get_rxq_by_label(evq, label)); 393 break; 394 395 case SFXGE_MAGIC_TX_QFLUSH_DONE: { 396 struct sfxge_txq *txq = sfxge_get_txq_by_label(evq, label); 397 398 KASSERT(txq != NULL, ("txq == NULL")); 399 KASSERT(evq->index == txq->evq_index, 400 ("evq->index != txq->evq_index")); 401 402 sfxge_tx_qflush_done(txq); 403 break; 404 } 405 default: 406 break; 407 } 408 409 return (B_FALSE); 410 } 411 412 static boolean_t 413 sfxge_ev_sram(void *arg, uint32_t code) 414 { 415 (void)arg; 416 (void)code; 417 418 switch (code) { 419 case EFX_SRAM_UPDATE: 420 EFSYS_PROBE(sram_update); 421 break; 422 423 case EFX_SRAM_CLEAR: 424 EFSYS_PROBE(sram_clear); 425 break; 426 427 case EFX_SRAM_ILLEGAL_CLEAR: 428 EFSYS_PROBE(sram_illegal_clear); 429 break; 430 431 default: 432 KASSERT(B_FALSE, ("Impossible SRAM event")); 433 break; 434 } 435 436 return (B_FALSE); 437 } 438 439 static boolean_t 440 sfxge_ev_timer(void *arg, uint32_t index) 441 { 442 (void)arg; 443 (void)index; 444 445 return (B_FALSE); 446 } 447 448 static boolean_t 449 sfxge_ev_wake_up(void *arg, uint32_t index) 450 { 451 (void)arg; 452 (void)index; 453 454 return (B_FALSE); 455 } 456 457 #if EFSYS_OPT_QSTATS 458 459 static void 460 sfxge_ev_stat_update(struct sfxge_softc *sc) 461 { 462 struct sfxge_evq *evq; 463 unsigned int index; 464 clock_t now; 465 466 SFXGE_ADAPTER_LOCK(sc); 467 468 if (__predict_false(sc->evq[0]->init_state != SFXGE_EVQ_STARTED)) 469 goto out; 470 471 now = ticks; 472 if (now - sc->ev_stats_update_time < hz) 473 goto out; 474 475 sc->ev_stats_update_time = now; 476 477 /* Add event counts from each event queue in turn */ 478 for (index = 0; index < sc->evq_count; index++) { 479 evq = sc->evq[index]; 480 SFXGE_EVQ_LOCK(evq); 481 efx_ev_qstats_update(evq->common, sc->ev_stats); 482 SFXGE_EVQ_UNLOCK(evq); 483 } 484 out: 485 SFXGE_ADAPTER_UNLOCK(sc); 486 } 487 488 static int 489 sfxge_ev_stat_handler(SYSCTL_HANDLER_ARGS) 490 { 491 struct sfxge_softc *sc = arg1; 492 unsigned int id = arg2; 493 494 sfxge_ev_stat_update(sc); 495 496 return (SYSCTL_OUT(req, &sc->ev_stats[id], sizeof(sc->ev_stats[id]))); 497 } 498 499 static void 500 sfxge_ev_stat_init(struct sfxge_softc *sc) 501 { 502 struct sysctl_ctx_list *ctx = device_get_sysctl_ctx(sc->dev); 503 struct sysctl_oid_list *stat_list; 504 unsigned int id; 505 char name[40]; 506 507 stat_list = SYSCTL_CHILDREN(sc->stats_node); 508 509 for (id = 0; id < EV_NQSTATS; id++) { 510 snprintf(name, sizeof(name), "ev_%s", 511 efx_ev_qstat_name(sc->enp, id)); 512 SYSCTL_ADD_PROC( 513 ctx, stat_list, 514 OID_AUTO, name, CTLTYPE_U64|CTLFLAG_RD, 515 sc, id, sfxge_ev_stat_handler, "Q", 516 ""); 517 } 518 } 519 520 #endif /* EFSYS_OPT_QSTATS */ 521 522 static void 523 sfxge_ev_qmoderate(struct sfxge_softc *sc, unsigned int idx, unsigned int us) 524 { 525 struct sfxge_evq *evq; 526 efx_evq_t *eep; 527 528 evq = sc->evq[idx]; 529 eep = evq->common; 530 531 KASSERT(evq->init_state == SFXGE_EVQ_STARTED, 532 ("evq->init_state != SFXGE_EVQ_STARTED")); 533 534 (void)efx_ev_qmoderate(eep, us); 535 } 536 537 static int 538 sfxge_int_mod_handler(SYSCTL_HANDLER_ARGS) 539 { 540 struct sfxge_softc *sc = arg1; 541 struct sfxge_intr *intr = &sc->intr; 542 unsigned int moderation; 543 int error; 544 unsigned int index; 545 546 SFXGE_ADAPTER_LOCK(sc); 547 548 if (req->newptr != NULL) { 549 if ((error = SYSCTL_IN(req, &moderation, sizeof(moderation))) 550 != 0) 551 goto out; 552 553 /* We may not be calling efx_ev_qmoderate() now, 554 * so we have to range-check the value ourselves. 555 */ 556 if (moderation > 557 efx_nic_cfg_get(sc->enp)->enc_evq_timer_max_us) { 558 error = EINVAL; 559 goto out; 560 } 561 562 sc->ev_moderation = moderation; 563 if (intr->state == SFXGE_INTR_STARTED) { 564 for (index = 0; index < sc->evq_count; index++) 565 sfxge_ev_qmoderate(sc, index, moderation); 566 } 567 } else { 568 error = SYSCTL_OUT(req, &sc->ev_moderation, 569 sizeof(sc->ev_moderation)); 570 } 571 572 out: 573 SFXGE_ADAPTER_UNLOCK(sc); 574 575 return (error); 576 } 577 578 static boolean_t 579 sfxge_ev_initialized(void *arg) 580 { 581 struct sfxge_evq *evq; 582 583 evq = (struct sfxge_evq *)arg; 584 SFXGE_EVQ_LOCK_ASSERT_OWNED(evq); 585 586 /* Init done events may be duplicated on 7xxx */ 587 KASSERT(evq->init_state == SFXGE_EVQ_STARTING || 588 evq->init_state == SFXGE_EVQ_STARTED, 589 ("evq not starting")); 590 591 evq->init_state = SFXGE_EVQ_STARTED; 592 593 return (0); 594 } 595 596 static boolean_t 597 sfxge_ev_link_change(void *arg, efx_link_mode_t link_mode) 598 { 599 struct sfxge_evq *evq; 600 struct sfxge_softc *sc; 601 602 evq = (struct sfxge_evq *)arg; 603 SFXGE_EVQ_LOCK_ASSERT_OWNED(evq); 604 605 sc = evq->sc; 606 607 sfxge_mac_link_update(sc, link_mode); 608 609 return (0); 610 } 611 612 static const efx_ev_callbacks_t sfxge_ev_callbacks = { 613 .eec_initialized = sfxge_ev_initialized, 614 .eec_rx = sfxge_ev_rx, 615 .eec_tx = sfxge_ev_tx, 616 .eec_exception = sfxge_ev_exception, 617 .eec_rxq_flush_done = sfxge_ev_rxq_flush_done, 618 .eec_rxq_flush_failed = sfxge_ev_rxq_flush_failed, 619 .eec_txq_flush_done = sfxge_ev_txq_flush_done, 620 .eec_software = sfxge_ev_software, 621 .eec_sram = sfxge_ev_sram, 622 .eec_wake_up = sfxge_ev_wake_up, 623 .eec_timer = sfxge_ev_timer, 624 .eec_link_change = sfxge_ev_link_change, 625 }; 626 627 628 int 629 sfxge_ev_qpoll(struct sfxge_evq *evq) 630 { 631 int rc; 632 633 SFXGE_EVQ_LOCK(evq); 634 635 if (__predict_false(evq->init_state != SFXGE_EVQ_STARTING && 636 evq->init_state != SFXGE_EVQ_STARTED)) { 637 rc = EINVAL; 638 goto fail; 639 } 640 641 /* Synchronize the DMA memory for reading */ 642 bus_dmamap_sync(evq->mem.esm_tag, evq->mem.esm_map, 643 BUS_DMASYNC_POSTREAD); 644 645 KASSERT(evq->rx_done == 0, ("evq->rx_done != 0")); 646 KASSERT(evq->tx_done == 0, ("evq->tx_done != 0")); 647 KASSERT(evq->txq == NULL, ("evq->txq != NULL")); 648 KASSERT(evq->txqs == &evq->txq, ("evq->txqs != &evq->txq")); 649 650 /* Poll the queue */ 651 efx_ev_qpoll(evq->common, &evq->read_ptr, &sfxge_ev_callbacks, evq); 652 653 evq->rx_done = 0; 654 evq->tx_done = 0; 655 656 /* Perform any pending completion processing */ 657 sfxge_ev_qcomplete(evq, B_TRUE); 658 659 /* Re-prime the event queue for interrupts */ 660 if ((rc = efx_ev_qprime(evq->common, evq->read_ptr)) != 0) 661 goto fail; 662 663 SFXGE_EVQ_UNLOCK(evq); 664 665 return (0); 666 667 fail: 668 SFXGE_EVQ_UNLOCK(evq); 669 return (rc); 670 } 671 672 static void 673 sfxge_ev_qstop(struct sfxge_softc *sc, unsigned int index) 674 { 675 struct sfxge_evq *evq; 676 677 evq = sc->evq[index]; 678 679 KASSERT(evq->init_state == SFXGE_EVQ_STARTED, 680 ("evq->init_state != SFXGE_EVQ_STARTED")); 681 682 SFXGE_EVQ_LOCK(evq); 683 evq->init_state = SFXGE_EVQ_INITIALIZED; 684 evq->read_ptr = 0; 685 evq->exception = B_FALSE; 686 687 #if EFSYS_OPT_QSTATS 688 /* Add event counts before discarding the common evq state */ 689 efx_ev_qstats_update(evq->common, sc->ev_stats); 690 #endif 691 692 efx_ev_qdestroy(evq->common); 693 efx_sram_buf_tbl_clear(sc->enp, evq->buf_base_id, 694 EFX_EVQ_NBUFS(evq->entries)); 695 SFXGE_EVQ_UNLOCK(evq); 696 } 697 698 static int 699 sfxge_ev_qstart(struct sfxge_softc *sc, unsigned int index) 700 { 701 struct sfxge_evq *evq; 702 efsys_mem_t *esmp; 703 int count; 704 int rc; 705 706 evq = sc->evq[index]; 707 esmp = &evq->mem; 708 709 KASSERT(evq->init_state == SFXGE_EVQ_INITIALIZED, 710 ("evq->init_state != SFXGE_EVQ_INITIALIZED")); 711 712 /* Clear all events. */ 713 (void)memset(esmp->esm_base, 0xff, EFX_EVQ_SIZE(evq->entries)); 714 715 /* Program the buffer table. */ 716 if ((rc = efx_sram_buf_tbl_set(sc->enp, evq->buf_base_id, esmp, 717 EFX_EVQ_NBUFS(evq->entries))) != 0) 718 return (rc); 719 720 /* Create the common code event queue. */ 721 if ((rc = efx_ev_qcreate(sc->enp, index, esmp, evq->entries, 722 evq->buf_base_id, &evq->common)) != 0) 723 goto fail; 724 725 SFXGE_EVQ_LOCK(evq); 726 727 /* Set the default moderation */ 728 (void)efx_ev_qmoderate(evq->common, sc->ev_moderation); 729 730 /* Prime the event queue for interrupts */ 731 if ((rc = efx_ev_qprime(evq->common, evq->read_ptr)) != 0) 732 goto fail2; 733 734 evq->init_state = SFXGE_EVQ_STARTING; 735 736 SFXGE_EVQ_UNLOCK(evq); 737 738 /* Wait for the initialization event */ 739 count = 0; 740 do { 741 /* Pause for 100 ms */ 742 pause("sfxge evq init", hz / 10); 743 744 /* Check to see if the test event has been processed */ 745 if (evq->init_state == SFXGE_EVQ_STARTED) 746 goto done; 747 748 } while (++count < 20); 749 750 rc = ETIMEDOUT; 751 goto fail3; 752 753 done: 754 return (0); 755 756 fail3: 757 SFXGE_EVQ_LOCK(evq); 758 evq->init_state = SFXGE_EVQ_INITIALIZED; 759 fail2: 760 SFXGE_EVQ_UNLOCK(evq); 761 efx_ev_qdestroy(evq->common); 762 fail: 763 efx_sram_buf_tbl_clear(sc->enp, evq->buf_base_id, 764 EFX_EVQ_NBUFS(evq->entries)); 765 766 return (rc); 767 } 768 769 void 770 sfxge_ev_stop(struct sfxge_softc *sc) 771 { 772 struct sfxge_intr *intr; 773 efx_nic_t *enp; 774 int index; 775 776 intr = &sc->intr; 777 enp = sc->enp; 778 779 KASSERT(intr->state == SFXGE_INTR_STARTED, 780 ("Interrupts not started")); 781 782 /* Stop the event queue(s) */ 783 index = sc->evq_count; 784 while (--index >= 0) 785 sfxge_ev_qstop(sc, index); 786 787 /* Tear down the event module */ 788 efx_ev_fini(enp); 789 } 790 791 int 792 sfxge_ev_start(struct sfxge_softc *sc) 793 { 794 struct sfxge_intr *intr; 795 int index; 796 int rc; 797 798 intr = &sc->intr; 799 800 KASSERT(intr->state == SFXGE_INTR_STARTED, 801 ("intr->state != SFXGE_INTR_STARTED")); 802 803 /* Initialize the event module */ 804 if ((rc = efx_ev_init(sc->enp)) != 0) 805 return (rc); 806 807 /* Start the event queues */ 808 for (index = 0; index < sc->evq_count; index++) { 809 if ((rc = sfxge_ev_qstart(sc, index)) != 0) 810 goto fail; 811 } 812 813 return (0); 814 815 fail: 816 /* Stop the event queue(s) */ 817 while (--index >= 0) 818 sfxge_ev_qstop(sc, index); 819 820 /* Tear down the event module */ 821 efx_ev_fini(sc->enp); 822 823 return (rc); 824 } 825 826 static void 827 sfxge_ev_qfini(struct sfxge_softc *sc, unsigned int index) 828 { 829 struct sfxge_evq *evq; 830 831 evq = sc->evq[index]; 832 833 KASSERT(evq->init_state == SFXGE_EVQ_INITIALIZED, 834 ("evq->init_state != SFXGE_EVQ_INITIALIZED")); 835 KASSERT(evq->txqs == &evq->txq, ("evq->txqs != &evq->txq")); 836 837 sfxge_dma_free(&evq->mem); 838 839 sc->evq[index] = NULL; 840 841 SFXGE_EVQ_LOCK_DESTROY(evq); 842 843 free(evq, M_SFXGE); 844 } 845 846 static int 847 sfxge_ev_qinit(struct sfxge_softc *sc, unsigned int index) 848 { 849 struct sfxge_evq *evq; 850 efsys_mem_t *esmp; 851 int rc; 852 853 KASSERT(index < SFXGE_RX_SCALE_MAX, ("index >= SFXGE_RX_SCALE_MAX")); 854 855 evq = malloc(sizeof(struct sfxge_evq), M_SFXGE, M_ZERO | M_WAITOK); 856 evq->sc = sc; 857 evq->index = index; 858 sc->evq[index] = evq; 859 esmp = &evq->mem; 860 861 /* Build an event queue with room for one event per tx and rx buffer, 862 * plus some extra for link state events and MCDI completions. 863 * There are three tx queues in the first event queue and one in 864 * other. 865 */ 866 if (index == 0) 867 evq->entries = 868 ROUNDUP_POW_OF_TWO(sc->rxq_entries + 869 3 * sc->txq_entries + 870 128); 871 else 872 evq->entries = 873 ROUNDUP_POW_OF_TWO(sc->rxq_entries + 874 sc->txq_entries + 875 128); 876 877 /* Initialise TX completion list */ 878 evq->txqs = &evq->txq; 879 880 /* Allocate DMA space. */ 881 if ((rc = sfxge_dma_alloc(sc, EFX_EVQ_SIZE(evq->entries), esmp)) != 0) 882 return (rc); 883 884 /* Allocate buffer table entries. */ 885 sfxge_sram_buf_tbl_alloc(sc, EFX_EVQ_NBUFS(evq->entries), 886 &evq->buf_base_id); 887 888 SFXGE_EVQ_LOCK_INIT(evq, device_get_nameunit(sc->dev), index); 889 890 evq->init_state = SFXGE_EVQ_INITIALIZED; 891 892 return (0); 893 } 894 895 void 896 sfxge_ev_fini(struct sfxge_softc *sc) 897 { 898 struct sfxge_intr *intr; 899 int index; 900 901 intr = &sc->intr; 902 903 KASSERT(intr->state == SFXGE_INTR_INITIALIZED, 904 ("intr->state != SFXGE_INTR_INITIALIZED")); 905 906 sc->ev_moderation = 0; 907 908 /* Tear down the event queue(s). */ 909 index = sc->evq_count; 910 while (--index >= 0) 911 sfxge_ev_qfini(sc, index); 912 913 sc->evq_count = 0; 914 } 915 916 int 917 sfxge_ev_init(struct sfxge_softc *sc) 918 { 919 struct sysctl_ctx_list *sysctl_ctx = device_get_sysctl_ctx(sc->dev); 920 struct sysctl_oid *sysctl_tree = device_get_sysctl_tree(sc->dev); 921 struct sfxge_intr *intr; 922 int index; 923 int rc; 924 925 intr = &sc->intr; 926 927 sc->evq_count = intr->n_alloc; 928 929 KASSERT(intr->state == SFXGE_INTR_INITIALIZED, 930 ("intr->state != SFXGE_INTR_INITIALIZED")); 931 932 /* Set default interrupt moderation; add a sysctl to 933 * read and change it. 934 */ 935 sc->ev_moderation = SFXGE_MODERATION; 936 SYSCTL_ADD_PROC(sysctl_ctx, SYSCTL_CHILDREN(sysctl_tree), 937 OID_AUTO, "int_mod", CTLTYPE_UINT|CTLFLAG_RW, 938 sc, 0, sfxge_int_mod_handler, "IU", 939 "sfxge interrupt moderation (us)"); 940 941 /* 942 * Initialize the event queue(s) - one per interrupt. 943 */ 944 for (index = 0; index < sc->evq_count; index++) { 945 if ((rc = sfxge_ev_qinit(sc, index)) != 0) 946 goto fail; 947 } 948 949 #if EFSYS_OPT_QSTATS 950 sfxge_ev_stat_init(sc); 951 #endif 952 953 return (0); 954 955 fail: 956 while (--index >= 0) 957 sfxge_ev_qfini(sc, index); 958 959 sc->evq_count = 0; 960 return (rc); 961 } 962