1 /*- 2 * Copyright (c) 2010-2011 Solarflare Communications, Inc. 3 * All rights reserved. 4 * 5 * This software was developed in part by Philip Paeps under contract for 6 * Solarflare Communications, Inc. 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 21 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 27 * SUCH DAMAGE. 28 */ 29 30 #include <sys/cdefs.h> 31 __FBSDID("$FreeBSD$"); 32 33 #include <sys/param.h> 34 #include <sys/kernel.h> 35 #include <sys/malloc.h> 36 #include <sys/param.h> 37 #include <sys/queue.h> 38 #include <sys/systm.h> 39 #include <sys/taskqueue.h> 40 41 #include "common/efx.h" 42 43 #include "sfxge.h" 44 45 static void 46 sfxge_ev_qcomplete(struct sfxge_evq *evq, boolean_t eop) 47 { 48 struct sfxge_softc *sc; 49 unsigned int index; 50 struct sfxge_rxq *rxq; 51 struct sfxge_txq *txq; 52 53 sc = evq->sc; 54 index = evq->index; 55 rxq = sc->rxq[index]; 56 57 if ((txq = evq->txq) != NULL) { 58 evq->txq = NULL; 59 evq->txqs = &(evq->txq); 60 61 do { 62 struct sfxge_txq *next; 63 64 next = txq->next; 65 txq->next = NULL; 66 67 KASSERT(txq->evq_index == index, 68 ("txq->evq_index != index")); 69 70 if (txq->pending != txq->completed) 71 sfxge_tx_qcomplete(txq); 72 73 txq = next; 74 } while (txq != NULL); 75 } 76 77 if (rxq->pending != rxq->completed) 78 sfxge_rx_qcomplete(rxq, eop); 79 } 80 81 static boolean_t 82 sfxge_ev_rx(void *arg, uint32_t label, uint32_t id, uint32_t size, 83 uint16_t flags) 84 { 85 struct sfxge_evq *evq; 86 struct sfxge_softc *sc; 87 struct sfxge_rxq *rxq; 88 unsigned int expected; 89 struct sfxge_rx_sw_desc *rx_desc; 90 91 evq = arg; 92 sc = evq->sc; 93 94 if (evq->exception) 95 goto done; 96 97 rxq = sc->rxq[label]; 98 KASSERT(rxq != NULL, ("rxq == NULL")); 99 KASSERT(evq->index == rxq->index, 100 ("evq->index != rxq->index")); 101 102 if (rxq->init_state != SFXGE_RXQ_STARTED) 103 goto done; 104 105 expected = rxq->pending++ & (SFXGE_NDESCS - 1); 106 if (id != expected) { 107 evq->exception = B_TRUE; 108 109 device_printf(sc->dev, "RX completion out of order" 110 " (id=%#x expected=%#x flags=%#x); resetting\n", 111 id, expected, flags); 112 sfxge_schedule_reset(sc); 113 114 goto done; 115 } 116 117 rx_desc = &rxq->queue[id]; 118 119 KASSERT(rx_desc->flags == EFX_DISCARD, 120 ("rx_desc->flags != EFX_DISCARD")); 121 rx_desc->flags = flags; 122 123 KASSERT(size < (1 << 16), ("size > (1 << 16)")); 124 rx_desc->size = (uint16_t)size; 125 prefetch_read_many(rx_desc->mbuf); 126 127 evq->rx_done++; 128 129 if (rxq->pending - rxq->completed >= SFXGE_RX_BATCH) 130 sfxge_ev_qcomplete(evq, B_FALSE); 131 132 done: 133 return (evq->rx_done >= SFXGE_EV_BATCH); 134 } 135 136 static boolean_t 137 sfxge_ev_exception(void *arg, uint32_t code, uint32_t data) 138 { 139 struct sfxge_evq *evq; 140 struct sfxge_softc *sc; 141 142 evq = (struct sfxge_evq *)arg; 143 sc = evq->sc; 144 145 evq->exception = B_TRUE; 146 147 if (code != EFX_EXCEPTION_UNKNOWN_SENSOREVT) { 148 device_printf(sc->dev, 149 "hardware exception (code=%u); resetting\n", 150 code); 151 sfxge_schedule_reset(sc); 152 } 153 154 return (B_FALSE); 155 } 156 157 static boolean_t 158 sfxge_ev_rxq_flush_done(void *arg, uint32_t label) 159 { 160 struct sfxge_evq *evq; 161 struct sfxge_softc *sc; 162 struct sfxge_rxq *rxq; 163 unsigned int index; 164 uint16_t magic; 165 166 evq = (struct sfxge_evq *)arg; 167 sc = evq->sc; 168 rxq = sc->rxq[label]; 169 170 KASSERT(rxq != NULL, ("rxq == NULL")); 171 172 /* Resend a software event on the correct queue */ 173 index = rxq->index; 174 evq = sc->evq[index]; 175 176 KASSERT((label & SFXGE_MAGIC_DMAQ_LABEL_MASK) == label, 177 ("(label & SFXGE_MAGIC_DMAQ_LABEL_MASK) != level")); 178 magic = SFXGE_MAGIC_RX_QFLUSH_DONE | label; 179 180 KASSERT(evq->init_state == SFXGE_EVQ_STARTED, 181 ("evq not started")); 182 efx_ev_qpost(evq->common, magic); 183 184 return (B_FALSE); 185 } 186 187 static boolean_t 188 sfxge_ev_rxq_flush_failed(void *arg, uint32_t label) 189 { 190 struct sfxge_evq *evq; 191 struct sfxge_softc *sc; 192 struct sfxge_rxq *rxq; 193 unsigned int index; 194 uint16_t magic; 195 196 evq = (struct sfxge_evq *)arg; 197 sc = evq->sc; 198 rxq = sc->rxq[label]; 199 200 KASSERT(rxq != NULL, ("rxq == NULL")); 201 202 /* Resend a software event on the correct queue */ 203 index = rxq->index; 204 evq = sc->evq[index]; 205 206 KASSERT((label & SFXGE_MAGIC_DMAQ_LABEL_MASK) == label, 207 ("(label & SFXGE_MAGIC_DMAQ_LABEL_MASK) != label")); 208 magic = SFXGE_MAGIC_RX_QFLUSH_FAILED | label; 209 210 KASSERT(evq->init_state == SFXGE_EVQ_STARTED, 211 ("evq not started")); 212 efx_ev_qpost(evq->common, magic); 213 214 return (B_FALSE); 215 } 216 217 static boolean_t 218 sfxge_ev_tx(void *arg, uint32_t label, uint32_t id) 219 { 220 struct sfxge_evq *evq; 221 struct sfxge_softc *sc; 222 struct sfxge_txq *txq; 223 unsigned int stop; 224 unsigned int delta; 225 226 evq = (struct sfxge_evq *)arg; 227 sc = evq->sc; 228 txq = sc->txq[label]; 229 230 KASSERT(txq != NULL, ("txq == NULL")); 231 KASSERT(evq->index == txq->evq_index, 232 ("evq->index != txq->evq_index")); 233 234 if (txq->init_state != SFXGE_TXQ_STARTED) 235 goto done; 236 237 stop = (id + 1) & (SFXGE_NDESCS - 1); 238 id = txq->pending & (SFXGE_NDESCS - 1); 239 240 delta = (stop >= id) ? (stop - id) : (SFXGE_NDESCS - id + stop); 241 txq->pending += delta; 242 243 evq->tx_done++; 244 245 if (txq->next == NULL && 246 evq->txqs != &(txq->next)) { 247 *(evq->txqs) = txq; 248 evq->txqs = &(txq->next); 249 } 250 251 if (txq->pending - txq->completed >= SFXGE_TX_BATCH) 252 sfxge_tx_qcomplete(txq); 253 254 done: 255 return (evq->tx_done >= SFXGE_EV_BATCH); 256 } 257 258 static boolean_t 259 sfxge_ev_txq_flush_done(void *arg, uint32_t label) 260 { 261 struct sfxge_evq *evq; 262 struct sfxge_softc *sc; 263 struct sfxge_txq *txq; 264 uint16_t magic; 265 266 evq = (struct sfxge_evq *)arg; 267 sc = evq->sc; 268 txq = sc->txq[label]; 269 270 KASSERT(txq != NULL, ("txq == NULL")); 271 KASSERT(txq->init_state == SFXGE_TXQ_INITIALIZED, 272 ("txq not initialized")); 273 274 /* Resend a software event on the correct queue */ 275 evq = sc->evq[txq->evq_index]; 276 277 KASSERT((label & SFXGE_MAGIC_DMAQ_LABEL_MASK) == label, 278 ("(label & SFXGE_MAGIC_DMAQ_LABEL_MASK) != label")); 279 magic = SFXGE_MAGIC_TX_QFLUSH_DONE | label; 280 281 KASSERT(evq->init_state == SFXGE_EVQ_STARTED, 282 ("evq not started")); 283 efx_ev_qpost(evq->common, magic); 284 285 return (B_FALSE); 286 } 287 288 static boolean_t 289 sfxge_ev_software(void *arg, uint16_t magic) 290 { 291 struct sfxge_evq *evq; 292 struct sfxge_softc *sc; 293 unsigned int label; 294 295 evq = (struct sfxge_evq *)arg; 296 sc = evq->sc; 297 298 label = magic & SFXGE_MAGIC_DMAQ_LABEL_MASK; 299 magic &= ~SFXGE_MAGIC_DMAQ_LABEL_MASK; 300 301 switch (magic) { 302 case SFXGE_MAGIC_RX_QFLUSH_DONE: { 303 struct sfxge_rxq *rxq = sc->rxq[label]; 304 305 KASSERT(rxq != NULL, ("rxq == NULL")); 306 KASSERT(evq->index == rxq->index, 307 ("evq->index != rxq->index")); 308 309 sfxge_rx_qflush_done(rxq); 310 break; 311 } 312 case SFXGE_MAGIC_RX_QFLUSH_FAILED: { 313 struct sfxge_rxq *rxq = sc->rxq[label]; 314 315 KASSERT(rxq != NULL, ("rxq == NULL")); 316 KASSERT(evq->index == rxq->index, 317 ("evq->index != rxq->index")); 318 319 sfxge_rx_qflush_failed(rxq); 320 break; 321 } 322 case SFXGE_MAGIC_RX_QREFILL: { 323 struct sfxge_rxq *rxq = sc->rxq[label]; 324 325 KASSERT(rxq != NULL, ("rxq == NULL")); 326 KASSERT(evq->index == rxq->index, 327 ("evq->index != rxq->index")); 328 329 sfxge_rx_qrefill(rxq); 330 break; 331 } 332 case SFXGE_MAGIC_TX_QFLUSH_DONE: { 333 struct sfxge_txq *txq = sc->txq[label]; 334 335 KASSERT(txq != NULL, ("txq == NULL")); 336 KASSERT(evq->index == txq->evq_index, 337 ("evq->index != txq->evq_index")); 338 339 sfxge_tx_qflush_done(txq); 340 break; 341 } 342 default: 343 break; 344 } 345 346 return (B_FALSE); 347 } 348 349 static boolean_t 350 sfxge_ev_sram(void *arg, uint32_t code) 351 { 352 (void)arg; 353 (void)code; 354 355 switch (code) { 356 case EFX_SRAM_UPDATE: 357 EFSYS_PROBE(sram_update); 358 break; 359 360 case EFX_SRAM_CLEAR: 361 EFSYS_PROBE(sram_clear); 362 break; 363 364 case EFX_SRAM_ILLEGAL_CLEAR: 365 EFSYS_PROBE(sram_illegal_clear); 366 break; 367 368 default: 369 KASSERT(B_FALSE, ("Impossible SRAM event")); 370 break; 371 } 372 373 return (B_FALSE); 374 } 375 376 static boolean_t 377 sfxge_ev_timer(void *arg, uint32_t index) 378 { 379 (void)arg; 380 (void)index; 381 382 return (B_FALSE); 383 } 384 385 static boolean_t 386 sfxge_ev_wake_up(void *arg, uint32_t index) 387 { 388 (void)arg; 389 (void)index; 390 391 return (B_FALSE); 392 } 393 394 static void 395 sfxge_ev_stat_update(struct sfxge_softc *sc) 396 { 397 struct sfxge_evq *evq; 398 unsigned int index; 399 clock_t now; 400 401 sx_xlock(&sc->softc_lock); 402 403 if (sc->evq[0]->init_state != SFXGE_EVQ_STARTED) 404 goto out; 405 406 now = ticks; 407 if (now - sc->ev_stats_update_time < hz) 408 goto out; 409 410 sc->ev_stats_update_time = now; 411 412 /* Add event counts from each event queue in turn */ 413 for (index = 0; index < sc->intr.n_alloc; index++) { 414 evq = sc->evq[index]; 415 mtx_lock(&evq->lock); 416 efx_ev_qstats_update(evq->common, sc->ev_stats); 417 mtx_unlock(&evq->lock); 418 } 419 out: 420 sx_xunlock(&sc->softc_lock); 421 } 422 423 static int 424 sfxge_ev_stat_handler(SYSCTL_HANDLER_ARGS) 425 { 426 struct sfxge_softc *sc = arg1; 427 unsigned int id = arg2; 428 429 sfxge_ev_stat_update(sc); 430 431 return SYSCTL_OUT(req, &sc->ev_stats[id], sizeof(sc->ev_stats[id])); 432 } 433 434 static void 435 sfxge_ev_stat_init(struct sfxge_softc *sc) 436 { 437 struct sysctl_ctx_list *ctx = device_get_sysctl_ctx(sc->dev); 438 struct sysctl_oid_list *stat_list; 439 unsigned int id; 440 char name[40]; 441 442 stat_list = SYSCTL_CHILDREN(sc->stats_node); 443 444 for (id = 0; id < EV_NQSTATS; id++) { 445 snprintf(name, sizeof(name), "ev_%s", 446 efx_ev_qstat_name(sc->enp, id)); 447 SYSCTL_ADD_PROC( 448 ctx, stat_list, 449 OID_AUTO, name, CTLTYPE_U64|CTLFLAG_RD, 450 sc, id, sfxge_ev_stat_handler, "Q", 451 ""); 452 } 453 } 454 455 static void 456 sfxge_ev_qmoderate(struct sfxge_softc *sc, unsigned int idx, unsigned int us) 457 { 458 struct sfxge_evq *evq; 459 efx_evq_t *eep; 460 461 evq = sc->evq[idx]; 462 eep = evq->common; 463 464 KASSERT(evq->init_state == SFXGE_EVQ_STARTED, 465 ("evq->init_state != SFXGE_EVQ_STARTED")); 466 467 (void)efx_ev_qmoderate(eep, us); 468 } 469 470 static int 471 sfxge_int_mod_handler(SYSCTL_HANDLER_ARGS) 472 { 473 struct sfxge_softc *sc = arg1; 474 struct sfxge_intr *intr = &sc->intr; 475 unsigned int moderation; 476 int error; 477 int index; 478 479 sx_xlock(&sc->softc_lock); 480 481 if (req->newptr) { 482 if ((error = SYSCTL_IN(req, &moderation, sizeof(moderation))) 483 != 0) 484 goto out; 485 486 /* We may not be calling efx_ev_qmoderate() now, 487 * so we have to range-check the value ourselves. 488 */ 489 if (moderation > 490 efx_nic_cfg_get(sc->enp)->enc_evq_moderation_max) { 491 error = EINVAL; 492 goto out; 493 } 494 495 sc->ev_moderation = moderation; 496 if (intr->state == SFXGE_INTR_STARTED) { 497 for (index = 0; index < intr->n_alloc; index++) 498 sfxge_ev_qmoderate(sc, index, moderation); 499 } 500 } else { 501 error = SYSCTL_OUT(req, &sc->ev_moderation, 502 sizeof(sc->ev_moderation)); 503 } 504 505 out: 506 sx_xunlock(&sc->softc_lock); 507 508 return error; 509 } 510 511 static boolean_t 512 sfxge_ev_initialized(void *arg) 513 { 514 struct sfxge_evq *evq; 515 516 evq = (struct sfxge_evq *)arg; 517 518 KASSERT(evq->init_state == SFXGE_EVQ_STARTING, 519 ("evq not starting")); 520 521 evq->init_state = SFXGE_EVQ_STARTED; 522 523 return (0); 524 } 525 526 static boolean_t 527 sfxge_ev_link_change(void *arg, efx_link_mode_t link_mode) 528 { 529 struct sfxge_evq *evq; 530 struct sfxge_softc *sc; 531 532 evq = (struct sfxge_evq *)arg; 533 sc = evq->sc; 534 535 sfxge_mac_link_update(sc, link_mode); 536 537 return (0); 538 } 539 540 static const efx_ev_callbacks_t sfxge_ev_callbacks = { 541 .eec_initialized = sfxge_ev_initialized, 542 .eec_rx = sfxge_ev_rx, 543 .eec_tx = sfxge_ev_tx, 544 .eec_exception = sfxge_ev_exception, 545 .eec_rxq_flush_done = sfxge_ev_rxq_flush_done, 546 .eec_rxq_flush_failed = sfxge_ev_rxq_flush_failed, 547 .eec_txq_flush_done = sfxge_ev_txq_flush_done, 548 .eec_software = sfxge_ev_software, 549 .eec_sram = sfxge_ev_sram, 550 .eec_wake_up = sfxge_ev_wake_up, 551 .eec_timer = sfxge_ev_timer, 552 .eec_link_change = sfxge_ev_link_change, 553 }; 554 555 556 int 557 sfxge_ev_qpoll(struct sfxge_softc *sc, unsigned int index) 558 { 559 struct sfxge_evq *evq; 560 int rc; 561 562 evq = sc->evq[index]; 563 564 mtx_lock(&evq->lock); 565 566 if (evq->init_state != SFXGE_EVQ_STARTING && 567 evq->init_state != SFXGE_EVQ_STARTED) { 568 rc = EINVAL; 569 goto fail; 570 } 571 572 /* Synchronize the DMA memory for reading */ 573 bus_dmamap_sync(evq->mem.esm_tag, evq->mem.esm_map, 574 BUS_DMASYNC_POSTREAD); 575 576 KASSERT(evq->rx_done == 0, ("evq->rx_done != 0")); 577 KASSERT(evq->tx_done == 0, ("evq->tx_done != 0")); 578 KASSERT(evq->txq == NULL, ("evq->txq != NULL")); 579 KASSERT(evq->txqs == &evq->txq, ("evq->txqs != &evq->txq")); 580 581 /* Poll the queue */ 582 efx_ev_qpoll(evq->common, &evq->read_ptr, &sfxge_ev_callbacks, evq); 583 584 evq->rx_done = 0; 585 evq->tx_done = 0; 586 587 /* Perform any pending completion processing */ 588 sfxge_ev_qcomplete(evq, B_TRUE); 589 590 /* Re-prime the event queue for interrupts */ 591 if ((rc = efx_ev_qprime(evq->common, evq->read_ptr)) != 0) 592 goto fail; 593 594 mtx_unlock(&evq->lock); 595 596 return (0); 597 598 fail: 599 mtx_unlock(&(evq->lock)); 600 return (rc); 601 } 602 603 static void 604 sfxge_ev_qstop(struct sfxge_softc *sc, unsigned int index) 605 { 606 struct sfxge_evq *evq; 607 608 evq = sc->evq[index]; 609 610 KASSERT(evq->init_state == SFXGE_EVQ_STARTED, 611 ("evq->init_state != SFXGE_EVQ_STARTED")); 612 613 mtx_lock(&evq->lock); 614 evq->init_state = SFXGE_EVQ_INITIALIZED; 615 evq->read_ptr = 0; 616 evq->exception = B_FALSE; 617 618 /* Add event counts before discarding the common evq state */ 619 efx_ev_qstats_update(evq->common, sc->ev_stats); 620 621 efx_ev_qdestroy(evq->common); 622 efx_sram_buf_tbl_clear(sc->enp, evq->buf_base_id, 623 EFX_EVQ_NBUFS(SFXGE_NEVS)); 624 mtx_unlock(&evq->lock); 625 } 626 627 static int 628 sfxge_ev_qstart(struct sfxge_softc *sc, unsigned int index) 629 { 630 struct sfxge_evq *evq; 631 efsys_mem_t *esmp; 632 int count; 633 int rc; 634 635 evq = sc->evq[index]; 636 esmp = &evq->mem; 637 638 KASSERT(evq->init_state == SFXGE_EVQ_INITIALIZED, 639 ("evq->init_state != SFXGE_EVQ_INITIALIZED")); 640 641 /* Clear all events. */ 642 (void)memset(esmp->esm_base, 0xff, EFX_EVQ_SIZE(SFXGE_NEVS)); 643 644 /* Program the buffer table. */ 645 if ((rc = efx_sram_buf_tbl_set(sc->enp, evq->buf_base_id, esmp, 646 EFX_EVQ_NBUFS(SFXGE_NEVS))) != 0) 647 return rc; 648 649 /* Create the common code event queue. */ 650 if ((rc = efx_ev_qcreate(sc->enp, index, esmp, SFXGE_NEVS, 651 evq->buf_base_id, &evq->common)) != 0) 652 goto fail; 653 654 mtx_lock(&evq->lock); 655 656 /* Set the default moderation */ 657 (void)efx_ev_qmoderate(evq->common, sc->ev_moderation); 658 659 /* Prime the event queue for interrupts */ 660 if ((rc = efx_ev_qprime(evq->common, evq->read_ptr)) != 0) 661 goto fail2; 662 663 evq->init_state = SFXGE_EVQ_STARTING; 664 665 mtx_unlock(&evq->lock); 666 667 /* Wait for the initialization event */ 668 count = 0; 669 do { 670 /* Pause for 100 ms */ 671 pause("sfxge evq init", hz / 10); 672 673 /* Check to see if the test event has been processed */ 674 if (evq->init_state == SFXGE_EVQ_STARTED) 675 goto done; 676 677 } while (++count < 20); 678 679 rc = ETIMEDOUT; 680 goto fail3; 681 682 done: 683 return (0); 684 685 fail3: 686 mtx_lock(&evq->lock); 687 evq->init_state = SFXGE_EVQ_INITIALIZED; 688 fail2: 689 mtx_unlock(&evq->lock); 690 efx_ev_qdestroy(evq->common); 691 fail: 692 efx_sram_buf_tbl_clear(sc->enp, evq->buf_base_id, 693 EFX_EVQ_NBUFS(SFXGE_NEVS)); 694 695 return (rc); 696 } 697 698 void 699 sfxge_ev_stop(struct sfxge_softc *sc) 700 { 701 struct sfxge_intr *intr; 702 efx_nic_t *enp; 703 int index; 704 705 intr = &sc->intr; 706 enp = sc->enp; 707 708 KASSERT(intr->state == SFXGE_INTR_STARTED, 709 ("Interrupts not started")); 710 711 /* Stop the event queue(s) */ 712 index = intr->n_alloc; 713 while (--index >= 0) 714 sfxge_ev_qstop(sc, index); 715 716 /* Tear down the event module */ 717 efx_ev_fini(enp); 718 } 719 720 int 721 sfxge_ev_start(struct sfxge_softc *sc) 722 { 723 struct sfxge_intr *intr; 724 int index; 725 int rc; 726 727 intr = &sc->intr; 728 729 KASSERT(intr->state == SFXGE_INTR_STARTED, 730 ("intr->state != SFXGE_INTR_STARTED")); 731 732 /* Initialize the event module */ 733 if ((rc = efx_ev_init(sc->enp)) != 0) 734 return rc; 735 736 /* Start the event queues */ 737 for (index = 0; index < intr->n_alloc; index++) { 738 if ((rc = sfxge_ev_qstart(sc, index)) != 0) 739 goto fail; 740 } 741 742 return (0); 743 744 fail: 745 /* Stop the event queue(s) */ 746 while (--index >= 0) 747 sfxge_ev_qstop(sc, index); 748 749 /* Tear down the event module */ 750 efx_ev_fini(sc->enp); 751 752 return (rc); 753 } 754 755 static void 756 sfxge_ev_qfini(struct sfxge_softc *sc, unsigned int index) 757 { 758 struct sfxge_evq *evq; 759 760 evq = sc->evq[index]; 761 762 KASSERT(evq->init_state == SFXGE_EVQ_INITIALIZED, 763 ("evq->init_state != SFXGE_EVQ_INITIALIZED")); 764 KASSERT(evq->txqs == &evq->txq, ("evq->txqs != &evq->txq")); 765 766 sfxge_dma_free(&evq->mem); 767 768 sc->evq[index] = NULL; 769 770 mtx_destroy(&evq->lock); 771 772 free(evq, M_SFXGE); 773 } 774 775 static int 776 sfxge_ev_qinit(struct sfxge_softc *sc, unsigned int index) 777 { 778 struct sfxge_evq *evq; 779 efsys_mem_t *esmp; 780 int rc; 781 782 KASSERT(index < SFXGE_RX_SCALE_MAX, ("index >= SFXGE_RX_SCALE_MAX")); 783 784 evq = malloc(sizeof(struct sfxge_evq), M_SFXGE, M_ZERO | M_WAITOK); 785 evq->sc = sc; 786 evq->index = index; 787 sc->evq[index] = evq; 788 esmp = &evq->mem; 789 790 /* Initialise TX completion list */ 791 evq->txqs = &evq->txq; 792 793 /* Allocate DMA space. */ 794 if ((rc = sfxge_dma_alloc(sc, EFX_EVQ_SIZE(SFXGE_NEVS), esmp)) != 0) 795 return (rc); 796 797 /* Allocate buffer table entries. */ 798 sfxge_sram_buf_tbl_alloc(sc, EFX_EVQ_NBUFS(SFXGE_NEVS), 799 &evq->buf_base_id); 800 801 mtx_init(&evq->lock, "evq", NULL, MTX_DEF); 802 803 evq->init_state = SFXGE_EVQ_INITIALIZED; 804 805 return (0); 806 } 807 808 void 809 sfxge_ev_fini(struct sfxge_softc *sc) 810 { 811 struct sfxge_intr *intr; 812 int index; 813 814 intr = &sc->intr; 815 816 KASSERT(intr->state == SFXGE_INTR_INITIALIZED, 817 ("intr->state != SFXGE_INTR_INITIALIZED")); 818 819 sc->ev_moderation = 0; 820 821 /* Tear down the event queue(s). */ 822 index = intr->n_alloc; 823 while (--index >= 0) 824 sfxge_ev_qfini(sc, index); 825 } 826 827 int 828 sfxge_ev_init(struct sfxge_softc *sc) 829 { 830 struct sysctl_ctx_list *sysctl_ctx = device_get_sysctl_ctx(sc->dev); 831 struct sysctl_oid *sysctl_tree = device_get_sysctl_tree(sc->dev); 832 struct sfxge_intr *intr; 833 int index; 834 int rc; 835 836 intr = &sc->intr; 837 838 KASSERT(intr->state == SFXGE_INTR_INITIALIZED, 839 ("intr->state != SFXGE_INTR_INITIALIZED")); 840 841 /* Set default interrupt moderation; add a sysctl to 842 * read and change it. 843 */ 844 sc->ev_moderation = 30; 845 SYSCTL_ADD_PROC(sysctl_ctx, SYSCTL_CHILDREN(sysctl_tree), 846 OID_AUTO, "int_mod", CTLTYPE_UINT|CTLFLAG_RW, 847 sc, 0, sfxge_int_mod_handler, "IU", 848 "sfxge interrupt moderation (us)"); 849 850 /* 851 * Initialize the event queue(s) - one per interrupt. 852 */ 853 for (index = 0; index < intr->n_alloc; index++) { 854 if ((rc = sfxge_ev_qinit(sc, index)) != 0) 855 goto fail; 856 } 857 858 sfxge_ev_stat_init(sc); 859 860 return (0); 861 862 fail: 863 while (--index >= 0) 864 sfxge_ev_qfini(sc, index); 865 866 return (rc); 867 } 868