1 /*- 2 * Copyright (c) 2010-2011 Solarflare Communications, Inc. 3 * All rights reserved. 4 * 5 * This software was developed in part by Philip Paeps under contract for 6 * Solarflare Communications, Inc. 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 21 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 27 * SUCH DAMAGE. 28 */ 29 30 #include <sys/cdefs.h> 31 __FBSDID("$FreeBSD$"); 32 33 #include <sys/param.h> 34 #include <sys/systm.h> 35 36 #include "common/efx.h" 37 38 #include "sfxge.h" 39 40 static void 41 sfxge_ev_qcomplete(struct sfxge_evq *evq, boolean_t eop) 42 { 43 struct sfxge_softc *sc; 44 unsigned int index; 45 struct sfxge_rxq *rxq; 46 struct sfxge_txq *txq; 47 48 sc = evq->sc; 49 index = evq->index; 50 rxq = sc->rxq[index]; 51 52 if ((txq = evq->txq) != NULL) { 53 evq->txq = NULL; 54 evq->txqs = &(evq->txq); 55 56 do { 57 struct sfxge_txq *next; 58 59 next = txq->next; 60 txq->next = NULL; 61 62 KASSERT(txq->evq_index == index, 63 ("txq->evq_index != index")); 64 65 if (txq->pending != txq->completed) 66 sfxge_tx_qcomplete(txq); 67 68 txq = next; 69 } while (txq != NULL); 70 } 71 72 if (rxq->pending != rxq->completed) 73 sfxge_rx_qcomplete(rxq, eop); 74 } 75 76 static boolean_t 77 sfxge_ev_rx(void *arg, uint32_t label, uint32_t id, uint32_t size, 78 uint16_t flags) 79 { 80 struct sfxge_evq *evq; 81 struct sfxge_softc *sc; 82 struct sfxge_rxq *rxq; 83 unsigned int expected; 84 struct sfxge_rx_sw_desc *rx_desc; 85 86 evq = arg; 87 sc = evq->sc; 88 89 if (evq->exception) 90 goto done; 91 92 rxq = sc->rxq[label]; 93 KASSERT(rxq != NULL, ("rxq == NULL")); 94 KASSERT(evq->index == rxq->index, 95 ("evq->index != rxq->index")); 96 97 if (rxq->init_state != SFXGE_RXQ_STARTED) 98 goto done; 99 100 expected = rxq->pending++ & (SFXGE_NDESCS - 1); 101 if (id != expected) { 102 evq->exception = B_TRUE; 103 104 device_printf(sc->dev, "RX completion out of order" 105 " (id=%#x expected=%#x flags=%#x); resetting\n", 106 id, expected, flags); 107 sfxge_schedule_reset(sc); 108 109 goto done; 110 } 111 112 rx_desc = &rxq->queue[id]; 113 114 KASSERT(rx_desc->flags == EFX_DISCARD, 115 ("rx_desc->flags != EFX_DISCARD")); 116 rx_desc->flags = flags; 117 118 KASSERT(size < (1 << 16), ("size > (1 << 16)")); 119 rx_desc->size = (uint16_t)size; 120 prefetch_read_many(rx_desc->mbuf); 121 122 evq->rx_done++; 123 124 if (rxq->pending - rxq->completed >= SFXGE_RX_BATCH) 125 sfxge_ev_qcomplete(evq, B_FALSE); 126 127 done: 128 return (evq->rx_done >= SFXGE_EV_BATCH); 129 } 130 131 static boolean_t 132 sfxge_ev_exception(void *arg, uint32_t code, uint32_t data) 133 { 134 struct sfxge_evq *evq; 135 struct sfxge_softc *sc; 136 137 evq = (struct sfxge_evq *)arg; 138 sc = evq->sc; 139 140 evq->exception = B_TRUE; 141 142 if (code != EFX_EXCEPTION_UNKNOWN_SENSOREVT) { 143 device_printf(sc->dev, 144 "hardware exception (code=%u); resetting\n", 145 code); 146 sfxge_schedule_reset(sc); 147 } 148 149 return (B_FALSE); 150 } 151 152 static boolean_t 153 sfxge_ev_rxq_flush_done(void *arg, uint32_t label) 154 { 155 struct sfxge_evq *evq; 156 struct sfxge_softc *sc; 157 struct sfxge_rxq *rxq; 158 unsigned int index; 159 uint16_t magic; 160 161 evq = (struct sfxge_evq *)arg; 162 sc = evq->sc; 163 rxq = sc->rxq[label]; 164 165 KASSERT(rxq != NULL, ("rxq == NULL")); 166 167 /* Resend a software event on the correct queue */ 168 index = rxq->index; 169 evq = sc->evq[index]; 170 171 KASSERT((label & SFXGE_MAGIC_DMAQ_LABEL_MASK) == label, 172 ("(label & SFXGE_MAGIC_DMAQ_LABEL_MASK) != level")); 173 magic = SFXGE_MAGIC_RX_QFLUSH_DONE | label; 174 175 KASSERT(evq->init_state == SFXGE_EVQ_STARTED, 176 ("evq not started")); 177 efx_ev_qpost(evq->common, magic); 178 179 return (B_FALSE); 180 } 181 182 static boolean_t 183 sfxge_ev_rxq_flush_failed(void *arg, uint32_t label) 184 { 185 struct sfxge_evq *evq; 186 struct sfxge_softc *sc; 187 struct sfxge_rxq *rxq; 188 unsigned int index; 189 uint16_t magic; 190 191 evq = (struct sfxge_evq *)arg; 192 sc = evq->sc; 193 rxq = sc->rxq[label]; 194 195 KASSERT(rxq != NULL, ("rxq == NULL")); 196 197 /* Resend a software event on the correct queue */ 198 index = rxq->index; 199 evq = sc->evq[index]; 200 201 KASSERT((label & SFXGE_MAGIC_DMAQ_LABEL_MASK) == label, 202 ("(label & SFXGE_MAGIC_DMAQ_LABEL_MASK) != label")); 203 magic = SFXGE_MAGIC_RX_QFLUSH_FAILED | label; 204 205 KASSERT(evq->init_state == SFXGE_EVQ_STARTED, 206 ("evq not started")); 207 efx_ev_qpost(evq->common, magic); 208 209 return (B_FALSE); 210 } 211 212 static boolean_t 213 sfxge_ev_tx(void *arg, uint32_t label, uint32_t id) 214 { 215 struct sfxge_evq *evq; 216 struct sfxge_softc *sc; 217 struct sfxge_txq *txq; 218 unsigned int stop; 219 unsigned int delta; 220 221 evq = (struct sfxge_evq *)arg; 222 sc = evq->sc; 223 txq = sc->txq[label]; 224 225 KASSERT(txq != NULL, ("txq == NULL")); 226 KASSERT(evq->index == txq->evq_index, 227 ("evq->index != txq->evq_index")); 228 229 if (txq->init_state != SFXGE_TXQ_STARTED) 230 goto done; 231 232 stop = (id + 1) & (SFXGE_NDESCS - 1); 233 id = txq->pending & (SFXGE_NDESCS - 1); 234 235 delta = (stop >= id) ? (stop - id) : (SFXGE_NDESCS - id + stop); 236 txq->pending += delta; 237 238 evq->tx_done++; 239 240 if (txq->next == NULL && 241 evq->txqs != &(txq->next)) { 242 *(evq->txqs) = txq; 243 evq->txqs = &(txq->next); 244 } 245 246 if (txq->pending - txq->completed >= SFXGE_TX_BATCH) 247 sfxge_tx_qcomplete(txq); 248 249 done: 250 return (evq->tx_done >= SFXGE_EV_BATCH); 251 } 252 253 static boolean_t 254 sfxge_ev_txq_flush_done(void *arg, uint32_t label) 255 { 256 struct sfxge_evq *evq; 257 struct sfxge_softc *sc; 258 struct sfxge_txq *txq; 259 uint16_t magic; 260 261 evq = (struct sfxge_evq *)arg; 262 sc = evq->sc; 263 txq = sc->txq[label]; 264 265 KASSERT(txq != NULL, ("txq == NULL")); 266 KASSERT(txq->init_state == SFXGE_TXQ_INITIALIZED, 267 ("txq not initialized")); 268 269 /* Resend a software event on the correct queue */ 270 evq = sc->evq[txq->evq_index]; 271 272 KASSERT((label & SFXGE_MAGIC_DMAQ_LABEL_MASK) == label, 273 ("(label & SFXGE_MAGIC_DMAQ_LABEL_MASK) != label")); 274 magic = SFXGE_MAGIC_TX_QFLUSH_DONE | label; 275 276 KASSERT(evq->init_state == SFXGE_EVQ_STARTED, 277 ("evq not started")); 278 efx_ev_qpost(evq->common, magic); 279 280 return (B_FALSE); 281 } 282 283 static boolean_t 284 sfxge_ev_software(void *arg, uint16_t magic) 285 { 286 struct sfxge_evq *evq; 287 struct sfxge_softc *sc; 288 unsigned int label; 289 290 evq = (struct sfxge_evq *)arg; 291 sc = evq->sc; 292 293 label = magic & SFXGE_MAGIC_DMAQ_LABEL_MASK; 294 magic &= ~SFXGE_MAGIC_DMAQ_LABEL_MASK; 295 296 switch (magic) { 297 case SFXGE_MAGIC_RX_QFLUSH_DONE: { 298 struct sfxge_rxq *rxq = sc->rxq[label]; 299 300 KASSERT(rxq != NULL, ("rxq == NULL")); 301 KASSERT(evq->index == rxq->index, 302 ("evq->index != rxq->index")); 303 304 sfxge_rx_qflush_done(rxq); 305 break; 306 } 307 case SFXGE_MAGIC_RX_QFLUSH_FAILED: { 308 struct sfxge_rxq *rxq = sc->rxq[label]; 309 310 KASSERT(rxq != NULL, ("rxq == NULL")); 311 KASSERT(evq->index == rxq->index, 312 ("evq->index != rxq->index")); 313 314 sfxge_rx_qflush_failed(rxq); 315 break; 316 } 317 case SFXGE_MAGIC_RX_QREFILL: { 318 struct sfxge_rxq *rxq = sc->rxq[label]; 319 320 KASSERT(rxq != NULL, ("rxq == NULL")); 321 KASSERT(evq->index == rxq->index, 322 ("evq->index != rxq->index")); 323 324 sfxge_rx_qrefill(rxq); 325 break; 326 } 327 case SFXGE_MAGIC_TX_QFLUSH_DONE: { 328 struct sfxge_txq *txq = sc->txq[label]; 329 330 KASSERT(txq != NULL, ("txq == NULL")); 331 KASSERT(evq->index == txq->evq_index, 332 ("evq->index != txq->evq_index")); 333 334 sfxge_tx_qflush_done(txq); 335 break; 336 } 337 default: 338 break; 339 } 340 341 return (B_FALSE); 342 } 343 344 static boolean_t 345 sfxge_ev_sram(void *arg, uint32_t code) 346 { 347 (void)arg; 348 (void)code; 349 350 switch (code) { 351 case EFX_SRAM_UPDATE: 352 EFSYS_PROBE(sram_update); 353 break; 354 355 case EFX_SRAM_CLEAR: 356 EFSYS_PROBE(sram_clear); 357 break; 358 359 case EFX_SRAM_ILLEGAL_CLEAR: 360 EFSYS_PROBE(sram_illegal_clear); 361 break; 362 363 default: 364 KASSERT(B_FALSE, ("Impossible SRAM event")); 365 break; 366 } 367 368 return (B_FALSE); 369 } 370 371 static boolean_t 372 sfxge_ev_timer(void *arg, uint32_t index) 373 { 374 (void)arg; 375 (void)index; 376 377 return (B_FALSE); 378 } 379 380 static boolean_t 381 sfxge_ev_wake_up(void *arg, uint32_t index) 382 { 383 (void)arg; 384 (void)index; 385 386 return (B_FALSE); 387 } 388 389 static void 390 sfxge_ev_stat_update(struct sfxge_softc *sc) 391 { 392 struct sfxge_evq *evq; 393 unsigned int index; 394 clock_t now; 395 396 sx_xlock(&sc->softc_lock); 397 398 if (sc->evq[0]->init_state != SFXGE_EVQ_STARTED) 399 goto out; 400 401 now = ticks; 402 if (now - sc->ev_stats_update_time < hz) 403 goto out; 404 405 sc->ev_stats_update_time = now; 406 407 /* Add event counts from each event queue in turn */ 408 for (index = 0; index < sc->intr.n_alloc; index++) { 409 evq = sc->evq[index]; 410 mtx_lock(&evq->lock); 411 efx_ev_qstats_update(evq->common, sc->ev_stats); 412 mtx_unlock(&evq->lock); 413 } 414 out: 415 sx_xunlock(&sc->softc_lock); 416 } 417 418 static int 419 sfxge_ev_stat_handler(SYSCTL_HANDLER_ARGS) 420 { 421 struct sfxge_softc *sc = arg1; 422 unsigned int id = arg2; 423 424 sfxge_ev_stat_update(sc); 425 426 return SYSCTL_OUT(req, &sc->ev_stats[id], sizeof(sc->ev_stats[id])); 427 } 428 429 static void 430 sfxge_ev_stat_init(struct sfxge_softc *sc) 431 { 432 struct sysctl_ctx_list *ctx = device_get_sysctl_ctx(sc->dev); 433 struct sysctl_oid_list *stat_list; 434 unsigned int id; 435 char name[40]; 436 437 stat_list = SYSCTL_CHILDREN(sc->stats_node); 438 439 for (id = 0; id < EV_NQSTATS; id++) { 440 snprintf(name, sizeof(name), "ev_%s", 441 efx_ev_qstat_name(sc->enp, id)); 442 SYSCTL_ADD_PROC( 443 ctx, stat_list, 444 OID_AUTO, name, CTLTYPE_U64|CTLFLAG_RD, 445 sc, id, sfxge_ev_stat_handler, "Q", 446 ""); 447 } 448 } 449 450 static void 451 sfxge_ev_qmoderate(struct sfxge_softc *sc, unsigned int idx, unsigned int us) 452 { 453 struct sfxge_evq *evq; 454 efx_evq_t *eep; 455 456 evq = sc->evq[idx]; 457 eep = evq->common; 458 459 KASSERT(evq->init_state == SFXGE_EVQ_STARTED, 460 ("evq->init_state != SFXGE_EVQ_STARTED")); 461 462 (void)efx_ev_qmoderate(eep, us); 463 } 464 465 static int 466 sfxge_int_mod_handler(SYSCTL_HANDLER_ARGS) 467 { 468 struct sfxge_softc *sc = arg1; 469 struct sfxge_intr *intr = &sc->intr; 470 unsigned int moderation; 471 int error; 472 int index; 473 474 sx_xlock(&sc->softc_lock); 475 476 if (req->newptr) { 477 if ((error = SYSCTL_IN(req, &moderation, sizeof(moderation))) 478 != 0) 479 goto out; 480 481 /* We may not be calling efx_ev_qmoderate() now, 482 * so we have to range-check the value ourselves. 483 */ 484 if (moderation > 485 efx_nic_cfg_get(sc->enp)->enc_evq_moderation_max) { 486 error = EINVAL; 487 goto out; 488 } 489 490 sc->ev_moderation = moderation; 491 if (intr->state == SFXGE_INTR_STARTED) { 492 for (index = 0; index < intr->n_alloc; index++) 493 sfxge_ev_qmoderate(sc, index, moderation); 494 } 495 } else { 496 error = SYSCTL_OUT(req, &sc->ev_moderation, 497 sizeof(sc->ev_moderation)); 498 } 499 500 out: 501 sx_xunlock(&sc->softc_lock); 502 503 return error; 504 } 505 506 static boolean_t 507 sfxge_ev_initialized(void *arg) 508 { 509 struct sfxge_evq *evq; 510 511 evq = (struct sfxge_evq *)arg; 512 513 KASSERT(evq->init_state == SFXGE_EVQ_STARTING, 514 ("evq not starting")); 515 516 evq->init_state = SFXGE_EVQ_STARTED; 517 518 return (0); 519 } 520 521 static boolean_t 522 sfxge_ev_link_change(void *arg, efx_link_mode_t link_mode) 523 { 524 struct sfxge_evq *evq; 525 struct sfxge_softc *sc; 526 527 evq = (struct sfxge_evq *)arg; 528 sc = evq->sc; 529 530 sfxge_mac_link_update(sc, link_mode); 531 532 return (0); 533 } 534 535 static const efx_ev_callbacks_t sfxge_ev_callbacks = { 536 .eec_initialized = sfxge_ev_initialized, 537 .eec_rx = sfxge_ev_rx, 538 .eec_tx = sfxge_ev_tx, 539 .eec_exception = sfxge_ev_exception, 540 .eec_rxq_flush_done = sfxge_ev_rxq_flush_done, 541 .eec_rxq_flush_failed = sfxge_ev_rxq_flush_failed, 542 .eec_txq_flush_done = sfxge_ev_txq_flush_done, 543 .eec_software = sfxge_ev_software, 544 .eec_sram = sfxge_ev_sram, 545 .eec_wake_up = sfxge_ev_wake_up, 546 .eec_timer = sfxge_ev_timer, 547 .eec_link_change = sfxge_ev_link_change, 548 }; 549 550 551 int 552 sfxge_ev_qpoll(struct sfxge_softc *sc, unsigned int index) 553 { 554 struct sfxge_evq *evq; 555 int rc; 556 557 evq = sc->evq[index]; 558 559 mtx_lock(&evq->lock); 560 561 if (evq->init_state != SFXGE_EVQ_STARTING && 562 evq->init_state != SFXGE_EVQ_STARTED) { 563 rc = EINVAL; 564 goto fail; 565 } 566 567 /* Synchronize the DMA memory for reading */ 568 bus_dmamap_sync(evq->mem.esm_tag, evq->mem.esm_map, 569 BUS_DMASYNC_POSTREAD); 570 571 KASSERT(evq->rx_done == 0, ("evq->rx_done != 0")); 572 KASSERT(evq->tx_done == 0, ("evq->tx_done != 0")); 573 KASSERT(evq->txq == NULL, ("evq->txq != NULL")); 574 KASSERT(evq->txqs == &evq->txq, ("evq->txqs != &evq->txq")); 575 576 /* Poll the queue */ 577 efx_ev_qpoll(evq->common, &evq->read_ptr, &sfxge_ev_callbacks, evq); 578 579 evq->rx_done = 0; 580 evq->tx_done = 0; 581 582 /* Perform any pending completion processing */ 583 sfxge_ev_qcomplete(evq, B_TRUE); 584 585 /* Re-prime the event queue for interrupts */ 586 if ((rc = efx_ev_qprime(evq->common, evq->read_ptr)) != 0) 587 goto fail; 588 589 mtx_unlock(&evq->lock); 590 591 return (0); 592 593 fail: 594 mtx_unlock(&(evq->lock)); 595 return (rc); 596 } 597 598 static void 599 sfxge_ev_qstop(struct sfxge_softc *sc, unsigned int index) 600 { 601 struct sfxge_evq *evq; 602 603 evq = sc->evq[index]; 604 605 KASSERT(evq->init_state == SFXGE_EVQ_STARTED, 606 ("evq->init_state != SFXGE_EVQ_STARTED")); 607 608 mtx_lock(&evq->lock); 609 evq->init_state = SFXGE_EVQ_INITIALIZED; 610 evq->read_ptr = 0; 611 evq->exception = B_FALSE; 612 613 /* Add event counts before discarding the common evq state */ 614 efx_ev_qstats_update(evq->common, sc->ev_stats); 615 616 efx_ev_qdestroy(evq->common); 617 efx_sram_buf_tbl_clear(sc->enp, evq->buf_base_id, 618 EFX_EVQ_NBUFS(SFXGE_NEVS)); 619 mtx_unlock(&evq->lock); 620 } 621 622 static int 623 sfxge_ev_qstart(struct sfxge_softc *sc, unsigned int index) 624 { 625 struct sfxge_evq *evq; 626 efsys_mem_t *esmp; 627 int count; 628 int rc; 629 630 evq = sc->evq[index]; 631 esmp = &evq->mem; 632 633 KASSERT(evq->init_state == SFXGE_EVQ_INITIALIZED, 634 ("evq->init_state != SFXGE_EVQ_INITIALIZED")); 635 636 /* Clear all events. */ 637 (void)memset(esmp->esm_base, 0xff, EFX_EVQ_SIZE(SFXGE_NEVS)); 638 639 /* Program the buffer table. */ 640 if ((rc = efx_sram_buf_tbl_set(sc->enp, evq->buf_base_id, esmp, 641 EFX_EVQ_NBUFS(SFXGE_NEVS))) != 0) 642 return rc; 643 644 /* Create the common code event queue. */ 645 if ((rc = efx_ev_qcreate(sc->enp, index, esmp, SFXGE_NEVS, 646 evq->buf_base_id, &evq->common)) != 0) 647 goto fail; 648 649 mtx_lock(&evq->lock); 650 651 /* Set the default moderation */ 652 (void)efx_ev_qmoderate(evq->common, sc->ev_moderation); 653 654 /* Prime the event queue for interrupts */ 655 if ((rc = efx_ev_qprime(evq->common, evq->read_ptr)) != 0) 656 goto fail2; 657 658 evq->init_state = SFXGE_EVQ_STARTING; 659 660 mtx_unlock(&evq->lock); 661 662 /* Wait for the initialization event */ 663 count = 0; 664 do { 665 /* Pause for 100 ms */ 666 pause("sfxge evq init", hz / 10); 667 668 /* Check to see if the test event has been processed */ 669 if (evq->init_state == SFXGE_EVQ_STARTED) 670 goto done; 671 672 } while (++count < 20); 673 674 rc = ETIMEDOUT; 675 goto fail3; 676 677 done: 678 return (0); 679 680 fail3: 681 mtx_lock(&evq->lock); 682 evq->init_state = SFXGE_EVQ_INITIALIZED; 683 fail2: 684 mtx_unlock(&evq->lock); 685 efx_ev_qdestroy(evq->common); 686 fail: 687 efx_sram_buf_tbl_clear(sc->enp, evq->buf_base_id, 688 EFX_EVQ_NBUFS(SFXGE_NEVS)); 689 690 return (rc); 691 } 692 693 void 694 sfxge_ev_stop(struct sfxge_softc *sc) 695 { 696 struct sfxge_intr *intr; 697 efx_nic_t *enp; 698 int index; 699 700 intr = &sc->intr; 701 enp = sc->enp; 702 703 KASSERT(intr->state == SFXGE_INTR_STARTED, 704 ("Interrupts not started")); 705 706 /* Stop the event queue(s) */ 707 index = intr->n_alloc; 708 while (--index >= 0) 709 sfxge_ev_qstop(sc, index); 710 711 /* Tear down the event module */ 712 efx_ev_fini(enp); 713 } 714 715 int 716 sfxge_ev_start(struct sfxge_softc *sc) 717 { 718 struct sfxge_intr *intr; 719 int index; 720 int rc; 721 722 intr = &sc->intr; 723 724 KASSERT(intr->state == SFXGE_INTR_STARTED, 725 ("intr->state != SFXGE_INTR_STARTED")); 726 727 /* Initialize the event module */ 728 if ((rc = efx_ev_init(sc->enp)) != 0) 729 return rc; 730 731 /* Start the event queues */ 732 for (index = 0; index < intr->n_alloc; index++) { 733 if ((rc = sfxge_ev_qstart(sc, index)) != 0) 734 goto fail; 735 } 736 737 return (0); 738 739 fail: 740 /* Stop the event queue(s) */ 741 while (--index >= 0) 742 sfxge_ev_qstop(sc, index); 743 744 /* Tear down the event module */ 745 efx_ev_fini(sc->enp); 746 747 return (rc); 748 } 749 750 static void 751 sfxge_ev_qfini(struct sfxge_softc *sc, unsigned int index) 752 { 753 struct sfxge_evq *evq; 754 755 evq = sc->evq[index]; 756 757 KASSERT(evq->init_state == SFXGE_EVQ_INITIALIZED, 758 ("evq->init_state != SFXGE_EVQ_INITIALIZED")); 759 KASSERT(evq->txqs == &evq->txq, ("evq->txqs != &evq->txq")); 760 761 sfxge_dma_free(&evq->mem); 762 763 sc->evq[index] = NULL; 764 765 mtx_destroy(&evq->lock); 766 767 free(evq, M_SFXGE); 768 } 769 770 static int 771 sfxge_ev_qinit(struct sfxge_softc *sc, unsigned int index) 772 { 773 struct sfxge_evq *evq; 774 efsys_mem_t *esmp; 775 int rc; 776 777 KASSERT(index < SFXGE_RX_SCALE_MAX, ("index >= SFXGE_RX_SCALE_MAX")); 778 779 evq = malloc(sizeof(struct sfxge_evq), M_SFXGE, M_ZERO | M_WAITOK); 780 evq->sc = sc; 781 evq->index = index; 782 sc->evq[index] = evq; 783 esmp = &evq->mem; 784 785 /* Initialise TX completion list */ 786 evq->txqs = &evq->txq; 787 788 /* Allocate DMA space. */ 789 if ((rc = sfxge_dma_alloc(sc, EFX_EVQ_SIZE(SFXGE_NEVS), esmp)) != 0) 790 return (rc); 791 792 /* Allocate buffer table entries. */ 793 sfxge_sram_buf_tbl_alloc(sc, EFX_EVQ_NBUFS(SFXGE_NEVS), 794 &evq->buf_base_id); 795 796 mtx_init(&evq->lock, "evq", NULL, MTX_DEF); 797 798 evq->init_state = SFXGE_EVQ_INITIALIZED; 799 800 return (0); 801 } 802 803 void 804 sfxge_ev_fini(struct sfxge_softc *sc) 805 { 806 struct sfxge_intr *intr; 807 int index; 808 809 intr = &sc->intr; 810 811 KASSERT(intr->state == SFXGE_INTR_INITIALIZED, 812 ("intr->state != SFXGE_INTR_INITIALIZED")); 813 814 sc->ev_moderation = 0; 815 816 /* Tear down the event queue(s). */ 817 index = intr->n_alloc; 818 while (--index >= 0) 819 sfxge_ev_qfini(sc, index); 820 } 821 822 int 823 sfxge_ev_init(struct sfxge_softc *sc) 824 { 825 struct sysctl_ctx_list *sysctl_ctx = device_get_sysctl_ctx(sc->dev); 826 struct sysctl_oid *sysctl_tree = device_get_sysctl_tree(sc->dev); 827 struct sfxge_intr *intr; 828 int index; 829 int rc; 830 831 intr = &sc->intr; 832 833 KASSERT(intr->state == SFXGE_INTR_INITIALIZED, 834 ("intr->state != SFXGE_INTR_INITIALIZED")); 835 836 /* Set default interrupt moderation; add a sysctl to 837 * read and change it. 838 */ 839 sc->ev_moderation = 30; 840 SYSCTL_ADD_PROC(sysctl_ctx, SYSCTL_CHILDREN(sysctl_tree), 841 OID_AUTO, "int_mod", CTLTYPE_UINT|CTLFLAG_RW, 842 sc, 0, sfxge_int_mod_handler, "IU", 843 "sfxge interrupt moderation (us)"); 844 845 /* 846 * Initialize the event queue(s) - one per interrupt. 847 */ 848 for (index = 0; index < intr->n_alloc; index++) { 849 if ((rc = sfxge_ev_qinit(sc, index)) != 0) 850 goto fail; 851 } 852 853 sfxge_ev_stat_init(sc); 854 855 return (0); 856 857 fail: 858 while (--index >= 0) 859 sfxge_ev_qfini(sc, index); 860 861 return (rc); 862 } 863