1 /*- 2 * Copyright (c) 2010-2011 Solarflare Communications, Inc. 3 * All rights reserved. 4 * 5 * This software was developed in part by Philip Paeps under contract for 6 * Solarflare Communications, Inc. 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 21 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 27 * SUCH DAMAGE. 28 */ 29 30 #include <sys/cdefs.h> 31 __FBSDID("$FreeBSD$"); 32 33 #include <sys/param.h> 34 #include <sys/kernel.h> 35 #include <sys/malloc.h> 36 #include <sys/param.h> 37 #include <sys/queue.h> 38 #include <sys/systm.h> 39 #include <sys/taskqueue.h> 40 41 #include "common/efx.h" 42 43 #include "sfxge.h" 44 45 static void 46 sfxge_ev_qcomplete(struct sfxge_evq *evq, boolean_t eop) 47 { 48 struct sfxge_softc *sc; 49 unsigned int index; 50 struct sfxge_rxq *rxq; 51 struct sfxge_txq *txq; 52 53 sc = evq->sc; 54 index = evq->index; 55 rxq = sc->rxq[index]; 56 57 if ((txq = evq->txq) != NULL) { 58 evq->txq = NULL; 59 evq->txqs = &(evq->txq); 60 61 do { 62 struct sfxge_txq *next; 63 64 next = txq->next; 65 txq->next = NULL; 66 67 KASSERT(txq->evq_index == index, 68 ("txq->evq_index != index")); 69 70 if (txq->pending != txq->completed) 71 sfxge_tx_qcomplete(txq, evq); 72 73 txq = next; 74 } while (txq != NULL); 75 } 76 77 if (rxq->pending != rxq->completed) 78 sfxge_rx_qcomplete(rxq, eop); 79 } 80 81 static boolean_t 82 sfxge_ev_rx(void *arg, uint32_t label, uint32_t id, uint32_t size, 83 uint16_t flags) 84 { 85 struct sfxge_evq *evq; 86 struct sfxge_softc *sc; 87 struct sfxge_rxq *rxq; 88 unsigned int expected; 89 struct sfxge_rx_sw_desc *rx_desc; 90 91 evq = arg; 92 sc = evq->sc; 93 94 if (evq->exception) 95 goto done; 96 97 rxq = sc->rxq[label]; 98 KASSERT(rxq != NULL, ("rxq == NULL")); 99 KASSERT(evq->index == rxq->index, 100 ("evq->index != rxq->index")); 101 102 if (rxq->init_state != SFXGE_RXQ_STARTED) 103 goto done; 104 105 expected = rxq->pending++ & rxq->ptr_mask; 106 if (id != expected) { 107 evq->exception = B_TRUE; 108 109 device_printf(sc->dev, "RX completion out of order" 110 " (id=%#x expected=%#x flags=%#x); resetting\n", 111 id, expected, flags); 112 sfxge_schedule_reset(sc); 113 114 goto done; 115 } 116 117 rx_desc = &rxq->queue[id]; 118 119 KASSERT(rx_desc->flags == EFX_DISCARD, 120 ("rx_desc->flags != EFX_DISCARD")); 121 rx_desc->flags = flags; 122 123 KASSERT(size < (1 << 16), ("size > (1 << 16)")); 124 rx_desc->size = (uint16_t)size; 125 prefetch_read_many(rx_desc->mbuf); 126 127 evq->rx_done++; 128 129 if (rxq->pending - rxq->completed >= SFXGE_RX_BATCH) 130 sfxge_ev_qcomplete(evq, B_FALSE); 131 132 done: 133 return (evq->rx_done >= SFXGE_EV_BATCH); 134 } 135 136 static boolean_t 137 sfxge_ev_exception(void *arg, uint32_t code, uint32_t data) 138 { 139 struct sfxge_evq *evq; 140 struct sfxge_softc *sc; 141 142 evq = (struct sfxge_evq *)arg; 143 sc = evq->sc; 144 145 evq->exception = B_TRUE; 146 147 if (code != EFX_EXCEPTION_UNKNOWN_SENSOREVT) { 148 device_printf(sc->dev, 149 "hardware exception (code=%u); resetting\n", 150 code); 151 sfxge_schedule_reset(sc); 152 } 153 154 return (B_FALSE); 155 } 156 157 static boolean_t 158 sfxge_ev_rxq_flush_done(void *arg, uint32_t rxq_index) 159 { 160 struct sfxge_evq *evq; 161 struct sfxge_softc *sc; 162 struct sfxge_rxq *rxq; 163 unsigned int index; 164 unsigned int label; 165 uint16_t magic; 166 167 evq = (struct sfxge_evq *)arg; 168 sc = evq->sc; 169 rxq = sc->rxq[rxq_index]; 170 171 KASSERT(rxq != NULL, ("rxq == NULL")); 172 173 /* Resend a software event on the correct queue */ 174 index = rxq->index; 175 evq = sc->evq[index]; 176 177 label = rxq_index; 178 KASSERT((label & SFXGE_MAGIC_DMAQ_LABEL_MASK) == label, 179 ("(label & SFXGE_MAGIC_DMAQ_LABEL_MASK) != level")); 180 magic = SFXGE_MAGIC_RX_QFLUSH_DONE | label; 181 182 KASSERT(evq->init_state == SFXGE_EVQ_STARTED, 183 ("evq not started")); 184 efx_ev_qpost(evq->common, magic); 185 186 return (B_FALSE); 187 } 188 189 static boolean_t 190 sfxge_ev_rxq_flush_failed(void *arg, uint32_t rxq_index) 191 { 192 struct sfxge_evq *evq; 193 struct sfxge_softc *sc; 194 struct sfxge_rxq *rxq; 195 unsigned int index; 196 unsigned int label; 197 uint16_t magic; 198 199 evq = (struct sfxge_evq *)arg; 200 sc = evq->sc; 201 rxq = sc->rxq[rxq_index]; 202 203 KASSERT(rxq != NULL, ("rxq == NULL")); 204 205 /* Resend a software event on the correct queue */ 206 index = rxq->index; 207 evq = sc->evq[index]; 208 209 label = rxq_index; 210 KASSERT((label & SFXGE_MAGIC_DMAQ_LABEL_MASK) == label, 211 ("(label & SFXGE_MAGIC_DMAQ_LABEL_MASK) != label")); 212 magic = SFXGE_MAGIC_RX_QFLUSH_FAILED | label; 213 214 KASSERT(evq->init_state == SFXGE_EVQ_STARTED, 215 ("evq not started")); 216 efx_ev_qpost(evq->common, magic); 217 218 return (B_FALSE); 219 } 220 221 static struct sfxge_txq * 222 sfxge_get_txq_by_label(struct sfxge_evq *evq, enum sfxge_txq_type label) 223 { 224 unsigned int index; 225 226 KASSERT((evq->index == 0 && label < SFXGE_TXQ_NTYPES) || 227 (label == SFXGE_TXQ_IP_TCP_UDP_CKSUM), ("unexpected txq label")); 228 index = (evq->index == 0) ? label : (evq->index - 1 + SFXGE_TXQ_NTYPES); 229 return (evq->sc->txq[index]); 230 } 231 232 static boolean_t 233 sfxge_ev_tx(void *arg, uint32_t label, uint32_t id) 234 { 235 struct sfxge_evq *evq; 236 struct sfxge_txq *txq; 237 unsigned int stop; 238 unsigned int delta; 239 240 evq = (struct sfxge_evq *)arg; 241 txq = sfxge_get_txq_by_label(evq, label); 242 243 KASSERT(txq != NULL, ("txq == NULL")); 244 KASSERT(evq->index == txq->evq_index, 245 ("evq->index != txq->evq_index")); 246 247 if (txq->init_state != SFXGE_TXQ_STARTED) 248 goto done; 249 250 stop = (id + 1) & txq->ptr_mask; 251 id = txq->pending & txq->ptr_mask; 252 253 delta = (stop >= id) ? (stop - id) : (txq->entries - id + stop); 254 txq->pending += delta; 255 256 evq->tx_done++; 257 258 if (txq->next == NULL && 259 evq->txqs != &(txq->next)) { 260 *(evq->txqs) = txq; 261 evq->txqs = &(txq->next); 262 } 263 264 if (txq->pending - txq->completed >= SFXGE_TX_BATCH) 265 sfxge_tx_qcomplete(txq, evq); 266 267 done: 268 return (evq->tx_done >= SFXGE_EV_BATCH); 269 } 270 271 static boolean_t 272 sfxge_ev_txq_flush_done(void *arg, uint32_t txq_index) 273 { 274 struct sfxge_evq *evq; 275 struct sfxge_softc *sc; 276 struct sfxge_txq *txq; 277 unsigned int label; 278 uint16_t magic; 279 280 evq = (struct sfxge_evq *)arg; 281 sc = evq->sc; 282 txq = sc->txq[txq_index]; 283 284 KASSERT(txq != NULL, ("txq == NULL")); 285 KASSERT(txq->init_state == SFXGE_TXQ_INITIALIZED, 286 ("txq not initialized")); 287 288 /* Resend a software event on the correct queue */ 289 evq = sc->evq[txq->evq_index]; 290 291 label = txq->type; 292 KASSERT((label & SFXGE_MAGIC_DMAQ_LABEL_MASK) == label, 293 ("(label & SFXGE_MAGIC_DMAQ_LABEL_MASK) != label")); 294 magic = SFXGE_MAGIC_TX_QFLUSH_DONE | label; 295 296 KASSERT(evq->init_state == SFXGE_EVQ_STARTED, 297 ("evq not started")); 298 efx_ev_qpost(evq->common, magic); 299 300 return (B_FALSE); 301 } 302 303 static boolean_t 304 sfxge_ev_software(void *arg, uint16_t magic) 305 { 306 struct sfxge_evq *evq; 307 struct sfxge_softc *sc; 308 unsigned int label; 309 310 evq = (struct sfxge_evq *)arg; 311 sc = evq->sc; 312 313 label = magic & SFXGE_MAGIC_DMAQ_LABEL_MASK; 314 magic &= ~SFXGE_MAGIC_DMAQ_LABEL_MASK; 315 316 switch (magic) { 317 case SFXGE_MAGIC_RX_QFLUSH_DONE: { 318 struct sfxge_rxq *rxq = sc->rxq[label]; 319 320 KASSERT(rxq != NULL, ("rxq == NULL")); 321 KASSERT(evq->index == rxq->index, 322 ("evq->index != rxq->index")); 323 324 sfxge_rx_qflush_done(rxq); 325 break; 326 } 327 case SFXGE_MAGIC_RX_QFLUSH_FAILED: { 328 struct sfxge_rxq *rxq = sc->rxq[label]; 329 330 KASSERT(rxq != NULL, ("rxq == NULL")); 331 KASSERT(evq->index == rxq->index, 332 ("evq->index != rxq->index")); 333 334 sfxge_rx_qflush_failed(rxq); 335 break; 336 } 337 case SFXGE_MAGIC_RX_QREFILL: { 338 struct sfxge_rxq *rxq = sc->rxq[label]; 339 340 KASSERT(rxq != NULL, ("rxq == NULL")); 341 KASSERT(evq->index == rxq->index, 342 ("evq->index != rxq->index")); 343 344 sfxge_rx_qrefill(rxq); 345 break; 346 } 347 case SFXGE_MAGIC_TX_QFLUSH_DONE: { 348 struct sfxge_txq *txq = sfxge_get_txq_by_label(evq, label); 349 350 KASSERT(txq != NULL, ("txq == NULL")); 351 KASSERT(evq->index == txq->evq_index, 352 ("evq->index != txq->evq_index")); 353 354 sfxge_tx_qflush_done(txq); 355 break; 356 } 357 default: 358 break; 359 } 360 361 return (B_FALSE); 362 } 363 364 static boolean_t 365 sfxge_ev_sram(void *arg, uint32_t code) 366 { 367 (void)arg; 368 (void)code; 369 370 switch (code) { 371 case EFX_SRAM_UPDATE: 372 EFSYS_PROBE(sram_update); 373 break; 374 375 case EFX_SRAM_CLEAR: 376 EFSYS_PROBE(sram_clear); 377 break; 378 379 case EFX_SRAM_ILLEGAL_CLEAR: 380 EFSYS_PROBE(sram_illegal_clear); 381 break; 382 383 default: 384 KASSERT(B_FALSE, ("Impossible SRAM event")); 385 break; 386 } 387 388 return (B_FALSE); 389 } 390 391 static boolean_t 392 sfxge_ev_timer(void *arg, uint32_t index) 393 { 394 (void)arg; 395 (void)index; 396 397 return (B_FALSE); 398 } 399 400 static boolean_t 401 sfxge_ev_wake_up(void *arg, uint32_t index) 402 { 403 (void)arg; 404 (void)index; 405 406 return (B_FALSE); 407 } 408 409 #if EFSYS_OPT_QSTATS 410 411 static void 412 sfxge_ev_stat_update(struct sfxge_softc *sc) 413 { 414 struct sfxge_evq *evq; 415 unsigned int index; 416 clock_t now; 417 418 SFXGE_ADAPTER_LOCK(sc); 419 420 if (sc->evq[0]->init_state != SFXGE_EVQ_STARTED) 421 goto out; 422 423 now = ticks; 424 if (now - sc->ev_stats_update_time < hz) 425 goto out; 426 427 sc->ev_stats_update_time = now; 428 429 /* Add event counts from each event queue in turn */ 430 for (index = 0; index < sc->intr.n_alloc; index++) { 431 evq = sc->evq[index]; 432 SFXGE_EVQ_LOCK(evq); 433 efx_ev_qstats_update(evq->common, sc->ev_stats); 434 SFXGE_EVQ_UNLOCK(evq); 435 } 436 out: 437 SFXGE_ADAPTER_UNLOCK(sc); 438 } 439 440 static int 441 sfxge_ev_stat_handler(SYSCTL_HANDLER_ARGS) 442 { 443 struct sfxge_softc *sc = arg1; 444 unsigned int id = arg2; 445 446 sfxge_ev_stat_update(sc); 447 448 return (SYSCTL_OUT(req, &sc->ev_stats[id], sizeof(sc->ev_stats[id]))); 449 } 450 451 static void 452 sfxge_ev_stat_init(struct sfxge_softc *sc) 453 { 454 struct sysctl_ctx_list *ctx = device_get_sysctl_ctx(sc->dev); 455 struct sysctl_oid_list *stat_list; 456 unsigned int id; 457 char name[40]; 458 459 stat_list = SYSCTL_CHILDREN(sc->stats_node); 460 461 for (id = 0; id < EV_NQSTATS; id++) { 462 snprintf(name, sizeof(name), "ev_%s", 463 efx_ev_qstat_name(sc->enp, id)); 464 SYSCTL_ADD_PROC( 465 ctx, stat_list, 466 OID_AUTO, name, CTLTYPE_U64|CTLFLAG_RD, 467 sc, id, sfxge_ev_stat_handler, "Q", 468 ""); 469 } 470 } 471 472 #endif /* EFSYS_OPT_QSTATS */ 473 474 static void 475 sfxge_ev_qmoderate(struct sfxge_softc *sc, unsigned int idx, unsigned int us) 476 { 477 struct sfxge_evq *evq; 478 efx_evq_t *eep; 479 480 evq = sc->evq[idx]; 481 eep = evq->common; 482 483 KASSERT(evq->init_state == SFXGE_EVQ_STARTED, 484 ("evq->init_state != SFXGE_EVQ_STARTED")); 485 486 (void)efx_ev_qmoderate(eep, us); 487 } 488 489 static int 490 sfxge_int_mod_handler(SYSCTL_HANDLER_ARGS) 491 { 492 struct sfxge_softc *sc = arg1; 493 struct sfxge_intr *intr = &sc->intr; 494 unsigned int moderation; 495 int error; 496 int index; 497 498 SFXGE_ADAPTER_LOCK(sc); 499 500 if (req->newptr != NULL) { 501 if ((error = SYSCTL_IN(req, &moderation, sizeof(moderation))) 502 != 0) 503 goto out; 504 505 /* We may not be calling efx_ev_qmoderate() now, 506 * so we have to range-check the value ourselves. 507 */ 508 if (moderation > 509 efx_nic_cfg_get(sc->enp)->enc_evq_moderation_max) { 510 error = EINVAL; 511 goto out; 512 } 513 514 sc->ev_moderation = moderation; 515 if (intr->state == SFXGE_INTR_STARTED) { 516 for (index = 0; index < intr->n_alloc; index++) 517 sfxge_ev_qmoderate(sc, index, moderation); 518 } 519 } else { 520 error = SYSCTL_OUT(req, &sc->ev_moderation, 521 sizeof(sc->ev_moderation)); 522 } 523 524 out: 525 SFXGE_ADAPTER_UNLOCK(sc); 526 527 return (error); 528 } 529 530 static boolean_t 531 sfxge_ev_initialized(void *arg) 532 { 533 struct sfxge_evq *evq; 534 535 evq = (struct sfxge_evq *)arg; 536 537 KASSERT(evq->init_state == SFXGE_EVQ_STARTING, 538 ("evq not starting")); 539 540 evq->init_state = SFXGE_EVQ_STARTED; 541 542 return (0); 543 } 544 545 static boolean_t 546 sfxge_ev_link_change(void *arg, efx_link_mode_t link_mode) 547 { 548 struct sfxge_evq *evq; 549 struct sfxge_softc *sc; 550 551 evq = (struct sfxge_evq *)arg; 552 sc = evq->sc; 553 554 sfxge_mac_link_update(sc, link_mode); 555 556 return (0); 557 } 558 559 static const efx_ev_callbacks_t sfxge_ev_callbacks = { 560 .eec_initialized = sfxge_ev_initialized, 561 .eec_rx = sfxge_ev_rx, 562 .eec_tx = sfxge_ev_tx, 563 .eec_exception = sfxge_ev_exception, 564 .eec_rxq_flush_done = sfxge_ev_rxq_flush_done, 565 .eec_rxq_flush_failed = sfxge_ev_rxq_flush_failed, 566 .eec_txq_flush_done = sfxge_ev_txq_flush_done, 567 .eec_software = sfxge_ev_software, 568 .eec_sram = sfxge_ev_sram, 569 .eec_wake_up = sfxge_ev_wake_up, 570 .eec_timer = sfxge_ev_timer, 571 .eec_link_change = sfxge_ev_link_change, 572 }; 573 574 575 int 576 sfxge_ev_qpoll(struct sfxge_evq *evq) 577 { 578 int rc; 579 580 SFXGE_EVQ_LOCK(evq); 581 582 if (evq->init_state != SFXGE_EVQ_STARTING && 583 evq->init_state != SFXGE_EVQ_STARTED) { 584 rc = EINVAL; 585 goto fail; 586 } 587 588 /* Synchronize the DMA memory for reading */ 589 bus_dmamap_sync(evq->mem.esm_tag, evq->mem.esm_map, 590 BUS_DMASYNC_POSTREAD); 591 592 KASSERT(evq->rx_done == 0, ("evq->rx_done != 0")); 593 KASSERT(evq->tx_done == 0, ("evq->tx_done != 0")); 594 KASSERT(evq->txq == NULL, ("evq->txq != NULL")); 595 KASSERT(evq->txqs == &evq->txq, ("evq->txqs != &evq->txq")); 596 597 /* Poll the queue */ 598 efx_ev_qpoll(evq->common, &evq->read_ptr, &sfxge_ev_callbacks, evq); 599 600 evq->rx_done = 0; 601 evq->tx_done = 0; 602 603 /* Perform any pending completion processing */ 604 sfxge_ev_qcomplete(evq, B_TRUE); 605 606 /* Re-prime the event queue for interrupts */ 607 if ((rc = efx_ev_qprime(evq->common, evq->read_ptr)) != 0) 608 goto fail; 609 610 SFXGE_EVQ_UNLOCK(evq); 611 612 return (0); 613 614 fail: 615 SFXGE_EVQ_UNLOCK(evq); 616 return (rc); 617 } 618 619 static void 620 sfxge_ev_qstop(struct sfxge_softc *sc, unsigned int index) 621 { 622 struct sfxge_evq *evq; 623 624 evq = sc->evq[index]; 625 626 KASSERT(evq->init_state == SFXGE_EVQ_STARTED, 627 ("evq->init_state != SFXGE_EVQ_STARTED")); 628 629 SFXGE_EVQ_LOCK(evq); 630 evq->init_state = SFXGE_EVQ_INITIALIZED; 631 evq->read_ptr = 0; 632 evq->exception = B_FALSE; 633 634 #if EFSYS_OPT_QSTATS 635 /* Add event counts before discarding the common evq state */ 636 efx_ev_qstats_update(evq->common, sc->ev_stats); 637 #endif 638 639 efx_ev_qdestroy(evq->common); 640 efx_sram_buf_tbl_clear(sc->enp, evq->buf_base_id, 641 EFX_EVQ_NBUFS(evq->entries)); 642 SFXGE_EVQ_UNLOCK(evq); 643 } 644 645 static int 646 sfxge_ev_qstart(struct sfxge_softc *sc, unsigned int index) 647 { 648 struct sfxge_evq *evq; 649 efsys_mem_t *esmp; 650 int count; 651 int rc; 652 653 evq = sc->evq[index]; 654 esmp = &evq->mem; 655 656 KASSERT(evq->init_state == SFXGE_EVQ_INITIALIZED, 657 ("evq->init_state != SFXGE_EVQ_INITIALIZED")); 658 659 /* Clear all events. */ 660 (void)memset(esmp->esm_base, 0xff, EFX_EVQ_SIZE(evq->entries)); 661 662 /* Program the buffer table. */ 663 if ((rc = efx_sram_buf_tbl_set(sc->enp, evq->buf_base_id, esmp, 664 EFX_EVQ_NBUFS(evq->entries))) != 0) 665 return (rc); 666 667 /* Create the common code event queue. */ 668 if ((rc = efx_ev_qcreate(sc->enp, index, esmp, evq->entries, 669 evq->buf_base_id, &evq->common)) != 0) 670 goto fail; 671 672 SFXGE_EVQ_LOCK(evq); 673 674 /* Set the default moderation */ 675 (void)efx_ev_qmoderate(evq->common, sc->ev_moderation); 676 677 /* Prime the event queue for interrupts */ 678 if ((rc = efx_ev_qprime(evq->common, evq->read_ptr)) != 0) 679 goto fail2; 680 681 evq->init_state = SFXGE_EVQ_STARTING; 682 683 SFXGE_EVQ_UNLOCK(evq); 684 685 /* Wait for the initialization event */ 686 count = 0; 687 do { 688 /* Pause for 100 ms */ 689 pause("sfxge evq init", hz / 10); 690 691 /* Check to see if the test event has been processed */ 692 if (evq->init_state == SFXGE_EVQ_STARTED) 693 goto done; 694 695 } while (++count < 20); 696 697 rc = ETIMEDOUT; 698 goto fail3; 699 700 done: 701 return (0); 702 703 fail3: 704 SFXGE_EVQ_LOCK(evq); 705 evq->init_state = SFXGE_EVQ_INITIALIZED; 706 fail2: 707 SFXGE_EVQ_UNLOCK(evq); 708 efx_ev_qdestroy(evq->common); 709 fail: 710 efx_sram_buf_tbl_clear(sc->enp, evq->buf_base_id, 711 EFX_EVQ_NBUFS(evq->entries)); 712 713 return (rc); 714 } 715 716 void 717 sfxge_ev_stop(struct sfxge_softc *sc) 718 { 719 struct sfxge_intr *intr; 720 efx_nic_t *enp; 721 int index; 722 723 intr = &sc->intr; 724 enp = sc->enp; 725 726 KASSERT(intr->state == SFXGE_INTR_STARTED, 727 ("Interrupts not started")); 728 729 /* Stop the event queue(s) */ 730 index = intr->n_alloc; 731 while (--index >= 0) 732 sfxge_ev_qstop(sc, index); 733 734 /* Tear down the event module */ 735 efx_ev_fini(enp); 736 } 737 738 int 739 sfxge_ev_start(struct sfxge_softc *sc) 740 { 741 struct sfxge_intr *intr; 742 int index; 743 int rc; 744 745 intr = &sc->intr; 746 747 KASSERT(intr->state == SFXGE_INTR_STARTED, 748 ("intr->state != SFXGE_INTR_STARTED")); 749 750 /* Initialize the event module */ 751 if ((rc = efx_ev_init(sc->enp)) != 0) 752 return (rc); 753 754 /* Start the event queues */ 755 for (index = 0; index < intr->n_alloc; index++) { 756 if ((rc = sfxge_ev_qstart(sc, index)) != 0) 757 goto fail; 758 } 759 760 return (0); 761 762 fail: 763 /* Stop the event queue(s) */ 764 while (--index >= 0) 765 sfxge_ev_qstop(sc, index); 766 767 /* Tear down the event module */ 768 efx_ev_fini(sc->enp); 769 770 return (rc); 771 } 772 773 static void 774 sfxge_ev_qfini(struct sfxge_softc *sc, unsigned int index) 775 { 776 struct sfxge_evq *evq; 777 778 evq = sc->evq[index]; 779 780 KASSERT(evq->init_state == SFXGE_EVQ_INITIALIZED, 781 ("evq->init_state != SFXGE_EVQ_INITIALIZED")); 782 KASSERT(evq->txqs == &evq->txq, ("evq->txqs != &evq->txq")); 783 784 sfxge_dma_free(&evq->mem); 785 786 sc->evq[index] = NULL; 787 788 SFXGE_EVQ_LOCK_DESTROY(evq); 789 790 free(evq, M_SFXGE); 791 } 792 793 static int 794 sfxge_ev_qinit(struct sfxge_softc *sc, unsigned int index) 795 { 796 struct sfxge_evq *evq; 797 efsys_mem_t *esmp; 798 int rc; 799 800 KASSERT(index < SFXGE_RX_SCALE_MAX, ("index >= SFXGE_RX_SCALE_MAX")); 801 802 evq = malloc(sizeof(struct sfxge_evq), M_SFXGE, M_ZERO | M_WAITOK); 803 evq->sc = sc; 804 evq->index = index; 805 sc->evq[index] = evq; 806 esmp = &evq->mem; 807 808 /* Build an event queue with room for one event per tx and rx buffer, 809 * plus some extra for link state events and MCDI completions. 810 * There are three tx queues in the first event queue and one in 811 * other. 812 */ 813 if (index == 0) 814 evq->entries = 815 ROUNDUP_POW_OF_TWO(sc->rxq_entries + 816 3 * sc->txq_entries + 817 128); 818 else 819 evq->entries = 820 ROUNDUP_POW_OF_TWO(sc->rxq_entries + 821 sc->txq_entries + 822 128); 823 824 /* Initialise TX completion list */ 825 evq->txqs = &evq->txq; 826 827 /* Allocate DMA space. */ 828 if ((rc = sfxge_dma_alloc(sc, EFX_EVQ_SIZE(evq->entries), esmp)) != 0) 829 return (rc); 830 831 /* Allocate buffer table entries. */ 832 sfxge_sram_buf_tbl_alloc(sc, EFX_EVQ_NBUFS(evq->entries), 833 &evq->buf_base_id); 834 835 SFXGE_EVQ_LOCK_INIT(evq, device_get_nameunit(sc->dev), index); 836 837 evq->init_state = SFXGE_EVQ_INITIALIZED; 838 839 return (0); 840 } 841 842 void 843 sfxge_ev_fini(struct sfxge_softc *sc) 844 { 845 struct sfxge_intr *intr; 846 int index; 847 848 intr = &sc->intr; 849 850 KASSERT(intr->state == SFXGE_INTR_INITIALIZED, 851 ("intr->state != SFXGE_INTR_INITIALIZED")); 852 853 sc->ev_moderation = 0; 854 855 /* Tear down the event queue(s). */ 856 index = intr->n_alloc; 857 while (--index >= 0) 858 sfxge_ev_qfini(sc, index); 859 } 860 861 int 862 sfxge_ev_init(struct sfxge_softc *sc) 863 { 864 struct sysctl_ctx_list *sysctl_ctx = device_get_sysctl_ctx(sc->dev); 865 struct sysctl_oid *sysctl_tree = device_get_sysctl_tree(sc->dev); 866 struct sfxge_intr *intr; 867 int index; 868 int rc; 869 870 intr = &sc->intr; 871 872 KASSERT(intr->state == SFXGE_INTR_INITIALIZED, 873 ("intr->state != SFXGE_INTR_INITIALIZED")); 874 875 /* Set default interrupt moderation; add a sysctl to 876 * read and change it. 877 */ 878 sc->ev_moderation = SFXGE_MODERATION; 879 SYSCTL_ADD_PROC(sysctl_ctx, SYSCTL_CHILDREN(sysctl_tree), 880 OID_AUTO, "int_mod", CTLTYPE_UINT|CTLFLAG_RW, 881 sc, 0, sfxge_int_mod_handler, "IU", 882 "sfxge interrupt moderation (us)"); 883 884 /* 885 * Initialize the event queue(s) - one per interrupt. 886 */ 887 for (index = 0; index < intr->n_alloc; index++) { 888 if ((rc = sfxge_ev_qinit(sc, index)) != 0) 889 goto fail; 890 } 891 892 #if EFSYS_OPT_QSTATS 893 sfxge_ev_stat_init(sc); 894 #endif 895 896 return (0); 897 898 fail: 899 while (--index >= 0) 900 sfxge_ev_qfini(sc, index); 901 902 return (rc); 903 } 904