1 /*- 2 * Copyright (c) 2010-2016 Solarflare Communications Inc. 3 * All rights reserved. 4 * 5 * This software was developed in part by Philip Paeps under contract for 6 * Solarflare Communications, Inc. 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions are met: 10 * 11 * 1. Redistributions of source code must retain the above copyright notice, 12 * this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright notice, 14 * this list of conditions and the following disclaimer in the documentation 15 * and/or other materials provided with the distribution. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 18 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, 19 * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 20 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR 21 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, 22 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 23 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; 24 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 25 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR 26 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, 27 * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 * 29 * The views and conclusions contained in the software and documentation are 30 * those of the authors and should not be interpreted as representing official 31 * policies, either expressed or implied, of the FreeBSD Project. 32 */ 33 34 #include <sys/cdefs.h> 35 __FBSDID("$FreeBSD$"); 36 37 #include <sys/param.h> 38 #include <sys/kernel.h> 39 #include <sys/malloc.h> 40 #include <sys/param.h> 41 #include <sys/queue.h> 42 #include <sys/systm.h> 43 #include <sys/taskqueue.h> 44 45 #include "common/efx.h" 46 47 #include "sfxge.h" 48 49 static void 50 sfxge_ev_qcomplete(struct sfxge_evq *evq, boolean_t eop) 51 { 52 struct sfxge_softc *sc; 53 unsigned int index; 54 struct sfxge_rxq *rxq; 55 struct sfxge_txq *txq; 56 57 SFXGE_EVQ_LOCK_ASSERT_OWNED(evq); 58 59 sc = evq->sc; 60 index = evq->index; 61 rxq = sc->rxq[index]; 62 63 if ((txq = evq->txq) != NULL) { 64 evq->txq = NULL; 65 evq->txqs = &(evq->txq); 66 67 do { 68 struct sfxge_txq *next; 69 70 next = txq->next; 71 txq->next = NULL; 72 73 KASSERT(txq->evq_index == index, 74 ("txq->evq_index != index")); 75 76 if (txq->pending != txq->completed) 77 sfxge_tx_qcomplete(txq, evq); 78 79 txq = next; 80 } while (txq != NULL); 81 } 82 83 if (rxq->pending != rxq->completed) 84 sfxge_rx_qcomplete(rxq, eop); 85 } 86 87 static struct sfxge_rxq * 88 sfxge_get_rxq_by_label(struct sfxge_evq *evq, uint32_t label) 89 { 90 struct sfxge_rxq *rxq; 91 92 KASSERT(label == 0, ("unexpected rxq label != 0")); 93 94 rxq = evq->sc->rxq[evq->index]; 95 96 KASSERT(rxq != NULL, ("rxq == NULL")); 97 KASSERT(evq->index == rxq->index, ("evq->index != rxq->index")); 98 99 return (rxq); 100 } 101 102 static boolean_t 103 sfxge_ev_rx(void *arg, uint32_t label, uint32_t id, uint32_t size, 104 uint16_t flags) 105 { 106 struct sfxge_evq *evq; 107 struct sfxge_softc *sc; 108 struct sfxge_rxq *rxq; 109 unsigned int stop; 110 unsigned int delta; 111 struct sfxge_rx_sw_desc *rx_desc; 112 113 evq = arg; 114 SFXGE_EVQ_LOCK_ASSERT_OWNED(evq); 115 116 sc = evq->sc; 117 118 if (evq->exception) 119 goto done; 120 121 rxq = sfxge_get_rxq_by_label(evq, label); 122 if (__predict_false(rxq->init_state != SFXGE_RXQ_STARTED)) 123 goto done; 124 125 stop = (id + 1) & rxq->ptr_mask; 126 id = rxq->pending & rxq->ptr_mask; 127 delta = (stop >= id) ? (stop - id) : (rxq->entries - id + stop); 128 rxq->pending += delta; 129 130 if (delta != 1) { 131 if ((!efx_nic_cfg_get(sc->enp)->enc_rx_batching_enabled) || 132 (delta <= 0) || 133 (delta > efx_nic_cfg_get(sc->enp)->enc_rx_batch_max)) { 134 evq->exception = B_TRUE; 135 136 device_printf(sc->dev, "RX completion out of order" 137 " (id=%#x delta=%u flags=%#x); resetting\n", 138 id, delta, flags); 139 sfxge_schedule_reset(sc); 140 141 goto done; 142 } 143 } 144 145 rx_desc = &rxq->queue[id]; 146 147 prefetch_read_many(rx_desc->mbuf); 148 149 for (; id != stop; id = (id + 1) & rxq->ptr_mask) { 150 rx_desc = &rxq->queue[id]; 151 KASSERT(rx_desc->flags == EFX_DISCARD, 152 ("rx_desc->flags != EFX_DISCARD")); 153 rx_desc->flags = flags; 154 155 KASSERT(size < (1 << 16), ("size > (1 << 16)")); 156 rx_desc->size = (uint16_t)size; 157 } 158 159 evq->rx_done++; 160 161 if (rxq->pending - rxq->completed >= SFXGE_RX_BATCH) 162 sfxge_ev_qcomplete(evq, B_FALSE); 163 164 done: 165 return (evq->rx_done >= SFXGE_EV_BATCH); 166 } 167 168 static boolean_t 169 sfxge_ev_exception(void *arg, uint32_t code, uint32_t data) 170 { 171 struct sfxge_evq *evq; 172 struct sfxge_softc *sc; 173 174 evq = (struct sfxge_evq *)arg; 175 SFXGE_EVQ_LOCK_ASSERT_OWNED(evq); 176 177 sc = evq->sc; 178 179 DBGPRINT(sc->dev, "[%d] %s", evq->index, 180 (code == EFX_EXCEPTION_RX_RECOVERY) ? "RX_RECOVERY" : 181 (code == EFX_EXCEPTION_RX_DSC_ERROR) ? "RX_DSC_ERROR" : 182 (code == EFX_EXCEPTION_TX_DSC_ERROR) ? "TX_DSC_ERROR" : 183 (code == EFX_EXCEPTION_UNKNOWN_SENSOREVT) ? "UNKNOWN_SENSOREVT" : 184 (code == EFX_EXCEPTION_FWALERT_SRAM) ? "FWALERT_SRAM" : 185 (code == EFX_EXCEPTION_UNKNOWN_FWALERT) ? "UNKNOWN_FWALERT" : 186 (code == EFX_EXCEPTION_RX_ERROR) ? "RX_ERROR" : 187 (code == EFX_EXCEPTION_TX_ERROR) ? "TX_ERROR" : 188 (code == EFX_EXCEPTION_EV_ERROR) ? "EV_ERROR" : 189 "UNKNOWN"); 190 191 evq->exception = B_TRUE; 192 193 if (code != EFX_EXCEPTION_UNKNOWN_SENSOREVT) { 194 device_printf(sc->dev, 195 "hardware exception (code=%u); resetting\n", 196 code); 197 sfxge_schedule_reset(sc); 198 } 199 200 return (B_FALSE); 201 } 202 203 static boolean_t 204 sfxge_ev_rxq_flush_done(void *arg, uint32_t rxq_index) 205 { 206 struct sfxge_evq *evq; 207 struct sfxge_softc *sc; 208 struct sfxge_rxq *rxq; 209 unsigned int index; 210 uint16_t magic; 211 212 evq = (struct sfxge_evq *)arg; 213 SFXGE_EVQ_LOCK_ASSERT_OWNED(evq); 214 215 sc = evq->sc; 216 rxq = sc->rxq[rxq_index]; 217 218 KASSERT(rxq != NULL, ("rxq == NULL")); 219 220 /* Resend a software event on the correct queue */ 221 index = rxq->index; 222 if (index == evq->index) { 223 sfxge_rx_qflush_done(rxq); 224 return (B_FALSE); 225 } 226 227 evq = sc->evq[index]; 228 magic = sfxge_sw_ev_rxq_magic(SFXGE_SW_EV_RX_QFLUSH_DONE, rxq); 229 230 KASSERT(evq->init_state == SFXGE_EVQ_STARTED, 231 ("evq not started")); 232 efx_ev_qpost(evq->common, magic); 233 234 return (B_FALSE); 235 } 236 237 static boolean_t 238 sfxge_ev_rxq_flush_failed(void *arg, uint32_t rxq_index) 239 { 240 struct sfxge_evq *evq; 241 struct sfxge_softc *sc; 242 struct sfxge_rxq *rxq; 243 unsigned int index; 244 uint16_t magic; 245 246 evq = (struct sfxge_evq *)arg; 247 SFXGE_EVQ_LOCK_ASSERT_OWNED(evq); 248 249 sc = evq->sc; 250 rxq = sc->rxq[rxq_index]; 251 252 KASSERT(rxq != NULL, ("rxq == NULL")); 253 254 /* Resend a software event on the correct queue */ 255 index = rxq->index; 256 evq = sc->evq[index]; 257 magic = sfxge_sw_ev_rxq_magic(SFXGE_SW_EV_RX_QFLUSH_FAILED, rxq); 258 259 KASSERT(evq->init_state == SFXGE_EVQ_STARTED, 260 ("evq not started")); 261 efx_ev_qpost(evq->common, magic); 262 263 return (B_FALSE); 264 } 265 266 static struct sfxge_txq * 267 sfxge_get_txq_by_label(struct sfxge_evq *evq, enum sfxge_txq_type label) 268 { 269 unsigned int index; 270 271 KASSERT((evq->index == 0 && label < SFXGE_TXQ_NTYPES) || 272 (label == SFXGE_TXQ_IP_TCP_UDP_CKSUM), ("unexpected txq label")); 273 index = (evq->index == 0) ? label : (evq->index - 1 + SFXGE_TXQ_NTYPES); 274 return (evq->sc->txq[index]); 275 } 276 277 static boolean_t 278 sfxge_ev_tx(void *arg, uint32_t label, uint32_t id) 279 { 280 struct sfxge_evq *evq; 281 struct sfxge_txq *txq; 282 unsigned int stop; 283 unsigned int delta; 284 285 evq = (struct sfxge_evq *)arg; 286 SFXGE_EVQ_LOCK_ASSERT_OWNED(evq); 287 288 txq = sfxge_get_txq_by_label(evq, label); 289 290 KASSERT(txq != NULL, ("txq == NULL")); 291 KASSERT(evq->index == txq->evq_index, 292 ("evq->index != txq->evq_index")); 293 294 if (__predict_false(txq->init_state != SFXGE_TXQ_STARTED)) 295 goto done; 296 297 stop = (id + 1) & txq->ptr_mask; 298 id = txq->pending & txq->ptr_mask; 299 300 delta = (stop >= id) ? (stop - id) : (txq->entries - id + stop); 301 txq->pending += delta; 302 303 evq->tx_done++; 304 305 if (txq->next == NULL && 306 evq->txqs != &(txq->next)) { 307 *(evq->txqs) = txq; 308 evq->txqs = &(txq->next); 309 } 310 311 if (txq->pending - txq->completed >= SFXGE_TX_BATCH) 312 sfxge_tx_qcomplete(txq, evq); 313 314 done: 315 return (evq->tx_done >= SFXGE_EV_BATCH); 316 } 317 318 static boolean_t 319 sfxge_ev_txq_flush_done(void *arg, uint32_t txq_index) 320 { 321 struct sfxge_evq *evq; 322 struct sfxge_softc *sc; 323 struct sfxge_txq *txq; 324 uint16_t magic; 325 326 evq = (struct sfxge_evq *)arg; 327 SFXGE_EVQ_LOCK_ASSERT_OWNED(evq); 328 329 sc = evq->sc; 330 txq = sc->txq[txq_index]; 331 332 KASSERT(txq != NULL, ("txq == NULL")); 333 KASSERT(txq->init_state == SFXGE_TXQ_INITIALIZED, 334 ("txq not initialized")); 335 336 if (txq->evq_index == evq->index) { 337 sfxge_tx_qflush_done(txq); 338 return (B_FALSE); 339 } 340 341 /* Resend a software event on the correct queue */ 342 evq = sc->evq[txq->evq_index]; 343 magic = sfxge_sw_ev_txq_magic(SFXGE_SW_EV_TX_QFLUSH_DONE, txq); 344 345 KASSERT(evq->init_state == SFXGE_EVQ_STARTED, 346 ("evq not started")); 347 efx_ev_qpost(evq->common, magic); 348 349 return (B_FALSE); 350 } 351 352 static boolean_t 353 sfxge_ev_software(void *arg, uint16_t magic) 354 { 355 struct sfxge_evq *evq; 356 struct sfxge_softc *sc; 357 unsigned int label; 358 359 evq = (struct sfxge_evq *)arg; 360 SFXGE_EVQ_LOCK_ASSERT_OWNED(evq); 361 362 sc = evq->sc; 363 364 label = magic & SFXGE_MAGIC_DMAQ_LABEL_MASK; 365 magic &= ~SFXGE_MAGIC_DMAQ_LABEL_MASK; 366 367 switch (magic) { 368 case SFXGE_SW_EV_MAGIC(SFXGE_SW_EV_RX_QFLUSH_DONE): 369 sfxge_rx_qflush_done(sfxge_get_rxq_by_label(evq, label)); 370 break; 371 372 case SFXGE_SW_EV_MAGIC(SFXGE_SW_EV_RX_QFLUSH_FAILED): 373 sfxge_rx_qflush_failed(sfxge_get_rxq_by_label(evq, label)); 374 break; 375 376 case SFXGE_SW_EV_MAGIC(SFXGE_SW_EV_RX_QREFILL): 377 sfxge_rx_qrefill(sfxge_get_rxq_by_label(evq, label)); 378 break; 379 380 case SFXGE_SW_EV_MAGIC(SFXGE_SW_EV_TX_QFLUSH_DONE): { 381 struct sfxge_txq *txq = sfxge_get_txq_by_label(evq, label); 382 383 KASSERT(txq != NULL, ("txq == NULL")); 384 KASSERT(evq->index == txq->evq_index, 385 ("evq->index != txq->evq_index")); 386 387 sfxge_tx_qflush_done(txq); 388 break; 389 } 390 default: 391 break; 392 } 393 394 return (B_FALSE); 395 } 396 397 static boolean_t 398 sfxge_ev_sram(void *arg, uint32_t code) 399 { 400 (void)arg; 401 (void)code; 402 403 switch (code) { 404 case EFX_SRAM_UPDATE: 405 EFSYS_PROBE(sram_update); 406 break; 407 408 case EFX_SRAM_CLEAR: 409 EFSYS_PROBE(sram_clear); 410 break; 411 412 case EFX_SRAM_ILLEGAL_CLEAR: 413 EFSYS_PROBE(sram_illegal_clear); 414 break; 415 416 default: 417 KASSERT(B_FALSE, ("Impossible SRAM event")); 418 break; 419 } 420 421 return (B_FALSE); 422 } 423 424 static boolean_t 425 sfxge_ev_timer(void *arg, uint32_t index) 426 { 427 (void)arg; 428 (void)index; 429 430 return (B_FALSE); 431 } 432 433 static boolean_t 434 sfxge_ev_wake_up(void *arg, uint32_t index) 435 { 436 (void)arg; 437 (void)index; 438 439 return (B_FALSE); 440 } 441 442 #if EFSYS_OPT_QSTATS 443 444 static void 445 sfxge_ev_stat_update(struct sfxge_softc *sc) 446 { 447 struct sfxge_evq *evq; 448 unsigned int index; 449 clock_t now; 450 451 SFXGE_ADAPTER_LOCK(sc); 452 453 if (__predict_false(sc->evq[0]->init_state != SFXGE_EVQ_STARTED)) 454 goto out; 455 456 now = ticks; 457 if (now - sc->ev_stats_update_time < hz) 458 goto out; 459 460 sc->ev_stats_update_time = now; 461 462 /* Add event counts from each event queue in turn */ 463 for (index = 0; index < sc->evq_count; index++) { 464 evq = sc->evq[index]; 465 SFXGE_EVQ_LOCK(evq); 466 efx_ev_qstats_update(evq->common, sc->ev_stats); 467 SFXGE_EVQ_UNLOCK(evq); 468 } 469 out: 470 SFXGE_ADAPTER_UNLOCK(sc); 471 } 472 473 static int 474 sfxge_ev_stat_handler(SYSCTL_HANDLER_ARGS) 475 { 476 struct sfxge_softc *sc = arg1; 477 unsigned int id = arg2; 478 479 sfxge_ev_stat_update(sc); 480 481 return (SYSCTL_OUT(req, &sc->ev_stats[id], sizeof(sc->ev_stats[id]))); 482 } 483 484 static void 485 sfxge_ev_stat_init(struct sfxge_softc *sc) 486 { 487 struct sysctl_ctx_list *ctx = device_get_sysctl_ctx(sc->dev); 488 struct sysctl_oid_list *stat_list; 489 unsigned int id; 490 char name[40]; 491 492 stat_list = SYSCTL_CHILDREN(sc->stats_node); 493 494 for (id = 0; id < EV_NQSTATS; id++) { 495 snprintf(name, sizeof(name), "ev_%s", 496 efx_ev_qstat_name(sc->enp, id)); 497 SYSCTL_ADD_PROC( 498 ctx, stat_list, 499 OID_AUTO, name, CTLTYPE_U64|CTLFLAG_RD, 500 sc, id, sfxge_ev_stat_handler, "Q", 501 ""); 502 } 503 } 504 505 #endif /* EFSYS_OPT_QSTATS */ 506 507 static void 508 sfxge_ev_qmoderate(struct sfxge_softc *sc, unsigned int idx, unsigned int us) 509 { 510 struct sfxge_evq *evq; 511 efx_evq_t *eep; 512 513 evq = sc->evq[idx]; 514 eep = evq->common; 515 516 KASSERT(evq->init_state == SFXGE_EVQ_STARTED, 517 ("evq->init_state != SFXGE_EVQ_STARTED")); 518 519 (void)efx_ev_qmoderate(eep, us); 520 } 521 522 static int 523 sfxge_int_mod_handler(SYSCTL_HANDLER_ARGS) 524 { 525 struct sfxge_softc *sc = arg1; 526 struct sfxge_intr *intr = &sc->intr; 527 unsigned int moderation; 528 int error; 529 unsigned int index; 530 531 SFXGE_ADAPTER_LOCK(sc); 532 533 if (req->newptr != NULL) { 534 if ((error = SYSCTL_IN(req, &moderation, sizeof(moderation))) 535 != 0) 536 goto out; 537 538 /* We may not be calling efx_ev_qmoderate() now, 539 * so we have to range-check the value ourselves. 540 */ 541 if (moderation > 542 efx_nic_cfg_get(sc->enp)->enc_evq_timer_max_us) { 543 error = EINVAL; 544 goto out; 545 } 546 547 sc->ev_moderation = moderation; 548 if (intr->state == SFXGE_INTR_STARTED) { 549 for (index = 0; index < sc->evq_count; index++) 550 sfxge_ev_qmoderate(sc, index, moderation); 551 } 552 } else { 553 error = SYSCTL_OUT(req, &sc->ev_moderation, 554 sizeof(sc->ev_moderation)); 555 } 556 557 out: 558 SFXGE_ADAPTER_UNLOCK(sc); 559 560 return (error); 561 } 562 563 static boolean_t 564 sfxge_ev_initialized(void *arg) 565 { 566 struct sfxge_evq *evq; 567 568 evq = (struct sfxge_evq *)arg; 569 SFXGE_EVQ_LOCK_ASSERT_OWNED(evq); 570 571 /* Init done events may be duplicated on 7xxx */ 572 KASSERT(evq->init_state == SFXGE_EVQ_STARTING || 573 evq->init_state == SFXGE_EVQ_STARTED, 574 ("evq not starting")); 575 576 evq->init_state = SFXGE_EVQ_STARTED; 577 578 return (0); 579 } 580 581 static boolean_t 582 sfxge_ev_link_change(void *arg, efx_link_mode_t link_mode) 583 { 584 struct sfxge_evq *evq; 585 struct sfxge_softc *sc; 586 587 evq = (struct sfxge_evq *)arg; 588 SFXGE_EVQ_LOCK_ASSERT_OWNED(evq); 589 590 sc = evq->sc; 591 592 sfxge_mac_link_update(sc, link_mode); 593 594 return (0); 595 } 596 597 static const efx_ev_callbacks_t sfxge_ev_callbacks = { 598 .eec_initialized = sfxge_ev_initialized, 599 .eec_rx = sfxge_ev_rx, 600 .eec_tx = sfxge_ev_tx, 601 .eec_exception = sfxge_ev_exception, 602 .eec_rxq_flush_done = sfxge_ev_rxq_flush_done, 603 .eec_rxq_flush_failed = sfxge_ev_rxq_flush_failed, 604 .eec_txq_flush_done = sfxge_ev_txq_flush_done, 605 .eec_software = sfxge_ev_software, 606 .eec_sram = sfxge_ev_sram, 607 .eec_wake_up = sfxge_ev_wake_up, 608 .eec_timer = sfxge_ev_timer, 609 .eec_link_change = sfxge_ev_link_change, 610 }; 611 612 613 int 614 sfxge_ev_qpoll(struct sfxge_evq *evq) 615 { 616 int rc; 617 618 SFXGE_EVQ_LOCK(evq); 619 620 if (__predict_false(evq->init_state != SFXGE_EVQ_STARTING && 621 evq->init_state != SFXGE_EVQ_STARTED)) { 622 rc = EINVAL; 623 goto fail; 624 } 625 626 /* Synchronize the DMA memory for reading */ 627 bus_dmamap_sync(evq->mem.esm_tag, evq->mem.esm_map, 628 BUS_DMASYNC_POSTREAD); 629 630 KASSERT(evq->rx_done == 0, ("evq->rx_done != 0")); 631 KASSERT(evq->tx_done == 0, ("evq->tx_done != 0")); 632 KASSERT(evq->txq == NULL, ("evq->txq != NULL")); 633 KASSERT(evq->txqs == &evq->txq, ("evq->txqs != &evq->txq")); 634 635 /* Poll the queue */ 636 efx_ev_qpoll(evq->common, &evq->read_ptr, &sfxge_ev_callbacks, evq); 637 638 evq->rx_done = 0; 639 evq->tx_done = 0; 640 641 /* Perform any pending completion processing */ 642 sfxge_ev_qcomplete(evq, B_TRUE); 643 644 /* Re-prime the event queue for interrupts */ 645 if ((rc = efx_ev_qprime(evq->common, evq->read_ptr)) != 0) 646 goto fail; 647 648 SFXGE_EVQ_UNLOCK(evq); 649 650 return (0); 651 652 fail: 653 SFXGE_EVQ_UNLOCK(evq); 654 return (rc); 655 } 656 657 static void 658 sfxge_ev_qstop(struct sfxge_softc *sc, unsigned int index) 659 { 660 struct sfxge_evq *evq; 661 662 evq = sc->evq[index]; 663 664 KASSERT(evq->init_state == SFXGE_EVQ_STARTED, 665 ("evq->init_state != SFXGE_EVQ_STARTED")); 666 667 SFXGE_EVQ_LOCK(evq); 668 evq->init_state = SFXGE_EVQ_INITIALIZED; 669 evq->read_ptr = 0; 670 evq->exception = B_FALSE; 671 672 #if EFSYS_OPT_QSTATS 673 /* Add event counts before discarding the common evq state */ 674 efx_ev_qstats_update(evq->common, sc->ev_stats); 675 #endif 676 677 efx_ev_qdestroy(evq->common); 678 efx_sram_buf_tbl_clear(sc->enp, evq->buf_base_id, 679 EFX_EVQ_NBUFS(evq->entries)); 680 SFXGE_EVQ_UNLOCK(evq); 681 } 682 683 static int 684 sfxge_ev_qstart(struct sfxge_softc *sc, unsigned int index) 685 { 686 struct sfxge_evq *evq; 687 efsys_mem_t *esmp; 688 int count; 689 int rc; 690 691 evq = sc->evq[index]; 692 esmp = &evq->mem; 693 694 KASSERT(evq->init_state == SFXGE_EVQ_INITIALIZED, 695 ("evq->init_state != SFXGE_EVQ_INITIALIZED")); 696 697 /* Clear all events. */ 698 (void)memset(esmp->esm_base, 0xff, EFX_EVQ_SIZE(evq->entries)); 699 700 /* Program the buffer table. */ 701 if ((rc = efx_sram_buf_tbl_set(sc->enp, evq->buf_base_id, esmp, 702 EFX_EVQ_NBUFS(evq->entries))) != 0) 703 return (rc); 704 705 /* Create the common code event queue. */ 706 if ((rc = efx_ev_qcreate(sc->enp, index, esmp, evq->entries, 707 evq->buf_base_id, sc->ev_moderation, &evq->common)) != 0) 708 goto fail; 709 710 SFXGE_EVQ_LOCK(evq); 711 712 /* Prime the event queue for interrupts */ 713 if ((rc = efx_ev_qprime(evq->common, evq->read_ptr)) != 0) 714 goto fail2; 715 716 evq->init_state = SFXGE_EVQ_STARTING; 717 718 SFXGE_EVQ_UNLOCK(evq); 719 720 /* Wait for the initialization event */ 721 count = 0; 722 do { 723 /* Pause for 100 ms */ 724 pause("sfxge evq init", hz / 10); 725 726 /* Check to see if the test event has been processed */ 727 if (evq->init_state == SFXGE_EVQ_STARTED) 728 goto done; 729 730 } while (++count < 20); 731 732 rc = ETIMEDOUT; 733 goto fail3; 734 735 done: 736 return (0); 737 738 fail3: 739 SFXGE_EVQ_LOCK(evq); 740 evq->init_state = SFXGE_EVQ_INITIALIZED; 741 fail2: 742 SFXGE_EVQ_UNLOCK(evq); 743 efx_ev_qdestroy(evq->common); 744 fail: 745 efx_sram_buf_tbl_clear(sc->enp, evq->buf_base_id, 746 EFX_EVQ_NBUFS(evq->entries)); 747 748 return (rc); 749 } 750 751 void 752 sfxge_ev_stop(struct sfxge_softc *sc) 753 { 754 struct sfxge_intr *intr; 755 efx_nic_t *enp; 756 int index; 757 758 intr = &sc->intr; 759 enp = sc->enp; 760 761 KASSERT(intr->state == SFXGE_INTR_STARTED, 762 ("Interrupts not started")); 763 764 /* Stop the event queue(s) */ 765 index = sc->evq_count; 766 while (--index >= 0) 767 sfxge_ev_qstop(sc, index); 768 769 /* Tear down the event module */ 770 efx_ev_fini(enp); 771 } 772 773 int 774 sfxge_ev_start(struct sfxge_softc *sc) 775 { 776 struct sfxge_intr *intr; 777 int index; 778 int rc; 779 780 intr = &sc->intr; 781 782 KASSERT(intr->state == SFXGE_INTR_STARTED, 783 ("intr->state != SFXGE_INTR_STARTED")); 784 785 /* Initialize the event module */ 786 if ((rc = efx_ev_init(sc->enp)) != 0) 787 return (rc); 788 789 /* Start the event queues */ 790 for (index = 0; index < sc->evq_count; index++) { 791 if ((rc = sfxge_ev_qstart(sc, index)) != 0) 792 goto fail; 793 } 794 795 return (0); 796 797 fail: 798 /* Stop the event queue(s) */ 799 while (--index >= 0) 800 sfxge_ev_qstop(sc, index); 801 802 /* Tear down the event module */ 803 efx_ev_fini(sc->enp); 804 805 return (rc); 806 } 807 808 static void 809 sfxge_ev_qfini(struct sfxge_softc *sc, unsigned int index) 810 { 811 struct sfxge_evq *evq; 812 813 evq = sc->evq[index]; 814 815 KASSERT(evq->init_state == SFXGE_EVQ_INITIALIZED, 816 ("evq->init_state != SFXGE_EVQ_INITIALIZED")); 817 KASSERT(evq->txqs == &evq->txq, ("evq->txqs != &evq->txq")); 818 819 sfxge_dma_free(&evq->mem); 820 821 sc->evq[index] = NULL; 822 823 SFXGE_EVQ_LOCK_DESTROY(evq); 824 825 free(evq, M_SFXGE); 826 } 827 828 static int 829 sfxge_ev_qinit(struct sfxge_softc *sc, unsigned int index) 830 { 831 struct sfxge_evq *evq; 832 efsys_mem_t *esmp; 833 int rc; 834 835 KASSERT(index < SFXGE_RX_SCALE_MAX, ("index >= SFXGE_RX_SCALE_MAX")); 836 837 evq = malloc(sizeof(struct sfxge_evq), M_SFXGE, M_ZERO | M_WAITOK); 838 evq->sc = sc; 839 evq->index = index; 840 sc->evq[index] = evq; 841 esmp = &evq->mem; 842 843 /* Build an event queue with room for one event per tx and rx buffer, 844 * plus some extra for link state events and MCDI completions. 845 * There are three tx queues in the first event queue and one in 846 * other. 847 */ 848 if (index == 0) 849 evq->entries = 850 ROUNDUP_POW_OF_TWO(sc->rxq_entries + 851 3 * sc->txq_entries + 852 128); 853 else 854 evq->entries = 855 ROUNDUP_POW_OF_TWO(sc->rxq_entries + 856 sc->txq_entries + 857 128); 858 859 /* Initialise TX completion list */ 860 evq->txqs = &evq->txq; 861 862 /* Allocate DMA space. */ 863 if ((rc = sfxge_dma_alloc(sc, EFX_EVQ_SIZE(evq->entries), esmp)) != 0) 864 return (rc); 865 866 /* Allocate buffer table entries. */ 867 sfxge_sram_buf_tbl_alloc(sc, EFX_EVQ_NBUFS(evq->entries), 868 &evq->buf_base_id); 869 870 SFXGE_EVQ_LOCK_INIT(evq, device_get_nameunit(sc->dev), index); 871 872 evq->init_state = SFXGE_EVQ_INITIALIZED; 873 874 return (0); 875 } 876 877 void 878 sfxge_ev_fini(struct sfxge_softc *sc) 879 { 880 struct sfxge_intr *intr; 881 int index; 882 883 intr = &sc->intr; 884 885 KASSERT(intr->state == SFXGE_INTR_INITIALIZED, 886 ("intr->state != SFXGE_INTR_INITIALIZED")); 887 888 sc->ev_moderation = 0; 889 890 /* Tear down the event queue(s). */ 891 index = sc->evq_count; 892 while (--index >= 0) 893 sfxge_ev_qfini(sc, index); 894 895 sc->evq_count = 0; 896 } 897 898 int 899 sfxge_ev_init(struct sfxge_softc *sc) 900 { 901 struct sysctl_ctx_list *sysctl_ctx = device_get_sysctl_ctx(sc->dev); 902 struct sysctl_oid *sysctl_tree = device_get_sysctl_tree(sc->dev); 903 struct sfxge_intr *intr; 904 int index; 905 int rc; 906 907 intr = &sc->intr; 908 909 sc->evq_count = intr->n_alloc; 910 911 KASSERT(intr->state == SFXGE_INTR_INITIALIZED, 912 ("intr->state != SFXGE_INTR_INITIALIZED")); 913 914 /* Set default interrupt moderation; add a sysctl to 915 * read and change it. 916 */ 917 sc->ev_moderation = SFXGE_MODERATION; 918 SYSCTL_ADD_PROC(sysctl_ctx, SYSCTL_CHILDREN(sysctl_tree), 919 OID_AUTO, "int_mod", CTLTYPE_UINT|CTLFLAG_RW, 920 sc, 0, sfxge_int_mod_handler, "IU", 921 "sfxge interrupt moderation (us)"); 922 923 /* 924 * Initialize the event queue(s) - one per interrupt. 925 */ 926 for (index = 0; index < sc->evq_count; index++) { 927 if ((rc = sfxge_ev_qinit(sc, index)) != 0) 928 goto fail; 929 } 930 931 #if EFSYS_OPT_QSTATS 932 sfxge_ev_stat_init(sc); 933 #endif 934 935 return (0); 936 937 fail: 938 while (--index >= 0) 939 sfxge_ev_qfini(sc, index); 940 941 sc->evq_count = 0; 942 return (rc); 943 } 944