1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD 3 * 4 * Copyright (c) 2010-2016 Solarflare Communications Inc. 5 * All rights reserved. 6 * 7 * This software was developed in part by Philip Paeps under contract for 8 * Solarflare Communications, Inc. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions are met: 12 * 13 * 1. Redistributions of source code must retain the above copyright notice, 14 * this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright notice, 16 * this list of conditions and the following disclaimer in the documentation 17 * and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 20 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, 21 * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR 23 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, 24 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 25 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; 26 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 27 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR 28 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, 29 * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 * 31 * The views and conclusions contained in the software and documentation are 32 * those of the authors and should not be interpreted as representing official 33 * policies, either expressed or implied, of the FreeBSD Project. 34 */ 35 36 #include <sys/cdefs.h> 37 __FBSDID("$FreeBSD$"); 38 39 #include <sys/param.h> 40 #include <sys/kernel.h> 41 #include <sys/malloc.h> 42 #include <sys/param.h> 43 #include <sys/queue.h> 44 #include <sys/systm.h> 45 #include <sys/taskqueue.h> 46 47 #include "common/efx.h" 48 49 #include "sfxge.h" 50 51 static void 52 sfxge_ev_qcomplete(struct sfxge_evq *evq, boolean_t eop) 53 { 54 struct sfxge_softc *sc; 55 unsigned int index; 56 struct sfxge_rxq *rxq; 57 struct sfxge_txq *txq; 58 59 SFXGE_EVQ_LOCK_ASSERT_OWNED(evq); 60 61 sc = evq->sc; 62 index = evq->index; 63 rxq = sc->rxq[index]; 64 65 if ((txq = evq->txq) != NULL) { 66 evq->txq = NULL; 67 evq->txqs = &(evq->txq); 68 69 do { 70 struct sfxge_txq *next; 71 72 next = txq->next; 73 txq->next = NULL; 74 75 KASSERT(txq->evq_index == index, 76 ("txq->evq_index != index")); 77 78 if (txq->pending != txq->completed) 79 sfxge_tx_qcomplete(txq, evq); 80 81 txq = next; 82 } while (txq != NULL); 83 } 84 85 if (rxq->pending != rxq->completed) 86 sfxge_rx_qcomplete(rxq, eop); 87 } 88 89 static struct sfxge_rxq * 90 sfxge_get_rxq_by_label(struct sfxge_evq *evq, uint32_t label) 91 { 92 struct sfxge_rxq *rxq; 93 94 KASSERT(label == 0, ("unexpected rxq label != 0")); 95 96 rxq = evq->sc->rxq[evq->index]; 97 98 KASSERT(rxq != NULL, ("rxq == NULL")); 99 KASSERT(evq->index == rxq->index, ("evq->index != rxq->index")); 100 101 return (rxq); 102 } 103 104 static boolean_t 105 sfxge_ev_rx(void *arg, uint32_t label, uint32_t id, uint32_t size, 106 uint16_t flags) 107 { 108 struct sfxge_evq *evq; 109 struct sfxge_softc *sc; 110 struct sfxge_rxq *rxq; 111 unsigned int stop; 112 unsigned int delta; 113 struct sfxge_rx_sw_desc *rx_desc; 114 115 evq = arg; 116 SFXGE_EVQ_LOCK_ASSERT_OWNED(evq); 117 118 sc = evq->sc; 119 120 if (evq->exception) 121 goto done; 122 123 rxq = sfxge_get_rxq_by_label(evq, label); 124 if (__predict_false(rxq->init_state != SFXGE_RXQ_STARTED)) 125 goto done; 126 127 stop = (id + 1) & rxq->ptr_mask; 128 id = rxq->pending & rxq->ptr_mask; 129 delta = (stop >= id) ? (stop - id) : (rxq->entries - id + stop); 130 rxq->pending += delta; 131 132 if (delta != 1) { 133 if ((delta <= 0) || 134 (delta > efx_nic_cfg_get(sc->enp)->enc_rx_batch_max)) { 135 evq->exception = B_TRUE; 136 137 device_printf(sc->dev, "RX completion out of order" 138 " (id=%#x delta=%u flags=%#x); resetting\n", 139 id, delta, flags); 140 sfxge_schedule_reset(sc); 141 142 goto done; 143 } 144 } 145 146 rx_desc = &rxq->queue[id]; 147 148 prefetch_read_many(rx_desc->mbuf); 149 150 for (; id != stop; id = (id + 1) & rxq->ptr_mask) { 151 rx_desc = &rxq->queue[id]; 152 KASSERT(rx_desc->flags == EFX_DISCARD, 153 ("rx_desc->flags != EFX_DISCARD")); 154 rx_desc->flags = flags; 155 156 KASSERT(size < (1 << 16), ("size > (1 << 16)")); 157 rx_desc->size = (uint16_t)size; 158 } 159 160 evq->rx_done++; 161 162 if (rxq->pending - rxq->completed >= SFXGE_RX_BATCH) 163 sfxge_ev_qcomplete(evq, B_FALSE); 164 165 done: 166 return (evq->rx_done >= SFXGE_EV_BATCH); 167 } 168 169 static boolean_t 170 sfxge_ev_exception(void *arg, uint32_t code, uint32_t data) 171 { 172 struct sfxge_evq *evq; 173 struct sfxge_softc *sc; 174 175 evq = (struct sfxge_evq *)arg; 176 SFXGE_EVQ_LOCK_ASSERT_OWNED(evq); 177 178 sc = evq->sc; 179 180 DBGPRINT(sc->dev, "[%d] %s", evq->index, 181 (code == EFX_EXCEPTION_RX_RECOVERY) ? "RX_RECOVERY" : 182 (code == EFX_EXCEPTION_RX_DSC_ERROR) ? "RX_DSC_ERROR" : 183 (code == EFX_EXCEPTION_TX_DSC_ERROR) ? "TX_DSC_ERROR" : 184 (code == EFX_EXCEPTION_UNKNOWN_SENSOREVT) ? "UNKNOWN_SENSOREVT" : 185 (code == EFX_EXCEPTION_FWALERT_SRAM) ? "FWALERT_SRAM" : 186 (code == EFX_EXCEPTION_UNKNOWN_FWALERT) ? "UNKNOWN_FWALERT" : 187 (code == EFX_EXCEPTION_RX_ERROR) ? "RX_ERROR" : 188 (code == EFX_EXCEPTION_TX_ERROR) ? "TX_ERROR" : 189 (code == EFX_EXCEPTION_EV_ERROR) ? "EV_ERROR" : 190 "UNKNOWN"); 191 192 evq->exception = B_TRUE; 193 194 if (code != EFX_EXCEPTION_UNKNOWN_SENSOREVT) { 195 device_printf(sc->dev, 196 "hardware exception (code=%u); resetting\n", 197 code); 198 sfxge_schedule_reset(sc); 199 } 200 201 return (B_FALSE); 202 } 203 204 static boolean_t 205 sfxge_ev_rxq_flush_done(void *arg, uint32_t rxq_index) 206 { 207 struct sfxge_evq *evq; 208 struct sfxge_softc *sc; 209 struct sfxge_rxq *rxq; 210 unsigned int index; 211 uint16_t magic; 212 213 evq = (struct sfxge_evq *)arg; 214 SFXGE_EVQ_LOCK_ASSERT_OWNED(evq); 215 216 sc = evq->sc; 217 rxq = sc->rxq[rxq_index]; 218 219 KASSERT(rxq != NULL, ("rxq == NULL")); 220 221 /* Resend a software event on the correct queue */ 222 index = rxq->index; 223 if (index == evq->index) { 224 sfxge_rx_qflush_done(rxq); 225 return (B_FALSE); 226 } 227 228 evq = sc->evq[index]; 229 magic = sfxge_sw_ev_rxq_magic(SFXGE_SW_EV_RX_QFLUSH_DONE, rxq); 230 231 KASSERT(evq->init_state == SFXGE_EVQ_STARTED, 232 ("evq not started")); 233 efx_ev_qpost(evq->common, magic); 234 235 return (B_FALSE); 236 } 237 238 static boolean_t 239 sfxge_ev_rxq_flush_failed(void *arg, uint32_t rxq_index) 240 { 241 struct sfxge_evq *evq; 242 struct sfxge_softc *sc; 243 struct sfxge_rxq *rxq; 244 unsigned int index; 245 uint16_t magic; 246 247 evq = (struct sfxge_evq *)arg; 248 SFXGE_EVQ_LOCK_ASSERT_OWNED(evq); 249 250 sc = evq->sc; 251 rxq = sc->rxq[rxq_index]; 252 253 KASSERT(rxq != NULL, ("rxq == NULL")); 254 255 /* Resend a software event on the correct queue */ 256 index = rxq->index; 257 evq = sc->evq[index]; 258 magic = sfxge_sw_ev_rxq_magic(SFXGE_SW_EV_RX_QFLUSH_FAILED, rxq); 259 260 KASSERT(evq->init_state == SFXGE_EVQ_STARTED, 261 ("evq not started")); 262 efx_ev_qpost(evq->common, magic); 263 264 return (B_FALSE); 265 } 266 267 static struct sfxge_txq * 268 sfxge_get_txq_by_label(struct sfxge_evq *evq, enum sfxge_txq_type label) 269 { 270 unsigned int index; 271 272 KASSERT((evq->sc->txq_dynamic_cksum_toggle_supported) ? (label == 0) : 273 ((evq->index == 0 && label < SFXGE_TXQ_NTYPES) || 274 (label == SFXGE_TXQ_IP_TCP_UDP_CKSUM)), 275 ("unexpected txq label")); 276 277 index = (evq->index == 0) ? 278 label : (evq->index - 1 + SFXGE_EVQ0_N_TXQ(evq->sc)); 279 return (evq->sc->txq[index]); 280 } 281 282 static boolean_t 283 sfxge_ev_tx(void *arg, uint32_t label, uint32_t id) 284 { 285 struct sfxge_evq *evq; 286 struct sfxge_txq *txq; 287 unsigned int stop; 288 unsigned int delta; 289 290 evq = (struct sfxge_evq *)arg; 291 SFXGE_EVQ_LOCK_ASSERT_OWNED(evq); 292 293 txq = sfxge_get_txq_by_label(evq, label); 294 295 KASSERT(txq != NULL, ("txq == NULL")); 296 KASSERT(evq->index == txq->evq_index, 297 ("evq->index != txq->evq_index")); 298 299 if (__predict_false(txq->init_state != SFXGE_TXQ_STARTED)) 300 goto done; 301 302 stop = (id + 1) & txq->ptr_mask; 303 id = txq->pending & txq->ptr_mask; 304 305 delta = (stop >= id) ? (stop - id) : (txq->entries - id + stop); 306 txq->pending += delta; 307 308 evq->tx_done++; 309 310 if (txq->next == NULL && 311 evq->txqs != &(txq->next)) { 312 *(evq->txqs) = txq; 313 evq->txqs = &(txq->next); 314 } 315 316 if (txq->pending - txq->completed >= SFXGE_TX_BATCH) 317 sfxge_tx_qcomplete(txq, evq); 318 319 done: 320 return (evq->tx_done >= SFXGE_EV_BATCH); 321 } 322 323 static boolean_t 324 sfxge_ev_txq_flush_done(void *arg, uint32_t txq_index) 325 { 326 struct sfxge_evq *evq; 327 struct sfxge_softc *sc; 328 struct sfxge_txq *txq; 329 uint16_t magic; 330 331 evq = (struct sfxge_evq *)arg; 332 SFXGE_EVQ_LOCK_ASSERT_OWNED(evq); 333 334 sc = evq->sc; 335 txq = sc->txq[txq_index]; 336 337 KASSERT(txq != NULL, ("txq == NULL")); 338 KASSERT(txq->init_state == SFXGE_TXQ_INITIALIZED, 339 ("txq not initialized")); 340 341 if (txq->evq_index == evq->index) { 342 sfxge_tx_qflush_done(txq); 343 return (B_FALSE); 344 } 345 346 /* Resend a software event on the correct queue */ 347 evq = sc->evq[txq->evq_index]; 348 magic = sfxge_sw_ev_txq_magic(SFXGE_SW_EV_TX_QFLUSH_DONE, txq); 349 350 KASSERT(evq->init_state == SFXGE_EVQ_STARTED, 351 ("evq not started")); 352 efx_ev_qpost(evq->common, magic); 353 354 return (B_FALSE); 355 } 356 357 static boolean_t 358 sfxge_ev_software(void *arg, uint16_t magic) 359 { 360 struct sfxge_evq *evq; 361 struct sfxge_softc *sc; 362 unsigned int label; 363 364 evq = (struct sfxge_evq *)arg; 365 SFXGE_EVQ_LOCK_ASSERT_OWNED(evq); 366 367 sc = evq->sc; 368 369 label = magic & SFXGE_MAGIC_DMAQ_LABEL_MASK; 370 magic &= ~SFXGE_MAGIC_DMAQ_LABEL_MASK; 371 372 switch (magic) { 373 case SFXGE_SW_EV_MAGIC(SFXGE_SW_EV_RX_QFLUSH_DONE): 374 sfxge_rx_qflush_done(sfxge_get_rxq_by_label(evq, label)); 375 break; 376 377 case SFXGE_SW_EV_MAGIC(SFXGE_SW_EV_RX_QFLUSH_FAILED): 378 sfxge_rx_qflush_failed(sfxge_get_rxq_by_label(evq, label)); 379 break; 380 381 case SFXGE_SW_EV_MAGIC(SFXGE_SW_EV_RX_QREFILL): 382 sfxge_rx_qrefill(sfxge_get_rxq_by_label(evq, label)); 383 break; 384 385 case SFXGE_SW_EV_MAGIC(SFXGE_SW_EV_TX_QFLUSH_DONE): { 386 struct sfxge_txq *txq = sfxge_get_txq_by_label(evq, label); 387 388 KASSERT(txq != NULL, ("txq == NULL")); 389 KASSERT(evq->index == txq->evq_index, 390 ("evq->index != txq->evq_index")); 391 392 sfxge_tx_qflush_done(txq); 393 break; 394 } 395 default: 396 break; 397 } 398 399 return (B_FALSE); 400 } 401 402 static boolean_t 403 sfxge_ev_sram(void *arg, uint32_t code) 404 { 405 (void)arg; 406 (void)code; 407 408 switch (code) { 409 case EFX_SRAM_UPDATE: 410 EFSYS_PROBE(sram_update); 411 break; 412 413 case EFX_SRAM_CLEAR: 414 EFSYS_PROBE(sram_clear); 415 break; 416 417 case EFX_SRAM_ILLEGAL_CLEAR: 418 EFSYS_PROBE(sram_illegal_clear); 419 break; 420 421 default: 422 KASSERT(B_FALSE, ("Impossible SRAM event")); 423 break; 424 } 425 426 return (B_FALSE); 427 } 428 429 static boolean_t 430 sfxge_ev_timer(void *arg, uint32_t index) 431 { 432 (void)arg; 433 (void)index; 434 435 return (B_FALSE); 436 } 437 438 static boolean_t 439 sfxge_ev_wake_up(void *arg, uint32_t index) 440 { 441 (void)arg; 442 (void)index; 443 444 return (B_FALSE); 445 } 446 447 #if EFSYS_OPT_QSTATS 448 449 static void 450 sfxge_evq_stat_update(struct sfxge_evq *evq) 451 { 452 clock_t now; 453 454 SFXGE_EVQ_LOCK(evq); 455 456 if (__predict_false(evq->init_state != SFXGE_EVQ_STARTED)) 457 goto out; 458 459 now = ticks; 460 if ((unsigned int)(now - evq->stats_update_time) < (unsigned int)hz) 461 goto out; 462 463 evq->stats_update_time = now; 464 efx_ev_qstats_update(evq->common, evq->stats); 465 466 out: 467 SFXGE_EVQ_UNLOCK(evq); 468 } 469 470 static int 471 sfxge_evq_stat_handler(SYSCTL_HANDLER_ARGS) 472 { 473 struct sfxge_evq *evq = arg1; 474 struct sfxge_softc *sc = evq->sc; 475 unsigned int id = arg2; 476 477 SFXGE_ADAPTER_LOCK(sc); 478 479 sfxge_evq_stat_update(evq); 480 481 SFXGE_ADAPTER_UNLOCK(sc); 482 483 return (SYSCTL_OUT(req, &evq->stats[id], sizeof(evq->stats[id]))); 484 } 485 486 static int 487 sfxge_evq_stat_init(struct sfxge_evq *evq) 488 { 489 struct sfxge_softc *sc = evq->sc; 490 struct sysctl_ctx_list *ctx = device_get_sysctl_ctx(sc->dev); 491 char name[16]; 492 struct sysctl_oid *evq_stats_node; 493 unsigned int id; 494 495 snprintf(name, sizeof(name), "%u", evq->index); 496 evq_stats_node = SYSCTL_ADD_NODE(ctx, 497 SYSCTL_CHILDREN(sc->evqs_stats_node), OID_AUTO, name, 498 CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, ""); 499 if (evq_stats_node == NULL) 500 return (ENOMEM); 501 502 for (id = 0; id < EV_NQSTATS; id++) { 503 SYSCTL_ADD_PROC(ctx, SYSCTL_CHILDREN(evq_stats_node), 504 OID_AUTO, efx_ev_qstat_name(sc->enp, id), 505 CTLTYPE_U64 | CTLFLAG_RD | CTLFLAG_MPSAFE, evq, id, 506 sfxge_evq_stat_handler, "Q", ""); 507 } 508 509 return (0); 510 } 511 512 static void 513 sfxge_ev_stat_update(struct sfxge_softc *sc) 514 { 515 struct sfxge_evq *evq; 516 unsigned int index; 517 clock_t now; 518 unsigned int id; 519 520 SFXGE_ADAPTER_LOCK(sc); 521 522 now = ticks; 523 if ((unsigned int)(now - sc->ev_stats_update_time) < (unsigned int)hz) 524 goto out; 525 526 sc->ev_stats_update_time = now; 527 528 memset(sc->ev_stats, 0, sizeof(sc->ev_stats)); 529 530 /* Update and add event counts from each event queue in turn */ 531 for (index = 0; index < sc->evq_count; index++) { 532 evq = sc->evq[index]; 533 sfxge_evq_stat_update(evq); 534 for (id = 0; id < EV_NQSTATS; id++) 535 sc->ev_stats[id] += evq->stats[id]; 536 } 537 out: 538 SFXGE_ADAPTER_UNLOCK(sc); 539 } 540 541 static int 542 sfxge_ev_stat_handler(SYSCTL_HANDLER_ARGS) 543 { 544 struct sfxge_softc *sc = arg1; 545 unsigned int id = arg2; 546 547 sfxge_ev_stat_update(sc); 548 549 return (SYSCTL_OUT(req, &sc->ev_stats[id], sizeof(sc->ev_stats[id]))); 550 } 551 552 static void 553 sfxge_ev_stat_init(struct sfxge_softc *sc) 554 { 555 struct sysctl_ctx_list *ctx = device_get_sysctl_ctx(sc->dev); 556 struct sysctl_oid_list *stat_list; 557 unsigned int id; 558 char name[40]; 559 560 stat_list = SYSCTL_CHILDREN(sc->stats_node); 561 562 for (id = 0; id < EV_NQSTATS; id++) { 563 snprintf(name, sizeof(name), "ev_%s", 564 efx_ev_qstat_name(sc->enp, id)); 565 SYSCTL_ADD_PROC(ctx, stat_list, OID_AUTO, name, 566 CTLTYPE_U64 | CTLFLAG_RD | CTLFLAG_MPSAFE, 567 sc, id, sfxge_ev_stat_handler, "Q", ""); 568 } 569 } 570 571 #endif /* EFSYS_OPT_QSTATS */ 572 573 static void 574 sfxge_ev_qmoderate(struct sfxge_softc *sc, unsigned int idx, unsigned int us) 575 { 576 struct sfxge_evq *evq; 577 efx_evq_t *eep; 578 579 evq = sc->evq[idx]; 580 eep = evq->common; 581 582 KASSERT(evq->init_state == SFXGE_EVQ_STARTED, 583 ("evq->init_state != SFXGE_EVQ_STARTED")); 584 585 (void)efx_ev_qmoderate(eep, us); 586 } 587 588 static int 589 sfxge_int_mod_handler(SYSCTL_HANDLER_ARGS) 590 { 591 struct sfxge_softc *sc = arg1; 592 struct sfxge_intr *intr = &sc->intr; 593 unsigned int moderation; 594 int error; 595 unsigned int index; 596 597 SFXGE_ADAPTER_LOCK(sc); 598 599 if (req->newptr != NULL) { 600 if ((error = SYSCTL_IN(req, &moderation, sizeof(moderation))) 601 != 0) 602 goto out; 603 604 /* We may not be calling efx_ev_qmoderate() now, 605 * so we have to range-check the value ourselves. 606 */ 607 if (moderation > 608 efx_nic_cfg_get(sc->enp)->enc_evq_timer_max_us) { 609 error = EINVAL; 610 goto out; 611 } 612 613 sc->ev_moderation = moderation; 614 if (intr->state == SFXGE_INTR_STARTED) { 615 for (index = 0; index < sc->evq_count; index++) 616 sfxge_ev_qmoderate(sc, index, moderation); 617 } 618 } else { 619 error = SYSCTL_OUT(req, &sc->ev_moderation, 620 sizeof(sc->ev_moderation)); 621 } 622 623 out: 624 SFXGE_ADAPTER_UNLOCK(sc); 625 626 return (error); 627 } 628 629 static boolean_t 630 sfxge_ev_initialized(void *arg) 631 { 632 struct sfxge_evq *evq; 633 634 evq = (struct sfxge_evq *)arg; 635 SFXGE_EVQ_LOCK_ASSERT_OWNED(evq); 636 637 /* Init done events may be duplicated on 7xxx */ 638 KASSERT(evq->init_state == SFXGE_EVQ_STARTING || 639 evq->init_state == SFXGE_EVQ_STARTED, 640 ("evq not starting")); 641 642 evq->init_state = SFXGE_EVQ_STARTED; 643 644 return (0); 645 } 646 647 static boolean_t 648 sfxge_ev_link_change(void *arg, efx_link_mode_t link_mode) 649 { 650 struct sfxge_evq *evq; 651 struct sfxge_softc *sc; 652 653 evq = (struct sfxge_evq *)arg; 654 SFXGE_EVQ_LOCK_ASSERT_OWNED(evq); 655 656 sc = evq->sc; 657 658 sfxge_mac_link_update(sc, link_mode); 659 660 return (0); 661 } 662 663 static const efx_ev_callbacks_t sfxge_ev_callbacks = { 664 .eec_initialized = sfxge_ev_initialized, 665 .eec_rx = sfxge_ev_rx, 666 .eec_tx = sfxge_ev_tx, 667 .eec_exception = sfxge_ev_exception, 668 .eec_rxq_flush_done = sfxge_ev_rxq_flush_done, 669 .eec_rxq_flush_failed = sfxge_ev_rxq_flush_failed, 670 .eec_txq_flush_done = sfxge_ev_txq_flush_done, 671 .eec_software = sfxge_ev_software, 672 .eec_sram = sfxge_ev_sram, 673 .eec_wake_up = sfxge_ev_wake_up, 674 .eec_timer = sfxge_ev_timer, 675 .eec_link_change = sfxge_ev_link_change, 676 }; 677 678 679 int 680 sfxge_ev_qpoll(struct sfxge_evq *evq) 681 { 682 int rc; 683 684 SFXGE_EVQ_LOCK(evq); 685 686 if (__predict_false(evq->init_state != SFXGE_EVQ_STARTING && 687 evq->init_state != SFXGE_EVQ_STARTED)) { 688 rc = EINVAL; 689 goto fail; 690 } 691 692 /* Synchronize the DMA memory for reading */ 693 bus_dmamap_sync(evq->mem.esm_tag, evq->mem.esm_map, 694 BUS_DMASYNC_POSTREAD); 695 696 KASSERT(evq->rx_done == 0, ("evq->rx_done != 0")); 697 KASSERT(evq->tx_done == 0, ("evq->tx_done != 0")); 698 KASSERT(evq->txq == NULL, ("evq->txq != NULL")); 699 KASSERT(evq->txqs == &evq->txq, ("evq->txqs != &evq->txq")); 700 701 /* Poll the queue */ 702 efx_ev_qpoll(evq->common, &evq->read_ptr, &sfxge_ev_callbacks, evq); 703 704 evq->rx_done = 0; 705 evq->tx_done = 0; 706 707 /* Perform any pending completion processing */ 708 sfxge_ev_qcomplete(evq, B_TRUE); 709 710 /* Re-prime the event queue for interrupts */ 711 if ((rc = efx_ev_qprime(evq->common, evq->read_ptr)) != 0) 712 goto fail; 713 714 SFXGE_EVQ_UNLOCK(evq); 715 716 return (0); 717 718 fail: 719 SFXGE_EVQ_UNLOCK(evq); 720 return (rc); 721 } 722 723 static void 724 sfxge_ev_qstop(struct sfxge_softc *sc, unsigned int index) 725 { 726 struct sfxge_evq *evq; 727 728 evq = sc->evq[index]; 729 730 KASSERT(evq->init_state == SFXGE_EVQ_STARTED, 731 ("evq->init_state != SFXGE_EVQ_STARTED")); 732 733 SFXGE_EVQ_LOCK(evq); 734 evq->init_state = SFXGE_EVQ_INITIALIZED; 735 evq->read_ptr = 0; 736 evq->exception = B_FALSE; 737 738 #if EFSYS_OPT_QSTATS 739 /* Add event counts before discarding the common evq state */ 740 efx_ev_qstats_update(evq->common, evq->stats); 741 #endif 742 743 efx_ev_qdestroy(evq->common); 744 efx_sram_buf_tbl_clear(sc->enp, evq->buf_base_id, 745 EFX_EVQ_NBUFS(evq->entries)); 746 SFXGE_EVQ_UNLOCK(evq); 747 } 748 749 static int 750 sfxge_ev_qstart(struct sfxge_softc *sc, unsigned int index) 751 { 752 struct sfxge_evq *evq; 753 efsys_mem_t *esmp; 754 int count; 755 int rc; 756 757 evq = sc->evq[index]; 758 esmp = &evq->mem; 759 760 KASSERT(evq->init_state == SFXGE_EVQ_INITIALIZED, 761 ("evq->init_state != SFXGE_EVQ_INITIALIZED")); 762 763 /* Clear all events. */ 764 (void)memset(esmp->esm_base, 0xff, EFX_EVQ_SIZE(evq->entries)); 765 766 /* Program the buffer table. */ 767 if ((rc = efx_sram_buf_tbl_set(sc->enp, evq->buf_base_id, esmp, 768 EFX_EVQ_NBUFS(evq->entries))) != 0) 769 return (rc); 770 771 /* Create the common code event queue. */ 772 if ((rc = efx_ev_qcreate(sc->enp, index, esmp, evq->entries, 773 evq->buf_base_id, sc->ev_moderation, EFX_EVQ_FLAGS_TYPE_AUTO, 774 &evq->common)) != 0) 775 goto fail; 776 777 SFXGE_EVQ_LOCK(evq); 778 779 /* Prime the event queue for interrupts */ 780 if ((rc = efx_ev_qprime(evq->common, evq->read_ptr)) != 0) 781 goto fail2; 782 783 evq->init_state = SFXGE_EVQ_STARTING; 784 785 SFXGE_EVQ_UNLOCK(evq); 786 787 /* Wait for the initialization event */ 788 count = 0; 789 do { 790 /* Pause for 100 ms */ 791 pause("sfxge evq init", hz / 10); 792 793 /* Check to see if the test event has been processed */ 794 if (evq->init_state == SFXGE_EVQ_STARTED) 795 goto done; 796 797 } while (++count < 20); 798 799 rc = ETIMEDOUT; 800 goto fail3; 801 802 done: 803 return (0); 804 805 fail3: 806 SFXGE_EVQ_LOCK(evq); 807 evq->init_state = SFXGE_EVQ_INITIALIZED; 808 fail2: 809 SFXGE_EVQ_UNLOCK(evq); 810 efx_ev_qdestroy(evq->common); 811 fail: 812 efx_sram_buf_tbl_clear(sc->enp, evq->buf_base_id, 813 EFX_EVQ_NBUFS(evq->entries)); 814 815 return (rc); 816 } 817 818 void 819 sfxge_ev_stop(struct sfxge_softc *sc) 820 { 821 struct sfxge_intr *intr; 822 efx_nic_t *enp; 823 int index; 824 825 intr = &sc->intr; 826 enp = sc->enp; 827 828 KASSERT(intr->state == SFXGE_INTR_STARTED, 829 ("Interrupts not started")); 830 831 /* Stop the event queue(s) */ 832 index = sc->evq_count; 833 while (--index >= 0) 834 sfxge_ev_qstop(sc, index); 835 836 /* Tear down the event module */ 837 efx_ev_fini(enp); 838 } 839 840 int 841 sfxge_ev_start(struct sfxge_softc *sc) 842 { 843 struct sfxge_intr *intr; 844 int index; 845 int rc; 846 847 intr = &sc->intr; 848 849 KASSERT(intr->state == SFXGE_INTR_STARTED, 850 ("intr->state != SFXGE_INTR_STARTED")); 851 852 /* Initialize the event module */ 853 if ((rc = efx_ev_init(sc->enp)) != 0) 854 return (rc); 855 856 /* Start the event queues */ 857 for (index = 0; index < sc->evq_count; index++) { 858 if ((rc = sfxge_ev_qstart(sc, index)) != 0) 859 goto fail; 860 } 861 862 return (0); 863 864 fail: 865 /* Stop the event queue(s) */ 866 while (--index >= 0) 867 sfxge_ev_qstop(sc, index); 868 869 /* Tear down the event module */ 870 efx_ev_fini(sc->enp); 871 872 return (rc); 873 } 874 875 static void 876 sfxge_ev_qfini(struct sfxge_softc *sc, unsigned int index) 877 { 878 struct sfxge_evq *evq; 879 880 evq = sc->evq[index]; 881 882 KASSERT(evq->init_state == SFXGE_EVQ_INITIALIZED, 883 ("evq->init_state != SFXGE_EVQ_INITIALIZED")); 884 KASSERT(evq->txqs == &evq->txq, ("evq->txqs != &evq->txq")); 885 886 sfxge_dma_free(&evq->mem); 887 888 sc->evq[index] = NULL; 889 890 SFXGE_EVQ_LOCK_DESTROY(evq); 891 892 free(evq, M_SFXGE); 893 } 894 895 static int 896 sfxge_ev_qinit(struct sfxge_softc *sc, unsigned int index) 897 { 898 struct sfxge_evq *evq; 899 efsys_mem_t *esmp; 900 int rc; 901 902 KASSERT(index < SFXGE_RX_SCALE_MAX, ("index >= SFXGE_RX_SCALE_MAX")); 903 904 evq = malloc(sizeof(struct sfxge_evq), M_SFXGE, M_ZERO | M_WAITOK); 905 evq->sc = sc; 906 evq->index = index; 907 sc->evq[index] = evq; 908 esmp = &evq->mem; 909 910 /* Build an event queue with room for one event per tx and rx buffer, 911 * plus some extra for link state events and MCDI completions. 912 * There are three tx queues in the first event queue and one in 913 * other. 914 */ 915 if (index == 0) 916 evq->entries = 917 ROUNDUP_POW_OF_TWO(sc->rxq_entries + 918 3 * sc->txq_entries + 919 128); 920 else 921 evq->entries = 922 ROUNDUP_POW_OF_TWO(sc->rxq_entries + 923 sc->txq_entries + 924 128); 925 926 /* Initialise TX completion list */ 927 evq->txqs = &evq->txq; 928 929 /* Allocate DMA space. */ 930 if ((rc = sfxge_dma_alloc(sc, EFX_EVQ_SIZE(evq->entries), esmp)) != 0) 931 return (rc); 932 933 /* Allocate buffer table entries. */ 934 sfxge_sram_buf_tbl_alloc(sc, EFX_EVQ_NBUFS(evq->entries), 935 &evq->buf_base_id); 936 937 SFXGE_EVQ_LOCK_INIT(evq, device_get_nameunit(sc->dev), index); 938 939 evq->init_state = SFXGE_EVQ_INITIALIZED; 940 941 #if EFSYS_OPT_QSTATS 942 rc = sfxge_evq_stat_init(evq); 943 if (rc != 0) 944 goto fail_evq_stat_init; 945 #endif 946 947 return (0); 948 949 #if EFSYS_OPT_QSTATS 950 fail_evq_stat_init: 951 evq->init_state = SFXGE_EVQ_UNINITIALIZED; 952 SFXGE_EVQ_LOCK_DESTROY(evq); 953 sfxge_dma_free(esmp); 954 sc->evq[index] = NULL; 955 free(evq, M_SFXGE); 956 957 return (rc); 958 #endif 959 } 960 961 void 962 sfxge_ev_fini(struct sfxge_softc *sc) 963 { 964 struct sfxge_intr *intr; 965 int index; 966 967 intr = &sc->intr; 968 969 KASSERT(intr->state == SFXGE_INTR_INITIALIZED, 970 ("intr->state != SFXGE_INTR_INITIALIZED")); 971 972 sc->ev_moderation = 0; 973 974 /* Tear down the event queue(s). */ 975 index = sc->evq_count; 976 while (--index >= 0) 977 sfxge_ev_qfini(sc, index); 978 979 sc->evq_count = 0; 980 } 981 982 int 983 sfxge_ev_init(struct sfxge_softc *sc) 984 { 985 struct sysctl_ctx_list *sysctl_ctx = device_get_sysctl_ctx(sc->dev); 986 struct sysctl_oid *sysctl_tree = device_get_sysctl_tree(sc->dev); 987 struct sfxge_intr *intr; 988 int index; 989 int rc; 990 991 intr = &sc->intr; 992 993 sc->evq_count = intr->n_alloc; 994 995 KASSERT(intr->state == SFXGE_INTR_INITIALIZED, 996 ("intr->state != SFXGE_INTR_INITIALIZED")); 997 998 /* Set default interrupt moderation; add a sysctl to 999 * read and change it. 1000 */ 1001 sc->ev_moderation = SFXGE_MODERATION; 1002 SYSCTL_ADD_PROC(sysctl_ctx, SYSCTL_CHILDREN(sysctl_tree), 1003 OID_AUTO, "int_mod", CTLTYPE_UINT | CTLFLAG_RW | CTLFLAG_MPSAFE, 1004 sc, 0, sfxge_int_mod_handler, "IU", 1005 "sfxge interrupt moderation (us)"); 1006 1007 #if EFSYS_OPT_QSTATS 1008 sc->evqs_stats_node = SYSCTL_ADD_NODE( 1009 device_get_sysctl_ctx(sc->dev), SYSCTL_CHILDREN(sc->stats_node), 1010 OID_AUTO, "evq", CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, 1011 "Event queues stats"); 1012 if (sc->evqs_stats_node == NULL) { 1013 rc = ENOMEM; 1014 goto fail_evqs_stats_node; 1015 } 1016 #endif 1017 1018 /* 1019 * Initialize the event queue(s) - one per interrupt. 1020 */ 1021 for (index = 0; index < sc->evq_count; index++) { 1022 if ((rc = sfxge_ev_qinit(sc, index)) != 0) 1023 goto fail; 1024 } 1025 1026 #if EFSYS_OPT_QSTATS 1027 sfxge_ev_stat_init(sc); 1028 #endif 1029 1030 return (0); 1031 1032 fail: 1033 while (--index >= 0) 1034 sfxge_ev_qfini(sc, index); 1035 1036 #if EFSYS_OPT_QSTATS 1037 fail_evqs_stats_node: 1038 #endif 1039 sc->evq_count = 0; 1040 return (rc); 1041 } 1042