1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD 3 * 4 * Copyright (c) 2010-2016 Solarflare Communications Inc. 5 * All rights reserved. 6 * 7 * This software was developed in part by Philip Paeps under contract for 8 * Solarflare Communications, Inc. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions are met: 12 * 13 * 1. Redistributions of source code must retain the above copyright notice, 14 * this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright notice, 16 * this list of conditions and the following disclaimer in the documentation 17 * and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 20 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, 21 * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR 23 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, 24 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 25 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; 26 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 27 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR 28 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, 29 * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 * 31 * The views and conclusions contained in the software and documentation are 32 * those of the authors and should not be interpreted as representing official 33 * policies, either expressed or implied, of the FreeBSD Project. 34 */ 35 36 #include <sys/cdefs.h> 37 __FBSDID("$FreeBSD$"); 38 39 #include <sys/param.h> 40 #include <sys/kernel.h> 41 #include <sys/malloc.h> 42 #include <sys/param.h> 43 #include <sys/queue.h> 44 #include <sys/systm.h> 45 #include <sys/taskqueue.h> 46 47 #include "common/efx.h" 48 49 #include "sfxge.h" 50 51 static void 52 sfxge_ev_qcomplete(struct sfxge_evq *evq, boolean_t eop) 53 { 54 struct sfxge_softc *sc; 55 unsigned int index; 56 struct sfxge_rxq *rxq; 57 struct sfxge_txq *txq; 58 59 SFXGE_EVQ_LOCK_ASSERT_OWNED(evq); 60 61 sc = evq->sc; 62 index = evq->index; 63 rxq = sc->rxq[index]; 64 65 if ((txq = evq->txq) != NULL) { 66 evq->txq = NULL; 67 evq->txqs = &(evq->txq); 68 69 do { 70 struct sfxge_txq *next; 71 72 next = txq->next; 73 txq->next = NULL; 74 75 KASSERT(txq->evq_index == index, 76 ("txq->evq_index != index")); 77 78 if (txq->pending != txq->completed) 79 sfxge_tx_qcomplete(txq, evq); 80 81 txq = next; 82 } while (txq != NULL); 83 } 84 85 if (rxq->pending != rxq->completed) 86 sfxge_rx_qcomplete(rxq, eop); 87 } 88 89 static struct sfxge_rxq * 90 sfxge_get_rxq_by_label(struct sfxge_evq *evq, uint32_t label) 91 { 92 struct sfxge_rxq *rxq; 93 94 KASSERT(label == 0, ("unexpected rxq label != 0")); 95 96 rxq = evq->sc->rxq[evq->index]; 97 98 KASSERT(rxq != NULL, ("rxq == NULL")); 99 KASSERT(evq->index == rxq->index, ("evq->index != rxq->index")); 100 101 return (rxq); 102 } 103 104 static boolean_t 105 sfxge_ev_rx(void *arg, uint32_t label, uint32_t id, uint32_t size, 106 uint16_t flags) 107 { 108 struct sfxge_evq *evq; 109 struct sfxge_softc *sc; 110 struct sfxge_rxq *rxq; 111 unsigned int stop; 112 unsigned int delta; 113 struct sfxge_rx_sw_desc *rx_desc; 114 115 evq = arg; 116 SFXGE_EVQ_LOCK_ASSERT_OWNED(evq); 117 118 sc = evq->sc; 119 120 if (evq->exception) 121 goto done; 122 123 rxq = sfxge_get_rxq_by_label(evq, label); 124 if (__predict_false(rxq->init_state != SFXGE_RXQ_STARTED)) 125 goto done; 126 127 stop = (id + 1) & rxq->ptr_mask; 128 id = rxq->pending & rxq->ptr_mask; 129 delta = (stop >= id) ? (stop - id) : (rxq->entries - id + stop); 130 rxq->pending += delta; 131 132 if (delta != 1) { 133 if ((delta <= 0) || 134 (delta > efx_nic_cfg_get(sc->enp)->enc_rx_batch_max)) { 135 evq->exception = B_TRUE; 136 137 device_printf(sc->dev, "RX completion out of order" 138 " (id=%#x delta=%u flags=%#x); resetting\n", 139 id, delta, flags); 140 sfxge_schedule_reset(sc); 141 142 goto done; 143 } 144 } 145 146 rx_desc = &rxq->queue[id]; 147 148 prefetch_read_many(rx_desc->mbuf); 149 150 for (; id != stop; id = (id + 1) & rxq->ptr_mask) { 151 rx_desc = &rxq->queue[id]; 152 KASSERT(rx_desc->flags == EFX_DISCARD, 153 ("rx_desc->flags != EFX_DISCARD")); 154 rx_desc->flags = flags; 155 156 KASSERT(size < (1 << 16), ("size > (1 << 16)")); 157 rx_desc->size = (uint16_t)size; 158 } 159 160 evq->rx_done++; 161 162 if (rxq->pending - rxq->completed >= SFXGE_RX_BATCH) 163 sfxge_ev_qcomplete(evq, B_FALSE); 164 165 done: 166 return (evq->rx_done >= SFXGE_EV_BATCH); 167 } 168 169 static boolean_t 170 sfxge_ev_exception(void *arg, uint32_t code, uint32_t data) 171 { 172 struct sfxge_evq *evq; 173 struct sfxge_softc *sc; 174 175 evq = (struct sfxge_evq *)arg; 176 SFXGE_EVQ_LOCK_ASSERT_OWNED(evq); 177 178 sc = evq->sc; 179 180 DBGPRINT(sc->dev, "[%d] %s", evq->index, 181 (code == EFX_EXCEPTION_RX_RECOVERY) ? "RX_RECOVERY" : 182 (code == EFX_EXCEPTION_RX_DSC_ERROR) ? "RX_DSC_ERROR" : 183 (code == EFX_EXCEPTION_TX_DSC_ERROR) ? "TX_DSC_ERROR" : 184 (code == EFX_EXCEPTION_UNKNOWN_SENSOREVT) ? "UNKNOWN_SENSOREVT" : 185 (code == EFX_EXCEPTION_FWALERT_SRAM) ? "FWALERT_SRAM" : 186 (code == EFX_EXCEPTION_UNKNOWN_FWALERT) ? "UNKNOWN_FWALERT" : 187 (code == EFX_EXCEPTION_RX_ERROR) ? "RX_ERROR" : 188 (code == EFX_EXCEPTION_TX_ERROR) ? "TX_ERROR" : 189 (code == EFX_EXCEPTION_EV_ERROR) ? "EV_ERROR" : 190 "UNKNOWN"); 191 192 evq->exception = B_TRUE; 193 194 if (code != EFX_EXCEPTION_UNKNOWN_SENSOREVT) { 195 device_printf(sc->dev, 196 "hardware exception (code=%u); resetting\n", 197 code); 198 sfxge_schedule_reset(sc); 199 } 200 201 return (B_FALSE); 202 } 203 204 static boolean_t 205 sfxge_ev_rxq_flush_done(void *arg, uint32_t rxq_index) 206 { 207 struct sfxge_evq *evq; 208 struct sfxge_softc *sc; 209 struct sfxge_rxq *rxq; 210 unsigned int index; 211 uint16_t magic; 212 213 evq = (struct sfxge_evq *)arg; 214 SFXGE_EVQ_LOCK_ASSERT_OWNED(evq); 215 216 sc = evq->sc; 217 rxq = sc->rxq[rxq_index]; 218 219 KASSERT(rxq != NULL, ("rxq == NULL")); 220 221 /* Resend a software event on the correct queue */ 222 index = rxq->index; 223 if (index == evq->index) { 224 sfxge_rx_qflush_done(rxq); 225 return (B_FALSE); 226 } 227 228 evq = sc->evq[index]; 229 magic = sfxge_sw_ev_rxq_magic(SFXGE_SW_EV_RX_QFLUSH_DONE, rxq); 230 231 KASSERT(evq->init_state == SFXGE_EVQ_STARTED, 232 ("evq not started")); 233 efx_ev_qpost(evq->common, magic); 234 235 return (B_FALSE); 236 } 237 238 static boolean_t 239 sfxge_ev_rxq_flush_failed(void *arg, uint32_t rxq_index) 240 { 241 struct sfxge_evq *evq; 242 struct sfxge_softc *sc; 243 struct sfxge_rxq *rxq; 244 unsigned int index; 245 uint16_t magic; 246 247 evq = (struct sfxge_evq *)arg; 248 SFXGE_EVQ_LOCK_ASSERT_OWNED(evq); 249 250 sc = evq->sc; 251 rxq = sc->rxq[rxq_index]; 252 253 KASSERT(rxq != NULL, ("rxq == NULL")); 254 255 /* Resend a software event on the correct queue */ 256 index = rxq->index; 257 evq = sc->evq[index]; 258 magic = sfxge_sw_ev_rxq_magic(SFXGE_SW_EV_RX_QFLUSH_FAILED, rxq); 259 260 KASSERT(evq->init_state == SFXGE_EVQ_STARTED, 261 ("evq not started")); 262 efx_ev_qpost(evq->common, magic); 263 264 return (B_FALSE); 265 } 266 267 static struct sfxge_txq * 268 sfxge_get_txq_by_label(struct sfxge_evq *evq, enum sfxge_txq_type label) 269 { 270 unsigned int index; 271 272 KASSERT((evq->sc->txq_dynamic_cksum_toggle_supported) ? (label == 0) : 273 ((evq->index == 0 && label < SFXGE_TXQ_NTYPES) || 274 (label == SFXGE_TXQ_IP_TCP_UDP_CKSUM)), 275 ("unexpected txq label")); 276 277 index = (evq->index == 0) ? 278 label : (evq->index - 1 + SFXGE_EVQ0_N_TXQ(evq->sc)); 279 return (evq->sc->txq[index]); 280 } 281 282 static boolean_t 283 sfxge_ev_tx(void *arg, uint32_t label, uint32_t id) 284 { 285 struct sfxge_evq *evq; 286 struct sfxge_txq *txq; 287 unsigned int stop; 288 unsigned int delta; 289 290 evq = (struct sfxge_evq *)arg; 291 SFXGE_EVQ_LOCK_ASSERT_OWNED(evq); 292 293 txq = sfxge_get_txq_by_label(evq, label); 294 295 KASSERT(txq != NULL, ("txq == NULL")); 296 KASSERT(evq->index == txq->evq_index, 297 ("evq->index != txq->evq_index")); 298 299 if (__predict_false(txq->init_state != SFXGE_TXQ_STARTED)) 300 goto done; 301 302 stop = (id + 1) & txq->ptr_mask; 303 id = txq->pending & txq->ptr_mask; 304 305 delta = (stop >= id) ? (stop - id) : (txq->entries - id + stop); 306 txq->pending += delta; 307 308 evq->tx_done++; 309 310 if (txq->next == NULL && 311 evq->txqs != &(txq->next)) { 312 *(evq->txqs) = txq; 313 evq->txqs = &(txq->next); 314 } 315 316 if (txq->pending - txq->completed >= SFXGE_TX_BATCH) 317 sfxge_tx_qcomplete(txq, evq); 318 319 done: 320 return (evq->tx_done >= SFXGE_EV_BATCH); 321 } 322 323 static boolean_t 324 sfxge_ev_txq_flush_done(void *arg, uint32_t txq_index) 325 { 326 struct sfxge_evq *evq; 327 struct sfxge_softc *sc; 328 struct sfxge_txq *txq; 329 uint16_t magic; 330 331 evq = (struct sfxge_evq *)arg; 332 SFXGE_EVQ_LOCK_ASSERT_OWNED(evq); 333 334 sc = evq->sc; 335 txq = sc->txq[txq_index]; 336 337 KASSERT(txq != NULL, ("txq == NULL")); 338 KASSERT(txq->init_state == SFXGE_TXQ_INITIALIZED, 339 ("txq not initialized")); 340 341 if (txq->evq_index == evq->index) { 342 sfxge_tx_qflush_done(txq); 343 return (B_FALSE); 344 } 345 346 /* Resend a software event on the correct queue */ 347 evq = sc->evq[txq->evq_index]; 348 magic = sfxge_sw_ev_txq_magic(SFXGE_SW_EV_TX_QFLUSH_DONE, txq); 349 350 KASSERT(evq->init_state == SFXGE_EVQ_STARTED, 351 ("evq not started")); 352 efx_ev_qpost(evq->common, magic); 353 354 return (B_FALSE); 355 } 356 357 static boolean_t 358 sfxge_ev_software(void *arg, uint16_t magic) 359 { 360 struct sfxge_evq *evq; 361 unsigned int label; 362 363 evq = (struct sfxge_evq *)arg; 364 SFXGE_EVQ_LOCK_ASSERT_OWNED(evq); 365 366 label = magic & SFXGE_MAGIC_DMAQ_LABEL_MASK; 367 magic &= ~SFXGE_MAGIC_DMAQ_LABEL_MASK; 368 369 switch (magic) { 370 case SFXGE_SW_EV_MAGIC(SFXGE_SW_EV_RX_QFLUSH_DONE): 371 sfxge_rx_qflush_done(sfxge_get_rxq_by_label(evq, label)); 372 break; 373 374 case SFXGE_SW_EV_MAGIC(SFXGE_SW_EV_RX_QFLUSH_FAILED): 375 sfxge_rx_qflush_failed(sfxge_get_rxq_by_label(evq, label)); 376 break; 377 378 case SFXGE_SW_EV_MAGIC(SFXGE_SW_EV_RX_QREFILL): 379 sfxge_rx_qrefill(sfxge_get_rxq_by_label(evq, label)); 380 break; 381 382 case SFXGE_SW_EV_MAGIC(SFXGE_SW_EV_TX_QFLUSH_DONE): { 383 struct sfxge_txq *txq = sfxge_get_txq_by_label(evq, label); 384 385 KASSERT(txq != NULL, ("txq == NULL")); 386 KASSERT(evq->index == txq->evq_index, 387 ("evq->index != txq->evq_index")); 388 389 sfxge_tx_qflush_done(txq); 390 break; 391 } 392 default: 393 break; 394 } 395 396 return (B_FALSE); 397 } 398 399 static boolean_t 400 sfxge_ev_sram(void *arg, uint32_t code) 401 { 402 (void)arg; 403 (void)code; 404 405 switch (code) { 406 case EFX_SRAM_UPDATE: 407 EFSYS_PROBE(sram_update); 408 break; 409 410 case EFX_SRAM_CLEAR: 411 EFSYS_PROBE(sram_clear); 412 break; 413 414 case EFX_SRAM_ILLEGAL_CLEAR: 415 EFSYS_PROBE(sram_illegal_clear); 416 break; 417 418 default: 419 KASSERT(B_FALSE, ("Impossible SRAM event")); 420 break; 421 } 422 423 return (B_FALSE); 424 } 425 426 static boolean_t 427 sfxge_ev_timer(void *arg, uint32_t index) 428 { 429 (void)arg; 430 (void)index; 431 432 return (B_FALSE); 433 } 434 435 static boolean_t 436 sfxge_ev_wake_up(void *arg, uint32_t index) 437 { 438 (void)arg; 439 (void)index; 440 441 return (B_FALSE); 442 } 443 444 #if EFSYS_OPT_QSTATS 445 446 static void 447 sfxge_evq_stat_update(struct sfxge_evq *evq) 448 { 449 clock_t now; 450 451 SFXGE_EVQ_LOCK(evq); 452 453 if (__predict_false(evq->init_state != SFXGE_EVQ_STARTED)) 454 goto out; 455 456 now = ticks; 457 if ((unsigned int)(now - evq->stats_update_time) < (unsigned int)hz) 458 goto out; 459 460 evq->stats_update_time = now; 461 efx_ev_qstats_update(evq->common, evq->stats); 462 463 out: 464 SFXGE_EVQ_UNLOCK(evq); 465 } 466 467 static int 468 sfxge_evq_stat_handler(SYSCTL_HANDLER_ARGS) 469 { 470 struct sfxge_evq *evq = arg1; 471 struct sfxge_softc *sc = evq->sc; 472 unsigned int id = arg2; 473 474 SFXGE_ADAPTER_LOCK(sc); 475 476 sfxge_evq_stat_update(evq); 477 478 SFXGE_ADAPTER_UNLOCK(sc); 479 480 return (SYSCTL_OUT(req, &evq->stats[id], sizeof(evq->stats[id]))); 481 } 482 483 static int 484 sfxge_evq_stat_init(struct sfxge_evq *evq) 485 { 486 struct sfxge_softc *sc = evq->sc; 487 struct sysctl_ctx_list *ctx = device_get_sysctl_ctx(sc->dev); 488 char name[16]; 489 struct sysctl_oid *evq_stats_node; 490 unsigned int id; 491 492 snprintf(name, sizeof(name), "%u", evq->index); 493 evq_stats_node = SYSCTL_ADD_NODE(ctx, 494 SYSCTL_CHILDREN(sc->evqs_stats_node), OID_AUTO, name, 495 CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, ""); 496 if (evq_stats_node == NULL) 497 return (ENOMEM); 498 499 for (id = 0; id < EV_NQSTATS; id++) { 500 SYSCTL_ADD_PROC(ctx, SYSCTL_CHILDREN(evq_stats_node), 501 OID_AUTO, efx_ev_qstat_name(sc->enp, id), 502 CTLTYPE_U64 | CTLFLAG_RD | CTLFLAG_MPSAFE, evq, id, 503 sfxge_evq_stat_handler, "Q", ""); 504 } 505 506 return (0); 507 } 508 509 static void 510 sfxge_ev_stat_update(struct sfxge_softc *sc) 511 { 512 struct sfxge_evq *evq; 513 unsigned int index; 514 clock_t now; 515 unsigned int id; 516 517 SFXGE_ADAPTER_LOCK(sc); 518 519 now = ticks; 520 if ((unsigned int)(now - sc->ev_stats_update_time) < (unsigned int)hz) 521 goto out; 522 523 sc->ev_stats_update_time = now; 524 525 memset(sc->ev_stats, 0, sizeof(sc->ev_stats)); 526 527 /* Update and add event counts from each event queue in turn */ 528 for (index = 0; index < sc->evq_count; index++) { 529 evq = sc->evq[index]; 530 sfxge_evq_stat_update(evq); 531 for (id = 0; id < EV_NQSTATS; id++) 532 sc->ev_stats[id] += evq->stats[id]; 533 } 534 out: 535 SFXGE_ADAPTER_UNLOCK(sc); 536 } 537 538 static int 539 sfxge_ev_stat_handler(SYSCTL_HANDLER_ARGS) 540 { 541 struct sfxge_softc *sc = arg1; 542 unsigned int id = arg2; 543 544 sfxge_ev_stat_update(sc); 545 546 return (SYSCTL_OUT(req, &sc->ev_stats[id], sizeof(sc->ev_stats[id]))); 547 } 548 549 static void 550 sfxge_ev_stat_init(struct sfxge_softc *sc) 551 { 552 struct sysctl_ctx_list *ctx = device_get_sysctl_ctx(sc->dev); 553 struct sysctl_oid_list *stat_list; 554 unsigned int id; 555 char name[40]; 556 557 stat_list = SYSCTL_CHILDREN(sc->stats_node); 558 559 for (id = 0; id < EV_NQSTATS; id++) { 560 snprintf(name, sizeof(name), "ev_%s", 561 efx_ev_qstat_name(sc->enp, id)); 562 SYSCTL_ADD_PROC(ctx, stat_list, OID_AUTO, name, 563 CTLTYPE_U64 | CTLFLAG_RD | CTLFLAG_MPSAFE, 564 sc, id, sfxge_ev_stat_handler, "Q", ""); 565 } 566 } 567 568 #endif /* EFSYS_OPT_QSTATS */ 569 570 static void 571 sfxge_ev_qmoderate(struct sfxge_softc *sc, unsigned int idx, unsigned int us) 572 { 573 struct sfxge_evq *evq; 574 efx_evq_t *eep; 575 576 evq = sc->evq[idx]; 577 eep = evq->common; 578 579 KASSERT(evq->init_state == SFXGE_EVQ_STARTED, 580 ("evq->init_state != SFXGE_EVQ_STARTED")); 581 582 (void)efx_ev_qmoderate(eep, us); 583 } 584 585 static int 586 sfxge_int_mod_handler(SYSCTL_HANDLER_ARGS) 587 { 588 struct sfxge_softc *sc = arg1; 589 struct sfxge_intr *intr = &sc->intr; 590 unsigned int moderation; 591 int error; 592 unsigned int index; 593 594 SFXGE_ADAPTER_LOCK(sc); 595 596 if (req->newptr != NULL) { 597 if ((error = SYSCTL_IN(req, &moderation, sizeof(moderation))) 598 != 0) 599 goto out; 600 601 /* We may not be calling efx_ev_qmoderate() now, 602 * so we have to range-check the value ourselves. 603 */ 604 if (moderation > 605 efx_nic_cfg_get(sc->enp)->enc_evq_timer_max_us) { 606 error = EINVAL; 607 goto out; 608 } 609 610 sc->ev_moderation = moderation; 611 if (intr->state == SFXGE_INTR_STARTED) { 612 for (index = 0; index < sc->evq_count; index++) 613 sfxge_ev_qmoderate(sc, index, moderation); 614 } 615 } else { 616 error = SYSCTL_OUT(req, &sc->ev_moderation, 617 sizeof(sc->ev_moderation)); 618 } 619 620 out: 621 SFXGE_ADAPTER_UNLOCK(sc); 622 623 return (error); 624 } 625 626 static boolean_t 627 sfxge_ev_initialized(void *arg) 628 { 629 struct sfxge_evq *evq; 630 631 evq = (struct sfxge_evq *)arg; 632 SFXGE_EVQ_LOCK_ASSERT_OWNED(evq); 633 634 /* Init done events may be duplicated on 7xxx */ 635 KASSERT(evq->init_state == SFXGE_EVQ_STARTING || 636 evq->init_state == SFXGE_EVQ_STARTED, 637 ("evq not starting")); 638 639 evq->init_state = SFXGE_EVQ_STARTED; 640 641 return (0); 642 } 643 644 static boolean_t 645 sfxge_ev_link_change(void *arg, efx_link_mode_t link_mode) 646 { 647 struct sfxge_evq *evq; 648 struct sfxge_softc *sc; 649 650 evq = (struct sfxge_evq *)arg; 651 SFXGE_EVQ_LOCK_ASSERT_OWNED(evq); 652 653 sc = evq->sc; 654 655 sfxge_mac_link_update(sc, link_mode); 656 657 return (0); 658 } 659 660 static const efx_ev_callbacks_t sfxge_ev_callbacks = { 661 .eec_initialized = sfxge_ev_initialized, 662 .eec_rx = sfxge_ev_rx, 663 .eec_tx = sfxge_ev_tx, 664 .eec_exception = sfxge_ev_exception, 665 .eec_rxq_flush_done = sfxge_ev_rxq_flush_done, 666 .eec_rxq_flush_failed = sfxge_ev_rxq_flush_failed, 667 .eec_txq_flush_done = sfxge_ev_txq_flush_done, 668 .eec_software = sfxge_ev_software, 669 .eec_sram = sfxge_ev_sram, 670 .eec_wake_up = sfxge_ev_wake_up, 671 .eec_timer = sfxge_ev_timer, 672 .eec_link_change = sfxge_ev_link_change, 673 }; 674 675 int 676 sfxge_ev_qpoll(struct sfxge_evq *evq) 677 { 678 int rc; 679 680 SFXGE_EVQ_LOCK(evq); 681 682 if (__predict_false(evq->init_state != SFXGE_EVQ_STARTING && 683 evq->init_state != SFXGE_EVQ_STARTED)) { 684 rc = EINVAL; 685 goto fail; 686 } 687 688 /* Synchronize the DMA memory for reading */ 689 bus_dmamap_sync(evq->mem.esm_tag, evq->mem.esm_map, 690 BUS_DMASYNC_POSTREAD); 691 692 KASSERT(evq->rx_done == 0, ("evq->rx_done != 0")); 693 KASSERT(evq->tx_done == 0, ("evq->tx_done != 0")); 694 KASSERT(evq->txq == NULL, ("evq->txq != NULL")); 695 KASSERT(evq->txqs == &evq->txq, ("evq->txqs != &evq->txq")); 696 697 /* Poll the queue */ 698 efx_ev_qpoll(evq->common, &evq->read_ptr, &sfxge_ev_callbacks, evq); 699 700 evq->rx_done = 0; 701 evq->tx_done = 0; 702 703 /* Perform any pending completion processing */ 704 sfxge_ev_qcomplete(evq, B_TRUE); 705 706 /* Re-prime the event queue for interrupts */ 707 if ((rc = efx_ev_qprime(evq->common, evq->read_ptr)) != 0) 708 goto fail; 709 710 SFXGE_EVQ_UNLOCK(evq); 711 712 return (0); 713 714 fail: 715 SFXGE_EVQ_UNLOCK(evq); 716 return (rc); 717 } 718 719 static void 720 sfxge_ev_qstop(struct sfxge_softc *sc, unsigned int index) 721 { 722 struct sfxge_evq *evq; 723 724 evq = sc->evq[index]; 725 726 KASSERT(evq->init_state == SFXGE_EVQ_STARTED, 727 ("evq->init_state != SFXGE_EVQ_STARTED")); 728 729 SFXGE_EVQ_LOCK(evq); 730 evq->init_state = SFXGE_EVQ_INITIALIZED; 731 evq->read_ptr = 0; 732 evq->exception = B_FALSE; 733 734 #if EFSYS_OPT_QSTATS 735 /* Add event counts before discarding the common evq state */ 736 efx_ev_qstats_update(evq->common, evq->stats); 737 #endif 738 739 efx_ev_qdestroy(evq->common); 740 efx_sram_buf_tbl_clear(sc->enp, evq->buf_base_id, 741 EFX_EVQ_NBUFS(evq->entries)); 742 SFXGE_EVQ_UNLOCK(evq); 743 } 744 745 static int 746 sfxge_ev_qstart(struct sfxge_softc *sc, unsigned int index) 747 { 748 struct sfxge_evq *evq; 749 efsys_mem_t *esmp; 750 int count; 751 int rc; 752 753 evq = sc->evq[index]; 754 esmp = &evq->mem; 755 756 KASSERT(evq->init_state == SFXGE_EVQ_INITIALIZED, 757 ("evq->init_state != SFXGE_EVQ_INITIALIZED")); 758 759 /* Clear all events. */ 760 (void)memset(esmp->esm_base, 0xff, EFX_EVQ_SIZE(evq->entries)); 761 762 /* Program the buffer table. */ 763 if ((rc = efx_sram_buf_tbl_set(sc->enp, evq->buf_base_id, esmp, 764 EFX_EVQ_NBUFS(evq->entries))) != 0) 765 return (rc); 766 767 /* Create the common code event queue. */ 768 if ((rc = efx_ev_qcreate(sc->enp, index, esmp, evq->entries, 769 evq->buf_base_id, sc->ev_moderation, EFX_EVQ_FLAGS_TYPE_AUTO, 770 &evq->common)) != 0) 771 goto fail; 772 773 SFXGE_EVQ_LOCK(evq); 774 775 /* Prime the event queue for interrupts */ 776 if ((rc = efx_ev_qprime(evq->common, evq->read_ptr)) != 0) 777 goto fail2; 778 779 evq->init_state = SFXGE_EVQ_STARTING; 780 781 SFXGE_EVQ_UNLOCK(evq); 782 783 /* Wait for the initialization event */ 784 count = 0; 785 do { 786 /* Pause for 100 ms */ 787 pause("sfxge evq init", hz / 10); 788 789 /* Check to see if the test event has been processed */ 790 if (evq->init_state == SFXGE_EVQ_STARTED) 791 goto done; 792 793 } while (++count < 20); 794 795 rc = ETIMEDOUT; 796 goto fail3; 797 798 done: 799 return (0); 800 801 fail3: 802 SFXGE_EVQ_LOCK(evq); 803 evq->init_state = SFXGE_EVQ_INITIALIZED; 804 fail2: 805 SFXGE_EVQ_UNLOCK(evq); 806 efx_ev_qdestroy(evq->common); 807 fail: 808 efx_sram_buf_tbl_clear(sc->enp, evq->buf_base_id, 809 EFX_EVQ_NBUFS(evq->entries)); 810 811 return (rc); 812 } 813 814 void 815 sfxge_ev_stop(struct sfxge_softc *sc) 816 { 817 struct sfxge_intr *intr __diagused; 818 efx_nic_t *enp; 819 int index; 820 821 intr = &sc->intr; 822 enp = sc->enp; 823 824 KASSERT(intr->state == SFXGE_INTR_STARTED, 825 ("Interrupts not started")); 826 827 /* Stop the event queue(s) */ 828 index = sc->evq_count; 829 while (--index >= 0) 830 sfxge_ev_qstop(sc, index); 831 832 /* Tear down the event module */ 833 efx_ev_fini(enp); 834 } 835 836 int 837 sfxge_ev_start(struct sfxge_softc *sc) 838 { 839 struct sfxge_intr *intr __diagused; 840 int index; 841 int rc; 842 843 intr = &sc->intr; 844 845 KASSERT(intr->state == SFXGE_INTR_STARTED, 846 ("intr->state != SFXGE_INTR_STARTED")); 847 848 /* Initialize the event module */ 849 if ((rc = efx_ev_init(sc->enp)) != 0) 850 return (rc); 851 852 /* Start the event queues */ 853 for (index = 0; index < sc->evq_count; index++) { 854 if ((rc = sfxge_ev_qstart(sc, index)) != 0) 855 goto fail; 856 } 857 858 return (0); 859 860 fail: 861 /* Stop the event queue(s) */ 862 while (--index >= 0) 863 sfxge_ev_qstop(sc, index); 864 865 /* Tear down the event module */ 866 efx_ev_fini(sc->enp); 867 868 return (rc); 869 } 870 871 static void 872 sfxge_ev_qfini(struct sfxge_softc *sc, unsigned int index) 873 { 874 struct sfxge_evq *evq; 875 876 evq = sc->evq[index]; 877 878 KASSERT(evq->init_state == SFXGE_EVQ_INITIALIZED, 879 ("evq->init_state != SFXGE_EVQ_INITIALIZED")); 880 KASSERT(evq->txqs == &evq->txq, ("evq->txqs != &evq->txq")); 881 882 sfxge_dma_free(&evq->mem); 883 884 sc->evq[index] = NULL; 885 886 SFXGE_EVQ_LOCK_DESTROY(evq); 887 888 free(evq, M_SFXGE); 889 } 890 891 static int 892 sfxge_ev_qinit(struct sfxge_softc *sc, unsigned int index) 893 { 894 struct sfxge_evq *evq; 895 efsys_mem_t *esmp; 896 int rc; 897 898 KASSERT(index < SFXGE_RX_SCALE_MAX, ("index >= SFXGE_RX_SCALE_MAX")); 899 900 evq = malloc(sizeof(struct sfxge_evq), M_SFXGE, M_ZERO | M_WAITOK); 901 evq->sc = sc; 902 evq->index = index; 903 sc->evq[index] = evq; 904 esmp = &evq->mem; 905 906 /* Build an event queue with room for one event per tx and rx buffer, 907 * plus some extra for link state events and MCDI completions. 908 * There are three tx queues in the first event queue and one in 909 * other. 910 */ 911 if (index == 0) 912 evq->entries = 913 ROUNDUP_POW_OF_TWO(sc->rxq_entries + 914 3 * sc->txq_entries + 915 128); 916 else 917 evq->entries = 918 ROUNDUP_POW_OF_TWO(sc->rxq_entries + 919 sc->txq_entries + 920 128); 921 922 /* Initialise TX completion list */ 923 evq->txqs = &evq->txq; 924 925 /* Allocate DMA space. */ 926 if ((rc = sfxge_dma_alloc(sc, EFX_EVQ_SIZE(evq->entries), esmp)) != 0) 927 return (rc); 928 929 /* Allocate buffer table entries. */ 930 sfxge_sram_buf_tbl_alloc(sc, EFX_EVQ_NBUFS(evq->entries), 931 &evq->buf_base_id); 932 933 SFXGE_EVQ_LOCK_INIT(evq, device_get_nameunit(sc->dev), index); 934 935 evq->init_state = SFXGE_EVQ_INITIALIZED; 936 937 #if EFSYS_OPT_QSTATS 938 rc = sfxge_evq_stat_init(evq); 939 if (rc != 0) 940 goto fail_evq_stat_init; 941 #endif 942 943 return (0); 944 945 #if EFSYS_OPT_QSTATS 946 fail_evq_stat_init: 947 evq->init_state = SFXGE_EVQ_UNINITIALIZED; 948 SFXGE_EVQ_LOCK_DESTROY(evq); 949 sfxge_dma_free(esmp); 950 sc->evq[index] = NULL; 951 free(evq, M_SFXGE); 952 953 return (rc); 954 #endif 955 } 956 957 void 958 sfxge_ev_fini(struct sfxge_softc *sc) 959 { 960 struct sfxge_intr *intr __diagused; 961 int index; 962 963 intr = &sc->intr; 964 965 KASSERT(intr->state == SFXGE_INTR_INITIALIZED, 966 ("intr->state != SFXGE_INTR_INITIALIZED")); 967 968 sc->ev_moderation = 0; 969 970 /* Tear down the event queue(s). */ 971 index = sc->evq_count; 972 while (--index >= 0) 973 sfxge_ev_qfini(sc, index); 974 975 sc->evq_count = 0; 976 } 977 978 int 979 sfxge_ev_init(struct sfxge_softc *sc) 980 { 981 struct sysctl_ctx_list *sysctl_ctx = device_get_sysctl_ctx(sc->dev); 982 struct sysctl_oid *sysctl_tree = device_get_sysctl_tree(sc->dev); 983 struct sfxge_intr *intr; 984 int index; 985 int rc; 986 987 intr = &sc->intr; 988 989 sc->evq_count = intr->n_alloc; 990 991 KASSERT(intr->state == SFXGE_INTR_INITIALIZED, 992 ("intr->state != SFXGE_INTR_INITIALIZED")); 993 994 /* Set default interrupt moderation; add a sysctl to 995 * read and change it. 996 */ 997 sc->ev_moderation = SFXGE_MODERATION; 998 SYSCTL_ADD_PROC(sysctl_ctx, SYSCTL_CHILDREN(sysctl_tree), 999 OID_AUTO, "int_mod", CTLTYPE_UINT | CTLFLAG_RW | CTLFLAG_MPSAFE, 1000 sc, 0, sfxge_int_mod_handler, "IU", 1001 "sfxge interrupt moderation (us)"); 1002 1003 #if EFSYS_OPT_QSTATS 1004 sc->evqs_stats_node = SYSCTL_ADD_NODE( 1005 device_get_sysctl_ctx(sc->dev), SYSCTL_CHILDREN(sc->stats_node), 1006 OID_AUTO, "evq", CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, 1007 "Event queues stats"); 1008 if (sc->evqs_stats_node == NULL) { 1009 rc = ENOMEM; 1010 goto fail_evqs_stats_node; 1011 } 1012 #endif 1013 1014 /* 1015 * Initialize the event queue(s) - one per interrupt. 1016 */ 1017 for (index = 0; index < sc->evq_count; index++) { 1018 if ((rc = sfxge_ev_qinit(sc, index)) != 0) 1019 goto fail; 1020 } 1021 1022 #if EFSYS_OPT_QSTATS 1023 sfxge_ev_stat_init(sc); 1024 #endif 1025 1026 return (0); 1027 1028 fail: 1029 while (--index >= 0) 1030 sfxge_ev_qfini(sc, index); 1031 1032 #if EFSYS_OPT_QSTATS 1033 fail_evqs_stats_node: 1034 #endif 1035 sc->evq_count = 0; 1036 return (rc); 1037 } 1038