1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause 3 * 4 * Copyright (c) 2010-2016 Solarflare Communications Inc. 5 * All rights reserved. 6 * 7 * This software was developed in part by Philip Paeps under contract for 8 * Solarflare Communications, Inc. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions are met: 12 * 13 * 1. Redistributions of source code must retain the above copyright notice, 14 * this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright notice, 16 * this list of conditions and the following disclaimer in the documentation 17 * and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 20 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, 21 * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR 23 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, 24 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 25 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; 26 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 27 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR 28 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, 29 * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 * 31 * The views and conclusions contained in the software and documentation are 32 * those of the authors and should not be interpreted as representing official 33 * policies, either expressed or implied, of the FreeBSD Project. 34 */ 35 36 #include <sys/cdefs.h> 37 #include <sys/param.h> 38 #include <sys/kernel.h> 39 #include <sys/malloc.h> 40 #include <sys/param.h> 41 #include <sys/queue.h> 42 #include <sys/systm.h> 43 #include <sys/taskqueue.h> 44 45 #include "common/efx.h" 46 47 #include "sfxge.h" 48 49 static void 50 sfxge_ev_qcomplete(struct sfxge_evq *evq, boolean_t eop) 51 { 52 struct sfxge_softc *sc; 53 unsigned int index; 54 struct sfxge_rxq *rxq; 55 struct sfxge_txq *txq; 56 57 SFXGE_EVQ_LOCK_ASSERT_OWNED(evq); 58 59 sc = evq->sc; 60 index = evq->index; 61 rxq = sc->rxq[index]; 62 63 if ((txq = evq->txq) != NULL) { 64 evq->txq = NULL; 65 evq->txqs = &(evq->txq); 66 67 do { 68 struct sfxge_txq *next; 69 70 next = txq->next; 71 txq->next = NULL; 72 73 KASSERT(txq->evq_index == index, 74 ("txq->evq_index != index")); 75 76 if (txq->pending != txq->completed) 77 sfxge_tx_qcomplete(txq, evq); 78 79 txq = next; 80 } while (txq != NULL); 81 } 82 83 if (rxq->pending != rxq->completed) 84 sfxge_rx_qcomplete(rxq, eop); 85 } 86 87 static struct sfxge_rxq * 88 sfxge_get_rxq_by_label(struct sfxge_evq *evq, uint32_t label) 89 { 90 struct sfxge_rxq *rxq; 91 92 KASSERT(label == 0, ("unexpected rxq label != 0")); 93 94 rxq = evq->sc->rxq[evq->index]; 95 96 KASSERT(rxq != NULL, ("rxq == NULL")); 97 KASSERT(evq->index == rxq->index, ("evq->index != rxq->index")); 98 99 return (rxq); 100 } 101 102 static boolean_t 103 sfxge_ev_rx(void *arg, uint32_t label, uint32_t id, uint32_t size, 104 uint16_t flags) 105 { 106 struct sfxge_evq *evq; 107 struct sfxge_softc *sc; 108 struct sfxge_rxq *rxq; 109 unsigned int stop; 110 unsigned int delta; 111 struct sfxge_rx_sw_desc *rx_desc; 112 113 evq = arg; 114 SFXGE_EVQ_LOCK_ASSERT_OWNED(evq); 115 116 sc = evq->sc; 117 118 if (evq->exception) 119 goto done; 120 121 rxq = sfxge_get_rxq_by_label(evq, label); 122 if (__predict_false(rxq->init_state != SFXGE_RXQ_STARTED)) 123 goto done; 124 125 stop = (id + 1) & rxq->ptr_mask; 126 id = rxq->pending & rxq->ptr_mask; 127 delta = (stop >= id) ? (stop - id) : (rxq->entries - id + stop); 128 rxq->pending += delta; 129 130 if (delta != 1) { 131 if ((delta <= 0) || 132 (delta > efx_nic_cfg_get(sc->enp)->enc_rx_batch_max)) { 133 evq->exception = B_TRUE; 134 135 device_printf(sc->dev, "RX completion out of order" 136 " (id=%#x delta=%u flags=%#x); resetting\n", 137 id, delta, flags); 138 sfxge_schedule_reset(sc); 139 140 goto done; 141 } 142 } 143 144 rx_desc = &rxq->queue[id]; 145 146 prefetch_read_many(rx_desc->mbuf); 147 148 for (; id != stop; id = (id + 1) & rxq->ptr_mask) { 149 rx_desc = &rxq->queue[id]; 150 KASSERT(rx_desc->flags == EFX_DISCARD, 151 ("rx_desc->flags != EFX_DISCARD")); 152 rx_desc->flags = flags; 153 154 KASSERT(size < (1 << 16), ("size > (1 << 16)")); 155 rx_desc->size = (uint16_t)size; 156 } 157 158 evq->rx_done++; 159 160 if (rxq->pending - rxq->completed >= SFXGE_RX_BATCH) 161 sfxge_ev_qcomplete(evq, B_FALSE); 162 163 done: 164 return (evq->rx_done >= SFXGE_EV_BATCH); 165 } 166 167 static boolean_t 168 sfxge_ev_exception(void *arg, uint32_t code, uint32_t data) 169 { 170 struct sfxge_evq *evq; 171 struct sfxge_softc *sc; 172 173 evq = (struct sfxge_evq *)arg; 174 SFXGE_EVQ_LOCK_ASSERT_OWNED(evq); 175 176 sc = evq->sc; 177 178 DBGPRINT(sc->dev, "[%d] %s", evq->index, 179 (code == EFX_EXCEPTION_RX_RECOVERY) ? "RX_RECOVERY" : 180 (code == EFX_EXCEPTION_RX_DSC_ERROR) ? "RX_DSC_ERROR" : 181 (code == EFX_EXCEPTION_TX_DSC_ERROR) ? "TX_DSC_ERROR" : 182 (code == EFX_EXCEPTION_UNKNOWN_SENSOREVT) ? "UNKNOWN_SENSOREVT" : 183 (code == EFX_EXCEPTION_FWALERT_SRAM) ? "FWALERT_SRAM" : 184 (code == EFX_EXCEPTION_UNKNOWN_FWALERT) ? "UNKNOWN_FWALERT" : 185 (code == EFX_EXCEPTION_RX_ERROR) ? "RX_ERROR" : 186 (code == EFX_EXCEPTION_TX_ERROR) ? "TX_ERROR" : 187 (code == EFX_EXCEPTION_EV_ERROR) ? "EV_ERROR" : 188 "UNKNOWN"); 189 190 evq->exception = B_TRUE; 191 192 if (code != EFX_EXCEPTION_UNKNOWN_SENSOREVT) { 193 device_printf(sc->dev, 194 "hardware exception (code=%u); resetting\n", 195 code); 196 sfxge_schedule_reset(sc); 197 } 198 199 return (B_FALSE); 200 } 201 202 static boolean_t 203 sfxge_ev_rxq_flush_done(void *arg, uint32_t rxq_index) 204 { 205 struct sfxge_evq *evq; 206 struct sfxge_softc *sc; 207 struct sfxge_rxq *rxq; 208 unsigned int index; 209 uint16_t magic; 210 211 evq = (struct sfxge_evq *)arg; 212 SFXGE_EVQ_LOCK_ASSERT_OWNED(evq); 213 214 sc = evq->sc; 215 rxq = sc->rxq[rxq_index]; 216 217 KASSERT(rxq != NULL, ("rxq == NULL")); 218 219 /* Resend a software event on the correct queue */ 220 index = rxq->index; 221 if (index == evq->index) { 222 sfxge_rx_qflush_done(rxq); 223 return (B_FALSE); 224 } 225 226 evq = sc->evq[index]; 227 magic = sfxge_sw_ev_rxq_magic(SFXGE_SW_EV_RX_QFLUSH_DONE, rxq); 228 229 KASSERT(evq->init_state == SFXGE_EVQ_STARTED, 230 ("evq not started")); 231 efx_ev_qpost(evq->common, magic); 232 233 return (B_FALSE); 234 } 235 236 static boolean_t 237 sfxge_ev_rxq_flush_failed(void *arg, uint32_t rxq_index) 238 { 239 struct sfxge_evq *evq; 240 struct sfxge_softc *sc; 241 struct sfxge_rxq *rxq; 242 unsigned int index; 243 uint16_t magic; 244 245 evq = (struct sfxge_evq *)arg; 246 SFXGE_EVQ_LOCK_ASSERT_OWNED(evq); 247 248 sc = evq->sc; 249 rxq = sc->rxq[rxq_index]; 250 251 KASSERT(rxq != NULL, ("rxq == NULL")); 252 253 /* Resend a software event on the correct queue */ 254 index = rxq->index; 255 evq = sc->evq[index]; 256 magic = sfxge_sw_ev_rxq_magic(SFXGE_SW_EV_RX_QFLUSH_FAILED, rxq); 257 258 KASSERT(evq->init_state == SFXGE_EVQ_STARTED, 259 ("evq not started")); 260 efx_ev_qpost(evq->common, magic); 261 262 return (B_FALSE); 263 } 264 265 static struct sfxge_txq * 266 sfxge_get_txq_by_label(struct sfxge_evq *evq, enum sfxge_txq_type label) 267 { 268 unsigned int index; 269 270 KASSERT((evq->sc->txq_dynamic_cksum_toggle_supported) ? (label == 0) : 271 ((evq->index == 0 && label < SFXGE_TXQ_NTYPES) || 272 (label == SFXGE_TXQ_IP_TCP_UDP_CKSUM)), 273 ("unexpected txq label")); 274 275 index = (evq->index == 0) ? 276 label : (evq->index - 1 + SFXGE_EVQ0_N_TXQ(evq->sc)); 277 return (evq->sc->txq[index]); 278 } 279 280 static boolean_t 281 sfxge_ev_tx(void *arg, uint32_t label, uint32_t id) 282 { 283 struct sfxge_evq *evq; 284 struct sfxge_txq *txq; 285 unsigned int stop; 286 unsigned int delta; 287 288 evq = (struct sfxge_evq *)arg; 289 SFXGE_EVQ_LOCK_ASSERT_OWNED(evq); 290 291 txq = sfxge_get_txq_by_label(evq, label); 292 293 KASSERT(txq != NULL, ("txq == NULL")); 294 KASSERT(evq->index == txq->evq_index, 295 ("evq->index != txq->evq_index")); 296 297 if (__predict_false(txq->init_state != SFXGE_TXQ_STARTED)) 298 goto done; 299 300 stop = (id + 1) & txq->ptr_mask; 301 id = txq->pending & txq->ptr_mask; 302 303 delta = (stop >= id) ? (stop - id) : (txq->entries - id + stop); 304 txq->pending += delta; 305 306 evq->tx_done++; 307 308 if (txq->next == NULL && 309 evq->txqs != &(txq->next)) { 310 *(evq->txqs) = txq; 311 evq->txqs = &(txq->next); 312 } 313 314 if (txq->pending - txq->completed >= SFXGE_TX_BATCH) 315 sfxge_tx_qcomplete(txq, evq); 316 317 done: 318 return (evq->tx_done >= SFXGE_EV_BATCH); 319 } 320 321 static boolean_t 322 sfxge_ev_txq_flush_done(void *arg, uint32_t txq_index) 323 { 324 struct sfxge_evq *evq; 325 struct sfxge_softc *sc; 326 struct sfxge_txq *txq; 327 uint16_t magic; 328 329 evq = (struct sfxge_evq *)arg; 330 SFXGE_EVQ_LOCK_ASSERT_OWNED(evq); 331 332 sc = evq->sc; 333 txq = sc->txq[txq_index]; 334 335 KASSERT(txq != NULL, ("txq == NULL")); 336 KASSERT(txq->init_state == SFXGE_TXQ_INITIALIZED, 337 ("txq not initialized")); 338 339 if (txq->evq_index == evq->index) { 340 sfxge_tx_qflush_done(txq); 341 return (B_FALSE); 342 } 343 344 /* Resend a software event on the correct queue */ 345 evq = sc->evq[txq->evq_index]; 346 magic = sfxge_sw_ev_txq_magic(SFXGE_SW_EV_TX_QFLUSH_DONE, txq); 347 348 KASSERT(evq->init_state == SFXGE_EVQ_STARTED, 349 ("evq not started")); 350 efx_ev_qpost(evq->common, magic); 351 352 return (B_FALSE); 353 } 354 355 static boolean_t 356 sfxge_ev_software(void *arg, uint16_t magic) 357 { 358 struct sfxge_evq *evq; 359 unsigned int label; 360 361 evq = (struct sfxge_evq *)arg; 362 SFXGE_EVQ_LOCK_ASSERT_OWNED(evq); 363 364 label = magic & SFXGE_MAGIC_DMAQ_LABEL_MASK; 365 magic &= ~SFXGE_MAGIC_DMAQ_LABEL_MASK; 366 367 switch (magic) { 368 case SFXGE_SW_EV_MAGIC(SFXGE_SW_EV_RX_QFLUSH_DONE): 369 sfxge_rx_qflush_done(sfxge_get_rxq_by_label(evq, label)); 370 break; 371 372 case SFXGE_SW_EV_MAGIC(SFXGE_SW_EV_RX_QFLUSH_FAILED): 373 sfxge_rx_qflush_failed(sfxge_get_rxq_by_label(evq, label)); 374 break; 375 376 case SFXGE_SW_EV_MAGIC(SFXGE_SW_EV_RX_QREFILL): 377 sfxge_rx_qrefill(sfxge_get_rxq_by_label(evq, label)); 378 break; 379 380 case SFXGE_SW_EV_MAGIC(SFXGE_SW_EV_TX_QFLUSH_DONE): { 381 struct sfxge_txq *txq = sfxge_get_txq_by_label(evq, label); 382 383 KASSERT(txq != NULL, ("txq == NULL")); 384 KASSERT(evq->index == txq->evq_index, 385 ("evq->index != txq->evq_index")); 386 387 sfxge_tx_qflush_done(txq); 388 break; 389 } 390 default: 391 break; 392 } 393 394 return (B_FALSE); 395 } 396 397 static boolean_t 398 sfxge_ev_sram(void *arg, uint32_t code) 399 { 400 (void)arg; 401 (void)code; 402 403 switch (code) { 404 case EFX_SRAM_UPDATE: 405 EFSYS_PROBE(sram_update); 406 break; 407 408 case EFX_SRAM_CLEAR: 409 EFSYS_PROBE(sram_clear); 410 break; 411 412 case EFX_SRAM_ILLEGAL_CLEAR: 413 EFSYS_PROBE(sram_illegal_clear); 414 break; 415 416 default: 417 KASSERT(B_FALSE, ("Impossible SRAM event")); 418 break; 419 } 420 421 return (B_FALSE); 422 } 423 424 static boolean_t 425 sfxge_ev_timer(void *arg, uint32_t index) 426 { 427 (void)arg; 428 (void)index; 429 430 return (B_FALSE); 431 } 432 433 static boolean_t 434 sfxge_ev_wake_up(void *arg, uint32_t index) 435 { 436 (void)arg; 437 (void)index; 438 439 return (B_FALSE); 440 } 441 442 #if EFSYS_OPT_QSTATS 443 444 static void 445 sfxge_evq_stat_update(struct sfxge_evq *evq) 446 { 447 clock_t now; 448 449 SFXGE_EVQ_LOCK(evq); 450 451 if (__predict_false(evq->init_state != SFXGE_EVQ_STARTED)) 452 goto out; 453 454 now = ticks; 455 if ((unsigned int)(now - evq->stats_update_time) < (unsigned int)hz) 456 goto out; 457 458 evq->stats_update_time = now; 459 efx_ev_qstats_update(evq->common, evq->stats); 460 461 out: 462 SFXGE_EVQ_UNLOCK(evq); 463 } 464 465 static int 466 sfxge_evq_stat_handler(SYSCTL_HANDLER_ARGS) 467 { 468 struct sfxge_evq *evq = arg1; 469 struct sfxge_softc *sc = evq->sc; 470 unsigned int id = arg2; 471 472 SFXGE_ADAPTER_LOCK(sc); 473 474 sfxge_evq_stat_update(evq); 475 476 SFXGE_ADAPTER_UNLOCK(sc); 477 478 return (SYSCTL_OUT(req, &evq->stats[id], sizeof(evq->stats[id]))); 479 } 480 481 static int 482 sfxge_evq_stat_init(struct sfxge_evq *evq) 483 { 484 struct sfxge_softc *sc = evq->sc; 485 struct sysctl_ctx_list *ctx = device_get_sysctl_ctx(sc->dev); 486 char name[16]; 487 struct sysctl_oid *evq_stats_node; 488 unsigned int id; 489 490 snprintf(name, sizeof(name), "%u", evq->index); 491 evq_stats_node = SYSCTL_ADD_NODE(ctx, 492 SYSCTL_CHILDREN(sc->evqs_stats_node), OID_AUTO, name, 493 CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, ""); 494 if (evq_stats_node == NULL) 495 return (ENOMEM); 496 497 for (id = 0; id < EV_NQSTATS; id++) { 498 SYSCTL_ADD_PROC(ctx, SYSCTL_CHILDREN(evq_stats_node), 499 OID_AUTO, efx_ev_qstat_name(sc->enp, id), 500 CTLTYPE_U64 | CTLFLAG_RD | CTLFLAG_MPSAFE, evq, id, 501 sfxge_evq_stat_handler, "Q", ""); 502 } 503 504 return (0); 505 } 506 507 static void 508 sfxge_ev_stat_update(struct sfxge_softc *sc) 509 { 510 struct sfxge_evq *evq; 511 unsigned int index; 512 clock_t now; 513 unsigned int id; 514 515 SFXGE_ADAPTER_LOCK(sc); 516 517 now = ticks; 518 if ((unsigned int)(now - sc->ev_stats_update_time) < (unsigned int)hz) 519 goto out; 520 521 sc->ev_stats_update_time = now; 522 523 memset(sc->ev_stats, 0, sizeof(sc->ev_stats)); 524 525 /* Update and add event counts from each event queue in turn */ 526 for (index = 0; index < sc->evq_count; index++) { 527 evq = sc->evq[index]; 528 sfxge_evq_stat_update(evq); 529 for (id = 0; id < EV_NQSTATS; id++) 530 sc->ev_stats[id] += evq->stats[id]; 531 } 532 out: 533 SFXGE_ADAPTER_UNLOCK(sc); 534 } 535 536 static int 537 sfxge_ev_stat_handler(SYSCTL_HANDLER_ARGS) 538 { 539 struct sfxge_softc *sc = arg1; 540 unsigned int id = arg2; 541 542 sfxge_ev_stat_update(sc); 543 544 return (SYSCTL_OUT(req, &sc->ev_stats[id], sizeof(sc->ev_stats[id]))); 545 } 546 547 static void 548 sfxge_ev_stat_init(struct sfxge_softc *sc) 549 { 550 struct sysctl_ctx_list *ctx = device_get_sysctl_ctx(sc->dev); 551 struct sysctl_oid_list *stat_list; 552 unsigned int id; 553 char name[40]; 554 555 stat_list = SYSCTL_CHILDREN(sc->stats_node); 556 557 for (id = 0; id < EV_NQSTATS; id++) { 558 snprintf(name, sizeof(name), "ev_%s", 559 efx_ev_qstat_name(sc->enp, id)); 560 SYSCTL_ADD_PROC(ctx, stat_list, OID_AUTO, name, 561 CTLTYPE_U64 | CTLFLAG_RD | CTLFLAG_MPSAFE, 562 sc, id, sfxge_ev_stat_handler, "Q", ""); 563 } 564 } 565 566 #endif /* EFSYS_OPT_QSTATS */ 567 568 static void 569 sfxge_ev_qmoderate(struct sfxge_softc *sc, unsigned int idx, unsigned int us) 570 { 571 struct sfxge_evq *evq; 572 efx_evq_t *eep; 573 574 evq = sc->evq[idx]; 575 eep = evq->common; 576 577 KASSERT(evq->init_state == SFXGE_EVQ_STARTED, 578 ("evq->init_state != SFXGE_EVQ_STARTED")); 579 580 (void)efx_ev_qmoderate(eep, us); 581 } 582 583 static int 584 sfxge_int_mod_handler(SYSCTL_HANDLER_ARGS) 585 { 586 struct sfxge_softc *sc = arg1; 587 struct sfxge_intr *intr = &sc->intr; 588 unsigned int moderation; 589 int error; 590 unsigned int index; 591 592 SFXGE_ADAPTER_LOCK(sc); 593 594 if (req->newptr != NULL) { 595 if ((error = SYSCTL_IN(req, &moderation, sizeof(moderation))) 596 != 0) 597 goto out; 598 599 /* We may not be calling efx_ev_qmoderate() now, 600 * so we have to range-check the value ourselves. 601 */ 602 if (moderation > 603 efx_nic_cfg_get(sc->enp)->enc_evq_timer_max_us) { 604 error = EINVAL; 605 goto out; 606 } 607 608 sc->ev_moderation = moderation; 609 if (intr->state == SFXGE_INTR_STARTED) { 610 for (index = 0; index < sc->evq_count; index++) 611 sfxge_ev_qmoderate(sc, index, moderation); 612 } 613 } else { 614 error = SYSCTL_OUT(req, &sc->ev_moderation, 615 sizeof(sc->ev_moderation)); 616 } 617 618 out: 619 SFXGE_ADAPTER_UNLOCK(sc); 620 621 return (error); 622 } 623 624 static boolean_t 625 sfxge_ev_initialized(void *arg) 626 { 627 struct sfxge_evq *evq; 628 629 evq = (struct sfxge_evq *)arg; 630 SFXGE_EVQ_LOCK_ASSERT_OWNED(evq); 631 632 /* Init done events may be duplicated on 7xxx */ 633 KASSERT(evq->init_state == SFXGE_EVQ_STARTING || 634 evq->init_state == SFXGE_EVQ_STARTED, 635 ("evq not starting")); 636 637 evq->init_state = SFXGE_EVQ_STARTED; 638 639 return (0); 640 } 641 642 static boolean_t 643 sfxge_ev_link_change(void *arg, efx_link_mode_t link_mode) 644 { 645 struct sfxge_evq *evq; 646 struct sfxge_softc *sc; 647 648 evq = (struct sfxge_evq *)arg; 649 SFXGE_EVQ_LOCK_ASSERT_OWNED(evq); 650 651 sc = evq->sc; 652 653 sfxge_mac_link_update(sc, link_mode); 654 655 return (0); 656 } 657 658 static const efx_ev_callbacks_t sfxge_ev_callbacks = { 659 .eec_initialized = sfxge_ev_initialized, 660 .eec_rx = sfxge_ev_rx, 661 .eec_tx = sfxge_ev_tx, 662 .eec_exception = sfxge_ev_exception, 663 .eec_rxq_flush_done = sfxge_ev_rxq_flush_done, 664 .eec_rxq_flush_failed = sfxge_ev_rxq_flush_failed, 665 .eec_txq_flush_done = sfxge_ev_txq_flush_done, 666 .eec_software = sfxge_ev_software, 667 .eec_sram = sfxge_ev_sram, 668 .eec_wake_up = sfxge_ev_wake_up, 669 .eec_timer = sfxge_ev_timer, 670 .eec_link_change = sfxge_ev_link_change, 671 }; 672 673 int 674 sfxge_ev_qpoll(struct sfxge_evq *evq) 675 { 676 int rc; 677 678 SFXGE_EVQ_LOCK(evq); 679 680 if (__predict_false(evq->init_state != SFXGE_EVQ_STARTING && 681 evq->init_state != SFXGE_EVQ_STARTED)) { 682 rc = EINVAL; 683 goto fail; 684 } 685 686 /* Synchronize the DMA memory for reading */ 687 bus_dmamap_sync(evq->mem.esm_tag, evq->mem.esm_map, 688 BUS_DMASYNC_POSTREAD); 689 690 KASSERT(evq->rx_done == 0, ("evq->rx_done != 0")); 691 KASSERT(evq->tx_done == 0, ("evq->tx_done != 0")); 692 KASSERT(evq->txq == NULL, ("evq->txq != NULL")); 693 KASSERT(evq->txqs == &evq->txq, ("evq->txqs != &evq->txq")); 694 695 /* Poll the queue */ 696 efx_ev_qpoll(evq->common, &evq->read_ptr, &sfxge_ev_callbacks, evq); 697 698 evq->rx_done = 0; 699 evq->tx_done = 0; 700 701 /* Perform any pending completion processing */ 702 sfxge_ev_qcomplete(evq, B_TRUE); 703 704 /* Re-prime the event queue for interrupts */ 705 if ((rc = efx_ev_qprime(evq->common, evq->read_ptr)) != 0) 706 goto fail; 707 708 SFXGE_EVQ_UNLOCK(evq); 709 710 return (0); 711 712 fail: 713 SFXGE_EVQ_UNLOCK(evq); 714 return (rc); 715 } 716 717 static void 718 sfxge_ev_qstop(struct sfxge_softc *sc, unsigned int index) 719 { 720 struct sfxge_evq *evq; 721 722 evq = sc->evq[index]; 723 724 KASSERT(evq->init_state == SFXGE_EVQ_STARTED, 725 ("evq->init_state != SFXGE_EVQ_STARTED")); 726 727 SFXGE_EVQ_LOCK(evq); 728 evq->init_state = SFXGE_EVQ_INITIALIZED; 729 evq->read_ptr = 0; 730 evq->exception = B_FALSE; 731 732 #if EFSYS_OPT_QSTATS 733 /* Add event counts before discarding the common evq state */ 734 efx_ev_qstats_update(evq->common, evq->stats); 735 #endif 736 737 efx_ev_qdestroy(evq->common); 738 efx_sram_buf_tbl_clear(sc->enp, evq->buf_base_id, 739 EFX_EVQ_NBUFS(evq->entries)); 740 SFXGE_EVQ_UNLOCK(evq); 741 } 742 743 static int 744 sfxge_ev_qstart(struct sfxge_softc *sc, unsigned int index) 745 { 746 struct sfxge_evq *evq; 747 efsys_mem_t *esmp; 748 int count; 749 int rc; 750 751 evq = sc->evq[index]; 752 esmp = &evq->mem; 753 754 KASSERT(evq->init_state == SFXGE_EVQ_INITIALIZED, 755 ("evq->init_state != SFXGE_EVQ_INITIALIZED")); 756 757 /* Clear all events. */ 758 (void)memset(esmp->esm_base, 0xff, EFX_EVQ_SIZE(evq->entries)); 759 760 /* Program the buffer table. */ 761 if ((rc = efx_sram_buf_tbl_set(sc->enp, evq->buf_base_id, esmp, 762 EFX_EVQ_NBUFS(evq->entries))) != 0) 763 return (rc); 764 765 /* Create the common code event queue. */ 766 if ((rc = efx_ev_qcreate(sc->enp, index, esmp, evq->entries, 767 evq->buf_base_id, sc->ev_moderation, EFX_EVQ_FLAGS_TYPE_AUTO, 768 &evq->common)) != 0) 769 goto fail; 770 771 SFXGE_EVQ_LOCK(evq); 772 773 /* Prime the event queue for interrupts */ 774 if ((rc = efx_ev_qprime(evq->common, evq->read_ptr)) != 0) 775 goto fail2; 776 777 evq->init_state = SFXGE_EVQ_STARTING; 778 779 SFXGE_EVQ_UNLOCK(evq); 780 781 /* Wait for the initialization event */ 782 count = 0; 783 do { 784 /* Pause for 100 ms */ 785 pause("sfxge evq init", hz / 10); 786 787 /* Check to see if the test event has been processed */ 788 if (evq->init_state == SFXGE_EVQ_STARTED) 789 goto done; 790 791 } while (++count < 20); 792 793 rc = ETIMEDOUT; 794 goto fail3; 795 796 done: 797 return (0); 798 799 fail3: 800 SFXGE_EVQ_LOCK(evq); 801 evq->init_state = SFXGE_EVQ_INITIALIZED; 802 fail2: 803 SFXGE_EVQ_UNLOCK(evq); 804 efx_ev_qdestroy(evq->common); 805 fail: 806 efx_sram_buf_tbl_clear(sc->enp, evq->buf_base_id, 807 EFX_EVQ_NBUFS(evq->entries)); 808 809 return (rc); 810 } 811 812 void 813 sfxge_ev_stop(struct sfxge_softc *sc) 814 { 815 struct sfxge_intr *intr __diagused; 816 efx_nic_t *enp; 817 int index; 818 819 intr = &sc->intr; 820 enp = sc->enp; 821 822 KASSERT(intr->state == SFXGE_INTR_STARTED, 823 ("Interrupts not started")); 824 825 /* Stop the event queue(s) */ 826 index = sc->evq_count; 827 while (--index >= 0) 828 sfxge_ev_qstop(sc, index); 829 830 /* Tear down the event module */ 831 efx_ev_fini(enp); 832 } 833 834 int 835 sfxge_ev_start(struct sfxge_softc *sc) 836 { 837 struct sfxge_intr *intr __diagused; 838 int index; 839 int rc; 840 841 intr = &sc->intr; 842 843 KASSERT(intr->state == SFXGE_INTR_STARTED, 844 ("intr->state != SFXGE_INTR_STARTED")); 845 846 /* Initialize the event module */ 847 if ((rc = efx_ev_init(sc->enp)) != 0) 848 return (rc); 849 850 /* Start the event queues */ 851 for (index = 0; index < sc->evq_count; index++) { 852 if ((rc = sfxge_ev_qstart(sc, index)) != 0) 853 goto fail; 854 } 855 856 return (0); 857 858 fail: 859 /* Stop the event queue(s) */ 860 while (--index >= 0) 861 sfxge_ev_qstop(sc, index); 862 863 /* Tear down the event module */ 864 efx_ev_fini(sc->enp); 865 866 return (rc); 867 } 868 869 static void 870 sfxge_ev_qfini(struct sfxge_softc *sc, unsigned int index) 871 { 872 struct sfxge_evq *evq; 873 874 evq = sc->evq[index]; 875 876 KASSERT(evq->init_state == SFXGE_EVQ_INITIALIZED, 877 ("evq->init_state != SFXGE_EVQ_INITIALIZED")); 878 KASSERT(evq->txqs == &evq->txq, ("evq->txqs != &evq->txq")); 879 880 sfxge_dma_free(&evq->mem); 881 882 sc->evq[index] = NULL; 883 884 SFXGE_EVQ_LOCK_DESTROY(evq); 885 886 free(evq, M_SFXGE); 887 } 888 889 static int 890 sfxge_ev_qinit(struct sfxge_softc *sc, unsigned int index) 891 { 892 struct sfxge_evq *evq; 893 efsys_mem_t *esmp; 894 int rc; 895 896 KASSERT(index < SFXGE_RX_SCALE_MAX, ("index >= SFXGE_RX_SCALE_MAX")); 897 898 evq = malloc(sizeof(struct sfxge_evq), M_SFXGE, M_ZERO | M_WAITOK); 899 evq->sc = sc; 900 evq->index = index; 901 sc->evq[index] = evq; 902 esmp = &evq->mem; 903 904 /* Build an event queue with room for one event per tx and rx buffer, 905 * plus some extra for link state events and MCDI completions. 906 * There are three tx queues in the first event queue and one in 907 * other. 908 */ 909 if (index == 0) 910 evq->entries = 911 ROUNDUP_POW_OF_TWO(sc->rxq_entries + 912 3 * sc->txq_entries + 913 128); 914 else 915 evq->entries = 916 ROUNDUP_POW_OF_TWO(sc->rxq_entries + 917 sc->txq_entries + 918 128); 919 920 /* Initialise TX completion list */ 921 evq->txqs = &evq->txq; 922 923 /* Allocate DMA space. */ 924 if ((rc = sfxge_dma_alloc(sc, EFX_EVQ_SIZE(evq->entries), esmp)) != 0) 925 return (rc); 926 927 /* Allocate buffer table entries. */ 928 sfxge_sram_buf_tbl_alloc(sc, EFX_EVQ_NBUFS(evq->entries), 929 &evq->buf_base_id); 930 931 SFXGE_EVQ_LOCK_INIT(evq, device_get_nameunit(sc->dev), index); 932 933 evq->init_state = SFXGE_EVQ_INITIALIZED; 934 935 #if EFSYS_OPT_QSTATS 936 rc = sfxge_evq_stat_init(evq); 937 if (rc != 0) 938 goto fail_evq_stat_init; 939 #endif 940 941 return (0); 942 943 #if EFSYS_OPT_QSTATS 944 fail_evq_stat_init: 945 evq->init_state = SFXGE_EVQ_UNINITIALIZED; 946 SFXGE_EVQ_LOCK_DESTROY(evq); 947 sfxge_dma_free(esmp); 948 sc->evq[index] = NULL; 949 free(evq, M_SFXGE); 950 951 return (rc); 952 #endif 953 } 954 955 void 956 sfxge_ev_fini(struct sfxge_softc *sc) 957 { 958 struct sfxge_intr *intr __diagused; 959 int index; 960 961 intr = &sc->intr; 962 963 KASSERT(intr->state == SFXGE_INTR_INITIALIZED, 964 ("intr->state != SFXGE_INTR_INITIALIZED")); 965 966 sc->ev_moderation = 0; 967 968 /* Tear down the event queue(s). */ 969 index = sc->evq_count; 970 while (--index >= 0) 971 sfxge_ev_qfini(sc, index); 972 973 sc->evq_count = 0; 974 } 975 976 int 977 sfxge_ev_init(struct sfxge_softc *sc) 978 { 979 struct sysctl_ctx_list *sysctl_ctx = device_get_sysctl_ctx(sc->dev); 980 struct sysctl_oid *sysctl_tree = device_get_sysctl_tree(sc->dev); 981 struct sfxge_intr *intr; 982 int index; 983 int rc; 984 985 intr = &sc->intr; 986 987 sc->evq_count = intr->n_alloc; 988 989 KASSERT(intr->state == SFXGE_INTR_INITIALIZED, 990 ("intr->state != SFXGE_INTR_INITIALIZED")); 991 992 /* Set default interrupt moderation; add a sysctl to 993 * read and change it. 994 */ 995 sc->ev_moderation = SFXGE_MODERATION; 996 SYSCTL_ADD_PROC(sysctl_ctx, SYSCTL_CHILDREN(sysctl_tree), 997 OID_AUTO, "int_mod", CTLTYPE_UINT | CTLFLAG_RW | CTLFLAG_MPSAFE, 998 sc, 0, sfxge_int_mod_handler, "IU", 999 "sfxge interrupt moderation (us)"); 1000 1001 #if EFSYS_OPT_QSTATS 1002 sc->evqs_stats_node = SYSCTL_ADD_NODE( 1003 device_get_sysctl_ctx(sc->dev), SYSCTL_CHILDREN(sc->stats_node), 1004 OID_AUTO, "evq", CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, 1005 "Event queues stats"); 1006 if (sc->evqs_stats_node == NULL) { 1007 rc = ENOMEM; 1008 goto fail_evqs_stats_node; 1009 } 1010 #endif 1011 1012 /* 1013 * Initialize the event queue(s) - one per interrupt. 1014 */ 1015 for (index = 0; index < sc->evq_count; index++) { 1016 if ((rc = sfxge_ev_qinit(sc, index)) != 0) 1017 goto fail; 1018 } 1019 1020 #if EFSYS_OPT_QSTATS 1021 sfxge_ev_stat_init(sc); 1022 #endif 1023 1024 return (0); 1025 1026 fail: 1027 while (--index >= 0) 1028 sfxge_ev_qfini(sc, index); 1029 1030 #if EFSYS_OPT_QSTATS 1031 fail_evqs_stats_node: 1032 #endif 1033 sc->evq_count = 0; 1034 return (rc); 1035 } 1036