1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause 3 * 4 * Copyright (c) 2010-2016 Solarflare Communications Inc. 5 * All rights reserved. 6 * 7 * This software was developed in part by Philip Paeps under contract for 8 * Solarflare Communications, Inc. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions are met: 12 * 13 * 1. Redistributions of source code must retain the above copyright notice, 14 * this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright notice, 16 * this list of conditions and the following disclaimer in the documentation 17 * and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 20 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, 21 * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR 23 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, 24 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 25 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; 26 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 27 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR 28 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, 29 * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 * 31 * The views and conclusions contained in the software and documentation are 32 * those of the authors and should not be interpreted as representing official 33 * policies, either expressed or implied, of the FreeBSD Project. 34 */ 35 36 #include <sys/param.h> 37 #include <sys/kernel.h> 38 #include <sys/malloc.h> 39 #include <sys/param.h> 40 #include <sys/queue.h> 41 #include <sys/systm.h> 42 #include <sys/taskqueue.h> 43 44 #include "common/efx.h" 45 46 #include "sfxge.h" 47 48 static void 49 sfxge_ev_qcomplete(struct sfxge_evq *evq, boolean_t eop) 50 { 51 struct sfxge_softc *sc; 52 unsigned int index; 53 struct sfxge_rxq *rxq; 54 struct sfxge_txq *txq; 55 56 SFXGE_EVQ_LOCK_ASSERT_OWNED(evq); 57 58 sc = evq->sc; 59 index = evq->index; 60 rxq = sc->rxq[index]; 61 62 if ((txq = evq->txq) != NULL) { 63 evq->txq = NULL; 64 evq->txqs = &(evq->txq); 65 66 do { 67 struct sfxge_txq *next; 68 69 next = txq->next; 70 txq->next = NULL; 71 72 KASSERT(txq->evq_index == index, 73 ("txq->evq_index != index")); 74 75 if (txq->pending != txq->completed) 76 sfxge_tx_qcomplete(txq, evq); 77 78 txq = next; 79 } while (txq != NULL); 80 } 81 82 if (rxq->pending != rxq->completed) 83 sfxge_rx_qcomplete(rxq, eop); 84 } 85 86 static struct sfxge_rxq * 87 sfxge_get_rxq_by_label(struct sfxge_evq *evq, uint32_t label) 88 { 89 struct sfxge_rxq *rxq; 90 91 KASSERT(label == 0, ("unexpected rxq label != 0")); 92 93 rxq = evq->sc->rxq[evq->index]; 94 95 KASSERT(rxq != NULL, ("rxq == NULL")); 96 KASSERT(evq->index == rxq->index, ("evq->index != rxq->index")); 97 98 return (rxq); 99 } 100 101 static boolean_t 102 sfxge_ev_rx(void *arg, uint32_t label, uint32_t id, uint32_t size, 103 uint16_t flags) 104 { 105 struct sfxge_evq *evq; 106 struct sfxge_softc *sc; 107 struct sfxge_rxq *rxq; 108 unsigned int stop; 109 unsigned int delta; 110 struct sfxge_rx_sw_desc *rx_desc; 111 112 evq = arg; 113 SFXGE_EVQ_LOCK_ASSERT_OWNED(evq); 114 115 sc = evq->sc; 116 117 if (evq->exception) 118 goto done; 119 120 rxq = sfxge_get_rxq_by_label(evq, label); 121 if (__predict_false(rxq->init_state != SFXGE_RXQ_STARTED)) 122 goto done; 123 124 stop = (id + 1) & rxq->ptr_mask; 125 id = rxq->pending & rxq->ptr_mask; 126 delta = (stop >= id) ? (stop - id) : (rxq->entries - id + stop); 127 rxq->pending += delta; 128 129 if (delta != 1) { 130 if ((delta <= 0) || 131 (delta > efx_nic_cfg_get(sc->enp)->enc_rx_batch_max)) { 132 evq->exception = B_TRUE; 133 134 device_printf(sc->dev, "RX completion out of order" 135 " (id=%#x delta=%u flags=%#x); resetting\n", 136 id, delta, flags); 137 sfxge_schedule_reset(sc); 138 139 goto done; 140 } 141 } 142 143 rx_desc = &rxq->queue[id]; 144 145 prefetch_read_many(rx_desc->mbuf); 146 147 for (; id != stop; id = (id + 1) & rxq->ptr_mask) { 148 rx_desc = &rxq->queue[id]; 149 KASSERT(rx_desc->flags == EFX_DISCARD, 150 ("rx_desc->flags != EFX_DISCARD")); 151 rx_desc->flags = flags; 152 153 KASSERT(size < (1 << 16), ("size > (1 << 16)")); 154 rx_desc->size = (uint16_t)size; 155 } 156 157 evq->rx_done++; 158 159 if (rxq->pending - rxq->completed >= SFXGE_RX_BATCH) 160 sfxge_ev_qcomplete(evq, B_FALSE); 161 162 done: 163 return (evq->rx_done >= SFXGE_EV_BATCH); 164 } 165 166 static boolean_t 167 sfxge_ev_exception(void *arg, uint32_t code, uint32_t data) 168 { 169 struct sfxge_evq *evq; 170 struct sfxge_softc *sc; 171 172 evq = (struct sfxge_evq *)arg; 173 SFXGE_EVQ_LOCK_ASSERT_OWNED(evq); 174 175 sc = evq->sc; 176 177 DBGPRINT(sc->dev, "[%d] %s", evq->index, 178 (code == EFX_EXCEPTION_RX_RECOVERY) ? "RX_RECOVERY" : 179 (code == EFX_EXCEPTION_RX_DSC_ERROR) ? "RX_DSC_ERROR" : 180 (code == EFX_EXCEPTION_TX_DSC_ERROR) ? "TX_DSC_ERROR" : 181 (code == EFX_EXCEPTION_UNKNOWN_SENSOREVT) ? "UNKNOWN_SENSOREVT" : 182 (code == EFX_EXCEPTION_FWALERT_SRAM) ? "FWALERT_SRAM" : 183 (code == EFX_EXCEPTION_UNKNOWN_FWALERT) ? "UNKNOWN_FWALERT" : 184 (code == EFX_EXCEPTION_RX_ERROR) ? "RX_ERROR" : 185 (code == EFX_EXCEPTION_TX_ERROR) ? "TX_ERROR" : 186 (code == EFX_EXCEPTION_EV_ERROR) ? "EV_ERROR" : 187 "UNKNOWN"); 188 189 evq->exception = B_TRUE; 190 191 if (code != EFX_EXCEPTION_UNKNOWN_SENSOREVT) { 192 device_printf(sc->dev, 193 "hardware exception (code=%u); resetting\n", 194 code); 195 sfxge_schedule_reset(sc); 196 } 197 198 return (B_FALSE); 199 } 200 201 static boolean_t 202 sfxge_ev_rxq_flush_done(void *arg, uint32_t rxq_index) 203 { 204 struct sfxge_evq *evq; 205 struct sfxge_softc *sc; 206 struct sfxge_rxq *rxq; 207 unsigned int index; 208 uint16_t magic; 209 210 evq = (struct sfxge_evq *)arg; 211 SFXGE_EVQ_LOCK_ASSERT_OWNED(evq); 212 213 sc = evq->sc; 214 rxq = sc->rxq[rxq_index]; 215 216 KASSERT(rxq != NULL, ("rxq == NULL")); 217 218 /* Resend a software event on the correct queue */ 219 index = rxq->index; 220 if (index == evq->index) { 221 sfxge_rx_qflush_done(rxq); 222 return (B_FALSE); 223 } 224 225 evq = sc->evq[index]; 226 magic = sfxge_sw_ev_rxq_magic(SFXGE_SW_EV_RX_QFLUSH_DONE, rxq); 227 228 KASSERT(evq->init_state == SFXGE_EVQ_STARTED, 229 ("evq not started")); 230 efx_ev_qpost(evq->common, magic); 231 232 return (B_FALSE); 233 } 234 235 static boolean_t 236 sfxge_ev_rxq_flush_failed(void *arg, uint32_t rxq_index) 237 { 238 struct sfxge_evq *evq; 239 struct sfxge_softc *sc; 240 struct sfxge_rxq *rxq; 241 unsigned int index; 242 uint16_t magic; 243 244 evq = (struct sfxge_evq *)arg; 245 SFXGE_EVQ_LOCK_ASSERT_OWNED(evq); 246 247 sc = evq->sc; 248 rxq = sc->rxq[rxq_index]; 249 250 KASSERT(rxq != NULL, ("rxq == NULL")); 251 252 /* Resend a software event on the correct queue */ 253 index = rxq->index; 254 evq = sc->evq[index]; 255 magic = sfxge_sw_ev_rxq_magic(SFXGE_SW_EV_RX_QFLUSH_FAILED, rxq); 256 257 KASSERT(evq->init_state == SFXGE_EVQ_STARTED, 258 ("evq not started")); 259 efx_ev_qpost(evq->common, magic); 260 261 return (B_FALSE); 262 } 263 264 static struct sfxge_txq * 265 sfxge_get_txq_by_label(struct sfxge_evq *evq, enum sfxge_txq_type label) 266 { 267 unsigned int index; 268 269 KASSERT((evq->sc->txq_dynamic_cksum_toggle_supported) ? (label == 0) : 270 ((evq->index == 0 && label < SFXGE_TXQ_NTYPES) || 271 (label == SFXGE_TXQ_IP_TCP_UDP_CKSUM)), 272 ("unexpected txq label")); 273 274 index = (evq->index == 0) ? 275 label : (evq->index - 1 + SFXGE_EVQ0_N_TXQ(evq->sc)); 276 return (evq->sc->txq[index]); 277 } 278 279 static boolean_t 280 sfxge_ev_tx(void *arg, uint32_t label, uint32_t id) 281 { 282 struct sfxge_evq *evq; 283 struct sfxge_txq *txq; 284 unsigned int stop; 285 unsigned int delta; 286 287 evq = (struct sfxge_evq *)arg; 288 SFXGE_EVQ_LOCK_ASSERT_OWNED(evq); 289 290 txq = sfxge_get_txq_by_label(evq, label); 291 292 KASSERT(txq != NULL, ("txq == NULL")); 293 KASSERT(evq->index == txq->evq_index, 294 ("evq->index != txq->evq_index")); 295 296 if (__predict_false(txq->init_state != SFXGE_TXQ_STARTED)) 297 goto done; 298 299 stop = (id + 1) & txq->ptr_mask; 300 id = txq->pending & txq->ptr_mask; 301 302 delta = (stop >= id) ? (stop - id) : (txq->entries - id + stop); 303 txq->pending += delta; 304 305 evq->tx_done++; 306 307 if (txq->next == NULL && 308 evq->txqs != &(txq->next)) { 309 *(evq->txqs) = txq; 310 evq->txqs = &(txq->next); 311 } 312 313 if (txq->pending - txq->completed >= SFXGE_TX_BATCH) 314 sfxge_tx_qcomplete(txq, evq); 315 316 done: 317 return (evq->tx_done >= SFXGE_EV_BATCH); 318 } 319 320 static boolean_t 321 sfxge_ev_txq_flush_done(void *arg, uint32_t txq_index) 322 { 323 struct sfxge_evq *evq; 324 struct sfxge_softc *sc; 325 struct sfxge_txq *txq; 326 uint16_t magic; 327 328 evq = (struct sfxge_evq *)arg; 329 SFXGE_EVQ_LOCK_ASSERT_OWNED(evq); 330 331 sc = evq->sc; 332 txq = sc->txq[txq_index]; 333 334 KASSERT(txq != NULL, ("txq == NULL")); 335 KASSERT(txq->init_state == SFXGE_TXQ_INITIALIZED, 336 ("txq not initialized")); 337 338 if (txq->evq_index == evq->index) { 339 sfxge_tx_qflush_done(txq); 340 return (B_FALSE); 341 } 342 343 /* Resend a software event on the correct queue */ 344 evq = sc->evq[txq->evq_index]; 345 magic = sfxge_sw_ev_txq_magic(SFXGE_SW_EV_TX_QFLUSH_DONE, txq); 346 347 KASSERT(evq->init_state == SFXGE_EVQ_STARTED, 348 ("evq not started")); 349 efx_ev_qpost(evq->common, magic); 350 351 return (B_FALSE); 352 } 353 354 static boolean_t 355 sfxge_ev_software(void *arg, uint16_t magic) 356 { 357 struct sfxge_evq *evq; 358 unsigned int label; 359 360 evq = (struct sfxge_evq *)arg; 361 SFXGE_EVQ_LOCK_ASSERT_OWNED(evq); 362 363 label = magic & SFXGE_MAGIC_DMAQ_LABEL_MASK; 364 magic &= ~SFXGE_MAGIC_DMAQ_LABEL_MASK; 365 366 switch (magic) { 367 case SFXGE_SW_EV_MAGIC(SFXGE_SW_EV_RX_QFLUSH_DONE): 368 sfxge_rx_qflush_done(sfxge_get_rxq_by_label(evq, label)); 369 break; 370 371 case SFXGE_SW_EV_MAGIC(SFXGE_SW_EV_RX_QFLUSH_FAILED): 372 sfxge_rx_qflush_failed(sfxge_get_rxq_by_label(evq, label)); 373 break; 374 375 case SFXGE_SW_EV_MAGIC(SFXGE_SW_EV_RX_QREFILL): 376 sfxge_rx_qrefill(sfxge_get_rxq_by_label(evq, label)); 377 break; 378 379 case SFXGE_SW_EV_MAGIC(SFXGE_SW_EV_TX_QFLUSH_DONE): { 380 struct sfxge_txq *txq = sfxge_get_txq_by_label(evq, label); 381 382 KASSERT(txq != NULL, ("txq == NULL")); 383 KASSERT(evq->index == txq->evq_index, 384 ("evq->index != txq->evq_index")); 385 386 sfxge_tx_qflush_done(txq); 387 break; 388 } 389 default: 390 break; 391 } 392 393 return (B_FALSE); 394 } 395 396 static boolean_t 397 sfxge_ev_sram(void *arg, uint32_t code) 398 { 399 (void)arg; 400 (void)code; 401 402 switch (code) { 403 case EFX_SRAM_UPDATE: 404 EFSYS_PROBE(sram_update); 405 break; 406 407 case EFX_SRAM_CLEAR: 408 EFSYS_PROBE(sram_clear); 409 break; 410 411 case EFX_SRAM_ILLEGAL_CLEAR: 412 EFSYS_PROBE(sram_illegal_clear); 413 break; 414 415 default: 416 KASSERT(B_FALSE, ("Impossible SRAM event")); 417 break; 418 } 419 420 return (B_FALSE); 421 } 422 423 static boolean_t 424 sfxge_ev_timer(void *arg, uint32_t index) 425 { 426 (void)arg; 427 (void)index; 428 429 return (B_FALSE); 430 } 431 432 static boolean_t 433 sfxge_ev_wake_up(void *arg, uint32_t index) 434 { 435 (void)arg; 436 (void)index; 437 438 return (B_FALSE); 439 } 440 441 #if EFSYS_OPT_QSTATS 442 443 static void 444 sfxge_evq_stat_update(struct sfxge_evq *evq) 445 { 446 clock_t now; 447 448 SFXGE_EVQ_LOCK(evq); 449 450 if (__predict_false(evq->init_state != SFXGE_EVQ_STARTED)) 451 goto out; 452 453 now = ticks; 454 if ((unsigned int)(now - evq->stats_update_time) < (unsigned int)hz) 455 goto out; 456 457 evq->stats_update_time = now; 458 efx_ev_qstats_update(evq->common, evq->stats); 459 460 out: 461 SFXGE_EVQ_UNLOCK(evq); 462 } 463 464 static int 465 sfxge_evq_stat_handler(SYSCTL_HANDLER_ARGS) 466 { 467 struct sfxge_evq *evq = arg1; 468 struct sfxge_softc *sc = evq->sc; 469 unsigned int id = arg2; 470 471 SFXGE_ADAPTER_LOCK(sc); 472 473 sfxge_evq_stat_update(evq); 474 475 SFXGE_ADAPTER_UNLOCK(sc); 476 477 return (SYSCTL_OUT(req, &evq->stats[id], sizeof(evq->stats[id]))); 478 } 479 480 static int 481 sfxge_evq_stat_init(struct sfxge_evq *evq) 482 { 483 struct sfxge_softc *sc = evq->sc; 484 struct sysctl_ctx_list *ctx = device_get_sysctl_ctx(sc->dev); 485 char name[16]; 486 struct sysctl_oid *evq_stats_node; 487 unsigned int id; 488 489 snprintf(name, sizeof(name), "%u", evq->index); 490 evq_stats_node = SYSCTL_ADD_NODE(ctx, 491 SYSCTL_CHILDREN(sc->evqs_stats_node), OID_AUTO, name, 492 CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, ""); 493 if (evq_stats_node == NULL) 494 return (ENOMEM); 495 496 for (id = 0; id < EV_NQSTATS; id++) { 497 SYSCTL_ADD_PROC(ctx, SYSCTL_CHILDREN(evq_stats_node), 498 OID_AUTO, efx_ev_qstat_name(sc->enp, id), 499 CTLTYPE_U64 | CTLFLAG_RD | CTLFLAG_MPSAFE, evq, id, 500 sfxge_evq_stat_handler, "Q", ""); 501 } 502 503 return (0); 504 } 505 506 static void 507 sfxge_ev_stat_update(struct sfxge_softc *sc) 508 { 509 struct sfxge_evq *evq; 510 unsigned int index; 511 clock_t now; 512 unsigned int id; 513 514 SFXGE_ADAPTER_LOCK(sc); 515 516 now = ticks; 517 if ((unsigned int)(now - sc->ev_stats_update_time) < (unsigned int)hz) 518 goto out; 519 520 sc->ev_stats_update_time = now; 521 522 memset(sc->ev_stats, 0, sizeof(sc->ev_stats)); 523 524 /* Update and add event counts from each event queue in turn */ 525 for (index = 0; index < sc->evq_count; index++) { 526 evq = sc->evq[index]; 527 sfxge_evq_stat_update(evq); 528 for (id = 0; id < EV_NQSTATS; id++) 529 sc->ev_stats[id] += evq->stats[id]; 530 } 531 out: 532 SFXGE_ADAPTER_UNLOCK(sc); 533 } 534 535 static int 536 sfxge_ev_stat_handler(SYSCTL_HANDLER_ARGS) 537 { 538 struct sfxge_softc *sc = arg1; 539 unsigned int id = arg2; 540 541 sfxge_ev_stat_update(sc); 542 543 return (SYSCTL_OUT(req, &sc->ev_stats[id], sizeof(sc->ev_stats[id]))); 544 } 545 546 static void 547 sfxge_ev_stat_init(struct sfxge_softc *sc) 548 { 549 struct sysctl_ctx_list *ctx = device_get_sysctl_ctx(sc->dev); 550 struct sysctl_oid_list *stat_list; 551 unsigned int id; 552 char name[40]; 553 554 stat_list = SYSCTL_CHILDREN(sc->stats_node); 555 556 for (id = 0; id < EV_NQSTATS; id++) { 557 snprintf(name, sizeof(name), "ev_%s", 558 efx_ev_qstat_name(sc->enp, id)); 559 SYSCTL_ADD_PROC(ctx, stat_list, OID_AUTO, name, 560 CTLTYPE_U64 | CTLFLAG_RD | CTLFLAG_MPSAFE, 561 sc, id, sfxge_ev_stat_handler, "Q", ""); 562 } 563 } 564 565 #endif /* EFSYS_OPT_QSTATS */ 566 567 static void 568 sfxge_ev_qmoderate(struct sfxge_softc *sc, unsigned int idx, unsigned int us) 569 { 570 struct sfxge_evq *evq; 571 efx_evq_t *eep; 572 573 evq = sc->evq[idx]; 574 eep = evq->common; 575 576 KASSERT(evq->init_state == SFXGE_EVQ_STARTED, 577 ("evq->init_state != SFXGE_EVQ_STARTED")); 578 579 (void)efx_ev_qmoderate(eep, us); 580 } 581 582 static int 583 sfxge_int_mod_handler(SYSCTL_HANDLER_ARGS) 584 { 585 struct sfxge_softc *sc = arg1; 586 struct sfxge_intr *intr = &sc->intr; 587 unsigned int moderation; 588 int error; 589 unsigned int index; 590 591 SFXGE_ADAPTER_LOCK(sc); 592 593 if (req->newptr != NULL) { 594 if ((error = SYSCTL_IN(req, &moderation, sizeof(moderation))) 595 != 0) 596 goto out; 597 598 /* We may not be calling efx_ev_qmoderate() now, 599 * so we have to range-check the value ourselves. 600 */ 601 if (moderation > 602 efx_nic_cfg_get(sc->enp)->enc_evq_timer_max_us) { 603 error = EINVAL; 604 goto out; 605 } 606 607 sc->ev_moderation = moderation; 608 if (intr->state == SFXGE_INTR_STARTED) { 609 for (index = 0; index < sc->evq_count; index++) 610 sfxge_ev_qmoderate(sc, index, moderation); 611 } 612 } else { 613 error = SYSCTL_OUT(req, &sc->ev_moderation, 614 sizeof(sc->ev_moderation)); 615 } 616 617 out: 618 SFXGE_ADAPTER_UNLOCK(sc); 619 620 return (error); 621 } 622 623 static boolean_t 624 sfxge_ev_initialized(void *arg) 625 { 626 struct sfxge_evq *evq; 627 628 evq = (struct sfxge_evq *)arg; 629 SFXGE_EVQ_LOCK_ASSERT_OWNED(evq); 630 631 /* Init done events may be duplicated on 7xxx */ 632 KASSERT(evq->init_state == SFXGE_EVQ_STARTING || 633 evq->init_state == SFXGE_EVQ_STARTED, 634 ("evq not starting")); 635 636 evq->init_state = SFXGE_EVQ_STARTED; 637 638 return (0); 639 } 640 641 static boolean_t 642 sfxge_ev_link_change(void *arg, efx_link_mode_t link_mode) 643 { 644 struct sfxge_evq *evq; 645 struct sfxge_softc *sc; 646 647 evq = (struct sfxge_evq *)arg; 648 SFXGE_EVQ_LOCK_ASSERT_OWNED(evq); 649 650 sc = evq->sc; 651 652 sfxge_mac_link_update(sc, link_mode); 653 654 return (0); 655 } 656 657 static const efx_ev_callbacks_t sfxge_ev_callbacks = { 658 .eec_initialized = sfxge_ev_initialized, 659 .eec_rx = sfxge_ev_rx, 660 .eec_tx = sfxge_ev_tx, 661 .eec_exception = sfxge_ev_exception, 662 .eec_rxq_flush_done = sfxge_ev_rxq_flush_done, 663 .eec_rxq_flush_failed = sfxge_ev_rxq_flush_failed, 664 .eec_txq_flush_done = sfxge_ev_txq_flush_done, 665 .eec_software = sfxge_ev_software, 666 .eec_sram = sfxge_ev_sram, 667 .eec_wake_up = sfxge_ev_wake_up, 668 .eec_timer = sfxge_ev_timer, 669 .eec_link_change = sfxge_ev_link_change, 670 }; 671 672 int 673 sfxge_ev_qpoll(struct sfxge_evq *evq) 674 { 675 int rc; 676 677 SFXGE_EVQ_LOCK(evq); 678 679 if (__predict_false(evq->init_state != SFXGE_EVQ_STARTING && 680 evq->init_state != SFXGE_EVQ_STARTED)) { 681 rc = EINVAL; 682 goto fail; 683 } 684 685 /* Synchronize the DMA memory for reading */ 686 bus_dmamap_sync(evq->mem.esm_tag, evq->mem.esm_map, 687 BUS_DMASYNC_POSTREAD); 688 689 KASSERT(evq->rx_done == 0, ("evq->rx_done != 0")); 690 KASSERT(evq->tx_done == 0, ("evq->tx_done != 0")); 691 KASSERT(evq->txq == NULL, ("evq->txq != NULL")); 692 KASSERT(evq->txqs == &evq->txq, ("evq->txqs != &evq->txq")); 693 694 /* Poll the queue */ 695 efx_ev_qpoll(evq->common, &evq->read_ptr, &sfxge_ev_callbacks, evq); 696 697 evq->rx_done = 0; 698 evq->tx_done = 0; 699 700 /* Perform any pending completion processing */ 701 sfxge_ev_qcomplete(evq, B_TRUE); 702 703 /* Re-prime the event queue for interrupts */ 704 if ((rc = efx_ev_qprime(evq->common, evq->read_ptr)) != 0) 705 goto fail; 706 707 SFXGE_EVQ_UNLOCK(evq); 708 709 return (0); 710 711 fail: 712 SFXGE_EVQ_UNLOCK(evq); 713 return (rc); 714 } 715 716 static void 717 sfxge_ev_qstop(struct sfxge_softc *sc, unsigned int index) 718 { 719 struct sfxge_evq *evq; 720 721 evq = sc->evq[index]; 722 723 KASSERT(evq->init_state == SFXGE_EVQ_STARTED, 724 ("evq->init_state != SFXGE_EVQ_STARTED")); 725 726 SFXGE_EVQ_LOCK(evq); 727 evq->init_state = SFXGE_EVQ_INITIALIZED; 728 evq->read_ptr = 0; 729 evq->exception = B_FALSE; 730 731 #if EFSYS_OPT_QSTATS 732 /* Add event counts before discarding the common evq state */ 733 efx_ev_qstats_update(evq->common, evq->stats); 734 #endif 735 736 efx_ev_qdestroy(evq->common); 737 efx_sram_buf_tbl_clear(sc->enp, evq->buf_base_id, 738 EFX_EVQ_NBUFS(evq->entries)); 739 SFXGE_EVQ_UNLOCK(evq); 740 } 741 742 static int 743 sfxge_ev_qstart(struct sfxge_softc *sc, unsigned int index) 744 { 745 struct sfxge_evq *evq; 746 efsys_mem_t *esmp; 747 int count; 748 int rc; 749 750 evq = sc->evq[index]; 751 esmp = &evq->mem; 752 753 KASSERT(evq->init_state == SFXGE_EVQ_INITIALIZED, 754 ("evq->init_state != SFXGE_EVQ_INITIALIZED")); 755 756 /* Clear all events. */ 757 (void)memset(esmp->esm_base, 0xff, EFX_EVQ_SIZE(evq->entries)); 758 759 /* Program the buffer table. */ 760 if ((rc = efx_sram_buf_tbl_set(sc->enp, evq->buf_base_id, esmp, 761 EFX_EVQ_NBUFS(evq->entries))) != 0) 762 return (rc); 763 764 /* Create the common code event queue. */ 765 if ((rc = efx_ev_qcreate(sc->enp, index, esmp, evq->entries, 766 evq->buf_base_id, sc->ev_moderation, EFX_EVQ_FLAGS_TYPE_AUTO, 767 &evq->common)) != 0) 768 goto fail; 769 770 SFXGE_EVQ_LOCK(evq); 771 772 /* Prime the event queue for interrupts */ 773 if ((rc = efx_ev_qprime(evq->common, evq->read_ptr)) != 0) 774 goto fail2; 775 776 evq->init_state = SFXGE_EVQ_STARTING; 777 778 SFXGE_EVQ_UNLOCK(evq); 779 780 /* Wait for the initialization event */ 781 count = 0; 782 do { 783 /* Pause for 100 ms */ 784 pause("sfxge evq init", hz / 10); 785 786 /* Check to see if the test event has been processed */ 787 if (evq->init_state == SFXGE_EVQ_STARTED) 788 goto done; 789 790 } while (++count < 20); 791 792 rc = ETIMEDOUT; 793 goto fail3; 794 795 done: 796 return (0); 797 798 fail3: 799 SFXGE_EVQ_LOCK(evq); 800 evq->init_state = SFXGE_EVQ_INITIALIZED; 801 fail2: 802 SFXGE_EVQ_UNLOCK(evq); 803 efx_ev_qdestroy(evq->common); 804 fail: 805 efx_sram_buf_tbl_clear(sc->enp, evq->buf_base_id, 806 EFX_EVQ_NBUFS(evq->entries)); 807 808 return (rc); 809 } 810 811 void 812 sfxge_ev_stop(struct sfxge_softc *sc) 813 { 814 struct sfxge_intr *intr __diagused; 815 efx_nic_t *enp; 816 int index; 817 818 intr = &sc->intr; 819 enp = sc->enp; 820 821 KASSERT(intr->state == SFXGE_INTR_STARTED, 822 ("Interrupts not started")); 823 824 /* Stop the event queue(s) */ 825 index = sc->evq_count; 826 while (--index >= 0) 827 sfxge_ev_qstop(sc, index); 828 829 /* Tear down the event module */ 830 efx_ev_fini(enp); 831 } 832 833 int 834 sfxge_ev_start(struct sfxge_softc *sc) 835 { 836 struct sfxge_intr *intr __diagused; 837 int index; 838 int rc; 839 840 intr = &sc->intr; 841 842 KASSERT(intr->state == SFXGE_INTR_STARTED, 843 ("intr->state != SFXGE_INTR_STARTED")); 844 845 /* Initialize the event module */ 846 if ((rc = efx_ev_init(sc->enp)) != 0) 847 return (rc); 848 849 /* Start the event queues */ 850 for (index = 0; index < sc->evq_count; index++) { 851 if ((rc = sfxge_ev_qstart(sc, index)) != 0) 852 goto fail; 853 } 854 855 return (0); 856 857 fail: 858 /* Stop the event queue(s) */ 859 while (--index >= 0) 860 sfxge_ev_qstop(sc, index); 861 862 /* Tear down the event module */ 863 efx_ev_fini(sc->enp); 864 865 return (rc); 866 } 867 868 static void 869 sfxge_ev_qfini(struct sfxge_softc *sc, unsigned int index) 870 { 871 struct sfxge_evq *evq; 872 873 evq = sc->evq[index]; 874 875 KASSERT(evq->init_state == SFXGE_EVQ_INITIALIZED, 876 ("evq->init_state != SFXGE_EVQ_INITIALIZED")); 877 KASSERT(evq->txqs == &evq->txq, ("evq->txqs != &evq->txq")); 878 879 sfxge_dma_free(&evq->mem); 880 881 sc->evq[index] = NULL; 882 883 SFXGE_EVQ_LOCK_DESTROY(evq); 884 885 free(evq, M_SFXGE); 886 } 887 888 static int 889 sfxge_ev_qinit(struct sfxge_softc *sc, unsigned int index) 890 { 891 struct sfxge_evq *evq; 892 efsys_mem_t *esmp; 893 int rc; 894 895 KASSERT(index < SFXGE_RX_SCALE_MAX, ("index >= SFXGE_RX_SCALE_MAX")); 896 897 evq = malloc(sizeof(struct sfxge_evq), M_SFXGE, M_ZERO | M_WAITOK); 898 evq->sc = sc; 899 evq->index = index; 900 sc->evq[index] = evq; 901 esmp = &evq->mem; 902 903 /* Build an event queue with room for one event per tx and rx buffer, 904 * plus some extra for link state events and MCDI completions. 905 * There are three tx queues in the first event queue and one in 906 * other. 907 */ 908 if (index == 0) 909 evq->entries = 910 ROUNDUP_POW_OF_TWO(sc->rxq_entries + 911 3 * sc->txq_entries + 912 128); 913 else 914 evq->entries = 915 ROUNDUP_POW_OF_TWO(sc->rxq_entries + 916 sc->txq_entries + 917 128); 918 919 /* Initialise TX completion list */ 920 evq->txqs = &evq->txq; 921 922 /* Allocate DMA space. */ 923 if ((rc = sfxge_dma_alloc(sc, EFX_EVQ_SIZE(evq->entries), esmp)) != 0) 924 return (rc); 925 926 /* Allocate buffer table entries. */ 927 sfxge_sram_buf_tbl_alloc(sc, EFX_EVQ_NBUFS(evq->entries), 928 &evq->buf_base_id); 929 930 SFXGE_EVQ_LOCK_INIT(evq, device_get_nameunit(sc->dev), index); 931 932 evq->init_state = SFXGE_EVQ_INITIALIZED; 933 934 #if EFSYS_OPT_QSTATS 935 rc = sfxge_evq_stat_init(evq); 936 if (rc != 0) 937 goto fail_evq_stat_init; 938 #endif 939 940 return (0); 941 942 #if EFSYS_OPT_QSTATS 943 fail_evq_stat_init: 944 evq->init_state = SFXGE_EVQ_UNINITIALIZED; 945 SFXGE_EVQ_LOCK_DESTROY(evq); 946 sfxge_dma_free(esmp); 947 sc->evq[index] = NULL; 948 free(evq, M_SFXGE); 949 950 return (rc); 951 #endif 952 } 953 954 void 955 sfxge_ev_fini(struct sfxge_softc *sc) 956 { 957 struct sfxge_intr *intr __diagused; 958 int index; 959 960 intr = &sc->intr; 961 962 KASSERT(intr->state == SFXGE_INTR_INITIALIZED, 963 ("intr->state != SFXGE_INTR_INITIALIZED")); 964 965 sc->ev_moderation = 0; 966 967 /* Tear down the event queue(s). */ 968 index = sc->evq_count; 969 while (--index >= 0) 970 sfxge_ev_qfini(sc, index); 971 972 sc->evq_count = 0; 973 } 974 975 int 976 sfxge_ev_init(struct sfxge_softc *sc) 977 { 978 struct sysctl_ctx_list *sysctl_ctx = device_get_sysctl_ctx(sc->dev); 979 struct sysctl_oid *sysctl_tree = device_get_sysctl_tree(sc->dev); 980 struct sfxge_intr *intr; 981 int index; 982 int rc; 983 984 intr = &sc->intr; 985 986 sc->evq_count = intr->n_alloc; 987 988 KASSERT(intr->state == SFXGE_INTR_INITIALIZED, 989 ("intr->state != SFXGE_INTR_INITIALIZED")); 990 991 /* Set default interrupt moderation; add a sysctl to 992 * read and change it. 993 */ 994 sc->ev_moderation = SFXGE_MODERATION; 995 SYSCTL_ADD_PROC(sysctl_ctx, SYSCTL_CHILDREN(sysctl_tree), 996 OID_AUTO, "int_mod", CTLTYPE_UINT | CTLFLAG_RW | CTLFLAG_MPSAFE, 997 sc, 0, sfxge_int_mod_handler, "IU", 998 "sfxge interrupt moderation (us)"); 999 1000 #if EFSYS_OPT_QSTATS 1001 sc->evqs_stats_node = SYSCTL_ADD_NODE( 1002 device_get_sysctl_ctx(sc->dev), SYSCTL_CHILDREN(sc->stats_node), 1003 OID_AUTO, "evq", CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, 1004 "Event queues stats"); 1005 if (sc->evqs_stats_node == NULL) { 1006 rc = ENOMEM; 1007 goto fail_evqs_stats_node; 1008 } 1009 #endif 1010 1011 /* 1012 * Initialize the event queue(s) - one per interrupt. 1013 */ 1014 for (index = 0; index < sc->evq_count; index++) { 1015 if ((rc = sfxge_ev_qinit(sc, index)) != 0) 1016 goto fail; 1017 } 1018 1019 #if EFSYS_OPT_QSTATS 1020 sfxge_ev_stat_init(sc); 1021 #endif 1022 1023 return (0); 1024 1025 fail: 1026 while (--index >= 0) 1027 sfxge_ev_qfini(sc, index); 1028 1029 #if EFSYS_OPT_QSTATS 1030 fail_evqs_stats_node: 1031 #endif 1032 sc->evq_count = 0; 1033 return (rc); 1034 } 1035