1 /*- 2 * Copyright (c) 2010-2015 Solarflare Communications Inc. 3 * All rights reserved. 4 * 5 * This software was developed in part by Philip Paeps under contract for 6 * Solarflare Communications, Inc. 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions are met: 10 * 11 * 1. Redistributions of source code must retain the above copyright notice, 12 * this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright notice, 14 * this list of conditions and the following disclaimer in the documentation 15 * and/or other materials provided with the distribution. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 18 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, 19 * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 20 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR 21 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, 22 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 23 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; 24 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 25 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR 26 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, 27 * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 * 29 * The views and conclusions contained in the software and documentation are 30 * those of the authors and should not be interpreted as representing official 31 * policies, either expressed or implied, of the FreeBSD Project. 32 */ 33 34 #include <sys/cdefs.h> 35 __FBSDID("$FreeBSD$"); 36 37 #include <sys/param.h> 38 #include <sys/kernel.h> 39 #include <sys/malloc.h> 40 #include <sys/param.h> 41 #include <sys/queue.h> 42 #include <sys/systm.h> 43 #include <sys/taskqueue.h> 44 45 #include "common/efx.h" 46 47 #include "sfxge.h" 48 49 static void 50 sfxge_ev_qcomplete(struct sfxge_evq *evq, boolean_t eop) 51 { 52 struct sfxge_softc *sc; 53 unsigned int index; 54 struct sfxge_rxq *rxq; 55 struct sfxge_txq *txq; 56 57 SFXGE_EVQ_LOCK_ASSERT_OWNED(evq); 58 59 sc = evq->sc; 60 index = evq->index; 61 rxq = sc->rxq[index]; 62 63 if ((txq = evq->txq) != NULL) { 64 evq->txq = NULL; 65 evq->txqs = &(evq->txq); 66 67 do { 68 struct sfxge_txq *next; 69 70 next = txq->next; 71 txq->next = NULL; 72 73 KASSERT(txq->evq_index == index, 74 ("txq->evq_index != index")); 75 76 if (txq->pending != txq->completed) 77 sfxge_tx_qcomplete(txq, evq); 78 79 txq = next; 80 } while (txq != NULL); 81 } 82 83 if (rxq->pending != rxq->completed) 84 sfxge_rx_qcomplete(rxq, eop); 85 } 86 87 static boolean_t 88 sfxge_ev_rx(void *arg, uint32_t label, uint32_t id, uint32_t size, 89 uint16_t flags) 90 { 91 struct sfxge_evq *evq; 92 struct sfxge_softc *sc; 93 struct sfxge_rxq *rxq; 94 unsigned int stop; 95 unsigned int delta; 96 struct sfxge_rx_sw_desc *rx_desc; 97 98 evq = arg; 99 SFXGE_EVQ_LOCK_ASSERT_OWNED(evq); 100 101 sc = evq->sc; 102 103 if (evq->exception) 104 goto done; 105 106 rxq = sc->rxq[label]; 107 KASSERT(rxq != NULL, ("rxq == NULL")); 108 KASSERT(evq->index == rxq->index, 109 ("evq->index != rxq->index")); 110 111 if (__predict_false(rxq->init_state != SFXGE_RXQ_STARTED)) 112 goto done; 113 114 stop = (id + 1) & rxq->ptr_mask; 115 id = rxq->pending & rxq->ptr_mask; 116 delta = (stop >= id) ? (stop - id) : (rxq->entries - id + stop); 117 rxq->pending += delta; 118 119 if (delta != 1) { 120 if ((!efx_nic_cfg_get(sc->enp)->enc_rx_batching_enabled) || 121 (delta <= 0) || 122 (delta > efx_nic_cfg_get(sc->enp)->enc_rx_batch_max)) { 123 evq->exception = B_TRUE; 124 125 device_printf(sc->dev, "RX completion out of order" 126 " (id=%#x delta=%u flags=%#x); resetting\n", 127 id, delta, flags); 128 sfxge_schedule_reset(sc); 129 130 goto done; 131 } 132 } 133 134 rx_desc = &rxq->queue[id]; 135 136 prefetch_read_many(rx_desc->mbuf); 137 138 for (; id != stop; id = (id + 1) & rxq->ptr_mask) { 139 rx_desc = &rxq->queue[id]; 140 KASSERT(rx_desc->flags == EFX_DISCARD, 141 ("rx_desc->flags != EFX_DISCARD")); 142 rx_desc->flags = flags; 143 144 KASSERT(size < (1 << 16), ("size > (1 << 16)")); 145 rx_desc->size = (uint16_t)size; 146 } 147 148 evq->rx_done++; 149 150 if (rxq->pending - rxq->completed >= SFXGE_RX_BATCH) 151 sfxge_ev_qcomplete(evq, B_FALSE); 152 153 done: 154 return (evq->rx_done >= SFXGE_EV_BATCH); 155 } 156 157 static boolean_t 158 sfxge_ev_exception(void *arg, uint32_t code, uint32_t data) 159 { 160 struct sfxge_evq *evq; 161 struct sfxge_softc *sc; 162 163 evq = (struct sfxge_evq *)arg; 164 SFXGE_EVQ_LOCK_ASSERT_OWNED(evq); 165 166 sc = evq->sc; 167 168 DBGPRINT(sc->dev, "[%d] %s", evq->index, 169 (code == EFX_EXCEPTION_RX_RECOVERY) ? "RX_RECOVERY" : 170 (code == EFX_EXCEPTION_RX_DSC_ERROR) ? "RX_DSC_ERROR" : 171 (code == EFX_EXCEPTION_TX_DSC_ERROR) ? "TX_DSC_ERROR" : 172 (code == EFX_EXCEPTION_UNKNOWN_SENSOREVT) ? "UNKNOWN_SENSOREVT" : 173 (code == EFX_EXCEPTION_FWALERT_SRAM) ? "FWALERT_SRAM" : 174 (code == EFX_EXCEPTION_UNKNOWN_FWALERT) ? "UNKNOWN_FWALERT" : 175 (code == EFX_EXCEPTION_RX_ERROR) ? "RX_ERROR" : 176 (code == EFX_EXCEPTION_TX_ERROR) ? "TX_ERROR" : 177 (code == EFX_EXCEPTION_EV_ERROR) ? "EV_ERROR" : 178 "UNKNOWN"); 179 180 evq->exception = B_TRUE; 181 182 if (code != EFX_EXCEPTION_UNKNOWN_SENSOREVT) { 183 device_printf(sc->dev, 184 "hardware exception (code=%u); resetting\n", 185 code); 186 sfxge_schedule_reset(sc); 187 } 188 189 return (B_FALSE); 190 } 191 192 static boolean_t 193 sfxge_ev_rxq_flush_done(void *arg, uint32_t rxq_index) 194 { 195 struct sfxge_evq *evq; 196 struct sfxge_softc *sc; 197 struct sfxge_rxq *rxq; 198 unsigned int index; 199 unsigned int label; 200 uint16_t magic; 201 202 evq = (struct sfxge_evq *)arg; 203 SFXGE_EVQ_LOCK_ASSERT_OWNED(evq); 204 205 sc = evq->sc; 206 rxq = sc->rxq[rxq_index]; 207 208 KASSERT(rxq != NULL, ("rxq == NULL")); 209 210 /* Resend a software event on the correct queue */ 211 index = rxq->index; 212 if (index == evq->index) { 213 sfxge_rx_qflush_done(rxq); 214 return (B_FALSE); 215 } 216 217 evq = sc->evq[index]; 218 219 label = rxq_index; 220 KASSERT((label & SFXGE_MAGIC_DMAQ_LABEL_MASK) == label, 221 ("(label & SFXGE_MAGIC_DMAQ_LABEL_MASK) != level")); 222 magic = SFXGE_MAGIC_RX_QFLUSH_DONE | label; 223 224 KASSERT(evq->init_state == SFXGE_EVQ_STARTED, 225 ("evq not started")); 226 efx_ev_qpost(evq->common, magic); 227 228 return (B_FALSE); 229 } 230 231 static boolean_t 232 sfxge_ev_rxq_flush_failed(void *arg, uint32_t rxq_index) 233 { 234 struct sfxge_evq *evq; 235 struct sfxge_softc *sc; 236 struct sfxge_rxq *rxq; 237 unsigned int index; 238 unsigned int label; 239 uint16_t magic; 240 241 evq = (struct sfxge_evq *)arg; 242 SFXGE_EVQ_LOCK_ASSERT_OWNED(evq); 243 244 sc = evq->sc; 245 rxq = sc->rxq[rxq_index]; 246 247 KASSERT(rxq != NULL, ("rxq == NULL")); 248 249 /* Resend a software event on the correct queue */ 250 index = rxq->index; 251 evq = sc->evq[index]; 252 253 label = rxq_index; 254 KASSERT((label & SFXGE_MAGIC_DMAQ_LABEL_MASK) == label, 255 ("(label & SFXGE_MAGIC_DMAQ_LABEL_MASK) != label")); 256 magic = SFXGE_MAGIC_RX_QFLUSH_FAILED | label; 257 258 KASSERT(evq->init_state == SFXGE_EVQ_STARTED, 259 ("evq not started")); 260 efx_ev_qpost(evq->common, magic); 261 262 return (B_FALSE); 263 } 264 265 static struct sfxge_txq * 266 sfxge_get_txq_by_label(struct sfxge_evq *evq, enum sfxge_txq_type label) 267 { 268 unsigned int index; 269 270 KASSERT((evq->index == 0 && label < SFXGE_TXQ_NTYPES) || 271 (label == SFXGE_TXQ_IP_TCP_UDP_CKSUM), ("unexpected txq label")); 272 index = (evq->index == 0) ? label : (evq->index - 1 + SFXGE_TXQ_NTYPES); 273 return (evq->sc->txq[index]); 274 } 275 276 static boolean_t 277 sfxge_ev_tx(void *arg, uint32_t label, uint32_t id) 278 { 279 struct sfxge_evq *evq; 280 struct sfxge_txq *txq; 281 unsigned int stop; 282 unsigned int delta; 283 284 evq = (struct sfxge_evq *)arg; 285 SFXGE_EVQ_LOCK_ASSERT_OWNED(evq); 286 287 txq = sfxge_get_txq_by_label(evq, label); 288 289 KASSERT(txq != NULL, ("txq == NULL")); 290 KASSERT(evq->index == txq->evq_index, 291 ("evq->index != txq->evq_index")); 292 293 if (__predict_false(txq->init_state != SFXGE_TXQ_STARTED)) 294 goto done; 295 296 stop = (id + 1) & txq->ptr_mask; 297 id = txq->pending & txq->ptr_mask; 298 299 delta = (stop >= id) ? (stop - id) : (txq->entries - id + stop); 300 txq->pending += delta; 301 302 evq->tx_done++; 303 304 if (txq->next == NULL && 305 evq->txqs != &(txq->next)) { 306 *(evq->txqs) = txq; 307 evq->txqs = &(txq->next); 308 } 309 310 if (txq->pending - txq->completed >= SFXGE_TX_BATCH) 311 sfxge_tx_qcomplete(txq, evq); 312 313 done: 314 return (evq->tx_done >= SFXGE_EV_BATCH); 315 } 316 317 static boolean_t 318 sfxge_ev_txq_flush_done(void *arg, uint32_t txq_index) 319 { 320 struct sfxge_evq *evq; 321 struct sfxge_softc *sc; 322 struct sfxge_txq *txq; 323 unsigned int label; 324 uint16_t magic; 325 326 evq = (struct sfxge_evq *)arg; 327 SFXGE_EVQ_LOCK_ASSERT_OWNED(evq); 328 329 sc = evq->sc; 330 txq = sc->txq[txq_index]; 331 332 KASSERT(txq != NULL, ("txq == NULL")); 333 KASSERT(txq->init_state == SFXGE_TXQ_INITIALIZED, 334 ("txq not initialized")); 335 336 if (txq->evq_index == evq->index) { 337 sfxge_tx_qflush_done(txq); 338 return (B_FALSE); 339 } 340 341 /* Resend a software event on the correct queue */ 342 evq = sc->evq[txq->evq_index]; 343 344 label = txq->type; 345 KASSERT((label & SFXGE_MAGIC_DMAQ_LABEL_MASK) == label, 346 ("(label & SFXGE_MAGIC_DMAQ_LABEL_MASK) != label")); 347 magic = SFXGE_MAGIC_TX_QFLUSH_DONE | label; 348 349 KASSERT(evq->init_state == SFXGE_EVQ_STARTED, 350 ("evq not started")); 351 efx_ev_qpost(evq->common, magic); 352 353 return (B_FALSE); 354 } 355 356 static boolean_t 357 sfxge_ev_software(void *arg, uint16_t magic) 358 { 359 struct sfxge_evq *evq; 360 struct sfxge_softc *sc; 361 unsigned int label; 362 363 evq = (struct sfxge_evq *)arg; 364 SFXGE_EVQ_LOCK_ASSERT_OWNED(evq); 365 366 sc = evq->sc; 367 368 label = magic & SFXGE_MAGIC_DMAQ_LABEL_MASK; 369 magic &= ~SFXGE_MAGIC_DMAQ_LABEL_MASK; 370 371 switch (magic) { 372 case SFXGE_MAGIC_RX_QFLUSH_DONE: { 373 struct sfxge_rxq *rxq = sc->rxq[label]; 374 375 KASSERT(rxq != NULL, ("rxq == NULL")); 376 KASSERT(evq->index == rxq->index, 377 ("evq->index != rxq->index")); 378 379 sfxge_rx_qflush_done(rxq); 380 break; 381 } 382 case SFXGE_MAGIC_RX_QFLUSH_FAILED: { 383 struct sfxge_rxq *rxq = sc->rxq[label]; 384 385 KASSERT(rxq != NULL, ("rxq == NULL")); 386 KASSERT(evq->index == rxq->index, 387 ("evq->index != rxq->index")); 388 389 sfxge_rx_qflush_failed(rxq); 390 break; 391 } 392 case SFXGE_MAGIC_RX_QREFILL: { 393 struct sfxge_rxq *rxq = sc->rxq[label]; 394 395 KASSERT(rxq != NULL, ("rxq == NULL")); 396 KASSERT(evq->index == rxq->index, 397 ("evq->index != rxq->index")); 398 399 sfxge_rx_qrefill(rxq); 400 break; 401 } 402 case SFXGE_MAGIC_TX_QFLUSH_DONE: { 403 struct sfxge_txq *txq = sfxge_get_txq_by_label(evq, label); 404 405 KASSERT(txq != NULL, ("txq == NULL")); 406 KASSERT(evq->index == txq->evq_index, 407 ("evq->index != txq->evq_index")); 408 409 sfxge_tx_qflush_done(txq); 410 break; 411 } 412 default: 413 break; 414 } 415 416 return (B_FALSE); 417 } 418 419 static boolean_t 420 sfxge_ev_sram(void *arg, uint32_t code) 421 { 422 (void)arg; 423 (void)code; 424 425 switch (code) { 426 case EFX_SRAM_UPDATE: 427 EFSYS_PROBE(sram_update); 428 break; 429 430 case EFX_SRAM_CLEAR: 431 EFSYS_PROBE(sram_clear); 432 break; 433 434 case EFX_SRAM_ILLEGAL_CLEAR: 435 EFSYS_PROBE(sram_illegal_clear); 436 break; 437 438 default: 439 KASSERT(B_FALSE, ("Impossible SRAM event")); 440 break; 441 } 442 443 return (B_FALSE); 444 } 445 446 static boolean_t 447 sfxge_ev_timer(void *arg, uint32_t index) 448 { 449 (void)arg; 450 (void)index; 451 452 return (B_FALSE); 453 } 454 455 static boolean_t 456 sfxge_ev_wake_up(void *arg, uint32_t index) 457 { 458 (void)arg; 459 (void)index; 460 461 return (B_FALSE); 462 } 463 464 #if EFSYS_OPT_QSTATS 465 466 static void 467 sfxge_ev_stat_update(struct sfxge_softc *sc) 468 { 469 struct sfxge_evq *evq; 470 unsigned int index; 471 clock_t now; 472 473 SFXGE_ADAPTER_LOCK(sc); 474 475 if (__predict_false(sc->evq[0]->init_state != SFXGE_EVQ_STARTED)) 476 goto out; 477 478 now = ticks; 479 if (now - sc->ev_stats_update_time < hz) 480 goto out; 481 482 sc->ev_stats_update_time = now; 483 484 /* Add event counts from each event queue in turn */ 485 for (index = 0; index < sc->evq_count; index++) { 486 evq = sc->evq[index]; 487 SFXGE_EVQ_LOCK(evq); 488 efx_ev_qstats_update(evq->common, sc->ev_stats); 489 SFXGE_EVQ_UNLOCK(evq); 490 } 491 out: 492 SFXGE_ADAPTER_UNLOCK(sc); 493 } 494 495 static int 496 sfxge_ev_stat_handler(SYSCTL_HANDLER_ARGS) 497 { 498 struct sfxge_softc *sc = arg1; 499 unsigned int id = arg2; 500 501 sfxge_ev_stat_update(sc); 502 503 return (SYSCTL_OUT(req, &sc->ev_stats[id], sizeof(sc->ev_stats[id]))); 504 } 505 506 static void 507 sfxge_ev_stat_init(struct sfxge_softc *sc) 508 { 509 struct sysctl_ctx_list *ctx = device_get_sysctl_ctx(sc->dev); 510 struct sysctl_oid_list *stat_list; 511 unsigned int id; 512 char name[40]; 513 514 stat_list = SYSCTL_CHILDREN(sc->stats_node); 515 516 for (id = 0; id < EV_NQSTATS; id++) { 517 snprintf(name, sizeof(name), "ev_%s", 518 efx_ev_qstat_name(sc->enp, id)); 519 SYSCTL_ADD_PROC( 520 ctx, stat_list, 521 OID_AUTO, name, CTLTYPE_U64|CTLFLAG_RD, 522 sc, id, sfxge_ev_stat_handler, "Q", 523 ""); 524 } 525 } 526 527 #endif /* EFSYS_OPT_QSTATS */ 528 529 static void 530 sfxge_ev_qmoderate(struct sfxge_softc *sc, unsigned int idx, unsigned int us) 531 { 532 struct sfxge_evq *evq; 533 efx_evq_t *eep; 534 535 evq = sc->evq[idx]; 536 eep = evq->common; 537 538 KASSERT(evq->init_state == SFXGE_EVQ_STARTED, 539 ("evq->init_state != SFXGE_EVQ_STARTED")); 540 541 (void)efx_ev_qmoderate(eep, us); 542 } 543 544 static int 545 sfxge_int_mod_handler(SYSCTL_HANDLER_ARGS) 546 { 547 struct sfxge_softc *sc = arg1; 548 struct sfxge_intr *intr = &sc->intr; 549 unsigned int moderation; 550 int error; 551 unsigned int index; 552 553 SFXGE_ADAPTER_LOCK(sc); 554 555 if (req->newptr != NULL) { 556 if ((error = SYSCTL_IN(req, &moderation, sizeof(moderation))) 557 != 0) 558 goto out; 559 560 /* We may not be calling efx_ev_qmoderate() now, 561 * so we have to range-check the value ourselves. 562 */ 563 if (moderation > 564 efx_nic_cfg_get(sc->enp)->enc_evq_timer_max_us) { 565 error = EINVAL; 566 goto out; 567 } 568 569 sc->ev_moderation = moderation; 570 if (intr->state == SFXGE_INTR_STARTED) { 571 for (index = 0; index < sc->evq_count; index++) 572 sfxge_ev_qmoderate(sc, index, moderation); 573 } 574 } else { 575 error = SYSCTL_OUT(req, &sc->ev_moderation, 576 sizeof(sc->ev_moderation)); 577 } 578 579 out: 580 SFXGE_ADAPTER_UNLOCK(sc); 581 582 return (error); 583 } 584 585 static boolean_t 586 sfxge_ev_initialized(void *arg) 587 { 588 struct sfxge_evq *evq; 589 590 evq = (struct sfxge_evq *)arg; 591 SFXGE_EVQ_LOCK_ASSERT_OWNED(evq); 592 593 /* Init done events may be duplicated on 7xxx */ 594 KASSERT(evq->init_state == SFXGE_EVQ_STARTING || 595 evq->init_state == SFXGE_EVQ_STARTED, 596 ("evq not starting")); 597 598 evq->init_state = SFXGE_EVQ_STARTED; 599 600 return (0); 601 } 602 603 static boolean_t 604 sfxge_ev_link_change(void *arg, efx_link_mode_t link_mode) 605 { 606 struct sfxge_evq *evq; 607 struct sfxge_softc *sc; 608 609 evq = (struct sfxge_evq *)arg; 610 SFXGE_EVQ_LOCK_ASSERT_OWNED(evq); 611 612 sc = evq->sc; 613 614 sfxge_mac_link_update(sc, link_mode); 615 616 return (0); 617 } 618 619 static const efx_ev_callbacks_t sfxge_ev_callbacks = { 620 .eec_initialized = sfxge_ev_initialized, 621 .eec_rx = sfxge_ev_rx, 622 .eec_tx = sfxge_ev_tx, 623 .eec_exception = sfxge_ev_exception, 624 .eec_rxq_flush_done = sfxge_ev_rxq_flush_done, 625 .eec_rxq_flush_failed = sfxge_ev_rxq_flush_failed, 626 .eec_txq_flush_done = sfxge_ev_txq_flush_done, 627 .eec_software = sfxge_ev_software, 628 .eec_sram = sfxge_ev_sram, 629 .eec_wake_up = sfxge_ev_wake_up, 630 .eec_timer = sfxge_ev_timer, 631 .eec_link_change = sfxge_ev_link_change, 632 }; 633 634 635 int 636 sfxge_ev_qpoll(struct sfxge_evq *evq) 637 { 638 int rc; 639 640 SFXGE_EVQ_LOCK(evq); 641 642 if (__predict_false(evq->init_state != SFXGE_EVQ_STARTING && 643 evq->init_state != SFXGE_EVQ_STARTED)) { 644 rc = EINVAL; 645 goto fail; 646 } 647 648 /* Synchronize the DMA memory for reading */ 649 bus_dmamap_sync(evq->mem.esm_tag, evq->mem.esm_map, 650 BUS_DMASYNC_POSTREAD); 651 652 KASSERT(evq->rx_done == 0, ("evq->rx_done != 0")); 653 KASSERT(evq->tx_done == 0, ("evq->tx_done != 0")); 654 KASSERT(evq->txq == NULL, ("evq->txq != NULL")); 655 KASSERT(evq->txqs == &evq->txq, ("evq->txqs != &evq->txq")); 656 657 /* Poll the queue */ 658 efx_ev_qpoll(evq->common, &evq->read_ptr, &sfxge_ev_callbacks, evq); 659 660 evq->rx_done = 0; 661 evq->tx_done = 0; 662 663 /* Perform any pending completion processing */ 664 sfxge_ev_qcomplete(evq, B_TRUE); 665 666 /* Re-prime the event queue for interrupts */ 667 if ((rc = efx_ev_qprime(evq->common, evq->read_ptr)) != 0) 668 goto fail; 669 670 SFXGE_EVQ_UNLOCK(evq); 671 672 return (0); 673 674 fail: 675 SFXGE_EVQ_UNLOCK(evq); 676 return (rc); 677 } 678 679 static void 680 sfxge_ev_qstop(struct sfxge_softc *sc, unsigned int index) 681 { 682 struct sfxge_evq *evq; 683 684 evq = sc->evq[index]; 685 686 KASSERT(evq->init_state == SFXGE_EVQ_STARTED, 687 ("evq->init_state != SFXGE_EVQ_STARTED")); 688 689 SFXGE_EVQ_LOCK(evq); 690 evq->init_state = SFXGE_EVQ_INITIALIZED; 691 evq->read_ptr = 0; 692 evq->exception = B_FALSE; 693 694 #if EFSYS_OPT_QSTATS 695 /* Add event counts before discarding the common evq state */ 696 efx_ev_qstats_update(evq->common, sc->ev_stats); 697 #endif 698 699 efx_ev_qdestroy(evq->common); 700 efx_sram_buf_tbl_clear(sc->enp, evq->buf_base_id, 701 EFX_EVQ_NBUFS(evq->entries)); 702 SFXGE_EVQ_UNLOCK(evq); 703 } 704 705 static int 706 sfxge_ev_qstart(struct sfxge_softc *sc, unsigned int index) 707 { 708 struct sfxge_evq *evq; 709 efsys_mem_t *esmp; 710 int count; 711 int rc; 712 713 evq = sc->evq[index]; 714 esmp = &evq->mem; 715 716 KASSERT(evq->init_state == SFXGE_EVQ_INITIALIZED, 717 ("evq->init_state != SFXGE_EVQ_INITIALIZED")); 718 719 /* Clear all events. */ 720 (void)memset(esmp->esm_base, 0xff, EFX_EVQ_SIZE(evq->entries)); 721 722 /* Program the buffer table. */ 723 if ((rc = efx_sram_buf_tbl_set(sc->enp, evq->buf_base_id, esmp, 724 EFX_EVQ_NBUFS(evq->entries))) != 0) 725 return (rc); 726 727 /* Create the common code event queue. */ 728 if ((rc = efx_ev_qcreate(sc->enp, index, esmp, evq->entries, 729 evq->buf_base_id, &evq->common)) != 0) 730 goto fail; 731 732 SFXGE_EVQ_LOCK(evq); 733 734 /* Set the default moderation */ 735 (void)efx_ev_qmoderate(evq->common, sc->ev_moderation); 736 737 /* Prime the event queue for interrupts */ 738 if ((rc = efx_ev_qprime(evq->common, evq->read_ptr)) != 0) 739 goto fail2; 740 741 evq->init_state = SFXGE_EVQ_STARTING; 742 743 SFXGE_EVQ_UNLOCK(evq); 744 745 /* Wait for the initialization event */ 746 count = 0; 747 do { 748 /* Pause for 100 ms */ 749 pause("sfxge evq init", hz / 10); 750 751 /* Check to see if the test event has been processed */ 752 if (evq->init_state == SFXGE_EVQ_STARTED) 753 goto done; 754 755 } while (++count < 20); 756 757 rc = ETIMEDOUT; 758 goto fail3; 759 760 done: 761 return (0); 762 763 fail3: 764 SFXGE_EVQ_LOCK(evq); 765 evq->init_state = SFXGE_EVQ_INITIALIZED; 766 fail2: 767 SFXGE_EVQ_UNLOCK(evq); 768 efx_ev_qdestroy(evq->common); 769 fail: 770 efx_sram_buf_tbl_clear(sc->enp, evq->buf_base_id, 771 EFX_EVQ_NBUFS(evq->entries)); 772 773 return (rc); 774 } 775 776 void 777 sfxge_ev_stop(struct sfxge_softc *sc) 778 { 779 struct sfxge_intr *intr; 780 efx_nic_t *enp; 781 int index; 782 783 intr = &sc->intr; 784 enp = sc->enp; 785 786 KASSERT(intr->state == SFXGE_INTR_STARTED, 787 ("Interrupts not started")); 788 789 /* Stop the event queue(s) */ 790 index = sc->evq_count; 791 while (--index >= 0) 792 sfxge_ev_qstop(sc, index); 793 794 /* Tear down the event module */ 795 efx_ev_fini(enp); 796 } 797 798 int 799 sfxge_ev_start(struct sfxge_softc *sc) 800 { 801 struct sfxge_intr *intr; 802 int index; 803 int rc; 804 805 intr = &sc->intr; 806 807 KASSERT(intr->state == SFXGE_INTR_STARTED, 808 ("intr->state != SFXGE_INTR_STARTED")); 809 810 /* Initialize the event module */ 811 if ((rc = efx_ev_init(sc->enp)) != 0) 812 return (rc); 813 814 /* Start the event queues */ 815 for (index = 0; index < sc->evq_count; index++) { 816 if ((rc = sfxge_ev_qstart(sc, index)) != 0) 817 goto fail; 818 } 819 820 return (0); 821 822 fail: 823 /* Stop the event queue(s) */ 824 while (--index >= 0) 825 sfxge_ev_qstop(sc, index); 826 827 /* Tear down the event module */ 828 efx_ev_fini(sc->enp); 829 830 return (rc); 831 } 832 833 static void 834 sfxge_ev_qfini(struct sfxge_softc *sc, unsigned int index) 835 { 836 struct sfxge_evq *evq; 837 838 evq = sc->evq[index]; 839 840 KASSERT(evq->init_state == SFXGE_EVQ_INITIALIZED, 841 ("evq->init_state != SFXGE_EVQ_INITIALIZED")); 842 KASSERT(evq->txqs == &evq->txq, ("evq->txqs != &evq->txq")); 843 844 sfxge_dma_free(&evq->mem); 845 846 sc->evq[index] = NULL; 847 848 SFXGE_EVQ_LOCK_DESTROY(evq); 849 850 free(evq, M_SFXGE); 851 } 852 853 static int 854 sfxge_ev_qinit(struct sfxge_softc *sc, unsigned int index) 855 { 856 struct sfxge_evq *evq; 857 efsys_mem_t *esmp; 858 int rc; 859 860 KASSERT(index < SFXGE_RX_SCALE_MAX, ("index >= SFXGE_RX_SCALE_MAX")); 861 862 evq = malloc(sizeof(struct sfxge_evq), M_SFXGE, M_ZERO | M_WAITOK); 863 evq->sc = sc; 864 evq->index = index; 865 sc->evq[index] = evq; 866 esmp = &evq->mem; 867 868 /* Build an event queue with room for one event per tx and rx buffer, 869 * plus some extra for link state events and MCDI completions. 870 * There are three tx queues in the first event queue and one in 871 * other. 872 */ 873 if (index == 0) 874 evq->entries = 875 ROUNDUP_POW_OF_TWO(sc->rxq_entries + 876 3 * sc->txq_entries + 877 128); 878 else 879 evq->entries = 880 ROUNDUP_POW_OF_TWO(sc->rxq_entries + 881 sc->txq_entries + 882 128); 883 884 /* Initialise TX completion list */ 885 evq->txqs = &evq->txq; 886 887 /* Allocate DMA space. */ 888 if ((rc = sfxge_dma_alloc(sc, EFX_EVQ_SIZE(evq->entries), esmp)) != 0) 889 return (rc); 890 891 /* Allocate buffer table entries. */ 892 sfxge_sram_buf_tbl_alloc(sc, EFX_EVQ_NBUFS(evq->entries), 893 &evq->buf_base_id); 894 895 SFXGE_EVQ_LOCK_INIT(evq, device_get_nameunit(sc->dev), index); 896 897 evq->init_state = SFXGE_EVQ_INITIALIZED; 898 899 return (0); 900 } 901 902 void 903 sfxge_ev_fini(struct sfxge_softc *sc) 904 { 905 struct sfxge_intr *intr; 906 int index; 907 908 intr = &sc->intr; 909 910 KASSERT(intr->state == SFXGE_INTR_INITIALIZED, 911 ("intr->state != SFXGE_INTR_INITIALIZED")); 912 913 sc->ev_moderation = 0; 914 915 /* Tear down the event queue(s). */ 916 index = sc->evq_count; 917 while (--index >= 0) 918 sfxge_ev_qfini(sc, index); 919 920 sc->evq_count = 0; 921 } 922 923 int 924 sfxge_ev_init(struct sfxge_softc *sc) 925 { 926 struct sysctl_ctx_list *sysctl_ctx = device_get_sysctl_ctx(sc->dev); 927 struct sysctl_oid *sysctl_tree = device_get_sysctl_tree(sc->dev); 928 struct sfxge_intr *intr; 929 int index; 930 int rc; 931 932 intr = &sc->intr; 933 934 sc->evq_count = intr->n_alloc; 935 936 KASSERT(intr->state == SFXGE_INTR_INITIALIZED, 937 ("intr->state != SFXGE_INTR_INITIALIZED")); 938 939 /* Set default interrupt moderation; add a sysctl to 940 * read and change it. 941 */ 942 sc->ev_moderation = SFXGE_MODERATION; 943 SYSCTL_ADD_PROC(sysctl_ctx, SYSCTL_CHILDREN(sysctl_tree), 944 OID_AUTO, "int_mod", CTLTYPE_UINT|CTLFLAG_RW, 945 sc, 0, sfxge_int_mod_handler, "IU", 946 "sfxge interrupt moderation (us)"); 947 948 /* 949 * Initialize the event queue(s) - one per interrupt. 950 */ 951 for (index = 0; index < sc->evq_count; index++) { 952 if ((rc = sfxge_ev_qinit(sc, index)) != 0) 953 goto fail; 954 } 955 956 #if EFSYS_OPT_QSTATS 957 sfxge_ev_stat_init(sc); 958 #endif 959 960 return (0); 961 962 fail: 963 while (--index >= 0) 964 sfxge_ev_qfini(sc, index); 965 966 sc->evq_count = 0; 967 return (rc); 968 } 969