Lines Matching +full:tx +full:- +full:queue +full:- +full:2
2 SPDX-License-Identifier: BSD-2-Clause
4 Copyright (c) 2007-2009, Chelsio Inc.
13 2. Neither the name of the Chelsio Corporation nor the names of its
44 #include <sys/queue.h>
88 "size of per-queue mbuf ring");
96 #define COALESCE_START_MAX (TX_ETH_Q_SIZE-(TX_ETH_Q_SIZE>>3))
97 #define COALESCE_STOP_DEFAULT TX_ETH_Q_SIZE>>2
100 #define TX_RECLAIM_MAX TX_ETH_Q_SIZE>>2
115 "tx cleaning minimum threshold");
118 * XXX don't re-enable this until TOE stops assuming
135 * Period of the Tx buffer reclaim timer. This timer does not need to run
136 * frequently as Tx buffers are usually reclaimed by new Tx packets.
159 struct rsp_desc { /* response queue descriptor */
177 struct tx_sw_desc { /* SW state per Tx descriptor */
204 * Maps a number of flits to the number of Tx descriptors that can hold them.
207 * desc = 1 + (flits - 2) / (WR_FLITS - 1).
215 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
218 #elif SGE_NUM_GENBITS == 2
220 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
224 # error "SGE_NUM_GENBITS must be 1 or 2"
228 #define TXQ_LOCK_ASSERT(qs) mtx_assert(&(qs)->lock, MA_OWNED)
229 #define TXQ_TRYLOCK(qs) mtx_trylock(&(qs)->lock)
230 #define TXQ_LOCK(qs) mtx_lock(&(qs)->lock)
231 #define TXQ_UNLOCK(qs) mtx_unlock(&(qs)->lock)
232 #define TXQ_RING_EMPTY(qs) drbr_empty((qs)->port->ifp, (qs)->txq[TXQ_ETH].txq_mr)
234 drbr_needs_enqueue((qs)->port->ifp, (qs)->txq[TXQ_ETH].txq_mr)
235 #define TXQ_RING_FLUSH(qs) drbr_flush((qs)->port->ifp, (qs)->txq[TXQ_ETH].txq_mr)
237 drbr_dequeue_cond((qs)->port->ifp, (qs)->txq[TXQ_ETH].txq_mr, func, arg)
239 drbr_dequeue((qs)->port->ifp, (qs)->txq[TXQ_ETH].txq_mr)
262 txq = &qs->txq[TXQ_ETH]; in check_pkt_coalesce()
263 sc = qs->port->adapter; in check_pkt_coalesce()
264 fill = &sc->tunq_fill[qs->idx]; in check_pkt_coalesce()
271 * if the hardware transmit queue is more than 1/8 full in check_pkt_coalesce()
272 * we mark it as coalescing - we drop back from coalescing in check_pkt_coalesce()
276 if (*fill != 0 && (txq->in_use <= cxgb_tx_coalesce_enable_stop) && in check_pkt_coalesce()
277 TXQ_RING_EMPTY(qs) && (qs->coalescing == 0)) in check_pkt_coalesce()
279 else if (*fill == 0 && (txq->in_use >= cxgb_tx_coalesce_enable_start)) in check_pkt_coalesce()
282 return (sc->tunq_coalesce); in check_pkt_coalesce()
297 wrp->wrh_hilo = wr_hilo; in set_wr_hdr()
304 wrp->wrh_hi = wr_hi; in set_wr_hdr()
306 wrp->wrh_lo = wr_lo; in set_wr_hdr()
321 if ((m->m_next != NULL) || in coalesce_check()
322 ((mtod(m, vm_offset_t) & PAGE_MASK) + m->m_len > PAGE_SIZE)) in coalesce_check()
323 ci->noncoal = 1; in coalesce_check()
325 if ((ci->count == 0) || (ci->noncoal == 0 && (ci->count < 7) && in coalesce_check()
326 (ci->nbytes + m->m_len <= 10500))) { in coalesce_check()
327 ci->count++; in coalesce_check()
328 ci->nbytes += m->m_len; in coalesce_check()
351 m_tail->m_nextpkt = m; in cxgb_dequeue()
361 * reclaim_completed_tx - reclaims completed Tx descriptors
363 * @q: the Tx queue to reclaim completed descriptors from
365 * Reclaims Tx descriptors that the SGE has indicated it has processed,
366 * and frees the associated buffers if possible. Called with the Tx
367 * queue's lock held.
370 reclaim_completed_tx(struct sge_qset *qs, int reclaim_min, int queue) in reclaim_completed_tx() argument
372 struct sge_txq *q = &qs->txq[queue]; in reclaim_completed_tx()
382 mtx_assert(&qs->lock, MA_OWNED); in reclaim_completed_tx()
384 t3_free_tx_desc(qs, reclaim, queue); in reclaim_completed_tx()
385 q->cleaned += reclaim; in reclaim_completed_tx()
386 q->in_use -= reclaim; in reclaim_completed_tx()
388 if (isset(&qs->txq_stopped, TXQ_ETH)) in reclaim_completed_tx()
389 clrbit(&qs->txq_stopped, TXQ_ETH); in reclaim_completed_tx()
404 * should_restart_tx - are there enough resources to restart a Tx queue?
405 * @q: the Tx queue
407 * Checks if there are enough descriptors to restart a suspended Tx queue.
412 unsigned int r = q->processed - q->cleaned; in should_restart_tx()
414 return q->in_use - r < (q->size >> 1); in should_restart_tx()
418 * t3_sge_init - initialize SGE
423 * We do not initialize any of the queue sets here, instead the driver
424 * top-level must request those individually. We also do not enable DMA
432 ups = 0; /* = ffs(pci_resource_len(adap->pdev, 2) >> 12); */ in t3_sge_init()
434 ctrl = F_DROPPKT | V_PKTSHIFT(2) | F_FLMODE | F_AVOIDCQOVFL | in t3_sge_init()
436 V_HOSTPAGESIZE(PAGE_SHIFT - 11) | F_BIGENDIANINGRESS | in t3_sge_init()
437 V_USERSPACESIZE(ups ? ups - 1 : 0) | F_ISCSICOALESCING; in t3_sge_init()
441 if (adap->params.rev > 0) { in t3_sge_init()
442 if (!(adap->flags & (USING_MSIX | USING_MSI))) in t3_sge_init()
452 adap->params.rev < T3_REV_C ? 1000 : 500); in t3_sge_init()
462 * sgl_len - calculates the size of an SGL of the given capacity
471 return ((3 * n) / 2 + (n & 1)); in sgl_len()
475 * get_imm_packet - return the next ingress packet buffer from a response
484 if (resp->rss_hdr.opcode == CPL_RX_DATA) { in get_imm_packet()
485 const struct cpl_rx_data *cpl = (const void *)&resp->imm_data[0]; in get_imm_packet()
486 m->m_len = sizeof(*cpl) + ntohs(cpl->len); in get_imm_packet()
487 } else if (resp->rss_hdr.opcode == CPL_RX_PKT) { in get_imm_packet()
488 const struct cpl_rx_pkt *cpl = (const void *)&resp->imm_data[0]; in get_imm_packet()
489 m->m_len = sizeof(*cpl) + ntohs(cpl->len); in get_imm_packet()
491 m->m_len = IMMED_PKT_SIZE; in get_imm_packet()
492 m->m_ext.ext_buf = NULL; in get_imm_packet()
493 m->m_ext.ext_type = 0; in get_imm_packet()
494 memcpy(mtod(m, uint8_t *), resp->imm_data, m->m_len); in get_imm_packet()
514 * t3_sge_err_intr_handler - SGE async event interrupt handler
517 * Interrupt handler for SGE asynchronous (non-data) events.
532 CH_ALERT(adapter, "SGE response queue credit overflow\n"); in t3_sge_err_intr_handler()
538 "packet delivered to disabled response queue (0x%x)\n", in t3_sge_err_intr_handler()
552 nqsets = min(SGE_QSETS / adap->params.nports, mp_ncpus); in t3_sge_prep()
553 nqsets *= adap->params.nports; in t3_sge_prep()
558 use_16k = cxgb_use_16k_clusters != -1 ? cxgb_use_16k_clusters : in t3_sge_prep()
570 if (fl_q_size < (FL_Q_SIZE / 4) || jumbo_q_size < (JUMBO_Q_SIZE / 2)) in t3_sge_prep()
571 device_printf(adap->dev, in t3_sge_prep()
574 p->max_pkt_size = jumbo_buf_size - sizeof(struct cpl_rx_data); in t3_sge_prep()
577 struct qset_params *q = p->qset + i; in t3_sge_prep()
579 if (adap->params.nports > 2) { in t3_sge_prep()
580 q->coalesce_usecs = 50; in t3_sge_prep()
583 q->coalesce_usecs = 10; in t3_sge_prep()
585 q->coalesce_usecs = 5; in t3_sge_prep()
588 q->polling = 0; in t3_sge_prep()
589 q->rspq_size = RSPQ_Q_SIZE; in t3_sge_prep()
590 q->fl_size = fl_q_size; in t3_sge_prep()
591 q->jumbo_size = jumbo_q_size; in t3_sge_prep()
592 q->jumbo_buf_size = jumbo_buf_size; in t3_sge_prep()
593 q->txq_size[TXQ_ETH] = TX_ETH_Q_SIZE; in t3_sge_prep()
594 q->txq_size[TXQ_OFLD] = is_offload(adap) ? TX_OFLD_Q_SIZE : 16; in t3_sge_prep()
595 q->txq_size[TXQ_CTRL] = TX_CTRL_Q_SIZE; in t3_sge_prep()
596 q->cong_thres = 0; in t3_sge_prep()
605 if (bus_dma_tag_create( bus_get_dma_tag(sc->dev),/* PCI parent */ in t3_sge_alloc()
615 &sc->parent_dmat)) { in t3_sge_alloc()
616 device_printf(sc->dev, "Cannot allocate parent DMA tag\n"); in t3_sge_alloc()
623 if (bus_dma_tag_create(sc->parent_dmat, MCLBYTES, 0, BUS_SPACE_MAXADDR, in t3_sge_alloc()
625 MCLBYTES, BUS_DMA_ALLOCNOW, NULL, NULL, &sc->rx_dmat)) { in t3_sge_alloc()
626 device_printf(sc->dev, "Cannot allocate RX DMA tag\n"); in t3_sge_alloc()
633 if (bus_dma_tag_create(sc->parent_dmat, MJUM16BYTES, 0, BUS_SPACE_MAXADDR, in t3_sge_alloc()
635 BUS_DMA_ALLOCNOW, NULL, NULL, &sc->rx_jumbo_dmat)) { in t3_sge_alloc()
636 device_printf(sc->dev, "Cannot allocate RX jumbo DMA tag\n"); in t3_sge_alloc()
641 * DMA tag for TX frames. in t3_sge_alloc()
643 if (bus_dma_tag_create(sc->parent_dmat, 1, 0, BUS_SPACE_MAXADDR, in t3_sge_alloc()
646 NULL, NULL, &sc->tx_dmat)) { in t3_sge_alloc()
647 device_printf(sc->dev, "Cannot allocate TX DMA tag\n"); in t3_sge_alloc()
658 if (sc->tx_dmat != NULL) in t3_sge_free()
659 bus_dma_tag_destroy(sc->tx_dmat); in t3_sge_free()
661 if (sc->rx_jumbo_dmat != NULL) in t3_sge_free()
662 bus_dma_tag_destroy(sc->rx_jumbo_dmat); in t3_sge_free()
664 if (sc->rx_dmat != NULL) in t3_sge_free()
665 bus_dma_tag_destroy(sc->rx_dmat); in t3_sge_free()
667 if (sc->parent_dmat != NULL) in t3_sge_free()
668 bus_dma_tag_destroy(sc->parent_dmat); in t3_sge_free()
677 qs->rspq.holdoff_tmr = max(p->coalesce_usecs * 10, 1U); in t3_update_qset_coalesce()
678 qs->rspq.polling = 0 /* p->polling */; in t3_update_qset_coalesce()
687 cb_arg->error = error; in refill_fl_cb()
688 cb_arg->seg = segs[0]; in refill_fl_cb()
689 cb_arg->nseg = nseg; in refill_fl_cb()
694 * refill_fl - refill an SGE free-buffer list
696 * @q: the free-list to refill
699 * (Re)populate an SGE free-buffer list with up to @n new packet buffers.
700 * The caller must assure that @n does not exceed the queue's capacity.
705 struct rx_sw_desc *sd = &q->sdesc[q->pidx]; in refill_fl()
706 struct rx_desc *d = &q->desc[q->pidx]; in refill_fl()
713 while (n--) { in refill_fl()
718 if (q->zone == zone_pack) { in refill_fl()
721 cl = m->m_ext.ext_buf; in refill_fl()
723 if ((cl = m_cljget(NULL, M_NOWAIT, q->buf_size)) == NULL) in refill_fl()
726 uma_zfree(q->zone, cl); in refill_fl()
730 if ((sd->flags & RX_SW_DESC_MAP_CREATED) == 0) { in refill_fl()
731 if ((err = bus_dmamap_create(q->entry_tag, 0, &sd->map))) { in refill_fl()
733 uma_zfree(q->zone, cl); in refill_fl()
736 sd->flags |= RX_SW_DESC_MAP_CREATED; in refill_fl()
739 err = bus_dmamap_load(q->entry_tag, sd->map, in refill_fl()
740 cl, q->buf_size, refill_fl_cb, &cb_arg, 0); in refill_fl()
743 if (q->zone != zone_pack) in refill_fl()
744 uma_zfree(q->zone, cl); in refill_fl()
751 sd->flags |= RX_SW_DESC_INUSE; in refill_fl()
752 sd->rxsd_cl = cl; in refill_fl()
753 sd->m = m; in refill_fl()
754 d->addr_lo = htobe32(cb_arg.seg.ds_addr & 0xffffffff); in refill_fl()
755 d->addr_hi = htobe32(((uint64_t)cb_arg.seg.ds_addr >>32) & 0xffffffff); in refill_fl()
756 d->len_gen = htobe32(V_FLD_GEN1(q->gen)); in refill_fl()
757 d->gen2 = htobe32(V_FLD_GEN2(q->gen)); in refill_fl()
762 if (++q->pidx == q->size) { in refill_fl()
763 q->pidx = 0; in refill_fl()
764 q->gen ^= 1; in refill_fl()
765 sd = q->sdesc; in refill_fl()
766 d = q->desc; in refill_fl()
768 q->credits++; in refill_fl()
769 q->db_pending++; in refill_fl()
773 if (q->db_pending >= 32) { in refill_fl()
774 q->db_pending = 0; in refill_fl()
775 t3_write_reg(sc, A_SG_KDOORBELL, V_EGRCNTX(q->cntxt_id)); in refill_fl()
781 * free_rx_bufs - free the Rx buffers on an SGE free list
785 * Release the buffers on an SGE free-buffer Rx queue. HW fetching from
786 * this queue should be stopped before calling this function.
791 u_int cidx = q->cidx; in free_rx_bufs()
793 while (q->credits--) { in free_rx_bufs()
794 struct rx_sw_desc *d = &q->sdesc[cidx]; in free_rx_bufs()
796 if (d->flags & RX_SW_DESC_INUSE) { in free_rx_bufs()
797 bus_dmamap_unload(q->entry_tag, d->map); in free_rx_bufs()
798 bus_dmamap_destroy(q->entry_tag, d->map); in free_rx_bufs()
799 if (q->zone == zone_pack) { in free_rx_bufs()
800 m_init(d->m, M_NOWAIT, MT_DATA, M_EXT); in free_rx_bufs()
801 uma_zfree(zone_pack, d->m); in free_rx_bufs()
803 m_init(d->m, M_NOWAIT, MT_DATA, 0); in free_rx_bufs()
804 m_free_raw(d->m); in free_rx_bufs()
805 uma_zfree(q->zone, d->rxsd_cl); in free_rx_bufs()
809 d->rxsd_cl = NULL; in free_rx_bufs()
810 d->m = NULL; in free_rx_bufs()
811 if (++cidx == q->size) in free_rx_bufs()
819 refill_fl(adap, fl, min(16U, fl->size - fl->credits)); in __refill_fl()
825 uint32_t reclaimable = fl->size - fl->credits; in __refill_fl_lt()
832 * recycle_rx_buf - recycle a receive buffer
843 struct rx_desc *from = &q->desc[idx]; in recycle_rx_buf()
844 struct rx_desc *to = &q->desc[q->pidx]; in recycle_rx_buf()
846 q->sdesc[q->pidx] = q->sdesc[idx]; in recycle_rx_buf()
847 to->addr_lo = from->addr_lo; // already big endian in recycle_rx_buf()
848 to->addr_hi = from->addr_hi; // likewise in recycle_rx_buf()
850 to->len_gen = htobe32(V_FLD_GEN1(q->gen)); in recycle_rx_buf()
851 to->gen2 = htobe32(V_FLD_GEN2(q->gen)); in recycle_rx_buf()
852 q->credits++; in recycle_rx_buf()
854 if (++q->pidx == q->size) { in recycle_rx_buf()
855 q->pidx = 0; in recycle_rx_buf()
856 q->gen ^= 1; in recycle_rx_buf()
858 t3_write_reg(adap, A_SG_KDOORBELL, V_EGRCNTX(q->cntxt_id)); in recycle_rx_buf()
880 if ((err = bus_dma_tag_create(sc->parent_dmat, PAGE_SIZE, 0, in alloc_ring()
884 device_printf(sc->dev, "Cannot allocate descriptor tag\n"); in alloc_ring()
890 device_printf(sc->dev, "Cannot allocate descriptor memory\n"); in alloc_ring()
911 device_printf(sc->dev, "Cannot allocate descriptor entry tag\n"); in alloc_ring()
923 t3_write_reg(sc, A_PL_INT_ENABLE0, sc->slow_intr_mask); in sge_slow_intr_handler()
928 * sge_timer_cb - perform periodic maintenance of an SGE qset
929 * @data: the SGE queue set to maintain
931 * Runs periodically from a timer to perform maintenance of an SGE queue
934 * a) Cleans up any completed Tx descriptors that may still be pending.
935 * Normal descriptor cleanup happens when new packets are added to a Tx
936 * queue so this timer is relatively infrequent and does any cleanup only
937 * if the Tx queue has not seen any new packets in a while. We make a
939 * around if we cannot get a queue's lock (which most likely is because
946 * when out of memory a queue can become empty. We try to add only a few
947 * buffers here, the queue will be replenished fully as these new buffers
950 * c) Return coalesced response queue credits in case a response queue is
960 if ((sc->flags & USING_MSIX) == 0) { in sge_timer_cb()
968 if (sc->open_device_map == 0) in sge_timer_cb()
971 for (i = 0; i < sc->params.nports; i++) { in sge_timer_cb()
972 pi = &sc->port[i]; in sge_timer_cb()
973 for (j = 0; j < pi->nqsets; j++) { in sge_timer_cb()
974 qs = &sc->sge.qs[pi->first_qset + j]; in sge_timer_cb()
975 txq = &qs->txq[0]; in sge_timer_cb()
976 reclaim_ofl = txq[TXQ_OFLD].processed - txq[TXQ_OFLD].cleaned; in sge_timer_cb()
977 refill_rx = ((qs->fl[0].credits < qs->fl[0].size) || in sge_timer_cb()
978 (qs->fl[1].credits < qs->fl[1].size)); in sge_timer_cb()
980 taskqueue_enqueue(sc->tq, &pi->timer_reclaim_task); in sge_timer_cb()
987 if (sc->params.nports > 2) { in sge_timer_cb()
991 struct port_info *pi = &sc->port[i]; in sge_timer_cb()
995 (FW_TUNNEL_SGEEC_START + pi->first_qset)); in sge_timer_cb()
998 if (((sc->flags & USING_MSIX) == 0 || sc->params.nports > 2) && in sge_timer_cb()
999 sc->open_device_map != 0) in sge_timer_cb()
1000 callout_reset(&sc->sge_timer_ch, TX_RECLAIM_PERIOD, sge_timer_cb, sc); in sge_timer_cb()
1004 * This is meant to be a catch-all function to keep sge state private
1011 callout_init(&sc->sge_timer_ch, 1); in t3_sge_init_adapter()
1012 callout_reset(&sc->sge_timer_ch, TX_RECLAIM_PERIOD, sge_timer_cb, sc); in t3_sge_init_adapter()
1013 TASK_INIT(&sc->slow_intr_task, 0, sge_slow_intr_handler, sc); in t3_sge_init_adapter()
1020 callout_reset(&sc->sge_timer_ch, TX_RECLAIM_PERIOD, sge_timer_cb, sc); in t3_sge_reset_adapter()
1027 TASK_INIT(&pi->timer_reclaim_task, 0, sge_timer_reclaim, pi); in t3_sge_init_port()
1032 * refill_rspq - replenish an SGE response queue
1034 * @q: the response queue to replenish
1037 * Replenishes a response queue by making the supplied number of responses
1046 V_RSPQ(q->cntxt_id) | V_CREDITS(credits)); in refill_rspq()
1063 int i, nqsets = pi->nqsets; in sge_timer_reclaim()
1064 adapter_t *sc = pi->adapter; in sge_timer_reclaim()
1068 KASSERT((sc->flags & USING_MSIX) == 0, in sge_timer_reclaim()
1069 ("can't call timer reclaim for msi-x")); in sge_timer_reclaim()
1072 qs = &sc->sge.qs[pi->first_qset + i]; in sge_timer_reclaim()
1075 lock = (sc->flags & USING_MSIX) ? &qs->rspq.lock : in sge_timer_reclaim()
1076 &sc->sge.qs[0].rspq.lock; in sge_timer_reclaim()
1082 if (qs->fl[0].credits < qs->fl[0].size - 16) in sge_timer_reclaim()
1083 __refill_fl(sc, &qs->fl[0]); in sge_timer_reclaim()
1084 if (qs->fl[1].credits < qs->fl[1].size - 16) in sge_timer_reclaim()
1085 __refill_fl(sc, &qs->fl[1]); in sge_timer_reclaim()
1087 if (status & (1 << qs->rspq.cntxt_id)) { in sge_timer_reclaim()
1088 if (qs->rspq.credits) { in sge_timer_reclaim()
1089 refill_rspq(sc, &qs->rspq, 1); in sge_timer_reclaim()
1090 qs->rspq.credits--; in sge_timer_reclaim()
1092 1 << qs->rspq.cntxt_id); in sge_timer_reclaim()
1101 * init_qset_cntxt - initialize an SGE queue set context info
1102 * @qs: the queue set
1103 * @id: the queue set id
1105 * Initializes the TIDs and context ids for the queues of a queue set.
1111 qs->rspq.cntxt_id = id; in init_qset_cntxt()
1112 qs->fl[0].cntxt_id = 2 * id; in init_qset_cntxt()
1113 qs->fl[1].cntxt_id = 2 * id + 1; in init_qset_cntxt()
1114 qs->txq[TXQ_ETH].cntxt_id = FW_TUNNEL_SGEEC_START + id; in init_qset_cntxt()
1115 qs->txq[TXQ_ETH].token = FW_TUNNEL_TID_START + id; in init_qset_cntxt()
1116 qs->txq[TXQ_OFLD].cntxt_id = FW_OFLD_SGEEC_START + id; in init_qset_cntxt()
1117 qs->txq[TXQ_CTRL].cntxt_id = FW_CTRL_SGEEC_START + id; in init_qset_cntxt()
1118 qs->txq[TXQ_CTRL].token = FW_CTRL_TID_START + id; in init_qset_cntxt()
1121 mbufq_init(&qs->txq[TXQ_ETH].sendq, INT_MAX); in init_qset_cntxt()
1122 mbufq_init(&qs->txq[TXQ_OFLD].sendq, INT_MAX); in init_qset_cntxt()
1123 mbufq_init(&qs->txq[TXQ_CTRL].sendq, INT_MAX); in init_qset_cntxt()
1130 txq->in_use += ndesc; in txq_prod()
1132 * XXX we don't handle stopping of queue in txq_prod()
1135 txqs->gen = txq->gen; in txq_prod()
1136 txq->unacked += ndesc; in txq_prod()
1137 txqs->compl = (txq->unacked & 32) << (S_WR_COMPL - 5); in txq_prod()
1138 txq->unacked &= 31; in txq_prod()
1139 txqs->pidx = txq->pidx; in txq_prod()
1140 txq->pidx += ndesc; in txq_prod()
1142 if (((txqs->pidx > txq->cidx) && in txq_prod()
1143 (txq->pidx < txqs->pidx) && in txq_prod()
1144 (txq->pidx >= txq->cidx)) || in txq_prod()
1145 ((txqs->pidx < txq->cidx) && in txq_prod()
1146 (txq->pidx >= txq-> cidx)) || in txq_prod()
1147 ((txqs->pidx < txq->cidx) && in txq_prod()
1148 (txq->cidx < txqs->pidx))) in txq_prod()
1149 panic("txqs->pidx=%d txq->pidx=%d txq->cidx=%d", in txq_prod()
1150 txqs->pidx, txq->pidx, txq->cidx); in txq_prod()
1152 if (txq->pidx >= txq->size) { in txq_prod()
1153 txq->pidx -= txq->size; in txq_prod()
1154 txq->gen ^= 1; in txq_prod()
1160 * calc_tx_descs - calculate the number of Tx descriptors for a packet
1164 * Returns the number of Tx descriptors needed for the given Ethernet
1172 if (m->m_pkthdr.len <= PIO_LEN) in calc_tx_descs()
1175 flits = sgl_len(nsegs) + 2; in calc_tx_descs()
1176 if (m->m_pkthdr.csum_flags & CSUM_TSO) in calc_tx_descs()
1183 * make_sgl - populate a scatter/gather list for a packet
1189 * and returns the SGL size in 8-byte words. The caller must size the SGL
1206 sgp->len[idx] = htobe32(segs[i].ds_len); in make_sgl()
1207 sgp->addr[idx] = htobe64(segs[i].ds_addr); in make_sgl()
1212 sgp->len[idx] = 0; in make_sgl()
1213 sgp->addr[idx] = 0; in make_sgl()
1218 * check_ring_tx_db - check and potentially ring a Tx queue's doorbell
1220 * @q: the Tx queue
1222 * Ring the doorbell if a Tx queue is asleep. There is a natural race,
1224 * then the interrupt handler will detect the outstanding TX packet
1233 clear_bit(TXQ_LAST_PKT_DB, &q->flags); in check_ring_tx_db()
1234 if (test_and_set_bit(TXQ_RUNNING, &q->flags) == 0) { in check_ring_tx_db()
1235 set_bit(TXQ_LAST_PKT_DB, &q->flags); in check_ring_tx_db()
1237 T3_TRACE1(adap->tb[q->cntxt_id & 7], "doorbell Tx, cntxt %d", in check_ring_tx_db()
1238 q->cntxt_id); in check_ring_tx_db()
1241 F_SELEGRCNTX | V_EGRCNTX(q->cntxt_id)); in check_ring_tx_db()
1244 if (mustring || ++q->db_pending >= 32) { in check_ring_tx_db()
1247 F_SELEGRCNTX | V_EGRCNTX(q->cntxt_id)); in check_ring_tx_db()
1248 q->db_pending = 0; in check_ring_tx_db()
1256 #if SGE_NUM_GENBITS == 2 in wr_gen2()
1257 d->flit[TX_DESC_FLITS - 1] = htobe64(gen); in wr_gen2()
1262 * write_wr_hdr_sgl - write a WR header and, optionally, SGL
1263 * @ndesc: number of Tx descriptors spanned by the SGL
1264 * @txd: first Tx descriptor to be written
1266 * @txq: the SGE Tx queue
1274 * small enough to fit into one Tx descriptor it has already been written
1289 htonl(V_WR_LEN(flits + sgl_flits) | V_WR_GEN(txqs->gen)) | in write_wr_hdr_sgl()
1292 wr_gen2(txd, txqs->gen); in write_wr_hdr_sgl()
1295 unsigned int ogen = txqs->gen; in write_wr_hdr_sgl()
1299 wrp->wrh_hi = htonl(F_WR_SOP | V_WR_DATATYPE(1) | in write_wr_hdr_sgl()
1303 unsigned int avail = WR_FLITS - flits; in write_wr_hdr_sgl()
1307 memcpy(&txd->flit[flits], fp, avail * sizeof(*fp)); in write_wr_hdr_sgl()
1308 sgl_flits -= avail; in write_wr_hdr_sgl()
1309 ndesc--; in write_wr_hdr_sgl()
1315 if (++txqs->pidx == txq->size) { in write_wr_hdr_sgl()
1316 txqs->pidx = 0; in write_wr_hdr_sgl()
1317 txqs->gen ^= 1; in write_wr_hdr_sgl()
1318 txd = txq->desc; in write_wr_hdr_sgl()
1327 wrp->wrh_hi = htonl(V_WR_DATATYPE(1) | in write_wr_hdr_sgl()
1329 wrp->wrh_lo = htonl(V_WR_LEN(min(WR_FLITS, in write_wr_hdr_sgl()
1331 V_WR_GEN(txqs->gen)) | wr_lo; in write_wr_hdr_sgl()
1332 wr_gen2(txd, txqs->gen); in write_wr_hdr_sgl()
1335 wrp->wrh_hi |= htonl(F_WR_EOP); in write_wr_hdr_sgl()
1337 wp->wrh_lo = htonl(V_WR_LEN(WR_FLITS) | V_WR_GEN(ogen)) | wr_lo; in write_wr_hdr_sgl()
1347 if ((m)->m_flags & M_VLANTAG) \
1348 cntrl |= F_TXPKT_VLAN_VLD | V_TXPKT_VLAN((m)->m_pkthdr.ether_vtag); \
1370 pi = qs->port; in t3_encap()
1371 sc = pi->adapter; in t3_encap()
1372 txq = &qs->txq[TXQ_ETH]; in t3_encap()
1373 txd = &txq->desc[txq->pidx]; in t3_encap()
1374 txsd = &txq->sdesc[txq->pidx]; in t3_encap()
1375 sgl = txq->txq_sgl; in t3_encap()
1380 mtx_assert(&qs->lock, MA_OWNED); in t3_encap()
1381 cntrl = V_TXPKT_INTF(pi->txpkt_intf); in t3_encap()
1382 KASSERT(m0->m_flags & M_PKTHDR, ("not packet header\n")); in t3_encap()
1384 if (m0->m_nextpkt == NULL && m0->m_next != NULL && in t3_encap()
1385 m0->m_pkthdr.csum_flags & (CSUM_TSO)) in t3_encap()
1386 tso_info = V_LSO_MSS(m0->m_pkthdr.tso_segsz); in t3_encap()
1388 if (m0->m_nextpkt != NULL) { in t3_encap()
1389 busdma_map_sg_vec(txq->entry_tag, txsd->map, m0, segs, &nsegs); in t3_encap()
1393 if ((err = busdma_map_sg_collapse(txq->entry_tag, txsd->map, in t3_encap()
1399 mlen = m0->m_pkthdr.len; in t3_encap()
1404 KASSERT(m0->m_pkthdr.len, ("empty packet nsegs=%d", nsegs)); in t3_encap()
1405 txsd->m = m0; in t3_encap()
1407 if (m0->m_nextpkt != NULL) { in t3_encap()
1413 txq->txq_coalesced += nsegs; in t3_encap()
1415 flits = nsegs*2 + 1; in t3_encap()
1417 for (fidx = 1, i = 0; i < nsegs; i++, fidx += 2) { in t3_encap()
1421 int cflags = m0->m_pkthdr.csum_flags; in t3_encap()
1423 cntrl = V_TXPKT_INTF(pi->txpkt_intf); in t3_encap()
1435 cbe = &cpl_batch->pkt_entry[i]; in t3_encap()
1436 cbe->cntrl = hflit[0]; in t3_encap()
1437 cbe->len = hflit[1]; in t3_encap()
1438 cbe->addr = htobe64(segs[i].ds_addr); in t3_encap()
1445 V_WR_GEN(txqs.gen)) | htonl(V_WR_TID(txq->token)); in t3_encap()
1448 ETHER_BPF_MTAP(pi->ifp, m0); in t3_encap()
1459 txd->flit[2] = 0; in t3_encap()
1462 hdr->cntrl = htonl(cntrl); in t3_encap()
1463 hdr->len = htonl(mlen | 0x80000000); in t3_encap()
1467 m0, mlen, m0->m_pkthdr.tso_segsz, in t3_encap()
1468 (int)m0->m_pkthdr.csum_flags, CSUM_BITS, m0->m_flags); in t3_encap()
1469 panic("tx tso packet too small"); in t3_encap()
1473 if (__predict_false(m0->m_len < TCPPKTHDRSIZE)) { in t3_encap()
1482 eth_type = eh->ether_type; in t3_encap()
1488 eth_type = evh->evl_proto; in t3_encap()
1497 tso_info |= V_LSO_IPHDR_WORDS(ip->ip_hl); in t3_encap()
1502 KASSERT(ip6->ip6_nxt == IPPROTO_TCP, in t3_encap()
1504 __func__, ip6->ip6_nxt)); in t3_encap()
1507 tso_info |= V_LSO_IPHDR_WORDS(sizeof(*ip6) >> 2); in t3_encap()
1512 tso_info |= V_LSO_TCPHDR_WORDS(tcp->th_off); in t3_encap()
1513 hdr->lso_info = htonl(tso_info); in t3_encap()
1520 txsd->m = NULL; in t3_encap()
1521 m_copydata(m0, 0, mlen, (caddr_t)&txd->flit[3]); in t3_encap()
1527 V_WR_GEN(txqs.gen) | V_WR_TID(txq->token)); in t3_encap()
1528 set_wr_hdr(&hdr->wr, wr_hi, wr_lo); in t3_encap()
1530 ETHER_BPF_MTAP(pi->ifp, m0); in t3_encap()
1542 if (__predict_false(!(m0->m_pkthdr.csum_flags & CSUM_IP))) in t3_encap()
1544 if (__predict_false(!(m0->m_pkthdr.csum_flags & (CSUM_TCP | in t3_encap()
1547 cpl->cntrl = htonl(cntrl); in t3_encap()
1548 cpl->len = htonl(mlen | 0x80000000); in t3_encap()
1551 txsd->m = NULL; in t3_encap()
1552 m_copydata(m0, 0, mlen, (caddr_t)&txd->flit[2]); in t3_encap()
1553 flits = (mlen + 7) / 8 + 2; in t3_encap()
1559 V_WR_GEN(txqs.gen) | V_WR_TID(txq->token)); in t3_encap()
1560 set_wr_hdr(&cpl->wr, wr_hi, wr_lo); in t3_encap()
1562 ETHER_BPF_MTAP(pi->ifp, m0); in t3_encap()
1568 flits = 2; in t3_encap()
1571 sgp = (ndesc == 1) ? (struct sg_ent *)&txd->flit[flits] : sgl; in t3_encap()
1576 ETHER_BPF_MTAP(pi->ifp, m0); in t3_encap()
1580 wr_lo = htonl(V_WR_TID(txq->token)); in t3_encap()
1596 check_ring_tx_db(qs->port->adapter, &qs->txq[TXQ_ETH], 1); in cxgb_debugnet_encap()
1609 struct sge_txq *txq = &qs->txq[TXQ_ETH]; in cxgb_tx_watchdog()
1611 if (qs->coalescing != 0 && in cxgb_tx_watchdog()
1612 (txq->in_use <= cxgb_tx_coalesce_enable_stop) && in cxgb_tx_watchdog()
1614 qs->coalescing = 0; in cxgb_tx_watchdog()
1615 else if (qs->coalescing == 0 && in cxgb_tx_watchdog()
1616 (txq->in_use >= cxgb_tx_coalesce_enable_start)) in cxgb_tx_watchdog()
1617 qs->coalescing = 1; in cxgb_tx_watchdog()
1619 qs->qs_flags |= QS_FLUSHING; in cxgb_tx_watchdog()
1621 qs->qs_flags &= ~QS_FLUSHING; in cxgb_tx_watchdog()
1624 if (if_getdrvflags(qs->port->ifp) & IFF_DRV_RUNNING) in cxgb_tx_watchdog()
1625 callout_reset_on(&txq->txq_watchdog, hz/4, cxgb_tx_watchdog, in cxgb_tx_watchdog()
1626 qs, txq->txq_watchdog.c_cpu); in cxgb_tx_watchdog()
1633 struct sge_txq *txq = &qs->txq[TXQ_ETH]; in cxgb_tx_timeout()
1635 if (qs->coalescing == 0 && (txq->in_use >= (txq->size>>3))) in cxgb_tx_timeout()
1636 qs->coalescing = 1; in cxgb_tx_timeout()
1638 qs->qs_flags |= QS_TIMEOUT; in cxgb_tx_timeout()
1640 qs->qs_flags &= ~QS_TIMEOUT; in cxgb_tx_timeout()
1649 struct sge_txq *txq = &qs->txq[TXQ_ETH]; in cxgb_start_locked()
1650 struct port_info *pi = qs->port; in cxgb_start_locked()
1651 if_t ifp = pi->ifp; in cxgb_start_locked()
1653 if (qs->qs_flags & (QS_FLUSHING|QS_TIMEOUT)) in cxgb_start_locked()
1656 if (!pi->link_config.link_ok) { in cxgb_start_locked()
1662 pi->link_config.link_ok) { in cxgb_start_locked()
1665 if (txq->size - txq->in_use <= TX_MAX_DESC) in cxgb_start_locked()
1680 if (txq->db_pending) in cxgb_start_locked()
1681 check_ring_tx_db(pi->adapter, txq, 1); in cxgb_start_locked()
1683 if (!TXQ_RING_EMPTY(qs) && callout_pending(&txq->txq_timer) == 0 && in cxgb_start_locked()
1684 pi->link_config.link_ok) in cxgb_start_locked()
1685 callout_reset_on(&txq->txq_timer, 1, cxgb_tx_timeout, in cxgb_start_locked()
1686 qs, txq->txq_timer.c_cpu); in cxgb_start_locked()
1694 struct port_info *pi = qs->port; in cxgb_transmit_locked()
1695 struct sge_txq *txq = &qs->txq[TXQ_ETH]; in cxgb_transmit_locked()
1696 struct buf_ring *br = txq->txq_mr; in cxgb_transmit_locked()
1699 avail = txq->size - txq->in_use; in cxgb_transmit_locked()
1704 * - we aren't coalescing (ring < 3/4 full) in cxgb_transmit_locked()
1705 * - the link is up -- checked in caller in cxgb_transmit_locked()
1706 * - there are no packets enqueued already in cxgb_transmit_locked()
1707 * - there is space in hardware transmit queue in cxgb_transmit_locked()
1716 if (txq->db_pending) in cxgb_transmit_locked()
1717 check_ring_tx_db(pi->adapter, txq, 1); in cxgb_transmit_locked()
1723 txq->txq_direct_packets++; in cxgb_transmit_locked()
1724 txq->txq_direct_bytes += m->m_pkthdr.len; in cxgb_transmit_locked()
1730 if (!TXQ_RING_EMPTY(qs) && pi->link_config.link_ok && in cxgb_transmit_locked()
1733 else if (!TXQ_RING_EMPTY(qs) && !callout_pending(&txq->txq_timer)) in cxgb_transmit_locked()
1734 callout_reset_on(&txq->txq_timer, 1, cxgb_tx_timeout, in cxgb_transmit_locked()
1735 qs, txq->txq_timer.c_cpu); in cxgb_transmit_locked()
1744 int error, qidx = pi->first_qset; in cxgb_transmit()
1747 ||(!pi->link_config.link_ok)) { in cxgb_transmit()
1754 qidx = (m->m_pkthdr.flowid % pi->nqsets) + pi->first_qset; in cxgb_transmit()
1756 qs = &pi->adapter->sge.qs[qidx]; in cxgb_transmit()
1763 error = drbr_enqueue(ifp, qs->txq[TXQ_ETH].txq_mr, m); in cxgb_transmit()
1773 * no-op for now in cxgb_qflush()
1779 * write_imm - write a packet into a Tx descriptor as immediate data
1780 * @d: the Tx descriptor to write
1785 * Writes a packet as immediate data into a Tx descriptor. The packet
1801 memcpy(&to[1], &from[1], len - sizeof(*from)); in write_imm()
1802 wr_hi = from->wrh_hi | htonl(F_WR_SOP | F_WR_EOP | in write_imm()
1804 wr_lo = from->wrh_lo | htonl(V_WR_GEN(gen) | V_WR_LEN((len + 7) / 8)); in write_imm()
1811 * check_desc_avail - check descriptor availability on a send queue
1813 * @q: the TX queue
1815 * @ndesc: the number of Tx descriptors needed
1816 * @qid: the Tx queue number in its queue set (TXQ_OFLD or TXQ_CTRL)
1818 * Checks if the requested number of Tx descriptors is available on an
1819 * SGE send queue. If the queue is already suspended or not enough
1821 * Must be called with the Tx queue locked.
1824 * enough descriptors and the packet has been queued, and 2 if the caller
1834 * XXX We currently only use this for checking the control queue in check_desc_avail()
1835 * the control queue is only used for binding qsets which happens in check_desc_avail()
1838 if (__predict_false(!mbufq_empty(&q->sendq))) { in check_desc_avail()
1839 addq_exit: (void )mbufq_enqueue(&q->sendq, m); in check_desc_avail()
1842 if (__predict_false(q->size - q->in_use < ndesc)) { in check_desc_avail()
1846 setbit(&qs->txq_stopped, qid); in check_desc_avail()
1848 test_and_clear_bit(qid, &qs->txq_stopped)) in check_desc_avail()
1849 return 2; in check_desc_avail()
1851 q->stops++; in check_desc_avail()
1859 * reclaim_completed_tx_imm - reclaim completed control-queue Tx descs
1860 * @q: the SGE control Tx queue
1862 * This is a variant of reclaim_completed_tx() that is used for Tx queues
1869 unsigned int reclaim = q->processed - q->cleaned; in reclaim_completed_tx_imm()
1871 q->in_use -= reclaim; in reclaim_completed_tx_imm()
1872 q->cleaned += reclaim; in reclaim_completed_tx_imm()
1876 * ctrl_xmit - send a packet through an SGE control Tx queue
1878 * @q: the control queue
1881 * Send a packet through an SGE control Tx queue. Packets sent through
1882 * a control queue must fit entirely as immediate data in a single Tx
1890 struct sge_txq *q = &qs->txq[TXQ_CTRL]; in ctrl_xmit()
1892 KASSERT(m->m_len <= WR_LEN, ("%s: bad tx data", __func__)); in ctrl_xmit()
1894 wrp->wrh_hi |= htonl(F_WR_SOP | F_WR_EOP); in ctrl_xmit()
1895 wrp->wrh_lo = htonl(V_WR_TID(q->token)); in ctrl_xmit()
1908 write_imm(&q->desc[q->pidx], m->m_data, m->m_len, q->gen); in ctrl_xmit()
1910 q->in_use++; in ctrl_xmit()
1911 if (++q->pidx >= q->size) { in ctrl_xmit()
1912 q->pidx = 0; in ctrl_xmit()
1913 q->gen ^= 1; in ctrl_xmit()
1918 F_SELEGRCNTX | V_EGRCNTX(q->cntxt_id)); in ctrl_xmit()
1926 * restart_ctrlq - restart a suspended control queue
1927 * @qs: the queue set containing the control queue
1929 * Resumes transmission on a suspended Tx control queue.
1936 struct sge_txq *q = &qs->txq[TXQ_CTRL]; in restart_ctrlq()
1937 adapter_t *adap = qs->port->adapter; in restart_ctrlq()
1942 while (q->in_use < q->size && in restart_ctrlq()
1943 (m = mbufq_dequeue(&q->sendq)) != NULL) { in restart_ctrlq()
1945 write_imm(&q->desc[q->pidx], m->m_data, m->m_len, q->gen); in restart_ctrlq()
1948 if (++q->pidx >= q->size) { in restart_ctrlq()
1949 q->pidx = 0; in restart_ctrlq()
1950 q->gen ^= 1; in restart_ctrlq()
1952 q->in_use++; in restart_ctrlq()
1954 if (!mbufq_empty(&q->sendq)) { in restart_ctrlq()
1955 setbit(&qs->txq_stopped, TXQ_CTRL); in restart_ctrlq()
1958 test_and_clear_bit(TXQ_CTRL, &qs->txq_stopped)) in restart_ctrlq()
1960 q->stops++; in restart_ctrlq()
1964 F_SELEGRCNTX | V_EGRCNTX(q->cntxt_id)); in restart_ctrlq()
1969 * Send a management message through control queue 0
1974 return ctrl_xmit(adap, &adap->sge.qs[0], m); in t3_mgmt_tx()
1978 * free_qset - free the resources of an SGE queue set
1979 * @sc: the controller owning the queue set
1980 * @q: the queue set
1982 * Release the HW and SW resources associated with an SGE queue set, such
1984 * queue set must be quiesced prior to calling this.
1992 if (q->txq[TXQ_ETH].txq_mr != NULL) in t3_free_qset()
1993 buf_ring_free(q->txq[TXQ_ETH].txq_mr, M_DEVBUF); in t3_free_qset()
1994 if (q->txq[TXQ_ETH].txq_ifq != NULL) { in t3_free_qset()
1995 ifq_delete(q->txq[TXQ_ETH].txq_ifq); in t3_free_qset()
1996 free(q->txq[TXQ_ETH].txq_ifq, M_DEVBUF); in t3_free_qset()
2000 if (q->fl[i].desc) { in t3_free_qset()
2001 mtx_lock_spin(&sc->sge.reg_lock); in t3_free_qset()
2002 t3_sge_disable_fl(sc, q->fl[i].cntxt_id); in t3_free_qset()
2003 mtx_unlock_spin(&sc->sge.reg_lock); in t3_free_qset()
2004 bus_dmamap_unload(q->fl[i].desc_tag, q->fl[i].desc_map); in t3_free_qset()
2005 bus_dmamem_free(q->fl[i].desc_tag, q->fl[i].desc, in t3_free_qset()
2006 q->fl[i].desc_map); in t3_free_qset()
2007 bus_dma_tag_destroy(q->fl[i].desc_tag); in t3_free_qset()
2008 bus_dma_tag_destroy(q->fl[i].entry_tag); in t3_free_qset()
2010 if (q->fl[i].sdesc) { in t3_free_qset()
2011 free_rx_bufs(sc, &q->fl[i]); in t3_free_qset()
2012 free(q->fl[i].sdesc, M_DEVBUF); in t3_free_qset()
2016 mtx_unlock(&q->lock); in t3_free_qset()
2017 MTX_DESTROY(&q->lock); in t3_free_qset()
2019 if (q->txq[i].desc) { in t3_free_qset()
2020 mtx_lock_spin(&sc->sge.reg_lock); in t3_free_qset()
2021 t3_sge_enable_ecntxt(sc, q->txq[i].cntxt_id, 0); in t3_free_qset()
2022 mtx_unlock_spin(&sc->sge.reg_lock); in t3_free_qset()
2023 bus_dmamap_unload(q->txq[i].desc_tag, in t3_free_qset()
2024 q->txq[i].desc_map); in t3_free_qset()
2025 bus_dmamem_free(q->txq[i].desc_tag, q->txq[i].desc, in t3_free_qset()
2026 q->txq[i].desc_map); in t3_free_qset()
2027 bus_dma_tag_destroy(q->txq[i].desc_tag); in t3_free_qset()
2028 bus_dma_tag_destroy(q->txq[i].entry_tag); in t3_free_qset()
2030 if (q->txq[i].sdesc) { in t3_free_qset()
2031 free(q->txq[i].sdesc, M_DEVBUF); in t3_free_qset()
2035 if (q->rspq.desc) { in t3_free_qset()
2036 mtx_lock_spin(&sc->sge.reg_lock); in t3_free_qset()
2037 t3_sge_disable_rspcntxt(sc, q->rspq.cntxt_id); in t3_free_qset()
2038 mtx_unlock_spin(&sc->sge.reg_lock); in t3_free_qset()
2040 bus_dmamap_unload(q->rspq.desc_tag, q->rspq.desc_map); in t3_free_qset()
2041 bus_dmamem_free(q->rspq.desc_tag, q->rspq.desc, in t3_free_qset()
2042 q->rspq.desc_map); in t3_free_qset()
2043 bus_dma_tag_destroy(q->rspq.desc_tag); in t3_free_qset()
2044 MTX_DESTROY(&q->rspq.lock); in t3_free_qset()
2048 tcp_lro_free(&q->lro.ctrl); in t3_free_qset()
2055 * t3_free_sge_resources - free SGE resources
2058 * Frees resources used by the SGE queue sets.
2066 TXQ_LOCK(&sc->sge.qs[i]); in t3_free_sge_resources()
2067 t3_free_qset(sc, &sc->sge.qs[i]); in t3_free_sge_resources()
2072 * t3_sge_start - enable SGE
2085 * t3_sge_stop - disable SGE operation
2090 * case it also disables any pending queue restart tasklets. Note that
2105 * t3_free_tx_desc - reclaims Tx descriptors and their buffers
2107 * @q: the Tx queue to reclaim descriptors from
2112 * Reclaims Tx descriptors from an SGE Tx queue and frees the associated
2113 * Tx buffers. Called with the Tx queue lock held.
2118 t3_free_tx_desc(struct sge_qset *qs, int reclaimable, int queue) in t3_free_tx_desc() argument
2122 struct sge_txq *q = &qs->txq[queue]; in t3_free_tx_desc()
2125 T3_TRACE2(sc->tb[q->cntxt_id & 7], in t3_free_tx_desc()
2126 "reclaiming %u Tx descriptors at cidx %u", reclaimable, cidx); in t3_free_tx_desc()
2128 cidx = q->cidx; in t3_free_tx_desc()
2129 mask = q->size - 1; in t3_free_tx_desc()
2130 txsd = &q->sdesc[cidx]; in t3_free_tx_desc()
2132 mtx_assert(&qs->lock, MA_OWNED); in t3_free_tx_desc()
2133 while (reclaimable--) { in t3_free_tx_desc()
2134 prefetch(q->sdesc[(cidx + 1) & mask].m); in t3_free_tx_desc()
2135 prefetch(q->sdesc[(cidx + 2) & mask].m); in t3_free_tx_desc()
2137 if (txsd->m != NULL) { in t3_free_tx_desc()
2138 if (txsd->flags & TX_SW_DESC_MAPPED) { in t3_free_tx_desc()
2139 bus_dmamap_unload(q->entry_tag, txsd->map); in t3_free_tx_desc()
2140 txsd->flags &= ~TX_SW_DESC_MAPPED; in t3_free_tx_desc()
2142 m_freem_list(txsd->m); in t3_free_tx_desc()
2143 txsd->m = NULL; in t3_free_tx_desc()
2145 q->txq_skipped++; in t3_free_tx_desc()
2148 if (++cidx == q->size) { in t3_free_tx_desc()
2150 txsd = q->sdesc; in t3_free_tx_desc()
2153 q->cidx = cidx; in t3_free_tx_desc()
2158 * is_new_response - check if a response is newly written
2160 * @q: the response queue
2169 return (r->intr_gen & F_RSPD_GEN2) == q->gen; in is_new_response()
2183 * write_ofld_wr - write an offload work request
2186 * @q: the Tx queue
2187 * @pidx: index of the first Tx descriptor to write
2201 struct sg_ent *sgp, t3sgl[TX_MAX_SEGS / 2 + 1]; in write_ofld_wr()
2202 struct tx_desc *d = &q->desc[pidx]; in write_ofld_wr()
2209 wrlen = m->m_len - sizeof(*oh); in write_ofld_wr()
2211 if (!(oh->flags & F_HDR_SGL)) { in write_ofld_wr()
2215 * mbuf with "real" immediate tx data will be enqueue_wr'd by in write_ofld_wr()
2219 if (!(oh->flags & F_HDR_DF)) in write_ofld_wr()
2224 memcpy(&d->flit[1], &from[1], wrlen - sizeof(*from)); in write_ofld_wr()
2226 sgl = oh->sgl; in write_ofld_wr()
2228 sgp = (ndesc == 1) ? (struct sg_ent *)&d->flit[flits] : t3sgl; in write_ofld_wr()
2230 nsegs = sgl->sg_nseg; in write_ofld_wr()
2231 segs = sgl->sg_segs; in write_ofld_wr()
2236 sgp->len[idx] = htobe32(segs[i].ss_len); in write_ofld_wr()
2237 sgp->addr[idx] = htobe64(segs[i].ss_paddr); in write_ofld_wr()
2241 sgp->len[idx] = 0; in write_ofld_wr()
2242 sgp->addr[idx] = 0; in write_ofld_wr()
2251 from->wrh_hi, from->wrh_lo); in write_ofld_wr()
2255 * ofld_xmit - send a packet through an offload queue
2257 * @q: the Tx offload queue
2260 * Send an offload packet through an SGE offload queue.
2268 struct sge_txq *q = &qs->txq[TXQ_OFLD]; in ofld_xmit()
2271 ndesc = G_HDR_NDESC(oh->flags); in ofld_xmit()
2284 gen = q->gen; in ofld_xmit()
2285 q->in_use += ndesc; in ofld_xmit()
2286 pidx = q->pidx; in ofld_xmit()
2287 q->pidx += ndesc; in ofld_xmit()
2288 if (q->pidx >= q->size) { in ofld_xmit()
2289 q->pidx -= q->size; in ofld_xmit()
2290 q->gen ^= 1; in ofld_xmit()
2301 * restart_offloadq - restart a suspended offload queue
2302 * @qs: the queue set containing the offload queue
2304 * Resumes transmission on a suspended Tx offload queue.
2311 struct sge_txq *q = &qs->txq[TXQ_OFLD]; in restart_offloadq()
2312 adapter_t *adap = qs->port->adapter; in restart_offloadq()
2316 while ((m = mbufq_first(&q->sendq)) != NULL) { in restart_offloadq()
2319 unsigned int ndesc = G_HDR_NDESC(oh->flags); in restart_offloadq()
2321 if (__predict_false(q->size - q->in_use < ndesc)) { in restart_offloadq()
2322 setbit(&qs->txq_stopped, TXQ_OFLD); in restart_offloadq()
2324 test_and_clear_bit(TXQ_OFLD, &qs->txq_stopped)) in restart_offloadq()
2326 q->stops++; in restart_offloadq()
2330 gen = q->gen; in restart_offloadq()
2331 q->in_use += ndesc; in restart_offloadq()
2332 pidx = q->pidx; in restart_offloadq()
2333 q->pidx += ndesc; in restart_offloadq()
2334 if (q->pidx >= q->size) { in restart_offloadq()
2335 q->pidx -= q->size; in restart_offloadq()
2336 q->gen ^= 1; in restart_offloadq()
2339 (void)mbufq_dequeue(&q->sendq); in restart_offloadq()
2345 set_bit(TXQ_RUNNING, &q->flags); in restart_offloadq()
2346 set_bit(TXQ_LAST_PKT_DB, &q->flags); in restart_offloadq()
2351 F_SELEGRCNTX | V_EGRCNTX(q->cntxt_id)); in restart_offloadq()
2355 * t3_offload_tx - send an offload packet
2359 * appropriate Tx queue as follows: bit 0 indicates whether the packet
2360 * should be sent as regular or control, bits 1-3 select the queue set.
2366 struct sge_qset *qs = &sc->sge.qs[G_HDR_QSET(oh->flags)]; in t3_offload_tx()
2368 if (oh->flags & F_HDR_CTRL) { in t3_offload_tx()
2379 struct adapter *sc = qs->port->adapter; in restart_tx()
2381 if (isset(&qs->txq_stopped, TXQ_OFLD) && in restart_tx()
2382 should_restart_tx(&qs->txq[TXQ_OFLD]) && in restart_tx()
2383 test_and_clear_bit(TXQ_OFLD, &qs->txq_stopped)) { in restart_tx()
2384 qs->txq[TXQ_OFLD].restarts++; in restart_tx()
2385 taskqueue_enqueue(sc->tq, &qs->txq[TXQ_OFLD].qresume_task); in restart_tx()
2388 if (isset(&qs->txq_stopped, TXQ_CTRL) && in restart_tx()
2389 should_restart_tx(&qs->txq[TXQ_CTRL]) && in restart_tx()
2390 test_and_clear_bit(TXQ_CTRL, &qs->txq_stopped)) { in restart_tx()
2391 qs->txq[TXQ_CTRL].restarts++; in restart_tx()
2392 taskqueue_enqueue(sc->tq, &qs->txq[TXQ_CTRL].qresume_task); in restart_tx()
2397 * t3_sge_alloc_qset - initialize an SGE queue set
2399 * @id: the queue set id
2400 * @nports: how many Ethernet ports will be using this queue set
2401 * @irq_vec_idx: the IRQ vector index for response queue interrupts
2402 * @p: configuration parameters for this queue set
2403 * @ntxq: number of Tx queues for the queue set
2404 * @pi: port info for queue set
2406 * Allocate resources and initialize an SGE queue set. A queue set
2407 * comprises a response queue, two Rx free-buffer queues, and up to 3
2408 * Tx queues. The Tx queues are assigned roles in the order Ethernet
2409 * queue, offload queue, and control queue.
2415 struct sge_qset *q = &sc->sge.qs[id]; in t3_sge_alloc_qset()
2418 MTX_INIT(&q->lock, q->namebuf, NULL, MTX_DEF); in t3_sge_alloc_qset()
2419 q->port = pi; in t3_sge_alloc_qset()
2420 q->adap = sc; in t3_sge_alloc_qset()
2422 q->txq[TXQ_ETH].txq_mr = buf_ring_alloc(cxgb_txq_buf_ring_size, in t3_sge_alloc_qset()
2423 M_DEVBUF, M_WAITOK, &q->lock); in t3_sge_alloc_qset()
2424 if ((q->txq[TXQ_ETH].txq_ifq = malloc(sizeof(struct ifaltq), M_DEVBUF, in t3_sge_alloc_qset()
2426 device_printf(sc->dev, "failed to allocate ifq\n"); in t3_sge_alloc_qset()
2429 ifq_init(q->txq[TXQ_ETH].txq_ifq, pi->ifp); in t3_sge_alloc_qset()
2430 callout_init(&q->txq[TXQ_ETH].txq_timer, 1); in t3_sge_alloc_qset()
2431 callout_init(&q->txq[TXQ_ETH].txq_watchdog, 1); in t3_sge_alloc_qset()
2432 q->txq[TXQ_ETH].txq_timer.c_cpu = id % mp_ncpus; in t3_sge_alloc_qset()
2433 q->txq[TXQ_ETH].txq_watchdog.c_cpu = id % mp_ncpus; in t3_sge_alloc_qset()
2436 q->idx = id; in t3_sge_alloc_qset()
2437 if ((ret = alloc_ring(sc, p->fl_size, sizeof(struct rx_desc), in t3_sge_alloc_qset()
2438 sizeof(struct rx_sw_desc), &q->fl[0].phys_addr, in t3_sge_alloc_qset()
2439 &q->fl[0].desc, &q->fl[0].sdesc, in t3_sge_alloc_qset()
2440 &q->fl[0].desc_tag, &q->fl[0].desc_map, in t3_sge_alloc_qset()
2441 sc->rx_dmat, &q->fl[0].entry_tag)) != 0) { in t3_sge_alloc_qset()
2446 if ((ret = alloc_ring(sc, p->jumbo_size, sizeof(struct rx_desc), in t3_sge_alloc_qset()
2447 sizeof(struct rx_sw_desc), &q->fl[1].phys_addr, in t3_sge_alloc_qset()
2448 &q->fl[1].desc, &q->fl[1].sdesc, in t3_sge_alloc_qset()
2449 &q->fl[1].desc_tag, &q->fl[1].desc_map, in t3_sge_alloc_qset()
2450 sc->rx_jumbo_dmat, &q->fl[1].entry_tag)) != 0) { in t3_sge_alloc_qset()
2455 if ((ret = alloc_ring(sc, p->rspq_size, sizeof(struct rsp_desc), 0, in t3_sge_alloc_qset()
2456 &q->rspq.phys_addr, &q->rspq.desc, NULL, in t3_sge_alloc_qset()
2457 &q->rspq.desc_tag, &q->rspq.desc_map, in t3_sge_alloc_qset()
2463 snprintf(q->rspq.lockbuf, RSPQ_NAME_LEN, "t3 rspq lock %d:%d", in t3_sge_alloc_qset()
2464 device_get_unit(sc->dev), irq_vec_idx); in t3_sge_alloc_qset()
2465 MTX_INIT(&q->rspq.lock, q->rspq.lockbuf, NULL, MTX_DEF); in t3_sge_alloc_qset()
2470 if ((ret = alloc_ring(sc, p->txq_size[i], in t3_sge_alloc_qset()
2472 &q->txq[i].phys_addr, &q->txq[i].desc, in t3_sge_alloc_qset()
2473 &q->txq[i].sdesc, &q->txq[i].desc_tag, in t3_sge_alloc_qset()
2474 &q->txq[i].desc_map, in t3_sge_alloc_qset()
2475 sc->tx_dmat, &q->txq[i].entry_tag)) != 0) { in t3_sge_alloc_qset()
2476 printf("error %d from alloc ring tx %i\n", ret, i); in t3_sge_alloc_qset()
2479 mbufq_init(&q->txq[i].sendq, INT_MAX); in t3_sge_alloc_qset()
2480 q->txq[i].gen = 1; in t3_sge_alloc_qset()
2481 q->txq[i].size = p->txq_size[i]; in t3_sge_alloc_qset()
2485 TASK_INIT(&q->txq[TXQ_OFLD].qresume_task, 0, restart_offloadq, q); in t3_sge_alloc_qset()
2487 TASK_INIT(&q->txq[TXQ_CTRL].qresume_task, 0, restart_ctrlq, q); in t3_sge_alloc_qset()
2488 TASK_INIT(&q->txq[TXQ_ETH].qreclaim_task, 0, sge_txq_reclaim_handler, q); in t3_sge_alloc_qset()
2489 TASK_INIT(&q->txq[TXQ_OFLD].qreclaim_task, 0, sge_txq_reclaim_handler, q); in t3_sge_alloc_qset()
2491 q->fl[0].gen = q->fl[1].gen = 1; in t3_sge_alloc_qset()
2492 q->fl[0].size = p->fl_size; in t3_sge_alloc_qset()
2493 q->fl[1].size = p->jumbo_size; in t3_sge_alloc_qset()
2495 q->rspq.gen = 1; in t3_sge_alloc_qset()
2496 q->rspq.cidx = 0; in t3_sge_alloc_qset()
2497 q->rspq.size = p->rspq_size; in t3_sge_alloc_qset()
2499 q->txq[TXQ_ETH].stop_thres = nports * in t3_sge_alloc_qset()
2502 q->fl[0].buf_size = MCLBYTES; in t3_sge_alloc_qset()
2503 q->fl[0].zone = zone_pack; in t3_sge_alloc_qset()
2504 q->fl[0].type = EXT_PACKET; in t3_sge_alloc_qset()
2506 if (p->jumbo_buf_size == MJUM16BYTES) { in t3_sge_alloc_qset()
2507 q->fl[1].zone = zone_jumbo16; in t3_sge_alloc_qset()
2508 q->fl[1].type = EXT_JUMBO16; in t3_sge_alloc_qset()
2509 } else if (p->jumbo_buf_size == MJUM9BYTES) { in t3_sge_alloc_qset()
2510 q->fl[1].zone = zone_jumbo9; in t3_sge_alloc_qset()
2511 q->fl[1].type = EXT_JUMBO9; in t3_sge_alloc_qset()
2512 } else if (p->jumbo_buf_size == MJUMPAGESIZE) { in t3_sge_alloc_qset()
2513 q->fl[1].zone = zone_jumbop; in t3_sge_alloc_qset()
2514 q->fl[1].type = EXT_JUMBOP; in t3_sge_alloc_qset()
2516 KASSERT(0, ("can't deal with jumbo_buf_size %d.", p->jumbo_buf_size)); in t3_sge_alloc_qset()
2520 q->fl[1].buf_size = p->jumbo_buf_size; in t3_sge_alloc_qset()
2523 q->lro.enabled = !!(if_getcapenable(pi->ifp) & IFCAP_LRO); in t3_sge_alloc_qset()
2525 ret = tcp_lro_init(&q->lro.ctrl); in t3_sge_alloc_qset()
2531 q->lro.ctrl.ifp = pi->ifp; in t3_sge_alloc_qset()
2533 mtx_lock_spin(&sc->sge.reg_lock); in t3_sge_alloc_qset()
2534 ret = -t3_sge_init_rspcntxt(sc, q->rspq.cntxt_id, irq_vec_idx, in t3_sge_alloc_qset()
2535 q->rspq.phys_addr, q->rspq.size, in t3_sge_alloc_qset()
2536 q->fl[0].buf_size, 1, 0); in t3_sge_alloc_qset()
2543 ret = -t3_sge_init_flcntxt(sc, q->fl[i].cntxt_id, 0, in t3_sge_alloc_qset()
2544 q->fl[i].phys_addr, q->fl[i].size, in t3_sge_alloc_qset()
2545 q->fl[i].buf_size, p->cong_thres, 1, in t3_sge_alloc_qset()
2553 ret = -t3_sge_init_ecntxt(sc, q->txq[TXQ_ETH].cntxt_id, USE_GTS, in t3_sge_alloc_qset()
2554 SGE_CNTXT_ETH, id, q->txq[TXQ_ETH].phys_addr, in t3_sge_alloc_qset()
2555 q->txq[TXQ_ETH].size, q->txq[TXQ_ETH].token, in t3_sge_alloc_qset()
2563 ret = -t3_sge_init_ecntxt(sc, q->txq[TXQ_OFLD].cntxt_id, in t3_sge_alloc_qset()
2565 q->txq[TXQ_OFLD].phys_addr, in t3_sge_alloc_qset()
2566 q->txq[TXQ_OFLD].size, 0, 1, 0); in t3_sge_alloc_qset()
2573 if (ntxq > 2) { in t3_sge_alloc_qset()
2574 ret = -t3_sge_init_ecntxt(sc, q->txq[TXQ_CTRL].cntxt_id, 0, in t3_sge_alloc_qset()
2576 q->txq[TXQ_CTRL].phys_addr, in t3_sge_alloc_qset()
2577 q->txq[TXQ_CTRL].size, in t3_sge_alloc_qset()
2578 q->txq[TXQ_CTRL].token, 1, 0); in t3_sge_alloc_qset()
2585 mtx_unlock_spin(&sc->sge.reg_lock); in t3_sge_alloc_qset()
2588 refill_fl(sc, &q->fl[0], q->fl[0].size); in t3_sge_alloc_qset()
2589 refill_fl(sc, &q->fl[1], q->fl[1].size); in t3_sge_alloc_qset()
2590 refill_rspq(sc, &q->rspq, q->rspq.size - 1); in t3_sge_alloc_qset()
2592 t3_write_reg(sc, A_SG_GTS, V_RSPQ(q->rspq.cntxt_id) | in t3_sge_alloc_qset()
2593 V_NEWTIMER(q->rspq.holdoff_tmr)); in t3_sge_alloc_qset()
2598 mtx_unlock_spin(&sc->sge.reg_lock); in t3_sge_alloc_qset()
2615 struct port_info *pi = &adap->port[adap->rxpkt_map[cpl->iff]]; in t3_rx_eth()
2616 if_t ifp = pi->ifp; in t3_rx_eth()
2618 if (cpl->vlan_valid) { in t3_rx_eth()
2619 m->m_pkthdr.ether_vtag = ntohs(cpl->vlan); in t3_rx_eth()
2620 m->m_flags |= M_VLANTAG; in t3_rx_eth()
2623 m->m_pkthdr.rcvif = ifp; in t3_rx_eth()
2627 m->m_pkthdr.len -= (sizeof(*cpl) + ethpad); in t3_rx_eth()
2628 m->m_len -= (sizeof(*cpl) + ethpad); in t3_rx_eth()
2629 m->m_data += (sizeof(*cpl) + ethpad); in t3_rx_eth()
2631 if (!cpl->fragment && cpl->csum_valid && cpl->csum == 0xffff) { in t3_rx_eth()
2635 if (eh->ether_type == htons(ETHERTYPE_VLAN)) { in t3_rx_eth()
2638 eh_type = evh->evl_proto; in t3_rx_eth()
2640 eh_type = eh->ether_type; in t3_rx_eth()
2644 m->m_pkthdr.csum_flags = (CSUM_IP_CHECKED | in t3_rx_eth()
2646 m->m_pkthdr.csum_data = 0xffff; in t3_rx_eth()
2649 m->m_pkthdr.csum_flags = (CSUM_DATA_VALID_IPV6 | in t3_rx_eth()
2651 m->m_pkthdr.csum_data = 0xffff; in t3_rx_eth()
2657 * get_packet - return the next ingress packet buffer from a free list
2677 unsigned int len_cq = ntohl(r->len_cq); in get_packet()
2678 struct sge_fl *fl = (len_cq & F_RSPD_FLQ) ? &qs->fl[1] : &qs->fl[0]; in get_packet()
2679 int mask, cidx = fl->cidx; in get_packet()
2680 struct rx_sw_desc *sd = &fl->sdesc[cidx]; in get_packet()
2683 uint8_t sopeop = G_RSPD_SOP_EOP(ntohl(r->flags)); in get_packet()
2688 mask = fl->size - 1; in get_packet()
2689 prefetch(fl->sdesc[(cidx + 1) & mask].m); in get_packet()
2690 prefetch(fl->sdesc[(cidx + 2) & mask].m); in get_packet()
2691 prefetch(fl->sdesc[(cidx + 1) & mask].rxsd_cl); in get_packet()
2692 prefetch(fl->sdesc[(cidx + 2) & mask].rxsd_cl); in get_packet()
2694 fl->credits--; in get_packet()
2695 bus_dmamap_sync(fl->entry_tag, sd->map, BUS_DMASYNC_POSTREAD); in get_packet()
2702 memcpy(cl, sd->rxsd_cl, len); in get_packet()
2703 recycle_rx_buf(adap, fl, fl->cidx); in get_packet()
2704 m->m_pkthdr.len = m->m_len = len; in get_packet()
2705 m->m_flags = 0; in get_packet()
2706 mh->mh_head = mh->mh_tail = m; in get_packet()
2711 bus_dmamap_unload(fl->entry_tag, sd->map); in get_packet()
2712 cl = sd->rxsd_cl; in get_packet()
2713 m = sd->m; in get_packet()
2719 if (fl->zone == zone_pack) { in get_packet()
2723 m->m_data = m->m_ext.ext_buf; in get_packet()
2725 m_cljset(m, cl, fl->type); in get_packet()
2727 m->m_len = len; in get_packet()
2734 mh->mh_head = mh->mh_tail = m; in get_packet()
2735 m->m_pkthdr.len = len; in get_packet()
2741 if (mh->mh_tail == NULL) { in get_packet()
2747 mh->mh_tail->m_next = m; in get_packet()
2748 mh->mh_tail = m; in get_packet()
2749 mh->mh_head->m_pkthdr.len += len; in get_packet()
2753 printf("len=%d pktlen=%d\n", m->m_len, m->m_pkthdr.len); in get_packet()
2755 if (++fl->cidx == fl->size) in get_packet()
2756 fl->cidx = 0; in get_packet()
2762 * handle_rsp_cntrl_info - handles control information in a response
2763 * @qs: the queue set corresponding to the response
2767 * indications and completion credits for the queue set's Tx queues.
2777 clear_bit(TXQ_RUNNING, &qs->txq[TXQ_ETH].flags); in handle_rsp_cntrl_info()
2781 qs->txq[TXQ_ETH].processed += credits; in handle_rsp_cntrl_info()
2785 qs->txq[TXQ_CTRL].processed += credits; in handle_rsp_cntrl_info()
2789 clear_bit(TXQ_RUNNING, &qs->txq[TXQ_OFLD].flags); in handle_rsp_cntrl_info()
2793 qs->txq[TXQ_OFLD].processed += credits; in handle_rsp_cntrl_info()
2805 * process_responses - process responses from an SGE response queue
2807 * @qs: the queue set to which the response queue belongs
2810 * Process responses from an SGE response queue up to the supplied budget.
2812 * for the queues that belong to the response queue's queue set.
2816 * on this queue. If the system is under memory shortage use a fairly
2822 struct sge_rspq *rspq = &qs->rspq; in process_responses()
2823 struct rsp_desc *r = &rspq->desc[rspq->cidx]; in process_responses()
2827 int lro_enabled = qs->lro.enabled; in process_responses()
2829 struct lro_ctrl *lro_ctrl = &qs->lro.ctrl; in process_responses()
2831 struct t3_mbuf_hdr *mh = &rspq->rspq_mh; in process_responses()
2834 if (cxgb_debug && rspq->holdoff_tmr != last_holdoff) { in process_responses()
2835 printf("next_holdoff=%d\n", rspq->holdoff_tmr); in process_responses()
2836 last_holdoff = rspq->holdoff_tmr; in process_responses()
2839 rspq->next_holdoff = rspq->holdoff_tmr; in process_responses()
2843 uint32_t flags = ntohl(r->flags); in process_responses()
2844 uint32_t rss_hash = be32toh(r->rss_hdr.rss_hash_val); in process_responses()
2845 uint8_t opcode = r->rss_hdr.opcode; in process_responses()
2855 if (mh->mh_head == NULL) { in process_responses()
2856 mh->mh_head = m_gethdr(M_NOWAIT, MT_DATA); in process_responses()
2857 m = mh->mh_head; in process_responses()
2865 m->m_len = m->m_pkthdr.len = AN_PKT_SIZE; in process_responses()
2869 rspq->async_notif++; in process_responses()
2876 rspq->next_holdoff = NOMEM_INTR_DELAY; in process_responses()
2877 budget_left--; in process_responses()
2880 if (mh->mh_head == NULL) in process_responses()
2881 mh->mh_head = m; in process_responses()
2883 mh->mh_tail->m_next = m; in process_responses()
2884 mh->mh_tail = m; in process_responses()
2887 mh->mh_head->m_pkthdr.len += m->m_len; in process_responses()
2889 rspq->imm_data++; in process_responses()
2890 } else if (r->len_cq) { in process_responses()
2895 if (r->rss_hdr.hash_type && !adap->timestamp) { in process_responses()
2896 M_HASHTYPE_SET(mh->mh_head, in process_responses()
2898 mh->mh_head->m_pkthdr.flowid = rss_hash; in process_responses()
2902 ethpad = 2; in process_responses()
2904 rspq->pure_rsps++; in process_responses()
2913 rspq->offload_pkts++; in process_responses()
2915 adap->cpl_handler[opcode](qs, r, mh->mh_head); in process_responses()
2917 m_freem(mh->mh_head); in process_responses()
2919 mh->mh_head = NULL; in process_responses()
2921 struct mbuf *m = mh->mh_head; in process_responses()
2928 * lro_ctrl->ifp's input. That is incorrect. in process_responses()
2934 skip_lro = __predict_false(qs->port->ifp != m->m_pkthdr.rcvif); in process_responses()
2936 if (lro_enabled && lro_ctrl->lro_cnt && !skip_lro in process_responses()
2939 /* successfully queue'd for LRO */ in process_responses()
2945 * or unable to queue. Pass it up right now in in process_responses()
2948 if_t ifp = m->m_pkthdr.rcvif; in process_responses()
2951 mh->mh_head = NULL; in process_responses()
2956 if (__predict_false(++rspq->cidx == rspq->size)) { in process_responses()
2957 rspq->cidx = 0; in process_responses()
2958 rspq->gen ^= 1; in process_responses()
2959 r = rspq->desc; in process_responses()
2962 if (++rspq->credits >= 64) { in process_responses()
2963 refill_rspq(adap, rspq, rspq->credits); in process_responses()
2964 rspq->credits = 0; in process_responses()
2966 __refill_fl_lt(adap, &qs->fl[0], 32); in process_responses()
2967 __refill_fl_lt(adap, &qs->fl[1], 32); in process_responses()
2968 --budget_left; in process_responses()
2979 mb(); /* commit Tx queue processed updates */ in process_responses()
2980 if (__predict_false(qs->txq_stopped > 1)) in process_responses()
2983 __refill_fl_lt(adap, &qs->fl[0], 512); in process_responses()
2984 __refill_fl_lt(adap, &qs->fl[1], 512); in process_responses()
2985 budget -= budget_left; in process_responses()
2998 work = process_responses(adap, rspq_to_qset(rq), -1); in process_responses_gts()
3000 if (cxgb_debug && (rq->next_holdoff != last_holdoff)) { in process_responses_gts()
3001 printf("next_holdoff=%d\n", rq->next_holdoff); in process_responses_gts()
3002 last_holdoff = rq->next_holdoff; in process_responses_gts()
3004 t3_write_reg(adap, A_SG_GTS, V_RSPQ(rq->cntxt_id) | in process_responses_gts()
3005 V_NEWTIMER(rq->next_holdoff) | V_NEWINDEX(rq->cidx)); in process_responses_gts()
3015 return (process_responses_gts(adap, &qs->rspq)); in cxgb_debugnet_poll_rx()
3020 * Interrupt handler for legacy INTx interrupts for T3B-based cards.
3023 * response queue per port in this mode and protect all response queues with
3024 * queue 0's lock.
3031 struct sge_rspq *q0 = &adap->sge.qs[0].rspq; in t3b_intr()
3042 taskqueue_enqueue(adap->tq, &adap->slow_intr_task); in t3b_intr()
3045 mtx_lock(&q0->lock); in t3b_intr()
3048 process_responses_gts(adap, &adap->sge.qs[i].rspq); in t3b_intr()
3049 mtx_unlock(&q0->lock); in t3b_intr()
3055 * the same MSI vector. We use one SGE response queue per port in this mode
3056 * and protect all response queues with queue 0's lock.
3062 struct sge_rspq *q0 = &adap->sge.qs[0].rspq; in t3_intr_msi()
3065 mtx_lock(&q0->lock); in t3_intr_msi()
3068 if (process_responses_gts(adap, &adap->sge.qs[i].rspq)) in t3_intr_msi()
3070 mtx_unlock(&q0->lock); in t3_intr_msi()
3074 taskqueue_enqueue(adap->tq, &adap->slow_intr_task); in t3_intr_msi()
3082 adapter_t *adap = qs->port->adapter; in t3_intr_msix()
3083 struct sge_rspq *rspq = &qs->rspq; in t3_intr_msix()
3086 rspq->unhandled_irqs++; in t3_intr_msix()
3102 if (rspq->rspq_dump_count == 0) in t3_dump_rspq()
3104 if (rspq->rspq_dump_count > RSPQ_Q_SIZE) { in t3_dump_rspq()
3106 "dump count is too large %d\n", rspq->rspq_dump_count); in t3_dump_rspq()
3107 rspq->rspq_dump_count = 0; in t3_dump_rspq()
3110 if (rspq->rspq_dump_start > (RSPQ_Q_SIZE-1)) { in t3_dump_rspq()
3112 "dump start of %d is greater than queue size\n", in t3_dump_rspq()
3113 rspq->rspq_dump_start); in t3_dump_rspq()
3114 rspq->rspq_dump_start = 0; in t3_dump_rspq()
3117 err = t3_sge_read_rspq(qs->port->adapter, rspq->cntxt_id, data); in t3_dump_rspq()
3125 sbuf_printf(sb, " \n index=%u size=%u MSI-X/RspQ=%u intr enable=%u intr armed=%u\n", in t3_dump_rspq()
3126 (data[0] & 0xffff), data[0] >> 16, ((data[2] >> 20) & 0x3f), in t3_dump_rspq()
3127 ((data[2] >> 26) & 1), ((data[2] >> 27) & 1)); in t3_dump_rspq()
3129 ((data[2] >> 28) & 1), ((data[2] >> 31) & 1), data[3]); in t3_dump_rspq()
3131 sbuf_printf(sb, " start=%d -> end=%d\n", rspq->rspq_dump_start, in t3_dump_rspq()
3132 (rspq->rspq_dump_start + rspq->rspq_dump_count) & (RSPQ_Q_SIZE-1)); in t3_dump_rspq()
3134 dump_end = rspq->rspq_dump_start + rspq->rspq_dump_count; in t3_dump_rspq()
3135 for (i = rspq->rspq_dump_start; i < dump_end; i++) { in t3_dump_rspq()
3136 idx = i & (RSPQ_Q_SIZE-1); in t3_dump_rspq()
3138 rspd = &rspq->desc[idx]; in t3_dump_rspq()
3140 idx, rspd->rss_hdr.opcode, rspd->rss_hdr.cpu_idx, in t3_dump_rspq()
3141 rspd->rss_hdr.hash_type, be16toh(rspd->rss_hdr.cq_idx)); in t3_dump_rspq()
3143 rspd->rss_hdr.rss_hash_val, be32toh(rspd->flags), in t3_dump_rspq()
3144 be32toh(rspd->len_cq), rspd->intr_gen); in t3_dump_rspq()
3165 if (txq->txq_dump_count == 0) { in t3_dump_txq_eth()
3168 if (txq->txq_dump_count > TX_ETH_Q_SIZE) { in t3_dump_txq_eth()
3170 "dump count is too large %d\n", txq->txq_dump_count); in t3_dump_txq_eth()
3171 txq->txq_dump_count = 1; in t3_dump_txq_eth()
3174 if (txq->txq_dump_start > (TX_ETH_Q_SIZE-1)) { in t3_dump_txq_eth()
3176 "dump start of %d is greater than queue size\n", in t3_dump_txq_eth()
3177 txq->txq_dump_start); in t3_dump_txq_eth()
3178 txq->txq_dump_start = 0; in t3_dump_txq_eth()
3181 err = t3_sge_read_ecntxt(qs->port->adapter, qs->rspq.cntxt_id, data); in t3_dump_txq_eth()
3195 sbuf_printf(sb, " qid=%d start=%d -> end=%d\n", qs->idx, in t3_dump_txq_eth()
3196 txq->txq_dump_start, in t3_dump_txq_eth()
3197 (txq->txq_dump_start + txq->txq_dump_count) & (TX_ETH_Q_SIZE-1)); in t3_dump_txq_eth()
3199 dump_end = txq->txq_dump_start + txq->txq_dump_count; in t3_dump_txq_eth()
3200 for (i = txq->txq_dump_start; i < dump_end; i++) { in t3_dump_txq_eth()
3201 txd = &txq->desc[i & (TX_ETH_Q_SIZE-1)]; in t3_dump_txq_eth()
3202 WR = (uint32_t *)txd->flit; in t3_dump_txq_eth()
3209 for (j = 2; j < 30; j += 4) in t3_dump_txq_eth()
3211 WR[j], WR[j + 1], WR[j + 2], WR[j + 3]); in t3_dump_txq_eth()
3231 if (txq->txq_dump_count == 0) { in t3_dump_txq_ctrl()
3234 if (txq->txq_dump_count > 256) { in t3_dump_txq_ctrl()
3236 "dump count is too large %d\n", txq->txq_dump_count); in t3_dump_txq_ctrl()
3237 txq->txq_dump_count = 1; in t3_dump_txq_ctrl()
3240 if (txq->txq_dump_start > 255) { in t3_dump_txq_ctrl()
3242 "dump start of %d is greater than queue size\n", in t3_dump_txq_ctrl()
3243 txq->txq_dump_start); in t3_dump_txq_ctrl()
3244 txq->txq_dump_start = 0; in t3_dump_txq_ctrl()
3252 sbuf_printf(sb, " qid=%d start=%d -> end=%d\n", qs->idx, in t3_dump_txq_ctrl()
3253 txq->txq_dump_start, in t3_dump_txq_ctrl()
3254 (txq->txq_dump_start + txq->txq_dump_count) & 255); in t3_dump_txq_ctrl()
3256 dump_end = txq->txq_dump_start + txq->txq_dump_count; in t3_dump_txq_ctrl()
3257 for (i = txq->txq_dump_start; i < dump_end; i++) { in t3_dump_txq_ctrl()
3258 txd = &txq->desc[i & (255)]; in t3_dump_txq_ctrl()
3259 WR = (uint32_t *)txd->flit; in t3_dump_txq_ctrl()
3266 for (j = 2; j < 30; j += 4) in t3_dump_txq_ctrl()
3268 WR[j], WR[j + 1], WR[j + 2], WR[j + 3]); in t3_dump_txq_ctrl()
3280 struct qset_params *qsp = &sc->params.sge.qset[0]; in t3_set_coalesce_usecs()
3286 if ((sc->flags & FULL_INIT_DONE) == 0) in t3_set_coalesce_usecs()
3289 coalesce_usecs = qsp->coalesce_usecs; in t3_set_coalesce_usecs()
3295 if (coalesce_usecs == qsp->coalesce_usecs) in t3_set_coalesce_usecs()
3298 for (i = 0; i < sc->params.nports; i++) in t3_set_coalesce_usecs()
3299 for (j = 0; j < sc->port[i].nqsets; j++) in t3_set_coalesce_usecs()
3305 qs = &sc->sge.qs[i]; in t3_set_coalesce_usecs()
3306 qsp = &sc->params.sge.qset[i]; in t3_set_coalesce_usecs()
3307 qsp->coalesce_usecs = coalesce_usecs; in t3_set_coalesce_usecs()
3309 lock = (sc->flags & USING_MSIX) ? &qs->rspq.lock : in t3_set_coalesce_usecs()
3310 &sc->sge.qs[0].rspq.lock; in t3_set_coalesce_usecs()
3314 t3_write_reg(sc, A_SG_GTS, V_RSPQ(qs->rspq.cntxt_id) | in t3_set_coalesce_usecs()
3315 V_NEWTIMER(qs->rspq.holdoff_tmr)); in t3_set_coalesce_usecs()
3328 if ((sc->flags & FULL_INIT_DONE) == 0) in t3_pkt_timestamp()
3331 timestamp = sc->timestamp; in t3_pkt_timestamp()
3337 if (timestamp != sc->timestamp) { in t3_pkt_timestamp()
3340 sc->timestamp = timestamp; in t3_pkt_timestamp()
3352 ctx = device_get_sysctl_ctx(sc->dev); in t3_add_attach_sysctls()
3353 children = SYSCTL_CHILDREN(device_get_sysctl_tree(sc->dev)); in t3_add_attach_sysctls()
3358 CTLFLAG_RD, sc->fw_version, in t3_add_attach_sysctls()
3362 CTLFLAG_RD, &sc->params.rev, in t3_add_attach_sysctls()
3366 CTLFLAG_RD, sc->port_types, in t3_add_attach_sysctls()
3373 CTLFLAG_RD, &sc->tunq_coalesce, in t3_add_attach_sysctls()
3381 CTLFLAG_RD, &sc->params.vpd.cclk, in t3_add_attach_sysctls()
3404 parg = (uint64_t *) ((uint8_t *)&p->mac.stats + arg2); in sysctl_handle_macstat()
3416 ctx = device_get_sysctl_ctx(sc->dev); in t3_add_configured_sysctls()
3417 children = SYSCTL_CHILDREN(device_get_sysctl_tree(sc->dev)); in t3_add_configured_sysctls()
3431 for (i = 0; i < sc->params.nports; i++) { in t3_add_configured_sysctls()
3432 struct port_info *pi = &sc->port[i]; in t3_add_configured_sysctls()
3435 struct mac_stats *mstats = &pi->mac.stats; in t3_add_configured_sysctls()
3437 snprintf(pi->namebuf, PORT_NAME_LEN, "port%d", i); in t3_add_configured_sysctls()
3439 pi->namebuf, CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, in t3_add_configured_sysctls()
3443 "nqsets", CTLFLAG_RD, &pi->nqsets, in t3_add_configured_sysctls()
3444 0, "#queue sets"); in t3_add_configured_sysctls()
3446 for (j = 0; j < pi->nqsets; j++) { in t3_add_configured_sysctls()
3447 struct sge_qset *qs = &sc->sge.qs[pi->first_qset + j]; in t3_add_configured_sysctls()
3453 struct sge_txq *txq = &qs->txq[TXQ_ETH]; in t3_add_configured_sysctls()
3455 snprintf(qs->namebuf, QS_NAME_LEN, "qs%d", j); in t3_add_configured_sysctls()
3458 qs->namebuf, CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, in t3_add_configured_sysctls()
3463 CTLFLAG_RD, &qs->fl[0].empty, 0, in t3_add_configured_sysctls()
3466 CTLFLAG_RD, &qs->fl[1].empty, 0, in t3_add_configured_sysctls()
3480 txq_names[2], CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, in t3_add_configured_sysctls()
3490 CTLFLAG_RD, &qs->rspq.size, in t3_add_configured_sysctls()
3491 0, "#entries in response queue"); in t3_add_configured_sysctls()
3493 CTLFLAG_RD, &qs->rspq.cidx, in t3_add_configured_sysctls()
3496 CTLFLAG_RD, &qs->rspq.credits, in t3_add_configured_sysctls()
3499 CTLFLAG_RD, &qs->rspq.starved, in t3_add_configured_sysctls()
3502 CTLFLAG_RD, &qs->rspq.phys_addr, in t3_add_configured_sysctls()
3503 "physical_address_of the queue"); in t3_add_configured_sysctls()
3505 CTLFLAG_RW, &qs->rspq.rspq_dump_start, in t3_add_configured_sysctls()
3508 CTLFLAG_RW, &qs->rspq.rspq_dump_count, in t3_add_configured_sysctls()
3512 &qs->rspq, 0, t3_dump_rspq, "A", in t3_add_configured_sysctls()
3513 "dump of the response queue"); in t3_add_configured_sysctls()
3516 CTLFLAG_RD, &qs->txq[TXQ_ETH].txq_mr->br_drops, in t3_add_configured_sysctls()
3519 CTLFLAG_RD, &qs->txq[TXQ_ETH].sendq.mq_len, in t3_add_configured_sysctls()
3523 CTLFLAG_RD, (uint32_t *)(uintptr_t)&qs->txq[TXQ_ETH].txq_mr.br_prod, in t3_add_configured_sysctls()
3524 0, "#tunneled packets queue producer index"); in t3_add_configured_sysctls()
3526 CTLFLAG_RD, (uint32_t *)(uintptr_t)&qs->txq[TXQ_ETH].txq_mr.br_cons, in t3_add_configured_sysctls()
3527 0, "#tunneled packets queue consumer index"); in t3_add_configured_sysctls()
3530 CTLFLAG_RD, &qs->txq[TXQ_ETH].processed, in t3_add_configured_sysctls()
3533 CTLFLAG_RD, &txq->cleaned, in t3_add_configured_sysctls()
3536 CTLFLAG_RD, &txq->in_use, in t3_add_configured_sysctls()
3539 CTLFLAG_RD, &txq->txq_frees, in t3_add_configured_sysctls()
3542 CTLFLAG_RD, &txq->txq_skipped, in t3_add_configured_sysctls()
3545 CTLFLAG_RD, &txq->txq_coalesced, in t3_add_configured_sysctls()
3548 CTLFLAG_RD, &txq->txq_enqueued, in t3_add_configured_sysctls()
3551 CTLFLAG_RD, &qs->txq_stopped, in t3_add_configured_sysctls()
3552 0, "tx queues stopped"); in t3_add_configured_sysctls()
3554 CTLFLAG_RD, &txq->phys_addr, in t3_add_configured_sysctls()
3555 "physical_address_of the queue"); in t3_add_configured_sysctls()
3557 CTLFLAG_RW, &qs->txq[TXQ_ETH].gen, in t3_add_configured_sysctls()
3560 CTLFLAG_RD, &txq->cidx, in t3_add_configured_sysctls()
3561 0, "hardware queue cidx"); in t3_add_configured_sysctls()
3563 CTLFLAG_RD, &txq->pidx, in t3_add_configured_sysctls()
3564 0, "hardware queue pidx"); in t3_add_configured_sysctls()
3566 CTLFLAG_RW, &qs->txq[TXQ_ETH].txq_dump_start, in t3_add_configured_sysctls()
3569 CTLFLAG_RW, &qs->txq[TXQ_ETH].txq_dump_count, in t3_add_configured_sysctls()
3573 &qs->txq[TXQ_ETH], 0, t3_dump_txq_eth, "A", in t3_add_configured_sysctls()
3574 "dump of the transmit queue"); in t3_add_configured_sysctls()
3577 CTLFLAG_RW, &qs->txq[TXQ_CTRL].txq_dump_start, in t3_add_configured_sysctls()
3580 CTLFLAG_RW, &qs->txq[TXQ_CTRL].txq_dump_count, in t3_add_configured_sysctls()
3584 &qs->txq[TXQ_CTRL], 0, t3_dump_txq_ctrl, "A", in t3_add_configured_sysctls()
3585 "dump of the transmit queue"); in t3_add_configured_sysctls()
3588 CTLFLAG_RD, &qs->lro.ctrl.lro_queued, 0, NULL); in t3_add_configured_sysctls()
3590 CTLFLAG_RD, &qs->lro.ctrl.lro_flushed, 0, NULL); in t3_add_configured_sysctls()
3592 CTLFLAG_RD, &qs->lro.ctrl.lro_bad_csum, 0, NULL); in t3_add_configured_sysctls()
3594 CTLFLAG_RD, &qs->lro.ctrl.lro_cnt, 0, NULL); in t3_add_configured_sysctls()
3666 CTLFLAG_RD, &mstats->a, 0) in t3_add_configured_sysctls()
3682 * t3_get_desc - dump an SGE descriptor for debugging purposes
3683 * @qs: the queue set
3684 * @qnum: identifies the specific queue (0..2: Tx, 3:response, 4..5: Rx)
3685 * @idx: the descriptor index in the queue
3688 * Dumps the contents of a HW descriptor of an SGE queue. Returns the
3699 if (!qs->txq[qnum].desc || idx >= qs->txq[qnum].size) in t3_get_desc()
3700 return -EINVAL; in t3_get_desc()
3701 memcpy(data, &qs->txq[qnum].desc[idx], sizeof(struct tx_desc)); in t3_get_desc()
3706 if (!qs->rspq.desc || idx >= qs->rspq.size) in t3_get_desc()
3708 memcpy(data, &qs->rspq.desc[idx], sizeof(struct rsp_desc)); in t3_get_desc()
3712 qnum -= 4; in t3_get_desc()
3713 if (!qs->fl[qnum].desc || idx >= qs->fl[qnum].size) in t3_get_desc()
3715 memcpy(data, &qs->fl[qnum].desc[idx], sizeof(struct rx_desc)); in t3_get_desc()