Lines Matching +full:tx +full:- +full:queue +full:- +full:1
2 SPDX-License-Identifier: BSD-2-Clause
4 Copyright (c) 2007-2009, Chelsio Inc.
10 1. Redistributions of source code must retain the above copyright notice,
44 #include <sys/queue.h>
79 int multiq_tx_enable = 1;
88 "size of per-queue mbuf ring");
95 #define COALESCE_START_DEFAULT TX_ETH_Q_SIZE>>1
96 #define COALESCE_START_MAX (TX_ETH_Q_SIZE-(TX_ETH_Q_SIZE>>3))
115 "tx cleaning minimum threshold");
118 * XXX don't re-enable this until TOE stops assuming
135 * Period of the Tx buffer reclaim timer. This timer does not need to run
136 * frequently as Tx buffers are usually reclaimed by new Tx packets.
138 #define TX_RECLAIM_PERIOD (hz >> 1)
144 TXQ_RUNNING = 1 << 0, /* fetch engine is running */
145 TXQ_LAST_PKT_DB = 1 << 1, /* last packet rang the doorbell */
159 struct rsp_desc { /* response queue descriptor */
167 #define RX_SW_DESC_MAP_CREATED (1 << 0)
168 #define TX_SW_DESC_MAP_CREATED (1 << 1)
169 #define RX_SW_DESC_INUSE (1 << 3)
170 #define TX_SW_DESC_MAPPED (1 << 4)
177 struct tx_sw_desc { /* SW state per Tx descriptor */
204 * Maps a number of flits to the number of Tx descriptors that can hold them.
207 * desc = 1 + (flits - 2) / (WR_FLITS - 1).
213 #if SGE_NUM_GENBITS == 1
214 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
219 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,
224 # error "SGE_NUM_GENBITS must be 1 or 2"
228 #define TXQ_LOCK_ASSERT(qs) mtx_assert(&(qs)->lock, MA_OWNED)
229 #define TXQ_TRYLOCK(qs) mtx_trylock(&(qs)->lock)
230 #define TXQ_LOCK(qs) mtx_lock(&(qs)->lock)
231 #define TXQ_UNLOCK(qs) mtx_unlock(&(qs)->lock)
232 #define TXQ_RING_EMPTY(qs) drbr_empty((qs)->port->ifp, (qs)->txq[TXQ_ETH].txq_mr)
234 drbr_needs_enqueue((qs)->port->ifp, (qs)->txq[TXQ_ETH].txq_mr)
235 #define TXQ_RING_FLUSH(qs) drbr_flush((qs)->port->ifp, (qs)->txq[TXQ_ETH].txq_mr)
237 drbr_dequeue_cond((qs)->port->ifp, (qs)->txq[TXQ_ETH].txq_mr, func, arg)
239 drbr_dequeue((qs)->port->ifp, (qs)->txq[TXQ_ETH].txq_mr)
261 return (1); in check_pkt_coalesce()
262 txq = &qs->txq[TXQ_ETH]; in check_pkt_coalesce()
263 sc = qs->port->adapter; in check_pkt_coalesce()
264 fill = &sc->tunq_fill[qs->idx]; in check_pkt_coalesce()
271 * if the hardware transmit queue is more than 1/8 full in check_pkt_coalesce()
272 * we mark it as coalescing - we drop back from coalescing in check_pkt_coalesce()
273 * when we go below 1/32 full and there are no packets enqueued, in check_pkt_coalesce()
276 if (*fill != 0 && (txq->in_use <= cxgb_tx_coalesce_enable_stop) && in check_pkt_coalesce()
277 TXQ_RING_EMPTY(qs) && (qs->coalescing == 0)) in check_pkt_coalesce()
279 else if (*fill == 0 && (txq->in_use >= cxgb_tx_coalesce_enable_start)) in check_pkt_coalesce()
280 *fill = 1; in check_pkt_coalesce()
282 return (sc->tunq_coalesce); in check_pkt_coalesce()
297 wrp->wrh_hilo = wr_hilo; in set_wr_hdr()
304 wrp->wrh_hi = wr_hi; in set_wr_hdr()
306 wrp->wrh_lo = wr_lo; in set_wr_hdr()
321 if ((m->m_next != NULL) || in coalesce_check()
322 ((mtod(m, vm_offset_t) & PAGE_MASK) + m->m_len > PAGE_SIZE)) in coalesce_check()
323 ci->noncoal = 1; in coalesce_check()
325 if ((ci->count == 0) || (ci->noncoal == 0 && (ci->count < 7) && in coalesce_check()
326 (ci->nbytes + m->m_len <= 10500))) { in coalesce_check()
327 ci->count++; in coalesce_check()
328 ci->nbytes += m->m_len; in coalesce_check()
329 return (1); in coalesce_check()
351 m_tail->m_nextpkt = m; in cxgb_dequeue()
361 * reclaim_completed_tx - reclaims completed Tx descriptors
363 * @q: the Tx queue to reclaim completed descriptors from
365 * Reclaims Tx descriptors that the SGE has indicated it has processed,
366 * and frees the associated buffers if possible. Called with the Tx
367 * queue's lock held.
370 reclaim_completed_tx(struct sge_qset *qs, int reclaim_min, int queue) in reclaim_completed_tx() argument
372 struct sge_txq *q = &qs->txq[queue]; in reclaim_completed_tx()
382 mtx_assert(&qs->lock, MA_OWNED); in reclaim_completed_tx()
384 t3_free_tx_desc(qs, reclaim, queue); in reclaim_completed_tx()
385 q->cleaned += reclaim; in reclaim_completed_tx()
386 q->in_use -= reclaim; in reclaim_completed_tx()
388 if (isset(&qs->txq_stopped, TXQ_ETH)) in reclaim_completed_tx()
389 clrbit(&qs->txq_stopped, TXQ_ETH); in reclaim_completed_tx()
404 * should_restart_tx - are there enough resources to restart a Tx queue?
405 * @q: the Tx queue
407 * Checks if there are enough descriptors to restart a suspended Tx queue.
412 unsigned int r = q->processed - q->cleaned; in should_restart_tx()
414 return q->in_use - r < (q->size >> 1); in should_restart_tx()
418 * t3_sge_init - initialize SGE
423 * We do not initialize any of the queue sets here, instead the driver
424 * top-level must request those individually. We also do not enable DMA
432 ups = 0; /* = ffs(pci_resource_len(adap->pdev, 2) >> 12); */ in t3_sge_init()
436 V_HOSTPAGESIZE(PAGE_SHIFT - 11) | F_BIGENDIANINGRESS | in t3_sge_init()
437 V_USERSPACESIZE(ups ? ups - 1 : 0) | F_ISCSICOALESCING; in t3_sge_init()
438 #if SGE_NUM_GENBITS == 1 in t3_sge_init()
441 if (adap->params.rev > 0) { in t3_sge_init()
442 if (!(adap->flags & (USING_MSIX | USING_MSI))) in t3_sge_init()
452 adap->params.rev < T3_REV_C ? 1000 : 500); in t3_sge_init()
462 * sgl_len - calculates the size of an SGL of the given capacity
471 return ((3 * n) / 2 + (n & 1)); in sgl_len()
475 * get_imm_packet - return the next ingress packet buffer from a response
484 if (resp->rss_hdr.opcode == CPL_RX_DATA) { in get_imm_packet()
485 const struct cpl_rx_data *cpl = (const void *)&resp->imm_data[0]; in get_imm_packet()
486 m->m_len = sizeof(*cpl) + ntohs(cpl->len); in get_imm_packet()
487 } else if (resp->rss_hdr.opcode == CPL_RX_PKT) { in get_imm_packet()
488 const struct cpl_rx_pkt *cpl = (const void *)&resp->imm_data[0]; in get_imm_packet()
489 m->m_len = sizeof(*cpl) + ntohs(cpl->len); in get_imm_packet()
491 m->m_len = IMMED_PKT_SIZE; in get_imm_packet()
492 m->m_ext.ext_buf = NULL; in get_imm_packet()
493 m->m_ext.ext_type = 0; in get_imm_packet()
494 memcpy(mtod(m, uint8_t *), resp->imm_data, m->m_len); in get_imm_packet()
514 * t3_sge_err_intr_handler - SGE async event interrupt handler
517 * Interrupt handler for SGE asynchronous (non-data) events.
532 CH_ALERT(adapter, "SGE response queue credit overflow\n"); in t3_sge_err_intr_handler()
538 "packet delivered to disabled response queue (0x%x)\n", in t3_sge_err_intr_handler()
552 nqsets = min(SGE_QSETS / adap->params.nports, mp_ncpus); in t3_sge_prep()
553 nqsets *= adap->params.nports; in t3_sge_prep()
558 use_16k = cxgb_use_16k_clusters != -1 ? cxgb_use_16k_clusters : in t3_sge_prep()
571 device_printf(adap->dev, in t3_sge_prep()
574 p->max_pkt_size = jumbo_buf_size - sizeof(struct cpl_rx_data); in t3_sge_prep()
577 struct qset_params *q = p->qset + i; in t3_sge_prep()
579 if (adap->params.nports > 2) { in t3_sge_prep()
580 q->coalesce_usecs = 50; in t3_sge_prep()
583 q->coalesce_usecs = 10; in t3_sge_prep()
585 q->coalesce_usecs = 5; in t3_sge_prep()
588 q->polling = 0; in t3_sge_prep()
589 q->rspq_size = RSPQ_Q_SIZE; in t3_sge_prep()
590 q->fl_size = fl_q_size; in t3_sge_prep()
591 q->jumbo_size = jumbo_q_size; in t3_sge_prep()
592 q->jumbo_buf_size = jumbo_buf_size; in t3_sge_prep()
593 q->txq_size[TXQ_ETH] = TX_ETH_Q_SIZE; in t3_sge_prep()
594 q->txq_size[TXQ_OFLD] = is_offload(adap) ? TX_OFLD_Q_SIZE : 16; in t3_sge_prep()
595 q->txq_size[TXQ_CTRL] = TX_CTRL_Q_SIZE; in t3_sge_prep()
596 q->cong_thres = 0; in t3_sge_prep()
605 if (bus_dma_tag_create( bus_get_dma_tag(sc->dev),/* PCI parent */ in t3_sge_alloc()
606 1, 0, /* algnmnt, boundary */ in t3_sge_alloc()
615 &sc->parent_dmat)) { in t3_sge_alloc()
616 device_printf(sc->dev, "Cannot allocate parent DMA tag\n"); in t3_sge_alloc()
623 if (bus_dma_tag_create(sc->parent_dmat, MCLBYTES, 0, BUS_SPACE_MAXADDR, in t3_sge_alloc()
624 BUS_SPACE_MAXADDR, NULL, NULL, MCLBYTES, 1, in t3_sge_alloc()
625 MCLBYTES, BUS_DMA_ALLOCNOW, NULL, NULL, &sc->rx_dmat)) { in t3_sge_alloc()
626 device_printf(sc->dev, "Cannot allocate RX DMA tag\n"); in t3_sge_alloc()
633 if (bus_dma_tag_create(sc->parent_dmat, MJUM16BYTES, 0, BUS_SPACE_MAXADDR, in t3_sge_alloc()
634 BUS_SPACE_MAXADDR, NULL, NULL, MJUM16BYTES, 1, MJUM16BYTES, in t3_sge_alloc()
635 BUS_DMA_ALLOCNOW, NULL, NULL, &sc->rx_jumbo_dmat)) { in t3_sge_alloc()
636 device_printf(sc->dev, "Cannot allocate RX jumbo DMA tag\n"); in t3_sge_alloc()
641 * DMA tag for TX frames. in t3_sge_alloc()
643 if (bus_dma_tag_create(sc->parent_dmat, 1, 0, BUS_SPACE_MAXADDR, in t3_sge_alloc()
646 NULL, NULL, &sc->tx_dmat)) { in t3_sge_alloc()
647 device_printf(sc->dev, "Cannot allocate TX DMA tag\n"); in t3_sge_alloc()
658 if (sc->tx_dmat != NULL) in t3_sge_free()
659 bus_dma_tag_destroy(sc->tx_dmat); in t3_sge_free()
661 if (sc->rx_jumbo_dmat != NULL) in t3_sge_free()
662 bus_dma_tag_destroy(sc->rx_jumbo_dmat); in t3_sge_free()
664 if (sc->rx_dmat != NULL) in t3_sge_free()
665 bus_dma_tag_destroy(sc->rx_dmat); in t3_sge_free()
667 if (sc->parent_dmat != NULL) in t3_sge_free()
668 bus_dma_tag_destroy(sc->parent_dmat); in t3_sge_free()
677 qs->rspq.holdoff_tmr = max(p->coalesce_usecs * 10, 1U); in t3_update_qset_coalesce()
678 qs->rspq.polling = 0 /* p->polling */; in t3_update_qset_coalesce()
687 cb_arg->error = error; in refill_fl_cb()
688 cb_arg->seg = segs[0]; in refill_fl_cb()
689 cb_arg->nseg = nseg; in refill_fl_cb()
694 * refill_fl - refill an SGE free-buffer list
696 * @q: the free-list to refill
699 * (Re)populate an SGE free-buffer list with up to @n new packet buffers.
700 * The caller must assure that @n does not exceed the queue's capacity.
705 struct rx_sw_desc *sd = &q->sdesc[q->pidx]; in refill_fl()
706 struct rx_desc *d = &q->desc[q->pidx]; in refill_fl()
713 while (n--) { in refill_fl()
718 if (q->zone == zone_pack) { in refill_fl()
721 cl = m->m_ext.ext_buf; in refill_fl()
723 if ((cl = m_cljget(NULL, M_NOWAIT, q->buf_size)) == NULL) in refill_fl()
726 uma_zfree(q->zone, cl); in refill_fl()
730 if ((sd->flags & RX_SW_DESC_MAP_CREATED) == 0) { in refill_fl()
731 if ((err = bus_dmamap_create(q->entry_tag, 0, &sd->map))) { in refill_fl()
733 uma_zfree(q->zone, cl); in refill_fl()
736 sd->flags |= RX_SW_DESC_MAP_CREATED; in refill_fl()
739 err = bus_dmamap_load(q->entry_tag, sd->map, in refill_fl()
740 cl, q->buf_size, refill_fl_cb, &cb_arg, 0); in refill_fl()
743 if (q->zone != zone_pack) in refill_fl()
744 uma_zfree(q->zone, cl); in refill_fl()
751 sd->flags |= RX_SW_DESC_INUSE; in refill_fl()
752 sd->rxsd_cl = cl; in refill_fl()
753 sd->m = m; in refill_fl()
754 d->addr_lo = htobe32(cb_arg.seg.ds_addr & 0xffffffff); in refill_fl()
755 d->addr_hi = htobe32(((uint64_t)cb_arg.seg.ds_addr >>32) & 0xffffffff); in refill_fl()
756 d->len_gen = htobe32(V_FLD_GEN1(q->gen)); in refill_fl()
757 d->gen2 = htobe32(V_FLD_GEN2(q->gen)); in refill_fl()
762 if (++q->pidx == q->size) { in refill_fl()
763 q->pidx = 0; in refill_fl()
764 q->gen ^= 1; in refill_fl()
765 sd = q->sdesc; in refill_fl()
766 d = q->desc; in refill_fl()
768 q->credits++; in refill_fl()
769 q->db_pending++; in refill_fl()
773 if (q->db_pending >= 32) { in refill_fl()
774 q->db_pending = 0; in refill_fl()
775 t3_write_reg(sc, A_SG_KDOORBELL, V_EGRCNTX(q->cntxt_id)); in refill_fl()
781 * free_rx_bufs - free the Rx buffers on an SGE free list
785 * Release the buffers on an SGE free-buffer Rx queue. HW fetching from
786 * this queue should be stopped before calling this function.
791 u_int cidx = q->cidx; in free_rx_bufs()
793 while (q->credits--) { in free_rx_bufs()
794 struct rx_sw_desc *d = &q->sdesc[cidx]; in free_rx_bufs()
796 if (d->flags & RX_SW_DESC_INUSE) { in free_rx_bufs()
797 bus_dmamap_unload(q->entry_tag, d->map); in free_rx_bufs()
798 bus_dmamap_destroy(q->entry_tag, d->map); in free_rx_bufs()
799 if (q->zone == zone_pack) { in free_rx_bufs()
800 m_init(d->m, M_NOWAIT, MT_DATA, M_EXT); in free_rx_bufs()
801 uma_zfree(zone_pack, d->m); in free_rx_bufs()
803 m_init(d->m, M_NOWAIT, MT_DATA, 0); in free_rx_bufs()
804 m_free_raw(d->m); in free_rx_bufs()
805 uma_zfree(q->zone, d->rxsd_cl); in free_rx_bufs()
809 d->rxsd_cl = NULL; in free_rx_bufs()
810 d->m = NULL; in free_rx_bufs()
811 if (++cidx == q->size) in free_rx_bufs()
819 refill_fl(adap, fl, min(16U, fl->size - fl->credits)); in __refill_fl()
825 uint32_t reclaimable = fl->size - fl->credits; in __refill_fl_lt()
832 * recycle_rx_buf - recycle a receive buffer
843 struct rx_desc *from = &q->desc[idx]; in recycle_rx_buf()
844 struct rx_desc *to = &q->desc[q->pidx]; in recycle_rx_buf()
846 q->sdesc[q->pidx] = q->sdesc[idx]; in recycle_rx_buf()
847 to->addr_lo = from->addr_lo; // already big endian in recycle_rx_buf()
848 to->addr_hi = from->addr_hi; // likewise in recycle_rx_buf()
850 to->len_gen = htobe32(V_FLD_GEN1(q->gen)); in recycle_rx_buf()
851 to->gen2 = htobe32(V_FLD_GEN2(q->gen)); in recycle_rx_buf()
852 q->credits++; in recycle_rx_buf()
854 if (++q->pidx == q->size) { in recycle_rx_buf()
855 q->pidx = 0; in recycle_rx_buf()
856 q->gen ^= 1; in recycle_rx_buf()
858 t3_write_reg(adap, A_SG_KDOORBELL, V_EGRCNTX(q->cntxt_id)); in recycle_rx_buf()
880 if ((err = bus_dma_tag_create(sc->parent_dmat, PAGE_SIZE, 0, in alloc_ring()
882 BUS_SPACE_MAXADDR, NULL, NULL, len, 1, in alloc_ring()
884 device_printf(sc->dev, "Cannot allocate descriptor tag\n"); in alloc_ring()
890 device_printf(sc->dev, "Cannot allocate descriptor memory\n"); in alloc_ring()
906 if ((err = bus_dma_tag_create(parent_entry_tag, 1, 0, in alloc_ring()
911 device_printf(sc->dev, "Cannot allocate descriptor entry tag\n"); in alloc_ring()
923 t3_write_reg(sc, A_PL_INT_ENABLE0, sc->slow_intr_mask); in sge_slow_intr_handler()
928 * sge_timer_cb - perform periodic maintenance of an SGE qset
929 * @data: the SGE queue set to maintain
931 * Runs periodically from a timer to perform maintenance of an SGE queue
934 * a) Cleans up any completed Tx descriptors that may still be pending.
935 * Normal descriptor cleanup happens when new packets are added to a Tx
936 * queue so this timer is relatively infrequent and does any cleanup only
937 * if the Tx queue has not seen any new packets in a while. We make a
939 * around if we cannot get a queue's lock (which most likely is because
946 * when out of memory a queue can become empty. We try to add only a few
947 * buffers here, the queue will be replenished fully as these new buffers
950 * c) Return coalesced response queue credits in case a response queue is
960 if ((sc->flags & USING_MSIX) == 0) { in sge_timer_cb()
968 if (sc->open_device_map == 0) in sge_timer_cb()
971 for (i = 0; i < sc->params.nports; i++) { in sge_timer_cb()
972 pi = &sc->port[i]; in sge_timer_cb()
973 for (j = 0; j < pi->nqsets; j++) { in sge_timer_cb()
974 qs = &sc->sge.qs[pi->first_qset + j]; in sge_timer_cb()
975 txq = &qs->txq[0]; in sge_timer_cb()
976 reclaim_ofl = txq[TXQ_OFLD].processed - txq[TXQ_OFLD].cleaned; in sge_timer_cb()
977 refill_rx = ((qs->fl[0].credits < qs->fl[0].size) || in sge_timer_cb()
978 (qs->fl[1].credits < qs->fl[1].size)); in sge_timer_cb()
980 taskqueue_enqueue(sc->tq, &pi->timer_reclaim_task); in sge_timer_cb()
987 if (sc->params.nports > 2) { in sge_timer_cb()
991 struct port_info *pi = &sc->port[i]; in sge_timer_cb()
995 (FW_TUNNEL_SGEEC_START + pi->first_qset)); in sge_timer_cb()
998 if (((sc->flags & USING_MSIX) == 0 || sc->params.nports > 2) && in sge_timer_cb()
999 sc->open_device_map != 0) in sge_timer_cb()
1000 callout_reset(&sc->sge_timer_ch, TX_RECLAIM_PERIOD, sge_timer_cb, sc); in sge_timer_cb()
1004 * This is meant to be a catch-all function to keep sge state private
1011 callout_init(&sc->sge_timer_ch, 1); in t3_sge_init_adapter()
1012 callout_reset(&sc->sge_timer_ch, TX_RECLAIM_PERIOD, sge_timer_cb, sc); in t3_sge_init_adapter()
1013 TASK_INIT(&sc->slow_intr_task, 0, sge_slow_intr_handler, sc); in t3_sge_init_adapter()
1020 callout_reset(&sc->sge_timer_ch, TX_RECLAIM_PERIOD, sge_timer_cb, sc); in t3_sge_reset_adapter()
1027 TASK_INIT(&pi->timer_reclaim_task, 0, sge_timer_reclaim, pi); in t3_sge_init_port()
1032 * refill_rspq - replenish an SGE response queue
1034 * @q: the response queue to replenish
1037 * Replenishes a response queue by making the supplied number of responses
1046 V_RSPQ(q->cntxt_id) | V_CREDITS(credits)); in refill_rspq()
1063 int i, nqsets = pi->nqsets; in sge_timer_reclaim()
1064 adapter_t *sc = pi->adapter; in sge_timer_reclaim()
1068 KASSERT((sc->flags & USING_MSIX) == 0, in sge_timer_reclaim()
1069 ("can't call timer reclaim for msi-x")); in sge_timer_reclaim()
1072 qs = &sc->sge.qs[pi->first_qset + i]; in sge_timer_reclaim()
1075 lock = (sc->flags & USING_MSIX) ? &qs->rspq.lock : in sge_timer_reclaim()
1076 &sc->sge.qs[0].rspq.lock; in sge_timer_reclaim()
1082 if (qs->fl[0].credits < qs->fl[0].size - 16) in sge_timer_reclaim()
1083 __refill_fl(sc, &qs->fl[0]); in sge_timer_reclaim()
1084 if (qs->fl[1].credits < qs->fl[1].size - 16) in sge_timer_reclaim()
1085 __refill_fl(sc, &qs->fl[1]); in sge_timer_reclaim()
1087 if (status & (1 << qs->rspq.cntxt_id)) { in sge_timer_reclaim()
1088 if (qs->rspq.credits) { in sge_timer_reclaim()
1089 refill_rspq(sc, &qs->rspq, 1); in sge_timer_reclaim()
1090 qs->rspq.credits--; in sge_timer_reclaim()
1092 1 << qs->rspq.cntxt_id); in sge_timer_reclaim()
1101 * init_qset_cntxt - initialize an SGE queue set context info
1102 * @qs: the queue set
1103 * @id: the queue set id
1105 * Initializes the TIDs and context ids for the queues of a queue set.
1111 qs->rspq.cntxt_id = id; in init_qset_cntxt()
1112 qs->fl[0].cntxt_id = 2 * id; in init_qset_cntxt()
1113 qs->fl[1].cntxt_id = 2 * id + 1; in init_qset_cntxt()
1114 qs->txq[TXQ_ETH].cntxt_id = FW_TUNNEL_SGEEC_START + id; in init_qset_cntxt()
1115 qs->txq[TXQ_ETH].token = FW_TUNNEL_TID_START + id; in init_qset_cntxt()
1116 qs->txq[TXQ_OFLD].cntxt_id = FW_OFLD_SGEEC_START + id; in init_qset_cntxt()
1117 qs->txq[TXQ_CTRL].cntxt_id = FW_CTRL_SGEEC_START + id; in init_qset_cntxt()
1118 qs->txq[TXQ_CTRL].token = FW_CTRL_TID_START + id; in init_qset_cntxt()
1121 mbufq_init(&qs->txq[TXQ_ETH].sendq, INT_MAX); in init_qset_cntxt()
1122 mbufq_init(&qs->txq[TXQ_OFLD].sendq, INT_MAX); in init_qset_cntxt()
1123 mbufq_init(&qs->txq[TXQ_CTRL].sendq, INT_MAX); in init_qset_cntxt()
1130 txq->in_use += ndesc; in txq_prod()
1132 * XXX we don't handle stopping of queue in txq_prod()
1135 txqs->gen = txq->gen; in txq_prod()
1136 txq->unacked += ndesc; in txq_prod()
1137 txqs->compl = (txq->unacked & 32) << (S_WR_COMPL - 5); in txq_prod()
1138 txq->unacked &= 31; in txq_prod()
1139 txqs->pidx = txq->pidx; in txq_prod()
1140 txq->pidx += ndesc; in txq_prod()
1142 if (((txqs->pidx > txq->cidx) && in txq_prod()
1143 (txq->pidx < txqs->pidx) && in txq_prod()
1144 (txq->pidx >= txq->cidx)) || in txq_prod()
1145 ((txqs->pidx < txq->cidx) && in txq_prod()
1146 (txq->pidx >= txq-> cidx)) || in txq_prod()
1147 ((txqs->pidx < txq->cidx) && in txq_prod()
1148 (txq->cidx < txqs->pidx))) in txq_prod()
1149 panic("txqs->pidx=%d txq->pidx=%d txq->cidx=%d", in txq_prod()
1150 txqs->pidx, txq->pidx, txq->cidx); in txq_prod()
1152 if (txq->pidx >= txq->size) { in txq_prod()
1153 txq->pidx -= txq->size; in txq_prod()
1154 txq->gen ^= 1; in txq_prod()
1160 * calc_tx_descs - calculate the number of Tx descriptors for a packet
1164 * Returns the number of Tx descriptors needed for the given Ethernet
1172 if (m->m_pkthdr.len <= PIO_LEN) in calc_tx_descs()
1173 return 1; in calc_tx_descs()
1176 if (m->m_pkthdr.csum_flags & CSUM_TSO) in calc_tx_descs()
1183 * make_sgl - populate a scatter/gather list for a packet
1189 * and returns the SGL size in 8-byte words. The caller must size the SGL
1206 sgp->len[idx] = htobe32(segs[i].ds_len); in make_sgl()
1207 sgp->addr[idx] = htobe64(segs[i].ds_addr); in make_sgl()
1208 idx ^= 1; in make_sgl()
1212 sgp->len[idx] = 0; in make_sgl()
1213 sgp->addr[idx] = 0; in make_sgl()
1218 * check_ring_tx_db - check and potentially ring a Tx queue's doorbell
1220 * @q: the Tx queue
1222 * Ring the doorbell if a Tx queue is asleep. There is a natural race,
1224 * then the interrupt handler will detect the outstanding TX packet
1233 clear_bit(TXQ_LAST_PKT_DB, &q->flags); in check_ring_tx_db()
1234 if (test_and_set_bit(TXQ_RUNNING, &q->flags) == 0) { in check_ring_tx_db()
1235 set_bit(TXQ_LAST_PKT_DB, &q->flags); in check_ring_tx_db()
1237 T3_TRACE1(adap->tb[q->cntxt_id & 7], "doorbell Tx, cntxt %d", in check_ring_tx_db()
1238 q->cntxt_id); in check_ring_tx_db()
1241 F_SELEGRCNTX | V_EGRCNTX(q->cntxt_id)); in check_ring_tx_db()
1244 if (mustring || ++q->db_pending >= 32) { in check_ring_tx_db()
1247 F_SELEGRCNTX | V_EGRCNTX(q->cntxt_id)); in check_ring_tx_db()
1248 q->db_pending = 0; in check_ring_tx_db()
1257 d->flit[TX_DESC_FLITS - 1] = htobe64(gen); in wr_gen2()
1262 * write_wr_hdr_sgl - write a WR header and, optionally, SGL
1263 * @ndesc: number of Tx descriptors spanned by the SGL
1264 * @txd: first Tx descriptor to be written
1266 * @txq: the SGE Tx queue
1274 * small enough to fit into one Tx descriptor it has already been written
1286 if (__predict_true(ndesc == 1)) { in write_wr_hdr_sgl()
1287 set_wr_hdr(wrp, htonl(F_WR_SOP | F_WR_EOP | V_WR_DATATYPE(1) | in write_wr_hdr_sgl()
1289 htonl(V_WR_LEN(flits + sgl_flits) | V_WR_GEN(txqs->gen)) | in write_wr_hdr_sgl()
1292 wr_gen2(txd, txqs->gen); in write_wr_hdr_sgl()
1295 unsigned int ogen = txqs->gen; in write_wr_hdr_sgl()
1299 wrp->wrh_hi = htonl(F_WR_SOP | V_WR_DATATYPE(1) | in write_wr_hdr_sgl()
1303 unsigned int avail = WR_FLITS - flits; in write_wr_hdr_sgl()
1307 memcpy(&txd->flit[flits], fp, avail * sizeof(*fp)); in write_wr_hdr_sgl()
1308 sgl_flits -= avail; in write_wr_hdr_sgl()
1309 ndesc--; in write_wr_hdr_sgl()
1315 if (++txqs->pidx == txq->size) { in write_wr_hdr_sgl()
1316 txqs->pidx = 0; in write_wr_hdr_sgl()
1317 txqs->gen ^= 1; in write_wr_hdr_sgl()
1318 txd = txq->desc; in write_wr_hdr_sgl()
1327 wrp->wrh_hi = htonl(V_WR_DATATYPE(1) | in write_wr_hdr_sgl()
1328 V_WR_SGLSFLT(1)) | wr_hi; in write_wr_hdr_sgl()
1329 wrp->wrh_lo = htonl(V_WR_LEN(min(WR_FLITS, in write_wr_hdr_sgl()
1330 sgl_flits + 1)) | in write_wr_hdr_sgl()
1331 V_WR_GEN(txqs->gen)) | wr_lo; in write_wr_hdr_sgl()
1332 wr_gen2(txd, txqs->gen); in write_wr_hdr_sgl()
1333 flits = 1; in write_wr_hdr_sgl()
1335 wrp->wrh_hi |= htonl(F_WR_EOP); in write_wr_hdr_sgl()
1337 wp->wrh_lo = htonl(V_WR_LEN(WR_FLITS) | V_WR_GEN(ogen)) | wr_lo; in write_wr_hdr_sgl()
1347 if ((m)->m_flags & M_VLANTAG) \
1348 cntrl |= F_TXPKT_VLAN_VLD | V_TXPKT_VLAN((m)->m_pkthdr.ether_vtag); \
1370 pi = qs->port; in t3_encap()
1371 sc = pi->adapter; in t3_encap()
1372 txq = &qs->txq[TXQ_ETH]; in t3_encap()
1373 txd = &txq->desc[txq->pidx]; in t3_encap()
1374 txsd = &txq->sdesc[txq->pidx]; in t3_encap()
1375 sgl = txq->txq_sgl; in t3_encap()
1380 mtx_assert(&qs->lock, MA_OWNED); in t3_encap()
1381 cntrl = V_TXPKT_INTF(pi->txpkt_intf); in t3_encap()
1382 KASSERT(m0->m_flags & M_PKTHDR, ("not packet header\n")); in t3_encap()
1384 if (m0->m_nextpkt == NULL && m0->m_next != NULL && in t3_encap()
1385 m0->m_pkthdr.csum_flags & (CSUM_TSO)) in t3_encap()
1386 tso_info = V_LSO_MSS(m0->m_pkthdr.tso_segsz); in t3_encap()
1388 if (m0->m_nextpkt != NULL) { in t3_encap()
1389 busdma_map_sg_vec(txq->entry_tag, txsd->map, m0, segs, &nsegs); in t3_encap()
1390 ndesc = 1; in t3_encap()
1393 if ((err = busdma_map_sg_collapse(txq->entry_tag, txsd->map, in t3_encap()
1399 mlen = m0->m_pkthdr.len; in t3_encap()
1404 KASSERT(m0->m_pkthdr.len, ("empty packet nsegs=%d", nsegs)); in t3_encap()
1405 txsd->m = m0; in t3_encap()
1407 if (m0->m_nextpkt != NULL) { in t3_encap()
1413 txq->txq_coalesced += nsegs; in t3_encap()
1415 flits = nsegs*2 + 1; in t3_encap()
1417 for (fidx = 1, i = 0; i < nsegs; i++, fidx += 2) { in t3_encap()
1421 int cflags = m0->m_pkthdr.csum_flags; in t3_encap()
1423 cntrl = V_TXPKT_INTF(pi->txpkt_intf); in t3_encap()
1433 hflit[1] = htonl(segs[i].ds_len | 0x80000000); in t3_encap()
1434 flit |= htobe64(1 << 24); in t3_encap()
1435 cbe = &cpl_batch->pkt_entry[i]; in t3_encap()
1436 cbe->cntrl = hflit[0]; in t3_encap()
1437 cbe->len = hflit[1]; in t3_encap()
1438 cbe->addr = htobe64(segs[i].ds_addr); in t3_encap()
1441 wr_hi = htonl(F_WR_SOP | F_WR_EOP | V_WR_DATATYPE(1) | in t3_encap()
1445 V_WR_GEN(txqs.gen)) | htonl(V_WR_TID(txq->token)); in t3_encap()
1448 ETHER_BPF_MTAP(pi->ifp, m0); in t3_encap()
1459 txd->flit[2] = 0; in t3_encap()
1462 hdr->cntrl = htonl(cntrl); in t3_encap()
1463 hdr->len = htonl(mlen | 0x80000000); in t3_encap()
1467 m0, mlen, m0->m_pkthdr.tso_segsz, in t3_encap()
1468 (int)m0->m_pkthdr.csum_flags, CSUM_BITS, m0->m_flags); in t3_encap()
1469 panic("tx tso packet too small"); in t3_encap()
1473 if (__predict_false(m0->m_len < TCPPKTHDRSIZE)) { in t3_encap()
1482 eth_type = eh->ether_type; in t3_encap()
1487 l3hdr = evh + 1; in t3_encap()
1488 eth_type = evh->evl_proto; in t3_encap()
1491 l3hdr = eh + 1; in t3_encap()
1497 tso_info |= V_LSO_IPHDR_WORDS(ip->ip_hl); in t3_encap()
1498 tcp = (struct tcphdr *)(ip + 1); in t3_encap()
1502 KASSERT(ip6->ip6_nxt == IPPROTO_TCP, in t3_encap()
1504 __func__, ip6->ip6_nxt)); in t3_encap()
1508 tcp = (struct tcphdr *)(ip6 + 1); in t3_encap()
1512 tso_info |= V_LSO_TCPHDR_WORDS(tcp->th_off); in t3_encap()
1513 hdr->lso_info = htonl(tso_info); in t3_encap()
1520 txsd->m = NULL; in t3_encap()
1521 m_copydata(m0, 0, mlen, (caddr_t)&txd->flit[3]); in t3_encap()
1527 V_WR_GEN(txqs.gen) | V_WR_TID(txq->token)); in t3_encap()
1528 set_wr_hdr(&hdr->wr, wr_hi, wr_lo); in t3_encap()
1530 ETHER_BPF_MTAP(pi->ifp, m0); in t3_encap()
1542 if (__predict_false(!(m0->m_pkthdr.csum_flags & CSUM_IP))) in t3_encap()
1544 if (__predict_false(!(m0->m_pkthdr.csum_flags & (CSUM_TCP | in t3_encap()
1547 cpl->cntrl = htonl(cntrl); in t3_encap()
1548 cpl->len = htonl(mlen | 0x80000000); in t3_encap()
1551 txsd->m = NULL; in t3_encap()
1552 m_copydata(m0, 0, mlen, (caddr_t)&txd->flit[2]); in t3_encap()
1559 V_WR_GEN(txqs.gen) | V_WR_TID(txq->token)); in t3_encap()
1560 set_wr_hdr(&cpl->wr, wr_hi, wr_lo); in t3_encap()
1562 ETHER_BPF_MTAP(pi->ifp, m0); in t3_encap()
1571 sgp = (ndesc == 1) ? (struct sg_ent *)&txd->flit[flits] : sgl; in t3_encap()
1576 ETHER_BPF_MTAP(pi->ifp, m0); in t3_encap()
1580 wr_lo = htonl(V_WR_TID(txq->token)); in t3_encap()
1596 check_ring_tx_db(qs->port->adapter, &qs->txq[TXQ_ETH], 1); in cxgb_debugnet_encap()
1609 struct sge_txq *txq = &qs->txq[TXQ_ETH]; in cxgb_tx_watchdog()
1611 if (qs->coalescing != 0 && in cxgb_tx_watchdog()
1612 (txq->in_use <= cxgb_tx_coalesce_enable_stop) && in cxgb_tx_watchdog()
1614 qs->coalescing = 0; in cxgb_tx_watchdog()
1615 else if (qs->coalescing == 0 && in cxgb_tx_watchdog()
1616 (txq->in_use >= cxgb_tx_coalesce_enable_start)) in cxgb_tx_watchdog()
1617 qs->coalescing = 1; in cxgb_tx_watchdog()
1619 qs->qs_flags |= QS_FLUSHING; in cxgb_tx_watchdog()
1621 qs->qs_flags &= ~QS_FLUSHING; in cxgb_tx_watchdog()
1624 if (if_getdrvflags(qs->port->ifp) & IFF_DRV_RUNNING) in cxgb_tx_watchdog()
1625 callout_reset_on(&txq->txq_watchdog, hz/4, cxgb_tx_watchdog, in cxgb_tx_watchdog()
1626 qs, txq->txq_watchdog.c_cpu); in cxgb_tx_watchdog()
1633 struct sge_txq *txq = &qs->txq[TXQ_ETH]; in cxgb_tx_timeout()
1635 if (qs->coalescing == 0 && (txq->in_use >= (txq->size>>3))) in cxgb_tx_timeout()
1636 qs->coalescing = 1; in cxgb_tx_timeout()
1638 qs->qs_flags |= QS_TIMEOUT; in cxgb_tx_timeout()
1640 qs->qs_flags &= ~QS_TIMEOUT; in cxgb_tx_timeout()
1649 struct sge_txq *txq = &qs->txq[TXQ_ETH]; in cxgb_start_locked()
1650 struct port_info *pi = qs->port; in cxgb_start_locked()
1651 if_t ifp = pi->ifp; in cxgb_start_locked()
1653 if (qs->qs_flags & (QS_FLUSHING|QS_TIMEOUT)) in cxgb_start_locked()
1656 if (!pi->link_config.link_ok) { in cxgb_start_locked()
1662 pi->link_config.link_ok) { in cxgb_start_locked()
1665 if (txq->size - txq->in_use <= TX_MAX_DESC) in cxgb_start_locked()
1680 if (txq->db_pending) in cxgb_start_locked()
1681 check_ring_tx_db(pi->adapter, txq, 1); in cxgb_start_locked()
1683 if (!TXQ_RING_EMPTY(qs) && callout_pending(&txq->txq_timer) == 0 && in cxgb_start_locked()
1684 pi->link_config.link_ok) in cxgb_start_locked()
1685 callout_reset_on(&txq->txq_timer, 1, cxgb_tx_timeout, in cxgb_start_locked()
1686 qs, txq->txq_timer.c_cpu); in cxgb_start_locked()
1694 struct port_info *pi = qs->port; in cxgb_transmit_locked()
1695 struct sge_txq *txq = &qs->txq[TXQ_ETH]; in cxgb_transmit_locked()
1696 struct buf_ring *br = txq->txq_mr; in cxgb_transmit_locked()
1699 avail = txq->size - txq->in_use; in cxgb_transmit_locked()
1704 * - we aren't coalescing (ring < 3/4 full) in cxgb_transmit_locked()
1705 * - the link is up -- checked in caller in cxgb_transmit_locked()
1706 * - there are no packets enqueued already in cxgb_transmit_locked()
1707 * - there is space in hardware transmit queue in cxgb_transmit_locked()
1716 if (txq->db_pending) in cxgb_transmit_locked()
1717 check_ring_tx_db(pi->adapter, txq, 1); in cxgb_transmit_locked()
1723 txq->txq_direct_packets++; in cxgb_transmit_locked()
1724 txq->txq_direct_bytes += m->m_pkthdr.len; in cxgb_transmit_locked()
1730 if (!TXQ_RING_EMPTY(qs) && pi->link_config.link_ok && in cxgb_transmit_locked()
1733 else if (!TXQ_RING_EMPTY(qs) && !callout_pending(&txq->txq_timer)) in cxgb_transmit_locked()
1734 callout_reset_on(&txq->txq_timer, 1, cxgb_tx_timeout, in cxgb_transmit_locked()
1735 qs, txq->txq_timer.c_cpu); in cxgb_transmit_locked()
1744 int error, qidx = pi->first_qset; in cxgb_transmit()
1747 ||(!pi->link_config.link_ok)) { in cxgb_transmit()
1754 qidx = (m->m_pkthdr.flowid % pi->nqsets) + pi->first_qset; in cxgb_transmit()
1756 qs = &pi->adapter->sge.qs[qidx]; in cxgb_transmit()
1763 error = drbr_enqueue(ifp, qs->txq[TXQ_ETH].txq_mr, m); in cxgb_transmit()
1773 * no-op for now in cxgb_qflush()
1779 * write_imm - write a packet into a Tx descriptor as immediate data
1780 * @d: the Tx descriptor to write
1785 * Writes a packet as immediate data into a Tx descriptor. The packet
1801 memcpy(&to[1], &from[1], len - sizeof(*from)); in write_imm()
1802 wr_hi = from->wrh_hi | htonl(F_WR_SOP | F_WR_EOP | in write_imm()
1804 wr_lo = from->wrh_lo | htonl(V_WR_GEN(gen) | V_WR_LEN((len + 7) / 8)); in write_imm()
1811 * check_desc_avail - check descriptor availability on a send queue
1813 * @q: the TX queue
1815 * @ndesc: the number of Tx descriptors needed
1816 * @qid: the Tx queue number in its queue set (TXQ_OFLD or TXQ_CTRL)
1818 * Checks if the requested number of Tx descriptors is available on an
1819 * SGE send queue. If the queue is already suspended or not enough
1821 * Must be called with the Tx queue locked.
1823 * Returns 0 if enough descriptors are available, 1 if there aren't
1834 * XXX We currently only use this for checking the control queue in check_desc_avail()
1835 * the control queue is only used for binding qsets which happens in check_desc_avail()
1838 if (__predict_false(!mbufq_empty(&q->sendq))) { in check_desc_avail()
1839 addq_exit: (void )mbufq_enqueue(&q->sendq, m); in check_desc_avail()
1840 return 1; in check_desc_avail()
1842 if (__predict_false(q->size - q->in_use < ndesc)) { in check_desc_avail()
1846 setbit(&qs->txq_stopped, qid); in check_desc_avail()
1848 test_and_clear_bit(qid, &qs->txq_stopped)) in check_desc_avail()
1851 q->stops++; in check_desc_avail()
1859 * reclaim_completed_tx_imm - reclaim completed control-queue Tx descs
1860 * @q: the SGE control Tx queue
1862 * This is a variant of reclaim_completed_tx() that is used for Tx queues
1869 unsigned int reclaim = q->processed - q->cleaned; in reclaim_completed_tx_imm()
1871 q->in_use -= reclaim; in reclaim_completed_tx_imm()
1872 q->cleaned += reclaim; in reclaim_completed_tx_imm()
1876 * ctrl_xmit - send a packet through an SGE control Tx queue
1878 * @q: the control queue
1881 * Send a packet through an SGE control Tx queue. Packets sent through
1882 * a control queue must fit entirely as immediate data in a single Tx
1890 struct sge_txq *q = &qs->txq[TXQ_CTRL]; in ctrl_xmit()
1892 KASSERT(m->m_len <= WR_LEN, ("%s: bad tx data", __func__)); in ctrl_xmit()
1894 wrp->wrh_hi |= htonl(F_WR_SOP | F_WR_EOP); in ctrl_xmit()
1895 wrp->wrh_lo = htonl(V_WR_TID(q->token)); in ctrl_xmit()
1900 ret = check_desc_avail(adap, q, m, 1, TXQ_CTRL); in ctrl_xmit()
1902 if (ret == 1) { in ctrl_xmit()
1908 write_imm(&q->desc[q->pidx], m->m_data, m->m_len, q->gen); in ctrl_xmit()
1910 q->in_use++; in ctrl_xmit()
1911 if (++q->pidx >= q->size) { in ctrl_xmit()
1912 q->pidx = 0; in ctrl_xmit()
1913 q->gen ^= 1; in ctrl_xmit()
1918 F_SELEGRCNTX | V_EGRCNTX(q->cntxt_id)); in ctrl_xmit()
1926 * restart_ctrlq - restart a suspended control queue
1927 * @qs: the queue set containing the control queue
1929 * Resumes transmission on a suspended Tx control queue.
1936 struct sge_txq *q = &qs->txq[TXQ_CTRL]; in restart_ctrlq()
1937 adapter_t *adap = qs->port->adapter; in restart_ctrlq()
1942 while (q->in_use < q->size && in restart_ctrlq()
1943 (m = mbufq_dequeue(&q->sendq)) != NULL) { in restart_ctrlq()
1945 write_imm(&q->desc[q->pidx], m->m_data, m->m_len, q->gen); in restart_ctrlq()
1948 if (++q->pidx >= q->size) { in restart_ctrlq()
1949 q->pidx = 0; in restart_ctrlq()
1950 q->gen ^= 1; in restart_ctrlq()
1952 q->in_use++; in restart_ctrlq()
1954 if (!mbufq_empty(&q->sendq)) { in restart_ctrlq()
1955 setbit(&qs->txq_stopped, TXQ_CTRL); in restart_ctrlq()
1958 test_and_clear_bit(TXQ_CTRL, &qs->txq_stopped)) in restart_ctrlq()
1960 q->stops++; in restart_ctrlq()
1964 F_SELEGRCNTX | V_EGRCNTX(q->cntxt_id)); in restart_ctrlq()
1969 * Send a management message through control queue 0
1974 return ctrl_xmit(adap, &adap->sge.qs[0], m); in t3_mgmt_tx()
1978 * free_qset - free the resources of an SGE queue set
1979 * @sc: the controller owning the queue set
1980 * @q: the queue set
1982 * Release the HW and SW resources associated with an SGE queue set, such
1984 * queue set must be quiesced prior to calling this.
1992 if (q->txq[TXQ_ETH].txq_mr != NULL) in t3_free_qset()
1993 buf_ring_free(q->txq[TXQ_ETH].txq_mr, M_DEVBUF); in t3_free_qset()
1994 if (q->txq[TXQ_ETH].txq_ifq != NULL) { in t3_free_qset()
1995 ifq_delete(q->txq[TXQ_ETH].txq_ifq); in t3_free_qset()
1996 free(q->txq[TXQ_ETH].txq_ifq, M_DEVBUF); in t3_free_qset()
2000 if (q->fl[i].desc) { in t3_free_qset()
2001 mtx_lock_spin(&sc->sge.reg_lock); in t3_free_qset()
2002 t3_sge_disable_fl(sc, q->fl[i].cntxt_id); in t3_free_qset()
2003 mtx_unlock_spin(&sc->sge.reg_lock); in t3_free_qset()
2004 bus_dmamap_unload(q->fl[i].desc_tag, q->fl[i].desc_map); in t3_free_qset()
2005 bus_dmamem_free(q->fl[i].desc_tag, q->fl[i].desc, in t3_free_qset()
2006 q->fl[i].desc_map); in t3_free_qset()
2007 bus_dma_tag_destroy(q->fl[i].desc_tag); in t3_free_qset()
2008 bus_dma_tag_destroy(q->fl[i].entry_tag); in t3_free_qset()
2010 if (q->fl[i].sdesc) { in t3_free_qset()
2011 free_rx_bufs(sc, &q->fl[i]); in t3_free_qset()
2012 free(q->fl[i].sdesc, M_DEVBUF); in t3_free_qset()
2016 mtx_unlock(&q->lock); in t3_free_qset()
2017 MTX_DESTROY(&q->lock); in t3_free_qset()
2019 if (q->txq[i].desc) { in t3_free_qset()
2020 mtx_lock_spin(&sc->sge.reg_lock); in t3_free_qset()
2021 t3_sge_enable_ecntxt(sc, q->txq[i].cntxt_id, 0); in t3_free_qset()
2022 mtx_unlock_spin(&sc->sge.reg_lock); in t3_free_qset()
2023 bus_dmamap_unload(q->txq[i].desc_tag, in t3_free_qset()
2024 q->txq[i].desc_map); in t3_free_qset()
2025 bus_dmamem_free(q->txq[i].desc_tag, q->txq[i].desc, in t3_free_qset()
2026 q->txq[i].desc_map); in t3_free_qset()
2027 bus_dma_tag_destroy(q->txq[i].desc_tag); in t3_free_qset()
2028 bus_dma_tag_destroy(q->txq[i].entry_tag); in t3_free_qset()
2030 if (q->txq[i].sdesc) { in t3_free_qset()
2031 free(q->txq[i].sdesc, M_DEVBUF); in t3_free_qset()
2035 if (q->rspq.desc) { in t3_free_qset()
2036 mtx_lock_spin(&sc->sge.reg_lock); in t3_free_qset()
2037 t3_sge_disable_rspcntxt(sc, q->rspq.cntxt_id); in t3_free_qset()
2038 mtx_unlock_spin(&sc->sge.reg_lock); in t3_free_qset()
2040 bus_dmamap_unload(q->rspq.desc_tag, q->rspq.desc_map); in t3_free_qset()
2041 bus_dmamem_free(q->rspq.desc_tag, q->rspq.desc, in t3_free_qset()
2042 q->rspq.desc_map); in t3_free_qset()
2043 bus_dma_tag_destroy(q->rspq.desc_tag); in t3_free_qset()
2044 MTX_DESTROY(&q->rspq.lock); in t3_free_qset()
2048 tcp_lro_free(&q->lro.ctrl); in t3_free_qset()
2055 * t3_free_sge_resources - free SGE resources
2058 * Frees resources used by the SGE queue sets.
2066 TXQ_LOCK(&sc->sge.qs[i]); in t3_free_sge_resources()
2067 t3_free_qset(sc, &sc->sge.qs[i]); in t3_free_sge_resources()
2072 * t3_sge_start - enable SGE
2085 * t3_sge_stop - disable SGE operation
2090 * case it also disables any pending queue restart tasklets. Note that
2105 * t3_free_tx_desc - reclaims Tx descriptors and their buffers
2107 * @q: the Tx queue to reclaim descriptors from
2112 * Reclaims Tx descriptors from an SGE Tx queue and frees the associated
2113 * Tx buffers. Called with the Tx queue lock held.
2118 t3_free_tx_desc(struct sge_qset *qs, int reclaimable, int queue) in t3_free_tx_desc() argument
2122 struct sge_txq *q = &qs->txq[queue]; in t3_free_tx_desc()
2125 T3_TRACE2(sc->tb[q->cntxt_id & 7], in t3_free_tx_desc()
2126 "reclaiming %u Tx descriptors at cidx %u", reclaimable, cidx); in t3_free_tx_desc()
2128 cidx = q->cidx; in t3_free_tx_desc()
2129 mask = q->size - 1; in t3_free_tx_desc()
2130 txsd = &q->sdesc[cidx]; in t3_free_tx_desc()
2132 mtx_assert(&qs->lock, MA_OWNED); in t3_free_tx_desc()
2133 while (reclaimable--) { in t3_free_tx_desc()
2134 prefetch(q->sdesc[(cidx + 1) & mask].m); in t3_free_tx_desc()
2135 prefetch(q->sdesc[(cidx + 2) & mask].m); in t3_free_tx_desc()
2137 if (txsd->m != NULL) { in t3_free_tx_desc()
2138 if (txsd->flags & TX_SW_DESC_MAPPED) { in t3_free_tx_desc()
2139 bus_dmamap_unload(q->entry_tag, txsd->map); in t3_free_tx_desc()
2140 txsd->flags &= ~TX_SW_DESC_MAPPED; in t3_free_tx_desc()
2142 m_freem_list(txsd->m); in t3_free_tx_desc()
2143 txsd->m = NULL; in t3_free_tx_desc()
2145 q->txq_skipped++; in t3_free_tx_desc()
2148 if (++cidx == q->size) { in t3_free_tx_desc()
2150 txsd = q->sdesc; in t3_free_tx_desc()
2153 q->cidx = cidx; in t3_free_tx_desc()
2158 * is_new_response - check if a response is newly written
2160 * @q: the response queue
2169 return (r->intr_gen & F_RSPD_GEN2) == q->gen; in is_new_response()
2183 * write_ofld_wr - write an offload work request
2186 * @q: the Tx queue
2187 * @pidx: index of the first Tx descriptor to write
2201 struct sg_ent *sgp, t3sgl[TX_MAX_SEGS / 2 + 1]; in write_ofld_wr()
2202 struct tx_desc *d = &q->desc[pidx]; in write_ofld_wr()
2208 from = (void *)(oh + 1); /* Start of WR within mbuf */ in write_ofld_wr()
2209 wrlen = m->m_len - sizeof(*oh); in write_ofld_wr()
2211 if (!(oh->flags & F_HDR_SGL)) { in write_ofld_wr()
2215 * mbuf with "real" immediate tx data will be enqueue_wr'd by in write_ofld_wr()
2219 if (!(oh->flags & F_HDR_DF)) in write_ofld_wr()
2224 memcpy(&d->flit[1], &from[1], wrlen - sizeof(*from)); in write_ofld_wr()
2226 sgl = oh->sgl; in write_ofld_wr()
2228 sgp = (ndesc == 1) ? (struct sg_ent *)&d->flit[flits] : t3sgl; in write_ofld_wr()
2230 nsegs = sgl->sg_nseg; in write_ofld_wr()
2231 segs = sgl->sg_segs; in write_ofld_wr()
2236 sgp->len[idx] = htobe32(segs[i].ss_len); in write_ofld_wr()
2237 sgp->addr[idx] = htobe64(segs[i].ss_paddr); in write_ofld_wr()
2238 idx ^= 1; in write_ofld_wr()
2241 sgp->len[idx] = 0; in write_ofld_wr()
2242 sgp->addr[idx] = 0; in write_ofld_wr()
2251 from->wrh_hi, from->wrh_lo); in write_ofld_wr()
2255 * ofld_xmit - send a packet through an offload queue
2257 * @q: the Tx offload queue
2260 * Send an offload packet through an SGE offload queue.
2268 struct sge_txq *q = &qs->txq[TXQ_OFLD]; in ofld_xmit()
2271 ndesc = G_HDR_NDESC(oh->flags); in ofld_xmit()
2277 if (ret == 1) { in ofld_xmit()
2284 gen = q->gen; in ofld_xmit()
2285 q->in_use += ndesc; in ofld_xmit()
2286 pidx = q->pidx; in ofld_xmit()
2287 q->pidx += ndesc; in ofld_xmit()
2288 if (q->pidx >= q->size) { in ofld_xmit()
2289 q->pidx -= q->size; in ofld_xmit()
2290 q->gen ^= 1; in ofld_xmit()
2294 check_ring_tx_db(adap, q, 1); in ofld_xmit()
2301 * restart_offloadq - restart a suspended offload queue
2302 * @qs: the queue set containing the offload queue
2304 * Resumes transmission on a suspended Tx offload queue.
2311 struct sge_txq *q = &qs->txq[TXQ_OFLD]; in restart_offloadq()
2312 adapter_t *adap = qs->port->adapter; in restart_offloadq()
2316 while ((m = mbufq_first(&q->sendq)) != NULL) { in restart_offloadq()
2319 unsigned int ndesc = G_HDR_NDESC(oh->flags); in restart_offloadq()
2321 if (__predict_false(q->size - q->in_use < ndesc)) { in restart_offloadq()
2322 setbit(&qs->txq_stopped, TXQ_OFLD); in restart_offloadq()
2324 test_and_clear_bit(TXQ_OFLD, &qs->txq_stopped)) in restart_offloadq()
2326 q->stops++; in restart_offloadq()
2330 gen = q->gen; in restart_offloadq()
2331 q->in_use += ndesc; in restart_offloadq()
2332 pidx = q->pidx; in restart_offloadq()
2333 q->pidx += ndesc; in restart_offloadq()
2334 if (q->pidx >= q->size) { in restart_offloadq()
2335 q->pidx -= q->size; in restart_offloadq()
2336 q->gen ^= 1; in restart_offloadq()
2339 (void)mbufq_dequeue(&q->sendq); in restart_offloadq()
2345 set_bit(TXQ_RUNNING, &q->flags); in restart_offloadq()
2346 set_bit(TXQ_LAST_PKT_DB, &q->flags); in restart_offloadq()
2351 F_SELEGRCNTX | V_EGRCNTX(q->cntxt_id)); in restart_offloadq()
2355 * t3_offload_tx - send an offload packet
2359 * appropriate Tx queue as follows: bit 0 indicates whether the packet
2360 * should be sent as regular or control, bits 1-3 select the queue set.
2366 struct sge_qset *qs = &sc->sge.qs[G_HDR_QSET(oh->flags)]; in t3_offload_tx()
2368 if (oh->flags & F_HDR_CTRL) { in t3_offload_tx()
2379 struct adapter *sc = qs->port->adapter; in restart_tx()
2381 if (isset(&qs->txq_stopped, TXQ_OFLD) && in restart_tx()
2382 should_restart_tx(&qs->txq[TXQ_OFLD]) && in restart_tx()
2383 test_and_clear_bit(TXQ_OFLD, &qs->txq_stopped)) { in restart_tx()
2384 qs->txq[TXQ_OFLD].restarts++; in restart_tx()
2385 taskqueue_enqueue(sc->tq, &qs->txq[TXQ_OFLD].qresume_task); in restart_tx()
2388 if (isset(&qs->txq_stopped, TXQ_CTRL) && in restart_tx()
2389 should_restart_tx(&qs->txq[TXQ_CTRL]) && in restart_tx()
2390 test_and_clear_bit(TXQ_CTRL, &qs->txq_stopped)) { in restart_tx()
2391 qs->txq[TXQ_CTRL].restarts++; in restart_tx()
2392 taskqueue_enqueue(sc->tq, &qs->txq[TXQ_CTRL].qresume_task); in restart_tx()
2397 * t3_sge_alloc_qset - initialize an SGE queue set
2399 * @id: the queue set id
2400 * @nports: how many Ethernet ports will be using this queue set
2401 * @irq_vec_idx: the IRQ vector index for response queue interrupts
2402 * @p: configuration parameters for this queue set
2403 * @ntxq: number of Tx queues for the queue set
2404 * @pi: port info for queue set
2406 * Allocate resources and initialize an SGE queue set. A queue set
2407 * comprises a response queue, two Rx free-buffer queues, and up to 3
2408 * Tx queues. The Tx queues are assigned roles in the order Ethernet
2409 * queue, offload queue, and control queue.
2415 struct sge_qset *q = &sc->sge.qs[id]; in t3_sge_alloc_qset()
2418 MTX_INIT(&q->lock, q->namebuf, NULL, MTX_DEF); in t3_sge_alloc_qset()
2419 q->port = pi; in t3_sge_alloc_qset()
2420 q->adap = sc; in t3_sge_alloc_qset()
2422 q->txq[TXQ_ETH].txq_mr = buf_ring_alloc(cxgb_txq_buf_ring_size, in t3_sge_alloc_qset()
2423 M_DEVBUF, M_WAITOK, &q->lock); in t3_sge_alloc_qset()
2424 if ((q->txq[TXQ_ETH].txq_ifq = malloc(sizeof(struct ifaltq), M_DEVBUF, in t3_sge_alloc_qset()
2426 device_printf(sc->dev, "failed to allocate ifq\n"); in t3_sge_alloc_qset()
2429 ifq_init(q->txq[TXQ_ETH].txq_ifq, pi->ifp); in t3_sge_alloc_qset()
2430 callout_init(&q->txq[TXQ_ETH].txq_timer, 1); in t3_sge_alloc_qset()
2431 callout_init(&q->txq[TXQ_ETH].txq_watchdog, 1); in t3_sge_alloc_qset()
2432 q->txq[TXQ_ETH].txq_timer.c_cpu = id % mp_ncpus; in t3_sge_alloc_qset()
2433 q->txq[TXQ_ETH].txq_watchdog.c_cpu = id % mp_ncpus; in t3_sge_alloc_qset()
2436 q->idx = id; in t3_sge_alloc_qset()
2437 if ((ret = alloc_ring(sc, p->fl_size, sizeof(struct rx_desc), in t3_sge_alloc_qset()
2438 sizeof(struct rx_sw_desc), &q->fl[0].phys_addr, in t3_sge_alloc_qset()
2439 &q->fl[0].desc, &q->fl[0].sdesc, in t3_sge_alloc_qset()
2440 &q->fl[0].desc_tag, &q->fl[0].desc_map, in t3_sge_alloc_qset()
2441 sc->rx_dmat, &q->fl[0].entry_tag)) != 0) { in t3_sge_alloc_qset()
2446 if ((ret = alloc_ring(sc, p->jumbo_size, sizeof(struct rx_desc), in t3_sge_alloc_qset()
2447 sizeof(struct rx_sw_desc), &q->fl[1].phys_addr, in t3_sge_alloc_qset()
2448 &q->fl[1].desc, &q->fl[1].sdesc, in t3_sge_alloc_qset()
2449 &q->fl[1].desc_tag, &q->fl[1].desc_map, in t3_sge_alloc_qset()
2450 sc->rx_jumbo_dmat, &q->fl[1].entry_tag)) != 0) { in t3_sge_alloc_qset()
2455 if ((ret = alloc_ring(sc, p->rspq_size, sizeof(struct rsp_desc), 0, in t3_sge_alloc_qset()
2456 &q->rspq.phys_addr, &q->rspq.desc, NULL, in t3_sge_alloc_qset()
2457 &q->rspq.desc_tag, &q->rspq.desc_map, in t3_sge_alloc_qset()
2463 snprintf(q->rspq.lockbuf, RSPQ_NAME_LEN, "t3 rspq lock %d:%d", in t3_sge_alloc_qset()
2464 device_get_unit(sc->dev), irq_vec_idx); in t3_sge_alloc_qset()
2465 MTX_INIT(&q->rspq.lock, q->rspq.lockbuf, NULL, MTX_DEF); in t3_sge_alloc_qset()
2470 if ((ret = alloc_ring(sc, p->txq_size[i], in t3_sge_alloc_qset()
2472 &q->txq[i].phys_addr, &q->txq[i].desc, in t3_sge_alloc_qset()
2473 &q->txq[i].sdesc, &q->txq[i].desc_tag, in t3_sge_alloc_qset()
2474 &q->txq[i].desc_map, in t3_sge_alloc_qset()
2475 sc->tx_dmat, &q->txq[i].entry_tag)) != 0) { in t3_sge_alloc_qset()
2476 printf("error %d from alloc ring tx %i\n", ret, i); in t3_sge_alloc_qset()
2479 mbufq_init(&q->txq[i].sendq, INT_MAX); in t3_sge_alloc_qset()
2480 q->txq[i].gen = 1; in t3_sge_alloc_qset()
2481 q->txq[i].size = p->txq_size[i]; in t3_sge_alloc_qset()
2485 TASK_INIT(&q->txq[TXQ_OFLD].qresume_task, 0, restart_offloadq, q); in t3_sge_alloc_qset()
2487 TASK_INIT(&q->txq[TXQ_CTRL].qresume_task, 0, restart_ctrlq, q); in t3_sge_alloc_qset()
2488 TASK_INIT(&q->txq[TXQ_ETH].qreclaim_task, 0, sge_txq_reclaim_handler, q); in t3_sge_alloc_qset()
2489 TASK_INIT(&q->txq[TXQ_OFLD].qreclaim_task, 0, sge_txq_reclaim_handler, q); in t3_sge_alloc_qset()
2491 q->fl[0].gen = q->fl[1].gen = 1; in t3_sge_alloc_qset()
2492 q->fl[0].size = p->fl_size; in t3_sge_alloc_qset()
2493 q->fl[1].size = p->jumbo_size; in t3_sge_alloc_qset()
2495 q->rspq.gen = 1; in t3_sge_alloc_qset()
2496 q->rspq.cidx = 0; in t3_sge_alloc_qset()
2497 q->rspq.size = p->rspq_size; in t3_sge_alloc_qset()
2499 q->txq[TXQ_ETH].stop_thres = nports * in t3_sge_alloc_qset()
2500 flits_to_desc(sgl_len(TX_MAX_SEGS + 1) + 3); in t3_sge_alloc_qset()
2502 q->fl[0].buf_size = MCLBYTES; in t3_sge_alloc_qset()
2503 q->fl[0].zone = zone_pack; in t3_sge_alloc_qset()
2504 q->fl[0].type = EXT_PACKET; in t3_sge_alloc_qset()
2506 if (p->jumbo_buf_size == MJUM16BYTES) { in t3_sge_alloc_qset()
2507 q->fl[1].zone = zone_jumbo16; in t3_sge_alloc_qset()
2508 q->fl[1].type = EXT_JUMBO16; in t3_sge_alloc_qset()
2509 } else if (p->jumbo_buf_size == MJUM9BYTES) { in t3_sge_alloc_qset()
2510 q->fl[1].zone = zone_jumbo9; in t3_sge_alloc_qset()
2511 q->fl[1].type = EXT_JUMBO9; in t3_sge_alloc_qset()
2512 } else if (p->jumbo_buf_size == MJUMPAGESIZE) { in t3_sge_alloc_qset()
2513 q->fl[1].zone = zone_jumbop; in t3_sge_alloc_qset()
2514 q->fl[1].type = EXT_JUMBOP; in t3_sge_alloc_qset()
2516 KASSERT(0, ("can't deal with jumbo_buf_size %d.", p->jumbo_buf_size)); in t3_sge_alloc_qset()
2520 q->fl[1].buf_size = p->jumbo_buf_size; in t3_sge_alloc_qset()
2523 q->lro.enabled = !!(if_getcapenable(pi->ifp) & IFCAP_LRO); in t3_sge_alloc_qset()
2525 ret = tcp_lro_init(&q->lro.ctrl); in t3_sge_alloc_qset()
2531 q->lro.ctrl.ifp = pi->ifp; in t3_sge_alloc_qset()
2533 mtx_lock_spin(&sc->sge.reg_lock); in t3_sge_alloc_qset()
2534 ret = -t3_sge_init_rspcntxt(sc, q->rspq.cntxt_id, irq_vec_idx, in t3_sge_alloc_qset()
2535 q->rspq.phys_addr, q->rspq.size, in t3_sge_alloc_qset()
2536 q->fl[0].buf_size, 1, 0); in t3_sge_alloc_qset()
2543 ret = -t3_sge_init_flcntxt(sc, q->fl[i].cntxt_id, 0, in t3_sge_alloc_qset()
2544 q->fl[i].phys_addr, q->fl[i].size, in t3_sge_alloc_qset()
2545 q->fl[i].buf_size, p->cong_thres, 1, in t3_sge_alloc_qset()
2553 ret = -t3_sge_init_ecntxt(sc, q->txq[TXQ_ETH].cntxt_id, USE_GTS, in t3_sge_alloc_qset()
2554 SGE_CNTXT_ETH, id, q->txq[TXQ_ETH].phys_addr, in t3_sge_alloc_qset()
2555 q->txq[TXQ_ETH].size, q->txq[TXQ_ETH].token, in t3_sge_alloc_qset()
2556 1, 0); in t3_sge_alloc_qset()
2562 if (ntxq > 1) { in t3_sge_alloc_qset()
2563 ret = -t3_sge_init_ecntxt(sc, q->txq[TXQ_OFLD].cntxt_id, in t3_sge_alloc_qset()
2565 q->txq[TXQ_OFLD].phys_addr, in t3_sge_alloc_qset()
2566 q->txq[TXQ_OFLD].size, 0, 1, 0); in t3_sge_alloc_qset()
2574 ret = -t3_sge_init_ecntxt(sc, q->txq[TXQ_CTRL].cntxt_id, 0, in t3_sge_alloc_qset()
2576 q->txq[TXQ_CTRL].phys_addr, in t3_sge_alloc_qset()
2577 q->txq[TXQ_CTRL].size, in t3_sge_alloc_qset()
2578 q->txq[TXQ_CTRL].token, 1, 0); in t3_sge_alloc_qset()
2585 mtx_unlock_spin(&sc->sge.reg_lock); in t3_sge_alloc_qset()
2588 refill_fl(sc, &q->fl[0], q->fl[0].size); in t3_sge_alloc_qset()
2589 refill_fl(sc, &q->fl[1], q->fl[1].size); in t3_sge_alloc_qset()
2590 refill_rspq(sc, &q->rspq, q->rspq.size - 1); in t3_sge_alloc_qset()
2592 t3_write_reg(sc, A_SG_GTS, V_RSPQ(q->rspq.cntxt_id) | in t3_sge_alloc_qset()
2593 V_NEWTIMER(q->rspq.holdoff_tmr)); in t3_sge_alloc_qset()
2598 mtx_unlock_spin(&sc->sge.reg_lock); in t3_sge_alloc_qset()
2615 struct port_info *pi = &adap->port[adap->rxpkt_map[cpl->iff]]; in t3_rx_eth()
2616 if_t ifp = pi->ifp; in t3_rx_eth()
2618 if (cpl->vlan_valid) { in t3_rx_eth()
2619 m->m_pkthdr.ether_vtag = ntohs(cpl->vlan); in t3_rx_eth()
2620 m->m_flags |= M_VLANTAG; in t3_rx_eth()
2623 m->m_pkthdr.rcvif = ifp; in t3_rx_eth()
2627 m->m_pkthdr.len -= (sizeof(*cpl) + ethpad); in t3_rx_eth()
2628 m->m_len -= (sizeof(*cpl) + ethpad); in t3_rx_eth()
2629 m->m_data += (sizeof(*cpl) + ethpad); in t3_rx_eth()
2631 if (!cpl->fragment && cpl->csum_valid && cpl->csum == 0xffff) { in t3_rx_eth()
2635 if (eh->ether_type == htons(ETHERTYPE_VLAN)) { in t3_rx_eth()
2638 eh_type = evh->evl_proto; in t3_rx_eth()
2640 eh_type = eh->ether_type; in t3_rx_eth()
2644 m->m_pkthdr.csum_flags = (CSUM_IP_CHECKED | in t3_rx_eth()
2646 m->m_pkthdr.csum_data = 0xffff; in t3_rx_eth()
2649 m->m_pkthdr.csum_flags = (CSUM_DATA_VALID_IPV6 | in t3_rx_eth()
2651 m->m_pkthdr.csum_data = 0xffff; in t3_rx_eth()
2657 * get_packet - return the next ingress packet buffer from a free list
2677 unsigned int len_cq = ntohl(r->len_cq); in get_packet()
2678 struct sge_fl *fl = (len_cq & F_RSPD_FLQ) ? &qs->fl[1] : &qs->fl[0]; in get_packet()
2679 int mask, cidx = fl->cidx; in get_packet()
2680 struct rx_sw_desc *sd = &fl->sdesc[cidx]; in get_packet()
2683 uint8_t sopeop = G_RSPD_SOP_EOP(ntohl(r->flags)); in get_packet()
2688 mask = fl->size - 1; in get_packet()
2689 prefetch(fl->sdesc[(cidx + 1) & mask].m); in get_packet()
2690 prefetch(fl->sdesc[(cidx + 2) & mask].m); in get_packet()
2691 prefetch(fl->sdesc[(cidx + 1) & mask].rxsd_cl); in get_packet()
2692 prefetch(fl->sdesc[(cidx + 2) & mask].rxsd_cl); in get_packet()
2694 fl->credits--; in get_packet()
2695 bus_dmamap_sync(fl->entry_tag, sd->map, BUS_DMASYNC_POSTREAD); in get_packet()
2702 memcpy(cl, sd->rxsd_cl, len); in get_packet()
2703 recycle_rx_buf(adap, fl, fl->cidx); in get_packet()
2704 m->m_pkthdr.len = m->m_len = len; in get_packet()
2705 m->m_flags = 0; in get_packet()
2706 mh->mh_head = mh->mh_tail = m; in get_packet()
2707 ret = 1; in get_packet()
2711 bus_dmamap_unload(fl->entry_tag, sd->map); in get_packet()
2712 cl = sd->rxsd_cl; in get_packet()
2713 m = sd->m; in get_packet()
2719 if (fl->zone == zone_pack) { in get_packet()
2723 m->m_data = m->m_ext.ext_buf; in get_packet()
2725 m_cljset(m, cl, fl->type); in get_packet()
2727 m->m_len = len; in get_packet()
2731 ret = 1; in get_packet()
2734 mh->mh_head = mh->mh_tail = m; in get_packet()
2735 m->m_pkthdr.len = len; in get_packet()
2738 ret = 1; in get_packet()
2741 if (mh->mh_tail == NULL) { in get_packet()
2747 mh->mh_tail->m_next = m; in get_packet()
2748 mh->mh_tail = m; in get_packet()
2749 mh->mh_head->m_pkthdr.len += len; in get_packet()
2753 printf("len=%d pktlen=%d\n", m->m_len, m->m_pkthdr.len); in get_packet()
2755 if (++fl->cidx == fl->size) in get_packet()
2756 fl->cidx = 0; in get_packet()
2762 * handle_rsp_cntrl_info - handles control information in a response
2763 * @qs: the queue set corresponding to the response
2767 * indications and completion credits for the queue set's Tx queues.
2777 clear_bit(TXQ_RUNNING, &qs->txq[TXQ_ETH].flags); in handle_rsp_cntrl_info()
2781 qs->txq[TXQ_ETH].processed += credits; in handle_rsp_cntrl_info()
2785 qs->txq[TXQ_CTRL].processed += credits; in handle_rsp_cntrl_info()
2789 clear_bit(TXQ_RUNNING, &qs->txq[TXQ_OFLD].flags); in handle_rsp_cntrl_info()
2793 qs->txq[TXQ_OFLD].processed += credits; in handle_rsp_cntrl_info()
2805 * process_responses - process responses from an SGE response queue
2807 * @qs: the queue set to which the response queue belongs
2810 * Process responses from an SGE response queue up to the supplied budget.
2812 * for the queues that belong to the response queue's queue set.
2816 * on this queue. If the system is under memory shortage use a fairly
2822 struct sge_rspq *rspq = &qs->rspq; in process_responses()
2823 struct rsp_desc *r = &rspq->desc[rspq->cidx]; in process_responses()
2827 int lro_enabled = qs->lro.enabled; in process_responses()
2829 struct lro_ctrl *lro_ctrl = &qs->lro.ctrl; in process_responses()
2831 struct t3_mbuf_hdr *mh = &rspq->rspq_mh; in process_responses()
2834 if (cxgb_debug && rspq->holdoff_tmr != last_holdoff) { in process_responses()
2835 printf("next_holdoff=%d\n", rspq->holdoff_tmr); in process_responses()
2836 last_holdoff = rspq->holdoff_tmr; in process_responses()
2839 rspq->next_holdoff = rspq->holdoff_tmr; in process_responses()
2843 uint32_t flags = ntohl(r->flags); in process_responses()
2844 uint32_t rss_hash = be32toh(r->rss_hdr.rss_hash_val); in process_responses()
2845 uint8_t opcode = r->rss_hdr.opcode; in process_responses()
2855 if (mh->mh_head == NULL) { in process_responses()
2856 mh->mh_head = m_gethdr(M_NOWAIT, MT_DATA); in process_responses()
2857 m = mh->mh_head; in process_responses()
2865 m->m_len = m->m_pkthdr.len = AN_PKT_SIZE; in process_responses()
2868 eop = 1; in process_responses()
2869 rspq->async_notif++; in process_responses()
2876 rspq->next_holdoff = NOMEM_INTR_DELAY; in process_responses()
2877 budget_left--; in process_responses()
2880 if (mh->mh_head == NULL) in process_responses()
2881 mh->mh_head = m; in process_responses()
2883 mh->mh_tail->m_next = m; in process_responses()
2884 mh->mh_tail = m; in process_responses()
2887 mh->mh_head->m_pkthdr.len += m->m_len; in process_responses()
2888 eop = 1; in process_responses()
2889 rspq->imm_data++; in process_responses()
2890 } else if (r->len_cq) { in process_responses()
2895 if (r->rss_hdr.hash_type && !adap->timestamp) { in process_responses()
2896 M_HASHTYPE_SET(mh->mh_head, in process_responses()
2898 mh->mh_head->m_pkthdr.flowid = rss_hash; in process_responses()
2904 rspq->pure_rsps++; in process_responses()
2913 rspq->offload_pkts++; in process_responses()
2915 adap->cpl_handler[opcode](qs, r, mh->mh_head); in process_responses()
2917 m_freem(mh->mh_head); in process_responses()
2919 mh->mh_head = NULL; in process_responses()
2921 struct mbuf *m = mh->mh_head; in process_responses()
2928 * lro_ctrl->ifp's input. That is incorrect. in process_responses()
2934 skip_lro = __predict_false(qs->port->ifp != m->m_pkthdr.rcvif); in process_responses()
2936 if (lro_enabled && lro_ctrl->lro_cnt && !skip_lro in process_responses()
2939 /* successfully queue'd for LRO */ in process_responses()
2945 * or unable to queue. Pass it up right now in in process_responses()
2948 if_t ifp = m->m_pkthdr.rcvif; in process_responses()
2951 mh->mh_head = NULL; in process_responses()
2956 if (__predict_false(++rspq->cidx == rspq->size)) { in process_responses()
2957 rspq->cidx = 0; in process_responses()
2958 rspq->gen ^= 1; in process_responses()
2959 r = rspq->desc; in process_responses()
2962 if (++rspq->credits >= 64) { in process_responses()
2963 refill_rspq(adap, rspq, rspq->credits); in process_responses()
2964 rspq->credits = 0; in process_responses()
2966 __refill_fl_lt(adap, &qs->fl[0], 32); in process_responses()
2967 __refill_fl_lt(adap, &qs->fl[1], 32); in process_responses()
2968 --budget_left; in process_responses()
2979 mb(); /* commit Tx queue processed updates */ in process_responses()
2980 if (__predict_false(qs->txq_stopped > 1)) in process_responses()
2983 __refill_fl_lt(adap, &qs->fl[0], 512); in process_responses()
2984 __refill_fl_lt(adap, &qs->fl[1], 512); in process_responses()
2985 budget -= budget_left; in process_responses()
2998 work = process_responses(adap, rspq_to_qset(rq), -1); in process_responses_gts()
3000 if (cxgb_debug && (rq->next_holdoff != last_holdoff)) { in process_responses_gts()
3001 printf("next_holdoff=%d\n", rq->next_holdoff); in process_responses_gts()
3002 last_holdoff = rq->next_holdoff; in process_responses_gts()
3004 t3_write_reg(adap, A_SG_GTS, V_RSPQ(rq->cntxt_id) | in process_responses_gts()
3005 V_NEWTIMER(rq->next_holdoff) | V_NEWINDEX(rq->cidx)); in process_responses_gts()
3015 return (process_responses_gts(adap, &qs->rspq)); in cxgb_debugnet_poll_rx()
3020 * Interrupt handler for legacy INTx interrupts for T3B-based cards.
3023 * response queue per port in this mode and protect all response queues with
3024 * queue 0's lock.
3031 struct sge_rspq *q0 = &adap->sge.qs[0].rspq; in t3b_intr()
3042 taskqueue_enqueue(adap->tq, &adap->slow_intr_task); in t3b_intr()
3045 mtx_lock(&q0->lock); in t3b_intr()
3047 if (map & (1 << i)) in t3b_intr()
3048 process_responses_gts(adap, &adap->sge.qs[i].rspq); in t3b_intr()
3049 mtx_unlock(&q0->lock); in t3b_intr()
3055 * the same MSI vector. We use one SGE response queue per port in this mode
3056 * and protect all response queues with queue 0's lock.
3062 struct sge_rspq *q0 = &adap->sge.qs[0].rspq; in t3_intr_msi()
3065 mtx_lock(&q0->lock); in t3_intr_msi()
3068 if (process_responses_gts(adap, &adap->sge.qs[i].rspq)) in t3_intr_msi()
3069 new_packets = 1; in t3_intr_msi()
3070 mtx_unlock(&q0->lock); in t3_intr_msi()
3074 taskqueue_enqueue(adap->tq, &adap->slow_intr_task); in t3_intr_msi()
3082 adapter_t *adap = qs->port->adapter; in t3_intr_msix()
3083 struct sge_rspq *rspq = &qs->rspq; in t3_intr_msix()
3086 rspq->unhandled_irqs++; in t3_intr_msix()
3102 if (rspq->rspq_dump_count == 0) in t3_dump_rspq()
3104 if (rspq->rspq_dump_count > RSPQ_Q_SIZE) { in t3_dump_rspq()
3106 "dump count is too large %d\n", rspq->rspq_dump_count); in t3_dump_rspq()
3107 rspq->rspq_dump_count = 0; in t3_dump_rspq()
3110 if (rspq->rspq_dump_start > (RSPQ_Q_SIZE-1)) { in t3_dump_rspq()
3112 "dump start of %d is greater than queue size\n", in t3_dump_rspq()
3113 rspq->rspq_dump_start); in t3_dump_rspq()
3114 rspq->rspq_dump_start = 0; in t3_dump_rspq()
3117 err = t3_sge_read_rspq(qs->port->adapter, rspq->cntxt_id, data); in t3_dump_rspq()
3125 sbuf_printf(sb, " \n index=%u size=%u MSI-X/RspQ=%u intr enable=%u intr armed=%u\n", in t3_dump_rspq()
3127 ((data[2] >> 26) & 1), ((data[2] >> 27) & 1)); in t3_dump_rspq()
3129 ((data[2] >> 28) & 1), ((data[2] >> 31) & 1), data[3]); in t3_dump_rspq()
3131 sbuf_printf(sb, " start=%d -> end=%d\n", rspq->rspq_dump_start, in t3_dump_rspq()
3132 (rspq->rspq_dump_start + rspq->rspq_dump_count) & (RSPQ_Q_SIZE-1)); in t3_dump_rspq()
3134 dump_end = rspq->rspq_dump_start + rspq->rspq_dump_count; in t3_dump_rspq()
3135 for (i = rspq->rspq_dump_start; i < dump_end; i++) { in t3_dump_rspq()
3136 idx = i & (RSPQ_Q_SIZE-1); in t3_dump_rspq()
3138 rspd = &rspq->desc[idx]; in t3_dump_rspq()
3140 idx, rspd->rss_hdr.opcode, rspd->rss_hdr.cpu_idx, in t3_dump_rspq()
3141 rspd->rss_hdr.hash_type, be16toh(rspd->rss_hdr.cq_idx)); in t3_dump_rspq()
3143 rspd->rss_hdr.rss_hash_val, be32toh(rspd->flags), in t3_dump_rspq()
3144 be32toh(rspd->len_cq), rspd->intr_gen); in t3_dump_rspq()
3165 if (txq->txq_dump_count == 0) { in t3_dump_txq_eth()
3168 if (txq->txq_dump_count > TX_ETH_Q_SIZE) { in t3_dump_txq_eth()
3170 "dump count is too large %d\n", txq->txq_dump_count); in t3_dump_txq_eth()
3171 txq->txq_dump_count = 1; in t3_dump_txq_eth()
3174 if (txq->txq_dump_start > (TX_ETH_Q_SIZE-1)) { in t3_dump_txq_eth()
3176 "dump start of %d is greater than queue size\n", in t3_dump_txq_eth()
3177 txq->txq_dump_start); in t3_dump_txq_eth()
3178 txq->txq_dump_start = 0; in t3_dump_txq_eth()
3181 err = t3_sge_read_ecntxt(qs->port->adapter, qs->rspq.cntxt_id, data); in t3_dump_txq_eth()
3190 (data[0] & 0x7fff), ((data[0] >> 15) & 1), (data[0] >> 16), in t3_dump_txq_eth()
3191 (data[1] & 0xffff), ((data[3] >> 4) & 7), ((data[3] >> 7) & 1)); in t3_dump_txq_eth()
3193 ((data[3] >> 8) & 1), ((data[3] >> 9) & 1), ((data[3] >> 10) & 1), in t3_dump_txq_eth()
3194 ((data[3] >> 11) & 0xfffff), ((data[3] >> 31) & 1)); in t3_dump_txq_eth()
3195 sbuf_printf(sb, " qid=%d start=%d -> end=%d\n", qs->idx, in t3_dump_txq_eth()
3196 txq->txq_dump_start, in t3_dump_txq_eth()
3197 (txq->txq_dump_start + txq->txq_dump_count) & (TX_ETH_Q_SIZE-1)); in t3_dump_txq_eth()
3199 dump_end = txq->txq_dump_start + txq->txq_dump_count; in t3_dump_txq_eth()
3200 for (i = txq->txq_dump_start; i < dump_end; i++) { in t3_dump_txq_eth()
3201 txd = &txq->desc[i & (TX_ETH_Q_SIZE-1)]; in t3_dump_txq_eth()
3202 WR = (uint32_t *)txd->flit; in t3_dump_txq_eth()
3204 wr_lo = ntohl(WR[1]); in t3_dump_txq_eth()
3211 WR[j], WR[j + 1], WR[j + 2], WR[j + 3]); in t3_dump_txq_eth()
3231 if (txq->txq_dump_count == 0) { in t3_dump_txq_ctrl()
3234 if (txq->txq_dump_count > 256) { in t3_dump_txq_ctrl()
3236 "dump count is too large %d\n", txq->txq_dump_count); in t3_dump_txq_ctrl()
3237 txq->txq_dump_count = 1; in t3_dump_txq_ctrl()
3240 if (txq->txq_dump_start > 255) { in t3_dump_txq_ctrl()
3242 "dump start of %d is greater than queue size\n", in t3_dump_txq_ctrl()
3243 txq->txq_dump_start); in t3_dump_txq_ctrl()
3244 txq->txq_dump_start = 0; in t3_dump_txq_ctrl()
3252 sbuf_printf(sb, " qid=%d start=%d -> end=%d\n", qs->idx, in t3_dump_txq_ctrl()
3253 txq->txq_dump_start, in t3_dump_txq_ctrl()
3254 (txq->txq_dump_start + txq->txq_dump_count) & 255); in t3_dump_txq_ctrl()
3256 dump_end = txq->txq_dump_start + txq->txq_dump_count; in t3_dump_txq_ctrl()
3257 for (i = txq->txq_dump_start; i < dump_end; i++) { in t3_dump_txq_ctrl()
3258 txd = &txq->desc[i & (255)]; in t3_dump_txq_ctrl()
3259 WR = (uint32_t *)txd->flit; in t3_dump_txq_ctrl()
3261 wr_lo = ntohl(WR[1]); in t3_dump_txq_ctrl()
3268 WR[j], WR[j + 1], WR[j + 2], WR[j + 3]); in t3_dump_txq_ctrl()
3280 struct qset_params *qsp = &sc->params.sge.qset[0]; in t3_set_coalesce_usecs()
3286 if ((sc->flags & FULL_INIT_DONE) == 0) in t3_set_coalesce_usecs()
3289 coalesce_usecs = qsp->coalesce_usecs; in t3_set_coalesce_usecs()
3295 if (coalesce_usecs == qsp->coalesce_usecs) in t3_set_coalesce_usecs()
3298 for (i = 0; i < sc->params.nports; i++) in t3_set_coalesce_usecs()
3299 for (j = 0; j < sc->port[i].nqsets; j++) in t3_set_coalesce_usecs()
3302 coalesce_usecs = max(1, coalesce_usecs); in t3_set_coalesce_usecs()
3305 qs = &sc->sge.qs[i]; in t3_set_coalesce_usecs()
3306 qsp = &sc->params.sge.qset[i]; in t3_set_coalesce_usecs()
3307 qsp->coalesce_usecs = coalesce_usecs; in t3_set_coalesce_usecs()
3309 lock = (sc->flags & USING_MSIX) ? &qs->rspq.lock : in t3_set_coalesce_usecs()
3310 &sc->sge.qs[0].rspq.lock; in t3_set_coalesce_usecs()
3314 t3_write_reg(sc, A_SG_GTS, V_RSPQ(qs->rspq.cntxt_id) | in t3_set_coalesce_usecs()
3315 V_NEWTIMER(qs->rspq.holdoff_tmr)); in t3_set_coalesce_usecs()
3328 if ((sc->flags & FULL_INIT_DONE) == 0) in t3_pkt_timestamp()
3331 timestamp = sc->timestamp; in t3_pkt_timestamp()
3337 if (timestamp != sc->timestamp) { in t3_pkt_timestamp()
3340 sc->timestamp = timestamp; in t3_pkt_timestamp()
3352 ctx = device_get_sysctl_ctx(sc->dev); in t3_add_attach_sysctls()
3353 children = SYSCTL_CHILDREN(device_get_sysctl_tree(sc->dev)); in t3_add_attach_sysctls()
3358 CTLFLAG_RD, sc->fw_version, in t3_add_attach_sysctls()
3362 CTLFLAG_RD, &sc->params.rev, in t3_add_attach_sysctls()
3366 CTLFLAG_RD, sc->port_types, in t3_add_attach_sysctls()
3373 CTLFLAG_RD, &sc->tunq_coalesce, in t3_add_attach_sysctls()
3381 CTLFLAG_RD, &sc->params.vpd.cclk, in t3_add_attach_sysctls()
3404 parg = (uint64_t *) ((uint8_t *)&p->mac.stats + arg2); in sysctl_handle_macstat()
3416 ctx = device_get_sysctl_ctx(sc->dev); in t3_add_configured_sysctls()
3417 children = SYSCTL_CHILDREN(device_get_sysctl_tree(sc->dev)); in t3_add_configured_sysctls()
3431 for (i = 0; i < sc->params.nports; i++) { in t3_add_configured_sysctls()
3432 struct port_info *pi = &sc->port[i]; in t3_add_configured_sysctls()
3435 struct mac_stats *mstats = &pi->mac.stats; in t3_add_configured_sysctls()
3437 snprintf(pi->namebuf, PORT_NAME_LEN, "port%d", i); in t3_add_configured_sysctls()
3439 pi->namebuf, CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, in t3_add_configured_sysctls()
3443 "nqsets", CTLFLAG_RD, &pi->nqsets, in t3_add_configured_sysctls()
3444 0, "#queue sets"); in t3_add_configured_sysctls()
3446 for (j = 0; j < pi->nqsets; j++) { in t3_add_configured_sysctls()
3447 struct sge_qset *qs = &sc->sge.qs[pi->first_qset + j]; in t3_add_configured_sysctls()
3453 struct sge_txq *txq = &qs->txq[TXQ_ETH]; in t3_add_configured_sysctls()
3455 snprintf(qs->namebuf, QS_NAME_LEN, "qs%d", j); in t3_add_configured_sysctls()
3458 qs->namebuf, CTLFLAG_RD | CTLFLAG_MPSAFE, NULL, in t3_add_configured_sysctls()
3463 CTLFLAG_RD, &qs->fl[0].empty, 0, in t3_add_configured_sysctls()
3466 CTLFLAG_RD, &qs->fl[1].empty, 0, in t3_add_configured_sysctls()
3467 "freelist #1 empty"); in t3_add_configured_sysctls()
3490 CTLFLAG_RD, &qs->rspq.size, in t3_add_configured_sysctls()
3491 0, "#entries in response queue"); in t3_add_configured_sysctls()
3493 CTLFLAG_RD, &qs->rspq.cidx, in t3_add_configured_sysctls()
3496 CTLFLAG_RD, &qs->rspq.credits, in t3_add_configured_sysctls()
3499 CTLFLAG_RD, &qs->rspq.starved, in t3_add_configured_sysctls()
3502 CTLFLAG_RD, &qs->rspq.phys_addr, in t3_add_configured_sysctls()
3503 "physical_address_of the queue"); in t3_add_configured_sysctls()
3505 CTLFLAG_RW, &qs->rspq.rspq_dump_start, in t3_add_configured_sysctls()
3508 CTLFLAG_RW, &qs->rspq.rspq_dump_count, in t3_add_configured_sysctls()
3512 &qs->rspq, 0, t3_dump_rspq, "A", in t3_add_configured_sysctls()
3513 "dump of the response queue"); in t3_add_configured_sysctls()
3516 CTLFLAG_RD, &qs->txq[TXQ_ETH].txq_mr->br_drops, in t3_add_configured_sysctls()
3519 CTLFLAG_RD, &qs->txq[TXQ_ETH].sendq.mq_len, in t3_add_configured_sysctls()
3523 CTLFLAG_RD, (uint32_t *)(uintptr_t)&qs->txq[TXQ_ETH].txq_mr.br_prod, in t3_add_configured_sysctls()
3524 0, "#tunneled packets queue producer index"); in t3_add_configured_sysctls()
3526 CTLFLAG_RD, (uint32_t *)(uintptr_t)&qs->txq[TXQ_ETH].txq_mr.br_cons, in t3_add_configured_sysctls()
3527 0, "#tunneled packets queue consumer index"); in t3_add_configured_sysctls()
3530 CTLFLAG_RD, &qs->txq[TXQ_ETH].processed, in t3_add_configured_sysctls()
3533 CTLFLAG_RD, &txq->cleaned, in t3_add_configured_sysctls()
3536 CTLFLAG_RD, &txq->in_use, in t3_add_configured_sysctls()
3539 CTLFLAG_RD, &txq->txq_frees, in t3_add_configured_sysctls()
3542 CTLFLAG_RD, &txq->txq_skipped, in t3_add_configured_sysctls()
3545 CTLFLAG_RD, &txq->txq_coalesced, in t3_add_configured_sysctls()
3548 CTLFLAG_RD, &txq->txq_enqueued, in t3_add_configured_sysctls()
3551 CTLFLAG_RD, &qs->txq_stopped, in t3_add_configured_sysctls()
3552 0, "tx queues stopped"); in t3_add_configured_sysctls()
3554 CTLFLAG_RD, &txq->phys_addr, in t3_add_configured_sysctls()
3555 "physical_address_of the queue"); in t3_add_configured_sysctls()
3557 CTLFLAG_RW, &qs->txq[TXQ_ETH].gen, in t3_add_configured_sysctls()
3560 CTLFLAG_RD, &txq->cidx, in t3_add_configured_sysctls()
3561 0, "hardware queue cidx"); in t3_add_configured_sysctls()
3563 CTLFLAG_RD, &txq->pidx, in t3_add_configured_sysctls()
3564 0, "hardware queue pidx"); in t3_add_configured_sysctls()
3566 CTLFLAG_RW, &qs->txq[TXQ_ETH].txq_dump_start, in t3_add_configured_sysctls()
3569 CTLFLAG_RW, &qs->txq[TXQ_ETH].txq_dump_count, in t3_add_configured_sysctls()
3573 &qs->txq[TXQ_ETH], 0, t3_dump_txq_eth, "A", in t3_add_configured_sysctls()
3574 "dump of the transmit queue"); in t3_add_configured_sysctls()
3577 CTLFLAG_RW, &qs->txq[TXQ_CTRL].txq_dump_start, in t3_add_configured_sysctls()
3580 CTLFLAG_RW, &qs->txq[TXQ_CTRL].txq_dump_count, in t3_add_configured_sysctls()
3584 &qs->txq[TXQ_CTRL], 0, t3_dump_txq_ctrl, "A", in t3_add_configured_sysctls()
3585 "dump of the transmit queue"); in t3_add_configured_sysctls()
3588 CTLFLAG_RD, &qs->lro.ctrl.lro_queued, 0, NULL); in t3_add_configured_sysctls()
3590 CTLFLAG_RD, &qs->lro.ctrl.lro_flushed, 0, NULL); in t3_add_configured_sysctls()
3592 CTLFLAG_RD, &qs->lro.ctrl.lro_bad_csum, 0, NULL); in t3_add_configured_sysctls()
3594 CTLFLAG_RD, &qs->lro.ctrl.lro_cnt, 0, NULL); in t3_add_configured_sysctls()
3666 CTLFLAG_RD, &mstats->a, 0) in t3_add_configured_sysctls()
3682 * t3_get_desc - dump an SGE descriptor for debugging purposes
3683 * @qs: the queue set
3684 * @qnum: identifies the specific queue (0..2: Tx, 3:response, 4..5: Rx)
3685 * @idx: the descriptor index in the queue
3688 * Dumps the contents of a HW descriptor of an SGE queue. Returns the
3699 if (!qs->txq[qnum].desc || idx >= qs->txq[qnum].size) in t3_get_desc()
3700 return -EINVAL; in t3_get_desc()
3701 memcpy(data, &qs->txq[qnum].desc[idx], sizeof(struct tx_desc)); in t3_get_desc()
3706 if (!qs->rspq.desc || idx >= qs->rspq.size) in t3_get_desc()
3708 memcpy(data, &qs->rspq.desc[idx], sizeof(struct rsp_desc)); in t3_get_desc()
3712 qnum -= 4; in t3_get_desc()
3713 if (!qs->fl[qnum].desc || idx >= qs->fl[qnum].size) in t3_get_desc()
3715 memcpy(data, &qs->fl[qnum].desc[idx], sizeof(struct rx_desc)); in t3_get_desc()