1 // SPDX-License-Identifier: GPL-2.0 2 /* Marvell RVU Ethernet driver 3 * 4 * Copyright (C) 2024 Marvell. 5 * 6 */ 7 8 #include <linux/bpf_trace.h> 9 #include <linux/stringify.h> 10 #include <net/xdp_sock_drv.h> 11 #include <net/xdp.h> 12 13 #include "otx2_common.h" 14 #include "otx2_struct.h" 15 #include "otx2_xsk.h" 16 17 int otx2_xsk_pool_alloc_buf(struct otx2_nic *pfvf, struct otx2_pool *pool, 18 dma_addr_t *dma, int idx) 19 { 20 struct xdp_buff *xdp; 21 int delta; 22 23 xdp = xsk_buff_alloc(pool->xsk_pool); 24 if (!xdp) 25 return -ENOMEM; 26 27 pool->xdp[pool->xdp_top++] = xdp; 28 *dma = OTX2_DATA_ALIGN(xsk_buff_xdp_get_dma(xdp)); 29 /* Adjust xdp->data for unaligned addresses */ 30 delta = *dma - xsk_buff_xdp_get_dma(xdp); 31 xdp->data += delta; 32 33 return 0; 34 } 35 36 static int otx2_xsk_ctx_disable(struct otx2_nic *pfvf, u16 qidx, int aura_id) 37 { 38 struct nix_cn10k_aq_enq_req *cn10k_rq_aq; 39 struct npa_aq_enq_req *aura_aq; 40 struct npa_aq_enq_req *pool_aq; 41 struct nix_aq_enq_req *rq_aq; 42 43 if (test_bit(CN10K_LMTST, &pfvf->hw.cap_flag)) { 44 cn10k_rq_aq = otx2_mbox_alloc_msg_nix_cn10k_aq_enq(&pfvf->mbox); 45 if (!cn10k_rq_aq) 46 return -ENOMEM; 47 cn10k_rq_aq->qidx = qidx; 48 cn10k_rq_aq->rq.ena = 0; 49 cn10k_rq_aq->rq_mask.ena = 1; 50 cn10k_rq_aq->ctype = NIX_AQ_CTYPE_RQ; 51 cn10k_rq_aq->op = NIX_AQ_INSTOP_WRITE; 52 } else { 53 rq_aq = otx2_mbox_alloc_msg_nix_aq_enq(&pfvf->mbox); 54 if (!rq_aq) 55 return -ENOMEM; 56 rq_aq->qidx = qidx; 57 rq_aq->sq.ena = 0; 58 rq_aq->sq_mask.ena = 1; 59 rq_aq->ctype = NIX_AQ_CTYPE_RQ; 60 rq_aq->op = NIX_AQ_INSTOP_WRITE; 61 } 62 63 aura_aq = otx2_mbox_alloc_msg_npa_aq_enq(&pfvf->mbox); 64 if (!aura_aq) 65 goto fail; 66 67 aura_aq->aura_id = aura_id; 68 aura_aq->aura.ena = 0; 69 aura_aq->aura_mask.ena = 1; 70 aura_aq->ctype = NPA_AQ_CTYPE_AURA; 71 aura_aq->op = NPA_AQ_INSTOP_WRITE; 72 73 pool_aq = otx2_mbox_alloc_msg_npa_aq_enq(&pfvf->mbox); 74 if (!pool_aq) 75 goto fail; 76 77 pool_aq->aura_id = aura_id; 78 pool_aq->pool.ena = 0; 79 pool_aq->pool_mask.ena = 1; 80 81 pool_aq->ctype = NPA_AQ_CTYPE_POOL; 82 pool_aq->op = NPA_AQ_INSTOP_WRITE; 83 84 return otx2_sync_mbox_msg(&pfvf->mbox); 85 86 fail: 87 otx2_mbox_reset(&pfvf->mbox.mbox, 0); 88 return -ENOMEM; 89 } 90 91 static void otx2_clean_up_rq(struct otx2_nic *pfvf, int qidx) 92 { 93 struct otx2_qset *qset = &pfvf->qset; 94 struct otx2_cq_queue *cq; 95 struct otx2_pool *pool; 96 u64 iova; 97 98 /* If the DOWN flag is set SQs are already freed */ 99 if (pfvf->flags & OTX2_FLAG_INTF_DOWN) 100 return; 101 102 cq = &qset->cq[qidx]; 103 if (cq) 104 otx2_cleanup_rx_cqes(pfvf, cq, qidx); 105 106 pool = &pfvf->qset.pool[qidx]; 107 iova = otx2_aura_allocptr(pfvf, qidx); 108 while (iova) { 109 iova -= OTX2_HEAD_ROOM; 110 otx2_free_bufs(pfvf, pool, iova, pfvf->rbsize); 111 iova = otx2_aura_allocptr(pfvf, qidx); 112 } 113 114 mutex_lock(&pfvf->mbox.lock); 115 otx2_xsk_ctx_disable(pfvf, qidx, qidx); 116 mutex_unlock(&pfvf->mbox.lock); 117 } 118 119 int otx2_xsk_pool_enable(struct otx2_nic *pf, struct xsk_buff_pool *pool, u16 qidx) 120 { 121 u16 rx_queues = pf->hw.rx_queues; 122 u16 tx_queues = pf->hw.tx_queues; 123 int err; 124 125 if (qidx >= rx_queues || qidx >= tx_queues) 126 return -EINVAL; 127 128 err = xsk_pool_dma_map(pool, pf->dev, DMA_ATTR_SKIP_CPU_SYNC | DMA_ATTR_WEAK_ORDERING); 129 if (err) 130 return err; 131 132 set_bit(qidx, pf->af_xdp_zc_qidx); 133 otx2_clean_up_rq(pf, qidx); 134 /* Reconfigure RSS table as 'qidx' cannot be part of RSS now */ 135 otx2_set_rss_table(pf, DEFAULT_RSS_CONTEXT_GROUP, NULL); 136 /* Kick start the NAPI context so that receiving will start */ 137 return otx2_xsk_wakeup(pf->netdev, qidx, XDP_WAKEUP_RX); 138 } 139 140 int otx2_xsk_pool_disable(struct otx2_nic *pf, u16 qidx) 141 { 142 struct net_device *netdev = pf->netdev; 143 struct xsk_buff_pool *pool; 144 struct otx2_snd_queue *sq; 145 146 pool = xsk_get_pool_from_qid(netdev, qidx); 147 if (!pool) 148 return -EINVAL; 149 150 sq = &pf->qset.sq[qidx + pf->hw.tx_queues]; 151 sq->xsk_pool = NULL; 152 otx2_clean_up_rq(pf, qidx); 153 clear_bit(qidx, pf->af_xdp_zc_qidx); 154 xsk_pool_dma_unmap(pool, DMA_ATTR_SKIP_CPU_SYNC | DMA_ATTR_WEAK_ORDERING); 155 /* Reconfigure RSS table as 'qidx' now need to be part of RSS now */ 156 otx2_set_rss_table(pf, DEFAULT_RSS_CONTEXT_GROUP, NULL); 157 158 return 0; 159 } 160 161 int otx2_xsk_pool_setup(struct otx2_nic *pf, struct xsk_buff_pool *pool, u16 qidx) 162 { 163 if (pool) 164 return otx2_xsk_pool_enable(pf, pool, qidx); 165 166 return otx2_xsk_pool_disable(pf, qidx); 167 } 168 169 int otx2_xsk_wakeup(struct net_device *dev, u32 queue_id, u32 flags) 170 { 171 struct otx2_nic *pf = netdev_priv(dev); 172 struct otx2_cq_poll *cq_poll = NULL; 173 struct otx2_qset *qset = &pf->qset; 174 175 if (pf->flags & OTX2_FLAG_INTF_DOWN) 176 return -ENETDOWN; 177 178 if (queue_id >= pf->hw.rx_queues || queue_id >= pf->hw.tx_queues) 179 return -EINVAL; 180 181 cq_poll = &qset->napi[queue_id]; 182 if (!cq_poll) 183 return -EINVAL; 184 185 /* Trigger interrupt */ 186 if (!napi_if_scheduled_mark_missed(&cq_poll->napi)) { 187 otx2_write64(pf, NIX_LF_CINTX_ENA_W1S(cq_poll->cint_idx), BIT_ULL(0)); 188 otx2_write64(pf, NIX_LF_CINTX_INT_W1S(cq_poll->cint_idx), BIT_ULL(0)); 189 } 190 191 return 0; 192 } 193 194 void otx2_attach_xsk_buff(struct otx2_nic *pfvf, struct otx2_snd_queue *sq, int qidx) 195 { 196 if (test_bit(qidx, pfvf->af_xdp_zc_qidx)) 197 sq->xsk_pool = xsk_get_pool_from_qid(pfvf->netdev, qidx); 198 } 199 200 static void otx2_xsk_sq_append_pkt(struct otx2_nic *pfvf, u64 iova, int len, 201 u16 qidx) 202 { 203 struct nix_sqe_hdr_s *sqe_hdr; 204 struct otx2_snd_queue *sq; 205 int offset; 206 207 sq = &pfvf->qset.sq[qidx]; 208 memset(sq->sqe_base + 8, 0, sq->sqe_size - 8); 209 210 sqe_hdr = (struct nix_sqe_hdr_s *)(sq->sqe_base); 211 212 if (!sqe_hdr->total) { 213 sqe_hdr->aura = sq->aura_id; 214 sqe_hdr->df = 1; 215 sqe_hdr->sq = qidx; 216 sqe_hdr->pnc = 1; 217 } 218 sqe_hdr->total = len; 219 sqe_hdr->sqe_id = sq->head; 220 221 offset = sizeof(*sqe_hdr); 222 223 otx2_xdp_sqe_add_sg(sq, NULL, iova, len, &offset, OTX2_AF_XDP_FRAME); 224 sqe_hdr->sizem1 = (offset / 16) - 1; 225 pfvf->hw_ops->sqe_flush(pfvf, sq, offset, qidx); 226 } 227 228 void otx2_zc_napi_handler(struct otx2_nic *pfvf, struct xsk_buff_pool *pool, 229 int queue, int budget) 230 { 231 struct xdp_desc *xdp_desc = pool->tx_descs; 232 int i, batch; 233 234 budget = min(budget, otx2_read_free_sqe(pfvf, queue)); 235 batch = xsk_tx_peek_release_desc_batch(pool, budget); 236 if (!batch) 237 return; 238 239 for (i = 0; i < batch; i++) { 240 dma_addr_t dma_addr; 241 242 dma_addr = xsk_buff_raw_get_dma(pool, xdp_desc[i].addr); 243 otx2_xsk_sq_append_pkt(pfvf, dma_addr, xdp_desc[i].len, queue); 244 } 245 } 246