1 // SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) 2 /* Copyright (C) 2015-2019 Netronome Systems, Inc. */ 3 4 #include <linux/seq_file.h> 5 6 #include "../nfp_net.h" 7 #include "../nfp_net_dp.h" 8 #include "../nfp_net_xsk.h" 9 #include "nfd3.h" 10 11 static void nfp_nfd3_xsk_tx_bufs_free(struct nfp_net_tx_ring *tx_ring) 12 { 13 struct nfp_nfd3_tx_buf *txbuf; 14 unsigned int idx; 15 16 while (tx_ring->rd_p != tx_ring->wr_p) { 17 idx = D_IDX(tx_ring, tx_ring->rd_p); 18 txbuf = &tx_ring->txbufs[idx]; 19 20 txbuf->real_len = 0; 21 22 tx_ring->qcp_rd_p++; 23 tx_ring->rd_p++; 24 25 if (tx_ring->r_vec->xsk_pool) { 26 if (txbuf->is_xsk_tx) 27 nfp_nfd3_xsk_tx_free(txbuf); 28 29 xsk_tx_completed(tx_ring->r_vec->xsk_pool, 1); 30 } 31 } 32 } 33 34 /** 35 * nfp_nfd3_tx_ring_reset() - Free any untransmitted buffers and reset pointers 36 * @dp: NFP Net data path struct 37 * @tx_ring: TX ring structure 38 * 39 * Assumes that the device is stopped, must be idempotent. 40 */ 41 static void 42 nfp_nfd3_tx_ring_reset(struct nfp_net_dp *dp, struct nfp_net_tx_ring *tx_ring) 43 { 44 struct netdev_queue *nd_q; 45 const skb_frag_t *frag; 46 47 while (!tx_ring->is_xdp && tx_ring->rd_p != tx_ring->wr_p) { 48 struct nfp_nfd3_tx_buf *tx_buf; 49 struct sk_buff *skb; 50 int idx, nr_frags; 51 52 idx = D_IDX(tx_ring, tx_ring->rd_p); 53 tx_buf = &tx_ring->txbufs[idx]; 54 55 skb = tx_ring->txbufs[idx].skb; 56 nr_frags = skb_shinfo(skb)->nr_frags; 57 58 if (tx_buf->fidx == -1) { 59 /* unmap head */ 60 dma_unmap_single(dp->dev, tx_buf->dma_addr, 61 skb_headlen(skb), DMA_TO_DEVICE); 62 } else { 63 /* unmap fragment */ 64 frag = &skb_shinfo(skb)->frags[tx_buf->fidx]; 65 dma_unmap_page(dp->dev, tx_buf->dma_addr, 66 skb_frag_size(frag), DMA_TO_DEVICE); 67 } 68 69 /* check for last gather fragment */ 70 if (tx_buf->fidx == nr_frags - 1) 71 dev_kfree_skb_any(skb); 72 73 tx_buf->dma_addr = 0; 74 tx_buf->skb = NULL; 75 tx_buf->fidx = -2; 76 77 tx_ring->qcp_rd_p++; 78 tx_ring->rd_p++; 79 } 80 81 if (tx_ring->is_xdp) 82 nfp_nfd3_xsk_tx_bufs_free(tx_ring); 83 84 memset(tx_ring->txds, 0, tx_ring->size); 85 tx_ring->wr_p = 0; 86 tx_ring->rd_p = 0; 87 tx_ring->qcp_rd_p = 0; 88 tx_ring->wr_ptr_add = 0; 89 90 if (tx_ring->is_xdp || !dp->netdev) 91 return; 92 93 nd_q = netdev_get_tx_queue(dp->netdev, tx_ring->idx); 94 netdev_tx_reset_queue(nd_q); 95 } 96 97 /** 98 * nfp_nfd3_tx_ring_free() - Free resources allocated to a TX ring 99 * @tx_ring: TX ring to free 100 */ 101 static void nfp_nfd3_tx_ring_free(struct nfp_net_tx_ring *tx_ring) 102 { 103 struct nfp_net_r_vector *r_vec = tx_ring->r_vec; 104 struct nfp_net_dp *dp = &r_vec->nfp_net->dp; 105 106 kvfree(tx_ring->txbufs); 107 108 if (tx_ring->txds) 109 dma_free_coherent(dp->dev, tx_ring->size, 110 tx_ring->txds, tx_ring->dma); 111 112 tx_ring->cnt = 0; 113 tx_ring->txbufs = NULL; 114 tx_ring->txds = NULL; 115 tx_ring->dma = 0; 116 tx_ring->size = 0; 117 } 118 119 /** 120 * nfp_nfd3_tx_ring_alloc() - Allocate resource for a TX ring 121 * @dp: NFP Net data path struct 122 * @tx_ring: TX Ring structure to allocate 123 * 124 * Return: 0 on success, negative errno otherwise. 125 */ 126 static int 127 nfp_nfd3_tx_ring_alloc(struct nfp_net_dp *dp, struct nfp_net_tx_ring *tx_ring) 128 { 129 struct nfp_net_r_vector *r_vec = tx_ring->r_vec; 130 131 tx_ring->cnt = dp->txd_cnt; 132 133 tx_ring->size = array_size(tx_ring->cnt, sizeof(*tx_ring->txds)); 134 tx_ring->txds = dma_alloc_coherent(dp->dev, tx_ring->size, 135 &tx_ring->dma, 136 GFP_KERNEL | __GFP_NOWARN); 137 if (!tx_ring->txds) { 138 netdev_warn(dp->netdev, "failed to allocate TX descriptor ring memory, requested descriptor count: %d, consider lowering descriptor count\n", 139 tx_ring->cnt); 140 goto err_alloc; 141 } 142 143 tx_ring->txbufs = kvcalloc(tx_ring->cnt, sizeof(*tx_ring->txbufs), 144 GFP_KERNEL); 145 if (!tx_ring->txbufs) 146 goto err_alloc; 147 148 if (!tx_ring->is_xdp && dp->netdev) 149 netif_set_xps_queue(dp->netdev, &r_vec->affinity_mask, 150 tx_ring->idx); 151 152 return 0; 153 154 err_alloc: 155 nfp_nfd3_tx_ring_free(tx_ring); 156 return -ENOMEM; 157 } 158 159 static void 160 nfp_nfd3_tx_ring_bufs_free(struct nfp_net_dp *dp, 161 struct nfp_net_tx_ring *tx_ring) 162 { 163 unsigned int i; 164 165 if (!tx_ring->is_xdp) 166 return; 167 168 for (i = 0; i < tx_ring->cnt; i++) { 169 if (!tx_ring->txbufs[i].frag) 170 return; 171 172 nfp_net_dma_unmap_rx(dp, tx_ring->txbufs[i].dma_addr); 173 __free_page(virt_to_page(tx_ring->txbufs[i].frag)); 174 } 175 } 176 177 static int 178 nfp_nfd3_tx_ring_bufs_alloc(struct nfp_net_dp *dp, 179 struct nfp_net_tx_ring *tx_ring) 180 { 181 struct nfp_nfd3_tx_buf *txbufs = tx_ring->txbufs; 182 unsigned int i; 183 184 if (!tx_ring->is_xdp) 185 return 0; 186 187 for (i = 0; i < tx_ring->cnt; i++) { 188 txbufs[i].frag = nfp_net_rx_alloc_one(dp, &txbufs[i].dma_addr); 189 if (!txbufs[i].frag) { 190 nfp_nfd3_tx_ring_bufs_free(dp, tx_ring); 191 return -ENOMEM; 192 } 193 } 194 195 return 0; 196 } 197 198 static void 199 nfp_nfd3_print_tx_descs(struct seq_file *file, 200 struct nfp_net_r_vector *r_vec, 201 struct nfp_net_tx_ring *tx_ring, 202 u32 d_rd_p, u32 d_wr_p) 203 { 204 struct nfp_nfd3_tx_desc *txd; 205 u32 txd_cnt = tx_ring->cnt; 206 int i; 207 208 for (i = 0; i < txd_cnt; i++) { 209 struct xdp_buff *xdp; 210 struct sk_buff *skb; 211 212 txd = &tx_ring->txds[i]; 213 seq_printf(file, "%04d: 0x%08x 0x%08x 0x%08x 0x%08x", i, 214 txd->vals[0], txd->vals[1], 215 txd->vals[2], txd->vals[3]); 216 217 if (!tx_ring->is_xdp) { 218 skb = READ_ONCE(tx_ring->txbufs[i].skb); 219 if (skb) 220 seq_printf(file, " skb->head=%p skb->data=%p", 221 skb->head, skb->data); 222 } else { 223 xdp = READ_ONCE(tx_ring->txbufs[i].xdp); 224 if (xdp) 225 seq_printf(file, " xdp->data=%p", xdp->data); 226 } 227 228 if (tx_ring->txbufs[i].dma_addr) 229 seq_printf(file, " dma_addr=%pad", 230 &tx_ring->txbufs[i].dma_addr); 231 232 if (i == tx_ring->rd_p % txd_cnt) 233 seq_puts(file, " H_RD"); 234 if (i == tx_ring->wr_p % txd_cnt) 235 seq_puts(file, " H_WR"); 236 if (i == d_rd_p % txd_cnt) 237 seq_puts(file, " D_RD"); 238 if (i == d_wr_p % txd_cnt) 239 seq_puts(file, " D_WR"); 240 241 seq_putc(file, '\n'); 242 } 243 } 244 245 #define NFP_NFD3_CFG_CTRL_SUPPORTED \ 246 (NFP_NET_CFG_CTRL_ENABLE | NFP_NET_CFG_CTRL_PROMISC | \ 247 NFP_NET_CFG_CTRL_L2BC | NFP_NET_CFG_CTRL_L2MC | \ 248 NFP_NET_CFG_CTRL_RXCSUM | NFP_NET_CFG_CTRL_TXCSUM | \ 249 NFP_NET_CFG_CTRL_RXVLAN | NFP_NET_CFG_CTRL_TXVLAN | \ 250 NFP_NET_CFG_CTRL_RXVLAN_V2 | NFP_NET_CFG_CTRL_RXQINQ | \ 251 NFP_NET_CFG_CTRL_TXVLAN_V2 | \ 252 NFP_NET_CFG_CTRL_GATHER | NFP_NET_CFG_CTRL_LSO | \ 253 NFP_NET_CFG_CTRL_CTAG_FILTER | NFP_NET_CFG_CTRL_CMSG_DATA | \ 254 NFP_NET_CFG_CTRL_RINGCFG | NFP_NET_CFG_CTRL_RSS | \ 255 NFP_NET_CFG_CTRL_IRQMOD | NFP_NET_CFG_CTRL_TXRWB | \ 256 NFP_NET_CFG_CTRL_VEPA | \ 257 NFP_NET_CFG_CTRL_VXLAN | NFP_NET_CFG_CTRL_NVGRE | \ 258 NFP_NET_CFG_CTRL_BPF | NFP_NET_CFG_CTRL_LSO2 | \ 259 NFP_NET_CFG_CTRL_RSS2 | NFP_NET_CFG_CTRL_CSUM_COMPLETE | \ 260 NFP_NET_CFG_CTRL_LIVE_ADDR) 261 262 const struct nfp_dp_ops nfp_nfd3_ops = { 263 .version = NFP_NFD_VER_NFD3, 264 .tx_min_desc_per_pkt = 1, 265 .cap_mask = NFP_NFD3_CFG_CTRL_SUPPORTED, 266 .dma_mask = DMA_BIT_MASK(40), 267 .poll = nfp_nfd3_poll, 268 .xsk_poll = nfp_nfd3_xsk_poll, 269 .ctrl_poll = nfp_nfd3_ctrl_poll, 270 .xmit = nfp_nfd3_tx, 271 .ctrl_tx_one = nfp_nfd3_ctrl_tx_one, 272 .rx_ring_fill_freelist = nfp_nfd3_rx_ring_fill_freelist, 273 .tx_ring_alloc = nfp_nfd3_tx_ring_alloc, 274 .tx_ring_reset = nfp_nfd3_tx_ring_reset, 275 .tx_ring_free = nfp_nfd3_tx_ring_free, 276 .tx_ring_bufs_alloc = nfp_nfd3_tx_ring_bufs_alloc, 277 .tx_ring_bufs_free = nfp_nfd3_tx_ring_bufs_free, 278 .print_tx_descs = nfp_nfd3_print_tx_descs 279 }; 280