1 // SPDX-License-Identifier: (GPL-2.0-only OR BSD-2-Clause) 2 /* Copyright (C) 2018 Netronome Systems, Inc */ 3 /* Copyright (C) 2021 Corigine, Inc */ 4 5 #include <linux/dma-direction.h> 6 #include <linux/dma-mapping.h> 7 #include <linux/slab.h> 8 #include <net/xdp_sock_drv.h> 9 #include <trace/events/xdp.h> 10 11 #include "nfp_app.h" 12 #include "nfp_net.h" 13 #include "nfp_net_dp.h" 14 #include "nfp_net_xsk.h" 15 16 static void 17 nfp_net_xsk_rx_bufs_stash(struct nfp_net_rx_ring *rx_ring, unsigned int idx, 18 struct xdp_buff *xdp) 19 { 20 unsigned int headroom; 21 22 headroom = xsk_pool_get_headroom(rx_ring->r_vec->xsk_pool); 23 24 rx_ring->rxds[idx].fld.reserved = 0; 25 rx_ring->rxds[idx].fld.meta_len_dd = 0; 26 27 rx_ring->xsk_rxbufs[idx].xdp = xdp; 28 rx_ring->xsk_rxbufs[idx].dma_addr = 29 xsk_buff_xdp_get_frame_dma(xdp) + headroom; 30 } 31 32 void nfp_net_xsk_rx_unstash(struct nfp_net_xsk_rx_buf *rxbuf) 33 { 34 rxbuf->dma_addr = 0; 35 rxbuf->xdp = NULL; 36 } 37 38 void nfp_net_xsk_rx_free(struct nfp_net_xsk_rx_buf *rxbuf) 39 { 40 if (rxbuf->xdp) 41 xsk_buff_free(rxbuf->xdp); 42 43 nfp_net_xsk_rx_unstash(rxbuf); 44 } 45 46 void nfp_net_xsk_rx_bufs_free(struct nfp_net_rx_ring *rx_ring) 47 { 48 unsigned int i; 49 50 if (!rx_ring->cnt) 51 return; 52 53 for (i = 0; i < rx_ring->cnt - 1; i++) 54 nfp_net_xsk_rx_free(&rx_ring->xsk_rxbufs[i]); 55 } 56 57 void nfp_net_xsk_rx_ring_fill_freelist(struct nfp_net_rx_ring *rx_ring) 58 { 59 struct nfp_net_r_vector *r_vec = rx_ring->r_vec; 60 struct xsk_buff_pool *pool = r_vec->xsk_pool; 61 unsigned int wr_idx, wr_ptr_add = 0; 62 struct xdp_buff *xdp; 63 64 while (nfp_net_rx_space(rx_ring)) { 65 wr_idx = D_IDX(rx_ring, rx_ring->wr_p); 66 67 xdp = xsk_buff_alloc(pool); 68 if (!xdp) 69 break; 70 71 nfp_net_xsk_rx_bufs_stash(rx_ring, wr_idx, xdp); 72 73 nfp_desc_set_dma_addr(&rx_ring->rxds[wr_idx].fld, 74 rx_ring->xsk_rxbufs[wr_idx].dma_addr); 75 76 rx_ring->wr_p++; 77 wr_ptr_add++; 78 } 79 80 /* Ensure all records are visible before incrementing write counter. */ 81 wmb(); 82 nfp_qcp_wr_ptr_add(rx_ring->qcp_fl, wr_ptr_add); 83 } 84 85 void nfp_net_xsk_rx_drop(struct nfp_net_r_vector *r_vec, 86 struct nfp_net_xsk_rx_buf *xrxbuf) 87 { 88 u64_stats_update_begin(&r_vec->rx_sync); 89 r_vec->rx_drops++; 90 u64_stats_update_end(&r_vec->rx_sync); 91 92 nfp_net_xsk_rx_free(xrxbuf); 93 } 94 95 static void nfp_net_xsk_pool_unmap(struct device *dev, 96 struct xsk_buff_pool *pool) 97 { 98 return xsk_pool_dma_unmap(pool, 0); 99 } 100 101 static int nfp_net_xsk_pool_map(struct device *dev, struct xsk_buff_pool *pool) 102 { 103 return xsk_pool_dma_map(pool, dev, 0); 104 } 105 106 int nfp_net_xsk_setup_pool(struct net_device *netdev, 107 struct xsk_buff_pool *pool, u16 queue_id) 108 { 109 struct nfp_net *nn = netdev_priv(netdev); 110 111 struct xsk_buff_pool *prev_pool; 112 struct nfp_net_dp *dp; 113 int err; 114 115 /* NFDK doesn't implement xsk yet. */ 116 if (nn->dp.ops->version == NFP_NFD_VER_NFDK) 117 return -EOPNOTSUPP; 118 119 /* Reject on old FWs so we can drop some checks on datapath. */ 120 if (nn->dp.rx_offset != NFP_NET_CFG_RX_OFFSET_DYNAMIC) 121 return -EOPNOTSUPP; 122 if (!nn->dp.chained_metadata_format) 123 return -EOPNOTSUPP; 124 125 /* Install */ 126 if (pool) { 127 err = nfp_net_xsk_pool_map(nn->dp.dev, pool); 128 if (err) 129 return err; 130 } 131 132 /* Reconfig/swap */ 133 dp = nfp_net_clone_dp(nn); 134 if (!dp) { 135 err = -ENOMEM; 136 goto err_unmap; 137 } 138 139 prev_pool = dp->xsk_pools[queue_id]; 140 dp->xsk_pools[queue_id] = pool; 141 142 err = nfp_net_ring_reconfig(nn, dp, NULL); 143 if (err) 144 goto err_unmap; 145 146 /* Uninstall */ 147 if (prev_pool) 148 nfp_net_xsk_pool_unmap(nn->dp.dev, prev_pool); 149 150 return 0; 151 err_unmap: 152 if (pool) 153 nfp_net_xsk_pool_unmap(nn->dp.dev, pool); 154 155 return err; 156 } 157 158 int nfp_net_xsk_wakeup(struct net_device *netdev, u32 queue_id, u32 flags) 159 { 160 struct nfp_net *nn = netdev_priv(netdev); 161 162 /* queue_id comes from a zero-copy socket, installed with XDP_SETUP_XSK_POOL, 163 * so it must be within our vector range. Moreover, our napi structs 164 * are statically allocated, so we can always kick them without worrying 165 * if reconfig is in progress or interface down. 166 */ 167 napi_schedule(&nn->r_vecs[queue_id].napi); 168 169 return 0; 170 } 171