1 /* SPDX-License-Identifier: GPL-2.0-or-later */ 2 /* 3 * Device memory TCP support 4 * 5 * Authors: Mina Almasry <almasrymina@google.com> 6 * Willem de Bruijn <willemb@google.com> 7 * Kaiyuan Zhang <kaiyuanz@google.com> 8 * 9 */ 10 #ifndef _NET_DEVMEM_H 11 #define _NET_DEVMEM_H 12 13 #include <net/netmem.h> 14 #include <net/netdev_netlink.h> 15 16 struct netlink_ext_ack; 17 18 struct net_devmem_dmabuf_binding { 19 struct dma_buf *dmabuf; 20 struct dma_buf_attachment *attachment; 21 struct sg_table *sgt; 22 /* Physical NIC that does the actual DMA for this binding. */ 23 struct net_device *dev; 24 /* Opaque cookie identifying the virtual device (e.g. netkit) the user 25 * called bind-tx on. Used only for pointer comparison. Never 26 * dereferenced. 27 */ 28 void *vdev; 29 struct gen_pool *chunk_pool; 30 /* Protect dev */ 31 struct mutex lock; 32 33 /* The user holds a ref (via the netlink API) for as long as they want 34 * the binding to remain alive. Each page pool using this binding holds 35 * a ref to keep the binding alive. The page_pool does not release the 36 * ref until all the net_iovs allocated from this binding are released 37 * back to the page_pool. 38 * 39 * The binding undos itself and unmaps the underlying dmabuf once all 40 * those refs are dropped and the binding is no longer desired or in 41 * use. 42 * 43 * net_devmem_get_net_iov() on dmabuf net_iovs will increment this 44 * reference, making sure that the binding remains alive until all the 45 * net_iovs are no longer used. net_iovs allocated from this binding 46 * that are stuck in the TX path for any reason (such as awaiting 47 * retransmits) hold a reference to the binding until the skb holding 48 * them is freed. 49 */ 50 struct percpu_ref ref; 51 52 /* The list of bindings currently active. Used for netlink to notify us 53 * of the user dropping the bind. 54 */ 55 struct list_head list; 56 57 /* rxq's this binding is active on. */ 58 struct xarray bound_rxqs; 59 60 /* ID of this binding. Globally unique to all bindings currently 61 * active. 62 */ 63 u32 id; 64 65 /* DMA direction, FROM_DEVICE for Rx binding, TO_DEVICE for Tx. */ 66 enum dma_data_direction direction; 67 68 /* Array of net_iov pointers for this binding, sorted by virtual 69 * address. This array is convenient to map the virtual addresses to 70 * net_iovs in the TX path. 71 */ 72 struct net_iov **tx_vec; 73 74 struct work_struct unbind_w; 75 }; 76 77 #if defined(CONFIG_NET_DEVMEM) 78 /* Owner of the dma-buf chunks inserted into the gen pool. Each scatterlist 79 * entry from the dmabuf is inserted into the genpool as a chunk, and needs 80 * this owner struct to keep track of some metadata necessary to create 81 * allocations from this chunk. 82 */ 83 struct dmabuf_genpool_chunk_owner { 84 struct net_iov_area area; 85 struct net_devmem_dmabuf_binding *binding; 86 87 /* dma_addr of the start of the chunk. */ 88 dma_addr_t base_dma_addr; 89 }; 90 91 void __net_devmem_dmabuf_binding_free(struct work_struct *wq); 92 struct net_devmem_dmabuf_binding * 93 net_devmem_bind_dmabuf(struct net_device *dev, void *vdev, 94 struct device *dma_dev, 95 enum dma_data_direction direction, 96 unsigned int dmabuf_fd, struct netdev_nl_sock *priv, 97 struct netlink_ext_ack *extack); 98 struct net_devmem_dmabuf_binding *net_devmem_lookup_dmabuf(u32 id); 99 void net_devmem_unbind_dmabuf(struct net_devmem_dmabuf_binding *binding); 100 int net_devmem_bind_dmabuf_to_queue(struct net_device *dev, u32 rxq_idx, 101 struct net_devmem_dmabuf_binding *binding, 102 struct netlink_ext_ack *extack); 103 104 static inline struct dmabuf_genpool_chunk_owner * 105 net_devmem_iov_to_chunk_owner(const struct net_iov *niov) 106 { 107 struct net_iov_area *owner = net_iov_owner(niov); 108 109 return container_of(owner, struct dmabuf_genpool_chunk_owner, area); 110 } 111 112 static inline struct net_devmem_dmabuf_binding * 113 net_devmem_iov_binding(const struct net_iov *niov) 114 { 115 return net_devmem_iov_to_chunk_owner(niov)->binding; 116 } 117 118 static inline u32 net_devmem_iov_binding_id(const struct net_iov *niov) 119 { 120 return net_devmem_iov_binding(niov)->id; 121 } 122 123 static inline unsigned long net_iov_virtual_addr(const struct net_iov *niov) 124 { 125 struct net_iov_area *owner = net_iov_owner(niov); 126 127 return owner->base_virtual + 128 ((unsigned long)net_iov_idx(niov) << PAGE_SHIFT); 129 } 130 131 static inline bool 132 net_devmem_dmabuf_binding_get(struct net_devmem_dmabuf_binding *binding) 133 { 134 return percpu_ref_tryget(&binding->ref); 135 } 136 137 static inline void 138 net_devmem_dmabuf_binding_put(struct net_devmem_dmabuf_binding *binding) 139 { 140 percpu_ref_put(&binding->ref); 141 } 142 143 void net_devmem_get_net_iov(struct net_iov *niov); 144 void net_devmem_put_net_iov(struct net_iov *niov); 145 146 struct net_iov * 147 net_devmem_alloc_dmabuf(struct net_devmem_dmabuf_binding *binding); 148 void net_devmem_free_dmabuf(struct net_iov *ppiov); 149 150 151 struct net_devmem_dmabuf_binding * 152 net_devmem_get_binding(struct sock *sk, unsigned int dmabuf_id); 153 struct net_iov * 154 net_devmem_get_niov_at(struct net_devmem_dmabuf_binding *binding, size_t addr, 155 size_t *off, size_t *size); 156 157 #else 158 struct net_devmem_dmabuf_binding; 159 160 static inline void 161 net_devmem_dmabuf_binding_put(struct net_devmem_dmabuf_binding *binding) 162 { 163 } 164 165 static inline void net_devmem_get_net_iov(struct net_iov *niov) 166 { 167 } 168 169 static inline void net_devmem_put_net_iov(struct net_iov *niov) 170 { 171 } 172 173 static inline struct net_devmem_dmabuf_binding * 174 net_devmem_bind_dmabuf(struct net_device *dev, void *vdev, 175 struct device *dma_dev, 176 enum dma_data_direction direction, 177 unsigned int dmabuf_fd, 178 struct netdev_nl_sock *priv, 179 struct netlink_ext_ack *extack) 180 { 181 return ERR_PTR(-EOPNOTSUPP); 182 } 183 184 static inline struct net_devmem_dmabuf_binding *net_devmem_lookup_dmabuf(u32 id) 185 { 186 return NULL; 187 } 188 189 static inline void 190 net_devmem_unbind_dmabuf(struct net_devmem_dmabuf_binding *binding) 191 { 192 } 193 194 static inline int 195 net_devmem_bind_dmabuf_to_queue(struct net_device *dev, u32 rxq_idx, 196 struct net_devmem_dmabuf_binding *binding, 197 struct netlink_ext_ack *extack) 198 199 { 200 return -EOPNOTSUPP; 201 } 202 203 static inline struct net_iov * 204 net_devmem_alloc_dmabuf(struct net_devmem_dmabuf_binding *binding) 205 { 206 return NULL; 207 } 208 209 static inline void net_devmem_free_dmabuf(struct net_iov *ppiov) 210 { 211 } 212 213 static inline unsigned long net_iov_virtual_addr(const struct net_iov *niov) 214 { 215 return 0; 216 } 217 218 static inline u32 net_devmem_iov_binding_id(const struct net_iov *niov) 219 { 220 return 0; 221 } 222 223 static inline struct net_devmem_dmabuf_binding * 224 net_devmem_get_binding(struct sock *sk, unsigned int dmabuf_id) 225 { 226 return ERR_PTR(-EOPNOTSUPP); 227 } 228 229 static inline struct net_iov * 230 net_devmem_get_niov_at(struct net_devmem_dmabuf_binding *binding, size_t addr, 231 size_t *off, size_t *size) 232 { 233 return NULL; 234 } 235 236 static inline struct net_devmem_dmabuf_binding * 237 net_devmem_iov_binding(const struct net_iov *niov) 238 { 239 return NULL; 240 } 241 #endif 242 243 #endif /* _NET_DEVMEM_H */ 244