1 /* SPDX-License-Identifier: GPL-2.0-or-later */ 2 /* 3 * Device memory TCP support 4 * 5 * Authors: Mina Almasry <almasrymina@google.com> 6 * Willem de Bruijn <willemb@google.com> 7 * Kaiyuan Zhang <kaiyuanz@google.com> 8 * 9 */ 10 #ifndef _NET_DEVMEM_H 11 #define _NET_DEVMEM_H 12 13 #include <net/netmem.h> 14 #include <net/netdev_netlink.h> 15 16 struct netlink_ext_ack; 17 18 struct net_devmem_dmabuf_binding { 19 struct dma_buf *dmabuf; 20 struct dma_buf_attachment *attachment; 21 struct sg_table *sgt; 22 /* Physical NIC that does the actual DMA for this binding. */ 23 struct net_device *dev; 24 /* Opaque cookie identifying the virtual device (e.g. netkit) the user 25 * called bind-tx on. Used only for pointer comparison. Never 26 * dereferenced. 27 */ 28 void *vdev; 29 struct gen_pool *chunk_pool; 30 /* Protect dev */ 31 struct mutex lock; 32 33 /* The user holds a ref (via the netlink API) for as long as they want 34 * the binding to remain alive. Each page pool using this binding holds 35 * a ref to keep the binding alive. The page_pool does not release the 36 * ref until all the net_iovs allocated from this binding are released 37 * back to the page_pool. 38 * 39 * The binding undos itself and unmaps the underlying dmabuf once all 40 * those refs are dropped and the binding is no longer desired or in 41 * use. 42 * 43 * net_devmem_get_net_iov() on dmabuf net_iovs will increment this 44 * reference, making sure that the binding remains alive until all the 45 * net_iovs are no longer used. net_iovs allocated from this binding 46 * that are stuck in the TX path for any reason (such as awaiting 47 * retransmits) hold a reference to the binding until the skb holding 48 * them is freed. 49 */ 50 struct percpu_ref ref; 51 52 /* The list of bindings currently active. Used for netlink to notify us 53 * of the user dropping the bind. 54 */ 55 struct list_head list; 56 57 /* rxq's this binding is active on. */ 58 struct xarray bound_rxqs; 59 60 /* ID of this binding. Globally unique to all bindings currently 61 * active. 62 */ 63 u32 id; 64 65 /* DMA direction, FROM_DEVICE for Rx binding, TO_DEVICE for Tx. */ 66 enum dma_data_direction direction; 67 68 /* Array of net_iov pointers for this binding, sorted by virtual 69 * address. This array is convenient to map the virtual addresses to 70 * net_iovs in the TX path. 71 */ 72 struct net_iov **tx_vec; 73 74 unsigned int niov_shift; 75 76 struct work_struct unbind_w; 77 }; 78 79 #if defined(CONFIG_NET_DEVMEM) 80 /* Owner of the dma-buf chunks inserted into the gen pool. Each scatterlist 81 * entry from the dmabuf is inserted into the genpool as a chunk, and needs 82 * this owner struct to keep track of some metadata necessary to create 83 * allocations from this chunk. 84 */ 85 struct dmabuf_genpool_chunk_owner { 86 struct net_iov_area area; 87 struct net_devmem_dmabuf_binding *binding; 88 89 /* dma_addr of the start of the chunk. */ 90 dma_addr_t base_dma_addr; 91 }; 92 93 void __net_devmem_dmabuf_binding_free(struct work_struct *wq); 94 struct net_devmem_dmabuf_binding * 95 net_devmem_bind_dmabuf(struct net_device *dev, void *vdev, 96 struct device *dma_dev, 97 enum dma_data_direction direction, 98 unsigned int dmabuf_fd, unsigned int niov_shift, 99 struct netdev_nl_sock *priv, 100 struct netlink_ext_ack *extack); 101 struct net_devmem_dmabuf_binding *net_devmem_lookup_dmabuf(u32 id); 102 void net_devmem_unbind_dmabuf(struct net_devmem_dmabuf_binding *binding); 103 int net_devmem_bind_dmabuf_to_queue(struct net_device *dev, u32 rxq_idx, 104 struct net_devmem_dmabuf_binding *binding, 105 struct netlink_ext_ack *extack); 106 107 static inline struct dmabuf_genpool_chunk_owner * 108 net_devmem_iov_to_chunk_owner(const struct net_iov *niov) 109 { 110 struct net_iov_area *owner = net_iov_owner(niov); 111 112 return container_of(owner, struct dmabuf_genpool_chunk_owner, area); 113 } 114 115 static inline struct net_devmem_dmabuf_binding * 116 net_devmem_iov_binding(const struct net_iov *niov) 117 { 118 return net_devmem_iov_to_chunk_owner(niov)->binding; 119 } 120 121 static inline u32 net_devmem_iov_binding_id(const struct net_iov *niov) 122 { 123 return net_devmem_iov_binding(niov)->id; 124 } 125 126 static inline unsigned long net_iov_virtual_addr(const struct net_iov *niov) 127 { 128 struct dmabuf_genpool_chunk_owner *co = 129 net_devmem_iov_to_chunk_owner(niov); 130 131 return net_iov_owner(niov)->base_virtual + 132 ((unsigned long)net_iov_idx(niov) << co->binding->niov_shift); 133 } 134 135 static inline bool 136 net_devmem_dmabuf_binding_get(struct net_devmem_dmabuf_binding *binding) 137 { 138 return percpu_ref_tryget(&binding->ref); 139 } 140 141 static inline void 142 net_devmem_dmabuf_binding_put(struct net_devmem_dmabuf_binding *binding) 143 { 144 percpu_ref_put(&binding->ref); 145 } 146 147 void net_devmem_get_net_iov(struct net_iov *niov); 148 void net_devmem_put_net_iov(struct net_iov *niov); 149 150 struct net_iov * 151 net_devmem_alloc_dmabuf(struct net_devmem_dmabuf_binding *binding); 152 void net_devmem_free_dmabuf(struct net_iov *ppiov); 153 154 155 struct net_devmem_dmabuf_binding * 156 net_devmem_get_binding(struct sock *sk, unsigned int dmabuf_id); 157 struct net_iov * 158 net_devmem_get_niov_at(struct net_devmem_dmabuf_binding *binding, size_t addr, 159 size_t *off, size_t *size); 160 161 #else 162 struct net_devmem_dmabuf_binding; 163 164 static inline void 165 net_devmem_dmabuf_binding_put(struct net_devmem_dmabuf_binding *binding) 166 { 167 } 168 169 static inline void net_devmem_get_net_iov(struct net_iov *niov) 170 { 171 } 172 173 static inline void net_devmem_put_net_iov(struct net_iov *niov) 174 { 175 } 176 177 static inline struct net_devmem_dmabuf_binding * 178 net_devmem_bind_dmabuf(struct net_device *dev, void *vdev, 179 struct device *dma_dev, 180 enum dma_data_direction direction, 181 unsigned int dmabuf_fd, 182 unsigned int niov_shift, 183 struct netdev_nl_sock *priv, 184 struct netlink_ext_ack *extack) 185 { 186 return ERR_PTR(-EOPNOTSUPP); 187 } 188 189 static inline struct net_devmem_dmabuf_binding *net_devmem_lookup_dmabuf(u32 id) 190 { 191 return NULL; 192 } 193 194 static inline void 195 net_devmem_unbind_dmabuf(struct net_devmem_dmabuf_binding *binding) 196 { 197 } 198 199 static inline int 200 net_devmem_bind_dmabuf_to_queue(struct net_device *dev, u32 rxq_idx, 201 struct net_devmem_dmabuf_binding *binding, 202 struct netlink_ext_ack *extack) 203 204 { 205 return -EOPNOTSUPP; 206 } 207 208 static inline struct net_iov * 209 net_devmem_alloc_dmabuf(struct net_devmem_dmabuf_binding *binding) 210 { 211 return NULL; 212 } 213 214 static inline void net_devmem_free_dmabuf(struct net_iov *ppiov) 215 { 216 } 217 218 static inline unsigned long net_iov_virtual_addr(const struct net_iov *niov) 219 { 220 return 0; 221 } 222 223 static inline u32 net_devmem_iov_binding_id(const struct net_iov *niov) 224 { 225 return 0; 226 } 227 228 static inline struct net_devmem_dmabuf_binding * 229 net_devmem_get_binding(struct sock *sk, unsigned int dmabuf_id) 230 { 231 return ERR_PTR(-EOPNOTSUPP); 232 } 233 234 static inline struct net_iov * 235 net_devmem_get_niov_at(struct net_devmem_dmabuf_binding *binding, size_t addr, 236 size_t *off, size_t *size) 237 { 238 return NULL; 239 } 240 241 static inline struct net_devmem_dmabuf_binding * 242 net_devmem_iov_binding(const struct net_iov *niov) 243 { 244 return NULL; 245 } 246 #endif 247 248 #endif /* _NET_DEVMEM_H */ 249