1 /* SPDX-License-Identifier: GPL-2.0-or-later */ 2 /* 3 * Device memory TCP support 4 * 5 * Authors: Mina Almasry <almasrymina@google.com> 6 * Willem de Bruijn <willemb@google.com> 7 * Kaiyuan Zhang <kaiyuanz@google.com> 8 * 9 */ 10 #ifndef _NET_DEVMEM_H 11 #define _NET_DEVMEM_H 12 13 #include <net/netmem.h> 14 #include <net/netdev_netlink.h> 15 16 struct netlink_ext_ack; 17 18 struct net_devmem_dmabuf_binding { 19 struct net_iov_area area; 20 struct dma_buf *dmabuf; 21 struct dma_buf_attachment *attachment; 22 struct sg_table *sgt; 23 /* Physical NIC that does the actual DMA for this binding. */ 24 struct net_device *dev; 25 /* Opaque cookie identifying the virtual device (e.g. netkit) the user 26 * called bind-tx on. Used only for pointer comparison. Never 27 * dereferenced. 28 */ 29 void *vdev; 30 /* Protect dev */ 31 struct mutex lock; 32 33 /* The user holds a ref (via the netlink API) for as long as they want 34 * the binding to remain alive. Each page pool using this binding holds 35 * a ref to keep the binding alive. The page_pool does not release the 36 * ref until all the net_iovs allocated from this binding are released 37 * back to the page_pool. 38 * 39 * The binding undos itself and unmaps the underlying dmabuf once all 40 * those refs are dropped and the binding is no longer desired or in 41 * use. 42 * 43 * net_devmem_get_net_iov() on dmabuf net_iovs will increment this 44 * reference, making sure that the binding remains alive until all the 45 * net_iovs are no longer used. net_iovs allocated from this binding 46 * that are stuck in the TX path for any reason (such as awaiting 47 * retransmits) hold a reference to the binding until the skb holding 48 * them is freed. 49 */ 50 struct percpu_ref ref; 51 52 /* The list of bindings currently active. Used for netlink to notify us 53 * of the user dropping the bind. 54 */ 55 struct list_head list; 56 57 /* rxq's this binding is active on. */ 58 struct xarray bound_rxqs; 59 60 spinlock_t freelist_lock ____cacheline_aligned_in_smp; 61 size_t free_count; 62 struct net_iov **freelist; 63 64 /* ID of this binding. Globally unique to all bindings currently 65 * active. 66 */ 67 u32 id; 68 69 /* DMA direction, FROM_DEVICE for Rx binding, TO_DEVICE for Tx. */ 70 enum dma_data_direction direction; 71 72 /* Array of net_iov pointers for this binding, sorted by virtual 73 * address. This array is convenient to map the virtual addresses to 74 * net_iovs in the TX path. 75 */ 76 struct net_iov **tx_vec; 77 78 unsigned int niov_shift; 79 80 struct work_struct unbind_w; 81 }; 82 83 #if defined(CONFIG_NET_DEVMEM) 84 void __net_devmem_dmabuf_binding_free(struct work_struct *wq); 85 struct net_devmem_dmabuf_binding * 86 net_devmem_bind_dmabuf(struct net_device *dev, void *vdev, 87 struct device *dma_dev, 88 enum dma_data_direction direction, 89 unsigned int dmabuf_fd, unsigned int niov_shift, 90 struct netdev_nl_sock *priv, 91 struct netlink_ext_ack *extack); 92 struct net_devmem_dmabuf_binding *net_devmem_lookup_dmabuf(u32 id); 93 void net_devmem_unbind_dmabuf(struct net_devmem_dmabuf_binding *binding); 94 int net_devmem_bind_dmabuf_to_queue(struct net_device *dev, u32 rxq_idx, 95 struct net_devmem_dmabuf_binding *binding, 96 struct netlink_ext_ack *extack); 97 98 static inline struct net_devmem_dmabuf_binding * 99 net_devmem_iov_binding(const struct net_iov *niov) 100 { 101 return container_of(net_iov_owner(niov), 102 struct net_devmem_dmabuf_binding, area); 103 } 104 105 static inline u32 net_devmem_iov_binding_id(const struct net_iov *niov) 106 { 107 return net_devmem_iov_binding(niov)->id; 108 } 109 110 static inline unsigned long net_iov_virtual_addr(const struct net_iov *niov) 111 { 112 struct net_devmem_dmabuf_binding *binding = 113 net_devmem_iov_binding(niov); 114 115 return net_iov_owner(niov)->base_virtual + 116 ((unsigned long)net_iov_idx(niov) << binding->niov_shift); 117 } 118 119 static inline bool 120 net_devmem_dmabuf_binding_get(struct net_devmem_dmabuf_binding *binding) 121 { 122 return percpu_ref_tryget(&binding->ref); 123 } 124 125 static inline void 126 net_devmem_dmabuf_binding_put(struct net_devmem_dmabuf_binding *binding) 127 { 128 percpu_ref_put(&binding->ref); 129 } 130 131 void net_devmem_get_net_iov(struct net_iov *niov); 132 void net_devmem_put_net_iov(struct net_iov *niov); 133 134 void net_devmem_free_dmabuf(struct net_iov *ppiov); 135 136 137 struct net_devmem_dmabuf_binding * 138 net_devmem_get_binding(struct sock *sk, unsigned int dmabuf_id); 139 struct net_iov * 140 net_devmem_get_niov_at(struct net_devmem_dmabuf_binding *binding, size_t addr, 141 size_t *off, size_t *size); 142 143 #else 144 struct net_devmem_dmabuf_binding; 145 146 static inline void 147 net_devmem_dmabuf_binding_put(struct net_devmem_dmabuf_binding *binding) 148 { 149 } 150 151 static inline void net_devmem_get_net_iov(struct net_iov *niov) 152 { 153 } 154 155 static inline void net_devmem_put_net_iov(struct net_iov *niov) 156 { 157 } 158 159 static inline struct net_devmem_dmabuf_binding * 160 net_devmem_bind_dmabuf(struct net_device *dev, void *vdev, 161 struct device *dma_dev, 162 enum dma_data_direction direction, 163 unsigned int dmabuf_fd, 164 unsigned int niov_shift, 165 struct netdev_nl_sock *priv, 166 struct netlink_ext_ack *extack) 167 { 168 return ERR_PTR(-EOPNOTSUPP); 169 } 170 171 static inline struct net_devmem_dmabuf_binding *net_devmem_lookup_dmabuf(u32 id) 172 { 173 return NULL; 174 } 175 176 static inline void 177 net_devmem_unbind_dmabuf(struct net_devmem_dmabuf_binding *binding) 178 { 179 } 180 181 static inline int 182 net_devmem_bind_dmabuf_to_queue(struct net_device *dev, u32 rxq_idx, 183 struct net_devmem_dmabuf_binding *binding, 184 struct netlink_ext_ack *extack) 185 186 { 187 return -EOPNOTSUPP; 188 } 189 190 static inline void net_devmem_free_dmabuf(struct net_iov *ppiov) 191 { 192 } 193 194 static inline unsigned long net_iov_virtual_addr(const struct net_iov *niov) 195 { 196 return 0; 197 } 198 199 static inline u32 net_devmem_iov_binding_id(const struct net_iov *niov) 200 { 201 return 0; 202 } 203 204 static inline struct net_devmem_dmabuf_binding * 205 net_devmem_get_binding(struct sock *sk, unsigned int dmabuf_id) 206 { 207 return ERR_PTR(-EOPNOTSUPP); 208 } 209 210 static inline struct net_iov * 211 net_devmem_get_niov_at(struct net_devmem_dmabuf_binding *binding, size_t addr, 212 size_t *off, size_t *size) 213 { 214 return NULL; 215 } 216 217 static inline struct net_devmem_dmabuf_binding * 218 net_devmem_iov_binding(const struct net_iov *niov) 219 { 220 return NULL; 221 } 222 #endif 223 224 #endif /* _NET_DEVMEM_H */ 225