1 /* SPDX-License-Identifier: GPL-2.0-or-later */ 2 /* 3 * Device memory TCP support 4 * 5 * Authors: Mina Almasry <almasrymina@google.com> 6 * Willem de Bruijn <willemb@google.com> 7 * Kaiyuan Zhang <kaiyuanz@google.com> 8 * 9 */ 10 #ifndef _NET_DEVMEM_H 11 #define _NET_DEVMEM_H 12 13 #include <net/net_namespace.h> 14 #include <net/netmem.h> 15 #include <net/netdev_netlink.h> 16 17 struct netlink_ext_ack; 18 19 struct net_devmem_dmabuf_binding { 20 struct net_iov_area area; 21 struct dma_buf *dmabuf; 22 struct dma_buf_attachment *attachment; 23 struct sg_table *sgt; 24 /* Physical NIC that does the actual DMA for this binding. */ 25 struct net_device *dev; 26 /* Opaque cookie identifying the virtual device (e.g. netkit) the user 27 * called bind-tx on. Used only for pointer comparison. Never 28 * dereferenced. 29 */ 30 void *vdev; 31 possible_net_t net; 32 kuid_t uid; 33 /* Protect dev */ 34 struct mutex lock; 35 36 /* The user holds a ref (via the netlink API) for as long as they want 37 * the binding to remain alive. Each page pool using this binding holds 38 * a ref to keep the binding alive. The page_pool does not release the 39 * ref until all the net_iovs allocated from this binding are released 40 * back to the page_pool. 41 * 42 * The binding undos itself and unmaps the underlying dmabuf once all 43 * those refs are dropped and the binding is no longer desired or in 44 * use. 45 * 46 * net_devmem_get_net_iov() on dmabuf net_iovs will increment this 47 * reference, making sure that the binding remains alive until all the 48 * net_iovs are no longer used. net_iovs allocated from this binding 49 * that are stuck in the TX path for any reason (such as awaiting 50 * retransmits) hold a reference to the binding until the skb holding 51 * them is freed. 52 */ 53 struct percpu_ref ref; 54 55 /* The list of bindings currently active. Used for netlink to notify us 56 * of the user dropping the bind. 57 */ 58 struct list_head list; 59 60 /* rxq's this binding is active on. */ 61 struct xarray bound_rxqs; 62 63 spinlock_t freelist_lock ____cacheline_aligned_in_smp; 64 u32 free_count; 65 u32 *freelist; 66 67 /* ID of this binding. Globally unique to all bindings currently 68 * active. 69 */ 70 u32 id; 71 72 /* DMA direction, FROM_DEVICE for Rx binding, TO_DEVICE for Tx. */ 73 enum dma_data_direction direction; 74 75 /* Array of net_iov pointers for this binding, sorted by virtual 76 * address. This array is convenient to map the virtual addresses to 77 * net_iovs in the TX path. 78 */ 79 struct net_iov **tx_vec; 80 81 unsigned int niov_shift; 82 83 struct work_struct unbind_w; 84 }; 85 86 #if defined(CONFIG_NET_DEVMEM) 87 void __net_devmem_dmabuf_binding_free(struct work_struct *wq); 88 struct net_devmem_dmabuf_binding * 89 net_devmem_bind_dmabuf(struct net_device *dev, void *vdev, 90 struct device *dma_dev, 91 enum dma_data_direction direction, 92 unsigned int dmabuf_fd, unsigned int niov_shift, 93 struct netdev_nl_sock *priv, 94 const struct sock *nl_sk, 95 struct netlink_ext_ack *extack); 96 struct net_devmem_dmabuf_binding *net_devmem_lookup_dmabuf(u32 id); 97 void net_devmem_unbind_dmabuf(struct net_devmem_dmabuf_binding *binding); 98 int net_devmem_bind_dmabuf_to_queue(struct net_device *dev, u32 rxq_idx, 99 struct net_devmem_dmabuf_binding *binding, 100 struct netlink_ext_ack *extack); 101 102 static inline struct net_devmem_dmabuf_binding * 103 net_devmem_iov_binding(const struct net_iov *niov) 104 { 105 return container_of(net_iov_owner(niov), 106 struct net_devmem_dmabuf_binding, area); 107 } 108 109 static inline u32 net_devmem_iov_binding_id(const struct net_iov *niov) 110 { 111 return net_devmem_iov_binding(niov)->id; 112 } 113 114 static inline unsigned long net_iov_virtual_addr(const struct net_iov *niov) 115 { 116 struct net_devmem_dmabuf_binding *binding = 117 net_devmem_iov_binding(niov); 118 119 return (unsigned long)net_iov_idx(niov) << binding->niov_shift; 120 } 121 122 static inline bool 123 net_devmem_dmabuf_binding_get(struct net_devmem_dmabuf_binding *binding) 124 { 125 return percpu_ref_tryget(&binding->ref); 126 } 127 128 static inline void 129 net_devmem_dmabuf_binding_put(struct net_devmem_dmabuf_binding *binding) 130 { 131 percpu_ref_put(&binding->ref); 132 } 133 134 void net_devmem_get_net_iov(struct net_iov *niov); 135 void net_devmem_put_net_iov(struct net_iov *niov); 136 137 void net_devmem_free_dmabuf(struct net_iov *ppiov); 138 139 140 struct net_devmem_dmabuf_binding * 141 net_devmem_get_binding(struct sock *sk, unsigned int dmabuf_id); 142 struct net_iov * 143 net_devmem_get_niov_at(struct net_devmem_dmabuf_binding *binding, size_t addr, 144 size_t *off, size_t *size); 145 146 #else 147 struct net_devmem_dmabuf_binding; 148 149 static inline void 150 net_devmem_dmabuf_binding_put(struct net_devmem_dmabuf_binding *binding) 151 { 152 } 153 154 static inline void net_devmem_get_net_iov(struct net_iov *niov) 155 { 156 } 157 158 static inline void net_devmem_put_net_iov(struct net_iov *niov) 159 { 160 } 161 162 static inline struct net_devmem_dmabuf_binding * 163 net_devmem_bind_dmabuf(struct net_device *dev, void *vdev, 164 struct device *dma_dev, 165 enum dma_data_direction direction, 166 unsigned int dmabuf_fd, 167 unsigned int niov_shift, 168 struct netdev_nl_sock *priv, 169 const struct sock *nl_sk, 170 struct netlink_ext_ack *extack) 171 { 172 return ERR_PTR(-EOPNOTSUPP); 173 } 174 175 static inline struct net_devmem_dmabuf_binding *net_devmem_lookup_dmabuf(u32 id) 176 { 177 return NULL; 178 } 179 180 static inline void 181 net_devmem_unbind_dmabuf(struct net_devmem_dmabuf_binding *binding) 182 { 183 } 184 185 static inline int 186 net_devmem_bind_dmabuf_to_queue(struct net_device *dev, u32 rxq_idx, 187 struct net_devmem_dmabuf_binding *binding, 188 struct netlink_ext_ack *extack) 189 190 { 191 return -EOPNOTSUPP; 192 } 193 194 static inline void net_devmem_free_dmabuf(struct net_iov *ppiov) 195 { 196 } 197 198 static inline unsigned long net_iov_virtual_addr(const struct net_iov *niov) 199 { 200 return 0; 201 } 202 203 static inline u32 net_devmem_iov_binding_id(const struct net_iov *niov) 204 { 205 return 0; 206 } 207 208 static inline struct net_devmem_dmabuf_binding * 209 net_devmem_get_binding(struct sock *sk, unsigned int dmabuf_id) 210 { 211 return ERR_PTR(-EOPNOTSUPP); 212 } 213 214 static inline struct net_iov * 215 net_devmem_get_niov_at(struct net_devmem_dmabuf_binding *binding, size_t addr, 216 size_t *off, size_t *size) 217 { 218 return NULL; 219 } 220 221 static inline struct net_devmem_dmabuf_binding * 222 net_devmem_iov_binding(const struct net_iov *niov) 223 { 224 return NULL; 225 } 226 #endif 227 228 #endif /* _NET_DEVMEM_H */ 229