1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /*
3 * Devmem TCP
4 *
5 * Authors: Mina Almasry <almasrymina@google.com>
6 * Willem de Bruijn <willemdebruijn.kernel@gmail.com>
7 * Kaiyuan Zhang <kaiyuanz@google.com
8 */
9
10 #include <linux/dma-buf.h>
11 #include <linux/genalloc.h>
12 #include <linux/mm.h>
13 #include <linux/netdevice.h>
14 #include <linux/types.h>
15 #include <net/netdev_queues.h>
16 #include <net/netdev_rx_queue.h>
17 #include <net/page_pool/helpers.h>
18 #include <net/page_pool/memory_provider.h>
19 #include <net/sock.h>
20 #include <net/tcp.h>
21 #include <trace/events/page_pool.h>
22
23 #include "devmem.h"
24 #include "mp_dmabuf_devmem.h"
25 #include "page_pool_priv.h"
26
27 /* Device memory support */
28
29 static DEFINE_XARRAY_FLAGS(net_devmem_dmabuf_bindings, XA_FLAGS_ALLOC1);
30
31 static const struct memory_provider_ops dmabuf_devmem_ops;
32
net_is_devmem_iov(struct net_iov * niov)33 bool net_is_devmem_iov(struct net_iov *niov)
34 {
35 return niov->type == NET_IOV_DMABUF;
36 }
37
net_devmem_dmabuf_free_chunk_owner(struct gen_pool * genpool,struct gen_pool_chunk * chunk,void * not_used)38 static void net_devmem_dmabuf_free_chunk_owner(struct gen_pool *genpool,
39 struct gen_pool_chunk *chunk,
40 void *not_used)
41 {
42 struct dmabuf_genpool_chunk_owner *owner = chunk->owner;
43
44 kvfree(owner->area.niovs);
45 kfree(owner);
46 }
47
net_devmem_get_dma_addr(const struct net_iov * niov)48 static dma_addr_t net_devmem_get_dma_addr(const struct net_iov *niov)
49 {
50 struct dmabuf_genpool_chunk_owner *owner;
51
52 owner = net_devmem_iov_to_chunk_owner(niov);
53 return owner->base_dma_addr +
54 ((dma_addr_t)net_iov_idx(niov) << PAGE_SHIFT);
55 }
56
__net_devmem_dmabuf_binding_free(struct work_struct * wq)57 void __net_devmem_dmabuf_binding_free(struct work_struct *wq)
58 {
59 struct net_devmem_dmabuf_binding *binding = container_of(wq, typeof(*binding), unbind_w);
60
61 size_t size, avail;
62
63 gen_pool_for_each_chunk(binding->chunk_pool,
64 net_devmem_dmabuf_free_chunk_owner, NULL);
65
66 size = gen_pool_size(binding->chunk_pool);
67 avail = gen_pool_avail(binding->chunk_pool);
68
69 if (!WARN(size != avail, "can't destroy genpool. size=%zu, avail=%zu",
70 size, avail))
71 gen_pool_destroy(binding->chunk_pool);
72
73 dma_buf_unmap_attachment_unlocked(binding->attachment, binding->sgt,
74 binding->direction);
75 dma_buf_detach(binding->dmabuf, binding->attachment);
76 dma_buf_put(binding->dmabuf);
77 xa_destroy(&binding->bound_rxqs);
78 kvfree(binding->tx_vec);
79 kfree(binding);
80 }
81
82 struct net_iov *
net_devmem_alloc_dmabuf(struct net_devmem_dmabuf_binding * binding)83 net_devmem_alloc_dmabuf(struct net_devmem_dmabuf_binding *binding)
84 {
85 struct dmabuf_genpool_chunk_owner *owner;
86 unsigned long dma_addr;
87 struct net_iov *niov;
88 ssize_t offset;
89 ssize_t index;
90
91 dma_addr = gen_pool_alloc_owner(binding->chunk_pool, PAGE_SIZE,
92 (void **)&owner);
93 if (!dma_addr)
94 return NULL;
95
96 offset = dma_addr - owner->base_dma_addr;
97 index = offset / PAGE_SIZE;
98 niov = &owner->area.niovs[index];
99
100 niov->pp_magic = 0;
101 niov->pp = NULL;
102 atomic_long_set(&niov->pp_ref_count, 0);
103
104 return niov;
105 }
106
net_devmem_free_dmabuf(struct net_iov * niov)107 void net_devmem_free_dmabuf(struct net_iov *niov)
108 {
109 struct net_devmem_dmabuf_binding *binding = net_devmem_iov_binding(niov);
110 unsigned long dma_addr = net_devmem_get_dma_addr(niov);
111
112 if (WARN_ON(!gen_pool_has_addr(binding->chunk_pool, dma_addr,
113 PAGE_SIZE)))
114 return;
115
116 gen_pool_free(binding->chunk_pool, dma_addr, PAGE_SIZE);
117 }
118
net_devmem_unbind_dmabuf(struct net_devmem_dmabuf_binding * binding)119 void net_devmem_unbind_dmabuf(struct net_devmem_dmabuf_binding *binding)
120 {
121 struct netdev_rx_queue *rxq;
122 unsigned long xa_idx;
123 unsigned int rxq_idx;
124
125 xa_erase(&net_devmem_dmabuf_bindings, binding->id);
126
127 /* Ensure no tx net_devmem_lookup_dmabuf() are in flight after the
128 * erase.
129 */
130 synchronize_net();
131
132 if (binding->list.next)
133 list_del(&binding->list);
134
135 xa_for_each(&binding->bound_rxqs, xa_idx, rxq) {
136 const struct pp_memory_provider_params mp_params = {
137 .mp_priv = binding,
138 .mp_ops = &dmabuf_devmem_ops,
139 };
140
141 rxq_idx = get_netdev_rx_queue_index(rxq);
142
143 __net_mp_close_rxq(binding->dev, rxq_idx, &mp_params);
144 }
145
146 net_devmem_dmabuf_binding_put(binding);
147 }
148
net_devmem_bind_dmabuf_to_queue(struct net_device * dev,u32 rxq_idx,struct net_devmem_dmabuf_binding * binding,struct netlink_ext_ack * extack)149 int net_devmem_bind_dmabuf_to_queue(struct net_device *dev, u32 rxq_idx,
150 struct net_devmem_dmabuf_binding *binding,
151 struct netlink_ext_ack *extack)
152 {
153 struct pp_memory_provider_params mp_params = {
154 .mp_priv = binding,
155 .mp_ops = &dmabuf_devmem_ops,
156 };
157 struct netdev_rx_queue *rxq;
158 u32 xa_idx;
159 int err;
160
161 err = __net_mp_open_rxq(dev, rxq_idx, &mp_params, extack);
162 if (err)
163 return err;
164
165 rxq = __netif_get_rx_queue(dev, rxq_idx);
166 err = xa_alloc(&binding->bound_rxqs, &xa_idx, rxq, xa_limit_32b,
167 GFP_KERNEL);
168 if (err)
169 goto err_close_rxq;
170
171 return 0;
172
173 err_close_rxq:
174 __net_mp_close_rxq(dev, rxq_idx, &mp_params);
175 return err;
176 }
177
178 struct net_devmem_dmabuf_binding *
net_devmem_bind_dmabuf(struct net_device * dev,struct device * dma_dev,enum dma_data_direction direction,unsigned int dmabuf_fd,struct netdev_nl_sock * priv,struct netlink_ext_ack * extack)179 net_devmem_bind_dmabuf(struct net_device *dev,
180 struct device *dma_dev,
181 enum dma_data_direction direction,
182 unsigned int dmabuf_fd, struct netdev_nl_sock *priv,
183 struct netlink_ext_ack *extack)
184 {
185 struct net_devmem_dmabuf_binding *binding;
186 static u32 id_alloc_next;
187 struct scatterlist *sg;
188 struct dma_buf *dmabuf;
189 unsigned int sg_idx, i;
190 unsigned long virtual;
191 int err;
192
193 if (!dma_dev) {
194 NL_SET_ERR_MSG(extack, "Device doesn't support DMA");
195 return ERR_PTR(-EOPNOTSUPP);
196 }
197
198 dmabuf = dma_buf_get(dmabuf_fd);
199 if (IS_ERR(dmabuf))
200 return ERR_CAST(dmabuf);
201
202 binding = kzalloc_node(sizeof(*binding), GFP_KERNEL,
203 dev_to_node(&dev->dev));
204 if (!binding) {
205 err = -ENOMEM;
206 goto err_put_dmabuf;
207 }
208
209 binding->dev = dev;
210 xa_init_flags(&binding->bound_rxqs, XA_FLAGS_ALLOC);
211
212 refcount_set(&binding->ref, 1);
213
214 mutex_init(&binding->lock);
215
216 binding->dmabuf = dmabuf;
217 binding->direction = direction;
218
219 binding->attachment = dma_buf_attach(binding->dmabuf, dma_dev);
220 if (IS_ERR(binding->attachment)) {
221 err = PTR_ERR(binding->attachment);
222 NL_SET_ERR_MSG(extack, "Failed to bind dmabuf to device");
223 goto err_free_binding;
224 }
225
226 binding->sgt = dma_buf_map_attachment_unlocked(binding->attachment,
227 direction);
228 if (IS_ERR(binding->sgt)) {
229 err = PTR_ERR(binding->sgt);
230 NL_SET_ERR_MSG(extack, "Failed to map dmabuf attachment");
231 goto err_detach;
232 }
233
234 if (direction == DMA_TO_DEVICE) {
235 binding->tx_vec = kvmalloc_array(dmabuf->size / PAGE_SIZE,
236 sizeof(struct net_iov *),
237 GFP_KERNEL);
238 if (!binding->tx_vec) {
239 err = -ENOMEM;
240 goto err_unmap;
241 }
242 }
243
244 /* For simplicity we expect to make PAGE_SIZE allocations, but the
245 * binding can be much more flexible than that. We may be able to
246 * allocate MTU sized chunks here. Leave that for future work...
247 */
248 binding->chunk_pool = gen_pool_create(PAGE_SHIFT,
249 dev_to_node(&dev->dev));
250 if (!binding->chunk_pool) {
251 err = -ENOMEM;
252 goto err_tx_vec;
253 }
254
255 virtual = 0;
256 for_each_sgtable_dma_sg(binding->sgt, sg, sg_idx) {
257 dma_addr_t dma_addr = sg_dma_address(sg);
258 struct dmabuf_genpool_chunk_owner *owner;
259 size_t len = sg_dma_len(sg);
260 struct net_iov *niov;
261
262 owner = kzalloc_node(sizeof(*owner), GFP_KERNEL,
263 dev_to_node(&dev->dev));
264 if (!owner) {
265 err = -ENOMEM;
266 goto err_free_chunks;
267 }
268
269 owner->area.base_virtual = virtual;
270 owner->base_dma_addr = dma_addr;
271 owner->area.num_niovs = len / PAGE_SIZE;
272 owner->binding = binding;
273
274 err = gen_pool_add_owner(binding->chunk_pool, dma_addr,
275 dma_addr, len, dev_to_node(&dev->dev),
276 owner);
277 if (err) {
278 kfree(owner);
279 err = -EINVAL;
280 goto err_free_chunks;
281 }
282
283 owner->area.niovs = kvmalloc_array(owner->area.num_niovs,
284 sizeof(*owner->area.niovs),
285 GFP_KERNEL);
286 if (!owner->area.niovs) {
287 err = -ENOMEM;
288 goto err_free_chunks;
289 }
290
291 for (i = 0; i < owner->area.num_niovs; i++) {
292 niov = &owner->area.niovs[i];
293 niov->type = NET_IOV_DMABUF;
294 niov->owner = &owner->area;
295 page_pool_set_dma_addr_netmem(net_iov_to_netmem(niov),
296 net_devmem_get_dma_addr(niov));
297 if (direction == DMA_TO_DEVICE)
298 binding->tx_vec[owner->area.base_virtual / PAGE_SIZE + i] = niov;
299 }
300
301 virtual += len;
302 }
303
304 err = xa_alloc_cyclic(&net_devmem_dmabuf_bindings, &binding->id,
305 binding, xa_limit_32b, &id_alloc_next,
306 GFP_KERNEL);
307 if (err < 0)
308 goto err_free_chunks;
309
310 list_add(&binding->list, &priv->bindings);
311
312 return binding;
313
314 err_free_chunks:
315 gen_pool_for_each_chunk(binding->chunk_pool,
316 net_devmem_dmabuf_free_chunk_owner, NULL);
317 gen_pool_destroy(binding->chunk_pool);
318 err_tx_vec:
319 kvfree(binding->tx_vec);
320 err_unmap:
321 dma_buf_unmap_attachment_unlocked(binding->attachment, binding->sgt,
322 direction);
323 err_detach:
324 dma_buf_detach(dmabuf, binding->attachment);
325 err_free_binding:
326 kfree(binding);
327 err_put_dmabuf:
328 dma_buf_put(dmabuf);
329 return ERR_PTR(err);
330 }
331
net_devmem_lookup_dmabuf(u32 id)332 struct net_devmem_dmabuf_binding *net_devmem_lookup_dmabuf(u32 id)
333 {
334 struct net_devmem_dmabuf_binding *binding;
335
336 rcu_read_lock();
337 binding = xa_load(&net_devmem_dmabuf_bindings, id);
338 if (binding) {
339 if (!net_devmem_dmabuf_binding_get(binding))
340 binding = NULL;
341 }
342 rcu_read_unlock();
343
344 return binding;
345 }
346
net_devmem_get_net_iov(struct net_iov * niov)347 void net_devmem_get_net_iov(struct net_iov *niov)
348 {
349 net_devmem_dmabuf_binding_get(net_devmem_iov_binding(niov));
350 }
351
net_devmem_put_net_iov(struct net_iov * niov)352 void net_devmem_put_net_iov(struct net_iov *niov)
353 {
354 net_devmem_dmabuf_binding_put(net_devmem_iov_binding(niov));
355 }
356
net_devmem_get_binding(struct sock * sk,unsigned int dmabuf_id)357 struct net_devmem_dmabuf_binding *net_devmem_get_binding(struct sock *sk,
358 unsigned int dmabuf_id)
359 {
360 struct net_devmem_dmabuf_binding *binding;
361 struct net_device *dst_dev;
362 struct dst_entry *dst;
363 int err = 0;
364
365 binding = net_devmem_lookup_dmabuf(dmabuf_id);
366 if (!binding || !binding->tx_vec) {
367 err = -EINVAL;
368 goto out_err;
369 }
370
371 rcu_read_lock();
372 dst = __sk_dst_get(sk);
373 /* If dst is NULL (route expired), attempt to rebuild it. */
374 if (unlikely(!dst)) {
375 if (inet_csk(sk)->icsk_af_ops->rebuild_header(sk)) {
376 err = -EHOSTUNREACH;
377 goto out_unlock;
378 }
379 dst = __sk_dst_get(sk);
380 if (unlikely(!dst)) {
381 err = -ENODEV;
382 goto out_unlock;
383 }
384 }
385
386 /* The dma-addrs in this binding are only reachable to the corresponding
387 * net_device.
388 */
389 dst_dev = dst_dev_rcu(dst);
390 if (unlikely(!dst_dev) || unlikely(dst_dev != binding->dev)) {
391 err = -ENODEV;
392 goto out_unlock;
393 }
394
395 rcu_read_unlock();
396 return binding;
397
398 out_unlock:
399 rcu_read_unlock();
400 out_err:
401 if (binding)
402 net_devmem_dmabuf_binding_put(binding);
403
404 return ERR_PTR(err);
405 }
406
407 struct net_iov *
net_devmem_get_niov_at(struct net_devmem_dmabuf_binding * binding,size_t virt_addr,size_t * off,size_t * size)408 net_devmem_get_niov_at(struct net_devmem_dmabuf_binding *binding,
409 size_t virt_addr, size_t *off, size_t *size)
410 {
411 if (virt_addr >= binding->dmabuf->size)
412 return NULL;
413
414 *off = virt_addr % PAGE_SIZE;
415 *size = PAGE_SIZE - *off;
416
417 return binding->tx_vec[virt_addr / PAGE_SIZE];
418 }
419
420 /*** "Dmabuf devmem memory provider" ***/
421
mp_dmabuf_devmem_init(struct page_pool * pool)422 int mp_dmabuf_devmem_init(struct page_pool *pool)
423 {
424 struct net_devmem_dmabuf_binding *binding = pool->mp_priv;
425
426 if (!binding)
427 return -EINVAL;
428
429 /* dma-buf dma addresses do not need and should not be used with
430 * dma_sync_for_cpu/device. Force disable dma_sync.
431 */
432 pool->dma_sync = false;
433 pool->dma_sync_for_cpu = false;
434
435 if (pool->p.order != 0)
436 return -E2BIG;
437
438 net_devmem_dmabuf_binding_get(binding);
439 return 0;
440 }
441
mp_dmabuf_devmem_alloc_netmems(struct page_pool * pool,gfp_t gfp)442 netmem_ref mp_dmabuf_devmem_alloc_netmems(struct page_pool *pool, gfp_t gfp)
443 {
444 struct net_devmem_dmabuf_binding *binding = pool->mp_priv;
445 struct net_iov *niov;
446 netmem_ref netmem;
447
448 niov = net_devmem_alloc_dmabuf(binding);
449 if (!niov)
450 return 0;
451
452 netmem = net_iov_to_netmem(niov);
453
454 page_pool_set_pp_info(pool, netmem);
455
456 pool->pages_state_hold_cnt++;
457 trace_page_pool_state_hold(pool, netmem, pool->pages_state_hold_cnt);
458 return netmem;
459 }
460
mp_dmabuf_devmem_destroy(struct page_pool * pool)461 void mp_dmabuf_devmem_destroy(struct page_pool *pool)
462 {
463 struct net_devmem_dmabuf_binding *binding = pool->mp_priv;
464
465 net_devmem_dmabuf_binding_put(binding);
466 }
467
mp_dmabuf_devmem_release_page(struct page_pool * pool,netmem_ref netmem)468 bool mp_dmabuf_devmem_release_page(struct page_pool *pool, netmem_ref netmem)
469 {
470 long refcount = atomic_long_read(netmem_get_pp_ref_count_ref(netmem));
471
472 if (WARN_ON_ONCE(!netmem_is_net_iov(netmem)))
473 return false;
474
475 if (WARN_ON_ONCE(refcount != 1))
476 return false;
477
478 page_pool_clear_pp_info(netmem);
479
480 net_devmem_free_dmabuf(netmem_to_net_iov(netmem));
481
482 /* We don't want the page pool put_page()ing our net_iovs. */
483 return false;
484 }
485
mp_dmabuf_devmem_nl_fill(void * mp_priv,struct sk_buff * rsp,struct netdev_rx_queue * rxq)486 static int mp_dmabuf_devmem_nl_fill(void *mp_priv, struct sk_buff *rsp,
487 struct netdev_rx_queue *rxq)
488 {
489 const struct net_devmem_dmabuf_binding *binding = mp_priv;
490 int type = rxq ? NETDEV_A_QUEUE_DMABUF : NETDEV_A_PAGE_POOL_DMABUF;
491
492 return nla_put_u32(rsp, type, binding->id);
493 }
494
mp_dmabuf_devmem_uninstall(void * mp_priv,struct netdev_rx_queue * rxq)495 static void mp_dmabuf_devmem_uninstall(void *mp_priv,
496 struct netdev_rx_queue *rxq)
497 {
498 struct net_devmem_dmabuf_binding *binding = mp_priv;
499 struct netdev_rx_queue *bound_rxq;
500 unsigned long xa_idx;
501
502 xa_for_each(&binding->bound_rxqs, xa_idx, bound_rxq) {
503 if (bound_rxq == rxq) {
504 xa_erase(&binding->bound_rxqs, xa_idx);
505 if (xa_empty(&binding->bound_rxqs)) {
506 mutex_lock(&binding->lock);
507 binding->dev = NULL;
508 mutex_unlock(&binding->lock);
509 }
510 break;
511 }
512 }
513 }
514
515 static const struct memory_provider_ops dmabuf_devmem_ops = {
516 .init = mp_dmabuf_devmem_init,
517 .destroy = mp_dmabuf_devmem_destroy,
518 .alloc_netmems = mp_dmabuf_devmem_alloc_netmems,
519 .release_netmem = mp_dmabuf_devmem_release_page,
520 .nl_fill = mp_dmabuf_devmem_nl_fill,
521 .uninstall = mp_dmabuf_devmem_uninstall,
522 };
523