1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /*
3 * Devmem TCP
4 *
5 * Authors: Mina Almasry <almasrymina@google.com>
6 * Willem de Bruijn <willemdebruijn.kernel@gmail.com>
7 * Kaiyuan Zhang <kaiyuanz@google.com
8 */
9
10 #include <linux/dma-buf.h>
11 #include <linux/genalloc.h>
12 #include <linux/mm.h>
13 #include <linux/netdevice.h>
14 #include <linux/types.h>
15 #include <net/netdev_queues.h>
16 #include <net/netdev_rx_queue.h>
17 #include <net/page_pool/helpers.h>
18 #include <net/page_pool/memory_provider.h>
19 #include <net/sock.h>
20 #include <net/tcp.h>
21 #include <trace/events/page_pool.h>
22
23 #include "devmem.h"
24 #include "mp_dmabuf_devmem.h"
25 #include "page_pool_priv.h"
26
27 /* Device memory support */
28
29 static DEFINE_XARRAY_FLAGS(net_devmem_dmabuf_bindings, XA_FLAGS_ALLOC1);
30
31 static const struct memory_provider_ops dmabuf_devmem_ops;
32
net_devmem_dmabuf_free_chunk_owner(struct gen_pool * genpool,struct gen_pool_chunk * chunk,void * not_used)33 static void net_devmem_dmabuf_free_chunk_owner(struct gen_pool *genpool,
34 struct gen_pool_chunk *chunk,
35 void *not_used)
36 {
37 struct dmabuf_genpool_chunk_owner *owner = chunk->owner;
38
39 kvfree(owner->area.niovs);
40 kfree(owner);
41 }
42
net_devmem_get_dma_addr(const struct net_iov * niov)43 static dma_addr_t net_devmem_get_dma_addr(const struct net_iov *niov)
44 {
45 struct dmabuf_genpool_chunk_owner *owner;
46
47 owner = net_devmem_iov_to_chunk_owner(niov);
48 return owner->base_dma_addr +
49 ((dma_addr_t)net_iov_idx(niov) << PAGE_SHIFT);
50 }
51
net_devmem_dmabuf_binding_release(struct percpu_ref * ref)52 static void net_devmem_dmabuf_binding_release(struct percpu_ref *ref)
53 {
54 struct net_devmem_dmabuf_binding *binding =
55 container_of(ref, struct net_devmem_dmabuf_binding, ref);
56
57 INIT_WORK(&binding->unbind_w, __net_devmem_dmabuf_binding_free);
58 schedule_work(&binding->unbind_w);
59 }
60
__net_devmem_dmabuf_binding_free(struct work_struct * wq)61 void __net_devmem_dmabuf_binding_free(struct work_struct *wq)
62 {
63 struct net_devmem_dmabuf_binding *binding = container_of(wq, typeof(*binding), unbind_w);
64
65 size_t size, avail;
66
67 gen_pool_for_each_chunk(binding->chunk_pool,
68 net_devmem_dmabuf_free_chunk_owner, NULL);
69
70 size = gen_pool_size(binding->chunk_pool);
71 avail = gen_pool_avail(binding->chunk_pool);
72
73 if (!WARN(size != avail, "can't destroy genpool. size=%zu, avail=%zu",
74 size, avail))
75 gen_pool_destroy(binding->chunk_pool);
76
77 dma_buf_unmap_attachment_unlocked(binding->attachment, binding->sgt,
78 binding->direction);
79 dma_buf_detach(binding->dmabuf, binding->attachment);
80 dma_buf_put(binding->dmabuf);
81 xa_destroy(&binding->bound_rxqs);
82 percpu_ref_exit(&binding->ref);
83 kvfree(binding->tx_vec);
84 kfree(binding);
85 }
86
87 struct net_iov *
net_devmem_alloc_dmabuf(struct net_devmem_dmabuf_binding * binding)88 net_devmem_alloc_dmabuf(struct net_devmem_dmabuf_binding *binding)
89 {
90 struct dmabuf_genpool_chunk_owner *owner;
91 unsigned long dma_addr;
92 struct net_iov *niov;
93 ssize_t offset;
94 ssize_t index;
95
96 dma_addr = gen_pool_alloc_owner(binding->chunk_pool, PAGE_SIZE,
97 (void **)&owner);
98 if (!dma_addr)
99 return NULL;
100
101 offset = dma_addr - owner->base_dma_addr;
102 index = offset / PAGE_SIZE;
103 niov = &owner->area.niovs[index];
104
105 niov->desc.pp_magic = 0;
106 niov->desc.pp = NULL;
107 atomic_long_set(&niov->desc.pp_ref_count, 0);
108
109 return niov;
110 }
111
net_devmem_free_dmabuf(struct net_iov * niov)112 void net_devmem_free_dmabuf(struct net_iov *niov)
113 {
114 struct net_devmem_dmabuf_binding *binding = net_devmem_iov_binding(niov);
115 unsigned long dma_addr = net_devmem_get_dma_addr(niov);
116
117 if (WARN_ON(!gen_pool_has_addr(binding->chunk_pool, dma_addr,
118 PAGE_SIZE)))
119 return;
120
121 gen_pool_free(binding->chunk_pool, dma_addr, PAGE_SIZE);
122 }
123
net_devmem_unbind_dmabuf(struct net_devmem_dmabuf_binding * binding)124 void net_devmem_unbind_dmabuf(struct net_devmem_dmabuf_binding *binding)
125 {
126 struct netdev_rx_queue *rxq;
127 unsigned long xa_idx;
128 unsigned int rxq_idx;
129
130 xa_erase(&net_devmem_dmabuf_bindings, binding->id);
131
132 /* Ensure no tx net_devmem_lookup_dmabuf() are in flight after the
133 * erase.
134 */
135 synchronize_net();
136
137 if (binding->list.next)
138 list_del(&binding->list);
139
140 xa_for_each(&binding->bound_rxqs, xa_idx, rxq) {
141 const struct pp_memory_provider_params mp_params = {
142 .mp_priv = binding,
143 .mp_ops = &dmabuf_devmem_ops,
144 };
145
146 rxq_idx = get_netdev_rx_queue_index(rxq);
147
148 netif_mp_close_rxq(binding->dev, rxq_idx, &mp_params);
149 }
150
151 percpu_ref_kill(&binding->ref);
152 }
153
net_devmem_bind_dmabuf_to_queue(struct net_device * dev,u32 rxq_idx,struct net_devmem_dmabuf_binding * binding,struct netlink_ext_ack * extack)154 int net_devmem_bind_dmabuf_to_queue(struct net_device *dev, u32 rxq_idx,
155 struct net_devmem_dmabuf_binding *binding,
156 struct netlink_ext_ack *extack)
157 {
158 struct pp_memory_provider_params mp_params = {
159 .mp_priv = binding,
160 .mp_ops = &dmabuf_devmem_ops,
161 };
162 struct netdev_rx_queue *rxq;
163 u32 xa_idx;
164 int err;
165
166 err = netif_mp_open_rxq(dev, rxq_idx, &mp_params, extack);
167 if (err)
168 return err;
169
170 rxq = __netif_get_rx_queue(dev, rxq_idx);
171 err = xa_alloc(&binding->bound_rxqs, &xa_idx, rxq, xa_limit_32b,
172 GFP_KERNEL);
173 if (err)
174 goto err_close_rxq;
175
176 return 0;
177
178 err_close_rxq:
179 netif_mp_close_rxq(dev, rxq_idx, &mp_params);
180 return err;
181 }
182
183 struct net_devmem_dmabuf_binding *
net_devmem_bind_dmabuf(struct net_device * dev,struct device * dma_dev,enum dma_data_direction direction,unsigned int dmabuf_fd,struct netdev_nl_sock * priv,struct netlink_ext_ack * extack)184 net_devmem_bind_dmabuf(struct net_device *dev,
185 struct device *dma_dev,
186 enum dma_data_direction direction,
187 unsigned int dmabuf_fd, struct netdev_nl_sock *priv,
188 struct netlink_ext_ack *extack)
189 {
190 struct net_devmem_dmabuf_binding *binding;
191 static u32 id_alloc_next;
192 struct scatterlist *sg;
193 struct dma_buf *dmabuf;
194 unsigned int sg_idx, i;
195 unsigned long virtual;
196 int err;
197
198 if (!dma_dev) {
199 NL_SET_ERR_MSG(extack, "Device doesn't support DMA");
200 return ERR_PTR(-EOPNOTSUPP);
201 }
202
203 dmabuf = dma_buf_get(dmabuf_fd);
204 if (IS_ERR(dmabuf))
205 return ERR_CAST(dmabuf);
206
207 binding = kzalloc_node(sizeof(*binding), GFP_KERNEL,
208 dev_to_node(&dev->dev));
209 if (!binding) {
210 err = -ENOMEM;
211 goto err_put_dmabuf;
212 }
213
214 binding->dev = dev;
215 xa_init_flags(&binding->bound_rxqs, XA_FLAGS_ALLOC);
216
217 err = percpu_ref_init(&binding->ref,
218 net_devmem_dmabuf_binding_release,
219 0, GFP_KERNEL);
220 if (err < 0)
221 goto err_free_binding;
222
223 mutex_init(&binding->lock);
224
225 binding->dmabuf = dmabuf;
226 binding->direction = direction;
227
228 binding->attachment = dma_buf_attach(binding->dmabuf, dma_dev);
229 if (IS_ERR(binding->attachment)) {
230 err = PTR_ERR(binding->attachment);
231 NL_SET_ERR_MSG(extack, "Failed to bind dmabuf to device");
232 goto err_exit_ref;
233 }
234
235 binding->sgt = dma_buf_map_attachment_unlocked(binding->attachment,
236 direction);
237 if (IS_ERR(binding->sgt)) {
238 err = PTR_ERR(binding->sgt);
239 NL_SET_ERR_MSG(extack, "Failed to map dmabuf attachment");
240 goto err_detach;
241 }
242
243 if (direction == DMA_TO_DEVICE) {
244 if (!IS_ALIGNED(dmabuf->size, PAGE_SIZE)) {
245 err = -EINVAL;
246 NL_SET_ERR_MSG(extack, "TX dma-buf size must be a multiple of PAGE_SIZE");
247 goto err_unmap;
248 }
249 binding->tx_vec = kvmalloc_objs(struct net_iov *,
250 dmabuf->size / PAGE_SIZE);
251 if (!binding->tx_vec) {
252 err = -ENOMEM;
253 goto err_unmap;
254 }
255 }
256
257 /* For simplicity we expect to make PAGE_SIZE allocations, but the
258 * binding can be much more flexible than that. We may be able to
259 * allocate MTU sized chunks here. Leave that for future work...
260 */
261 binding->chunk_pool = gen_pool_create(PAGE_SHIFT,
262 dev_to_node(&dev->dev));
263 if (!binding->chunk_pool) {
264 err = -ENOMEM;
265 goto err_tx_vec;
266 }
267
268 virtual = 0;
269 for_each_sgtable_dma_sg(binding->sgt, sg, sg_idx) {
270 dma_addr_t dma_addr = sg_dma_address(sg);
271 struct dmabuf_genpool_chunk_owner *owner;
272 size_t len = sg_dma_len(sg);
273 struct net_iov *niov;
274
275 if (!IS_ALIGNED(len, PAGE_SIZE)) {
276 err = -EINVAL;
277 NL_SET_ERR_MSG(extack, "dma-buf SG length must be PAGE_SIZE aligned");
278 goto err_free_chunks;
279 }
280
281 owner = kzalloc_node(sizeof(*owner), GFP_KERNEL,
282 dev_to_node(&dev->dev));
283 if (!owner) {
284 err = -ENOMEM;
285 goto err_free_chunks;
286 }
287
288 owner->area.base_virtual = virtual;
289 owner->base_dma_addr = dma_addr;
290 owner->area.num_niovs = len / PAGE_SIZE;
291 owner->binding = binding;
292
293 err = gen_pool_add_owner(binding->chunk_pool, dma_addr,
294 dma_addr, len, dev_to_node(&dev->dev),
295 owner);
296 if (err) {
297 kfree(owner);
298 err = -EINVAL;
299 goto err_free_chunks;
300 }
301
302 owner->area.niovs = kvmalloc_objs(*owner->area.niovs,
303 owner->area.num_niovs);
304 if (!owner->area.niovs) {
305 err = -ENOMEM;
306 goto err_free_chunks;
307 }
308
309 for (i = 0; i < owner->area.num_niovs; i++) {
310 niov = &owner->area.niovs[i];
311 net_iov_init(niov, &owner->area, NET_IOV_DMABUF);
312 page_pool_set_dma_addr_netmem(net_iov_to_netmem(niov),
313 net_devmem_get_dma_addr(niov));
314 if (direction == DMA_TO_DEVICE)
315 binding->tx_vec[owner->area.base_virtual / PAGE_SIZE + i] = niov;
316 }
317
318 virtual += len;
319 }
320
321 err = xa_alloc_cyclic(&net_devmem_dmabuf_bindings, &binding->id,
322 binding, xa_limit_32b, &id_alloc_next,
323 GFP_KERNEL);
324 if (err < 0)
325 goto err_free_chunks;
326
327 list_add(&binding->list, &priv->bindings);
328
329 return binding;
330
331 err_free_chunks:
332 gen_pool_for_each_chunk(binding->chunk_pool,
333 net_devmem_dmabuf_free_chunk_owner, NULL);
334 gen_pool_destroy(binding->chunk_pool);
335 err_tx_vec:
336 kvfree(binding->tx_vec);
337 err_unmap:
338 dma_buf_unmap_attachment_unlocked(binding->attachment, binding->sgt,
339 direction);
340 err_detach:
341 dma_buf_detach(dmabuf, binding->attachment);
342 err_exit_ref:
343 percpu_ref_exit(&binding->ref);
344 err_free_binding:
345 kfree(binding);
346 err_put_dmabuf:
347 dma_buf_put(dmabuf);
348 return ERR_PTR(err);
349 }
350
net_devmem_lookup_dmabuf(u32 id)351 struct net_devmem_dmabuf_binding *net_devmem_lookup_dmabuf(u32 id)
352 {
353 struct net_devmem_dmabuf_binding *binding;
354
355 rcu_read_lock();
356 binding = xa_load(&net_devmem_dmabuf_bindings, id);
357 if (binding) {
358 if (!net_devmem_dmabuf_binding_get(binding))
359 binding = NULL;
360 }
361 rcu_read_unlock();
362
363 return binding;
364 }
365
net_devmem_get_net_iov(struct net_iov * niov)366 void net_devmem_get_net_iov(struct net_iov *niov)
367 {
368 net_devmem_dmabuf_binding_get(net_devmem_iov_binding(niov));
369 }
370
net_devmem_put_net_iov(struct net_iov * niov)371 void net_devmem_put_net_iov(struct net_iov *niov)
372 {
373 net_devmem_dmabuf_binding_put(net_devmem_iov_binding(niov));
374 }
375
net_devmem_get_binding(struct sock * sk,unsigned int dmabuf_id)376 struct net_devmem_dmabuf_binding *net_devmem_get_binding(struct sock *sk,
377 unsigned int dmabuf_id)
378 {
379 struct net_devmem_dmabuf_binding *binding;
380 struct net_device *dst_dev;
381 struct dst_entry *dst;
382 int err = 0;
383
384 binding = net_devmem_lookup_dmabuf(dmabuf_id);
385 if (!binding || !binding->tx_vec) {
386 err = -EINVAL;
387 goto out_err;
388 }
389
390 rcu_read_lock();
391 dst = __sk_dst_get(sk);
392 /* If dst is NULL (route expired), attempt to rebuild it. */
393 if (unlikely(!dst)) {
394 if (inet_csk(sk)->icsk_af_ops->rebuild_header(sk)) {
395 err = -EHOSTUNREACH;
396 goto out_unlock;
397 }
398 dst = __sk_dst_get(sk);
399 if (unlikely(!dst)) {
400 err = -ENODEV;
401 goto out_unlock;
402 }
403 }
404
405 /* The dma-addrs in this binding are only reachable to the corresponding
406 * net_device.
407 */
408 dst_dev = dst_dev_rcu(dst);
409 if (unlikely(!dst_dev) ||
410 unlikely(dst_dev != READ_ONCE(binding->dev))) {
411 err = -ENODEV;
412 goto out_unlock;
413 }
414
415 rcu_read_unlock();
416 return binding;
417
418 out_unlock:
419 rcu_read_unlock();
420 out_err:
421 if (binding)
422 net_devmem_dmabuf_binding_put(binding);
423
424 return ERR_PTR(err);
425 }
426
427 struct net_iov *
net_devmem_get_niov_at(struct net_devmem_dmabuf_binding * binding,size_t virt_addr,size_t * off,size_t * size)428 net_devmem_get_niov_at(struct net_devmem_dmabuf_binding *binding,
429 size_t virt_addr, size_t *off, size_t *size)
430 {
431 if (virt_addr >= binding->dmabuf->size)
432 return NULL;
433
434 *off = virt_addr % PAGE_SIZE;
435 *size = PAGE_SIZE - *off;
436
437 return binding->tx_vec[virt_addr / PAGE_SIZE];
438 }
439
440 /*** "Dmabuf devmem memory provider" ***/
441
mp_dmabuf_devmem_init(struct page_pool * pool)442 int mp_dmabuf_devmem_init(struct page_pool *pool)
443 {
444 struct net_devmem_dmabuf_binding *binding = pool->mp_priv;
445
446 if (!binding)
447 return -EINVAL;
448
449 /* dma-buf dma addresses do not need and should not be used with
450 * dma_sync_for_cpu/device. Force disable dma_sync.
451 */
452 pool->dma_sync = false;
453 pool->dma_sync_for_cpu = false;
454
455 if (pool->p.order != 0)
456 return -E2BIG;
457
458 net_devmem_dmabuf_binding_get(binding);
459 return 0;
460 }
461
mp_dmabuf_devmem_alloc_netmems(struct page_pool * pool,gfp_t gfp)462 netmem_ref mp_dmabuf_devmem_alloc_netmems(struct page_pool *pool, gfp_t gfp)
463 {
464 struct net_devmem_dmabuf_binding *binding = pool->mp_priv;
465 struct net_iov *niov;
466 netmem_ref netmem;
467
468 niov = net_devmem_alloc_dmabuf(binding);
469 if (!niov)
470 return 0;
471
472 netmem = net_iov_to_netmem(niov);
473
474 page_pool_set_pp_info(pool, netmem);
475
476 pool->pages_state_hold_cnt++;
477 trace_page_pool_state_hold(pool, netmem, pool->pages_state_hold_cnt);
478 return netmem;
479 }
480
mp_dmabuf_devmem_destroy(struct page_pool * pool)481 void mp_dmabuf_devmem_destroy(struct page_pool *pool)
482 {
483 struct net_devmem_dmabuf_binding *binding = pool->mp_priv;
484
485 net_devmem_dmabuf_binding_put(binding);
486 }
487
mp_dmabuf_devmem_release_page(struct page_pool * pool,netmem_ref netmem)488 bool mp_dmabuf_devmem_release_page(struct page_pool *pool, netmem_ref netmem)
489 {
490 long refcount = atomic_long_read(netmem_get_pp_ref_count_ref(netmem));
491
492 if (WARN_ON_ONCE(!netmem_is_net_iov(netmem)))
493 return false;
494
495 if (WARN_ON_ONCE(refcount != 1))
496 return false;
497
498 page_pool_clear_pp_info(netmem);
499
500 net_devmem_free_dmabuf(netmem_to_net_iov(netmem));
501
502 /* We don't want the page pool put_page()ing our net_iovs. */
503 return false;
504 }
505
mp_dmabuf_devmem_nl_fill(void * mp_priv,struct sk_buff * rsp,struct netdev_rx_queue * rxq)506 static int mp_dmabuf_devmem_nl_fill(void *mp_priv, struct sk_buff *rsp,
507 struct netdev_rx_queue *rxq)
508 {
509 const struct net_devmem_dmabuf_binding *binding = mp_priv;
510 int type = rxq ? NETDEV_A_QUEUE_DMABUF : NETDEV_A_PAGE_POOL_DMABUF;
511
512 return nla_put_u32(rsp, type, binding->id);
513 }
514
mp_dmabuf_devmem_uninstall(void * mp_priv,struct netdev_rx_queue * rxq)515 static void mp_dmabuf_devmem_uninstall(void *mp_priv,
516 struct netdev_rx_queue *rxq)
517 {
518 struct net_devmem_dmabuf_binding *binding = mp_priv;
519 struct netdev_rx_queue *bound_rxq;
520 unsigned long xa_idx;
521
522 xa_for_each(&binding->bound_rxqs, xa_idx, bound_rxq) {
523 if (bound_rxq == rxq) {
524 xa_erase(&binding->bound_rxqs, xa_idx);
525 if (xa_empty(&binding->bound_rxqs)) {
526 mutex_lock(&binding->lock);
527 ASSERT_EXCLUSIVE_WRITER(binding->dev);
528 WRITE_ONCE(binding->dev, NULL);
529 mutex_unlock(&binding->lock);
530 }
531 break;
532 }
533 }
534 }
535
536 static const struct memory_provider_ops dmabuf_devmem_ops = {
537 .init = mp_dmabuf_devmem_init,
538 .destroy = mp_dmabuf_devmem_destroy,
539 .alloc_netmems = mp_dmabuf_devmem_alloc_netmems,
540 .release_netmem = mp_dmabuf_devmem_release_page,
541 .nl_fill = mp_dmabuf_devmem_nl_fill,
542 .uninstall = mp_dmabuf_devmem_uninstall,
543 };
544