xref: /linux/net/core/devmem.c (revision cf85f810f911234a06a4ef2439e8694b93b717fc) 
1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /*
3  *      Devmem TCP
4  *
5  *      Authors:	Mina Almasry <almasrymina@google.com>
6  *			Willem de Bruijn <willemdebruijn.kernel@gmail.com>
7  *			Kaiyuan Zhang <kaiyuanz@google.com
8  */
9 
10 #include <linux/dma-buf.h>
11 #include <linux/genalloc.h>
12 #include <linux/mm.h>
13 #include <linux/netdevice.h>
14 #include <linux/types.h>
15 #include <net/netdev_queues.h>
16 #include <net/netdev_rx_queue.h>
17 #include <net/page_pool/helpers.h>
18 #include <net/page_pool/memory_provider.h>
19 #include <net/sock.h>
20 #include <net/tcp.h>
21 #include <trace/events/page_pool.h>
22 
23 #include "devmem.h"
24 #include "mp_dmabuf_devmem.h"
25 #include "page_pool_priv.h"
26 
27 /* Device memory support */
28 
29 static DEFINE_XARRAY_FLAGS(net_devmem_dmabuf_bindings, XA_FLAGS_ALLOC1);
30 
31 static const struct memory_provider_ops dmabuf_devmem_ops;
32 
33 static void net_devmem_dmabuf_free_chunk_owner(struct gen_pool *genpool,
34 					       struct gen_pool_chunk *chunk,
35 					       void *not_used)
36 {
37 	struct dmabuf_genpool_chunk_owner *owner = chunk->owner;
38 
39 	kvfree(owner->area.niovs);
40 	kfree(owner);
41 }
42 
43 static dma_addr_t net_devmem_get_dma_addr(const struct net_iov *niov)
44 {
45 	struct dmabuf_genpool_chunk_owner *owner;
46 
47 	owner = net_devmem_iov_to_chunk_owner(niov);
48 	return owner->base_dma_addr +
49 	       ((dma_addr_t)net_iov_idx(niov) << owner->binding->niov_shift);
50 }
51 
52 static void net_devmem_dmabuf_binding_release(struct percpu_ref *ref)
53 {
54 	struct net_devmem_dmabuf_binding *binding =
55 		container_of(ref, struct net_devmem_dmabuf_binding, ref);
56 
57 	INIT_WORK(&binding->unbind_w, __net_devmem_dmabuf_binding_free);
58 	schedule_work(&binding->unbind_w);
59 }
60 
61 void __net_devmem_dmabuf_binding_free(struct work_struct *wq)
62 {
63 	struct net_devmem_dmabuf_binding *binding = container_of(wq, typeof(*binding), unbind_w);
64 
65 	size_t size, avail;
66 
67 	gen_pool_for_each_chunk(binding->chunk_pool,
68 				net_devmem_dmabuf_free_chunk_owner, NULL);
69 
70 	size = gen_pool_size(binding->chunk_pool);
71 	avail = gen_pool_avail(binding->chunk_pool);
72 
73 	if (!WARN(size != avail, "can't destroy genpool. size=%zu, avail=%zu",
74 		  size, avail))
75 		gen_pool_destroy(binding->chunk_pool);
76 
77 	dma_buf_unmap_attachment_unlocked(binding->attachment, binding->sgt,
78 					  binding->direction);
79 	dma_buf_detach(binding->dmabuf, binding->attachment);
80 	dma_buf_put(binding->dmabuf);
81 	xa_destroy(&binding->bound_rxqs);
82 	percpu_ref_exit(&binding->ref);
83 	kvfree(binding->tx_vec);
84 	kfree(binding);
85 }
86 
87 struct net_iov *
88 net_devmem_alloc_dmabuf(struct net_devmem_dmabuf_binding *binding)
89 {
90 	struct dmabuf_genpool_chunk_owner *owner;
91 	unsigned long dma_addr;
92 	struct net_iov *niov;
93 	ssize_t offset;
94 	ssize_t index;
95 
96 	dma_addr = gen_pool_alloc_owner(binding->chunk_pool,
97 					1UL << binding->niov_shift,
98 					(void **)&owner);
99 	if (!dma_addr)
100 		return NULL;
101 
102 	offset = dma_addr - owner->base_dma_addr;
103 	index = offset >> binding->niov_shift;
104 	niov = &owner->area.niovs[index];
105 
106 	niov->desc.pp_magic = 0;
107 	niov->desc.pp = NULL;
108 	atomic_long_set(&niov->desc.pp_ref_count, 0);
109 
110 	return niov;
111 }
112 
113 void net_devmem_free_dmabuf(struct net_iov *niov)
114 {
115 	struct net_devmem_dmabuf_binding *binding = net_devmem_iov_binding(niov);
116 	unsigned long dma_addr = net_devmem_get_dma_addr(niov);
117 	size_t niov_size = 1UL << binding->niov_shift;
118 
119 	if (WARN_ON(!gen_pool_has_addr(binding->chunk_pool, dma_addr,
120 				       niov_size)))
121 		return;
122 
123 	gen_pool_free(binding->chunk_pool, dma_addr, niov_size);
124 }
125 
126 void net_devmem_unbind_dmabuf(struct net_devmem_dmabuf_binding *binding)
127 {
128 	struct netdev_rx_queue *rxq;
129 	unsigned long xa_idx;
130 	unsigned int rxq_idx;
131 
132 	xa_erase(&net_devmem_dmabuf_bindings, binding->id);
133 
134 	/* Ensure no tx net_devmem_lookup_dmabuf() are in flight after the
135 	 * erase.
136 	 */
137 	synchronize_net();
138 
139 	if (binding->list.next)
140 		list_del(&binding->list);
141 
142 	xa_for_each(&binding->bound_rxqs, xa_idx, rxq) {
143 		const struct pp_memory_provider_params mp_params = {
144 			.mp_priv	= binding,
145 			.mp_ops		= &dmabuf_devmem_ops,
146 		};
147 
148 		rxq_idx = get_netdev_rx_queue_index(rxq);
149 
150 		netif_mp_close_rxq(binding->dev, rxq_idx, &mp_params);
151 	}
152 
153 	percpu_ref_kill(&binding->ref);
154 }
155 
156 int net_devmem_bind_dmabuf_to_queue(struct net_device *dev, u32 rxq_idx,
157 				    struct net_devmem_dmabuf_binding *binding,
158 				    struct netlink_ext_ack *extack)
159 {
160 	struct pp_memory_provider_params mp_params = {
161 		.mp_priv	= binding,
162 		.mp_ops		= &dmabuf_devmem_ops,
163 	};
164 	struct netdev_rx_queue *rxq;
165 	u32 xa_idx;
166 	int err;
167 
168 	if (binding->niov_shift != PAGE_SHIFT)
169 		mp_params.rx_page_size = 1U << binding->niov_shift;
170 
171 	err = netif_mp_open_rxq(dev, rxq_idx, &mp_params, extack);
172 	if (err)
173 		return err;
174 
175 	rxq = __netif_get_rx_queue(dev, rxq_idx);
176 	err = xa_alloc(&binding->bound_rxqs, &xa_idx, rxq, xa_limit_32b,
177 		       GFP_KERNEL);
178 	if (err)
179 		goto err_close_rxq;
180 
181 	return 0;
182 
183 err_close_rxq:
184 	netif_mp_close_rxq(dev, rxq_idx, &mp_params);
185 	return err;
186 }
187 
188 struct net_devmem_dmabuf_binding *
189 net_devmem_bind_dmabuf(struct net_device *dev, void *vdev,
190 		       struct device *dma_dev,
191 		       enum dma_data_direction direction,
192 		       unsigned int dmabuf_fd, unsigned int niov_shift,
193 		       struct netdev_nl_sock *priv,
194 		       struct netlink_ext_ack *extack)
195 {
196 	struct net_devmem_dmabuf_binding *binding;
197 	size_t niov_size = 1UL << niov_shift;
198 	static u32 id_alloc_next;
199 	struct scatterlist *sg;
200 	struct dma_buf *dmabuf;
201 	unsigned int sg_idx, i;
202 	unsigned long virtual;
203 	int err;
204 
205 	if (!dma_dev) {
206 		NL_SET_ERR_MSG(extack, "Device doesn't support DMA");
207 		return ERR_PTR(-EOPNOTSUPP);
208 	}
209 
210 	dmabuf = dma_buf_get(dmabuf_fd);
211 	if (IS_ERR(dmabuf))
212 		return ERR_CAST(dmabuf);
213 
214 	binding = kzalloc_node(sizeof(*binding), GFP_KERNEL,
215 			       dev_to_node(&dev->dev));
216 	if (!binding) {
217 		err = -ENOMEM;
218 		goto err_put_dmabuf;
219 	}
220 
221 	binding->dev = dev;
222 	binding->vdev = vdev;
223 	binding->niov_shift = niov_shift;
224 	xa_init_flags(&binding->bound_rxqs, XA_FLAGS_ALLOC);
225 
226 	err = percpu_ref_init(&binding->ref,
227 			      net_devmem_dmabuf_binding_release,
228 			      0, GFP_KERNEL);
229 	if (err < 0)
230 		goto err_free_binding;
231 
232 	mutex_init(&binding->lock);
233 
234 	binding->dmabuf = dmabuf;
235 	binding->direction = direction;
236 
237 	binding->attachment = dma_buf_attach(binding->dmabuf, dma_dev);
238 	if (IS_ERR(binding->attachment)) {
239 		err = PTR_ERR(binding->attachment);
240 		NL_SET_ERR_MSG(extack, "Failed to bind dmabuf to device");
241 		goto err_exit_ref;
242 	}
243 
244 	binding->sgt = dma_buf_map_attachment_unlocked(binding->attachment,
245 						       direction);
246 	if (IS_ERR(binding->sgt)) {
247 		err = PTR_ERR(binding->sgt);
248 		NL_SET_ERR_MSG(extack, "Failed to map dmabuf attachment");
249 		goto err_detach;
250 	}
251 
252 	if (direction == DMA_TO_DEVICE) {
253 		if (!IS_ALIGNED(dmabuf->size, PAGE_SIZE)) {
254 			err = -EINVAL;
255 			NL_SET_ERR_MSG(extack, "TX dma-buf size must be a multiple of PAGE_SIZE");
256 			goto err_unmap;
257 		}
258 		binding->tx_vec = kvmalloc_objs(struct net_iov *,
259 						dmabuf->size / PAGE_SIZE);
260 		if (!binding->tx_vec) {
261 			err = -ENOMEM;
262 			goto err_unmap;
263 		}
264 	}
265 
266 	binding->chunk_pool = gen_pool_create(niov_shift,
267 					      dev_to_node(&dev->dev));
268 	if (!binding->chunk_pool) {
269 		err = -ENOMEM;
270 		goto err_tx_vec;
271 	}
272 
273 	virtual = 0;
274 	for_each_sgtable_dma_sg(binding->sgt, sg, sg_idx) {
275 		dma_addr_t dma_addr = sg_dma_address(sg);
276 		struct dmabuf_genpool_chunk_owner *owner;
277 		size_t len = sg_dma_len(sg);
278 		struct net_iov *niov;
279 
280 		if (!IS_ALIGNED(dma_addr, niov_size) ||
281 		    !IS_ALIGNED(len, niov_size)) {
282 			err = -EINVAL;
283 			NL_SET_ERR_MSG_FMT(extack,
284 					   "dmabuf sg entry (addr=%pad, len=%zu) not aligned to niov size %zu",
285 					   &dma_addr, len, niov_size);
286 			goto err_free_chunks;
287 		}
288 
289 		owner = kzalloc_node(sizeof(*owner), GFP_KERNEL,
290 				     dev_to_node(&dev->dev));
291 		if (!owner) {
292 			err = -ENOMEM;
293 			goto err_free_chunks;
294 		}
295 
296 		owner->area.base_virtual = virtual;
297 		owner->base_dma_addr = dma_addr;
298 		owner->area.num_niovs = len >> niov_shift;
299 		owner->binding = binding;
300 
301 		err = gen_pool_add_owner(binding->chunk_pool, dma_addr,
302 					 dma_addr, len, dev_to_node(&dev->dev),
303 					 owner);
304 		if (err) {
305 			kfree(owner);
306 			err = -EINVAL;
307 			goto err_free_chunks;
308 		}
309 
310 		owner->area.niovs = kvmalloc_objs(*owner->area.niovs,
311 						  owner->area.num_niovs);
312 		if (!owner->area.niovs) {
313 			err = -ENOMEM;
314 			goto err_free_chunks;
315 		}
316 
317 		for (i = 0; i < owner->area.num_niovs; i++) {
318 			niov = &owner->area.niovs[i];
319 			net_iov_init(niov, &owner->area, NET_IOV_DMABUF);
320 			page_pool_set_dma_addr_netmem(net_iov_to_netmem(niov),
321 						      net_devmem_get_dma_addr(niov));
322 			if (direction == DMA_TO_DEVICE)
323 				binding->tx_vec[owner->area.base_virtual / PAGE_SIZE + i] = niov;
324 		}
325 
326 		virtual += len;
327 	}
328 
329 	err = xa_alloc_cyclic(&net_devmem_dmabuf_bindings, &binding->id,
330 			      binding, xa_limit_32b, &id_alloc_next,
331 			      GFP_KERNEL);
332 	if (err < 0)
333 		goto err_free_chunks;
334 
335 	list_add(&binding->list, &priv->bindings);
336 
337 	return binding;
338 
339 err_free_chunks:
340 	gen_pool_for_each_chunk(binding->chunk_pool,
341 				net_devmem_dmabuf_free_chunk_owner, NULL);
342 	gen_pool_destroy(binding->chunk_pool);
343 err_tx_vec:
344 	kvfree(binding->tx_vec);
345 err_unmap:
346 	dma_buf_unmap_attachment_unlocked(binding->attachment, binding->sgt,
347 					  direction);
348 err_detach:
349 	dma_buf_detach(dmabuf, binding->attachment);
350 err_exit_ref:
351 	percpu_ref_exit(&binding->ref);
352 err_free_binding:
353 	kfree(binding);
354 err_put_dmabuf:
355 	dma_buf_put(dmabuf);
356 	return ERR_PTR(err);
357 }
358 
359 struct net_devmem_dmabuf_binding *net_devmem_lookup_dmabuf(u32 id)
360 {
361 	struct net_devmem_dmabuf_binding *binding;
362 
363 	rcu_read_lock();
364 	binding = xa_load(&net_devmem_dmabuf_bindings, id);
365 	if (binding) {
366 		if (!net_devmem_dmabuf_binding_get(binding))
367 			binding = NULL;
368 	}
369 	rcu_read_unlock();
370 
371 	return binding;
372 }
373 
374 void net_devmem_get_net_iov(struct net_iov *niov)
375 {
376 	net_devmem_dmabuf_binding_get(net_devmem_iov_binding(niov));
377 }
378 
379 void net_devmem_put_net_iov(struct net_iov *niov)
380 {
381 	net_devmem_dmabuf_binding_put(net_devmem_iov_binding(niov));
382 }
383 
384 struct net_devmem_dmabuf_binding *net_devmem_get_binding(struct sock *sk,
385 							 unsigned int dmabuf_id)
386 {
387 	struct net_devmem_dmabuf_binding *binding;
388 	struct net_device *dst_dev;
389 	struct dst_entry *dst;
390 	int err = 0;
391 
392 	binding = net_devmem_lookup_dmabuf(dmabuf_id);
393 	if (!binding || !binding->tx_vec) {
394 		err = -EINVAL;
395 		goto out_err;
396 	}
397 
398 	rcu_read_lock();
399 	dst = __sk_dst_get(sk);
400 	/* If dst is NULL (route expired), attempt to rebuild it. */
401 	if (unlikely(!dst)) {
402 		if (inet_csk(sk)->icsk_af_ops->rebuild_header(sk)) {
403 			err = -EHOSTUNREACH;
404 			goto out_unlock;
405 		}
406 		dst = __sk_dst_get(sk);
407 		if (unlikely(!dst)) {
408 			err = -ENODEV;
409 			goto out_unlock;
410 		}
411 	}
412 
413 	/* The dma-addrs in this binding are only reachable to the corresponding
414 	 * net_device.
415 	 */
416 	dst_dev = dst_dev_rcu(dst);
417 	if (unlikely(!dst_dev) ||
418 	    unlikely(dst_dev != READ_ONCE(binding->dev) &&
419 		     dst_dev != READ_ONCE(binding->vdev))) {
420 		err = -ENODEV;
421 		goto out_unlock;
422 	}
423 
424 	rcu_read_unlock();
425 	return binding;
426 
427 out_unlock:
428 	rcu_read_unlock();
429 out_err:
430 	if (binding)
431 		net_devmem_dmabuf_binding_put(binding);
432 
433 	return ERR_PTR(err);
434 }
435 
436 struct net_iov *
437 net_devmem_get_niov_at(struct net_devmem_dmabuf_binding *binding,
438 		       size_t virt_addr, size_t *off, size_t *size)
439 {
440 	if (virt_addr >= binding->dmabuf->size)
441 		return NULL;
442 
443 	*off = virt_addr % PAGE_SIZE;
444 	*size = PAGE_SIZE - *off;
445 
446 	return binding->tx_vec[virt_addr / PAGE_SIZE];
447 }
448 
449 /*** "Dmabuf devmem memory provider" ***/
450 
451 int mp_dmabuf_devmem_init(struct page_pool *pool)
452 {
453 	struct net_devmem_dmabuf_binding *binding = pool->mp_priv;
454 
455 	if (!binding)
456 		return -EINVAL;
457 
458 	/* dma-buf dma addresses do not need and should not be used with
459 	 * dma_sync_for_cpu/device. Force disable dma_sync.
460 	 */
461 	pool->dma_sync = false;
462 	pool->dma_sync_for_cpu = false;
463 
464 	if (pool->p.order != binding->niov_shift - PAGE_SHIFT)
465 		return -E2BIG;
466 
467 	net_devmem_dmabuf_binding_get(binding);
468 	return 0;
469 }
470 
471 netmem_ref mp_dmabuf_devmem_alloc_netmems(struct page_pool *pool, gfp_t gfp)
472 {
473 	struct net_devmem_dmabuf_binding *binding = pool->mp_priv;
474 	struct net_iov *niov;
475 	netmem_ref netmem;
476 
477 	niov = net_devmem_alloc_dmabuf(binding);
478 	if (!niov)
479 		return 0;
480 
481 	netmem = net_iov_to_netmem(niov);
482 
483 	page_pool_set_pp_info(pool, netmem);
484 
485 	pool->pages_state_hold_cnt++;
486 	trace_page_pool_state_hold(pool, netmem, pool->pages_state_hold_cnt);
487 	return netmem;
488 }
489 
490 void mp_dmabuf_devmem_destroy(struct page_pool *pool)
491 {
492 	struct net_devmem_dmabuf_binding *binding = pool->mp_priv;
493 
494 	net_devmem_dmabuf_binding_put(binding);
495 }
496 
497 bool mp_dmabuf_devmem_release_page(struct page_pool *pool, netmem_ref netmem)
498 {
499 	long refcount = atomic_long_read(netmem_get_pp_ref_count_ref(netmem));
500 
501 	if (WARN_ON_ONCE(!netmem_is_net_iov(netmem)))
502 		return false;
503 
504 	if (WARN_ON_ONCE(refcount != 1))
505 		return false;
506 
507 	page_pool_clear_pp_info(netmem);
508 
509 	net_devmem_free_dmabuf(netmem_to_net_iov(netmem));
510 
511 	/* We don't want the page pool put_page()ing our net_iovs. */
512 	return false;
513 }
514 
515 static int mp_dmabuf_devmem_nl_fill(void *mp_priv, struct sk_buff *rsp,
516 				    struct netdev_rx_queue *rxq)
517 {
518 	const struct net_devmem_dmabuf_binding *binding = mp_priv;
519 	int type = rxq ? NETDEV_A_QUEUE_DMABUF : NETDEV_A_PAGE_POOL_DMABUF;
520 
521 	return nla_put_u32(rsp, type, binding->id);
522 }
523 
524 static void mp_dmabuf_devmem_uninstall(void *mp_priv,
525 				       struct netdev_rx_queue *rxq)
526 {
527 	struct net_devmem_dmabuf_binding *binding = mp_priv;
528 	struct netdev_rx_queue *bound_rxq;
529 	unsigned long xa_idx;
530 
531 	xa_for_each(&binding->bound_rxqs, xa_idx, bound_rxq) {
532 		if (bound_rxq == rxq) {
533 			xa_erase(&binding->bound_rxqs, xa_idx);
534 			if (xa_empty(&binding->bound_rxqs)) {
535 				mutex_lock(&binding->lock);
536 				ASSERT_EXCLUSIVE_WRITER(binding->dev);
537 				WRITE_ONCE(binding->dev, NULL);
538 				mutex_unlock(&binding->lock);
539 			}
540 			break;
541 		}
542 	}
543 }
544 
545 static const struct memory_provider_ops dmabuf_devmem_ops = {
546 	.init			= mp_dmabuf_devmem_init,
547 	.destroy		= mp_dmabuf_devmem_destroy,
548 	.alloc_netmems		= mp_dmabuf_devmem_alloc_netmems,
549 	.release_netmem		= mp_dmabuf_devmem_release_page,
550 	.nl_fill		= mp_dmabuf_devmem_nl_fill,
551 	.uninstall		= mp_dmabuf_devmem_uninstall,
552 };
553