1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * Devmem TCP 4 * 5 * Authors: Mina Almasry <almasrymina@google.com> 6 * Willem de Bruijn <willemdebruijn.kernel@gmail.com> 7 * Kaiyuan Zhang <kaiyuanz@google.com 8 */ 9 10 #include <linux/dma-buf.h> 11 #include <linux/genalloc.h> 12 #include <linux/mm.h> 13 #include <linux/netdevice.h> 14 #include <linux/types.h> 15 #include <net/netdev_queues.h> 16 #include <net/netdev_rx_queue.h> 17 #include <net/page_pool/helpers.h> 18 #include <net/page_pool/memory_provider.h> 19 #include <net/sock.h> 20 #include <net/tcp.h> 21 #include <trace/events/page_pool.h> 22 23 #include "devmem.h" 24 #include "mp_dmabuf_devmem.h" 25 #include "page_pool_priv.h" 26 27 /* Device memory support */ 28 29 static DEFINE_XARRAY_FLAGS(net_devmem_dmabuf_bindings, XA_FLAGS_ALLOC1); 30 31 static const struct memory_provider_ops dmabuf_devmem_ops; 32 33 static void net_devmem_dmabuf_free_chunk_owner(struct gen_pool *genpool, 34 struct gen_pool_chunk *chunk, 35 void *not_used) 36 { 37 struct dmabuf_genpool_chunk_owner *owner = chunk->owner; 38 39 kvfree(owner->area.niovs); 40 kfree(owner); 41 } 42 43 static dma_addr_t net_devmem_get_dma_addr(const struct net_iov *niov) 44 { 45 struct dmabuf_genpool_chunk_owner *owner; 46 47 owner = net_devmem_iov_to_chunk_owner(niov); 48 return owner->base_dma_addr + 49 ((dma_addr_t)net_iov_idx(niov) << PAGE_SHIFT); 50 } 51 52 static void net_devmem_dmabuf_binding_release(struct percpu_ref *ref) 53 { 54 struct net_devmem_dmabuf_binding *binding = 55 container_of(ref, struct net_devmem_dmabuf_binding, ref); 56 57 INIT_WORK(&binding->unbind_w, __net_devmem_dmabuf_binding_free); 58 schedule_work(&binding->unbind_w); 59 } 60 61 void __net_devmem_dmabuf_binding_free(struct work_struct *wq) 62 { 63 struct net_devmem_dmabuf_binding *binding = container_of(wq, typeof(*binding), unbind_w); 64 65 size_t size, avail; 66 67 gen_pool_for_each_chunk(binding->chunk_pool, 68 net_devmem_dmabuf_free_chunk_owner, NULL); 69 70 size = gen_pool_size(binding->chunk_pool); 71 avail = gen_pool_avail(binding->chunk_pool); 72 73 if (!WARN(size != avail, "can't destroy genpool. size=%zu, avail=%zu", 74 size, avail)) 75 gen_pool_destroy(binding->chunk_pool); 76 77 dma_buf_unmap_attachment_unlocked(binding->attachment, binding->sgt, 78 binding->direction); 79 dma_buf_detach(binding->dmabuf, binding->attachment); 80 dma_buf_put(binding->dmabuf); 81 xa_destroy(&binding->bound_rxqs); 82 percpu_ref_exit(&binding->ref); 83 kvfree(binding->tx_vec); 84 kfree(binding); 85 } 86 87 struct net_iov * 88 net_devmem_alloc_dmabuf(struct net_devmem_dmabuf_binding *binding) 89 { 90 struct dmabuf_genpool_chunk_owner *owner; 91 unsigned long dma_addr; 92 struct net_iov *niov; 93 ssize_t offset; 94 ssize_t index; 95 96 dma_addr = gen_pool_alloc_owner(binding->chunk_pool, PAGE_SIZE, 97 (void **)&owner); 98 if (!dma_addr) 99 return NULL; 100 101 offset = dma_addr - owner->base_dma_addr; 102 index = offset / PAGE_SIZE; 103 niov = &owner->area.niovs[index]; 104 105 niov->desc.pp_magic = 0; 106 niov->desc.pp = NULL; 107 atomic_long_set(&niov->desc.pp_ref_count, 0); 108 109 return niov; 110 } 111 112 void net_devmem_free_dmabuf(struct net_iov *niov) 113 { 114 struct net_devmem_dmabuf_binding *binding = net_devmem_iov_binding(niov); 115 unsigned long dma_addr = net_devmem_get_dma_addr(niov); 116 117 if (WARN_ON(!gen_pool_has_addr(binding->chunk_pool, dma_addr, 118 PAGE_SIZE))) 119 return; 120 121 gen_pool_free(binding->chunk_pool, dma_addr, PAGE_SIZE); 122 } 123 124 void net_devmem_unbind_dmabuf(struct net_devmem_dmabuf_binding *binding) 125 { 126 struct netdev_rx_queue *rxq; 127 unsigned long xa_idx; 128 unsigned int rxq_idx; 129 130 xa_erase(&net_devmem_dmabuf_bindings, binding->id); 131 132 /* Ensure no tx net_devmem_lookup_dmabuf() are in flight after the 133 * erase. 134 */ 135 synchronize_net(); 136 137 if (binding->list.next) 138 list_del(&binding->list); 139 140 xa_for_each(&binding->bound_rxqs, xa_idx, rxq) { 141 const struct pp_memory_provider_params mp_params = { 142 .mp_priv = binding, 143 .mp_ops = &dmabuf_devmem_ops, 144 }; 145 146 rxq_idx = get_netdev_rx_queue_index(rxq); 147 148 netif_mp_close_rxq(binding->dev, rxq_idx, &mp_params); 149 } 150 151 percpu_ref_kill(&binding->ref); 152 } 153 154 int net_devmem_bind_dmabuf_to_queue(struct net_device *dev, u32 rxq_idx, 155 struct net_devmem_dmabuf_binding *binding, 156 struct netlink_ext_ack *extack) 157 { 158 struct pp_memory_provider_params mp_params = { 159 .mp_priv = binding, 160 .mp_ops = &dmabuf_devmem_ops, 161 }; 162 struct netdev_rx_queue *rxq; 163 u32 xa_idx; 164 int err; 165 166 err = netif_mp_open_rxq(dev, rxq_idx, &mp_params, extack); 167 if (err) 168 return err; 169 170 rxq = __netif_get_rx_queue(dev, rxq_idx); 171 err = xa_alloc(&binding->bound_rxqs, &xa_idx, rxq, xa_limit_32b, 172 GFP_KERNEL); 173 if (err) 174 goto err_close_rxq; 175 176 return 0; 177 178 err_close_rxq: 179 netif_mp_close_rxq(dev, rxq_idx, &mp_params); 180 return err; 181 } 182 183 struct net_devmem_dmabuf_binding * 184 net_devmem_bind_dmabuf(struct net_device *dev, void *vdev, 185 struct device *dma_dev, 186 enum dma_data_direction direction, 187 unsigned int dmabuf_fd, struct netdev_nl_sock *priv, 188 struct netlink_ext_ack *extack) 189 { 190 struct net_devmem_dmabuf_binding *binding; 191 static u32 id_alloc_next; 192 struct scatterlist *sg; 193 struct dma_buf *dmabuf; 194 unsigned int sg_idx, i; 195 unsigned long virtual; 196 int err; 197 198 if (!dma_dev) { 199 NL_SET_ERR_MSG(extack, "Device doesn't support DMA"); 200 return ERR_PTR(-EOPNOTSUPP); 201 } 202 203 dmabuf = dma_buf_get(dmabuf_fd); 204 if (IS_ERR(dmabuf)) 205 return ERR_CAST(dmabuf); 206 207 binding = kzalloc_node(sizeof(*binding), GFP_KERNEL, 208 dev_to_node(&dev->dev)); 209 if (!binding) { 210 err = -ENOMEM; 211 goto err_put_dmabuf; 212 } 213 214 binding->dev = dev; 215 binding->vdev = vdev; 216 xa_init_flags(&binding->bound_rxqs, XA_FLAGS_ALLOC); 217 218 err = percpu_ref_init(&binding->ref, 219 net_devmem_dmabuf_binding_release, 220 0, GFP_KERNEL); 221 if (err < 0) 222 goto err_free_binding; 223 224 mutex_init(&binding->lock); 225 226 binding->dmabuf = dmabuf; 227 binding->direction = direction; 228 229 binding->attachment = dma_buf_attach(binding->dmabuf, dma_dev); 230 if (IS_ERR(binding->attachment)) { 231 err = PTR_ERR(binding->attachment); 232 NL_SET_ERR_MSG(extack, "Failed to bind dmabuf to device"); 233 goto err_exit_ref; 234 } 235 236 binding->sgt = dma_buf_map_attachment_unlocked(binding->attachment, 237 direction); 238 if (IS_ERR(binding->sgt)) { 239 err = PTR_ERR(binding->sgt); 240 NL_SET_ERR_MSG(extack, "Failed to map dmabuf attachment"); 241 goto err_detach; 242 } 243 244 if (direction == DMA_TO_DEVICE) { 245 if (!IS_ALIGNED(dmabuf->size, PAGE_SIZE)) { 246 err = -EINVAL; 247 NL_SET_ERR_MSG(extack, "TX dma-buf size must be a multiple of PAGE_SIZE"); 248 goto err_unmap; 249 } 250 binding->tx_vec = kvmalloc_objs(struct net_iov *, 251 dmabuf->size / PAGE_SIZE); 252 if (!binding->tx_vec) { 253 err = -ENOMEM; 254 goto err_unmap; 255 } 256 } 257 258 /* For simplicity we expect to make PAGE_SIZE allocations, but the 259 * binding can be much more flexible than that. We may be able to 260 * allocate MTU sized chunks here. Leave that for future work... 261 */ 262 binding->chunk_pool = gen_pool_create(PAGE_SHIFT, 263 dev_to_node(&dev->dev)); 264 if (!binding->chunk_pool) { 265 err = -ENOMEM; 266 goto err_tx_vec; 267 } 268 269 virtual = 0; 270 for_each_sgtable_dma_sg(binding->sgt, sg, sg_idx) { 271 dma_addr_t dma_addr = sg_dma_address(sg); 272 struct dmabuf_genpool_chunk_owner *owner; 273 size_t len = sg_dma_len(sg); 274 struct net_iov *niov; 275 276 if (!IS_ALIGNED(len, PAGE_SIZE)) { 277 err = -EINVAL; 278 NL_SET_ERR_MSG(extack, "dma-buf SG length must be PAGE_SIZE aligned"); 279 goto err_free_chunks; 280 } 281 282 owner = kzalloc_node(sizeof(*owner), GFP_KERNEL, 283 dev_to_node(&dev->dev)); 284 if (!owner) { 285 err = -ENOMEM; 286 goto err_free_chunks; 287 } 288 289 owner->area.base_virtual = virtual; 290 owner->base_dma_addr = dma_addr; 291 owner->area.num_niovs = len / PAGE_SIZE; 292 owner->binding = binding; 293 294 err = gen_pool_add_owner(binding->chunk_pool, dma_addr, 295 dma_addr, len, dev_to_node(&dev->dev), 296 owner); 297 if (err) { 298 kfree(owner); 299 err = -EINVAL; 300 goto err_free_chunks; 301 } 302 303 owner->area.niovs = kvmalloc_objs(*owner->area.niovs, 304 owner->area.num_niovs); 305 if (!owner->area.niovs) { 306 err = -ENOMEM; 307 goto err_free_chunks; 308 } 309 310 for (i = 0; i < owner->area.num_niovs; i++) { 311 niov = &owner->area.niovs[i]; 312 net_iov_init(niov, &owner->area, NET_IOV_DMABUF); 313 page_pool_set_dma_addr_netmem(net_iov_to_netmem(niov), 314 net_devmem_get_dma_addr(niov)); 315 if (direction == DMA_TO_DEVICE) 316 binding->tx_vec[owner->area.base_virtual / PAGE_SIZE + i] = niov; 317 } 318 319 virtual += len; 320 } 321 322 err = xa_alloc_cyclic(&net_devmem_dmabuf_bindings, &binding->id, 323 binding, xa_limit_32b, &id_alloc_next, 324 GFP_KERNEL); 325 if (err < 0) 326 goto err_free_chunks; 327 328 list_add(&binding->list, &priv->bindings); 329 330 return binding; 331 332 err_free_chunks: 333 gen_pool_for_each_chunk(binding->chunk_pool, 334 net_devmem_dmabuf_free_chunk_owner, NULL); 335 gen_pool_destroy(binding->chunk_pool); 336 err_tx_vec: 337 kvfree(binding->tx_vec); 338 err_unmap: 339 dma_buf_unmap_attachment_unlocked(binding->attachment, binding->sgt, 340 direction); 341 err_detach: 342 dma_buf_detach(dmabuf, binding->attachment); 343 err_exit_ref: 344 percpu_ref_exit(&binding->ref); 345 err_free_binding: 346 kfree(binding); 347 err_put_dmabuf: 348 dma_buf_put(dmabuf); 349 return ERR_PTR(err); 350 } 351 352 struct net_devmem_dmabuf_binding *net_devmem_lookup_dmabuf(u32 id) 353 { 354 struct net_devmem_dmabuf_binding *binding; 355 356 rcu_read_lock(); 357 binding = xa_load(&net_devmem_dmabuf_bindings, id); 358 if (binding) { 359 if (!net_devmem_dmabuf_binding_get(binding)) 360 binding = NULL; 361 } 362 rcu_read_unlock(); 363 364 return binding; 365 } 366 367 void net_devmem_get_net_iov(struct net_iov *niov) 368 { 369 net_devmem_dmabuf_binding_get(net_devmem_iov_binding(niov)); 370 } 371 372 void net_devmem_put_net_iov(struct net_iov *niov) 373 { 374 net_devmem_dmabuf_binding_put(net_devmem_iov_binding(niov)); 375 } 376 377 struct net_devmem_dmabuf_binding *net_devmem_get_binding(struct sock *sk, 378 unsigned int dmabuf_id) 379 { 380 struct net_devmem_dmabuf_binding *binding; 381 struct net_device *dst_dev; 382 struct dst_entry *dst; 383 int err = 0; 384 385 binding = net_devmem_lookup_dmabuf(dmabuf_id); 386 if (!binding || !binding->tx_vec) { 387 err = -EINVAL; 388 goto out_err; 389 } 390 391 rcu_read_lock(); 392 dst = __sk_dst_get(sk); 393 /* If dst is NULL (route expired), attempt to rebuild it. */ 394 if (unlikely(!dst)) { 395 if (inet_csk(sk)->icsk_af_ops->rebuild_header(sk)) { 396 err = -EHOSTUNREACH; 397 goto out_unlock; 398 } 399 dst = __sk_dst_get(sk); 400 if (unlikely(!dst)) { 401 err = -ENODEV; 402 goto out_unlock; 403 } 404 } 405 406 /* The dma-addrs in this binding are only reachable to the corresponding 407 * net_device. 408 */ 409 dst_dev = dst_dev_rcu(dst); 410 if (unlikely(!dst_dev) || 411 unlikely(dst_dev != READ_ONCE(binding->dev) && 412 dst_dev != READ_ONCE(binding->vdev))) { 413 err = -ENODEV; 414 goto out_unlock; 415 } 416 417 rcu_read_unlock(); 418 return binding; 419 420 out_unlock: 421 rcu_read_unlock(); 422 out_err: 423 if (binding) 424 net_devmem_dmabuf_binding_put(binding); 425 426 return ERR_PTR(err); 427 } 428 429 struct net_iov * 430 net_devmem_get_niov_at(struct net_devmem_dmabuf_binding *binding, 431 size_t virt_addr, size_t *off, size_t *size) 432 { 433 if (virt_addr >= binding->dmabuf->size) 434 return NULL; 435 436 *off = virt_addr % PAGE_SIZE; 437 *size = PAGE_SIZE - *off; 438 439 return binding->tx_vec[virt_addr / PAGE_SIZE]; 440 } 441 442 /*** "Dmabuf devmem memory provider" ***/ 443 444 int mp_dmabuf_devmem_init(struct page_pool *pool) 445 { 446 struct net_devmem_dmabuf_binding *binding = pool->mp_priv; 447 448 if (!binding) 449 return -EINVAL; 450 451 /* dma-buf dma addresses do not need and should not be used with 452 * dma_sync_for_cpu/device. Force disable dma_sync. 453 */ 454 pool->dma_sync = false; 455 pool->dma_sync_for_cpu = false; 456 457 if (pool->p.order != 0) 458 return -E2BIG; 459 460 net_devmem_dmabuf_binding_get(binding); 461 return 0; 462 } 463 464 netmem_ref mp_dmabuf_devmem_alloc_netmems(struct page_pool *pool, gfp_t gfp) 465 { 466 struct net_devmem_dmabuf_binding *binding = pool->mp_priv; 467 struct net_iov *niov; 468 netmem_ref netmem; 469 470 niov = net_devmem_alloc_dmabuf(binding); 471 if (!niov) 472 return 0; 473 474 netmem = net_iov_to_netmem(niov); 475 476 page_pool_set_pp_info(pool, netmem); 477 478 pool->pages_state_hold_cnt++; 479 trace_page_pool_state_hold(pool, netmem, pool->pages_state_hold_cnt); 480 return netmem; 481 } 482 483 void mp_dmabuf_devmem_destroy(struct page_pool *pool) 484 { 485 struct net_devmem_dmabuf_binding *binding = pool->mp_priv; 486 487 net_devmem_dmabuf_binding_put(binding); 488 } 489 490 bool mp_dmabuf_devmem_release_page(struct page_pool *pool, netmem_ref netmem) 491 { 492 long refcount = atomic_long_read(netmem_get_pp_ref_count_ref(netmem)); 493 494 if (WARN_ON_ONCE(!netmem_is_net_iov(netmem))) 495 return false; 496 497 if (WARN_ON_ONCE(refcount != 1)) 498 return false; 499 500 page_pool_clear_pp_info(netmem); 501 502 net_devmem_free_dmabuf(netmem_to_net_iov(netmem)); 503 504 /* We don't want the page pool put_page()ing our net_iovs. */ 505 return false; 506 } 507 508 static int mp_dmabuf_devmem_nl_fill(void *mp_priv, struct sk_buff *rsp, 509 struct netdev_rx_queue *rxq) 510 { 511 const struct net_devmem_dmabuf_binding *binding = mp_priv; 512 int type = rxq ? NETDEV_A_QUEUE_DMABUF : NETDEV_A_PAGE_POOL_DMABUF; 513 514 return nla_put_u32(rsp, type, binding->id); 515 } 516 517 static void mp_dmabuf_devmem_uninstall(void *mp_priv, 518 struct netdev_rx_queue *rxq) 519 { 520 struct net_devmem_dmabuf_binding *binding = mp_priv; 521 struct netdev_rx_queue *bound_rxq; 522 unsigned long xa_idx; 523 524 xa_for_each(&binding->bound_rxqs, xa_idx, bound_rxq) { 525 if (bound_rxq == rxq) { 526 xa_erase(&binding->bound_rxqs, xa_idx); 527 if (xa_empty(&binding->bound_rxqs)) { 528 mutex_lock(&binding->lock); 529 ASSERT_EXCLUSIVE_WRITER(binding->dev); 530 WRITE_ONCE(binding->dev, NULL); 531 mutex_unlock(&binding->lock); 532 } 533 break; 534 } 535 } 536 } 537 538 static const struct memory_provider_ops dmabuf_devmem_ops = { 539 .init = mp_dmabuf_devmem_init, 540 .destroy = mp_dmabuf_devmem_destroy, 541 .alloc_netmems = mp_dmabuf_devmem_alloc_netmems, 542 .release_netmem = mp_dmabuf_devmem_release_page, 543 .nl_fill = mp_dmabuf_devmem_nl_fill, 544 .uninstall = mp_dmabuf_devmem_uninstall, 545 }; 546