1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * Devmem TCP 4 * 5 * Authors: Mina Almasry <almasrymina@google.com> 6 * Willem de Bruijn <willemdebruijn.kernel@gmail.com> 7 * Kaiyuan Zhang <kaiyuanz@google.com 8 */ 9 10 #include <linux/dma-buf.h> 11 #include <linux/dma-resv.h> 12 #include <linux/mm.h> 13 #include <linux/netdevice.h> 14 #include <linux/types.h> 15 #include <net/netdev_queues.h> 16 #include <net/netdev_rx_queue.h> 17 #include <net/page_pool/helpers.h> 18 #include <net/page_pool/memory_provider.h> 19 #include <net/sock.h> 20 #include <net/tcp.h> 21 22 #include "devmem.h" 23 #include "mp_dmabuf_devmem.h" 24 25 /* Device memory support */ 26 27 static DEFINE_XARRAY_FLAGS(net_devmem_dmabuf_bindings, XA_FLAGS_ALLOC1); 28 29 static const struct memory_provider_ops dmabuf_devmem_ops; 30 31 static struct net_iov *net_iov_free_pop(struct net_iov_area *area, 32 u32 *freelist, u32 *free_count) 33 { 34 u32 idx; 35 36 if (unlikely(!*free_count)) 37 return NULL; 38 39 idx = freelist[--(*free_count)]; 40 return &area->niovs[idx]; 41 } 42 43 static void net_iov_free_push(struct net_iov_area *area, 44 u32 *freelist, u32 *free_count, 45 struct net_iov *niov) 46 { 47 if (WARN_ON_ONCE(net_iov_owner(niov) != area || 48 *free_count >= area->num_niovs)) 49 return; 50 51 freelist[(*free_count)++] = net_iov_idx(niov); 52 } 53 54 static const struct dma_buf_attach_ops net_devmem_dmabuf_attach_ops = { 55 .allow_peer2peer = true, 56 }; 57 58 static void net_devmem_dmabuf_binding_release(struct percpu_ref *ref) 59 { 60 struct net_devmem_dmabuf_binding *binding = 61 container_of(ref, struct net_devmem_dmabuf_binding, ref); 62 63 INIT_WORK(&binding->unbind_w, __net_devmem_dmabuf_binding_free); 64 schedule_work(&binding->unbind_w); 65 } 66 67 void __net_devmem_dmabuf_binding_free(struct work_struct *wq) 68 { 69 struct net_devmem_dmabuf_binding *binding = container_of(wq, typeof(*binding), unbind_w); 70 71 if (binding->freelist) 72 WARN(binding->free_count != binding->area.num_niovs, 73 "destroying dmabuf binding with outstanding net_iovs: total=%zu, free=%u", 74 binding->area.num_niovs, binding->free_count); 75 76 kvfree(binding->area.niovs); 77 dma_resv_lock(binding->dmabuf->resv, NULL); 78 dma_buf_unmap_attachment(binding->attachment, binding->sgt, 79 binding->direction); 80 dma_buf_unpin(binding->attachment); 81 dma_resv_unlock(binding->dmabuf->resv); 82 dma_buf_detach(binding->dmabuf, binding->attachment); 83 dma_buf_put(binding->dmabuf); 84 xa_destroy(&binding->bound_rxqs); 85 percpu_ref_exit(&binding->ref); 86 kvfree(binding->freelist); 87 kvfree(binding->tx_vec); 88 kfree(binding); 89 } 90 91 static unsigned int 92 net_devmem_alloc_dmabuf_bulk(struct net_devmem_dmabuf_binding *binding, 93 netmem_ref *netmems, unsigned int count) 94 { 95 struct net_iov *niov; 96 unsigned int i; 97 98 spin_lock_bh(&binding->freelist_lock); 99 100 count = min(count, binding->free_count); 101 for (i = 0; i < count; i++) { 102 niov = net_iov_free_pop(&binding->area, binding->freelist, 103 &binding->free_count); 104 netmems[i] = net_iov_to_netmem(niov); 105 } 106 107 spin_unlock_bh(&binding->freelist_lock); 108 109 return count; 110 } 111 112 void net_devmem_free_dmabuf(struct net_iov *niov) 113 { 114 struct net_devmem_dmabuf_binding *binding = net_devmem_iov_binding(niov); 115 116 spin_lock_bh(&binding->freelist_lock); 117 net_iov_free_push(&binding->area, binding->freelist, 118 &binding->free_count, niov); 119 spin_unlock_bh(&binding->freelist_lock); 120 } 121 122 void net_devmem_unbind_dmabuf(struct net_devmem_dmabuf_binding *binding) 123 { 124 struct netdev_rx_queue *rxq; 125 unsigned long xa_idx; 126 unsigned int rxq_idx; 127 128 xa_erase(&net_devmem_dmabuf_bindings, binding->id); 129 130 /* Ensure no tx net_devmem_lookup_dmabuf() are in flight after the 131 * erase. 132 */ 133 synchronize_net(); 134 135 if (binding->list.next) 136 list_del(&binding->list); 137 138 xa_for_each(&binding->bound_rxqs, xa_idx, rxq) { 139 const struct pp_memory_provider_params mp_params = { 140 .mp_priv = binding, 141 .mp_ops = &dmabuf_devmem_ops, 142 }; 143 144 rxq_idx = get_netdev_rx_queue_index(rxq); 145 146 netif_mp_close_rxq(binding->dev, rxq_idx, &mp_params); 147 } 148 149 percpu_ref_kill(&binding->ref); 150 } 151 152 int net_devmem_bind_dmabuf_to_queue(struct net_device *dev, u32 rxq_idx, 153 struct net_devmem_dmabuf_binding *binding, 154 struct netlink_ext_ack *extack) 155 { 156 struct pp_memory_provider_params mp_params = { 157 .mp_priv = binding, 158 .mp_ops = &dmabuf_devmem_ops, 159 }; 160 struct netdev_rx_queue *rxq; 161 u32 xa_idx; 162 int err; 163 164 if (binding->niov_shift != PAGE_SHIFT) 165 mp_params.rx_page_size = 1U << binding->niov_shift; 166 167 err = netif_mp_open_rxq(dev, rxq_idx, &mp_params, extack); 168 if (err) 169 return err; 170 171 rxq = __netif_get_rx_queue(dev, rxq_idx); 172 err = xa_alloc(&binding->bound_rxqs, &xa_idx, rxq, xa_limit_32b, 173 GFP_KERNEL); 174 if (err) 175 goto err_close_rxq; 176 177 return 0; 178 179 err_close_rxq: 180 netif_mp_close_rxq(dev, rxq_idx, &mp_params); 181 return err; 182 } 183 184 struct net_devmem_dmabuf_binding * 185 net_devmem_bind_dmabuf(struct net_device *dev, void *vdev, 186 struct device *dma_dev, 187 enum dma_data_direction direction, 188 unsigned int dmabuf_fd, unsigned int niov_shift, 189 struct netdev_nl_sock *priv, 190 const struct sock *nl_sk, 191 struct netlink_ext_ack *extack) 192 { 193 struct net_devmem_dmabuf_binding *binding; 194 size_t niov_size = 1UL << niov_shift; 195 static u32 id_alloc_next; 196 struct scatterlist *sg; 197 struct dma_buf *dmabuf; 198 unsigned int sg_idx; 199 size_t niov_idx; 200 size_t i; 201 long err; 202 203 if (!dma_dev) { 204 NL_SET_ERR_MSG(extack, "Device doesn't support DMA"); 205 return ERR_PTR(-EOPNOTSUPP); 206 } 207 208 dmabuf = dma_buf_get(dmabuf_fd); 209 if (IS_ERR(dmabuf)) 210 return ERR_CAST(dmabuf); 211 212 binding = kzalloc_node(sizeof(*binding), GFP_KERNEL, 213 dev_to_node(&dev->dev)); 214 if (!binding) { 215 err = -ENOMEM; 216 goto err_put_dmabuf; 217 } 218 219 binding->dev = dev; 220 binding->vdev = vdev; 221 write_pnet(&binding->net, sock_net(nl_sk)); 222 binding->uid = sk_uid(nl_sk); 223 binding->niov_shift = niov_shift; 224 xa_init_flags(&binding->bound_rxqs, XA_FLAGS_ALLOC); 225 226 err = percpu_ref_init(&binding->ref, 227 net_devmem_dmabuf_binding_release, 228 0, GFP_KERNEL); 229 if (err < 0) 230 goto err_free_binding; 231 232 mutex_init(&binding->lock); 233 234 binding->dmabuf = dmabuf; 235 binding->direction = direction; 236 237 binding->attachment = 238 dma_buf_dynamic_attach(binding->dmabuf, dma_dev, 239 &net_devmem_dmabuf_attach_ops, binding); 240 if (IS_ERR(binding->attachment)) { 241 err = PTR_ERR(binding->attachment); 242 NL_SET_ERR_MSG(extack, "Failed to bind dmabuf to device"); 243 goto err_exit_ref; 244 } 245 246 dma_resv_lock(dmabuf->resv, NULL); 247 err = dma_buf_pin(binding->attachment); 248 if (err) { 249 NL_SET_ERR_MSG(extack, "Failed to pin dmabuf"); 250 goto err_unlock; 251 } 252 253 binding->sgt = dma_buf_map_attachment(binding->attachment, direction); 254 if (IS_ERR(binding->sgt)) { 255 err = PTR_ERR(binding->sgt); 256 NL_SET_ERR_MSG(extack, "Failed to map dmabuf attachment"); 257 goto err_unpin; 258 } 259 dma_resv_unlock(dmabuf->resv); 260 261 err = dma_resv_wait_timeout(dmabuf->resv, DMA_RESV_USAGE_KERNEL, true, 262 MAX_SCHEDULE_TIMEOUT); 263 if (err < 0) { 264 NL_SET_ERR_MSG(extack, "Failed to wait for dmabuf fences"); 265 goto err_unmap; 266 } 267 268 if (!IS_ALIGNED(dmabuf->size, niov_size)) { 269 err = -EINVAL; 270 NL_SET_ERR_MSG_FMT(extack, 271 "dmabuf size %zu not aligned to niov size %zu", 272 dmabuf->size, niov_size); 273 goto err_unmap; 274 } 275 276 if ((dmabuf->size >> niov_shift) > U32_MAX) { 277 err = -E2BIG; 278 NL_SET_ERR_MSG(extack, "dmabuf contains too many net_iovs"); 279 goto err_unmap; 280 } 281 282 binding->area.num_niovs = dmabuf->size >> niov_shift; 283 if (direction == DMA_TO_DEVICE) { 284 binding->tx_vec = kvmalloc_objs(struct net_iov *, 285 binding->area.num_niovs); 286 if (!binding->tx_vec) { 287 err = -ENOMEM; 288 goto err_unmap; 289 } 290 } else { 291 spin_lock_init(&binding->freelist_lock); 292 binding->freelist = kvmalloc_array(binding->area.num_niovs, 293 sizeof(binding->freelist[0]), 294 GFP_KERNEL); 295 if (!binding->freelist) { 296 err = -ENOMEM; 297 goto err_unmap; 298 } 299 } 300 binding->area.niovs = kvzalloc_objs(*binding->area.niovs, 301 binding->area.num_niovs); 302 if (!binding->area.niovs) { 303 err = -ENOMEM; 304 goto err_free_freelist; 305 } 306 307 niov_idx = 0; 308 for_each_sgtable_dma_sg(binding->sgt, sg, sg_idx) { 309 dma_addr_t dma_addr = sg_dma_address(sg); 310 size_t len = sg_dma_len(sg); 311 struct net_iov *niov; 312 size_t nr_niovs; 313 314 if (!IS_ALIGNED(dma_addr, niov_size) || 315 !IS_ALIGNED(len, niov_size)) { 316 err = -EINVAL; 317 NL_SET_ERR_MSG_FMT(extack, 318 "dmabuf sg entry (addr=%pad, len=%zu) not aligned to niov size %zu", 319 &dma_addr, len, niov_size); 320 goto err_free_niovs; 321 } 322 323 nr_niovs = len >> niov_shift; 324 for (i = 0; i < nr_niovs; i++, niov_idx++) { 325 niov = &binding->area.niovs[niov_idx]; 326 net_iov_init(niov, &binding->area, NET_IOV_DMABUF); 327 if (net_mp_niov_set_dma_addr(niov, dma_addr)) { 328 err = -EFAULT; 329 NL_SET_ERR_MSG(extack, 330 "dmabuf DMA address cannot be represented"); 331 goto err_free_niovs; 332 } 333 if (direction == DMA_TO_DEVICE) 334 binding->tx_vec[niov_idx] = niov; 335 else 336 net_iov_free_push(&binding->area, 337 binding->freelist, 338 &binding->free_count, niov); 339 dma_addr += niov_size; 340 } 341 } 342 343 err = xa_alloc_cyclic(&net_devmem_dmabuf_bindings, &binding->id, 344 binding, xa_limit_32b, &id_alloc_next, 345 GFP_KERNEL); 346 if (err < 0) 347 goto err_free_niovs; 348 349 list_add(&binding->list, &priv->bindings); 350 351 return binding; 352 353 err_free_niovs: 354 kvfree(binding->area.niovs); 355 err_free_freelist: 356 kvfree(binding->freelist); 357 kvfree(binding->tx_vec); 358 err_unmap: 359 dma_resv_lock(dmabuf->resv, NULL); 360 dma_buf_unmap_attachment(binding->attachment, binding->sgt, direction); 361 err_unpin: 362 dma_buf_unpin(binding->attachment); 363 err_unlock: 364 dma_resv_unlock(dmabuf->resv); 365 dma_buf_detach(dmabuf, binding->attachment); 366 err_exit_ref: 367 percpu_ref_exit(&binding->ref); 368 err_free_binding: 369 kfree(binding); 370 err_put_dmabuf: 371 dma_buf_put(dmabuf); 372 return ERR_PTR(err); 373 } 374 375 struct net_devmem_dmabuf_binding *net_devmem_lookup_dmabuf(u32 id) 376 { 377 struct net_devmem_dmabuf_binding *binding; 378 379 rcu_read_lock(); 380 binding = xa_load(&net_devmem_dmabuf_bindings, id); 381 if (binding) { 382 if (!net_devmem_dmabuf_binding_get(binding)) 383 binding = NULL; 384 } 385 rcu_read_unlock(); 386 387 return binding; 388 } 389 390 void net_devmem_get_net_iov(struct net_iov *niov) 391 { 392 net_devmem_dmabuf_binding_get(net_devmem_iov_binding(niov)); 393 } 394 395 void net_devmem_put_net_iov(struct net_iov *niov) 396 { 397 net_devmem_dmabuf_binding_put(net_devmem_iov_binding(niov)); 398 } 399 400 struct net_devmem_dmabuf_binding *net_devmem_get_binding(struct sock *sk, 401 unsigned int dmabuf_id) 402 { 403 struct net_devmem_dmabuf_binding *binding; 404 struct net_device *dst_dev; 405 struct dst_entry *dst; 406 int err = 0; 407 408 binding = net_devmem_lookup_dmabuf(dmabuf_id); 409 if (!binding || !binding->tx_vec || 410 !net_eq(sock_net(sk), read_pnet(&binding->net)) || 411 !uid_eq(sk_uid(sk), binding->uid)) { 412 err = -EINVAL; 413 goto out_err; 414 } 415 416 rcu_read_lock(); 417 dst = __sk_dst_get(sk); 418 /* If dst is NULL (route expired), attempt to rebuild it. */ 419 if (unlikely(!dst)) { 420 if (inet_csk(sk)->icsk_af_ops->rebuild_header(sk)) { 421 err = -EHOSTUNREACH; 422 goto out_unlock; 423 } 424 dst = __sk_dst_get(sk); 425 if (unlikely(!dst)) { 426 err = -ENODEV; 427 goto out_unlock; 428 } 429 } 430 431 /* The dma-addrs in this binding are only reachable to the corresponding 432 * net_device. 433 */ 434 dst_dev = dst_dev_rcu(dst); 435 if (unlikely(!dst_dev) || 436 unlikely(dst_dev != READ_ONCE(binding->dev) && 437 dst_dev != READ_ONCE(binding->vdev))) { 438 err = -ENODEV; 439 goto out_unlock; 440 } 441 442 rcu_read_unlock(); 443 return binding; 444 445 out_unlock: 446 rcu_read_unlock(); 447 out_err: 448 if (binding) 449 net_devmem_dmabuf_binding_put(binding); 450 451 return ERR_PTR(err); 452 } 453 454 struct net_iov * 455 net_devmem_get_niov_at(struct net_devmem_dmabuf_binding *binding, 456 size_t virt_addr, size_t *off, size_t *size) 457 { 458 if (virt_addr >= binding->dmabuf->size) 459 return NULL; 460 461 *off = virt_addr % PAGE_SIZE; 462 *size = PAGE_SIZE - *off; 463 464 return binding->tx_vec[virt_addr / PAGE_SIZE]; 465 } 466 467 /*** "Dmabuf devmem memory provider" ***/ 468 469 int mp_dmabuf_devmem_init(struct page_pool *pool) 470 { 471 struct net_devmem_dmabuf_binding *binding = pool->mp_priv; 472 473 if (!binding) 474 return -EINVAL; 475 476 /* dma-buf dma addresses do not need and should not be used with 477 * dma_sync_for_cpu/device. Force disable dma_sync. 478 */ 479 pool->dma_sync = false; 480 pool->dma_sync_for_cpu = false; 481 482 if (pool->p.order != binding->niov_shift - PAGE_SHIFT) 483 return -E2BIG; 484 485 net_devmem_dmabuf_binding_get(binding); 486 return 0; 487 } 488 489 netmem_ref mp_dmabuf_devmem_alloc_netmems(struct page_pool *pool, gfp_t gfp) 490 { 491 struct net_devmem_dmabuf_binding *binding = pool->mp_priv; 492 netmem_ref *netmems = pool->alloc.cache; 493 unsigned int allocated, i; 494 495 if (WARN_ON_ONCE(pool->alloc.count)) 496 return 0; 497 498 allocated = net_devmem_alloc_dmabuf_bulk(binding, netmems, 499 PP_ALLOC_CACHE_REFILL); 500 if (unlikely(!allocated)) 501 return 0; 502 503 for (i = 0; i < allocated; i++) 504 net_mp_niov_set_page_pool(pool, 505 netmem_to_net_iov(netmems[i])); 506 507 /* Return the last one, the rest stay in the page_pool cache. */ 508 allocated--; 509 pool->alloc.count = allocated; 510 return netmems[allocated]; 511 } 512 513 void mp_dmabuf_devmem_destroy(struct page_pool *pool) 514 { 515 struct net_devmem_dmabuf_binding *binding = pool->mp_priv; 516 517 net_devmem_dmabuf_binding_put(binding); 518 } 519 520 bool mp_dmabuf_devmem_release_page(struct page_pool *pool, netmem_ref netmem) 521 { 522 long refcount = atomic_long_read(netmem_get_pp_ref_count_ref(netmem)); 523 524 if (WARN_ON_ONCE(!netmem_is_net_iov(netmem))) 525 return false; 526 527 if (WARN_ON_ONCE(refcount != 1)) 528 return false; 529 530 net_mp_niov_clear_page_pool(netmem_to_net_iov(netmem)); 531 net_devmem_free_dmabuf(netmem_to_net_iov(netmem)); 532 533 /* We don't want the page pool put_page()ing our net_iovs. */ 534 return false; 535 } 536 537 static int mp_dmabuf_devmem_nl_fill(void *mp_priv, struct sk_buff *rsp, 538 struct netdev_rx_queue *rxq) 539 { 540 const struct net_devmem_dmabuf_binding *binding = mp_priv; 541 int type = rxq ? NETDEV_A_QUEUE_DMABUF : NETDEV_A_PAGE_POOL_DMABUF; 542 543 return nla_put_u32(rsp, type, binding->id); 544 } 545 546 static void mp_dmabuf_devmem_uninstall(void *mp_priv, 547 struct netdev_rx_queue *rxq) 548 { 549 struct net_devmem_dmabuf_binding *binding = mp_priv; 550 struct netdev_rx_queue *bound_rxq; 551 unsigned long xa_idx; 552 553 xa_for_each(&binding->bound_rxqs, xa_idx, bound_rxq) { 554 if (bound_rxq == rxq) { 555 xa_erase(&binding->bound_rxqs, xa_idx); 556 if (xa_empty(&binding->bound_rxqs)) { 557 mutex_lock(&binding->lock); 558 ASSERT_EXCLUSIVE_WRITER(binding->dev); 559 WRITE_ONCE(binding->dev, NULL); 560 mutex_unlock(&binding->lock); 561 } 562 break; 563 } 564 } 565 } 566 567 static const struct memory_provider_ops dmabuf_devmem_ops = { 568 .init = mp_dmabuf_devmem_init, 569 .destroy = mp_dmabuf_devmem_destroy, 570 .alloc_netmems = mp_dmabuf_devmem_alloc_netmems, 571 .release_netmem = mp_dmabuf_devmem_release_page, 572 .nl_fill = mp_dmabuf_devmem_nl_fill, 573 .uninstall = mp_dmabuf_devmem_uninstall, 574 }; 575