1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * Devmem TCP 4 * 5 * Authors: Mina Almasry <almasrymina@google.com> 6 * Willem de Bruijn <willemdebruijn.kernel@gmail.com> 7 * Kaiyuan Zhang <kaiyuanz@google.com 8 */ 9 10 #include <linux/dma-buf.h> 11 #include <linux/mm.h> 12 #include <linux/netdevice.h> 13 #include <linux/types.h> 14 #include <net/netdev_queues.h> 15 #include <net/netdev_rx_queue.h> 16 #include <net/page_pool/helpers.h> 17 #include <net/page_pool/memory_provider.h> 18 #include <net/sock.h> 19 #include <net/tcp.h> 20 21 #include "devmem.h" 22 #include "mp_dmabuf_devmem.h" 23 24 /* Device memory support */ 25 26 static DEFINE_XARRAY_FLAGS(net_devmem_dmabuf_bindings, XA_FLAGS_ALLOC1); 27 28 static const struct memory_provider_ops dmabuf_devmem_ops; 29 30 static struct net_iov *net_iov_free_pop(struct net_iov_area *area, 31 u32 *freelist, u32 *free_count) 32 { 33 u32 idx; 34 35 if (unlikely(!*free_count)) 36 return NULL; 37 38 idx = freelist[--(*free_count)]; 39 return &area->niovs[idx]; 40 } 41 42 static void net_iov_free_push(struct net_iov_area *area, 43 u32 *freelist, u32 *free_count, 44 struct net_iov *niov) 45 { 46 if (WARN_ON_ONCE(net_iov_owner(niov) != area || 47 *free_count >= area->num_niovs)) 48 return; 49 50 freelist[(*free_count)++] = net_iov_idx(niov); 51 } 52 53 static void net_devmem_dmabuf_binding_release(struct percpu_ref *ref) 54 { 55 struct net_devmem_dmabuf_binding *binding = 56 container_of(ref, struct net_devmem_dmabuf_binding, ref); 57 58 INIT_WORK(&binding->unbind_w, __net_devmem_dmabuf_binding_free); 59 schedule_work(&binding->unbind_w); 60 } 61 62 void __net_devmem_dmabuf_binding_free(struct work_struct *wq) 63 { 64 struct net_devmem_dmabuf_binding *binding = container_of(wq, typeof(*binding), unbind_w); 65 66 if (binding->freelist) 67 WARN(binding->free_count != binding->area.num_niovs, 68 "destroying dmabuf binding with outstanding net_iovs: total=%zu, free=%u", 69 binding->area.num_niovs, binding->free_count); 70 71 kvfree(binding->area.niovs); 72 dma_buf_unmap_attachment_unlocked(binding->attachment, binding->sgt, 73 binding->direction); 74 dma_buf_detach(binding->dmabuf, binding->attachment); 75 dma_buf_put(binding->dmabuf); 76 xa_destroy(&binding->bound_rxqs); 77 percpu_ref_exit(&binding->ref); 78 kvfree(binding->freelist); 79 kvfree(binding->tx_vec); 80 kfree(binding); 81 } 82 83 static unsigned int 84 net_devmem_alloc_dmabuf_bulk(struct net_devmem_dmabuf_binding *binding, 85 netmem_ref *netmems, unsigned int count) 86 { 87 struct net_iov *niov; 88 unsigned int i; 89 90 spin_lock_bh(&binding->freelist_lock); 91 92 count = min(count, binding->free_count); 93 for (i = 0; i < count; i++) { 94 niov = net_iov_free_pop(&binding->area, binding->freelist, 95 &binding->free_count); 96 netmems[i] = net_iov_to_netmem(niov); 97 } 98 99 spin_unlock_bh(&binding->freelist_lock); 100 101 return count; 102 } 103 104 void net_devmem_free_dmabuf(struct net_iov *niov) 105 { 106 struct net_devmem_dmabuf_binding *binding = net_devmem_iov_binding(niov); 107 108 spin_lock_bh(&binding->freelist_lock); 109 net_iov_free_push(&binding->area, binding->freelist, 110 &binding->free_count, niov); 111 spin_unlock_bh(&binding->freelist_lock); 112 } 113 114 void net_devmem_unbind_dmabuf(struct net_devmem_dmabuf_binding *binding) 115 { 116 struct netdev_rx_queue *rxq; 117 unsigned long xa_idx; 118 unsigned int rxq_idx; 119 120 xa_erase(&net_devmem_dmabuf_bindings, binding->id); 121 122 /* Ensure no tx net_devmem_lookup_dmabuf() are in flight after the 123 * erase. 124 */ 125 synchronize_net(); 126 127 if (binding->list.next) 128 list_del(&binding->list); 129 130 xa_for_each(&binding->bound_rxqs, xa_idx, rxq) { 131 const struct pp_memory_provider_params mp_params = { 132 .mp_priv = binding, 133 .mp_ops = &dmabuf_devmem_ops, 134 }; 135 136 rxq_idx = get_netdev_rx_queue_index(rxq); 137 138 netif_mp_close_rxq(binding->dev, rxq_idx, &mp_params); 139 } 140 141 percpu_ref_kill(&binding->ref); 142 } 143 144 int net_devmem_bind_dmabuf_to_queue(struct net_device *dev, u32 rxq_idx, 145 struct net_devmem_dmabuf_binding *binding, 146 struct netlink_ext_ack *extack) 147 { 148 struct pp_memory_provider_params mp_params = { 149 .mp_priv = binding, 150 .mp_ops = &dmabuf_devmem_ops, 151 }; 152 struct netdev_rx_queue *rxq; 153 u32 xa_idx; 154 int err; 155 156 if (binding->niov_shift != PAGE_SHIFT) 157 mp_params.rx_page_size = 1U << binding->niov_shift; 158 159 err = netif_mp_open_rxq(dev, rxq_idx, &mp_params, extack); 160 if (err) 161 return err; 162 163 rxq = __netif_get_rx_queue(dev, rxq_idx); 164 err = xa_alloc(&binding->bound_rxqs, &xa_idx, rxq, xa_limit_32b, 165 GFP_KERNEL); 166 if (err) 167 goto err_close_rxq; 168 169 return 0; 170 171 err_close_rxq: 172 netif_mp_close_rxq(dev, rxq_idx, &mp_params); 173 return err; 174 } 175 176 struct net_devmem_dmabuf_binding * 177 net_devmem_bind_dmabuf(struct net_device *dev, void *vdev, 178 struct device *dma_dev, 179 enum dma_data_direction direction, 180 unsigned int dmabuf_fd, unsigned int niov_shift, 181 struct netdev_nl_sock *priv, 182 struct netlink_ext_ack *extack) 183 { 184 struct net_devmem_dmabuf_binding *binding; 185 size_t niov_size = 1UL << niov_shift; 186 static u32 id_alloc_next; 187 struct scatterlist *sg; 188 struct dma_buf *dmabuf; 189 unsigned int sg_idx; 190 size_t niov_idx; 191 size_t i; 192 int err; 193 194 if (!dma_dev) { 195 NL_SET_ERR_MSG(extack, "Device doesn't support DMA"); 196 return ERR_PTR(-EOPNOTSUPP); 197 } 198 199 dmabuf = dma_buf_get(dmabuf_fd); 200 if (IS_ERR(dmabuf)) 201 return ERR_CAST(dmabuf); 202 203 binding = kzalloc_node(sizeof(*binding), GFP_KERNEL, 204 dev_to_node(&dev->dev)); 205 if (!binding) { 206 err = -ENOMEM; 207 goto err_put_dmabuf; 208 } 209 210 binding->dev = dev; 211 binding->vdev = vdev; 212 binding->niov_shift = niov_shift; 213 xa_init_flags(&binding->bound_rxqs, XA_FLAGS_ALLOC); 214 215 err = percpu_ref_init(&binding->ref, 216 net_devmem_dmabuf_binding_release, 217 0, GFP_KERNEL); 218 if (err < 0) 219 goto err_free_binding; 220 221 mutex_init(&binding->lock); 222 223 binding->dmabuf = dmabuf; 224 binding->direction = direction; 225 226 binding->attachment = dma_buf_attach(binding->dmabuf, dma_dev); 227 if (IS_ERR(binding->attachment)) { 228 err = PTR_ERR(binding->attachment); 229 NL_SET_ERR_MSG(extack, "Failed to bind dmabuf to device"); 230 goto err_exit_ref; 231 } 232 233 binding->sgt = dma_buf_map_attachment_unlocked(binding->attachment, 234 direction); 235 if (IS_ERR(binding->sgt)) { 236 err = PTR_ERR(binding->sgt); 237 NL_SET_ERR_MSG(extack, "Failed to map dmabuf attachment"); 238 goto err_detach; 239 } 240 241 if (!IS_ALIGNED(dmabuf->size, niov_size)) { 242 err = -EINVAL; 243 NL_SET_ERR_MSG_FMT(extack, 244 "dmabuf size %zu not aligned to niov size %zu", 245 dmabuf->size, niov_size); 246 goto err_unmap; 247 } 248 249 if ((dmabuf->size >> niov_shift) > U32_MAX) { 250 err = -E2BIG; 251 NL_SET_ERR_MSG(extack, "dmabuf contains too many net_iovs"); 252 goto err_unmap; 253 } 254 255 binding->area.num_niovs = dmabuf->size >> niov_shift; 256 if (direction == DMA_TO_DEVICE) { 257 binding->tx_vec = kvmalloc_objs(struct net_iov *, 258 binding->area.num_niovs); 259 if (!binding->tx_vec) { 260 err = -ENOMEM; 261 goto err_unmap; 262 } 263 } else { 264 spin_lock_init(&binding->freelist_lock); 265 binding->freelist = kvmalloc_array(binding->area.num_niovs, 266 sizeof(binding->freelist[0]), 267 GFP_KERNEL); 268 if (!binding->freelist) { 269 err = -ENOMEM; 270 goto err_unmap; 271 } 272 } 273 binding->area.niovs = kvzalloc_objs(*binding->area.niovs, 274 binding->area.num_niovs); 275 if (!binding->area.niovs) { 276 err = -ENOMEM; 277 goto err_free_freelist; 278 } 279 280 niov_idx = 0; 281 for_each_sgtable_dma_sg(binding->sgt, sg, sg_idx) { 282 dma_addr_t dma_addr = sg_dma_address(sg); 283 size_t len = sg_dma_len(sg); 284 struct net_iov *niov; 285 size_t nr_niovs; 286 287 if (!IS_ALIGNED(dma_addr, niov_size) || 288 !IS_ALIGNED(len, niov_size)) { 289 err = -EINVAL; 290 NL_SET_ERR_MSG_FMT(extack, 291 "dmabuf sg entry (addr=%pad, len=%zu) not aligned to niov size %zu", 292 &dma_addr, len, niov_size); 293 goto err_free_niovs; 294 } 295 296 nr_niovs = len >> niov_shift; 297 for (i = 0; i < nr_niovs; i++, niov_idx++) { 298 niov = &binding->area.niovs[niov_idx]; 299 net_iov_init(niov, &binding->area, NET_IOV_DMABUF); 300 if (net_mp_niov_set_dma_addr(niov, dma_addr)) { 301 err = -EFAULT; 302 NL_SET_ERR_MSG(extack, 303 "dmabuf DMA address cannot be represented"); 304 goto err_free_niovs; 305 } 306 if (direction == DMA_TO_DEVICE) 307 binding->tx_vec[niov_idx] = niov; 308 else 309 net_iov_free_push(&binding->area, 310 binding->freelist, 311 &binding->free_count, niov); 312 dma_addr += niov_size; 313 } 314 } 315 316 err = xa_alloc_cyclic(&net_devmem_dmabuf_bindings, &binding->id, 317 binding, xa_limit_32b, &id_alloc_next, 318 GFP_KERNEL); 319 if (err < 0) 320 goto err_free_niovs; 321 322 list_add(&binding->list, &priv->bindings); 323 324 return binding; 325 326 err_free_niovs: 327 kvfree(binding->area.niovs); 328 err_free_freelist: 329 kvfree(binding->freelist); 330 kvfree(binding->tx_vec); 331 err_unmap: 332 dma_buf_unmap_attachment_unlocked(binding->attachment, binding->sgt, 333 direction); 334 err_detach: 335 dma_buf_detach(dmabuf, binding->attachment); 336 err_exit_ref: 337 percpu_ref_exit(&binding->ref); 338 err_free_binding: 339 kfree(binding); 340 err_put_dmabuf: 341 dma_buf_put(dmabuf); 342 return ERR_PTR(err); 343 } 344 345 struct net_devmem_dmabuf_binding *net_devmem_lookup_dmabuf(u32 id) 346 { 347 struct net_devmem_dmabuf_binding *binding; 348 349 rcu_read_lock(); 350 binding = xa_load(&net_devmem_dmabuf_bindings, id); 351 if (binding) { 352 if (!net_devmem_dmabuf_binding_get(binding)) 353 binding = NULL; 354 } 355 rcu_read_unlock(); 356 357 return binding; 358 } 359 360 void net_devmem_get_net_iov(struct net_iov *niov) 361 { 362 net_devmem_dmabuf_binding_get(net_devmem_iov_binding(niov)); 363 } 364 365 void net_devmem_put_net_iov(struct net_iov *niov) 366 { 367 net_devmem_dmabuf_binding_put(net_devmem_iov_binding(niov)); 368 } 369 370 struct net_devmem_dmabuf_binding *net_devmem_get_binding(struct sock *sk, 371 unsigned int dmabuf_id) 372 { 373 struct net_devmem_dmabuf_binding *binding; 374 struct net_device *dst_dev; 375 struct dst_entry *dst; 376 int err = 0; 377 378 binding = net_devmem_lookup_dmabuf(dmabuf_id); 379 if (!binding || !binding->tx_vec) { 380 err = -EINVAL; 381 goto out_err; 382 } 383 384 rcu_read_lock(); 385 dst = __sk_dst_get(sk); 386 /* If dst is NULL (route expired), attempt to rebuild it. */ 387 if (unlikely(!dst)) { 388 if (inet_csk(sk)->icsk_af_ops->rebuild_header(sk)) { 389 err = -EHOSTUNREACH; 390 goto out_unlock; 391 } 392 dst = __sk_dst_get(sk); 393 if (unlikely(!dst)) { 394 err = -ENODEV; 395 goto out_unlock; 396 } 397 } 398 399 /* The dma-addrs in this binding are only reachable to the corresponding 400 * net_device. 401 */ 402 dst_dev = dst_dev_rcu(dst); 403 if (unlikely(!dst_dev) || 404 unlikely(dst_dev != READ_ONCE(binding->dev) && 405 dst_dev != READ_ONCE(binding->vdev))) { 406 err = -ENODEV; 407 goto out_unlock; 408 } 409 410 rcu_read_unlock(); 411 return binding; 412 413 out_unlock: 414 rcu_read_unlock(); 415 out_err: 416 if (binding) 417 net_devmem_dmabuf_binding_put(binding); 418 419 return ERR_PTR(err); 420 } 421 422 struct net_iov * 423 net_devmem_get_niov_at(struct net_devmem_dmabuf_binding *binding, 424 size_t virt_addr, size_t *off, size_t *size) 425 { 426 if (virt_addr >= binding->dmabuf->size) 427 return NULL; 428 429 *off = virt_addr % PAGE_SIZE; 430 *size = PAGE_SIZE - *off; 431 432 return binding->tx_vec[virt_addr / PAGE_SIZE]; 433 } 434 435 /*** "Dmabuf devmem memory provider" ***/ 436 437 int mp_dmabuf_devmem_init(struct page_pool *pool) 438 { 439 struct net_devmem_dmabuf_binding *binding = pool->mp_priv; 440 441 if (!binding) 442 return -EINVAL; 443 444 /* dma-buf dma addresses do not need and should not be used with 445 * dma_sync_for_cpu/device. Force disable dma_sync. 446 */ 447 pool->dma_sync = false; 448 pool->dma_sync_for_cpu = false; 449 450 if (pool->p.order != binding->niov_shift - PAGE_SHIFT) 451 return -E2BIG; 452 453 net_devmem_dmabuf_binding_get(binding); 454 return 0; 455 } 456 457 netmem_ref mp_dmabuf_devmem_alloc_netmems(struct page_pool *pool, gfp_t gfp) 458 { 459 struct net_devmem_dmabuf_binding *binding = pool->mp_priv; 460 netmem_ref *netmems = pool->alloc.cache; 461 unsigned int allocated, i; 462 463 if (WARN_ON_ONCE(pool->alloc.count)) 464 return 0; 465 466 allocated = net_devmem_alloc_dmabuf_bulk(binding, netmems, 467 PP_ALLOC_CACHE_REFILL); 468 if (unlikely(!allocated)) 469 return 0; 470 471 for (i = 0; i < allocated; i++) 472 net_mp_niov_set_page_pool(pool, 473 netmem_to_net_iov(netmems[i])); 474 475 /* Return the last one, the rest stay in the page_pool cache. */ 476 allocated--; 477 pool->alloc.count = allocated; 478 return netmems[allocated]; 479 } 480 481 void mp_dmabuf_devmem_destroy(struct page_pool *pool) 482 { 483 struct net_devmem_dmabuf_binding *binding = pool->mp_priv; 484 485 net_devmem_dmabuf_binding_put(binding); 486 } 487 488 bool mp_dmabuf_devmem_release_page(struct page_pool *pool, netmem_ref netmem) 489 { 490 long refcount = atomic_long_read(netmem_get_pp_ref_count_ref(netmem)); 491 492 if (WARN_ON_ONCE(!netmem_is_net_iov(netmem))) 493 return false; 494 495 if (WARN_ON_ONCE(refcount != 1)) 496 return false; 497 498 net_mp_niov_clear_page_pool(netmem_to_net_iov(netmem)); 499 net_devmem_free_dmabuf(netmem_to_net_iov(netmem)); 500 501 /* We don't want the page pool put_page()ing our net_iovs. */ 502 return false; 503 } 504 505 static int mp_dmabuf_devmem_nl_fill(void *mp_priv, struct sk_buff *rsp, 506 struct netdev_rx_queue *rxq) 507 { 508 const struct net_devmem_dmabuf_binding *binding = mp_priv; 509 int type = rxq ? NETDEV_A_QUEUE_DMABUF : NETDEV_A_PAGE_POOL_DMABUF; 510 511 return nla_put_u32(rsp, type, binding->id); 512 } 513 514 static void mp_dmabuf_devmem_uninstall(void *mp_priv, 515 struct netdev_rx_queue *rxq) 516 { 517 struct net_devmem_dmabuf_binding *binding = mp_priv; 518 struct netdev_rx_queue *bound_rxq; 519 unsigned long xa_idx; 520 521 xa_for_each(&binding->bound_rxqs, xa_idx, bound_rxq) { 522 if (bound_rxq == rxq) { 523 xa_erase(&binding->bound_rxqs, xa_idx); 524 if (xa_empty(&binding->bound_rxqs)) { 525 mutex_lock(&binding->lock); 526 ASSERT_EXCLUSIVE_WRITER(binding->dev); 527 WRITE_ONCE(binding->dev, NULL); 528 mutex_unlock(&binding->lock); 529 } 530 break; 531 } 532 } 533 } 534 535 static const struct memory_provider_ops dmabuf_devmem_ops = { 536 .init = mp_dmabuf_devmem_init, 537 .destroy = mp_dmabuf_devmem_destroy, 538 .alloc_netmems = mp_dmabuf_devmem_alloc_netmems, 539 .release_netmem = mp_dmabuf_devmem_release_page, 540 .nl_fill = mp_dmabuf_devmem_nl_fill, 541 .uninstall = mp_dmabuf_devmem_uninstall, 542 }; 543