1 // SPDX-License-Identifier: GPL-2.0-or-later 2 /* 3 * Devmem TCP 4 * 5 * Authors: Mina Almasry <almasrymina@google.com> 6 * Willem de Bruijn <willemdebruijn.kernel@gmail.com> 7 * Kaiyuan Zhang <kaiyuanz@google.com 8 */ 9 10 #include <linux/dma-buf.h> 11 #include <linux/mm.h> 12 #include <linux/netdevice.h> 13 #include <linux/types.h> 14 #include <net/netdev_queues.h> 15 #include <net/netdev_rx_queue.h> 16 #include <net/page_pool/helpers.h> 17 #include <net/page_pool/memory_provider.h> 18 #include <net/sock.h> 19 #include <net/tcp.h> 20 #include <trace/events/page_pool.h> 21 22 #include "devmem.h" 23 #include "mp_dmabuf_devmem.h" 24 #include "page_pool_priv.h" 25 26 /* Device memory support */ 27 28 static DEFINE_XARRAY_FLAGS(net_devmem_dmabuf_bindings, XA_FLAGS_ALLOC1); 29 30 static const struct memory_provider_ops dmabuf_devmem_ops; 31 32 static void net_devmem_dmabuf_binding_release(struct percpu_ref *ref) 33 { 34 struct net_devmem_dmabuf_binding *binding = 35 container_of(ref, struct net_devmem_dmabuf_binding, ref); 36 37 INIT_WORK(&binding->unbind_w, __net_devmem_dmabuf_binding_free); 38 schedule_work(&binding->unbind_w); 39 } 40 41 void __net_devmem_dmabuf_binding_free(struct work_struct *wq) 42 { 43 struct net_devmem_dmabuf_binding *binding = container_of(wq, typeof(*binding), unbind_w); 44 45 if (binding->freelist) 46 WARN(binding->free_count != binding->area.num_niovs, 47 "destroying dmabuf binding with outstanding net_iovs: total=%zu, free=%zu", 48 binding->area.num_niovs, binding->free_count); 49 50 kvfree(binding->area.niovs); 51 dma_buf_unmap_attachment_unlocked(binding->attachment, binding->sgt, 52 binding->direction); 53 dma_buf_detach(binding->dmabuf, binding->attachment); 54 dma_buf_put(binding->dmabuf); 55 xa_destroy(&binding->bound_rxqs); 56 percpu_ref_exit(&binding->ref); 57 kvfree(binding->freelist); 58 kvfree(binding->tx_vec); 59 kfree(binding); 60 } 61 62 static unsigned int 63 net_devmem_alloc_dmabuf_bulk(struct net_devmem_dmabuf_binding *binding, 64 netmem_ref *netmems, unsigned int count) 65 { 66 unsigned int i; 67 68 spin_lock_bh(&binding->freelist_lock); 69 70 count = min_t(size_t, count, binding->free_count); 71 for (i = 0; i < count; i++) { 72 struct net_iov *niov = binding->freelist[--binding->free_count]; 73 74 netmems[i] = net_iov_to_netmem(niov); 75 } 76 77 spin_unlock_bh(&binding->freelist_lock); 78 79 return count; 80 } 81 82 void net_devmem_free_dmabuf(struct net_iov *niov) 83 { 84 struct net_devmem_dmabuf_binding *binding = net_devmem_iov_binding(niov); 85 86 spin_lock_bh(&binding->freelist_lock); 87 if (WARN_ON_ONCE(binding->free_count >= binding->area.num_niovs)) { 88 spin_unlock_bh(&binding->freelist_lock); 89 return; 90 } 91 92 binding->freelist[binding->free_count++] = niov; 93 spin_unlock_bh(&binding->freelist_lock); 94 } 95 96 void net_devmem_unbind_dmabuf(struct net_devmem_dmabuf_binding *binding) 97 { 98 struct netdev_rx_queue *rxq; 99 unsigned long xa_idx; 100 unsigned int rxq_idx; 101 102 xa_erase(&net_devmem_dmabuf_bindings, binding->id); 103 104 /* Ensure no tx net_devmem_lookup_dmabuf() are in flight after the 105 * erase. 106 */ 107 synchronize_net(); 108 109 if (binding->list.next) 110 list_del(&binding->list); 111 112 xa_for_each(&binding->bound_rxqs, xa_idx, rxq) { 113 const struct pp_memory_provider_params mp_params = { 114 .mp_priv = binding, 115 .mp_ops = &dmabuf_devmem_ops, 116 }; 117 118 rxq_idx = get_netdev_rx_queue_index(rxq); 119 120 netif_mp_close_rxq(binding->dev, rxq_idx, &mp_params); 121 } 122 123 percpu_ref_kill(&binding->ref); 124 } 125 126 int net_devmem_bind_dmabuf_to_queue(struct net_device *dev, u32 rxq_idx, 127 struct net_devmem_dmabuf_binding *binding, 128 struct netlink_ext_ack *extack) 129 { 130 struct pp_memory_provider_params mp_params = { 131 .mp_priv = binding, 132 .mp_ops = &dmabuf_devmem_ops, 133 }; 134 struct netdev_rx_queue *rxq; 135 u32 xa_idx; 136 int err; 137 138 if (binding->niov_shift != PAGE_SHIFT) 139 mp_params.rx_page_size = 1U << binding->niov_shift; 140 141 err = netif_mp_open_rxq(dev, rxq_idx, &mp_params, extack); 142 if (err) 143 return err; 144 145 rxq = __netif_get_rx_queue(dev, rxq_idx); 146 err = xa_alloc(&binding->bound_rxqs, &xa_idx, rxq, xa_limit_32b, 147 GFP_KERNEL); 148 if (err) 149 goto err_close_rxq; 150 151 return 0; 152 153 err_close_rxq: 154 netif_mp_close_rxq(dev, rxq_idx, &mp_params); 155 return err; 156 } 157 158 struct net_devmem_dmabuf_binding * 159 net_devmem_bind_dmabuf(struct net_device *dev, void *vdev, 160 struct device *dma_dev, 161 enum dma_data_direction direction, 162 unsigned int dmabuf_fd, unsigned int niov_shift, 163 struct netdev_nl_sock *priv, 164 struct netlink_ext_ack *extack) 165 { 166 struct net_devmem_dmabuf_binding *binding; 167 size_t niov_size = 1UL << niov_shift; 168 static u32 id_alloc_next; 169 struct scatterlist *sg; 170 struct dma_buf *dmabuf; 171 unsigned int sg_idx; 172 size_t niov_idx; 173 size_t i; 174 int err; 175 176 if (!dma_dev) { 177 NL_SET_ERR_MSG(extack, "Device doesn't support DMA"); 178 return ERR_PTR(-EOPNOTSUPP); 179 } 180 181 dmabuf = dma_buf_get(dmabuf_fd); 182 if (IS_ERR(dmabuf)) 183 return ERR_CAST(dmabuf); 184 185 binding = kzalloc_node(sizeof(*binding), GFP_KERNEL, 186 dev_to_node(&dev->dev)); 187 if (!binding) { 188 err = -ENOMEM; 189 goto err_put_dmabuf; 190 } 191 192 binding->dev = dev; 193 binding->vdev = vdev; 194 binding->niov_shift = niov_shift; 195 xa_init_flags(&binding->bound_rxqs, XA_FLAGS_ALLOC); 196 197 err = percpu_ref_init(&binding->ref, 198 net_devmem_dmabuf_binding_release, 199 0, GFP_KERNEL); 200 if (err < 0) 201 goto err_free_binding; 202 203 mutex_init(&binding->lock); 204 205 binding->dmabuf = dmabuf; 206 binding->direction = direction; 207 208 binding->attachment = dma_buf_attach(binding->dmabuf, dma_dev); 209 if (IS_ERR(binding->attachment)) { 210 err = PTR_ERR(binding->attachment); 211 NL_SET_ERR_MSG(extack, "Failed to bind dmabuf to device"); 212 goto err_exit_ref; 213 } 214 215 binding->sgt = dma_buf_map_attachment_unlocked(binding->attachment, 216 direction); 217 if (IS_ERR(binding->sgt)) { 218 err = PTR_ERR(binding->sgt); 219 NL_SET_ERR_MSG(extack, "Failed to map dmabuf attachment"); 220 goto err_detach; 221 } 222 223 if (!IS_ALIGNED(dmabuf->size, niov_size)) { 224 err = -EINVAL; 225 NL_SET_ERR_MSG_FMT(extack, 226 "dmabuf size %zu not aligned to niov size %zu", 227 dmabuf->size, niov_size); 228 goto err_unmap; 229 } 230 231 binding->area.base_virtual = 0; 232 binding->area.num_niovs = dmabuf->size >> niov_shift; 233 if (direction == DMA_TO_DEVICE) { 234 binding->tx_vec = kvmalloc_objs(struct net_iov *, 235 binding->area.num_niovs); 236 if (!binding->tx_vec) { 237 err = -ENOMEM; 238 goto err_unmap; 239 } 240 } else { 241 spin_lock_init(&binding->freelist_lock); 242 binding->freelist = kvmalloc_array(binding->area.num_niovs, 243 sizeof(binding->freelist[0]), 244 GFP_KERNEL); 245 if (!binding->freelist) { 246 err = -ENOMEM; 247 goto err_unmap; 248 } 249 } 250 binding->area.niovs = kvmalloc_objs(*binding->area.niovs, 251 binding->area.num_niovs); 252 if (!binding->area.niovs) { 253 err = -ENOMEM; 254 goto err_free_freelist; 255 } 256 257 niov_idx = 0; 258 for_each_sgtable_dma_sg(binding->sgt, sg, sg_idx) { 259 dma_addr_t dma_addr = sg_dma_address(sg); 260 size_t len = sg_dma_len(sg); 261 struct net_iov *niov; 262 size_t nr_niovs; 263 264 if (!IS_ALIGNED(dma_addr, niov_size) || 265 !IS_ALIGNED(len, niov_size)) { 266 err = -EINVAL; 267 NL_SET_ERR_MSG_FMT(extack, 268 "dmabuf sg entry (addr=%pad, len=%zu) not aligned to niov size %zu", 269 &dma_addr, len, niov_size); 270 goto err_free_niovs; 271 } 272 273 nr_niovs = len >> niov_shift; 274 for (i = 0; i < nr_niovs; i++, niov_idx++) { 275 niov = &binding->area.niovs[niov_idx]; 276 net_iov_init(niov, &binding->area, NET_IOV_DMABUF); 277 page_pool_set_dma_addr_netmem(net_iov_to_netmem(niov), 278 dma_addr); 279 if (direction == DMA_TO_DEVICE) 280 binding->tx_vec[niov_idx] = niov; 281 else 282 binding->freelist[binding->free_count++] = niov; 283 dma_addr += niov_size; 284 } 285 } 286 287 err = xa_alloc_cyclic(&net_devmem_dmabuf_bindings, &binding->id, 288 binding, xa_limit_32b, &id_alloc_next, 289 GFP_KERNEL); 290 if (err < 0) 291 goto err_free_niovs; 292 293 list_add(&binding->list, &priv->bindings); 294 295 return binding; 296 297 err_free_niovs: 298 kvfree(binding->area.niovs); 299 err_free_freelist: 300 kvfree(binding->freelist); 301 kvfree(binding->tx_vec); 302 err_unmap: 303 dma_buf_unmap_attachment_unlocked(binding->attachment, binding->sgt, 304 direction); 305 err_detach: 306 dma_buf_detach(dmabuf, binding->attachment); 307 err_exit_ref: 308 percpu_ref_exit(&binding->ref); 309 err_free_binding: 310 kfree(binding); 311 err_put_dmabuf: 312 dma_buf_put(dmabuf); 313 return ERR_PTR(err); 314 } 315 316 struct net_devmem_dmabuf_binding *net_devmem_lookup_dmabuf(u32 id) 317 { 318 struct net_devmem_dmabuf_binding *binding; 319 320 rcu_read_lock(); 321 binding = xa_load(&net_devmem_dmabuf_bindings, id); 322 if (binding) { 323 if (!net_devmem_dmabuf_binding_get(binding)) 324 binding = NULL; 325 } 326 rcu_read_unlock(); 327 328 return binding; 329 } 330 331 void net_devmem_get_net_iov(struct net_iov *niov) 332 { 333 net_devmem_dmabuf_binding_get(net_devmem_iov_binding(niov)); 334 } 335 336 void net_devmem_put_net_iov(struct net_iov *niov) 337 { 338 net_devmem_dmabuf_binding_put(net_devmem_iov_binding(niov)); 339 } 340 341 struct net_devmem_dmabuf_binding *net_devmem_get_binding(struct sock *sk, 342 unsigned int dmabuf_id) 343 { 344 struct net_devmem_dmabuf_binding *binding; 345 struct net_device *dst_dev; 346 struct dst_entry *dst; 347 int err = 0; 348 349 binding = net_devmem_lookup_dmabuf(dmabuf_id); 350 if (!binding || !binding->tx_vec) { 351 err = -EINVAL; 352 goto out_err; 353 } 354 355 rcu_read_lock(); 356 dst = __sk_dst_get(sk); 357 /* If dst is NULL (route expired), attempt to rebuild it. */ 358 if (unlikely(!dst)) { 359 if (inet_csk(sk)->icsk_af_ops->rebuild_header(sk)) { 360 err = -EHOSTUNREACH; 361 goto out_unlock; 362 } 363 dst = __sk_dst_get(sk); 364 if (unlikely(!dst)) { 365 err = -ENODEV; 366 goto out_unlock; 367 } 368 } 369 370 /* The dma-addrs in this binding are only reachable to the corresponding 371 * net_device. 372 */ 373 dst_dev = dst_dev_rcu(dst); 374 if (unlikely(!dst_dev) || 375 unlikely(dst_dev != READ_ONCE(binding->dev) && 376 dst_dev != READ_ONCE(binding->vdev))) { 377 err = -ENODEV; 378 goto out_unlock; 379 } 380 381 rcu_read_unlock(); 382 return binding; 383 384 out_unlock: 385 rcu_read_unlock(); 386 out_err: 387 if (binding) 388 net_devmem_dmabuf_binding_put(binding); 389 390 return ERR_PTR(err); 391 } 392 393 struct net_iov * 394 net_devmem_get_niov_at(struct net_devmem_dmabuf_binding *binding, 395 size_t virt_addr, size_t *off, size_t *size) 396 { 397 if (virt_addr >= binding->dmabuf->size) 398 return NULL; 399 400 *off = virt_addr % PAGE_SIZE; 401 *size = PAGE_SIZE - *off; 402 403 return binding->tx_vec[virt_addr / PAGE_SIZE]; 404 } 405 406 /*** "Dmabuf devmem memory provider" ***/ 407 408 int mp_dmabuf_devmem_init(struct page_pool *pool) 409 { 410 struct net_devmem_dmabuf_binding *binding = pool->mp_priv; 411 412 if (!binding) 413 return -EINVAL; 414 415 /* dma-buf dma addresses do not need and should not be used with 416 * dma_sync_for_cpu/device. Force disable dma_sync. 417 */ 418 pool->dma_sync = false; 419 pool->dma_sync_for_cpu = false; 420 421 if (pool->p.order != binding->niov_shift - PAGE_SHIFT) 422 return -E2BIG; 423 424 net_devmem_dmabuf_binding_get(binding); 425 return 0; 426 } 427 428 netmem_ref mp_dmabuf_devmem_alloc_netmems(struct page_pool *pool, gfp_t gfp) 429 { 430 struct net_devmem_dmabuf_binding *binding = pool->mp_priv; 431 netmem_ref *netmems = pool->alloc.cache; 432 unsigned int allocated, i; 433 434 if (WARN_ON_ONCE(pool->alloc.count)) 435 return 0; 436 437 allocated = net_devmem_alloc_dmabuf_bulk(binding, netmems, 438 PP_ALLOC_CACHE_REFILL); 439 if (unlikely(!allocated)) 440 return 0; 441 442 for (i = 0; i < allocated; i++) { 443 struct net_iov *niov = netmem_to_net_iov(netmems[i]); 444 445 niov->desc.pp_magic = 0; 446 niov->desc.pp = NULL; 447 atomic_long_set(&niov->desc.pp_ref_count, 0); 448 449 page_pool_set_pp_info(pool, netmems[i]); 450 451 pool->pages_state_hold_cnt++; 452 trace_page_pool_state_hold(pool, netmems[i], 453 pool->pages_state_hold_cnt); 454 } 455 456 /* Return the last one, the rest stay in the page_pool cache. */ 457 allocated--; 458 pool->alloc.count = allocated; 459 return netmems[allocated]; 460 } 461 462 void mp_dmabuf_devmem_destroy(struct page_pool *pool) 463 { 464 struct net_devmem_dmabuf_binding *binding = pool->mp_priv; 465 466 net_devmem_dmabuf_binding_put(binding); 467 } 468 469 bool mp_dmabuf_devmem_release_page(struct page_pool *pool, netmem_ref netmem) 470 { 471 long refcount = atomic_long_read(netmem_get_pp_ref_count_ref(netmem)); 472 473 if (WARN_ON_ONCE(!netmem_is_net_iov(netmem))) 474 return false; 475 476 if (WARN_ON_ONCE(refcount != 1)) 477 return false; 478 479 page_pool_clear_pp_info(netmem); 480 481 net_devmem_free_dmabuf(netmem_to_net_iov(netmem)); 482 483 /* We don't want the page pool put_page()ing our net_iovs. */ 484 return false; 485 } 486 487 static int mp_dmabuf_devmem_nl_fill(void *mp_priv, struct sk_buff *rsp, 488 struct netdev_rx_queue *rxq) 489 { 490 const struct net_devmem_dmabuf_binding *binding = mp_priv; 491 int type = rxq ? NETDEV_A_QUEUE_DMABUF : NETDEV_A_PAGE_POOL_DMABUF; 492 493 return nla_put_u32(rsp, type, binding->id); 494 } 495 496 static void mp_dmabuf_devmem_uninstall(void *mp_priv, 497 struct netdev_rx_queue *rxq) 498 { 499 struct net_devmem_dmabuf_binding *binding = mp_priv; 500 struct netdev_rx_queue *bound_rxq; 501 unsigned long xa_idx; 502 503 xa_for_each(&binding->bound_rxqs, xa_idx, bound_rxq) { 504 if (bound_rxq == rxq) { 505 xa_erase(&binding->bound_rxqs, xa_idx); 506 if (xa_empty(&binding->bound_rxqs)) { 507 mutex_lock(&binding->lock); 508 ASSERT_EXCLUSIVE_WRITER(binding->dev); 509 WRITE_ONCE(binding->dev, NULL); 510 mutex_unlock(&binding->lock); 511 } 512 break; 513 } 514 } 515 } 516 517 static const struct memory_provider_ops dmabuf_devmem_ops = { 518 .init = mp_dmabuf_devmem_init, 519 .destroy = mp_dmabuf_devmem_destroy, 520 .alloc_netmems = mp_dmabuf_devmem_alloc_netmems, 521 .release_netmem = mp_dmabuf_devmem_release_page, 522 .nl_fill = mp_dmabuf_devmem_nl_fill, 523 .uninstall = mp_dmabuf_devmem_uninstall, 524 }; 525