1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB 2 /* Copyright (c) 2022, NVIDIA CORPORATION & AFFILIATES. */ 3 4 #include <rdma/ib_umem_odp.h> 5 #include <rdma/iter.h> 6 #include "mlx5_ib.h" 7 #include "umr.h" 8 #include "wr.h" 9 10 /* 11 * We can't use an array for xlt_emergency_page because dma_map_single doesn't 12 * work on kernel modules memory 13 */ 14 void *xlt_emergency_page; 15 static DEFINE_MUTEX(xlt_emergency_page_mutex); 16 17 static __be64 get_umr_enable_mr_mask(void) 18 { 19 u64 result; 20 21 result = MLX5_MKEY_MASK_KEY | 22 MLX5_MKEY_MASK_FREE; 23 24 return cpu_to_be64(result); 25 } 26 27 static __be64 get_umr_disable_mr_mask(void) 28 { 29 u64 result; 30 31 result = MLX5_MKEY_MASK_FREE; 32 33 return cpu_to_be64(result); 34 } 35 36 static __be64 get_umr_update_translation_mask(struct mlx5_ib_dev *dev) 37 { 38 u64 result; 39 40 result = MLX5_MKEY_MASK_LEN | 41 MLX5_MKEY_MASK_PAGE_SIZE | 42 MLX5_MKEY_MASK_START_ADDR; 43 if (MLX5_CAP_GEN_2(dev->mdev, umr_log_entity_size_5)) 44 result |= MLX5_MKEY_MASK_PAGE_SIZE_5; 45 46 return cpu_to_be64(result); 47 } 48 49 static __be64 get_umr_update_access_mask(struct mlx5_ib_dev *dev) 50 { 51 u64 result; 52 53 result = MLX5_MKEY_MASK_LR | 54 MLX5_MKEY_MASK_LW | 55 MLX5_MKEY_MASK_RR | 56 MLX5_MKEY_MASK_RW; 57 58 if (MLX5_CAP_GEN(dev->mdev, atomic)) 59 result |= MLX5_MKEY_MASK_A; 60 61 if (MLX5_CAP_GEN(dev->mdev, relaxed_ordering_write_umr)) 62 result |= MLX5_MKEY_MASK_RELAXED_ORDERING_WRITE; 63 64 if (MLX5_CAP_GEN(dev->mdev, relaxed_ordering_read_umr)) 65 result |= MLX5_MKEY_MASK_RELAXED_ORDERING_READ; 66 67 return cpu_to_be64(result); 68 } 69 70 static __be64 get_umr_update_pd_mask(void) 71 { 72 u64 result; 73 74 result = MLX5_MKEY_MASK_PD; 75 76 return cpu_to_be64(result); 77 } 78 79 static int umr_check_mkey_mask(struct mlx5_ib_dev *dev, u64 mask) 80 { 81 if (mask & MLX5_MKEY_MASK_PAGE_SIZE && 82 MLX5_CAP_GEN(dev->mdev, umr_modify_entity_size_disabled)) 83 return -EPERM; 84 85 if (mask & MLX5_MKEY_MASK_A && 86 MLX5_CAP_GEN(dev->mdev, umr_modify_atomic_disabled)) 87 return -EPERM; 88 89 if (mask & MLX5_MKEY_MASK_RELAXED_ORDERING_WRITE && 90 !MLX5_CAP_GEN(dev->mdev, relaxed_ordering_write_umr)) 91 return -EPERM; 92 93 if (mask & MLX5_MKEY_MASK_RELAXED_ORDERING_READ && 94 !MLX5_CAP_GEN(dev->mdev, relaxed_ordering_read_umr)) 95 return -EPERM; 96 97 return 0; 98 } 99 100 enum { 101 MAX_UMR_WR = 128, 102 }; 103 104 static int mlx5r_umr_qp_rst2rts(struct mlx5_ib_dev *dev, struct ib_qp *qp) 105 { 106 struct ib_qp_attr attr = {}; 107 int ret; 108 109 attr.qp_state = IB_QPS_INIT; 110 attr.port_num = 1; 111 ret = ib_modify_qp(qp, &attr, 112 IB_QP_STATE | IB_QP_PKEY_INDEX | IB_QP_PORT); 113 if (ret) { 114 mlx5_ib_dbg(dev, "Couldn't modify UMR QP\n"); 115 return ret; 116 } 117 118 memset(&attr, 0, sizeof(attr)); 119 attr.qp_state = IB_QPS_RTR; 120 121 ret = ib_modify_qp(qp, &attr, IB_QP_STATE); 122 if (ret) { 123 mlx5_ib_dbg(dev, "Couldn't modify umr QP to rtr\n"); 124 return ret; 125 } 126 127 memset(&attr, 0, sizeof(attr)); 128 attr.qp_state = IB_QPS_RTS; 129 ret = ib_modify_qp(qp, &attr, IB_QP_STATE); 130 if (ret) { 131 mlx5_ib_dbg(dev, "Couldn't modify umr QP to rts\n"); 132 return ret; 133 } 134 135 return 0; 136 } 137 138 int mlx5r_umr_resource_init(struct mlx5_ib_dev *dev) 139 { 140 struct ib_qp_init_attr init_attr = {}; 141 struct ib_cq *cq; 142 struct ib_qp *qp; 143 int ret = 0; 144 145 146 /* 147 * UMR qp is set once, never changed until device unload. 148 * Avoid taking the mutex if initialization is already done. 149 */ 150 if (dev->umrc.qp) 151 return 0; 152 153 mutex_lock(&dev->umrc.init_lock); 154 /* First user allocates the UMR resources. Skip if already allocated. */ 155 if (dev->umrc.qp) 156 goto unlock; 157 158 cq = ib_alloc_cq(&dev->ib_dev, NULL, 128, 0, IB_POLL_SOFTIRQ); 159 if (IS_ERR(cq)) { 160 mlx5_ib_dbg(dev, "Couldn't create CQ for sync UMR QP\n"); 161 ret = PTR_ERR(cq); 162 goto unlock; 163 } 164 165 init_attr.send_cq = cq; 166 init_attr.recv_cq = cq; 167 init_attr.sq_sig_type = IB_SIGNAL_ALL_WR; 168 init_attr.cap.max_send_wr = MAX_UMR_WR; 169 init_attr.cap.max_send_sge = 1; 170 init_attr.qp_type = MLX5_IB_QPT_REG_UMR; 171 init_attr.port_num = 1; 172 qp = ib_create_qp(dev->umrc.pd, &init_attr); 173 if (IS_ERR(qp)) { 174 mlx5_ib_dbg(dev, "Couldn't create sync UMR QP\n"); 175 ret = PTR_ERR(qp); 176 goto destroy_cq; 177 } 178 179 ret = mlx5r_umr_qp_rst2rts(dev, qp); 180 if (ret) 181 goto destroy_qp; 182 183 dev->umrc.cq = cq; 184 185 sema_init(&dev->umrc.sem, MAX_UMR_WR); 186 mutex_init(&dev->umrc.lock); 187 dev->umrc.state = MLX5_UMR_STATE_ACTIVE; 188 dev->umrc.qp = qp; 189 190 mutex_unlock(&dev->umrc.init_lock); 191 return 0; 192 193 destroy_qp: 194 ib_destroy_qp(qp); 195 destroy_cq: 196 ib_free_cq(cq); 197 unlock: 198 mutex_unlock(&dev->umrc.init_lock); 199 return ret; 200 } 201 202 void mlx5r_umr_resource_cleanup(struct mlx5_ib_dev *dev) 203 { 204 if (dev->umrc.state == MLX5_UMR_STATE_UNINIT) 205 return; 206 mutex_destroy(&dev->umrc.lock); 207 /* After device init, UMR cp/qp are not unset during the lifetime. */ 208 ib_destroy_qp(dev->umrc.qp); 209 ib_free_cq(dev->umrc.cq); 210 } 211 212 int mlx5r_umr_init(struct mlx5_ib_dev *dev) 213 { 214 struct ib_pd *pd; 215 216 pd = ib_alloc_pd(&dev->ib_dev, 0); 217 if (IS_ERR(pd)) { 218 mlx5_ib_dbg(dev, "Couldn't create PD for sync UMR QP\n"); 219 return PTR_ERR(pd); 220 } 221 dev->umrc.pd = pd; 222 223 mutex_init(&dev->umrc.init_lock); 224 225 return 0; 226 } 227 228 void mlx5r_umr_cleanup(struct mlx5_ib_dev *dev) 229 { 230 if (!dev->umrc.pd) 231 return; 232 233 mutex_destroy(&dev->umrc.init_lock); 234 ib_dealloc_pd(dev->umrc.pd); 235 } 236 237 238 static int mlx5r_umr_post_send(struct ib_qp *ibqp, u32 mkey, struct ib_cqe *cqe, 239 struct mlx5r_umr_wqe *wqe, bool with_data) 240 { 241 unsigned int wqe_size = 242 with_data ? sizeof(struct mlx5r_umr_wqe) : 243 sizeof(struct mlx5r_umr_wqe) - 244 sizeof(struct mlx5_wqe_data_seg); 245 struct mlx5_ib_dev *dev = to_mdev(ibqp->device); 246 struct mlx5_core_dev *mdev = dev->mdev; 247 struct mlx5_ib_qp *qp = to_mqp(ibqp); 248 struct mlx5_wqe_ctrl_seg *ctrl; 249 union { 250 struct ib_cqe *ib_cqe; 251 u64 wr_id; 252 } id; 253 void *cur_edge, *seg; 254 unsigned long flags; 255 unsigned int idx; 256 int size, err; 257 258 if (unlikely(mdev->state == MLX5_DEVICE_STATE_INTERNAL_ERROR)) 259 return -EIO; 260 261 spin_lock_irqsave(&qp->sq.lock, flags); 262 263 err = mlx5r_begin_wqe(qp, &seg, &ctrl, &idx, &size, &cur_edge, 0, 264 cpu_to_be32(mkey), false, false); 265 if (WARN_ON(err)) 266 goto out; 267 268 qp->sq.wr_data[idx] = MLX5_IB_WR_UMR; 269 270 mlx5r_memcpy_send_wqe(&qp->sq, &cur_edge, &seg, &size, wqe, wqe_size); 271 272 id.ib_cqe = cqe; 273 mlx5r_finish_wqe(qp, ctrl, seg, size, cur_edge, idx, id.wr_id, 0, 274 MLX5_FENCE_MODE_INITIATOR_SMALL, MLX5_OPCODE_UMR); 275 276 mlx5r_ring_db(qp, 1, ctrl); 277 278 out: 279 spin_unlock_irqrestore(&qp->sq.lock, flags); 280 281 return err; 282 } 283 284 static int mlx5r_umr_recover(struct mlx5_ib_dev *dev, u32 mkey, 285 struct mlx5r_umr_context *umr_context, 286 struct mlx5r_umr_wqe *wqe, bool with_data) 287 { 288 struct umr_common *umrc = &dev->umrc; 289 struct ib_qp_attr attr; 290 int err; 291 292 mutex_lock(&umrc->lock); 293 /* Preventing any further WRs to be sent now */ 294 if (umrc->state != MLX5_UMR_STATE_RECOVER) { 295 mlx5_ib_warn(dev, "UMR recovery encountered an unexpected state=%d\n", 296 umrc->state); 297 umrc->state = MLX5_UMR_STATE_RECOVER; 298 } 299 mutex_unlock(&umrc->lock); 300 301 /* Sending a final/barrier WR (the failed one) and wait for its completion. 302 * This will ensure that all the previous WRs got a completion before 303 * we set the QP state to RESET. 304 */ 305 err = mlx5r_umr_post_send(umrc->qp, mkey, &umr_context->cqe, wqe, 306 with_data); 307 if (err) { 308 mlx5_ib_warn(dev, "UMR recovery post send failed, err %d\n", err); 309 goto err; 310 } 311 312 /* Since the QP is in an error state, it will only receive 313 * IB_WC_WR_FLUSH_ERR. However, as it serves only as a barrier 314 * we don't care about its status. 315 */ 316 wait_for_completion(&umr_context->done); 317 318 attr.qp_state = IB_QPS_RESET; 319 err = ib_modify_qp(umrc->qp, &attr, IB_QP_STATE); 320 if (err) { 321 mlx5_ib_warn(dev, "Couldn't modify UMR QP to RESET, err=%d\n", err); 322 goto err; 323 } 324 325 err = mlx5r_umr_qp_rst2rts(dev, umrc->qp); 326 if (err) { 327 mlx5_ib_warn(dev, "Couldn't modify UMR QP to RTS, err=%d\n", err); 328 goto err; 329 } 330 331 umrc->state = MLX5_UMR_STATE_ACTIVE; 332 return 0; 333 334 err: 335 umrc->state = MLX5_UMR_STATE_ERR; 336 return err; 337 } 338 339 static void mlx5r_umr_done(struct ib_cq *cq, struct ib_wc *wc) 340 { 341 struct mlx5_ib_umr_context *context = 342 container_of(wc->wr_cqe, struct mlx5_ib_umr_context, cqe); 343 344 context->status = wc->status; 345 complete(&context->done); 346 } 347 348 static inline void mlx5r_umr_init_context(struct mlx5r_umr_context *context) 349 { 350 context->cqe.done = mlx5r_umr_done; 351 init_completion(&context->done); 352 } 353 354 static int mlx5r_umr_post_send_wait(struct mlx5_ib_dev *dev, u32 mkey, 355 struct mlx5r_umr_wqe *wqe, bool with_data) 356 { 357 struct umr_common *umrc = &dev->umrc; 358 struct mlx5r_umr_context umr_context; 359 int err; 360 361 err = umr_check_mkey_mask(dev, be64_to_cpu(wqe->ctrl_seg.mkey_mask)); 362 if (WARN_ON(err)) 363 return err; 364 365 mlx5r_umr_init_context(&umr_context); 366 367 down(&umrc->sem); 368 while (true) { 369 mutex_lock(&umrc->lock); 370 if (umrc->state == MLX5_UMR_STATE_ERR) { 371 mutex_unlock(&umrc->lock); 372 err = -EFAULT; 373 break; 374 } 375 376 if (umrc->state == MLX5_UMR_STATE_RECOVER) { 377 mutex_unlock(&umrc->lock); 378 usleep_range(3000, 5000); 379 continue; 380 } 381 382 err = mlx5r_umr_post_send(umrc->qp, mkey, &umr_context.cqe, wqe, 383 with_data); 384 mutex_unlock(&umrc->lock); 385 if (err) { 386 mlx5_ib_warn(dev, "UMR post send failed, err %d\n", 387 err); 388 break; 389 } 390 391 wait_for_completion(&umr_context.done); 392 393 if (umr_context.status == IB_WC_SUCCESS) 394 break; 395 396 if (umr_context.status == IB_WC_WR_FLUSH_ERR) 397 continue; 398 399 WARN_ON_ONCE(1); 400 mlx5_ib_warn(dev, 401 "reg umr failed (%u). Trying to recover and resubmit the flushed WQEs, mkey = %u\n", 402 umr_context.status, mkey); 403 err = mlx5r_umr_recover(dev, mkey, &umr_context, wqe, with_data); 404 if (err) 405 mlx5_ib_warn(dev, "couldn't recover UMR, err %d\n", 406 err); 407 err = -EFAULT; 408 break; 409 } 410 up(&umrc->sem); 411 return err; 412 } 413 414 /** 415 * mlx5r_umr_revoke_mr - Fence all DMA on the MR 416 * @mr: The MR to fence 417 * 418 * Upon return the NIC will not be doing any DMA to the pages under the MR, 419 * and any DMA in progress will be completed. Failure of this function 420 * indicates the HW has failed catastrophically. 421 */ 422 int mlx5r_umr_revoke_mr(struct mlx5_ib_mr *mr) 423 { 424 struct mlx5_ib_dev *dev = mr_to_mdev(mr); 425 struct mlx5r_umr_wqe wqe = {}; 426 427 if (dev->mdev->state == MLX5_DEVICE_STATE_INTERNAL_ERROR) 428 return 0; 429 430 wqe.ctrl_seg.mkey_mask |= get_umr_update_pd_mask(); 431 wqe.ctrl_seg.mkey_mask |= get_umr_disable_mr_mask(); 432 wqe.ctrl_seg.flags |= MLX5_UMR_INLINE; 433 434 MLX5_SET(mkc, &wqe.mkey_seg, free, 1); 435 MLX5_SET(mkc, &wqe.mkey_seg, pd, to_mpd(dev->umrc.pd)->pdn); 436 MLX5_SET(mkc, &wqe.mkey_seg, qpn, 0xffffff); 437 MLX5_SET(mkc, &wqe.mkey_seg, mkey_7_0, 438 mlx5_mkey_variant(mr->mmkey.key)); 439 440 return mlx5r_umr_post_send_wait(dev, mr->mmkey.key, &wqe, false); 441 } 442 443 static void mlx5r_umr_set_access_flags(struct mlx5_ib_dev *dev, 444 struct mlx5_mkey_seg *seg, 445 unsigned int access_flags) 446 { 447 bool ro_read = (access_flags & IB_ACCESS_RELAXED_ORDERING) && 448 (MLX5_CAP_GEN(dev->mdev, relaxed_ordering_read) || 449 pcie_relaxed_ordering_enabled(dev->mdev->pdev)); 450 451 MLX5_SET(mkc, seg, a, !!(access_flags & IB_ACCESS_REMOTE_ATOMIC)); 452 MLX5_SET(mkc, seg, rw, !!(access_flags & IB_ACCESS_REMOTE_WRITE)); 453 MLX5_SET(mkc, seg, rr, !!(access_flags & IB_ACCESS_REMOTE_READ)); 454 MLX5_SET(mkc, seg, lw, !!(access_flags & IB_ACCESS_LOCAL_WRITE)); 455 MLX5_SET(mkc, seg, lr, 1); 456 MLX5_SET(mkc, seg, relaxed_ordering_write, 457 !!(access_flags & IB_ACCESS_RELAXED_ORDERING)); 458 MLX5_SET(mkc, seg, relaxed_ordering_read, ro_read); 459 } 460 461 int mlx5r_umr_rereg_pd_access(struct mlx5_ib_mr *mr, struct ib_pd *pd, 462 int access_flags) 463 { 464 struct mlx5_ib_dev *dev = mr_to_mdev(mr); 465 struct mlx5r_umr_wqe wqe = {}; 466 int err; 467 468 wqe.ctrl_seg.mkey_mask = get_umr_update_access_mask(dev); 469 wqe.ctrl_seg.mkey_mask |= get_umr_update_pd_mask(); 470 wqe.ctrl_seg.flags = MLX5_UMR_CHECK_FREE; 471 wqe.ctrl_seg.flags |= MLX5_UMR_INLINE; 472 473 mlx5r_umr_set_access_flags(dev, &wqe.mkey_seg, access_flags); 474 MLX5_SET(mkc, &wqe.mkey_seg, pd, to_mpd(pd)->pdn); 475 MLX5_SET(mkc, &wqe.mkey_seg, qpn, 0xffffff); 476 MLX5_SET(mkc, &wqe.mkey_seg, mkey_7_0, 477 mlx5_mkey_variant(mr->mmkey.key)); 478 479 err = mlx5r_umr_post_send_wait(dev, mr->mmkey.key, &wqe, false); 480 if (err) 481 return err; 482 483 mr->access_flags = access_flags; 484 return 0; 485 } 486 487 #define MLX5_MAX_UMR_CHUNK \ 488 ((1 << (MLX5_MAX_UMR_SHIFT + 4)) - MLX5_UMR_FLEX_ALIGNMENT) 489 #define MLX5_SPARE_UMR_CHUNK 0x10000 490 491 /* 492 * Allocate a temporary buffer to hold the per-page information to transfer to 493 * HW. For efficiency this should be as large as it can be, but buffer 494 * allocation failure is not allowed, so try smaller sizes. 495 */ 496 static void *mlx5r_umr_alloc_xlt(size_t *nents, size_t ent_size, gfp_t gfp_mask) 497 { 498 const size_t xlt_chunk_align = MLX5_UMR_FLEX_ALIGNMENT / ent_size; 499 size_t size; 500 void *res = NULL; 501 502 static_assert(PAGE_SIZE % MLX5_UMR_FLEX_ALIGNMENT == 0); 503 504 /* 505 * MLX5_IB_UPD_XLT_ATOMIC doesn't signal an atomic context just that the 506 * allocation can't trigger any kind of reclaim. 507 */ 508 might_sleep(); 509 510 gfp_mask |= __GFP_ZERO | __GFP_NORETRY; 511 512 /* 513 * If the system already has a suitable high order page then just use 514 * that, but don't try hard to create one. This max is about 1M, so a 515 * free x86 huge page will satisfy it. 516 */ 517 size = min_t(size_t, ent_size * ALIGN(*nents, xlt_chunk_align), 518 MLX5_MAX_UMR_CHUNK); 519 *nents = size / ent_size; 520 res = (void *)__get_free_pages(gfp_mask | __GFP_NOWARN, 521 get_order(size)); 522 if (res) 523 return res; 524 525 if (size > MLX5_SPARE_UMR_CHUNK) { 526 size = MLX5_SPARE_UMR_CHUNK; 527 *nents = size / ent_size; 528 res = (void *)__get_free_pages(gfp_mask | __GFP_NOWARN, 529 get_order(size)); 530 if (res) 531 return res; 532 } 533 534 *nents = PAGE_SIZE / ent_size; 535 res = (void *)__get_free_page(gfp_mask); 536 if (res) 537 return res; 538 539 mutex_lock(&xlt_emergency_page_mutex); 540 memset(xlt_emergency_page, 0, PAGE_SIZE); 541 return xlt_emergency_page; 542 } 543 544 static void mlx5r_umr_free_xlt(void *xlt, size_t length) 545 { 546 if (xlt == xlt_emergency_page) { 547 mutex_unlock(&xlt_emergency_page_mutex); 548 return; 549 } 550 551 free_pages((unsigned long)xlt, get_order(length)); 552 } 553 554 static void mlx5r_umr_unmap_free_xlt(struct mlx5_ib_dev *dev, void *xlt, 555 struct ib_sge *sg) 556 { 557 struct device *ddev = &dev->mdev->pdev->dev; 558 559 dma_unmap_single(ddev, sg->addr, sg->length, DMA_TO_DEVICE); 560 mlx5r_umr_free_xlt(xlt, sg->length); 561 } 562 563 /* 564 * Create an XLT buffer ready for submission. 565 */ 566 static void *mlx5r_umr_create_xlt(struct mlx5_ib_dev *dev, struct ib_sge *sg, 567 size_t nents, size_t ent_size, 568 unsigned int flags) 569 { 570 struct device *ddev = &dev->mdev->pdev->dev; 571 dma_addr_t dma; 572 void *xlt; 573 574 xlt = mlx5r_umr_alloc_xlt(&nents, ent_size, 575 flags & MLX5_IB_UPD_XLT_ATOMIC ? GFP_ATOMIC : 576 GFP_KERNEL); 577 sg->length = nents * ent_size; 578 dma = dma_map_single(ddev, xlt, sg->length, DMA_TO_DEVICE); 579 if (dma_mapping_error(ddev, dma)) { 580 mlx5_ib_err(dev, "unable to map DMA during XLT update.\n"); 581 mlx5r_umr_free_xlt(xlt, sg->length); 582 return NULL; 583 } 584 sg->addr = dma; 585 sg->lkey = dev->umrc.pd->local_dma_lkey; 586 587 return xlt; 588 } 589 590 static void 591 mlx5r_umr_set_update_xlt_ctrl_seg(struct mlx5_wqe_umr_ctrl_seg *ctrl_seg, 592 unsigned int flags, struct ib_sge *sg) 593 { 594 if (!(flags & MLX5_IB_UPD_XLT_ENABLE)) 595 /* fail if free */ 596 ctrl_seg->flags = MLX5_UMR_CHECK_FREE; 597 else 598 /* fail if not free */ 599 ctrl_seg->flags = MLX5_UMR_CHECK_NOT_FREE; 600 ctrl_seg->xlt_octowords = 601 cpu_to_be16(mlx5r_umr_get_xlt_octo(sg->length)); 602 } 603 604 static void mlx5r_umr_set_update_xlt_mkey_seg(struct mlx5_ib_dev *dev, 605 struct mlx5_mkey_seg *mkey_seg, 606 struct mlx5_ib_mr *mr, 607 unsigned int page_shift) 608 { 609 mlx5r_umr_set_access_flags(dev, mkey_seg, mr->access_flags); 610 MLX5_SET(mkc, mkey_seg, pd, to_mpd(mr->ibmr.pd)->pdn); 611 MLX5_SET64(mkc, mkey_seg, start_addr, mr->ibmr.iova); 612 MLX5_SET64(mkc, mkey_seg, len, mr->ibmr.length); 613 MLX5_SET(mkc, mkey_seg, log_page_size, page_shift); 614 MLX5_SET(mkc, mkey_seg, qpn, 0xffffff); 615 MLX5_SET(mkc, mkey_seg, mkey_7_0, mlx5_mkey_variant(mr->mmkey.key)); 616 } 617 618 static void 619 mlx5r_umr_set_update_xlt_data_seg(struct mlx5_wqe_data_seg *data_seg, 620 struct ib_sge *sg) 621 { 622 data_seg->byte_count = cpu_to_be32(sg->length); 623 data_seg->lkey = cpu_to_be32(sg->lkey); 624 data_seg->addr = cpu_to_be64(sg->addr); 625 } 626 627 static void mlx5r_umr_update_offset(struct mlx5_wqe_umr_ctrl_seg *ctrl_seg, 628 u64 offset) 629 { 630 u64 octo_offset = mlx5r_umr_get_xlt_octo(offset); 631 632 ctrl_seg->xlt_offset = cpu_to_be16(octo_offset & 0xffff); 633 ctrl_seg->xlt_offset_47_16 = cpu_to_be32(octo_offset >> 16); 634 ctrl_seg->flags |= MLX5_UMR_TRANSLATION_OFFSET_EN; 635 } 636 637 static void mlx5r_umr_final_update_xlt(struct mlx5_ib_dev *dev, 638 struct mlx5r_umr_wqe *wqe, 639 struct mlx5_ib_mr *mr, struct ib_sge *sg, 640 unsigned int flags) 641 { 642 bool update_pd_access, update_translation; 643 644 if (flags & MLX5_IB_UPD_XLT_ENABLE) 645 wqe->ctrl_seg.mkey_mask |= get_umr_enable_mr_mask(); 646 647 update_pd_access = flags & MLX5_IB_UPD_XLT_ENABLE || 648 flags & MLX5_IB_UPD_XLT_PD || 649 flags & MLX5_IB_UPD_XLT_ACCESS; 650 651 if (update_pd_access) { 652 wqe->ctrl_seg.mkey_mask |= get_umr_update_access_mask(dev); 653 wqe->ctrl_seg.mkey_mask |= get_umr_update_pd_mask(); 654 } 655 656 update_translation = 657 flags & MLX5_IB_UPD_XLT_ENABLE || flags & MLX5_IB_UPD_XLT_ADDR; 658 659 if (update_translation) { 660 wqe->ctrl_seg.mkey_mask |= get_umr_update_translation_mask(dev); 661 if (!mr->ibmr.length) 662 MLX5_SET(mkc, &wqe->mkey_seg, length64, 1); 663 if (flags & MLX5_IB_UPD_XLT_KEEP_PGSZ) 664 wqe->ctrl_seg.mkey_mask &= 665 cpu_to_be64(~MLX5_MKEY_MASK_PAGE_SIZE); 666 } 667 668 wqe->ctrl_seg.xlt_octowords = 669 cpu_to_be16(mlx5r_umr_get_xlt_octo(sg->length)); 670 wqe->data_seg.byte_count = cpu_to_be32(sg->length); 671 } 672 673 static void 674 _mlx5r_umr_init_wqe(struct mlx5_ib_mr *mr, struct mlx5r_umr_wqe *wqe, 675 struct ib_sge *sg, unsigned int flags, 676 unsigned int page_shift, bool dd) 677 { 678 struct mlx5_ib_dev *dev = mr_to_mdev(mr); 679 680 mlx5r_umr_set_update_xlt_ctrl_seg(&wqe->ctrl_seg, flags, sg); 681 mlx5r_umr_set_update_xlt_mkey_seg(dev, &wqe->mkey_seg, mr, page_shift); 682 if (dd) /* Use the data direct internal kernel PD */ 683 MLX5_SET(mkc, &wqe->mkey_seg, pd, dev->ddr.pdn); 684 mlx5r_umr_set_update_xlt_data_seg(&wqe->data_seg, sg); 685 } 686 687 static int 688 _mlx5r_umr_update_mr_pas(struct mlx5_ib_mr *mr, unsigned int flags, bool dd, 689 size_t start_block, size_t nblocks) 690 { 691 size_t ent_size = dd ? sizeof(struct mlx5_ksm) : sizeof(struct mlx5_mtt); 692 struct mlx5_ib_dev *dev = mr_to_mdev(mr); 693 struct device *ddev = &dev->mdev->pdev->dev; 694 struct mlx5r_umr_wqe wqe = {}; 695 size_t processed_blocks = 0; 696 struct ib_block_iter biter; 697 size_t cur_block_idx = 0; 698 struct mlx5_ksm *cur_ksm; 699 struct mlx5_mtt *cur_mtt; 700 size_t orig_sg_length; 701 size_t total_blocks; 702 size_t final_size; 703 void *curr_entry; 704 struct ib_sge sg; 705 void *entry; 706 u64 offset; 707 int err = 0; 708 709 total_blocks = ib_umem_num_dma_blocks(mr->umem, 1UL << mr->page_shift); 710 if (start_block > total_blocks) 711 return -EINVAL; 712 713 /* nblocks 0 means update all blocks starting from start_block */ 714 if (nblocks) 715 total_blocks = nblocks; 716 717 entry = mlx5r_umr_create_xlt(dev, &sg, total_blocks, ent_size, flags); 718 if (!entry) 719 return -ENOMEM; 720 721 orig_sg_length = sg.length; 722 723 _mlx5r_umr_init_wqe(mr, &wqe, &sg, flags, mr->page_shift, dd); 724 725 /* Set initial translation offset to start_block */ 726 offset = (u64)start_block * ent_size; 727 mlx5r_umr_update_offset(&wqe.ctrl_seg, offset); 728 729 if (dd) 730 cur_ksm = entry; 731 else 732 cur_mtt = entry; 733 734 curr_entry = entry; 735 736 rdma_umem_for_each_dma_block(mr->umem, &biter, BIT(mr->page_shift)) { 737 if (cur_block_idx < start_block) { 738 cur_block_idx++; 739 continue; 740 } 741 742 if (nblocks && processed_blocks >= nblocks) 743 break; 744 745 if (curr_entry == entry + sg.length) { 746 dma_sync_single_for_device(ddev, sg.addr, sg.length, 747 DMA_TO_DEVICE); 748 749 err = mlx5r_umr_post_send_wait(dev, mr->mmkey.key, &wqe, 750 true); 751 if (err) 752 goto err; 753 dma_sync_single_for_cpu(ddev, sg.addr, sg.length, 754 DMA_TO_DEVICE); 755 offset += sg.length; 756 mlx5r_umr_update_offset(&wqe.ctrl_seg, offset); 757 if (dd) 758 cur_ksm = entry; 759 else 760 cur_mtt = entry; 761 } 762 763 if (dd) { 764 cur_ksm->va = cpu_to_be64(rdma_block_iter_dma_address(&biter)); 765 if (mr->access_flags & IB_ACCESS_RELAXED_ORDERING && 766 dev->ddr.mkey_ro_valid) 767 cur_ksm->key = cpu_to_be32(dev->ddr.mkey_ro); 768 else 769 cur_ksm->key = cpu_to_be32(dev->ddr.mkey); 770 if (mr->umem->is_dmabuf && 771 (flags & MLX5_IB_UPD_XLT_ZAP)) { 772 cur_ksm->va = 0; 773 cur_ksm->key = 0; 774 } 775 cur_ksm++; 776 curr_entry = cur_ksm; 777 } else { 778 cur_mtt->ptag = 779 cpu_to_be64(rdma_block_iter_dma_address(&biter) | 780 MLX5_IB_MTT_PRESENT); 781 if (mr->umem->is_dmabuf && (flags & MLX5_IB_UPD_XLT_ZAP)) 782 cur_mtt->ptag = 0; 783 cur_mtt++; 784 curr_entry = cur_mtt; 785 } 786 787 processed_blocks++; 788 } 789 790 final_size = curr_entry - entry; 791 sg.length = ALIGN(final_size, MLX5_UMR_FLEX_ALIGNMENT); 792 memset(curr_entry, 0, sg.length - final_size); 793 mlx5r_umr_final_update_xlt(dev, &wqe, mr, &sg, flags); 794 795 dma_sync_single_for_device(ddev, sg.addr, sg.length, DMA_TO_DEVICE); 796 err = mlx5r_umr_post_send_wait(dev, mr->mmkey.key, &wqe, true); 797 798 err: 799 sg.length = orig_sg_length; 800 mlx5r_umr_unmap_free_xlt(dev, entry, &sg); 801 return err; 802 } 803 804 int mlx5r_umr_update_data_direct_ksm_pas_range(struct mlx5_ib_mr *mr, 805 unsigned int flags, 806 size_t start_block, 807 size_t nblocks) 808 { 809 /* No invalidation flow is expected */ 810 if (WARN_ON(!mr->umem->is_dmabuf) || ((flags & MLX5_IB_UPD_XLT_ZAP) && 811 !(flags & MLX5_IB_UPD_XLT_KEEP_PGSZ))) 812 return -EINVAL; 813 814 return _mlx5r_umr_update_mr_pas(mr, flags, true, start_block, nblocks); 815 } 816 817 int mlx5r_umr_update_data_direct_ksm_pas(struct mlx5_ib_mr *mr, 818 unsigned int flags) 819 { 820 return mlx5r_umr_update_data_direct_ksm_pas_range(mr, flags, 0, 0); 821 } 822 823 int mlx5r_umr_update_mr_pas_range(struct mlx5_ib_mr *mr, unsigned int flags, 824 size_t start_block, size_t nblocks) 825 { 826 if (WARN_ON(mr->umem->is_odp)) 827 return -EINVAL; 828 829 return _mlx5r_umr_update_mr_pas(mr, flags, false, start_block, nblocks); 830 } 831 832 /* 833 * Send the DMA list to the HW for a normal MR using UMR. 834 * Dmabuf MR is handled in a similar way, except that the MLX5_IB_UPD_XLT_ZAP 835 * flag may be used. 836 */ 837 int mlx5r_umr_update_mr_pas(struct mlx5_ib_mr *mr, unsigned int flags) 838 { 839 return mlx5r_umr_update_mr_pas_range(mr, flags, 0, 0); 840 } 841 842 static bool umr_can_use_indirect_mkey(struct mlx5_ib_dev *dev) 843 { 844 return !MLX5_CAP_GEN(dev->mdev, umr_indirect_mkey_disabled); 845 } 846 847 int mlx5r_umr_update_xlt(struct mlx5_ib_mr *mr, u64 idx, int npages, 848 int page_shift, int flags) 849 { 850 int desc_size = (flags & MLX5_IB_UPD_XLT_INDIRECT) 851 ? sizeof(struct mlx5_klm) 852 : sizeof(struct mlx5_mtt); 853 const int page_align = MLX5_UMR_FLEX_ALIGNMENT / desc_size; 854 struct mlx5_ib_dev *dev = mr_to_mdev(mr); 855 struct device *ddev = &dev->mdev->pdev->dev; 856 const int page_mask = page_align - 1; 857 struct mlx5r_umr_wqe wqe = {}; 858 size_t pages_mapped = 0; 859 size_t pages_to_map = 0; 860 size_t size_to_map = 0; 861 size_t orig_sg_length; 862 size_t pages_iter; 863 struct ib_sge sg; 864 int err = 0; 865 void *xlt; 866 867 if ((flags & MLX5_IB_UPD_XLT_INDIRECT) && 868 !umr_can_use_indirect_mkey(dev)) 869 return -EPERM; 870 871 if (WARN_ON(!mr->umem->is_odp)) 872 return -EINVAL; 873 874 /* UMR copies MTTs in units of MLX5_UMR_FLEX_ALIGNMENT bytes, 875 * so we need to align the offset and length accordingly 876 */ 877 if (idx & page_mask) { 878 npages += idx & page_mask; 879 idx &= ~page_mask; 880 } 881 pages_to_map = ALIGN(npages, page_align); 882 883 xlt = mlx5r_umr_create_xlt(dev, &sg, npages, desc_size, flags); 884 if (!xlt) 885 return -ENOMEM; 886 887 pages_iter = sg.length / desc_size; 888 orig_sg_length = sg.length; 889 890 if (!(flags & MLX5_IB_UPD_XLT_INDIRECT)) { 891 struct ib_umem_odp *odp = to_ib_umem_odp(mr->umem); 892 size_t max_pages = ib_umem_odp_num_pages(odp) - idx; 893 894 pages_to_map = min_t(size_t, pages_to_map, max_pages); 895 } 896 897 mlx5r_umr_set_update_xlt_ctrl_seg(&wqe.ctrl_seg, flags, &sg); 898 mlx5r_umr_set_update_xlt_mkey_seg(dev, &wqe.mkey_seg, mr, page_shift); 899 mlx5r_umr_set_update_xlt_data_seg(&wqe.data_seg, &sg); 900 901 for (pages_mapped = 0; 902 pages_mapped < pages_to_map && !err; 903 pages_mapped += pages_iter, idx += pages_iter) { 904 npages = min_t(int, pages_iter, pages_to_map - pages_mapped); 905 size_to_map = npages * desc_size; 906 dma_sync_single_for_cpu(ddev, sg.addr, sg.length, 907 DMA_TO_DEVICE); 908 /* 909 * npages is the maximum number of pages to map, but we 910 * can't guarantee that all pages are actually mapped. 911 * 912 * For example, if page is p2p of type which is not supported 913 * for mapping, the number of pages mapped will be less than 914 * requested. 915 */ 916 err = mlx5_odp_populate_xlt(xlt, idx, npages, mr, flags); 917 if (err) 918 return err; 919 dma_sync_single_for_device(ddev, sg.addr, sg.length, 920 DMA_TO_DEVICE); 921 sg.length = ALIGN(size_to_map, MLX5_UMR_FLEX_ALIGNMENT); 922 923 if (pages_mapped + pages_iter >= pages_to_map) 924 mlx5r_umr_final_update_xlt(dev, &wqe, mr, &sg, flags); 925 mlx5r_umr_update_offset(&wqe.ctrl_seg, idx * desc_size); 926 err = mlx5r_umr_post_send_wait(dev, mr->mmkey.key, &wqe, true); 927 } 928 sg.length = orig_sg_length; 929 mlx5r_umr_unmap_free_xlt(dev, xlt, &sg); 930 return err; 931 } 932 933 /* 934 * Update only the page-size (log_page_size) field of an existing memory key 935 * using UMR. This is useful when the MR's physical layout stays the same 936 * but the optimal page shift has changed (e.g. dmabuf after pages are 937 * pinned and the HW can switch from 4K to huge-page alignment). 938 */ 939 int mlx5r_umr_update_mr_page_shift(struct mlx5_ib_mr *mr, 940 unsigned int page_shift, 941 bool dd) 942 { 943 struct mlx5_ib_dev *dev = mr_to_mdev(mr); 944 struct mlx5r_umr_wqe wqe = {}; 945 int err; 946 947 /* Build UMR wqe: we touch only PAGE_SIZE, so use the dedicated mask */ 948 wqe.ctrl_seg.mkey_mask = get_umr_update_translation_mask(dev); 949 950 /* MR must be free while page size is modified */ 951 wqe.ctrl_seg.flags = MLX5_UMR_CHECK_FREE | MLX5_UMR_INLINE; 952 953 /* Fill mkey segment with the new page size, keep the rest unchanged */ 954 MLX5_SET(mkc, &wqe.mkey_seg, log_page_size, page_shift); 955 956 if (dd) 957 MLX5_SET(mkc, &wqe.mkey_seg, pd, dev->ddr.pdn); 958 else 959 MLX5_SET(mkc, &wqe.mkey_seg, pd, to_mpd(mr->ibmr.pd)->pdn); 960 961 MLX5_SET64(mkc, &wqe.mkey_seg, start_addr, mr->ibmr.iova); 962 MLX5_SET64(mkc, &wqe.mkey_seg, len, mr->ibmr.length); 963 MLX5_SET(mkc, &wqe.mkey_seg, qpn, 0xffffff); 964 MLX5_SET(mkc, &wqe.mkey_seg, mkey_7_0, 965 mlx5_mkey_variant(mr->mmkey.key)); 966 967 err = mlx5r_umr_post_send_wait(dev, mr->mmkey.key, &wqe, false); 968 if (!err) 969 mr->page_shift = page_shift; 970 971 return err; 972 } 973 974 static inline int 975 _mlx5r_dmabuf_umr_update_pas(struct mlx5_ib_mr *mr, unsigned int flags, 976 size_t start_block, size_t nblocks, bool dd) 977 { 978 if (dd) 979 return mlx5r_umr_update_data_direct_ksm_pas_range(mr, flags, 980 start_block, 981 nblocks); 982 else 983 return mlx5r_umr_update_mr_pas_range(mr, flags, start_block, 984 nblocks); 985 } 986 987 /** 988 * This function makes an mkey non-present by zapping the translation entries of 989 * the mkey by zapping (zeroing out) the first N entries, where N is determined 990 * by the largest page size supported by the device and the MR length. 991 * It then updates the mkey's page size to the largest possible value, ensuring 992 * the MR is completely non-present and safe for further updates. 993 * It is useful to update the page size of a dmabuf MR on a page fault. 994 * 995 * Return: On success, returns the number of entries that were zapped. 996 * On error, returns a negative error code. 997 */ 998 static int _mlx5r_umr_zap_mkey(struct mlx5_ib_mr *mr, 999 unsigned int flags, 1000 unsigned int page_shift, 1001 size_t *nblocks, 1002 bool dd) 1003 { 1004 unsigned int old_page_shift = mr->page_shift; 1005 struct mlx5_ib_dev *dev = mr_to_mdev(mr); 1006 unsigned int max_page_shift; 1007 size_t page_shift_nblocks; 1008 unsigned int max_log_size; 1009 int access_mode; 1010 int err; 1011 1012 access_mode = dd ? MLX5_MKC_ACCESS_MODE_KSM : MLX5_MKC_ACCESS_MODE_MTT; 1013 flags |= MLX5_IB_UPD_XLT_KEEP_PGSZ | MLX5_IB_UPD_XLT_ZAP | 1014 MLX5_IB_UPD_XLT_ATOMIC; 1015 max_log_size = get_max_log_entity_size_cap(dev, access_mode); 1016 max_page_shift = order_base_2(mr->ibmr.length); 1017 max_page_shift = min(max(max_page_shift, page_shift), max_log_size); 1018 /* Count blocks in units of max_page_shift, we will zap exactly this 1019 * many to make the whole MR non-present. 1020 * Block size must be aligned to MLX5_UMR_FLEX_ALIGNMENT since it may 1021 * be used as offset into the XLT later on. 1022 */ 1023 *nblocks = ib_umem_num_dma_blocks(mr->umem, 1UL << max_page_shift); 1024 if (dd) 1025 *nblocks = ALIGN(*nblocks, MLX5_UMR_KSM_NUM_ENTRIES_ALIGNMENT); 1026 else 1027 *nblocks = ALIGN(*nblocks, MLX5_UMR_MTT_NUM_ENTRIES_ALIGNMENT); 1028 page_shift_nblocks = ib_umem_num_dma_blocks(mr->umem, 1029 1UL << page_shift); 1030 /* If the number of blocks at max possible page shift is greater than 1031 * the number of blocks at the new page size, we should just go over the 1032 * whole mkey entries. 1033 */ 1034 if (*nblocks >= page_shift_nblocks) 1035 *nblocks = 0; 1036 1037 /* Make the first nblocks entries non-present without changing 1038 * page size yet. 1039 */ 1040 if (*nblocks) 1041 mr->page_shift = max_page_shift; 1042 err = _mlx5r_dmabuf_umr_update_pas(mr, flags, 0, *nblocks, dd); 1043 if (err) { 1044 mr->page_shift = old_page_shift; 1045 return err; 1046 } 1047 1048 /* Change page size to the max page size now that the MR is completely 1049 * non-present. 1050 */ 1051 if (*nblocks) { 1052 err = mlx5r_umr_update_mr_page_shift(mr, max_page_shift, dd); 1053 if (err) { 1054 mr->page_shift = old_page_shift; 1055 return err; 1056 } 1057 } 1058 1059 return 0; 1060 } 1061 1062 /** 1063 * mlx5r_umr_dmabuf_update_pgsz - Safely update DMABUF MR page size and its 1064 * entries accordingly 1065 * @mr: The memory region to update 1066 * @xlt_flags: Translation table update flags 1067 * @page_shift: The new (optimized) page shift to use 1068 * 1069 * This function updates the page size and mkey translation entries for a DMABUF 1070 * MR in a safe, multi-step process to avoid exposing partially updated mappings 1071 * The update is performed in 5 steps: 1072 * 1. Make the first X entries non-present, while X is calculated to be 1073 * minimal according to a large page shift that can be used to cover the 1074 * MR length. 1075 * 2. Update the page size to the large supported page size 1076 * 3. Load the remaining N-X entries according to the (optimized) page_shift 1077 * 4. Update the page size according to the (optimized) page_shift 1078 * 5. Load the first X entries with the correct translations 1079 * 1080 * This ensures that at no point is the MR accessible with a partially updated 1081 * translation table, maintaining correctness and preventing access to stale or 1082 * inconsistent mappings. 1083 * 1084 * Returns 0 on success or a negative error code on failure. 1085 */ 1086 int mlx5r_umr_dmabuf_update_pgsz(struct mlx5_ib_mr *mr, u32 xlt_flags, 1087 unsigned int page_shift) 1088 { 1089 unsigned int old_page_shift = mr->page_shift; 1090 size_t zapped_blocks; 1091 size_t total_blocks; 1092 int err; 1093 1094 err = _mlx5r_umr_zap_mkey(mr, xlt_flags, page_shift, &zapped_blocks, 1095 mr->data_direct); 1096 if (err) 1097 return err; 1098 1099 /* _mlx5r_umr_zap_mkey already enables the mkey */ 1100 xlt_flags &= ~MLX5_IB_UPD_XLT_ENABLE; 1101 mr->page_shift = page_shift; 1102 total_blocks = ib_umem_num_dma_blocks(mr->umem, 1UL << mr->page_shift); 1103 if (zapped_blocks && zapped_blocks < total_blocks) { 1104 /* Update PAS according to the new page size but don't update 1105 * the page size in the mkey yet. 1106 */ 1107 err = _mlx5r_dmabuf_umr_update_pas( 1108 mr, 1109 xlt_flags | MLX5_IB_UPD_XLT_KEEP_PGSZ, 1110 zapped_blocks, 1111 total_blocks - zapped_blocks, 1112 mr->data_direct); 1113 if (err) 1114 goto err; 1115 } 1116 1117 err = mlx5r_umr_update_mr_page_shift(mr, mr->page_shift, 1118 mr->data_direct); 1119 if (err) 1120 goto err; 1121 err = _mlx5r_dmabuf_umr_update_pas(mr, xlt_flags, 0, zapped_blocks, 1122 mr->data_direct); 1123 if (err) 1124 goto err; 1125 1126 return 0; 1127 err: 1128 mr->page_shift = old_page_shift; 1129 return err; 1130 } 1131