1 /* 2 * Copyright (c) 2013-2015, Mellanox Technologies. All rights reserved. 3 * 4 * This software is available to you under a choice of one of two 5 * licenses. You may choose to be licensed under the terms of the GNU 6 * General Public License (GPL) Version 2, available from the file 7 * COPYING in the main directory of this source tree, or the 8 * OpenIB.org BSD license below: 9 * 10 * Redistribution and use in source and binary forms, with or 11 * without modification, are permitted provided that the following 12 * conditions are met: 13 * 14 * - Redistributions of source code must retain the above 15 * copyright notice, this list of conditions and the following 16 * disclaimer. 17 * 18 * - Redistributions in binary form must reproduce the above 19 * copyright notice, this list of conditions and the following 20 * disclaimer in the documentation and/or other materials 21 * provided with the distribution. 22 * 23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30 * SOFTWARE. 31 */ 32 33 #include <rdma/ib_umem.h> 34 #include <rdma/ib_umem_odp.h> 35 #include <linux/kernel.h> 36 37 #include "mlx5_ib.h" 38 #include "cmd.h" 39 40 #include <linux/mlx5/eq.h> 41 42 /* Contains the details of a pagefault. */ 43 struct mlx5_pagefault { 44 u32 bytes_committed; 45 u32 token; 46 u8 event_subtype; 47 u8 type; 48 union { 49 /* Initiator or send message responder pagefault details. */ 50 struct { 51 /* Received packet size, only valid for responders. */ 52 u32 packet_size; 53 /* 54 * Number of resource holding WQE, depends on type. 55 */ 56 u32 wq_num; 57 /* 58 * WQE index. Refers to either the send queue or 59 * receive queue, according to event_subtype. 60 */ 61 u16 wqe_index; 62 } wqe; 63 /* RDMA responder pagefault details */ 64 struct { 65 u32 r_key; 66 /* 67 * Received packet size, minimal size page fault 68 * resolution required for forward progress. 69 */ 70 u32 packet_size; 71 u32 rdma_op_len; 72 u64 rdma_va; 73 } rdma; 74 }; 75 76 struct mlx5_ib_pf_eq *eq; 77 struct work_struct work; 78 }; 79 80 #define MAX_PREFETCH_LEN (4*1024*1024U) 81 82 /* Timeout in ms to wait for an active mmu notifier to complete when handling 83 * a pagefault. */ 84 #define MMU_NOTIFIER_TIMEOUT 1000 85 86 #define MLX5_IMR_MTT_BITS (30 - PAGE_SHIFT) 87 #define MLX5_IMR_MTT_SHIFT (MLX5_IMR_MTT_BITS + PAGE_SHIFT) 88 #define MLX5_IMR_MTT_ENTRIES BIT_ULL(MLX5_IMR_MTT_BITS) 89 #define MLX5_IMR_MTT_SIZE BIT_ULL(MLX5_IMR_MTT_SHIFT) 90 #define MLX5_IMR_MTT_MASK (~(MLX5_IMR_MTT_SIZE - 1)) 91 92 #define MLX5_KSM_PAGE_SHIFT MLX5_IMR_MTT_SHIFT 93 94 static u64 mlx5_imr_ksm_entries; 95 96 static int check_parent(struct ib_umem_odp *odp, 97 struct mlx5_ib_mr *parent) 98 { 99 struct mlx5_ib_mr *mr = odp->private; 100 101 return mr && mr->parent == parent && !odp->dying; 102 } 103 104 static struct ib_ucontext_per_mm *mr_to_per_mm(struct mlx5_ib_mr *mr) 105 { 106 if (WARN_ON(!mr || !is_odp_mr(mr))) 107 return NULL; 108 109 return to_ib_umem_odp(mr->umem)->per_mm; 110 } 111 112 static struct ib_umem_odp *odp_next(struct ib_umem_odp *odp) 113 { 114 struct mlx5_ib_mr *mr = odp->private, *parent = mr->parent; 115 struct ib_ucontext_per_mm *per_mm = odp->per_mm; 116 struct rb_node *rb; 117 118 down_read(&per_mm->umem_rwsem); 119 while (1) { 120 rb = rb_next(&odp->interval_tree.rb); 121 if (!rb) 122 goto not_found; 123 odp = rb_entry(rb, struct ib_umem_odp, interval_tree.rb); 124 if (check_parent(odp, parent)) 125 goto end; 126 } 127 not_found: 128 odp = NULL; 129 end: 130 up_read(&per_mm->umem_rwsem); 131 return odp; 132 } 133 134 static struct ib_umem_odp *odp_lookup(u64 start, u64 length, 135 struct mlx5_ib_mr *parent) 136 { 137 struct ib_ucontext_per_mm *per_mm = mr_to_per_mm(parent); 138 struct ib_umem_odp *odp; 139 struct rb_node *rb; 140 141 down_read(&per_mm->umem_rwsem); 142 odp = rbt_ib_umem_lookup(&per_mm->umem_tree, start, length); 143 if (!odp) 144 goto end; 145 146 while (1) { 147 if (check_parent(odp, parent)) 148 goto end; 149 rb = rb_next(&odp->interval_tree.rb); 150 if (!rb) 151 goto not_found; 152 odp = rb_entry(rb, struct ib_umem_odp, interval_tree.rb); 153 if (ib_umem_start(odp) > start + length) 154 goto not_found; 155 } 156 not_found: 157 odp = NULL; 158 end: 159 up_read(&per_mm->umem_rwsem); 160 return odp; 161 } 162 163 void mlx5_odp_populate_klm(struct mlx5_klm *pklm, size_t offset, 164 size_t nentries, struct mlx5_ib_mr *mr, int flags) 165 { 166 struct ib_pd *pd = mr->ibmr.pd; 167 struct mlx5_ib_dev *dev = to_mdev(pd->device); 168 struct ib_umem_odp *odp; 169 unsigned long va; 170 int i; 171 172 if (flags & MLX5_IB_UPD_XLT_ZAP) { 173 for (i = 0; i < nentries; i++, pklm++) { 174 pklm->bcount = cpu_to_be32(MLX5_IMR_MTT_SIZE); 175 pklm->key = cpu_to_be32(dev->null_mkey); 176 pklm->va = 0; 177 } 178 return; 179 } 180 181 /* 182 * The locking here is pretty subtle. Ideally the implicit children 183 * list would be protected by the umem_mutex, however that is not 184 * possible. Instead this uses a weaker update-then-lock pattern: 185 * 186 * srcu_read_lock() 187 * <change children list> 188 * mutex_lock(umem_mutex) 189 * mlx5_ib_update_xlt() 190 * mutex_unlock(umem_mutex) 191 * destroy lkey 192 * 193 * ie any change the children list must be followed by the locked 194 * update_xlt before destroying. 195 * 196 * The umem_mutex provides the acquire/release semantic needed to make 197 * the children list visible to a racing thread. While SRCU is not 198 * technically required, using it gives consistent use of the SRCU 199 * locking around the children list. 200 */ 201 lockdep_assert_held(&to_ib_umem_odp(mr->umem)->umem_mutex); 202 lockdep_assert_held(&mr->dev->mr_srcu); 203 204 odp = odp_lookup(offset * MLX5_IMR_MTT_SIZE, 205 nentries * MLX5_IMR_MTT_SIZE, mr); 206 207 for (i = 0; i < nentries; i++, pklm++) { 208 pklm->bcount = cpu_to_be32(MLX5_IMR_MTT_SIZE); 209 va = (offset + i) * MLX5_IMR_MTT_SIZE; 210 if (odp && ib_umem_start(odp) == va) { 211 struct mlx5_ib_mr *mtt = odp->private; 212 213 pklm->key = cpu_to_be32(mtt->ibmr.lkey); 214 odp = odp_next(odp); 215 } else { 216 pklm->key = cpu_to_be32(dev->null_mkey); 217 } 218 mlx5_ib_dbg(dev, "[%d] va %lx key %x\n", 219 i, va, be32_to_cpu(pklm->key)); 220 } 221 } 222 223 static void mr_leaf_free_action(struct work_struct *work) 224 { 225 struct ib_umem_odp *odp = container_of(work, struct ib_umem_odp, work); 226 int idx = ib_umem_start(odp) >> MLX5_IMR_MTT_SHIFT; 227 struct mlx5_ib_mr *mr = odp->private, *imr = mr->parent; 228 struct ib_umem_odp *odp_imr = to_ib_umem_odp(imr->umem); 229 int srcu_key; 230 231 mr->parent = NULL; 232 synchronize_srcu(&mr->dev->mr_srcu); 233 234 if (smp_load_acquire(&imr->live)) { 235 srcu_key = srcu_read_lock(&mr->dev->mr_srcu); 236 mutex_lock(&odp_imr->umem_mutex); 237 mlx5_ib_update_xlt(imr, idx, 1, 0, 238 MLX5_IB_UPD_XLT_INDIRECT | 239 MLX5_IB_UPD_XLT_ATOMIC); 240 mutex_unlock(&odp_imr->umem_mutex); 241 srcu_read_unlock(&mr->dev->mr_srcu, srcu_key); 242 } 243 ib_umem_odp_release(odp); 244 mlx5_mr_cache_free(mr->dev, mr); 245 246 if (atomic_dec_and_test(&imr->num_leaf_free)) 247 wake_up(&imr->q_leaf_free); 248 } 249 250 void mlx5_ib_invalidate_range(struct ib_umem_odp *umem_odp, unsigned long start, 251 unsigned long end) 252 { 253 struct mlx5_ib_mr *mr; 254 const u64 umr_block_mask = (MLX5_UMR_MTT_ALIGNMENT / 255 sizeof(struct mlx5_mtt)) - 1; 256 u64 idx = 0, blk_start_idx = 0; 257 int in_block = 0; 258 u64 addr; 259 260 if (!umem_odp) { 261 pr_err("invalidation called on NULL umem or non-ODP umem\n"); 262 return; 263 } 264 265 mr = umem_odp->private; 266 267 if (!mr || !mr->ibmr.pd) 268 return; 269 270 start = max_t(u64, ib_umem_start(umem_odp), start); 271 end = min_t(u64, ib_umem_end(umem_odp), end); 272 273 /* 274 * Iteration one - zap the HW's MTTs. The notifiers_count ensures that 275 * while we are doing the invalidation, no page fault will attempt to 276 * overwrite the same MTTs. Concurent invalidations might race us, 277 * but they will write 0s as well, so no difference in the end result. 278 */ 279 mutex_lock(&umem_odp->umem_mutex); 280 for (addr = start; addr < end; addr += BIT(umem_odp->page_shift)) { 281 idx = (addr - ib_umem_start(umem_odp)) >> umem_odp->page_shift; 282 /* 283 * Strive to write the MTTs in chunks, but avoid overwriting 284 * non-existing MTTs. The huristic here can be improved to 285 * estimate the cost of another UMR vs. the cost of bigger 286 * UMR. 287 */ 288 if (umem_odp->dma_list[idx] & 289 (ODP_READ_ALLOWED_BIT | ODP_WRITE_ALLOWED_BIT)) { 290 if (!in_block) { 291 blk_start_idx = idx; 292 in_block = 1; 293 } 294 } else { 295 u64 umr_offset = idx & umr_block_mask; 296 297 if (in_block && umr_offset == 0) { 298 mlx5_ib_update_xlt(mr, blk_start_idx, 299 idx - blk_start_idx, 0, 300 MLX5_IB_UPD_XLT_ZAP | 301 MLX5_IB_UPD_XLT_ATOMIC); 302 in_block = 0; 303 } 304 } 305 } 306 if (in_block) 307 mlx5_ib_update_xlt(mr, blk_start_idx, 308 idx - blk_start_idx + 1, 0, 309 MLX5_IB_UPD_XLT_ZAP | 310 MLX5_IB_UPD_XLT_ATOMIC); 311 /* 312 * We are now sure that the device will not access the 313 * memory. We can safely unmap it, and mark it as dirty if 314 * needed. 315 */ 316 317 ib_umem_odp_unmap_dma_pages(umem_odp, start, end); 318 319 if (unlikely(!umem_odp->npages && mr->parent && 320 !umem_odp->dying)) { 321 WRITE_ONCE(mr->live, 0); 322 umem_odp->dying = 1; 323 atomic_inc(&mr->parent->num_leaf_free); 324 schedule_work(&umem_odp->work); 325 } 326 mutex_unlock(&umem_odp->umem_mutex); 327 } 328 329 void mlx5_ib_internal_fill_odp_caps(struct mlx5_ib_dev *dev) 330 { 331 struct ib_odp_caps *caps = &dev->odp_caps; 332 333 memset(caps, 0, sizeof(*caps)); 334 335 if (!MLX5_CAP_GEN(dev->mdev, pg) || 336 !mlx5_ib_can_use_umr(dev, true)) 337 return; 338 339 caps->general_caps = IB_ODP_SUPPORT; 340 341 if (MLX5_CAP_GEN(dev->mdev, umr_extended_translation_offset)) 342 dev->odp_max_size = U64_MAX; 343 else 344 dev->odp_max_size = BIT_ULL(MLX5_MAX_UMR_SHIFT + PAGE_SHIFT); 345 346 if (MLX5_CAP_ODP(dev->mdev, ud_odp_caps.send)) 347 caps->per_transport_caps.ud_odp_caps |= IB_ODP_SUPPORT_SEND; 348 349 if (MLX5_CAP_ODP(dev->mdev, ud_odp_caps.srq_receive)) 350 caps->per_transport_caps.ud_odp_caps |= IB_ODP_SUPPORT_SRQ_RECV; 351 352 if (MLX5_CAP_ODP(dev->mdev, rc_odp_caps.send)) 353 caps->per_transport_caps.rc_odp_caps |= IB_ODP_SUPPORT_SEND; 354 355 if (MLX5_CAP_ODP(dev->mdev, rc_odp_caps.receive)) 356 caps->per_transport_caps.rc_odp_caps |= IB_ODP_SUPPORT_RECV; 357 358 if (MLX5_CAP_ODP(dev->mdev, rc_odp_caps.write)) 359 caps->per_transport_caps.rc_odp_caps |= IB_ODP_SUPPORT_WRITE; 360 361 if (MLX5_CAP_ODP(dev->mdev, rc_odp_caps.read)) 362 caps->per_transport_caps.rc_odp_caps |= IB_ODP_SUPPORT_READ; 363 364 if (MLX5_CAP_ODP(dev->mdev, rc_odp_caps.atomic)) 365 caps->per_transport_caps.rc_odp_caps |= IB_ODP_SUPPORT_ATOMIC; 366 367 if (MLX5_CAP_ODP(dev->mdev, rc_odp_caps.srq_receive)) 368 caps->per_transport_caps.rc_odp_caps |= IB_ODP_SUPPORT_SRQ_RECV; 369 370 if (MLX5_CAP_ODP(dev->mdev, xrc_odp_caps.send)) 371 caps->per_transport_caps.xrc_odp_caps |= IB_ODP_SUPPORT_SEND; 372 373 if (MLX5_CAP_ODP(dev->mdev, xrc_odp_caps.receive)) 374 caps->per_transport_caps.xrc_odp_caps |= IB_ODP_SUPPORT_RECV; 375 376 if (MLX5_CAP_ODP(dev->mdev, xrc_odp_caps.write)) 377 caps->per_transport_caps.xrc_odp_caps |= IB_ODP_SUPPORT_WRITE; 378 379 if (MLX5_CAP_ODP(dev->mdev, xrc_odp_caps.read)) 380 caps->per_transport_caps.xrc_odp_caps |= IB_ODP_SUPPORT_READ; 381 382 if (MLX5_CAP_ODP(dev->mdev, xrc_odp_caps.atomic)) 383 caps->per_transport_caps.xrc_odp_caps |= IB_ODP_SUPPORT_ATOMIC; 384 385 if (MLX5_CAP_ODP(dev->mdev, xrc_odp_caps.srq_receive)) 386 caps->per_transport_caps.xrc_odp_caps |= IB_ODP_SUPPORT_SRQ_RECV; 387 388 if (MLX5_CAP_GEN(dev->mdev, fixed_buffer_size) && 389 MLX5_CAP_GEN(dev->mdev, null_mkey) && 390 MLX5_CAP_GEN(dev->mdev, umr_extended_translation_offset) && 391 !MLX5_CAP_GEN(dev->mdev, umr_indirect_mkey_disabled)) 392 caps->general_caps |= IB_ODP_SUPPORT_IMPLICIT; 393 394 return; 395 } 396 397 static void mlx5_ib_page_fault_resume(struct mlx5_ib_dev *dev, 398 struct mlx5_pagefault *pfault, 399 int error) 400 { 401 int wq_num = pfault->event_subtype == MLX5_PFAULT_SUBTYPE_WQE ? 402 pfault->wqe.wq_num : pfault->token; 403 u32 out[MLX5_ST_SZ_DW(page_fault_resume_out)] = { }; 404 u32 in[MLX5_ST_SZ_DW(page_fault_resume_in)] = { }; 405 int err; 406 407 MLX5_SET(page_fault_resume_in, in, opcode, MLX5_CMD_OP_PAGE_FAULT_RESUME); 408 MLX5_SET(page_fault_resume_in, in, page_fault_type, pfault->type); 409 MLX5_SET(page_fault_resume_in, in, token, pfault->token); 410 MLX5_SET(page_fault_resume_in, in, wq_number, wq_num); 411 MLX5_SET(page_fault_resume_in, in, error, !!error); 412 413 err = mlx5_cmd_exec(dev->mdev, in, sizeof(in), out, sizeof(out)); 414 if (err) 415 mlx5_ib_err(dev, "Failed to resolve the page fault on WQ 0x%x err %d\n", 416 wq_num, err); 417 } 418 419 static struct mlx5_ib_mr *implicit_mr_alloc(struct ib_pd *pd, 420 struct ib_umem_odp *umem_odp, 421 bool ksm, int access_flags) 422 { 423 struct mlx5_ib_dev *dev = to_mdev(pd->device); 424 struct mlx5_ib_mr *mr; 425 int err; 426 427 mr = mlx5_mr_cache_alloc(dev, ksm ? MLX5_IMR_KSM_CACHE_ENTRY : 428 MLX5_IMR_MTT_CACHE_ENTRY); 429 430 if (IS_ERR(mr)) 431 return mr; 432 433 mr->ibmr.pd = pd; 434 435 mr->dev = dev; 436 mr->access_flags = access_flags; 437 mr->mmkey.iova = 0; 438 mr->umem = &umem_odp->umem; 439 440 if (ksm) { 441 err = mlx5_ib_update_xlt(mr, 0, 442 mlx5_imr_ksm_entries, 443 MLX5_KSM_PAGE_SHIFT, 444 MLX5_IB_UPD_XLT_INDIRECT | 445 MLX5_IB_UPD_XLT_ZAP | 446 MLX5_IB_UPD_XLT_ENABLE); 447 448 } else { 449 err = mlx5_ib_update_xlt(mr, 0, 450 MLX5_IMR_MTT_ENTRIES, 451 PAGE_SHIFT, 452 MLX5_IB_UPD_XLT_ZAP | 453 MLX5_IB_UPD_XLT_ENABLE | 454 MLX5_IB_UPD_XLT_ATOMIC); 455 } 456 457 if (err) 458 goto fail; 459 460 mr->ibmr.lkey = mr->mmkey.key; 461 mr->ibmr.rkey = mr->mmkey.key; 462 463 mlx5_ib_dbg(dev, "key %x dev %p mr %p\n", 464 mr->mmkey.key, dev->mdev, mr); 465 466 return mr; 467 468 fail: 469 mlx5_ib_err(dev, "Failed to register MKEY %d\n", err); 470 mlx5_mr_cache_free(dev, mr); 471 472 return ERR_PTR(err); 473 } 474 475 static struct ib_umem_odp *implicit_mr_get_data(struct mlx5_ib_mr *mr, 476 u64 io_virt, size_t bcnt) 477 { 478 struct mlx5_ib_dev *dev = to_mdev(mr->ibmr.pd->device); 479 struct ib_umem_odp *odp, *result = NULL; 480 struct ib_umem_odp *odp_mr = to_ib_umem_odp(mr->umem); 481 u64 addr = io_virt & MLX5_IMR_MTT_MASK; 482 int nentries = 0, start_idx = 0, ret; 483 struct mlx5_ib_mr *mtt; 484 485 mutex_lock(&odp_mr->umem_mutex); 486 odp = odp_lookup(addr, 1, mr); 487 488 mlx5_ib_dbg(dev, "io_virt:%llx bcnt:%zx addr:%llx odp:%p\n", 489 io_virt, bcnt, addr, odp); 490 491 next_mr: 492 if (likely(odp)) { 493 if (nentries) 494 nentries++; 495 } else { 496 odp = ib_umem_odp_alloc_child(odp_mr, addr, MLX5_IMR_MTT_SIZE); 497 if (IS_ERR(odp)) { 498 mutex_unlock(&odp_mr->umem_mutex); 499 return ERR_CAST(odp); 500 } 501 502 mtt = implicit_mr_alloc(mr->ibmr.pd, odp, 0, 503 mr->access_flags); 504 if (IS_ERR(mtt)) { 505 mutex_unlock(&odp_mr->umem_mutex); 506 ib_umem_odp_release(odp); 507 return ERR_CAST(mtt); 508 } 509 510 odp->private = mtt; 511 mtt->umem = &odp->umem; 512 mtt->mmkey.iova = addr; 513 mtt->parent = mr; 514 INIT_WORK(&odp->work, mr_leaf_free_action); 515 516 smp_store_release(&mtt->live, 1); 517 518 if (!nentries) 519 start_idx = addr >> MLX5_IMR_MTT_SHIFT; 520 nentries++; 521 } 522 523 /* Return first odp if region not covered by single one */ 524 if (likely(!result)) 525 result = odp; 526 527 addr += MLX5_IMR_MTT_SIZE; 528 if (unlikely(addr < io_virt + bcnt)) { 529 odp = odp_next(odp); 530 if (odp && ib_umem_start(odp) != addr) 531 odp = NULL; 532 goto next_mr; 533 } 534 535 if (unlikely(nentries)) { 536 ret = mlx5_ib_update_xlt(mr, start_idx, nentries, 0, 537 MLX5_IB_UPD_XLT_INDIRECT | 538 MLX5_IB_UPD_XLT_ATOMIC); 539 if (ret) { 540 mlx5_ib_err(dev, "Failed to update PAS\n"); 541 result = ERR_PTR(ret); 542 } 543 } 544 545 mutex_unlock(&odp_mr->umem_mutex); 546 return result; 547 } 548 549 struct mlx5_ib_mr *mlx5_ib_alloc_implicit_mr(struct mlx5_ib_pd *pd, 550 struct ib_udata *udata, 551 int access_flags) 552 { 553 struct mlx5_ib_mr *imr; 554 struct ib_umem_odp *umem_odp; 555 556 umem_odp = ib_umem_odp_alloc_implicit(udata, access_flags); 557 if (IS_ERR(umem_odp)) 558 return ERR_CAST(umem_odp); 559 560 imr = implicit_mr_alloc(&pd->ibpd, umem_odp, 1, access_flags); 561 if (IS_ERR(imr)) { 562 ib_umem_odp_release(umem_odp); 563 return ERR_CAST(imr); 564 } 565 566 imr->umem = &umem_odp->umem; 567 init_waitqueue_head(&imr->q_leaf_free); 568 atomic_set(&imr->num_leaf_free, 0); 569 atomic_set(&imr->num_pending_prefetch, 0); 570 smp_store_release(&imr->live, 1); 571 572 return imr; 573 } 574 575 void mlx5_ib_free_implicit_mr(struct mlx5_ib_mr *imr) 576 { 577 struct ib_ucontext_per_mm *per_mm = mr_to_per_mm(imr); 578 struct rb_node *node; 579 580 down_read(&per_mm->umem_rwsem); 581 for (node = rb_first_cached(&per_mm->umem_tree); node; 582 node = rb_next(node)) { 583 struct ib_umem_odp *umem_odp = 584 rb_entry(node, struct ib_umem_odp, interval_tree.rb); 585 struct mlx5_ib_mr *mr = umem_odp->private; 586 587 if (mr->parent != imr) 588 continue; 589 590 mutex_lock(&umem_odp->umem_mutex); 591 ib_umem_odp_unmap_dma_pages(umem_odp, ib_umem_start(umem_odp), 592 ib_umem_end(umem_odp)); 593 594 if (umem_odp->dying) { 595 mutex_unlock(&umem_odp->umem_mutex); 596 continue; 597 } 598 599 umem_odp->dying = 1; 600 atomic_inc(&imr->num_leaf_free); 601 schedule_work(&umem_odp->work); 602 mutex_unlock(&umem_odp->umem_mutex); 603 } 604 up_read(&per_mm->umem_rwsem); 605 606 wait_event(imr->q_leaf_free, !atomic_read(&imr->num_leaf_free)); 607 } 608 609 #define MLX5_PF_FLAGS_PREFETCH BIT(0) 610 #define MLX5_PF_FLAGS_DOWNGRADE BIT(1) 611 static int pagefault_mr(struct mlx5_ib_dev *dev, struct mlx5_ib_mr *mr, 612 u64 io_virt, size_t bcnt, u32 *bytes_mapped, 613 u32 flags) 614 { 615 int npages = 0, current_seq, page_shift, ret, np; 616 struct ib_umem_odp *odp_mr = to_ib_umem_odp(mr->umem); 617 bool downgrade = flags & MLX5_PF_FLAGS_DOWNGRADE; 618 bool prefetch = flags & MLX5_PF_FLAGS_PREFETCH; 619 u64 access_mask; 620 u64 start_idx, page_mask; 621 struct ib_umem_odp *odp; 622 size_t size; 623 624 if (odp_mr->is_implicit_odp) { 625 odp = implicit_mr_get_data(mr, io_virt, bcnt); 626 627 if (IS_ERR(odp)) 628 return PTR_ERR(odp); 629 mr = odp->private; 630 } else { 631 odp = odp_mr; 632 } 633 634 next_mr: 635 size = min_t(size_t, bcnt, ib_umem_end(odp) - io_virt); 636 637 page_shift = odp->page_shift; 638 page_mask = ~(BIT(page_shift) - 1); 639 start_idx = (io_virt - (mr->mmkey.iova & page_mask)) >> page_shift; 640 access_mask = ODP_READ_ALLOWED_BIT; 641 642 if (prefetch && !downgrade && !odp->umem.writable) { 643 /* prefetch with write-access must 644 * be supported by the MR 645 */ 646 ret = -EINVAL; 647 goto out; 648 } 649 650 if (odp->umem.writable && !downgrade) 651 access_mask |= ODP_WRITE_ALLOWED_BIT; 652 653 current_seq = READ_ONCE(odp->notifiers_seq); 654 /* 655 * Ensure the sequence number is valid for some time before we call 656 * gup. 657 */ 658 smp_rmb(); 659 660 ret = ib_umem_odp_map_dma_pages(odp, io_virt, size, access_mask, 661 current_seq); 662 663 if (ret < 0) 664 goto out; 665 666 np = ret; 667 668 mutex_lock(&odp->umem_mutex); 669 if (!ib_umem_mmu_notifier_retry(odp, current_seq)) { 670 /* 671 * No need to check whether the MTTs really belong to 672 * this MR, since ib_umem_odp_map_dma_pages already 673 * checks this. 674 */ 675 ret = mlx5_ib_update_xlt(mr, start_idx, np, 676 page_shift, MLX5_IB_UPD_XLT_ATOMIC); 677 } else { 678 ret = -EAGAIN; 679 } 680 mutex_unlock(&odp->umem_mutex); 681 682 if (ret < 0) { 683 if (ret != -EAGAIN) 684 mlx5_ib_err(dev, "Failed to update mkey page tables\n"); 685 goto out; 686 } 687 688 if (bytes_mapped) { 689 u32 new_mappings = (np << page_shift) - 690 (io_virt - round_down(io_virt, 1 << page_shift)); 691 *bytes_mapped += min_t(u32, new_mappings, size); 692 } 693 694 npages += np << (page_shift - PAGE_SHIFT); 695 bcnt -= size; 696 697 if (unlikely(bcnt)) { 698 struct ib_umem_odp *next; 699 700 io_virt += size; 701 next = odp_next(odp); 702 if (unlikely(!next || ib_umem_start(next) != io_virt)) { 703 mlx5_ib_dbg(dev, "next implicit leaf removed at 0x%llx. got %p\n", 704 io_virt, next); 705 return -EAGAIN; 706 } 707 odp = next; 708 mr = odp->private; 709 goto next_mr; 710 } 711 712 return npages; 713 714 out: 715 if (ret == -EAGAIN) { 716 unsigned long timeout = msecs_to_jiffies(MMU_NOTIFIER_TIMEOUT); 717 718 if (!wait_for_completion_timeout(&odp->notifier_completion, 719 timeout)) { 720 mlx5_ib_warn( 721 dev, 722 "timeout waiting for mmu notifier. seq %d against %d. notifiers_count=%d\n", 723 current_seq, odp->notifiers_seq, 724 odp->notifiers_count); 725 } 726 } 727 728 return ret; 729 } 730 731 struct pf_frame { 732 struct pf_frame *next; 733 u32 key; 734 u64 io_virt; 735 size_t bcnt; 736 int depth; 737 }; 738 739 static bool mkey_is_eq(struct mlx5_core_mkey *mmkey, u32 key) 740 { 741 if (!mmkey) 742 return false; 743 if (mmkey->type == MLX5_MKEY_MW) 744 return mlx5_base_mkey(mmkey->key) == mlx5_base_mkey(key); 745 return mmkey->key == key; 746 } 747 748 static int get_indirect_num_descs(struct mlx5_core_mkey *mmkey) 749 { 750 struct mlx5_ib_mw *mw; 751 struct mlx5_ib_devx_mr *devx_mr; 752 753 if (mmkey->type == MLX5_MKEY_MW) { 754 mw = container_of(mmkey, struct mlx5_ib_mw, mmkey); 755 return mw->ndescs; 756 } 757 758 devx_mr = container_of(mmkey, struct mlx5_ib_devx_mr, 759 mmkey); 760 return devx_mr->ndescs; 761 } 762 763 /* 764 * Handle a single data segment in a page-fault WQE or RDMA region. 765 * 766 * Returns number of OS pages retrieved on success. The caller may continue to 767 * the next data segment. 768 * Can return the following error codes: 769 * -EAGAIN to designate a temporary error. The caller will abort handling the 770 * page fault and resolve it. 771 * -EFAULT when there's an error mapping the requested pages. The caller will 772 * abort the page fault handling. 773 */ 774 static int pagefault_single_data_segment(struct mlx5_ib_dev *dev, 775 struct ib_pd *pd, u32 key, 776 u64 io_virt, size_t bcnt, 777 u32 *bytes_committed, 778 u32 *bytes_mapped, u32 flags) 779 { 780 int npages = 0, srcu_key, ret, i, outlen, cur_outlen = 0, depth = 0; 781 bool prefetch = flags & MLX5_PF_FLAGS_PREFETCH; 782 struct pf_frame *head = NULL, *frame; 783 struct mlx5_core_mkey *mmkey; 784 struct mlx5_ib_mr *mr; 785 struct mlx5_klm *pklm; 786 u32 *out = NULL; 787 size_t offset; 788 int ndescs; 789 790 srcu_key = srcu_read_lock(&dev->mr_srcu); 791 792 io_virt += *bytes_committed; 793 bcnt -= *bytes_committed; 794 795 next_mr: 796 mmkey = xa_load(&dev->mdev->priv.mkey_table, mlx5_base_mkey(key)); 797 if (!mkey_is_eq(mmkey, key)) { 798 mlx5_ib_dbg(dev, "failed to find mkey %x\n", key); 799 ret = -EFAULT; 800 goto srcu_unlock; 801 } 802 803 if (prefetch && mmkey->type != MLX5_MKEY_MR) { 804 mlx5_ib_dbg(dev, "prefetch is allowed only for MR\n"); 805 ret = -EINVAL; 806 goto srcu_unlock; 807 } 808 809 switch (mmkey->type) { 810 case MLX5_MKEY_MR: 811 mr = container_of(mmkey, struct mlx5_ib_mr, mmkey); 812 if (!smp_load_acquire(&mr->live) || !mr->ibmr.pd) { 813 mlx5_ib_dbg(dev, "got dead MR\n"); 814 ret = -EFAULT; 815 goto srcu_unlock; 816 } 817 818 if (prefetch) { 819 if (!is_odp_mr(mr) || 820 mr->ibmr.pd != pd) { 821 mlx5_ib_dbg(dev, "Invalid prefetch request: %s\n", 822 is_odp_mr(mr) ? "MR is not ODP" : 823 "PD is not of the MR"); 824 ret = -EINVAL; 825 goto srcu_unlock; 826 } 827 } 828 829 if (!is_odp_mr(mr)) { 830 mlx5_ib_dbg(dev, "skipping non ODP MR (lkey=0x%06x) in page fault handler.\n", 831 key); 832 if (bytes_mapped) 833 *bytes_mapped += bcnt; 834 ret = 0; 835 goto srcu_unlock; 836 } 837 838 ret = pagefault_mr(dev, mr, io_virt, bcnt, bytes_mapped, flags); 839 if (ret < 0) 840 goto srcu_unlock; 841 842 npages += ret; 843 ret = 0; 844 break; 845 846 case MLX5_MKEY_MW: 847 case MLX5_MKEY_INDIRECT_DEVX: 848 ndescs = get_indirect_num_descs(mmkey); 849 850 if (depth >= MLX5_CAP_GEN(dev->mdev, max_indirection)) { 851 mlx5_ib_dbg(dev, "indirection level exceeded\n"); 852 ret = -EFAULT; 853 goto srcu_unlock; 854 } 855 856 outlen = MLX5_ST_SZ_BYTES(query_mkey_out) + 857 sizeof(*pklm) * (ndescs - 2); 858 859 if (outlen > cur_outlen) { 860 kfree(out); 861 out = kzalloc(outlen, GFP_KERNEL); 862 if (!out) { 863 ret = -ENOMEM; 864 goto srcu_unlock; 865 } 866 cur_outlen = outlen; 867 } 868 869 pklm = (struct mlx5_klm *)MLX5_ADDR_OF(query_mkey_out, out, 870 bsf0_klm0_pas_mtt0_1); 871 872 ret = mlx5_core_query_mkey(dev->mdev, mmkey, out, outlen); 873 if (ret) 874 goto srcu_unlock; 875 876 offset = io_virt - MLX5_GET64(query_mkey_out, out, 877 memory_key_mkey_entry.start_addr); 878 879 for (i = 0; bcnt && i < ndescs; i++, pklm++) { 880 if (offset >= be32_to_cpu(pklm->bcount)) { 881 offset -= be32_to_cpu(pklm->bcount); 882 continue; 883 } 884 885 frame = kzalloc(sizeof(*frame), GFP_KERNEL); 886 if (!frame) { 887 ret = -ENOMEM; 888 goto srcu_unlock; 889 } 890 891 frame->key = be32_to_cpu(pklm->key); 892 frame->io_virt = be64_to_cpu(pklm->va) + offset; 893 frame->bcnt = min_t(size_t, bcnt, 894 be32_to_cpu(pklm->bcount) - offset); 895 frame->depth = depth + 1; 896 frame->next = head; 897 head = frame; 898 899 bcnt -= frame->bcnt; 900 offset = 0; 901 } 902 break; 903 904 default: 905 mlx5_ib_dbg(dev, "wrong mkey type %d\n", mmkey->type); 906 ret = -EFAULT; 907 goto srcu_unlock; 908 } 909 910 if (head) { 911 frame = head; 912 head = frame->next; 913 914 key = frame->key; 915 io_virt = frame->io_virt; 916 bcnt = frame->bcnt; 917 depth = frame->depth; 918 kfree(frame); 919 920 goto next_mr; 921 } 922 923 srcu_unlock: 924 while (head) { 925 frame = head; 926 head = frame->next; 927 kfree(frame); 928 } 929 kfree(out); 930 931 srcu_read_unlock(&dev->mr_srcu, srcu_key); 932 *bytes_committed = 0; 933 return ret ? ret : npages; 934 } 935 936 /** 937 * Parse a series of data segments for page fault handling. 938 * 939 * @pfault contains page fault information. 940 * @wqe points at the first data segment in the WQE. 941 * @wqe_end points after the end of the WQE. 942 * @bytes_mapped receives the number of bytes that the function was able to 943 * map. This allows the caller to decide intelligently whether 944 * enough memory was mapped to resolve the page fault 945 * successfully (e.g. enough for the next MTU, or the entire 946 * WQE). 947 * @total_wqe_bytes receives the total data size of this WQE in bytes (minus 948 * the committed bytes). 949 * 950 * Returns the number of pages loaded if positive, zero for an empty WQE, or a 951 * negative error code. 952 */ 953 static int pagefault_data_segments(struct mlx5_ib_dev *dev, 954 struct mlx5_pagefault *pfault, 955 void *wqe, 956 void *wqe_end, u32 *bytes_mapped, 957 u32 *total_wqe_bytes, bool receive_queue) 958 { 959 int ret = 0, npages = 0; 960 u64 io_virt; 961 u32 key; 962 u32 byte_count; 963 size_t bcnt; 964 int inline_segment; 965 966 if (bytes_mapped) 967 *bytes_mapped = 0; 968 if (total_wqe_bytes) 969 *total_wqe_bytes = 0; 970 971 while (wqe < wqe_end) { 972 struct mlx5_wqe_data_seg *dseg = wqe; 973 974 io_virt = be64_to_cpu(dseg->addr); 975 key = be32_to_cpu(dseg->lkey); 976 byte_count = be32_to_cpu(dseg->byte_count); 977 inline_segment = !!(byte_count & MLX5_INLINE_SEG); 978 bcnt = byte_count & ~MLX5_INLINE_SEG; 979 980 if (inline_segment) { 981 bcnt = bcnt & MLX5_WQE_INLINE_SEG_BYTE_COUNT_MASK; 982 wqe += ALIGN(sizeof(struct mlx5_wqe_inline_seg) + bcnt, 983 16); 984 } else { 985 wqe += sizeof(*dseg); 986 } 987 988 /* receive WQE end of sg list. */ 989 if (receive_queue && bcnt == 0 && key == MLX5_INVALID_LKEY && 990 io_virt == 0) 991 break; 992 993 if (!inline_segment && total_wqe_bytes) { 994 *total_wqe_bytes += bcnt - min_t(size_t, bcnt, 995 pfault->bytes_committed); 996 } 997 998 /* A zero length data segment designates a length of 2GB. */ 999 if (bcnt == 0) 1000 bcnt = 1U << 31; 1001 1002 if (inline_segment || bcnt <= pfault->bytes_committed) { 1003 pfault->bytes_committed -= 1004 min_t(size_t, bcnt, 1005 pfault->bytes_committed); 1006 continue; 1007 } 1008 1009 ret = pagefault_single_data_segment(dev, NULL, key, 1010 io_virt, bcnt, 1011 &pfault->bytes_committed, 1012 bytes_mapped, 0); 1013 if (ret < 0) 1014 break; 1015 npages += ret; 1016 } 1017 1018 return ret < 0 ? ret : npages; 1019 } 1020 1021 /* 1022 * Parse initiator WQE. Advances the wqe pointer to point at the 1023 * scatter-gather list, and set wqe_end to the end of the WQE. 1024 */ 1025 static int mlx5_ib_mr_initiator_pfault_handler( 1026 struct mlx5_ib_dev *dev, struct mlx5_pagefault *pfault, 1027 struct mlx5_ib_qp *qp, void **wqe, void **wqe_end, int wqe_length) 1028 { 1029 struct mlx5_wqe_ctrl_seg *ctrl = *wqe; 1030 u16 wqe_index = pfault->wqe.wqe_index; 1031 struct mlx5_base_av *av; 1032 unsigned ds, opcode; 1033 u32 qpn = qp->trans_qp.base.mqp.qpn; 1034 1035 ds = be32_to_cpu(ctrl->qpn_ds) & MLX5_WQE_CTRL_DS_MASK; 1036 if (ds * MLX5_WQE_DS_UNITS > wqe_length) { 1037 mlx5_ib_err(dev, "Unable to read the complete WQE. ds = 0x%x, ret = 0x%x\n", 1038 ds, wqe_length); 1039 return -EFAULT; 1040 } 1041 1042 if (ds == 0) { 1043 mlx5_ib_err(dev, "Got WQE with zero DS. wqe_index=%x, qpn=%x\n", 1044 wqe_index, qpn); 1045 return -EFAULT; 1046 } 1047 1048 *wqe_end = *wqe + ds * MLX5_WQE_DS_UNITS; 1049 *wqe += sizeof(*ctrl); 1050 1051 opcode = be32_to_cpu(ctrl->opmod_idx_opcode) & 1052 MLX5_WQE_CTRL_OPCODE_MASK; 1053 1054 if (qp->ibqp.qp_type == IB_QPT_XRC_INI) 1055 *wqe += sizeof(struct mlx5_wqe_xrc_seg); 1056 1057 if (qp->ibqp.qp_type == IB_QPT_UD || 1058 qp->qp_sub_type == MLX5_IB_QPT_DCI) { 1059 av = *wqe; 1060 if (av->dqp_dct & cpu_to_be32(MLX5_EXTENDED_UD_AV)) 1061 *wqe += sizeof(struct mlx5_av); 1062 else 1063 *wqe += sizeof(struct mlx5_base_av); 1064 } 1065 1066 switch (opcode) { 1067 case MLX5_OPCODE_RDMA_WRITE: 1068 case MLX5_OPCODE_RDMA_WRITE_IMM: 1069 case MLX5_OPCODE_RDMA_READ: 1070 *wqe += sizeof(struct mlx5_wqe_raddr_seg); 1071 break; 1072 case MLX5_OPCODE_ATOMIC_CS: 1073 case MLX5_OPCODE_ATOMIC_FA: 1074 *wqe += sizeof(struct mlx5_wqe_raddr_seg); 1075 *wqe += sizeof(struct mlx5_wqe_atomic_seg); 1076 break; 1077 } 1078 1079 return 0; 1080 } 1081 1082 /* 1083 * Parse responder WQE and set wqe_end to the end of the WQE. 1084 */ 1085 static int mlx5_ib_mr_responder_pfault_handler_srq(struct mlx5_ib_dev *dev, 1086 struct mlx5_ib_srq *srq, 1087 void **wqe, void **wqe_end, 1088 int wqe_length) 1089 { 1090 int wqe_size = 1 << srq->msrq.wqe_shift; 1091 1092 if (wqe_size > wqe_length) { 1093 mlx5_ib_err(dev, "Couldn't read all of the receive WQE's content\n"); 1094 return -EFAULT; 1095 } 1096 1097 *wqe_end = *wqe + wqe_size; 1098 *wqe += sizeof(struct mlx5_wqe_srq_next_seg); 1099 1100 return 0; 1101 } 1102 1103 static int mlx5_ib_mr_responder_pfault_handler_rq(struct mlx5_ib_dev *dev, 1104 struct mlx5_ib_qp *qp, 1105 void *wqe, void **wqe_end, 1106 int wqe_length) 1107 { 1108 struct mlx5_ib_wq *wq = &qp->rq; 1109 int wqe_size = 1 << wq->wqe_shift; 1110 1111 if (qp->wq_sig) { 1112 mlx5_ib_err(dev, "ODP fault with WQE signatures is not supported\n"); 1113 return -EFAULT; 1114 } 1115 1116 if (wqe_size > wqe_length) { 1117 mlx5_ib_err(dev, "Couldn't read all of the receive WQE's content\n"); 1118 return -EFAULT; 1119 } 1120 1121 *wqe_end = wqe + wqe_size; 1122 1123 return 0; 1124 } 1125 1126 static inline struct mlx5_core_rsc_common *odp_get_rsc(struct mlx5_ib_dev *dev, 1127 u32 wq_num, int pf_type) 1128 { 1129 struct mlx5_core_rsc_common *common = NULL; 1130 struct mlx5_core_srq *srq; 1131 1132 switch (pf_type) { 1133 case MLX5_WQE_PF_TYPE_RMP: 1134 srq = mlx5_cmd_get_srq(dev, wq_num); 1135 if (srq) 1136 common = &srq->common; 1137 break; 1138 case MLX5_WQE_PF_TYPE_REQ_SEND_OR_WRITE: 1139 case MLX5_WQE_PF_TYPE_RESP: 1140 case MLX5_WQE_PF_TYPE_REQ_READ_OR_ATOMIC: 1141 common = mlx5_core_res_hold(dev->mdev, wq_num, MLX5_RES_QP); 1142 break; 1143 default: 1144 break; 1145 } 1146 1147 return common; 1148 } 1149 1150 static inline struct mlx5_ib_qp *res_to_qp(struct mlx5_core_rsc_common *res) 1151 { 1152 struct mlx5_core_qp *mqp = (struct mlx5_core_qp *)res; 1153 1154 return to_mibqp(mqp); 1155 } 1156 1157 static inline struct mlx5_ib_srq *res_to_srq(struct mlx5_core_rsc_common *res) 1158 { 1159 struct mlx5_core_srq *msrq = 1160 container_of(res, struct mlx5_core_srq, common); 1161 1162 return to_mibsrq(msrq); 1163 } 1164 1165 static void mlx5_ib_mr_wqe_pfault_handler(struct mlx5_ib_dev *dev, 1166 struct mlx5_pagefault *pfault) 1167 { 1168 bool sq = pfault->type & MLX5_PFAULT_REQUESTOR; 1169 u16 wqe_index = pfault->wqe.wqe_index; 1170 void *wqe, *wqe_start = NULL, *wqe_end = NULL; 1171 u32 bytes_mapped, total_wqe_bytes; 1172 struct mlx5_core_rsc_common *res; 1173 int resume_with_error = 1; 1174 struct mlx5_ib_qp *qp; 1175 size_t bytes_copied; 1176 int ret = 0; 1177 1178 res = odp_get_rsc(dev, pfault->wqe.wq_num, pfault->type); 1179 if (!res) { 1180 mlx5_ib_dbg(dev, "wqe page fault for missing resource %d\n", pfault->wqe.wq_num); 1181 return; 1182 } 1183 1184 if (res->res != MLX5_RES_QP && res->res != MLX5_RES_SRQ && 1185 res->res != MLX5_RES_XSRQ) { 1186 mlx5_ib_err(dev, "wqe page fault for unsupported type %d\n", 1187 pfault->type); 1188 goto resolve_page_fault; 1189 } 1190 1191 wqe_start = (void *)__get_free_page(GFP_KERNEL); 1192 if (!wqe_start) { 1193 mlx5_ib_err(dev, "Error allocating memory for IO page fault handling.\n"); 1194 goto resolve_page_fault; 1195 } 1196 1197 wqe = wqe_start; 1198 qp = (res->res == MLX5_RES_QP) ? res_to_qp(res) : NULL; 1199 if (qp && sq) { 1200 ret = mlx5_ib_read_user_wqe_sq(qp, wqe_index, wqe, PAGE_SIZE, 1201 &bytes_copied); 1202 if (ret) 1203 goto read_user; 1204 ret = mlx5_ib_mr_initiator_pfault_handler( 1205 dev, pfault, qp, &wqe, &wqe_end, bytes_copied); 1206 } else if (qp && !sq) { 1207 ret = mlx5_ib_read_user_wqe_rq(qp, wqe_index, wqe, PAGE_SIZE, 1208 &bytes_copied); 1209 if (ret) 1210 goto read_user; 1211 ret = mlx5_ib_mr_responder_pfault_handler_rq( 1212 dev, qp, wqe, &wqe_end, bytes_copied); 1213 } else if (!qp) { 1214 struct mlx5_ib_srq *srq = res_to_srq(res); 1215 1216 ret = mlx5_ib_read_user_wqe_srq(srq, wqe_index, wqe, PAGE_SIZE, 1217 &bytes_copied); 1218 if (ret) 1219 goto read_user; 1220 ret = mlx5_ib_mr_responder_pfault_handler_srq( 1221 dev, srq, &wqe, &wqe_end, bytes_copied); 1222 } 1223 1224 if (ret < 0 || wqe >= wqe_end) 1225 goto resolve_page_fault; 1226 1227 ret = pagefault_data_segments(dev, pfault, wqe, wqe_end, &bytes_mapped, 1228 &total_wqe_bytes, !sq); 1229 if (ret == -EAGAIN) 1230 goto out; 1231 1232 if (ret < 0 || total_wqe_bytes > bytes_mapped) 1233 goto resolve_page_fault; 1234 1235 out: 1236 ret = 0; 1237 resume_with_error = 0; 1238 1239 read_user: 1240 if (ret) 1241 mlx5_ib_err( 1242 dev, 1243 "Failed reading a WQE following page fault, error %d, wqe_index %x, qpn %x\n", 1244 ret, wqe_index, pfault->token); 1245 1246 resolve_page_fault: 1247 mlx5_ib_page_fault_resume(dev, pfault, resume_with_error); 1248 mlx5_ib_dbg(dev, "PAGE FAULT completed. QP 0x%x resume_with_error=%d, type: 0x%x\n", 1249 pfault->wqe.wq_num, resume_with_error, 1250 pfault->type); 1251 mlx5_core_res_put(res); 1252 free_page((unsigned long)wqe_start); 1253 } 1254 1255 static int pages_in_range(u64 address, u32 length) 1256 { 1257 return (ALIGN(address + length, PAGE_SIZE) - 1258 (address & PAGE_MASK)) >> PAGE_SHIFT; 1259 } 1260 1261 static void mlx5_ib_mr_rdma_pfault_handler(struct mlx5_ib_dev *dev, 1262 struct mlx5_pagefault *pfault) 1263 { 1264 u64 address; 1265 u32 length; 1266 u32 prefetch_len = pfault->bytes_committed; 1267 int prefetch_activated = 0; 1268 u32 rkey = pfault->rdma.r_key; 1269 int ret; 1270 1271 /* The RDMA responder handler handles the page fault in two parts. 1272 * First it brings the necessary pages for the current packet 1273 * (and uses the pfault context), and then (after resuming the QP) 1274 * prefetches more pages. The second operation cannot use the pfault 1275 * context and therefore uses the dummy_pfault context allocated on 1276 * the stack */ 1277 pfault->rdma.rdma_va += pfault->bytes_committed; 1278 pfault->rdma.rdma_op_len -= min(pfault->bytes_committed, 1279 pfault->rdma.rdma_op_len); 1280 pfault->bytes_committed = 0; 1281 1282 address = pfault->rdma.rdma_va; 1283 length = pfault->rdma.rdma_op_len; 1284 1285 /* For some operations, the hardware cannot tell the exact message 1286 * length, and in those cases it reports zero. Use prefetch 1287 * logic. */ 1288 if (length == 0) { 1289 prefetch_activated = 1; 1290 length = pfault->rdma.packet_size; 1291 prefetch_len = min(MAX_PREFETCH_LEN, prefetch_len); 1292 } 1293 1294 ret = pagefault_single_data_segment(dev, NULL, rkey, address, length, 1295 &pfault->bytes_committed, NULL, 1296 0); 1297 if (ret == -EAGAIN) { 1298 /* We're racing with an invalidation, don't prefetch */ 1299 prefetch_activated = 0; 1300 } else if (ret < 0 || pages_in_range(address, length) > ret) { 1301 mlx5_ib_page_fault_resume(dev, pfault, 1); 1302 if (ret != -ENOENT) 1303 mlx5_ib_dbg(dev, "PAGE FAULT error %d. QP 0x%x, type: 0x%x\n", 1304 ret, pfault->token, pfault->type); 1305 return; 1306 } 1307 1308 mlx5_ib_page_fault_resume(dev, pfault, 0); 1309 mlx5_ib_dbg(dev, "PAGE FAULT completed. QP 0x%x, type: 0x%x, prefetch_activated: %d\n", 1310 pfault->token, pfault->type, 1311 prefetch_activated); 1312 1313 /* At this point, there might be a new pagefault already arriving in 1314 * the eq, switch to the dummy pagefault for the rest of the 1315 * processing. We're still OK with the objects being alive as the 1316 * work-queue is being fenced. */ 1317 1318 if (prefetch_activated) { 1319 u32 bytes_committed = 0; 1320 1321 ret = pagefault_single_data_segment(dev, NULL, rkey, address, 1322 prefetch_len, 1323 &bytes_committed, NULL, 1324 0); 1325 if (ret < 0 && ret != -EAGAIN) { 1326 mlx5_ib_dbg(dev, "Prefetch failed. ret: %d, QP 0x%x, address: 0x%.16llx, length = 0x%.16x\n", 1327 ret, pfault->token, address, prefetch_len); 1328 } 1329 } 1330 } 1331 1332 static void mlx5_ib_pfault(struct mlx5_ib_dev *dev, struct mlx5_pagefault *pfault) 1333 { 1334 u8 event_subtype = pfault->event_subtype; 1335 1336 switch (event_subtype) { 1337 case MLX5_PFAULT_SUBTYPE_WQE: 1338 mlx5_ib_mr_wqe_pfault_handler(dev, pfault); 1339 break; 1340 case MLX5_PFAULT_SUBTYPE_RDMA: 1341 mlx5_ib_mr_rdma_pfault_handler(dev, pfault); 1342 break; 1343 default: 1344 mlx5_ib_err(dev, "Invalid page fault event subtype: 0x%x\n", 1345 event_subtype); 1346 mlx5_ib_page_fault_resume(dev, pfault, 1); 1347 } 1348 } 1349 1350 static void mlx5_ib_eqe_pf_action(struct work_struct *work) 1351 { 1352 struct mlx5_pagefault *pfault = container_of(work, 1353 struct mlx5_pagefault, 1354 work); 1355 struct mlx5_ib_pf_eq *eq = pfault->eq; 1356 1357 mlx5_ib_pfault(eq->dev, pfault); 1358 mempool_free(pfault, eq->pool); 1359 } 1360 1361 static void mlx5_ib_eq_pf_process(struct mlx5_ib_pf_eq *eq) 1362 { 1363 struct mlx5_eqe_page_fault *pf_eqe; 1364 struct mlx5_pagefault *pfault; 1365 struct mlx5_eqe *eqe; 1366 int cc = 0; 1367 1368 while ((eqe = mlx5_eq_get_eqe(eq->core, cc))) { 1369 pfault = mempool_alloc(eq->pool, GFP_ATOMIC); 1370 if (!pfault) { 1371 schedule_work(&eq->work); 1372 break; 1373 } 1374 1375 pf_eqe = &eqe->data.page_fault; 1376 pfault->event_subtype = eqe->sub_type; 1377 pfault->bytes_committed = be32_to_cpu(pf_eqe->bytes_committed); 1378 1379 mlx5_ib_dbg(eq->dev, 1380 "PAGE_FAULT: subtype: 0x%02x, bytes_committed: 0x%06x\n", 1381 eqe->sub_type, pfault->bytes_committed); 1382 1383 switch (eqe->sub_type) { 1384 case MLX5_PFAULT_SUBTYPE_RDMA: 1385 /* RDMA based event */ 1386 pfault->type = 1387 be32_to_cpu(pf_eqe->rdma.pftype_token) >> 24; 1388 pfault->token = 1389 be32_to_cpu(pf_eqe->rdma.pftype_token) & 1390 MLX5_24BIT_MASK; 1391 pfault->rdma.r_key = 1392 be32_to_cpu(pf_eqe->rdma.r_key); 1393 pfault->rdma.packet_size = 1394 be16_to_cpu(pf_eqe->rdma.packet_length); 1395 pfault->rdma.rdma_op_len = 1396 be32_to_cpu(pf_eqe->rdma.rdma_op_len); 1397 pfault->rdma.rdma_va = 1398 be64_to_cpu(pf_eqe->rdma.rdma_va); 1399 mlx5_ib_dbg(eq->dev, 1400 "PAGE_FAULT: type:0x%x, token: 0x%06x, r_key: 0x%08x\n", 1401 pfault->type, pfault->token, 1402 pfault->rdma.r_key); 1403 mlx5_ib_dbg(eq->dev, 1404 "PAGE_FAULT: rdma_op_len: 0x%08x, rdma_va: 0x%016llx\n", 1405 pfault->rdma.rdma_op_len, 1406 pfault->rdma.rdma_va); 1407 break; 1408 1409 case MLX5_PFAULT_SUBTYPE_WQE: 1410 /* WQE based event */ 1411 pfault->type = 1412 (be32_to_cpu(pf_eqe->wqe.pftype_wq) >> 24) & 0x7; 1413 pfault->token = 1414 be32_to_cpu(pf_eqe->wqe.token); 1415 pfault->wqe.wq_num = 1416 be32_to_cpu(pf_eqe->wqe.pftype_wq) & 1417 MLX5_24BIT_MASK; 1418 pfault->wqe.wqe_index = 1419 be16_to_cpu(pf_eqe->wqe.wqe_index); 1420 pfault->wqe.packet_size = 1421 be16_to_cpu(pf_eqe->wqe.packet_length); 1422 mlx5_ib_dbg(eq->dev, 1423 "PAGE_FAULT: type:0x%x, token: 0x%06x, wq_num: 0x%06x, wqe_index: 0x%04x\n", 1424 pfault->type, pfault->token, 1425 pfault->wqe.wq_num, 1426 pfault->wqe.wqe_index); 1427 break; 1428 1429 default: 1430 mlx5_ib_warn(eq->dev, 1431 "Unsupported page fault event sub-type: 0x%02hhx\n", 1432 eqe->sub_type); 1433 /* Unsupported page faults should still be 1434 * resolved by the page fault handler 1435 */ 1436 } 1437 1438 pfault->eq = eq; 1439 INIT_WORK(&pfault->work, mlx5_ib_eqe_pf_action); 1440 queue_work(eq->wq, &pfault->work); 1441 1442 cc = mlx5_eq_update_cc(eq->core, ++cc); 1443 } 1444 1445 mlx5_eq_update_ci(eq->core, cc, 1); 1446 } 1447 1448 static int mlx5_ib_eq_pf_int(struct notifier_block *nb, unsigned long type, 1449 void *data) 1450 { 1451 struct mlx5_ib_pf_eq *eq = 1452 container_of(nb, struct mlx5_ib_pf_eq, irq_nb); 1453 unsigned long flags; 1454 1455 if (spin_trylock_irqsave(&eq->lock, flags)) { 1456 mlx5_ib_eq_pf_process(eq); 1457 spin_unlock_irqrestore(&eq->lock, flags); 1458 } else { 1459 schedule_work(&eq->work); 1460 } 1461 1462 return IRQ_HANDLED; 1463 } 1464 1465 /* mempool_refill() was proposed but unfortunately wasn't accepted 1466 * http://lkml.iu.edu/hypermail/linux/kernel/1512.1/05073.html 1467 * Cheap workaround. 1468 */ 1469 static void mempool_refill(mempool_t *pool) 1470 { 1471 while (pool->curr_nr < pool->min_nr) 1472 mempool_free(mempool_alloc(pool, GFP_KERNEL), pool); 1473 } 1474 1475 static void mlx5_ib_eq_pf_action(struct work_struct *work) 1476 { 1477 struct mlx5_ib_pf_eq *eq = 1478 container_of(work, struct mlx5_ib_pf_eq, work); 1479 1480 mempool_refill(eq->pool); 1481 1482 spin_lock_irq(&eq->lock); 1483 mlx5_ib_eq_pf_process(eq); 1484 spin_unlock_irq(&eq->lock); 1485 } 1486 1487 enum { 1488 MLX5_IB_NUM_PF_EQE = 0x1000, 1489 MLX5_IB_NUM_PF_DRAIN = 64, 1490 }; 1491 1492 static int 1493 mlx5_ib_create_pf_eq(struct mlx5_ib_dev *dev, struct mlx5_ib_pf_eq *eq) 1494 { 1495 struct mlx5_eq_param param = {}; 1496 int err; 1497 1498 INIT_WORK(&eq->work, mlx5_ib_eq_pf_action); 1499 spin_lock_init(&eq->lock); 1500 eq->dev = dev; 1501 1502 eq->pool = mempool_create_kmalloc_pool(MLX5_IB_NUM_PF_DRAIN, 1503 sizeof(struct mlx5_pagefault)); 1504 if (!eq->pool) 1505 return -ENOMEM; 1506 1507 eq->wq = alloc_workqueue("mlx5_ib_page_fault", 1508 WQ_HIGHPRI | WQ_UNBOUND | WQ_MEM_RECLAIM, 1509 MLX5_NUM_CMD_EQE); 1510 if (!eq->wq) { 1511 err = -ENOMEM; 1512 goto err_mempool; 1513 } 1514 1515 eq->irq_nb.notifier_call = mlx5_ib_eq_pf_int; 1516 param = (struct mlx5_eq_param) { 1517 .irq_index = 0, 1518 .nent = MLX5_IB_NUM_PF_EQE, 1519 }; 1520 param.mask[0] = 1ull << MLX5_EVENT_TYPE_PAGE_FAULT; 1521 eq->core = mlx5_eq_create_generic(dev->mdev, ¶m); 1522 if (IS_ERR(eq->core)) { 1523 err = PTR_ERR(eq->core); 1524 goto err_wq; 1525 } 1526 err = mlx5_eq_enable(dev->mdev, eq->core, &eq->irq_nb); 1527 if (err) { 1528 mlx5_ib_err(dev, "failed to enable odp EQ %d\n", err); 1529 goto err_eq; 1530 } 1531 1532 return 0; 1533 err_eq: 1534 mlx5_eq_destroy_generic(dev->mdev, eq->core); 1535 err_wq: 1536 destroy_workqueue(eq->wq); 1537 err_mempool: 1538 mempool_destroy(eq->pool); 1539 return err; 1540 } 1541 1542 static int 1543 mlx5_ib_destroy_pf_eq(struct mlx5_ib_dev *dev, struct mlx5_ib_pf_eq *eq) 1544 { 1545 int err; 1546 1547 mlx5_eq_disable(dev->mdev, eq->core, &eq->irq_nb); 1548 err = mlx5_eq_destroy_generic(dev->mdev, eq->core); 1549 cancel_work_sync(&eq->work); 1550 destroy_workqueue(eq->wq); 1551 mempool_destroy(eq->pool); 1552 1553 return err; 1554 } 1555 1556 void mlx5_odp_init_mr_cache_entry(struct mlx5_cache_ent *ent) 1557 { 1558 if (!(ent->dev->odp_caps.general_caps & IB_ODP_SUPPORT_IMPLICIT)) 1559 return; 1560 1561 switch (ent->order - 2) { 1562 case MLX5_IMR_MTT_CACHE_ENTRY: 1563 ent->page = PAGE_SHIFT; 1564 ent->xlt = MLX5_IMR_MTT_ENTRIES * 1565 sizeof(struct mlx5_mtt) / 1566 MLX5_IB_UMR_OCTOWORD; 1567 ent->access_mode = MLX5_MKC_ACCESS_MODE_MTT; 1568 ent->limit = 0; 1569 break; 1570 1571 case MLX5_IMR_KSM_CACHE_ENTRY: 1572 ent->page = MLX5_KSM_PAGE_SHIFT; 1573 ent->xlt = mlx5_imr_ksm_entries * 1574 sizeof(struct mlx5_klm) / 1575 MLX5_IB_UMR_OCTOWORD; 1576 ent->access_mode = MLX5_MKC_ACCESS_MODE_KSM; 1577 ent->limit = 0; 1578 break; 1579 } 1580 } 1581 1582 static const struct ib_device_ops mlx5_ib_dev_odp_ops = { 1583 .advise_mr = mlx5_ib_advise_mr, 1584 .invalidate_range = mlx5_ib_invalidate_range, 1585 }; 1586 1587 int mlx5_ib_odp_init_one(struct mlx5_ib_dev *dev) 1588 { 1589 int ret = 0; 1590 1591 if (!(dev->odp_caps.general_caps & IB_ODP_SUPPORT)) 1592 return ret; 1593 1594 ib_set_device_ops(&dev->ib_dev, &mlx5_ib_dev_odp_ops); 1595 1596 if (dev->odp_caps.general_caps & IB_ODP_SUPPORT_IMPLICIT) { 1597 ret = mlx5_cmd_null_mkey(dev->mdev, &dev->null_mkey); 1598 if (ret) { 1599 mlx5_ib_err(dev, "Error getting null_mkey %d\n", ret); 1600 return ret; 1601 } 1602 } 1603 1604 ret = mlx5_ib_create_pf_eq(dev, &dev->odp_pf_eq); 1605 1606 return ret; 1607 } 1608 1609 void mlx5_ib_odp_cleanup_one(struct mlx5_ib_dev *dev) 1610 { 1611 if (!(dev->odp_caps.general_caps & IB_ODP_SUPPORT)) 1612 return; 1613 1614 mlx5_ib_destroy_pf_eq(dev, &dev->odp_pf_eq); 1615 } 1616 1617 int mlx5_ib_odp_init(void) 1618 { 1619 mlx5_imr_ksm_entries = BIT_ULL(get_order(TASK_SIZE) - 1620 MLX5_IMR_MTT_BITS); 1621 1622 return 0; 1623 } 1624 1625 struct prefetch_mr_work { 1626 struct work_struct work; 1627 struct ib_pd *pd; 1628 u32 pf_flags; 1629 u32 num_sge; 1630 struct ib_sge sg_list[0]; 1631 }; 1632 1633 static void num_pending_prefetch_dec(struct mlx5_ib_dev *dev, 1634 struct ib_sge *sg_list, u32 num_sge, 1635 u32 from) 1636 { 1637 u32 i; 1638 int srcu_key; 1639 1640 srcu_key = srcu_read_lock(&dev->mr_srcu); 1641 1642 for (i = from; i < num_sge; ++i) { 1643 struct mlx5_core_mkey *mmkey; 1644 struct mlx5_ib_mr *mr; 1645 1646 mmkey = xa_load(&dev->mdev->priv.mkey_table, 1647 mlx5_base_mkey(sg_list[i].lkey)); 1648 mr = container_of(mmkey, struct mlx5_ib_mr, mmkey); 1649 atomic_dec(&mr->num_pending_prefetch); 1650 } 1651 1652 srcu_read_unlock(&dev->mr_srcu, srcu_key); 1653 } 1654 1655 static bool num_pending_prefetch_inc(struct ib_pd *pd, 1656 struct ib_sge *sg_list, u32 num_sge) 1657 { 1658 struct mlx5_ib_dev *dev = to_mdev(pd->device); 1659 bool ret = true; 1660 u32 i; 1661 1662 for (i = 0; i < num_sge; ++i) { 1663 struct mlx5_core_mkey *mmkey; 1664 struct mlx5_ib_mr *mr; 1665 1666 mmkey = xa_load(&dev->mdev->priv.mkey_table, 1667 mlx5_base_mkey(sg_list[i].lkey)); 1668 if (!mmkey || mmkey->key != sg_list[i].lkey) { 1669 ret = false; 1670 break; 1671 } 1672 1673 if (mmkey->type != MLX5_MKEY_MR) { 1674 ret = false; 1675 break; 1676 } 1677 1678 mr = container_of(mmkey, struct mlx5_ib_mr, mmkey); 1679 1680 if (!smp_load_acquire(&mr->live)) { 1681 ret = false; 1682 break; 1683 } 1684 1685 if (mr->ibmr.pd != pd) { 1686 ret = false; 1687 break; 1688 } 1689 1690 atomic_inc(&mr->num_pending_prefetch); 1691 } 1692 1693 if (!ret) 1694 num_pending_prefetch_dec(dev, sg_list, i, 0); 1695 1696 return ret; 1697 } 1698 1699 static int mlx5_ib_prefetch_sg_list(struct ib_pd *pd, u32 pf_flags, 1700 struct ib_sge *sg_list, u32 num_sge) 1701 { 1702 u32 i; 1703 int ret = 0; 1704 struct mlx5_ib_dev *dev = to_mdev(pd->device); 1705 1706 for (i = 0; i < num_sge; ++i) { 1707 struct ib_sge *sg = &sg_list[i]; 1708 int bytes_committed = 0; 1709 1710 ret = pagefault_single_data_segment(dev, pd, sg->lkey, sg->addr, 1711 sg->length, 1712 &bytes_committed, NULL, 1713 pf_flags); 1714 if (ret < 0) 1715 break; 1716 } 1717 1718 return ret < 0 ? ret : 0; 1719 } 1720 1721 static void mlx5_ib_prefetch_mr_work(struct work_struct *work) 1722 { 1723 struct prefetch_mr_work *w = 1724 container_of(work, struct prefetch_mr_work, work); 1725 1726 if (ib_device_try_get(w->pd->device)) { 1727 mlx5_ib_prefetch_sg_list(w->pd, w->pf_flags, w->sg_list, 1728 w->num_sge); 1729 ib_device_put(w->pd->device); 1730 } 1731 1732 num_pending_prefetch_dec(to_mdev(w->pd->device), w->sg_list, 1733 w->num_sge, 0); 1734 kvfree(w); 1735 } 1736 1737 int mlx5_ib_advise_mr_prefetch(struct ib_pd *pd, 1738 enum ib_uverbs_advise_mr_advice advice, 1739 u32 flags, struct ib_sge *sg_list, u32 num_sge) 1740 { 1741 struct mlx5_ib_dev *dev = to_mdev(pd->device); 1742 u32 pf_flags = MLX5_PF_FLAGS_PREFETCH; 1743 struct prefetch_mr_work *work; 1744 bool valid_req; 1745 int srcu_key; 1746 1747 if (advice == IB_UVERBS_ADVISE_MR_ADVICE_PREFETCH) 1748 pf_flags |= MLX5_PF_FLAGS_DOWNGRADE; 1749 1750 if (flags & IB_UVERBS_ADVISE_MR_FLAG_FLUSH) 1751 return mlx5_ib_prefetch_sg_list(pd, pf_flags, sg_list, 1752 num_sge); 1753 1754 work = kvzalloc(struct_size(work, sg_list, num_sge), GFP_KERNEL); 1755 if (!work) 1756 return -ENOMEM; 1757 1758 memcpy(work->sg_list, sg_list, num_sge * sizeof(struct ib_sge)); 1759 1760 /* It is guaranteed that the pd when work is executed is the pd when 1761 * work was queued since pd can't be destroyed while it holds MRs and 1762 * destroying a MR leads to flushing the workquque 1763 */ 1764 work->pd = pd; 1765 work->pf_flags = pf_flags; 1766 work->num_sge = num_sge; 1767 1768 INIT_WORK(&work->work, mlx5_ib_prefetch_mr_work); 1769 1770 srcu_key = srcu_read_lock(&dev->mr_srcu); 1771 1772 valid_req = num_pending_prefetch_inc(pd, sg_list, num_sge); 1773 if (valid_req) 1774 queue_work(system_unbound_wq, &work->work); 1775 else 1776 kvfree(work); 1777 1778 srcu_read_unlock(&dev->mr_srcu, srcu_key); 1779 1780 return valid_req ? 0 : -EINVAL; 1781 } 1782