1 /* 2 * Copyright (c) 2013-2015, Mellanox Technologies. All rights reserved. 3 * 4 * This software is available to you under a choice of one of two 5 * licenses. You may choose to be licensed under the terms of the GNU 6 * General Public License (GPL) Version 2, available from the file 7 * COPYING in the main directory of this source tree, or the 8 * OpenIB.org BSD license below: 9 * 10 * Redistribution and use in source and binary forms, with or 11 * without modification, are permitted provided that the following 12 * conditions are met: 13 * 14 * - Redistributions of source code must retain the above 15 * copyright notice, this list of conditions and the following 16 * disclaimer. 17 * 18 * - Redistributions in binary form must reproduce the above 19 * copyright notice, this list of conditions and the following 20 * disclaimer in the documentation and/or other materials 21 * provided with the distribution. 22 * 23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30 * SOFTWARE. 31 */ 32 33 #include <rdma/ib_umem_odp.h> 34 #include <linux/kernel.h> 35 #include <linux/dma-buf.h> 36 #include <linux/dma-resv.h> 37 #include <linux/hmm.h> 38 #include <linux/hmm-dma.h> 39 #include <linux/pci-p2pdma.h> 40 41 #include "mlx5_ib.h" 42 #include "cmd.h" 43 #include "umr.h" 44 #include "qp.h" 45 46 #include <linux/mlx5/eq.h> 47 48 /* Contains the details of a pagefault. */ 49 struct mlx5_pagefault { 50 u32 bytes_committed; 51 u64 token; 52 u8 event_subtype; 53 u8 type; 54 union { 55 /* Initiator or send message responder pagefault details. */ 56 struct { 57 /* Received packet size, only valid for responders. */ 58 u32 packet_size; 59 /* 60 * Number of resource holding WQE, depends on type. 61 */ 62 u32 wq_num; 63 /* 64 * WQE index. Refers to either the send queue or 65 * receive queue, according to event_subtype. 66 */ 67 u16 wqe_index; 68 } wqe; 69 /* RDMA responder pagefault details */ 70 struct { 71 u32 r_key; 72 /* 73 * Received packet size, minimal size page fault 74 * resolution required for forward progress. 75 */ 76 u32 packet_size; 77 u32 rdma_op_len; 78 u64 rdma_va; 79 } rdma; 80 struct { 81 u64 va; 82 u32 mkey; 83 u32 fault_byte_count; 84 u32 prefetch_before_byte_count; 85 u32 prefetch_after_byte_count; 86 u8 flags; 87 } memory; 88 }; 89 90 struct mlx5_ib_pf_eq *eq; 91 struct work_struct work; 92 }; 93 94 #define MAX_PREFETCH_LEN (4*1024*1024U) 95 96 /* Timeout in ms to wait for an active mmu notifier to complete when handling 97 * a pagefault. */ 98 #define MMU_NOTIFIER_TIMEOUT 1000 99 100 #define MLX5_IMR_MTT_BITS (30 - PAGE_SHIFT) 101 #define MLX5_IMR_MTT_SHIFT (MLX5_IMR_MTT_BITS + PAGE_SHIFT) 102 #define MLX5_IMR_MTT_ENTRIES BIT_ULL(MLX5_IMR_MTT_BITS) 103 #define MLX5_IMR_MTT_SIZE BIT_ULL(MLX5_IMR_MTT_SHIFT) 104 #define MLX5_IMR_MTT_MASK (~(MLX5_IMR_MTT_SIZE - 1)) 105 106 #define MLX5_KSM_PAGE_SHIFT MLX5_IMR_MTT_SHIFT 107 108 static u64 mlx5_imr_ksm_entries; 109 110 static void populate_klm(struct mlx5_klm *pklm, size_t idx, size_t nentries, 111 struct mlx5_ib_mr *imr, int flags) 112 { 113 struct mlx5_core_dev *dev = mr_to_mdev(imr)->mdev; 114 struct mlx5_klm *end = pklm + nentries; 115 int step = MLX5_CAP_ODP(dev, mem_page_fault) ? MLX5_IMR_MTT_SIZE : 0; 116 __be32 key = MLX5_CAP_ODP(dev, mem_page_fault) ? 117 cpu_to_be32(imr->null_mmkey.key) : 118 mr_to_mdev(imr)->mkeys.null_mkey; 119 u64 va = 120 MLX5_CAP_ODP(dev, mem_page_fault) ? idx * MLX5_IMR_MTT_SIZE : 0; 121 122 if (flags & MLX5_IB_UPD_XLT_ZAP) { 123 for (; pklm != end; pklm++, idx++, va += step) { 124 pklm->bcount = cpu_to_be32(MLX5_IMR_MTT_SIZE); 125 pklm->key = key; 126 pklm->va = cpu_to_be64(va); 127 } 128 return; 129 } 130 131 /* 132 * The locking here is pretty subtle. Ideally the implicit_children 133 * xarray would be protected by the umem_mutex, however that is not 134 * possible. Instead this uses a weaker update-then-lock pattern: 135 * 136 * xa_store() 137 * mutex_lock(umem_mutex) 138 * mlx5r_umr_update_xlt() 139 * mutex_unlock(umem_mutex) 140 * destroy lkey 141 * 142 * ie any change the xarray must be followed by the locked update_xlt 143 * before destroying. 144 * 145 * The umem_mutex provides the acquire/release semantic needed to make 146 * the xa_store() visible to a racing thread. 147 */ 148 lockdep_assert_held(&to_ib_umem_odp(imr->umem)->umem_mutex); 149 150 for (; pklm != end; pklm++, idx++, va += step) { 151 struct mlx5_ib_mr *mtt = xa_load(&imr->implicit_children, idx); 152 153 pklm->bcount = cpu_to_be32(MLX5_IMR_MTT_SIZE); 154 if (mtt) { 155 pklm->key = cpu_to_be32(mtt->ibmr.lkey); 156 pklm->va = cpu_to_be64(idx * MLX5_IMR_MTT_SIZE); 157 } else { 158 pklm->key = key; 159 pklm->va = cpu_to_be64(va); 160 } 161 } 162 } 163 164 static int populate_mtt(__be64 *pas, size_t start, size_t nentries, 165 struct mlx5_ib_mr *mr, int flags) 166 { 167 struct ib_umem_odp *odp = to_ib_umem_odp(mr->umem); 168 bool downgrade = flags & MLX5_IB_UPD_XLT_DOWNGRADE; 169 struct pci_p2pdma_map_state p2pdma_state = {}; 170 struct ib_device *dev = odp->umem.ibdev; 171 size_t i; 172 173 if (flags & MLX5_IB_UPD_XLT_ZAP) 174 return 0; 175 176 for (i = 0; i < nentries; i++) { 177 unsigned long pfn = odp->map.pfn_list[start + i]; 178 dma_addr_t dma_addr; 179 180 pfn = odp->map.pfn_list[start + i]; 181 if (!(pfn & HMM_PFN_VALID)) 182 /* ODP initialization */ 183 continue; 184 185 dma_addr = hmm_dma_map_pfn(dev->dma_device, &odp->map, 186 start + i, &p2pdma_state); 187 if (ib_dma_mapping_error(dev, dma_addr)) 188 return -EFAULT; 189 190 dma_addr |= MLX5_IB_MTT_READ; 191 if ((pfn & HMM_PFN_WRITE) && !downgrade) 192 dma_addr |= MLX5_IB_MTT_WRITE; 193 194 pas[i] = cpu_to_be64(dma_addr); 195 odp->npages++; 196 } 197 return 0; 198 } 199 200 int mlx5_odp_populate_xlt(void *xlt, size_t idx, size_t nentries, 201 struct mlx5_ib_mr *mr, int flags) 202 { 203 if (flags & MLX5_IB_UPD_XLT_INDIRECT) { 204 populate_klm(xlt, idx, nentries, mr, flags); 205 return 0; 206 } else { 207 return populate_mtt(xlt, idx, nentries, mr, flags); 208 } 209 } 210 211 /* 212 * This must be called after the mr has been removed from implicit_children. 213 * NOTE: The MR does not necessarily have to be 214 * empty here, parallel page faults could have raced with the free process and 215 * added pages to it. 216 */ 217 static void free_implicit_child_mr_work(struct work_struct *work) 218 { 219 struct mlx5_ib_mr *mr = 220 container_of(work, struct mlx5_ib_mr, odp_destroy.work); 221 struct mlx5_ib_mr *imr = mr->parent; 222 struct ib_umem_odp *odp_imr = to_ib_umem_odp(imr->umem); 223 struct ib_umem_odp *odp = to_ib_umem_odp(mr->umem); 224 225 mlx5r_deref_wait_odp_mkey(&mr->mmkey); 226 227 mutex_lock(&odp_imr->umem_mutex); 228 mlx5r_umr_update_xlt(mr->parent, 229 ib_umem_start(odp) >> MLX5_IMR_MTT_SHIFT, 1, 0, 230 MLX5_IB_UPD_XLT_INDIRECT | MLX5_IB_UPD_XLT_ATOMIC); 231 mutex_unlock(&odp_imr->umem_mutex); 232 mlx5_ib_dereg_mr(&mr->ibmr, NULL); 233 234 mlx5r_deref_odp_mkey(&imr->mmkey); 235 } 236 237 static void destroy_unused_implicit_child_mr(struct mlx5_ib_mr *mr) 238 { 239 struct ib_umem_odp *odp = to_ib_umem_odp(mr->umem); 240 unsigned long idx = ib_umem_start(odp) >> MLX5_IMR_MTT_SHIFT; 241 struct mlx5_ib_mr *imr = mr->parent; 242 243 /* 244 * If userspace is racing freeing the parent implicit ODP MR then we can 245 * loose the race with parent destruction. In this case 246 * mlx5_ib_free_odp_mr() will free everything in the implicit_children 247 * xarray so NOP is fine. This child MR cannot be destroyed here because 248 * we are under its umem_mutex. 249 */ 250 if (!refcount_inc_not_zero(&imr->mmkey.usecount)) 251 return; 252 253 xa_lock(&imr->implicit_children); 254 if (__xa_cmpxchg(&imr->implicit_children, idx, mr, NULL, GFP_KERNEL) != 255 mr) { 256 xa_unlock(&imr->implicit_children); 257 mlx5r_deref_odp_mkey(&imr->mmkey); 258 return; 259 } 260 261 if (MLX5_CAP_ODP(mr_to_mdev(mr)->mdev, mem_page_fault)) 262 xa_erase(&mr_to_mdev(mr)->odp_mkeys, 263 mlx5_base_mkey(mr->mmkey.key)); 264 xa_unlock(&imr->implicit_children); 265 266 /* Freeing a MR is a sleeping operation, so bounce to a work queue */ 267 INIT_WORK(&mr->odp_destroy.work, free_implicit_child_mr_work); 268 queue_work(system_unbound_wq, &mr->odp_destroy.work); 269 } 270 271 static bool mlx5_ib_invalidate_range(struct mmu_interval_notifier *mni, 272 const struct mmu_notifier_range *range, 273 unsigned long cur_seq) 274 { 275 struct ib_umem_odp *umem_odp = 276 container_of(mni, struct ib_umem_odp, notifier); 277 struct mlx5_ib_mr *mr; 278 const u64 umr_block_mask = MLX5_UMR_MTT_NUM_ENTRIES_ALIGNMENT - 1; 279 u64 idx = 0, blk_start_idx = 0; 280 u64 invalidations = 0; 281 unsigned long start; 282 unsigned long end; 283 int in_block = 0; 284 u64 addr; 285 286 if (!mmu_notifier_range_blockable(range)) 287 return false; 288 289 mutex_lock(&umem_odp->umem_mutex); 290 mmu_interval_set_seq(mni, cur_seq); 291 /* 292 * If npages is zero then umem_odp->private may not be setup yet. This 293 * does not complete until after the first page is mapped for DMA. 294 */ 295 if (!umem_odp->npages) 296 goto out; 297 mr = umem_odp->private; 298 if (!mr) 299 goto out; 300 301 start = max_t(u64, ib_umem_start(umem_odp), range->start); 302 end = min_t(u64, ib_umem_end(umem_odp), range->end); 303 304 /* 305 * Iteration one - zap the HW's MTTs. The notifiers_count ensures that 306 * while we are doing the invalidation, no page fault will attempt to 307 * overwrite the same MTTs. Concurent invalidations might race us, 308 * but they will write 0s as well, so no difference in the end result. 309 */ 310 for (addr = start; addr < end; addr += BIT(umem_odp->page_shift)) { 311 idx = (addr - ib_umem_start(umem_odp)) >> umem_odp->page_shift; 312 /* 313 * Strive to write the MTTs in chunks, but avoid overwriting 314 * non-existing MTTs. The huristic here can be improved to 315 * estimate the cost of another UMR vs. the cost of bigger 316 * UMR. 317 */ 318 if (umem_odp->map.pfn_list[idx] & HMM_PFN_VALID) { 319 if (!in_block) { 320 blk_start_idx = idx; 321 in_block = 1; 322 } 323 } else { 324 u64 umr_offset = idx & umr_block_mask; 325 326 if (in_block && umr_offset == 0) { 327 mlx5r_umr_update_xlt(mr, blk_start_idx, 328 idx - blk_start_idx, 0, 329 MLX5_IB_UPD_XLT_ZAP | 330 MLX5_IB_UPD_XLT_ATOMIC); 331 in_block = 0; 332 /* Count page invalidations */ 333 invalidations += idx - blk_start_idx + 1; 334 } 335 } 336 } 337 if (in_block) { 338 mlx5r_umr_update_xlt(mr, blk_start_idx, 339 idx - blk_start_idx + 1, 0, 340 MLX5_IB_UPD_XLT_ZAP | 341 MLX5_IB_UPD_XLT_ATOMIC); 342 /* Count page invalidations */ 343 invalidations += idx - blk_start_idx + 1; 344 } 345 346 mlx5_update_odp_stats_with_handled(mr, invalidations, invalidations); 347 348 /* 349 * We are now sure that the device will not access the 350 * memory. We can safely unmap it, and mark it as dirty if 351 * needed. 352 */ 353 354 ib_umem_odp_unmap_dma_pages(umem_odp, start, end); 355 356 if (unlikely(!umem_odp->npages && mr->parent)) 357 destroy_unused_implicit_child_mr(mr); 358 out: 359 mutex_unlock(&umem_odp->umem_mutex); 360 return true; 361 } 362 363 const struct mmu_interval_notifier_ops mlx5_mn_ops = { 364 .invalidate = mlx5_ib_invalidate_range, 365 }; 366 367 static void internal_fill_odp_caps(struct mlx5_ib_dev *dev) 368 { 369 struct ib_odp_caps *caps = &dev->odp_caps; 370 371 memset(caps, 0, sizeof(*caps)); 372 373 if (!MLX5_CAP_GEN(dev->mdev, pg) || !mlx5r_umr_can_load_pas(dev, 0)) 374 return; 375 376 caps->general_caps = IB_ODP_SUPPORT; 377 378 if (MLX5_CAP_GEN(dev->mdev, umr_extended_translation_offset)) 379 dev->odp_max_size = U64_MAX; 380 else 381 dev->odp_max_size = BIT_ULL(MLX5_MAX_UMR_SHIFT + PAGE_SHIFT); 382 383 if (MLX5_CAP_ODP_SCHEME(dev->mdev, ud_odp_caps.send)) 384 caps->per_transport_caps.ud_odp_caps |= IB_ODP_SUPPORT_SEND; 385 386 if (MLX5_CAP_ODP_SCHEME(dev->mdev, ud_odp_caps.srq_receive)) 387 caps->per_transport_caps.ud_odp_caps |= IB_ODP_SUPPORT_SRQ_RECV; 388 389 if (MLX5_CAP_ODP_SCHEME(dev->mdev, rc_odp_caps.send)) 390 caps->per_transport_caps.rc_odp_caps |= IB_ODP_SUPPORT_SEND; 391 392 if (MLX5_CAP_ODP_SCHEME(dev->mdev, rc_odp_caps.receive)) 393 caps->per_transport_caps.rc_odp_caps |= IB_ODP_SUPPORT_RECV; 394 395 if (MLX5_CAP_ODP_SCHEME(dev->mdev, rc_odp_caps.write)) 396 caps->per_transport_caps.rc_odp_caps |= IB_ODP_SUPPORT_WRITE; 397 398 if (MLX5_CAP_ODP_SCHEME(dev->mdev, rc_odp_caps.read)) 399 caps->per_transport_caps.rc_odp_caps |= IB_ODP_SUPPORT_READ; 400 401 if (MLX5_CAP_ODP_SCHEME(dev->mdev, rc_odp_caps.atomic)) 402 caps->per_transport_caps.rc_odp_caps |= IB_ODP_SUPPORT_ATOMIC; 403 404 if (MLX5_CAP_ODP_SCHEME(dev->mdev, rc_odp_caps.srq_receive)) 405 caps->per_transport_caps.rc_odp_caps |= IB_ODP_SUPPORT_SRQ_RECV; 406 407 if (MLX5_CAP_ODP_SCHEME(dev->mdev, xrc_odp_caps.send)) 408 caps->per_transport_caps.xrc_odp_caps |= IB_ODP_SUPPORT_SEND; 409 410 if (MLX5_CAP_ODP_SCHEME(dev->mdev, xrc_odp_caps.receive)) 411 caps->per_transport_caps.xrc_odp_caps |= IB_ODP_SUPPORT_RECV; 412 413 if (MLX5_CAP_ODP_SCHEME(dev->mdev, xrc_odp_caps.write)) 414 caps->per_transport_caps.xrc_odp_caps |= IB_ODP_SUPPORT_WRITE; 415 416 if (MLX5_CAP_ODP_SCHEME(dev->mdev, xrc_odp_caps.read)) 417 caps->per_transport_caps.xrc_odp_caps |= IB_ODP_SUPPORT_READ; 418 419 if (MLX5_CAP_ODP_SCHEME(dev->mdev, xrc_odp_caps.atomic)) 420 caps->per_transport_caps.xrc_odp_caps |= IB_ODP_SUPPORT_ATOMIC; 421 422 if (MLX5_CAP_ODP_SCHEME(dev->mdev, xrc_odp_caps.srq_receive)) 423 caps->per_transport_caps.xrc_odp_caps |= IB_ODP_SUPPORT_SRQ_RECV; 424 425 if (MLX5_CAP_GEN(dev->mdev, fixed_buffer_size) && 426 MLX5_CAP_GEN(dev->mdev, null_mkey) && 427 MLX5_CAP_GEN(dev->mdev, umr_extended_translation_offset) && 428 !MLX5_CAP_GEN(dev->mdev, umr_indirect_mkey_disabled)) 429 caps->general_caps |= IB_ODP_SUPPORT_IMPLICIT; 430 } 431 432 static void mlx5_ib_page_fault_resume(struct mlx5_ib_dev *dev, 433 struct mlx5_pagefault *pfault, 434 int error) 435 { 436 int wq_num = pfault->event_subtype == MLX5_PFAULT_SUBTYPE_WQE ? 437 pfault->wqe.wq_num : pfault->token; 438 u32 in[MLX5_ST_SZ_DW(page_fault_resume_in)] = {}; 439 void *info; 440 int err; 441 442 MLX5_SET(page_fault_resume_in, in, opcode, MLX5_CMD_OP_PAGE_FAULT_RESUME); 443 444 if (pfault->event_subtype == MLX5_PFAULT_SUBTYPE_MEMORY) { 445 info = MLX5_ADDR_OF(page_fault_resume_in, in, 446 page_fault_info.mem_page_fault_info); 447 MLX5_SET(mem_page_fault_info, info, fault_token_31_0, 448 pfault->token & 0xffffffff); 449 MLX5_SET(mem_page_fault_info, info, fault_token_47_32, 450 (pfault->token >> 32) & 0xffff); 451 MLX5_SET(mem_page_fault_info, info, error, !!error); 452 } else { 453 info = MLX5_ADDR_OF(page_fault_resume_in, in, 454 page_fault_info.trans_page_fault_info); 455 MLX5_SET(trans_page_fault_info, info, page_fault_type, 456 pfault->type); 457 MLX5_SET(trans_page_fault_info, info, fault_token, 458 pfault->token); 459 MLX5_SET(trans_page_fault_info, info, wq_number, wq_num); 460 MLX5_SET(trans_page_fault_info, info, error, !!error); 461 } 462 463 err = mlx5_cmd_exec_in(dev->mdev, page_fault_resume, in); 464 if (err) 465 mlx5_ib_err(dev, "Failed to resolve the page fault on WQ 0x%x err %d\n", 466 wq_num, err); 467 } 468 469 static struct mlx5_ib_mr *implicit_get_child_mr(struct mlx5_ib_mr *imr, 470 unsigned long idx) 471 { 472 struct mlx5_ib_dev *dev = mr_to_mdev(imr); 473 struct ib_umem_odp *odp; 474 struct mlx5_ib_mr *mr; 475 struct mlx5_ib_mr *ret; 476 int err; 477 478 odp = ib_umem_odp_alloc_child(to_ib_umem_odp(imr->umem), 479 idx * MLX5_IMR_MTT_SIZE, 480 MLX5_IMR_MTT_SIZE, &mlx5_mn_ops); 481 if (IS_ERR(odp)) 482 return ERR_CAST(odp); 483 484 mr = mlx5_mr_cache_alloc(dev, imr->access_flags, 485 MLX5_MKC_ACCESS_MODE_MTT, 486 MLX5_IMR_MTT_ENTRIES); 487 if (IS_ERR(mr)) { 488 ib_umem_odp_release(odp); 489 return mr; 490 } 491 492 mr->access_flags = imr->access_flags; 493 mr->ibmr.pd = imr->ibmr.pd; 494 mr->ibmr.device = &mr_to_mdev(imr)->ib_dev; 495 mr->umem = &odp->umem; 496 mr->ibmr.lkey = mr->mmkey.key; 497 mr->ibmr.rkey = mr->mmkey.key; 498 mr->ibmr.iova = idx * MLX5_IMR_MTT_SIZE; 499 mr->parent = imr; 500 odp->private = mr; 501 502 /* 503 * First refcount is owned by the xarray and second refconut 504 * is returned to the caller. 505 */ 506 refcount_set(&mr->mmkey.usecount, 2); 507 508 err = mlx5r_umr_update_xlt(mr, 0, 509 MLX5_IMR_MTT_ENTRIES, 510 PAGE_SHIFT, 511 MLX5_IB_UPD_XLT_ZAP | 512 MLX5_IB_UPD_XLT_ENABLE); 513 if (err) { 514 ret = ERR_PTR(err); 515 goto out_mr; 516 } 517 518 xa_lock(&imr->implicit_children); 519 ret = __xa_cmpxchg(&imr->implicit_children, idx, NULL, mr, 520 GFP_KERNEL); 521 if (unlikely(ret)) { 522 if (xa_is_err(ret)) { 523 ret = ERR_PTR(xa_err(ret)); 524 goto out_lock; 525 } 526 /* 527 * Another thread beat us to creating the child mr, use 528 * theirs. 529 */ 530 refcount_inc(&ret->mmkey.usecount); 531 goto out_lock; 532 } 533 534 if (MLX5_CAP_ODP(dev->mdev, mem_page_fault)) { 535 ret = xa_store(&dev->odp_mkeys, mlx5_base_mkey(mr->mmkey.key), 536 &mr->mmkey, GFP_KERNEL); 537 if (xa_is_err(ret)) { 538 ret = ERR_PTR(xa_err(ret)); 539 __xa_erase(&imr->implicit_children, idx); 540 goto out_lock; 541 } 542 mr->mmkey.type = MLX5_MKEY_IMPLICIT_CHILD; 543 } 544 xa_unlock(&imr->implicit_children); 545 mlx5_ib_dbg(mr_to_mdev(imr), "key %x mr %p\n", mr->mmkey.key, mr); 546 return mr; 547 548 out_lock: 549 xa_unlock(&imr->implicit_children); 550 out_mr: 551 mlx5_ib_dereg_mr(&mr->ibmr, NULL); 552 return ret; 553 } 554 555 /* 556 * When using memory scheme ODP, implicit MRs can't use the reserved null mkey 557 * and each implicit MR needs to assign a private null mkey to get the page 558 * faults on. 559 * The null mkey is created with the properties to enable getting the page 560 * fault for every time it is accessed and having all relevant access flags. 561 */ 562 static int alloc_implicit_mr_null_mkey(struct mlx5_ib_dev *dev, 563 struct mlx5_ib_mr *imr, 564 struct mlx5_ib_pd *pd) 565 { 566 size_t inlen = MLX5_ST_SZ_BYTES(create_mkey_in) + 64; 567 void *mkc; 568 u32 *in; 569 int err; 570 571 in = kzalloc(inlen, GFP_KERNEL); 572 if (!in) 573 return -ENOMEM; 574 575 MLX5_SET(create_mkey_in, in, translations_octword_actual_size, 4); 576 MLX5_SET(create_mkey_in, in, pg_access, 1); 577 578 mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry); 579 MLX5_SET(mkc, mkc, a, 1); 580 MLX5_SET(mkc, mkc, rw, 1); 581 MLX5_SET(mkc, mkc, rr, 1); 582 MLX5_SET(mkc, mkc, lw, 1); 583 MLX5_SET(mkc, mkc, lr, 1); 584 MLX5_SET(mkc, mkc, free, 0); 585 MLX5_SET(mkc, mkc, umr_en, 0); 586 MLX5_SET(mkc, mkc, access_mode_1_0, MLX5_MKC_ACCESS_MODE_MTT); 587 588 MLX5_SET(mkc, mkc, translations_octword_size, 4); 589 MLX5_SET(mkc, mkc, log_page_size, 61); 590 MLX5_SET(mkc, mkc, length64, 1); 591 MLX5_SET(mkc, mkc, pd, pd->pdn); 592 MLX5_SET64(mkc, mkc, start_addr, 0); 593 MLX5_SET(mkc, mkc, qpn, 0xffffff); 594 595 err = mlx5_core_create_mkey(dev->mdev, &imr->null_mmkey.key, in, inlen); 596 if (err) 597 goto free_in; 598 599 imr->null_mmkey.type = MLX5_MKEY_NULL; 600 601 free_in: 602 kfree(in); 603 return err; 604 } 605 606 struct mlx5_ib_mr *mlx5_ib_alloc_implicit_mr(struct mlx5_ib_pd *pd, 607 int access_flags) 608 { 609 struct mlx5_ib_dev *dev = to_mdev(pd->ibpd.device); 610 struct ib_umem_odp *umem_odp; 611 struct mlx5_ib_mr *imr; 612 int err; 613 614 if (!mlx5r_umr_can_load_pas(dev, MLX5_IMR_MTT_ENTRIES * PAGE_SIZE)) 615 return ERR_PTR(-EOPNOTSUPP); 616 617 umem_odp = ib_umem_odp_alloc_implicit(&dev->ib_dev, access_flags); 618 if (IS_ERR(umem_odp)) 619 return ERR_CAST(umem_odp); 620 621 imr = mlx5_mr_cache_alloc(dev, access_flags, MLX5_MKC_ACCESS_MODE_KSM, 622 mlx5_imr_ksm_entries); 623 if (IS_ERR(imr)) { 624 ib_umem_odp_release(umem_odp); 625 return imr; 626 } 627 628 imr->access_flags = access_flags; 629 imr->ibmr.pd = &pd->ibpd; 630 imr->ibmr.iova = 0; 631 imr->umem = &umem_odp->umem; 632 imr->ibmr.lkey = imr->mmkey.key; 633 imr->ibmr.rkey = imr->mmkey.key; 634 imr->ibmr.device = &dev->ib_dev; 635 imr->is_odp_implicit = true; 636 xa_init(&imr->implicit_children); 637 638 if (MLX5_CAP_ODP(dev->mdev, mem_page_fault)) { 639 err = alloc_implicit_mr_null_mkey(dev, imr, pd); 640 if (err) 641 goto out_mr; 642 643 err = mlx5r_store_odp_mkey(dev, &imr->null_mmkey); 644 if (err) 645 goto out_mr; 646 } 647 648 err = mlx5r_umr_update_xlt(imr, 0, 649 mlx5_imr_ksm_entries, 650 MLX5_KSM_PAGE_SHIFT, 651 MLX5_IB_UPD_XLT_INDIRECT | 652 MLX5_IB_UPD_XLT_ZAP | 653 MLX5_IB_UPD_XLT_ENABLE); 654 if (err) 655 goto out_mr; 656 657 err = mlx5r_store_odp_mkey(dev, &imr->mmkey); 658 if (err) 659 goto out_mr; 660 661 mlx5_ib_dbg(dev, "key %x mr %p\n", imr->mmkey.key, imr); 662 return imr; 663 out_mr: 664 mlx5_ib_err(dev, "Failed to register MKEY %d\n", err); 665 mlx5_ib_dereg_mr(&imr->ibmr, NULL); 666 return ERR_PTR(err); 667 } 668 669 void mlx5_ib_free_odp_mr(struct mlx5_ib_mr *mr) 670 { 671 struct mlx5_ib_mr *mtt; 672 unsigned long idx; 673 674 /* 675 * If this is an implicit MR it is already invalidated so we can just 676 * delete the children mkeys. 677 */ 678 xa_for_each(&mr->implicit_children, idx, mtt) { 679 xa_erase(&mr->implicit_children, idx); 680 mlx5_ib_dereg_mr(&mtt->ibmr, NULL); 681 } 682 683 if (mr->null_mmkey.key) { 684 xa_erase(&mr_to_mdev(mr)->odp_mkeys, 685 mlx5_base_mkey(mr->null_mmkey.key)); 686 687 mlx5_core_destroy_mkey(mr_to_mdev(mr)->mdev, 688 mr->null_mmkey.key); 689 } 690 } 691 692 #define MLX5_PF_FLAGS_DOWNGRADE BIT(1) 693 #define MLX5_PF_FLAGS_SNAPSHOT BIT(2) 694 #define MLX5_PF_FLAGS_ENABLE BIT(3) 695 static int pagefault_real_mr(struct mlx5_ib_mr *mr, struct ib_umem_odp *odp, 696 u64 user_va, size_t bcnt, u32 *bytes_mapped, 697 u32 flags) 698 { 699 int page_shift, ret, np; 700 bool downgrade = flags & MLX5_PF_FLAGS_DOWNGRADE; 701 u64 access_mask = 0; 702 u64 start_idx; 703 bool fault = !(flags & MLX5_PF_FLAGS_SNAPSHOT); 704 u32 xlt_flags = MLX5_IB_UPD_XLT_ATOMIC; 705 706 if (flags & MLX5_PF_FLAGS_ENABLE) 707 xlt_flags |= MLX5_IB_UPD_XLT_ENABLE; 708 709 if (flags & MLX5_PF_FLAGS_DOWNGRADE) 710 xlt_flags |= MLX5_IB_UPD_XLT_DOWNGRADE; 711 712 page_shift = odp->page_shift; 713 start_idx = (user_va - ib_umem_start(odp)) >> page_shift; 714 715 if (odp->umem.writable && !downgrade) 716 access_mask |= HMM_PFN_WRITE; 717 718 np = ib_umem_odp_map_dma_and_lock(odp, user_va, bcnt, access_mask, fault); 719 if (np < 0) 720 return np; 721 722 /* 723 * No need to check whether the MTTs really belong to this MR, since 724 * ib_umem_odp_map_dma_and_lock already checks this. 725 */ 726 ret = mlx5r_umr_update_xlt(mr, start_idx, np, page_shift, xlt_flags); 727 mutex_unlock(&odp->umem_mutex); 728 729 if (ret < 0) { 730 if (ret != -EAGAIN) 731 mlx5_ib_err(mr_to_mdev(mr), 732 "Failed to update mkey page tables\n"); 733 goto out; 734 } 735 736 if (bytes_mapped) { 737 u32 new_mappings = (np << page_shift) - 738 (user_va - round_down(user_va, 1 << page_shift)); 739 740 *bytes_mapped += min_t(u32, new_mappings, bcnt); 741 } 742 743 return np << (page_shift - PAGE_SHIFT); 744 745 out: 746 return ret; 747 } 748 749 static int pagefault_implicit_mr(struct mlx5_ib_mr *imr, 750 struct ib_umem_odp *odp_imr, u64 user_va, 751 size_t bcnt, u32 *bytes_mapped, u32 flags) 752 { 753 unsigned long end_idx = (user_va + bcnt - 1) >> MLX5_IMR_MTT_SHIFT; 754 unsigned long upd_start_idx = end_idx + 1; 755 unsigned long upd_len = 0; 756 unsigned long npages = 0; 757 int err; 758 int ret; 759 760 if (unlikely(user_va >= mlx5_imr_ksm_entries * MLX5_IMR_MTT_SIZE || 761 mlx5_imr_ksm_entries * MLX5_IMR_MTT_SIZE - user_va < bcnt)) 762 return -EFAULT; 763 764 /* Fault each child mr that intersects with our interval. */ 765 while (bcnt) { 766 unsigned long idx = user_va >> MLX5_IMR_MTT_SHIFT; 767 struct ib_umem_odp *umem_odp; 768 struct mlx5_ib_mr *mtt; 769 u64 len; 770 771 xa_lock(&imr->implicit_children); 772 mtt = xa_load(&imr->implicit_children, idx); 773 if (unlikely(!mtt)) { 774 xa_unlock(&imr->implicit_children); 775 mtt = implicit_get_child_mr(imr, idx); 776 if (IS_ERR(mtt)) { 777 ret = PTR_ERR(mtt); 778 goto out; 779 } 780 upd_start_idx = min(upd_start_idx, idx); 781 upd_len = idx - upd_start_idx + 1; 782 } else { 783 refcount_inc(&mtt->mmkey.usecount); 784 xa_unlock(&imr->implicit_children); 785 } 786 787 umem_odp = to_ib_umem_odp(mtt->umem); 788 len = min_t(u64, user_va + bcnt, ib_umem_end(umem_odp)) - 789 user_va; 790 791 ret = pagefault_real_mr(mtt, umem_odp, user_va, len, 792 bytes_mapped, flags); 793 794 mlx5r_deref_odp_mkey(&mtt->mmkey); 795 796 if (ret < 0) 797 goto out; 798 user_va += len; 799 bcnt -= len; 800 npages += ret; 801 } 802 803 ret = npages; 804 805 /* 806 * Any time the implicit_children are changed we must perform an 807 * update of the xlt before exiting to ensure the HW and the 808 * implicit_children remains synchronized. 809 */ 810 out: 811 if (likely(!upd_len)) 812 return ret; 813 814 /* 815 * Notice this is not strictly ordered right, the KSM is updated after 816 * the implicit_children is updated, so a parallel page fault could 817 * see a MR that is not yet visible in the KSM. This is similar to a 818 * parallel page fault seeing a MR that is being concurrently removed 819 * from the KSM. Both of these improbable situations are resolved 820 * safely by resuming the HW and then taking another page fault. The 821 * next pagefault handler will see the new information. 822 */ 823 mutex_lock(&odp_imr->umem_mutex); 824 err = mlx5r_umr_update_xlt(imr, upd_start_idx, upd_len, 0, 825 MLX5_IB_UPD_XLT_INDIRECT | 826 MLX5_IB_UPD_XLT_ATOMIC); 827 mutex_unlock(&odp_imr->umem_mutex); 828 if (err) { 829 mlx5_ib_err(mr_to_mdev(imr), "Failed to update PAS\n"); 830 return err; 831 } 832 return ret; 833 } 834 835 static int pagefault_dmabuf_mr(struct mlx5_ib_mr *mr, size_t bcnt, 836 u32 *bytes_mapped, u32 flags) 837 { 838 struct ib_umem_dmabuf *umem_dmabuf = to_ib_umem_dmabuf(mr->umem); 839 int access_mode = mr->data_direct ? MLX5_MKC_ACCESS_MODE_KSM : 840 MLX5_MKC_ACCESS_MODE_MTT; 841 unsigned int old_page_shift = mr->page_shift; 842 unsigned int page_shift; 843 unsigned long page_size; 844 u32 xlt_flags = 0; 845 int err; 846 847 if (flags & MLX5_PF_FLAGS_ENABLE) 848 xlt_flags |= MLX5_IB_UPD_XLT_ENABLE; 849 850 dma_resv_lock(umem_dmabuf->attach->dmabuf->resv, NULL); 851 err = ib_umem_dmabuf_map_pages(umem_dmabuf); 852 if (err) { 853 dma_resv_unlock(umem_dmabuf->attach->dmabuf->resv); 854 return err; 855 } 856 857 page_size = mlx5_umem_dmabuf_find_best_pgsz(umem_dmabuf, access_mode); 858 if (!page_size) { 859 ib_umem_dmabuf_unmap_pages(umem_dmabuf); 860 err = -EINVAL; 861 } else { 862 page_shift = order_base_2(page_size); 863 if (page_shift != mr->page_shift && mr->dmabuf_faulted) { 864 err = mlx5r_umr_dmabuf_update_pgsz(mr, xlt_flags, 865 page_shift); 866 } else { 867 mr->page_shift = page_shift; 868 if (mr->data_direct) 869 err = mlx5r_umr_update_data_direct_ksm_pas( 870 mr, xlt_flags); 871 else 872 err = mlx5r_umr_update_mr_pas(mr, 873 xlt_flags); 874 } 875 } 876 dma_resv_unlock(umem_dmabuf->attach->dmabuf->resv); 877 878 if (err) { 879 mr->page_shift = old_page_shift; 880 return err; 881 } 882 883 mr->dmabuf_faulted = 1; 884 885 if (bytes_mapped) 886 *bytes_mapped += bcnt; 887 888 return ib_umem_num_pages(mr->umem); 889 } 890 891 /* 892 * Returns: 893 * -EFAULT: The io_virt->bcnt is not within the MR, it covers pages that are 894 * not accessible, or the MR is no longer valid. 895 * -EAGAIN/-ENOMEM: The operation should be retried 896 * 897 * -EINVAL/others: General internal malfunction 898 * >0: Number of pages mapped 899 */ 900 static int pagefault_mr(struct mlx5_ib_mr *mr, u64 io_virt, size_t bcnt, 901 u32 *bytes_mapped, u32 flags, bool permissive_fault) 902 { 903 struct ib_umem_odp *odp = to_ib_umem_odp(mr->umem); 904 905 if (unlikely(io_virt < mr->ibmr.iova) && !permissive_fault) 906 return -EFAULT; 907 908 if (mr->umem->is_dmabuf) 909 return pagefault_dmabuf_mr(mr, bcnt, bytes_mapped, flags); 910 911 if (!odp->is_implicit_odp) { 912 u64 offset = io_virt < mr->ibmr.iova ? 0 : io_virt - mr->ibmr.iova; 913 u64 user_va; 914 915 if (check_add_overflow(offset, (u64)odp->umem.address, 916 &user_va)) 917 return -EFAULT; 918 919 if (permissive_fault) { 920 if (user_va < ib_umem_start(odp)) 921 user_va = ib_umem_start(odp); 922 if ((user_va + bcnt) > ib_umem_end(odp)) 923 bcnt = ib_umem_end(odp) - user_va; 924 } else if (unlikely(user_va >= ib_umem_end(odp) || 925 ib_umem_end(odp) - user_va < bcnt)) 926 return -EFAULT; 927 return pagefault_real_mr(mr, odp, user_va, bcnt, bytes_mapped, 928 flags); 929 } 930 return pagefault_implicit_mr(mr, odp, io_virt, bcnt, bytes_mapped, 931 flags); 932 } 933 934 int mlx5_ib_init_odp_mr(struct mlx5_ib_mr *mr) 935 { 936 int ret; 937 938 ret = pagefault_real_mr(mr, to_ib_umem_odp(mr->umem), mr->umem->address, 939 mr->umem->length, NULL, 940 MLX5_PF_FLAGS_SNAPSHOT | MLX5_PF_FLAGS_ENABLE); 941 return ret >= 0 ? 0 : ret; 942 } 943 944 int mlx5_ib_init_dmabuf_mr(struct mlx5_ib_mr *mr) 945 { 946 int ret; 947 948 ret = pagefault_dmabuf_mr(mr, mr->umem->length, NULL, 949 MLX5_PF_FLAGS_ENABLE); 950 951 return ret >= 0 ? 0 : ret; 952 } 953 954 struct pf_frame { 955 struct pf_frame *next; 956 u32 key; 957 u64 io_virt; 958 size_t bcnt; 959 int depth; 960 }; 961 962 static bool mkey_is_eq(struct mlx5_ib_mkey *mmkey, u32 key) 963 { 964 if (!mmkey) 965 return false; 966 if (mmkey->type == MLX5_MKEY_MW || 967 mmkey->type == MLX5_MKEY_INDIRECT_DEVX) 968 return mlx5_base_mkey(mmkey->key) == mlx5_base_mkey(key); 969 return mmkey->key == key; 970 } 971 972 static struct mlx5_ib_mkey *find_odp_mkey(struct mlx5_ib_dev *dev, u32 key) 973 { 974 struct mlx5_ib_mkey *mmkey; 975 976 xa_lock(&dev->odp_mkeys); 977 mmkey = xa_load(&dev->odp_mkeys, mlx5_base_mkey(key)); 978 if (!mmkey) { 979 mmkey = ERR_PTR(-ENOENT); 980 goto out; 981 } 982 if (!mkey_is_eq(mmkey, key)) { 983 mmkey = ERR_PTR(-EFAULT); 984 goto out; 985 } 986 refcount_inc(&mmkey->usecount); 987 out: 988 xa_unlock(&dev->odp_mkeys); 989 990 return mmkey; 991 } 992 993 /* 994 * Handle a single data segment in a page-fault WQE or RDMA region. 995 * 996 * Returns zero on success. The caller may continue to the next data segment. 997 * Can return the following error codes: 998 * -EAGAIN to designate a temporary error. The caller will abort handling the 999 * page fault and resolve it. 1000 * -EFAULT when there's an error mapping the requested pages. The caller will 1001 * abort the page fault handling. 1002 */ 1003 static int pagefault_single_data_segment(struct mlx5_ib_dev *dev, 1004 struct ib_pd *pd, u32 key, 1005 u64 io_virt, size_t bcnt, 1006 u32 *bytes_committed, 1007 u32 *bytes_mapped) 1008 { 1009 int ret, i, outlen, cur_outlen = 0, depth = 0, pages_in_range; 1010 struct pf_frame *head = NULL, *frame; 1011 struct mlx5_ib_mkey *mmkey; 1012 struct mlx5_ib_mr *mr; 1013 struct mlx5_klm *pklm; 1014 u32 *out = NULL; 1015 size_t offset; 1016 1017 io_virt += *bytes_committed; 1018 bcnt -= *bytes_committed; 1019 next_mr: 1020 mmkey = find_odp_mkey(dev, key); 1021 if (IS_ERR(mmkey)) { 1022 ret = PTR_ERR(mmkey); 1023 if (ret == -ENOENT) { 1024 mlx5_ib_dbg( 1025 dev, 1026 "skipping non ODP MR (lkey=0x%06x) in page fault handler.\n", 1027 key); 1028 if (bytes_mapped) 1029 *bytes_mapped += bcnt; 1030 /* 1031 * The user could specify a SGL with multiple lkeys and 1032 * only some of them are ODP. Treat the non-ODP ones as 1033 * fully faulted. 1034 */ 1035 ret = 0; 1036 } 1037 goto end; 1038 } 1039 1040 switch (mmkey->type) { 1041 case MLX5_MKEY_MR: 1042 mr = container_of(mmkey, struct mlx5_ib_mr, mmkey); 1043 1044 pages_in_range = (ALIGN(io_virt + bcnt, PAGE_SIZE) - 1045 (io_virt & PAGE_MASK)) >> 1046 PAGE_SHIFT; 1047 ret = pagefault_mr(mr, io_virt, bcnt, bytes_mapped, 0, false); 1048 if (ret < 0) 1049 goto end; 1050 1051 mlx5_update_odp_stats_with_handled(mr, faults, ret); 1052 1053 if (ret < pages_in_range) { 1054 ret = -EFAULT; 1055 goto end; 1056 } 1057 1058 ret = 0; 1059 break; 1060 1061 case MLX5_MKEY_MW: 1062 case MLX5_MKEY_INDIRECT_DEVX: 1063 if (depth >= MLX5_CAP_GEN(dev->mdev, max_indirection)) { 1064 mlx5_ib_dbg(dev, "indirection level exceeded\n"); 1065 ret = -EFAULT; 1066 goto end; 1067 } 1068 1069 outlen = MLX5_ST_SZ_BYTES(query_mkey_out) + 1070 sizeof(*pklm) * (mmkey->ndescs - 2); 1071 1072 if (outlen > cur_outlen) { 1073 kfree(out); 1074 out = kzalloc(outlen, GFP_KERNEL); 1075 if (!out) { 1076 ret = -ENOMEM; 1077 goto end; 1078 } 1079 cur_outlen = outlen; 1080 } 1081 1082 pklm = (struct mlx5_klm *)MLX5_ADDR_OF(query_mkey_out, out, 1083 bsf0_klm0_pas_mtt0_1); 1084 1085 ret = mlx5_core_query_mkey(dev->mdev, mmkey->key, out, outlen); 1086 if (ret) 1087 goto end; 1088 1089 offset = io_virt - MLX5_GET64(query_mkey_out, out, 1090 memory_key_mkey_entry.start_addr); 1091 1092 for (i = 0; bcnt && i < mmkey->ndescs; i++, pklm++) { 1093 if (offset >= be32_to_cpu(pklm->bcount)) { 1094 offset -= be32_to_cpu(pklm->bcount); 1095 continue; 1096 } 1097 1098 frame = kzalloc(sizeof(*frame), GFP_KERNEL); 1099 if (!frame) { 1100 ret = -ENOMEM; 1101 goto end; 1102 } 1103 1104 frame->key = be32_to_cpu(pklm->key); 1105 frame->io_virt = be64_to_cpu(pklm->va) + offset; 1106 frame->bcnt = min_t(size_t, bcnt, 1107 be32_to_cpu(pklm->bcount) - offset); 1108 frame->depth = depth + 1; 1109 frame->next = head; 1110 head = frame; 1111 1112 bcnt -= frame->bcnt; 1113 offset = 0; 1114 } 1115 break; 1116 1117 default: 1118 mlx5_ib_dbg(dev, "wrong mkey type %d\n", mmkey->type); 1119 ret = -EFAULT; 1120 goto end; 1121 } 1122 1123 if (head) { 1124 frame = head; 1125 head = frame->next; 1126 1127 key = frame->key; 1128 io_virt = frame->io_virt; 1129 bcnt = frame->bcnt; 1130 depth = frame->depth; 1131 kfree(frame); 1132 1133 mlx5r_deref_odp_mkey(mmkey); 1134 goto next_mr; 1135 } 1136 1137 end: 1138 if (!IS_ERR(mmkey)) 1139 mlx5r_deref_odp_mkey(mmkey); 1140 while (head) { 1141 frame = head; 1142 head = frame->next; 1143 kfree(frame); 1144 } 1145 kfree(out); 1146 1147 *bytes_committed = 0; 1148 return ret; 1149 } 1150 1151 /* 1152 * Parse a series of data segments for page fault handling. 1153 * 1154 * @dev: Pointer to mlx5 IB device 1155 * @pfault: contains page fault information. 1156 * @wqe: points at the first data segment in the WQE. 1157 * @wqe_end: points after the end of the WQE. 1158 * @bytes_mapped: receives the number of bytes that the function was able to 1159 * map. This allows the caller to decide intelligently whether 1160 * enough memory was mapped to resolve the page fault 1161 * successfully (e.g. enough for the next MTU, or the entire 1162 * WQE). 1163 * @total_wqe_bytes: receives the total data size of this WQE in bytes (minus 1164 * the committed bytes). 1165 * @receive_queue: receive WQE end of sg list 1166 * 1167 * Returns zero for success or a negative error code. 1168 */ 1169 static int pagefault_data_segments(struct mlx5_ib_dev *dev, 1170 struct mlx5_pagefault *pfault, 1171 void *wqe, 1172 void *wqe_end, u32 *bytes_mapped, 1173 u32 *total_wqe_bytes, bool receive_queue) 1174 { 1175 int ret = 0; 1176 u64 io_virt; 1177 __be32 key; 1178 u32 byte_count; 1179 size_t bcnt; 1180 int inline_segment; 1181 1182 if (bytes_mapped) 1183 *bytes_mapped = 0; 1184 if (total_wqe_bytes) 1185 *total_wqe_bytes = 0; 1186 1187 while (wqe < wqe_end) { 1188 struct mlx5_wqe_data_seg *dseg = wqe; 1189 1190 io_virt = be64_to_cpu(dseg->addr); 1191 key = dseg->lkey; 1192 byte_count = be32_to_cpu(dseg->byte_count); 1193 inline_segment = !!(byte_count & MLX5_INLINE_SEG); 1194 bcnt = byte_count & ~MLX5_INLINE_SEG; 1195 1196 if (inline_segment) { 1197 bcnt = bcnt & MLX5_WQE_INLINE_SEG_BYTE_COUNT_MASK; 1198 wqe += ALIGN(sizeof(struct mlx5_wqe_inline_seg) + bcnt, 1199 16); 1200 } else { 1201 wqe += sizeof(*dseg); 1202 } 1203 1204 /* receive WQE end of sg list. */ 1205 if (receive_queue && bcnt == 0 && 1206 key == dev->mkeys.terminate_scatter_list_mkey && 1207 io_virt == 0) 1208 break; 1209 1210 if (!inline_segment && total_wqe_bytes) { 1211 *total_wqe_bytes += bcnt - min_t(size_t, bcnt, 1212 pfault->bytes_committed); 1213 } 1214 1215 /* A zero length data segment designates a length of 2GB. */ 1216 if (bcnt == 0) 1217 bcnt = 1U << 31; 1218 1219 if (inline_segment || bcnt <= pfault->bytes_committed) { 1220 pfault->bytes_committed -= 1221 min_t(size_t, bcnt, 1222 pfault->bytes_committed); 1223 continue; 1224 } 1225 1226 ret = pagefault_single_data_segment(dev, NULL, be32_to_cpu(key), 1227 io_virt, bcnt, 1228 &pfault->bytes_committed, 1229 bytes_mapped); 1230 if (ret < 0) 1231 break; 1232 } 1233 1234 return ret; 1235 } 1236 1237 /* 1238 * Parse initiator WQE. Advances the wqe pointer to point at the 1239 * scatter-gather list, and set wqe_end to the end of the WQE. 1240 */ 1241 static int mlx5_ib_mr_initiator_pfault_handler( 1242 struct mlx5_ib_dev *dev, struct mlx5_pagefault *pfault, 1243 struct mlx5_ib_qp *qp, void **wqe, void **wqe_end, int wqe_length) 1244 { 1245 struct mlx5_wqe_ctrl_seg *ctrl = *wqe; 1246 u16 wqe_index = pfault->wqe.wqe_index; 1247 struct mlx5_base_av *av; 1248 unsigned ds, opcode; 1249 u32 qpn = qp->trans_qp.base.mqp.qpn; 1250 1251 ds = be32_to_cpu(ctrl->qpn_ds) & MLX5_WQE_CTRL_DS_MASK; 1252 if (ds * MLX5_WQE_DS_UNITS > wqe_length) { 1253 mlx5_ib_err(dev, "Unable to read the complete WQE. ds = 0x%x, ret = 0x%x\n", 1254 ds, wqe_length); 1255 return -EFAULT; 1256 } 1257 1258 if (ds == 0) { 1259 mlx5_ib_err(dev, "Got WQE with zero DS. wqe_index=%x, qpn=%x\n", 1260 wqe_index, qpn); 1261 return -EFAULT; 1262 } 1263 1264 *wqe_end = *wqe + ds * MLX5_WQE_DS_UNITS; 1265 *wqe += sizeof(*ctrl); 1266 1267 opcode = be32_to_cpu(ctrl->opmod_idx_opcode) & 1268 MLX5_WQE_CTRL_OPCODE_MASK; 1269 1270 if (qp->type == IB_QPT_XRC_INI) 1271 *wqe += sizeof(struct mlx5_wqe_xrc_seg); 1272 1273 if (qp->type == IB_QPT_UD || qp->type == MLX5_IB_QPT_DCI) { 1274 av = *wqe; 1275 if (av->dqp_dct & cpu_to_be32(MLX5_EXTENDED_UD_AV)) 1276 *wqe += sizeof(struct mlx5_av); 1277 else 1278 *wqe += sizeof(struct mlx5_base_av); 1279 } 1280 1281 switch (opcode) { 1282 case MLX5_OPCODE_RDMA_WRITE: 1283 case MLX5_OPCODE_RDMA_WRITE_IMM: 1284 case MLX5_OPCODE_RDMA_READ: 1285 *wqe += sizeof(struct mlx5_wqe_raddr_seg); 1286 break; 1287 case MLX5_OPCODE_ATOMIC_CS: 1288 case MLX5_OPCODE_ATOMIC_FA: 1289 *wqe += sizeof(struct mlx5_wqe_raddr_seg); 1290 *wqe += sizeof(struct mlx5_wqe_atomic_seg); 1291 break; 1292 } 1293 1294 return 0; 1295 } 1296 1297 /* 1298 * Parse responder WQE and set wqe_end to the end of the WQE. 1299 */ 1300 static int mlx5_ib_mr_responder_pfault_handler_srq(struct mlx5_ib_dev *dev, 1301 struct mlx5_ib_srq *srq, 1302 void **wqe, void **wqe_end, 1303 int wqe_length) 1304 { 1305 int wqe_size = 1 << srq->msrq.wqe_shift; 1306 1307 if (wqe_size > wqe_length) { 1308 mlx5_ib_err(dev, "Couldn't read all of the receive WQE's content\n"); 1309 return -EFAULT; 1310 } 1311 1312 *wqe_end = *wqe + wqe_size; 1313 *wqe += sizeof(struct mlx5_wqe_srq_next_seg); 1314 1315 return 0; 1316 } 1317 1318 static int mlx5_ib_mr_responder_pfault_handler_rq(struct mlx5_ib_dev *dev, 1319 struct mlx5_ib_qp *qp, 1320 void *wqe, void **wqe_end, 1321 int wqe_length) 1322 { 1323 struct mlx5_ib_wq *wq = &qp->rq; 1324 int wqe_size = 1 << wq->wqe_shift; 1325 1326 if (qp->flags_en & MLX5_QP_FLAG_SIGNATURE) { 1327 mlx5_ib_err(dev, "ODP fault with WQE signatures is not supported\n"); 1328 return -EFAULT; 1329 } 1330 1331 if (wqe_size > wqe_length) { 1332 mlx5_ib_err(dev, "Couldn't read all of the receive WQE's content\n"); 1333 return -EFAULT; 1334 } 1335 1336 *wqe_end = wqe + wqe_size; 1337 1338 return 0; 1339 } 1340 1341 static inline struct mlx5_core_rsc_common *odp_get_rsc(struct mlx5_ib_dev *dev, 1342 u32 wq_num, int pf_type) 1343 { 1344 struct mlx5_core_rsc_common *common = NULL; 1345 struct mlx5_core_srq *srq; 1346 1347 switch (pf_type) { 1348 case MLX5_WQE_PF_TYPE_RMP: 1349 srq = mlx5_cmd_get_srq(dev, wq_num); 1350 if (srq) 1351 common = &srq->common; 1352 break; 1353 case MLX5_WQE_PF_TYPE_REQ_SEND_OR_WRITE: 1354 case MLX5_WQE_PF_TYPE_RESP: 1355 case MLX5_WQE_PF_TYPE_REQ_READ_OR_ATOMIC: 1356 common = mlx5_core_res_hold(dev, wq_num, MLX5_RES_QP); 1357 break; 1358 default: 1359 break; 1360 } 1361 1362 return common; 1363 } 1364 1365 static inline struct mlx5_ib_qp *res_to_qp(struct mlx5_core_rsc_common *res) 1366 { 1367 struct mlx5_core_qp *mqp = (struct mlx5_core_qp *)res; 1368 1369 return to_mibqp(mqp); 1370 } 1371 1372 static inline struct mlx5_ib_srq *res_to_srq(struct mlx5_core_rsc_common *res) 1373 { 1374 struct mlx5_core_srq *msrq = 1375 container_of(res, struct mlx5_core_srq, common); 1376 1377 return to_mibsrq(msrq); 1378 } 1379 1380 static void mlx5_ib_mr_wqe_pfault_handler(struct mlx5_ib_dev *dev, 1381 struct mlx5_pagefault *pfault) 1382 { 1383 bool sq = pfault->type & MLX5_PFAULT_REQUESTOR; 1384 u16 wqe_index = pfault->wqe.wqe_index; 1385 void *wqe, *wqe_start = NULL, *wqe_end = NULL; 1386 u32 bytes_mapped, total_wqe_bytes; 1387 struct mlx5_core_rsc_common *res; 1388 int resume_with_error = 1; 1389 struct mlx5_ib_qp *qp; 1390 size_t bytes_copied; 1391 int ret = 0; 1392 1393 res = odp_get_rsc(dev, pfault->wqe.wq_num, pfault->type); 1394 if (!res) { 1395 mlx5_ib_dbg(dev, "wqe page fault for missing resource %d\n", pfault->wqe.wq_num); 1396 return; 1397 } 1398 1399 if (res->res != MLX5_RES_QP && res->res != MLX5_RES_SRQ && 1400 res->res != MLX5_RES_XSRQ) { 1401 mlx5_ib_err(dev, "wqe page fault for unsupported type %d\n", 1402 pfault->type); 1403 goto resolve_page_fault; 1404 } 1405 1406 wqe_start = (void *)__get_free_page(GFP_KERNEL); 1407 if (!wqe_start) { 1408 mlx5_ib_err(dev, "Error allocating memory for IO page fault handling.\n"); 1409 goto resolve_page_fault; 1410 } 1411 1412 wqe = wqe_start; 1413 qp = (res->res == MLX5_RES_QP) ? res_to_qp(res) : NULL; 1414 if (qp && sq) { 1415 ret = mlx5_ib_read_wqe_sq(qp, wqe_index, wqe, PAGE_SIZE, 1416 &bytes_copied); 1417 if (ret) 1418 goto read_user; 1419 ret = mlx5_ib_mr_initiator_pfault_handler( 1420 dev, pfault, qp, &wqe, &wqe_end, bytes_copied); 1421 } else if (qp && !sq) { 1422 ret = mlx5_ib_read_wqe_rq(qp, wqe_index, wqe, PAGE_SIZE, 1423 &bytes_copied); 1424 if (ret) 1425 goto read_user; 1426 ret = mlx5_ib_mr_responder_pfault_handler_rq( 1427 dev, qp, wqe, &wqe_end, bytes_copied); 1428 } else if (!qp) { 1429 struct mlx5_ib_srq *srq = res_to_srq(res); 1430 1431 ret = mlx5_ib_read_wqe_srq(srq, wqe_index, wqe, PAGE_SIZE, 1432 &bytes_copied); 1433 if (ret) 1434 goto read_user; 1435 ret = mlx5_ib_mr_responder_pfault_handler_srq( 1436 dev, srq, &wqe, &wqe_end, bytes_copied); 1437 } 1438 1439 if (ret < 0 || wqe >= wqe_end) 1440 goto resolve_page_fault; 1441 1442 ret = pagefault_data_segments(dev, pfault, wqe, wqe_end, &bytes_mapped, 1443 &total_wqe_bytes, !sq); 1444 if (ret == -EAGAIN) 1445 goto out; 1446 1447 if (ret < 0 || total_wqe_bytes > bytes_mapped) 1448 goto resolve_page_fault; 1449 1450 out: 1451 ret = 0; 1452 resume_with_error = 0; 1453 1454 read_user: 1455 if (ret) 1456 mlx5_ib_err( 1457 dev, 1458 "Failed reading a WQE following page fault, error %d, wqe_index %x, qpn %llx\n", 1459 ret, wqe_index, pfault->token); 1460 1461 resolve_page_fault: 1462 mlx5_ib_page_fault_resume(dev, pfault, resume_with_error); 1463 mlx5_ib_dbg(dev, "PAGE FAULT completed. QP 0x%x resume_with_error=%d, type: 0x%x\n", 1464 pfault->wqe.wq_num, resume_with_error, 1465 pfault->type); 1466 mlx5_core_res_put(res); 1467 free_page((unsigned long)wqe_start); 1468 } 1469 1470 static void mlx5_ib_mr_rdma_pfault_handler(struct mlx5_ib_dev *dev, 1471 struct mlx5_pagefault *pfault) 1472 { 1473 u64 address; 1474 u32 length; 1475 u32 prefetch_len = pfault->bytes_committed; 1476 int prefetch_activated = 0; 1477 u32 rkey = pfault->rdma.r_key; 1478 int ret; 1479 1480 /* The RDMA responder handler handles the page fault in two parts. 1481 * First it brings the necessary pages for the current packet 1482 * (and uses the pfault context), and then (after resuming the QP) 1483 * prefetches more pages. The second operation cannot use the pfault 1484 * context and therefore uses the dummy_pfault context allocated on 1485 * the stack */ 1486 pfault->rdma.rdma_va += pfault->bytes_committed; 1487 pfault->rdma.rdma_op_len -= min(pfault->bytes_committed, 1488 pfault->rdma.rdma_op_len); 1489 pfault->bytes_committed = 0; 1490 1491 address = pfault->rdma.rdma_va; 1492 length = pfault->rdma.rdma_op_len; 1493 1494 /* For some operations, the hardware cannot tell the exact message 1495 * length, and in those cases it reports zero. Use prefetch 1496 * logic. */ 1497 if (length == 0) { 1498 prefetch_activated = 1; 1499 length = pfault->rdma.packet_size; 1500 prefetch_len = min(MAX_PREFETCH_LEN, prefetch_len); 1501 } 1502 1503 ret = pagefault_single_data_segment(dev, NULL, rkey, address, length, 1504 &pfault->bytes_committed, NULL); 1505 if (ret == -EAGAIN) { 1506 /* We're racing with an invalidation, don't prefetch */ 1507 prefetch_activated = 0; 1508 } else if (ret < 0) { 1509 mlx5_ib_page_fault_resume(dev, pfault, 1); 1510 if (ret != -ENOENT) 1511 mlx5_ib_dbg(dev, "PAGE FAULT error %d. QP 0x%llx, type: 0x%x\n", 1512 ret, pfault->token, pfault->type); 1513 return; 1514 } 1515 1516 mlx5_ib_page_fault_resume(dev, pfault, 0); 1517 mlx5_ib_dbg(dev, "PAGE FAULT completed. QP 0x%llx, type: 0x%x, prefetch_activated: %d\n", 1518 pfault->token, pfault->type, 1519 prefetch_activated); 1520 1521 /* At this point, there might be a new pagefault already arriving in 1522 * the eq, switch to the dummy pagefault for the rest of the 1523 * processing. We're still OK with the objects being alive as the 1524 * work-queue is being fenced. */ 1525 1526 if (prefetch_activated) { 1527 u32 bytes_committed = 0; 1528 1529 ret = pagefault_single_data_segment(dev, NULL, rkey, address, 1530 prefetch_len, 1531 &bytes_committed, NULL); 1532 if (ret < 0 && ret != -EAGAIN) { 1533 mlx5_ib_dbg(dev, "Prefetch failed. ret: %d, QP 0x%llx, address: 0x%.16llx, length = 0x%.16x\n", 1534 ret, pfault->token, address, prefetch_len); 1535 } 1536 } 1537 } 1538 1539 #define MLX5_MEMORY_PAGE_FAULT_FLAGS_LAST BIT(7) 1540 static void mlx5_ib_mr_memory_pfault_handler(struct mlx5_ib_dev *dev, 1541 struct mlx5_pagefault *pfault) 1542 { 1543 u64 prefetch_va = 1544 pfault->memory.va - pfault->memory.prefetch_before_byte_count; 1545 size_t prefetch_size = pfault->memory.prefetch_before_byte_count + 1546 pfault->memory.fault_byte_count + 1547 pfault->memory.prefetch_after_byte_count; 1548 struct mlx5_ib_mkey *mmkey; 1549 struct mlx5_ib_mr *mr, *child_mr; 1550 int ret = 0; 1551 1552 mmkey = find_odp_mkey(dev, pfault->memory.mkey); 1553 if (IS_ERR(mmkey)) 1554 goto err; 1555 1556 switch (mmkey->type) { 1557 case MLX5_MKEY_IMPLICIT_CHILD: 1558 child_mr = container_of(mmkey, struct mlx5_ib_mr, mmkey); 1559 mr = child_mr->parent; 1560 break; 1561 case MLX5_MKEY_NULL: 1562 mr = container_of(mmkey, struct mlx5_ib_mr, null_mmkey); 1563 break; 1564 default: 1565 mr = container_of(mmkey, struct mlx5_ib_mr, mmkey); 1566 break; 1567 } 1568 1569 /* If prefetch fails, handle only demanded page fault */ 1570 ret = pagefault_mr(mr, prefetch_va, prefetch_size, NULL, 0, true); 1571 if (ret < 0) { 1572 ret = pagefault_mr(mr, pfault->memory.va, 1573 pfault->memory.fault_byte_count, NULL, 0, 1574 true); 1575 if (ret < 0) 1576 goto err; 1577 } 1578 1579 mlx5_update_odp_stats_with_handled(mr, faults, ret); 1580 mlx5r_deref_odp_mkey(mmkey); 1581 1582 if (pfault->memory.flags & MLX5_MEMORY_PAGE_FAULT_FLAGS_LAST) 1583 mlx5_ib_page_fault_resume(dev, pfault, 0); 1584 1585 mlx5_ib_dbg( 1586 dev, 1587 "PAGE FAULT completed %s. token 0x%llx, mkey: 0x%x, va: 0x%llx, byte_count: 0x%x\n", 1588 pfault->memory.flags & MLX5_MEMORY_PAGE_FAULT_FLAGS_LAST ? 1589 "" : 1590 "without resume cmd", 1591 pfault->token, pfault->memory.mkey, pfault->memory.va, 1592 pfault->memory.fault_byte_count); 1593 1594 return; 1595 1596 err: 1597 if (!IS_ERR(mmkey)) 1598 mlx5r_deref_odp_mkey(mmkey); 1599 mlx5_ib_page_fault_resume(dev, pfault, 1); 1600 mlx5_ib_dbg( 1601 dev, 1602 "PAGE FAULT error. token 0x%llx, mkey: 0x%x, va: 0x%llx, byte_count: 0x%x, err: %d\n", 1603 pfault->token, pfault->memory.mkey, pfault->memory.va, 1604 pfault->memory.fault_byte_count, ret); 1605 } 1606 1607 static void mlx5_ib_pfault(struct mlx5_ib_dev *dev, struct mlx5_pagefault *pfault) 1608 { 1609 u8 event_subtype = pfault->event_subtype; 1610 1611 switch (event_subtype) { 1612 case MLX5_PFAULT_SUBTYPE_WQE: 1613 mlx5_ib_mr_wqe_pfault_handler(dev, pfault); 1614 break; 1615 case MLX5_PFAULT_SUBTYPE_RDMA: 1616 mlx5_ib_mr_rdma_pfault_handler(dev, pfault); 1617 break; 1618 case MLX5_PFAULT_SUBTYPE_MEMORY: 1619 mlx5_ib_mr_memory_pfault_handler(dev, pfault); 1620 break; 1621 default: 1622 mlx5_ib_err(dev, "Invalid page fault event subtype: 0x%x\n", 1623 event_subtype); 1624 mlx5_ib_page_fault_resume(dev, pfault, 1); 1625 } 1626 } 1627 1628 static void mlx5_ib_eqe_pf_action(struct work_struct *work) 1629 { 1630 struct mlx5_pagefault *pfault = container_of(work, 1631 struct mlx5_pagefault, 1632 work); 1633 struct mlx5_ib_pf_eq *eq = pfault->eq; 1634 1635 mlx5_ib_pfault(eq->dev, pfault); 1636 mempool_free(pfault, eq->pool); 1637 } 1638 1639 #define MEMORY_SCHEME_PAGE_FAULT_GRANULARITY 4096 1640 static void mlx5_ib_eq_pf_process(struct mlx5_ib_pf_eq *eq) 1641 { 1642 struct mlx5_eqe_page_fault *pf_eqe; 1643 struct mlx5_pagefault *pfault; 1644 struct mlx5_eqe *eqe; 1645 int cc = 0; 1646 1647 while ((eqe = mlx5_eq_get_eqe(eq->core, cc))) { 1648 pfault = mempool_alloc(eq->pool, GFP_ATOMIC); 1649 if (!pfault) { 1650 schedule_work(&eq->work); 1651 break; 1652 } 1653 1654 pf_eqe = &eqe->data.page_fault; 1655 pfault->event_subtype = eqe->sub_type; 1656 1657 switch (eqe->sub_type) { 1658 case MLX5_PFAULT_SUBTYPE_RDMA: 1659 /* RDMA based event */ 1660 pfault->bytes_committed = 1661 be32_to_cpu(pf_eqe->rdma.bytes_committed); 1662 pfault->type = 1663 be32_to_cpu(pf_eqe->rdma.pftype_token) >> 24; 1664 pfault->token = 1665 be32_to_cpu(pf_eqe->rdma.pftype_token) & 1666 MLX5_24BIT_MASK; 1667 pfault->rdma.r_key = 1668 be32_to_cpu(pf_eqe->rdma.r_key); 1669 pfault->rdma.packet_size = 1670 be16_to_cpu(pf_eqe->rdma.packet_length); 1671 pfault->rdma.rdma_op_len = 1672 be32_to_cpu(pf_eqe->rdma.rdma_op_len); 1673 pfault->rdma.rdma_va = 1674 be64_to_cpu(pf_eqe->rdma.rdma_va); 1675 mlx5_ib_dbg( 1676 eq->dev, 1677 "PAGE_FAULT: subtype: 0x%02x, bytes_committed: 0x%06x, type:0x%x, token: 0x%06llx, r_key: 0x%08x\n", 1678 eqe->sub_type, pfault->bytes_committed, 1679 pfault->type, pfault->token, 1680 pfault->rdma.r_key); 1681 mlx5_ib_dbg(eq->dev, 1682 "PAGE_FAULT: rdma_op_len: 0x%08x, rdma_va: 0x%016llx\n", 1683 pfault->rdma.rdma_op_len, 1684 pfault->rdma.rdma_va); 1685 break; 1686 1687 case MLX5_PFAULT_SUBTYPE_WQE: 1688 /* WQE based event */ 1689 pfault->bytes_committed = 1690 be32_to_cpu(pf_eqe->wqe.bytes_committed); 1691 pfault->type = 1692 (be32_to_cpu(pf_eqe->wqe.pftype_wq) >> 24) & 0x7; 1693 pfault->token = 1694 be32_to_cpu(pf_eqe->wqe.token); 1695 pfault->wqe.wq_num = 1696 be32_to_cpu(pf_eqe->wqe.pftype_wq) & 1697 MLX5_24BIT_MASK; 1698 pfault->wqe.wqe_index = 1699 be16_to_cpu(pf_eqe->wqe.wqe_index); 1700 pfault->wqe.packet_size = 1701 be16_to_cpu(pf_eqe->wqe.packet_length); 1702 mlx5_ib_dbg( 1703 eq->dev, 1704 "PAGE_FAULT: subtype: 0x%02x, bytes_committed: 0x%06x, type:0x%x, token: 0x%06llx, wq_num: 0x%06x, wqe_index: 0x%04x\n", 1705 eqe->sub_type, pfault->bytes_committed, 1706 pfault->type, pfault->token, pfault->wqe.wq_num, 1707 pfault->wqe.wqe_index); 1708 break; 1709 1710 case MLX5_PFAULT_SUBTYPE_MEMORY: 1711 /* Memory based event */ 1712 pfault->bytes_committed = 0; 1713 pfault->token = 1714 be32_to_cpu(pf_eqe->memory.token31_0) | 1715 ((u64)be16_to_cpu(pf_eqe->memory.token47_32) 1716 << 32); 1717 pfault->memory.va = be64_to_cpu(pf_eqe->memory.va); 1718 pfault->memory.mkey = be32_to_cpu(pf_eqe->memory.mkey); 1719 pfault->memory.fault_byte_count = (be32_to_cpu( 1720 pf_eqe->memory.demand_fault_pages) >> 12) * 1721 MEMORY_SCHEME_PAGE_FAULT_GRANULARITY; 1722 pfault->memory.prefetch_before_byte_count = 1723 be16_to_cpu( 1724 pf_eqe->memory.pre_demand_fault_pages) * 1725 MEMORY_SCHEME_PAGE_FAULT_GRANULARITY; 1726 pfault->memory.prefetch_after_byte_count = 1727 be16_to_cpu( 1728 pf_eqe->memory.post_demand_fault_pages) * 1729 MEMORY_SCHEME_PAGE_FAULT_GRANULARITY; 1730 pfault->memory.flags = pf_eqe->memory.flags; 1731 mlx5_ib_dbg( 1732 eq->dev, 1733 "PAGE_FAULT: subtype: 0x%02x, token: 0x%06llx, mkey: 0x%06x, fault_byte_count: 0x%06x, va: 0x%016llx, flags: 0x%02x\n", 1734 eqe->sub_type, pfault->token, 1735 pfault->memory.mkey, 1736 pfault->memory.fault_byte_count, 1737 pfault->memory.va, pfault->memory.flags); 1738 mlx5_ib_dbg( 1739 eq->dev, 1740 "PAGE_FAULT: prefetch size: before: 0x%06x, after 0x%06x\n", 1741 pfault->memory.prefetch_before_byte_count, 1742 pfault->memory.prefetch_after_byte_count); 1743 break; 1744 1745 default: 1746 mlx5_ib_warn(eq->dev, 1747 "Unsupported page fault event sub-type: 0x%02hhx\n", 1748 eqe->sub_type); 1749 /* Unsupported page faults should still be 1750 * resolved by the page fault handler 1751 */ 1752 } 1753 1754 pfault->eq = eq; 1755 INIT_WORK(&pfault->work, mlx5_ib_eqe_pf_action); 1756 queue_work(eq->wq, &pfault->work); 1757 1758 cc = mlx5_eq_update_cc(eq->core, ++cc); 1759 } 1760 1761 mlx5_eq_update_ci(eq->core, cc, 1); 1762 } 1763 1764 static int mlx5_ib_eq_pf_int(struct notifier_block *nb, unsigned long type, 1765 void *data) 1766 { 1767 struct mlx5_ib_pf_eq *eq = 1768 container_of(nb, struct mlx5_ib_pf_eq, irq_nb); 1769 unsigned long flags; 1770 1771 if (spin_trylock_irqsave(&eq->lock, flags)) { 1772 mlx5_ib_eq_pf_process(eq); 1773 spin_unlock_irqrestore(&eq->lock, flags); 1774 } else { 1775 schedule_work(&eq->work); 1776 } 1777 1778 return IRQ_HANDLED; 1779 } 1780 1781 /* mempool_refill() was proposed but unfortunately wasn't accepted 1782 * http://lkml.iu.edu/hypermail/linux/kernel/1512.1/05073.html 1783 * Cheap workaround. 1784 */ 1785 static void mempool_refill(mempool_t *pool) 1786 { 1787 while (pool->curr_nr < pool->min_nr) 1788 mempool_free(mempool_alloc(pool, GFP_KERNEL), pool); 1789 } 1790 1791 static void mlx5_ib_eq_pf_action(struct work_struct *work) 1792 { 1793 struct mlx5_ib_pf_eq *eq = 1794 container_of(work, struct mlx5_ib_pf_eq, work); 1795 1796 mempool_refill(eq->pool); 1797 1798 spin_lock_irq(&eq->lock); 1799 mlx5_ib_eq_pf_process(eq); 1800 spin_unlock_irq(&eq->lock); 1801 } 1802 1803 enum { 1804 MLX5_IB_NUM_PF_EQE = 0x1000, 1805 MLX5_IB_NUM_PF_DRAIN = 64, 1806 }; 1807 1808 int mlx5r_odp_create_eq(struct mlx5_ib_dev *dev, struct mlx5_ib_pf_eq *eq) 1809 { 1810 struct mlx5_eq_param param = {}; 1811 int err = 0; 1812 1813 mutex_lock(&dev->odp_eq_mutex); 1814 if (eq->core) 1815 goto unlock; 1816 INIT_WORK(&eq->work, mlx5_ib_eq_pf_action); 1817 spin_lock_init(&eq->lock); 1818 eq->dev = dev; 1819 1820 eq->pool = mempool_create_kmalloc_pool(MLX5_IB_NUM_PF_DRAIN, 1821 sizeof(struct mlx5_pagefault)); 1822 if (!eq->pool) { 1823 err = -ENOMEM; 1824 goto unlock; 1825 } 1826 1827 eq->wq = alloc_workqueue("mlx5_ib_page_fault", 1828 WQ_HIGHPRI | WQ_UNBOUND | WQ_MEM_RECLAIM, 1829 MLX5_NUM_CMD_EQE); 1830 if (!eq->wq) { 1831 err = -ENOMEM; 1832 goto err_mempool; 1833 } 1834 1835 eq->irq_nb.notifier_call = mlx5_ib_eq_pf_int; 1836 param = (struct mlx5_eq_param) { 1837 .nent = MLX5_IB_NUM_PF_EQE, 1838 }; 1839 param.mask[0] = 1ull << MLX5_EVENT_TYPE_PAGE_FAULT; 1840 eq->core = mlx5_eq_create_generic(dev->mdev, ¶m); 1841 if (IS_ERR(eq->core)) { 1842 err = PTR_ERR(eq->core); 1843 goto err_wq; 1844 } 1845 err = mlx5_eq_enable(dev->mdev, eq->core, &eq->irq_nb); 1846 if (err) { 1847 mlx5_ib_err(dev, "failed to enable odp EQ %d\n", err); 1848 goto err_eq; 1849 } 1850 1851 mutex_unlock(&dev->odp_eq_mutex); 1852 return 0; 1853 err_eq: 1854 mlx5_eq_destroy_generic(dev->mdev, eq->core); 1855 err_wq: 1856 eq->core = NULL; 1857 destroy_workqueue(eq->wq); 1858 err_mempool: 1859 mempool_destroy(eq->pool); 1860 unlock: 1861 mutex_unlock(&dev->odp_eq_mutex); 1862 return err; 1863 } 1864 1865 static int 1866 mlx5_ib_odp_destroy_eq(struct mlx5_ib_dev *dev, struct mlx5_ib_pf_eq *eq) 1867 { 1868 int err; 1869 1870 if (!eq->core) 1871 return 0; 1872 mlx5_eq_disable(dev->mdev, eq->core, &eq->irq_nb); 1873 err = mlx5_eq_destroy_generic(dev->mdev, eq->core); 1874 cancel_work_sync(&eq->work); 1875 destroy_workqueue(eq->wq); 1876 mempool_destroy(eq->pool); 1877 1878 return err; 1879 } 1880 1881 int mlx5_odp_init_mkey_cache(struct mlx5_ib_dev *dev) 1882 { 1883 struct mlx5r_cache_rb_key rb_key = { 1884 .access_mode = MLX5_MKC_ACCESS_MODE_KSM, 1885 .ndescs = mlx5_imr_ksm_entries, 1886 .ph = MLX5_IB_NO_PH, 1887 }; 1888 struct mlx5_cache_ent *ent; 1889 1890 if (!(dev->odp_caps.general_caps & IB_ODP_SUPPORT_IMPLICIT)) 1891 return 0; 1892 1893 ent = mlx5r_cache_create_ent_locked(dev, rb_key, true); 1894 if (IS_ERR(ent)) 1895 return PTR_ERR(ent); 1896 1897 return 0; 1898 } 1899 1900 static const struct ib_device_ops mlx5_ib_dev_odp_ops = { 1901 .advise_mr = mlx5_ib_advise_mr, 1902 }; 1903 1904 int mlx5_ib_odp_init_one(struct mlx5_ib_dev *dev) 1905 { 1906 internal_fill_odp_caps(dev); 1907 1908 if (!(dev->odp_caps.general_caps & IB_ODP_SUPPORT)) 1909 return 0; 1910 1911 ib_set_device_ops(&dev->ib_dev, &mlx5_ib_dev_odp_ops); 1912 1913 mutex_init(&dev->odp_eq_mutex); 1914 return 0; 1915 } 1916 1917 void mlx5_ib_odp_cleanup_one(struct mlx5_ib_dev *dev) 1918 { 1919 if (!(dev->odp_caps.general_caps & IB_ODP_SUPPORT)) 1920 return; 1921 1922 mlx5_ib_odp_destroy_eq(dev, &dev->odp_pf_eq); 1923 } 1924 1925 int mlx5_ib_odp_init(void) 1926 { 1927 mlx5_imr_ksm_entries = BIT_ULL(get_order(TASK_SIZE) - 1928 MLX5_IMR_MTT_BITS); 1929 1930 return 0; 1931 } 1932 1933 struct prefetch_mr_work { 1934 struct work_struct work; 1935 u32 pf_flags; 1936 u32 num_sge; 1937 struct { 1938 u64 io_virt; 1939 struct mlx5_ib_mr *mr; 1940 size_t length; 1941 } frags[]; 1942 }; 1943 1944 static void destroy_prefetch_work(struct prefetch_mr_work *work) 1945 { 1946 u32 i; 1947 1948 for (i = 0; i < work->num_sge; ++i) 1949 mlx5r_deref_odp_mkey(&work->frags[i].mr->mmkey); 1950 1951 kvfree(work); 1952 } 1953 1954 static struct mlx5_ib_mr * 1955 get_prefetchable_mr(struct ib_pd *pd, enum ib_uverbs_advise_mr_advice advice, 1956 u32 lkey) 1957 { 1958 struct mlx5_ib_dev *dev = to_mdev(pd->device); 1959 struct mlx5_ib_mr *mr = NULL; 1960 struct mlx5_ib_mkey *mmkey; 1961 1962 xa_lock(&dev->odp_mkeys); 1963 mmkey = xa_load(&dev->odp_mkeys, mlx5_base_mkey(lkey)); 1964 if (!mmkey || mmkey->key != lkey) { 1965 mr = ERR_PTR(-ENOENT); 1966 goto end; 1967 } 1968 if (mmkey->type != MLX5_MKEY_MR) { 1969 mr = ERR_PTR(-EINVAL); 1970 goto end; 1971 } 1972 1973 mr = container_of(mmkey, struct mlx5_ib_mr, mmkey); 1974 1975 if (mr->ibmr.pd != pd) { 1976 mr = ERR_PTR(-EPERM); 1977 goto end; 1978 } 1979 1980 /* prefetch with write-access must be supported by the MR */ 1981 if (advice == IB_UVERBS_ADVISE_MR_ADVICE_PREFETCH_WRITE && 1982 !mr->umem->writable) { 1983 mr = ERR_PTR(-EPERM); 1984 goto end; 1985 } 1986 1987 refcount_inc(&mmkey->usecount); 1988 end: 1989 xa_unlock(&dev->odp_mkeys); 1990 return mr; 1991 } 1992 1993 static void mlx5_ib_prefetch_mr_work(struct work_struct *w) 1994 { 1995 struct prefetch_mr_work *work = 1996 container_of(w, struct prefetch_mr_work, work); 1997 u32 bytes_mapped = 0; 1998 int ret; 1999 u32 i; 2000 2001 /* We rely on IB/core that work is executed if we have num_sge != 0 only. */ 2002 WARN_ON(!work->num_sge); 2003 for (i = 0; i < work->num_sge; ++i) { 2004 ret = pagefault_mr(work->frags[i].mr, work->frags[i].io_virt, 2005 work->frags[i].length, &bytes_mapped, 2006 work->pf_flags, false); 2007 if (ret <= 0) 2008 continue; 2009 mlx5_update_odp_stats(work->frags[i].mr, prefetch, ret); 2010 } 2011 2012 destroy_prefetch_work(work); 2013 } 2014 2015 static int init_prefetch_work(struct ib_pd *pd, 2016 enum ib_uverbs_advise_mr_advice advice, 2017 u32 pf_flags, struct prefetch_mr_work *work, 2018 struct ib_sge *sg_list, u32 num_sge) 2019 { 2020 u32 i; 2021 2022 INIT_WORK(&work->work, mlx5_ib_prefetch_mr_work); 2023 work->pf_flags = pf_flags; 2024 2025 for (i = 0; i < num_sge; ++i) { 2026 struct mlx5_ib_mr *mr; 2027 2028 mr = get_prefetchable_mr(pd, advice, sg_list[i].lkey); 2029 if (IS_ERR(mr)) { 2030 work->num_sge = i; 2031 return PTR_ERR(mr); 2032 } 2033 work->frags[i].io_virt = sg_list[i].addr; 2034 work->frags[i].length = sg_list[i].length; 2035 work->frags[i].mr = mr; 2036 } 2037 work->num_sge = num_sge; 2038 return 0; 2039 } 2040 2041 static int mlx5_ib_prefetch_sg_list(struct ib_pd *pd, 2042 enum ib_uverbs_advise_mr_advice advice, 2043 u32 pf_flags, struct ib_sge *sg_list, 2044 u32 num_sge) 2045 { 2046 u32 bytes_mapped = 0; 2047 int ret = 0; 2048 u32 i; 2049 2050 for (i = 0; i < num_sge; ++i) { 2051 struct mlx5_ib_mr *mr; 2052 2053 mr = get_prefetchable_mr(pd, advice, sg_list[i].lkey); 2054 if (IS_ERR(mr)) 2055 return PTR_ERR(mr); 2056 ret = pagefault_mr(mr, sg_list[i].addr, sg_list[i].length, 2057 &bytes_mapped, pf_flags, false); 2058 if (ret < 0) { 2059 mlx5r_deref_odp_mkey(&mr->mmkey); 2060 return ret; 2061 } 2062 mlx5_update_odp_stats(mr, prefetch, ret); 2063 mlx5r_deref_odp_mkey(&mr->mmkey); 2064 } 2065 2066 return 0; 2067 } 2068 2069 int mlx5_ib_advise_mr_prefetch(struct ib_pd *pd, 2070 enum ib_uverbs_advise_mr_advice advice, 2071 u32 flags, struct ib_sge *sg_list, u32 num_sge) 2072 { 2073 u32 pf_flags = 0; 2074 struct prefetch_mr_work *work; 2075 int rc; 2076 2077 if (advice == IB_UVERBS_ADVISE_MR_ADVICE_PREFETCH) 2078 pf_flags |= MLX5_PF_FLAGS_DOWNGRADE; 2079 2080 if (advice == IB_UVERBS_ADVISE_MR_ADVICE_PREFETCH_NO_FAULT) 2081 pf_flags |= MLX5_PF_FLAGS_SNAPSHOT; 2082 2083 if (flags & IB_UVERBS_ADVISE_MR_FLAG_FLUSH) 2084 return mlx5_ib_prefetch_sg_list(pd, advice, pf_flags, sg_list, 2085 num_sge); 2086 2087 work = kvzalloc(struct_size(work, frags, num_sge), GFP_KERNEL); 2088 if (!work) 2089 return -ENOMEM; 2090 2091 rc = init_prefetch_work(pd, advice, pf_flags, work, sg_list, num_sge); 2092 if (rc) { 2093 destroy_prefetch_work(work); 2094 return rc; 2095 } 2096 queue_work(system_unbound_wq, &work->work); 2097 return 0; 2098 } 2099