1 /* 2 * Copyright (c) 2013-2015, Mellanox Technologies. All rights reserved. 3 * 4 * This software is available to you under a choice of one of two 5 * licenses. You may choose to be licensed under the terms of the GNU 6 * General Public License (GPL) Version 2, available from the file 7 * COPYING in the main directory of this source tree, or the 8 * OpenIB.org BSD license below: 9 * 10 * Redistribution and use in source and binary forms, with or 11 * without modification, are permitted provided that the following 12 * conditions are met: 13 * 14 * - Redistributions of source code must retain the above 15 * copyright notice, this list of conditions and the following 16 * disclaimer. 17 * 18 * - Redistributions in binary form must reproduce the above 19 * copyright notice, this list of conditions and the following 20 * disclaimer in the documentation and/or other materials 21 * provided with the distribution. 22 * 23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30 * SOFTWARE. 31 */ 32 33 #include <rdma/ib_umem_odp.h> 34 #include <linux/kernel.h> 35 #include <linux/dma-buf.h> 36 #include <linux/dma-resv.h> 37 #include <linux/hmm.h> 38 #include <linux/hmm-dma.h> 39 #include <linux/pci-p2pdma.h> 40 41 #include "mlx5_ib.h" 42 #include "cmd.h" 43 #include "umr.h" 44 #include "qp.h" 45 46 #include <linux/mlx5/eq.h> 47 48 /* Contains the details of a pagefault. */ 49 struct mlx5_pagefault { 50 u32 bytes_committed; 51 u64 token; 52 u8 event_subtype; 53 u8 type; 54 union { 55 /* Initiator or send message responder pagefault details. */ 56 struct { 57 /* Received packet size, only valid for responders. */ 58 u32 packet_size; 59 /* 60 * Number of resource holding WQE, depends on type. 61 */ 62 u32 wq_num; 63 /* 64 * WQE index. Refers to either the send queue or 65 * receive queue, according to event_subtype. 66 */ 67 u16 wqe_index; 68 } wqe; 69 /* RDMA responder pagefault details */ 70 struct { 71 u32 r_key; 72 /* 73 * Received packet size, minimal size page fault 74 * resolution required for forward progress. 75 */ 76 u32 packet_size; 77 u32 rdma_op_len; 78 u64 rdma_va; 79 } rdma; 80 struct { 81 u64 va; 82 u32 mkey; 83 u32 fault_byte_count; 84 u32 prefetch_before_byte_count; 85 u32 prefetch_after_byte_count; 86 u8 flags; 87 } memory; 88 }; 89 90 struct mlx5_ib_pf_eq *eq; 91 struct work_struct work; 92 }; 93 94 #define MAX_PREFETCH_LEN (4*1024*1024U) 95 96 /* Timeout in ms to wait for an active mmu notifier to complete when handling 97 * a pagefault. */ 98 #define MMU_NOTIFIER_TIMEOUT 1000 99 100 static u64 mlx5_imr_ksm_entries; 101 static u64 mlx5_imr_mtt_entries; 102 static u64 mlx5_imr_mtt_size; 103 static u8 mlx5_imr_mtt_shift; 104 static u8 mlx5_imr_ksm_page_shift; 105 106 static void populate_ksm(struct mlx5_ksm *pksm, size_t idx, size_t nentries, 107 struct mlx5_ib_mr *imr, int flags) 108 { 109 struct mlx5_core_dev *dev = mr_to_mdev(imr)->mdev; 110 struct mlx5_ksm *end = pksm + nentries; 111 u64 step = MLX5_CAP_ODP(dev, mem_page_fault) ? mlx5_imr_mtt_size : 0; 112 __be32 key = MLX5_CAP_ODP(dev, mem_page_fault) ? 113 cpu_to_be32(imr->null_mmkey.key) : 114 mr_to_mdev(imr)->mkeys.null_mkey; 115 u64 va = 116 MLX5_CAP_ODP(dev, mem_page_fault) ? idx * mlx5_imr_mtt_size : 0; 117 118 if (flags & MLX5_IB_UPD_XLT_ZAP) { 119 for (; pksm != end; pksm++, idx++, va += step) { 120 pksm->key = key; 121 pksm->va = cpu_to_be64(va); 122 } 123 return; 124 } 125 126 /* 127 * The locking here is pretty subtle. Ideally the implicit_children 128 * xarray would be protected by the umem_mutex, however that is not 129 * possible. Instead this uses a weaker update-then-lock pattern: 130 * 131 * xa_store() 132 * mutex_lock(umem_mutex) 133 * mlx5r_umr_update_xlt() 134 * mutex_unlock(umem_mutex) 135 * destroy lkey 136 * 137 * ie any change the xarray must be followed by the locked update_xlt 138 * before destroying. 139 * 140 * The umem_mutex provides the acquire/release semantic needed to make 141 * the xa_store() visible to a racing thread. 142 */ 143 lockdep_assert_held(&to_ib_umem_odp(imr->umem)->umem_mutex); 144 145 for (; pksm != end; pksm++, idx++, va += step) { 146 struct mlx5_ib_mr *mtt = xa_load(&imr->implicit_children, idx); 147 148 if (mtt) { 149 pksm->key = cpu_to_be32(mtt->ibmr.lkey); 150 pksm->va = cpu_to_be64(idx * mlx5_imr_mtt_size); 151 } else { 152 pksm->key = key; 153 pksm->va = cpu_to_be64(va); 154 } 155 } 156 } 157 158 static int populate_mtt(__be64 *pas, size_t start, size_t nentries, 159 struct mlx5_ib_mr *mr, int flags) 160 { 161 struct ib_umem_odp *odp = to_ib_umem_odp(mr->umem); 162 bool downgrade = flags & MLX5_IB_UPD_XLT_DOWNGRADE; 163 struct pci_p2pdma_map_state p2pdma_state = {}; 164 struct ib_device *dev = odp->umem.ibdev; 165 size_t i; 166 167 if (flags & MLX5_IB_UPD_XLT_ZAP) 168 return 0; 169 170 for (i = 0; i < nentries; i++) { 171 unsigned long pfn = odp->map.pfn_list[start + i]; 172 dma_addr_t dma_addr; 173 174 pfn = odp->map.pfn_list[start + i]; 175 if (!(pfn & HMM_PFN_VALID)) 176 /* ODP initialization */ 177 continue; 178 179 dma_addr = hmm_dma_map_pfn(dev->dma_device, &odp->map, 180 start + i, &p2pdma_state); 181 if (ib_dma_mapping_error(dev, dma_addr)) 182 return -EFAULT; 183 184 dma_addr |= MLX5_IB_MTT_READ; 185 if ((pfn & HMM_PFN_WRITE) && !downgrade) 186 dma_addr |= MLX5_IB_MTT_WRITE; 187 188 pas[i] = cpu_to_be64(dma_addr); 189 odp->npages++; 190 } 191 return 0; 192 } 193 194 int mlx5_odp_populate_xlt(void *xlt, size_t idx, size_t nentries, 195 struct mlx5_ib_mr *mr, int flags) 196 { 197 if (flags & MLX5_IB_UPD_XLT_INDIRECT) { 198 populate_ksm(xlt, idx, nentries, mr, flags); 199 return 0; 200 } else { 201 return populate_mtt(xlt, idx, nentries, mr, flags); 202 } 203 } 204 205 /* 206 * This must be called after the mr has been removed from implicit_children. 207 * NOTE: The MR does not necessarily have to be 208 * empty here, parallel page faults could have raced with the free process and 209 * added pages to it. 210 */ 211 static void free_implicit_child_mr_work(struct work_struct *work) 212 { 213 struct mlx5_ib_mr *mr = 214 container_of(work, struct mlx5_ib_mr, odp_destroy.work); 215 struct mlx5_ib_mr *imr = mr->parent; 216 struct ib_umem_odp *odp_imr = to_ib_umem_odp(imr->umem); 217 struct ib_umem_odp *odp = to_ib_umem_odp(mr->umem); 218 219 mlx5r_deref_wait_odp_mkey(&mr->mmkey); 220 221 mutex_lock(&odp_imr->umem_mutex); 222 mlx5r_umr_update_xlt(mr->parent, 223 ib_umem_start(odp) >> mlx5_imr_mtt_shift, 1, 0, 224 MLX5_IB_UPD_XLT_INDIRECT | MLX5_IB_UPD_XLT_ATOMIC, 225 0); 226 mutex_unlock(&odp_imr->umem_mutex); 227 mlx5_ib_dereg_mr(&mr->ibmr, NULL); 228 229 mlx5r_deref_odp_mkey(&imr->mmkey); 230 } 231 232 static void destroy_unused_implicit_child_mr(struct mlx5_ib_mr *mr) 233 { 234 struct ib_umem_odp *odp = to_ib_umem_odp(mr->umem); 235 unsigned long idx = ib_umem_start(odp) >> mlx5_imr_mtt_shift; 236 struct mlx5_ib_mr *imr = mr->parent; 237 238 /* 239 * If userspace is racing freeing the parent implicit ODP MR then we can 240 * loose the race with parent destruction. In this case 241 * mlx5_ib_free_odp_mr() will free everything in the implicit_children 242 * xarray so NOP is fine. This child MR cannot be destroyed here because 243 * we are under its umem_mutex. 244 */ 245 if (!refcount_inc_not_zero(&imr->mmkey.usecount)) 246 return; 247 248 xa_lock(&imr->implicit_children); 249 if (__xa_cmpxchg(&imr->implicit_children, idx, mr, NULL, GFP_KERNEL) != 250 mr) { 251 xa_unlock(&imr->implicit_children); 252 mlx5r_deref_odp_mkey(&imr->mmkey); 253 return; 254 } 255 256 if (MLX5_CAP_ODP(mr_to_mdev(mr)->mdev, mem_page_fault)) 257 xa_erase(&mr_to_mdev(mr)->odp_mkeys, 258 mlx5_base_mkey(mr->mmkey.key)); 259 xa_unlock(&imr->implicit_children); 260 261 /* Freeing a MR is a sleeping operation, so bounce to a work queue */ 262 INIT_WORK(&mr->odp_destroy.work, free_implicit_child_mr_work); 263 queue_work(system_dfl_wq, &mr->odp_destroy.work); 264 } 265 266 static bool mlx5_ib_invalidate_range(struct mmu_interval_notifier *mni, 267 const struct mmu_notifier_range *range, 268 unsigned long cur_seq) 269 { 270 struct ib_umem_odp *umem_odp = 271 container_of(mni, struct ib_umem_odp, notifier); 272 struct mlx5_ib_mr *mr; 273 const u64 umr_block_mask = MLX5_UMR_MTT_NUM_ENTRIES_ALIGNMENT - 1; 274 u64 idx = 0, blk_start_idx = 0; 275 u64 invalidations = 0; 276 unsigned long start; 277 unsigned long end; 278 int in_block = 0; 279 u64 addr; 280 281 if (!mmu_notifier_range_blockable(range)) 282 return false; 283 284 mutex_lock(&umem_odp->umem_mutex); 285 mmu_interval_set_seq(mni, cur_seq); 286 /* 287 * If npages is zero then umem_odp->private may not be setup yet. This 288 * does not complete until after the first page is mapped for DMA. 289 */ 290 if (!umem_odp->npages) 291 goto out; 292 mr = umem_odp->private; 293 if (!mr) 294 goto out; 295 296 start = max_t(u64, ib_umem_start(umem_odp), range->start); 297 end = min_t(u64, ib_umem_end(umem_odp), range->end); 298 299 /* 300 * Iteration one - zap the HW's MTTs. The notifiers_count ensures that 301 * while we are doing the invalidation, no page fault will attempt to 302 * overwrite the same MTTs. Concurent invalidations might race us, 303 * but they will write 0s as well, so no difference in the end result. 304 */ 305 for (addr = start; addr < end; addr += BIT(umem_odp->page_shift)) { 306 idx = (addr - ib_umem_start(umem_odp)) >> umem_odp->page_shift; 307 /* 308 * Strive to write the MTTs in chunks, but avoid overwriting 309 * non-existing MTTs. The huristic here can be improved to 310 * estimate the cost of another UMR vs. the cost of bigger 311 * UMR. 312 */ 313 if (umem_odp->map.pfn_list[idx] & HMM_PFN_VALID) { 314 if (!in_block) { 315 blk_start_idx = idx; 316 in_block = 1; 317 } 318 } else { 319 u64 umr_offset = idx & umr_block_mask; 320 321 if (in_block && umr_offset == 0) { 322 mlx5r_umr_update_xlt( 323 mr, blk_start_idx, idx - blk_start_idx, 324 0, 325 MLX5_IB_UPD_XLT_ZAP | 326 MLX5_IB_UPD_XLT_ATOMIC, 327 0); 328 in_block = 0; 329 /* Count page invalidations */ 330 invalidations += idx - blk_start_idx + 1; 331 } 332 } 333 } 334 if (in_block) { 335 mlx5r_umr_update_xlt( 336 mr, blk_start_idx, idx - blk_start_idx + 1, 0, 337 MLX5_IB_UPD_XLT_ZAP | MLX5_IB_UPD_XLT_ATOMIC, 0); 338 /* Count page invalidations */ 339 invalidations += idx - blk_start_idx + 1; 340 } 341 342 mlx5_update_odp_stats_with_handled(mr, invalidations, invalidations); 343 344 /* 345 * We are now sure that the device will not access the 346 * memory. We can safely unmap it, and mark it as dirty if 347 * needed. 348 */ 349 350 ib_umem_odp_unmap_dma_pages(umem_odp, start, end); 351 352 if (unlikely(!umem_odp->npages && mr->parent)) 353 destroy_unused_implicit_child_mr(mr); 354 out: 355 mutex_unlock(&umem_odp->umem_mutex); 356 return true; 357 } 358 359 const struct mmu_interval_notifier_ops mlx5_mn_ops = { 360 .invalidate = mlx5_ib_invalidate_range, 361 }; 362 363 static void internal_fill_odp_caps(struct mlx5_ib_dev *dev) 364 { 365 struct ib_odp_caps *caps = &dev->odp_caps; 366 367 memset(caps, 0, sizeof(*caps)); 368 369 if (!MLX5_CAP_GEN(dev->mdev, pg) || !mlx5r_umr_can_load_pas(dev, 0)) 370 return; 371 372 caps->general_caps = IB_ODP_SUPPORT; 373 374 if (MLX5_CAP_GEN(dev->mdev, umr_extended_translation_offset)) 375 dev->odp_max_size = U64_MAX; 376 else 377 dev->odp_max_size = BIT_ULL(MLX5_MAX_UMR_SHIFT + PAGE_SHIFT); 378 379 if (MLX5_CAP_ODP_SCHEME(dev->mdev, ud_odp_caps.send)) 380 caps->per_transport_caps.ud_odp_caps |= IB_ODP_SUPPORT_SEND; 381 382 if (MLX5_CAP_ODP_SCHEME(dev->mdev, ud_odp_caps.srq_receive)) 383 caps->per_transport_caps.ud_odp_caps |= IB_ODP_SUPPORT_SRQ_RECV; 384 385 if (MLX5_CAP_ODP_SCHEME(dev->mdev, rc_odp_caps.send)) 386 caps->per_transport_caps.rc_odp_caps |= IB_ODP_SUPPORT_SEND; 387 388 if (MLX5_CAP_ODP_SCHEME(dev->mdev, rc_odp_caps.receive)) 389 caps->per_transport_caps.rc_odp_caps |= IB_ODP_SUPPORT_RECV; 390 391 if (MLX5_CAP_ODP_SCHEME(dev->mdev, rc_odp_caps.write)) 392 caps->per_transport_caps.rc_odp_caps |= IB_ODP_SUPPORT_WRITE; 393 394 if (MLX5_CAP_ODP_SCHEME(dev->mdev, rc_odp_caps.read)) 395 caps->per_transport_caps.rc_odp_caps |= IB_ODP_SUPPORT_READ; 396 397 if (MLX5_CAP_ODP_SCHEME(dev->mdev, rc_odp_caps.atomic)) 398 caps->per_transport_caps.rc_odp_caps |= IB_ODP_SUPPORT_ATOMIC; 399 400 if (MLX5_CAP_ODP_SCHEME(dev->mdev, rc_odp_caps.srq_receive)) 401 caps->per_transport_caps.rc_odp_caps |= IB_ODP_SUPPORT_SRQ_RECV; 402 403 if (MLX5_CAP_ODP_SCHEME(dev->mdev, xrc_odp_caps.send)) 404 caps->per_transport_caps.xrc_odp_caps |= IB_ODP_SUPPORT_SEND; 405 406 if (MLX5_CAP_ODP_SCHEME(dev->mdev, xrc_odp_caps.receive)) 407 caps->per_transport_caps.xrc_odp_caps |= IB_ODP_SUPPORT_RECV; 408 409 if (MLX5_CAP_ODP_SCHEME(dev->mdev, xrc_odp_caps.write)) 410 caps->per_transport_caps.xrc_odp_caps |= IB_ODP_SUPPORT_WRITE; 411 412 if (MLX5_CAP_ODP_SCHEME(dev->mdev, xrc_odp_caps.read)) 413 caps->per_transport_caps.xrc_odp_caps |= IB_ODP_SUPPORT_READ; 414 415 if (MLX5_CAP_ODP_SCHEME(dev->mdev, xrc_odp_caps.atomic)) 416 caps->per_transport_caps.xrc_odp_caps |= IB_ODP_SUPPORT_ATOMIC; 417 418 if (MLX5_CAP_ODP_SCHEME(dev->mdev, xrc_odp_caps.srq_receive)) 419 caps->per_transport_caps.xrc_odp_caps |= IB_ODP_SUPPORT_SRQ_RECV; 420 421 if (MLX5_CAP_GEN(dev->mdev, fixed_buffer_size) && 422 MLX5_CAP_GEN(dev->mdev, null_mkey) && 423 MLX5_CAP_GEN(dev->mdev, umr_extended_translation_offset) && 424 !MLX5_CAP_GEN(dev->mdev, umr_indirect_mkey_disabled) && 425 mlx5_imr_ksm_entries != 0 && 426 !(mlx5_imr_ksm_page_shift > 427 get_max_log_entity_size_cap(dev, MLX5_MKC_ACCESS_MODE_KSM))) 428 caps->general_caps |= IB_ODP_SUPPORT_IMPLICIT; 429 } 430 431 static void mlx5_ib_page_fault_resume(struct mlx5_ib_dev *dev, 432 struct mlx5_pagefault *pfault, 433 int error) 434 { 435 int wq_num = pfault->event_subtype == MLX5_PFAULT_SUBTYPE_WQE ? 436 pfault->wqe.wq_num : pfault->token; 437 u32 in[MLX5_ST_SZ_DW(page_fault_resume_in)] = {}; 438 void *info; 439 int err; 440 441 MLX5_SET(page_fault_resume_in, in, opcode, MLX5_CMD_OP_PAGE_FAULT_RESUME); 442 443 if (pfault->event_subtype == MLX5_PFAULT_SUBTYPE_MEMORY) { 444 info = MLX5_ADDR_OF(page_fault_resume_in, in, 445 page_fault_info.mem_page_fault_info); 446 MLX5_SET(mem_page_fault_info, info, fault_token_31_0, 447 pfault->token & 0xffffffff); 448 MLX5_SET(mem_page_fault_info, info, fault_token_47_32, 449 (pfault->token >> 32) & 0xffff); 450 MLX5_SET(mem_page_fault_info, info, error, !!error); 451 } else { 452 info = MLX5_ADDR_OF(page_fault_resume_in, in, 453 page_fault_info.trans_page_fault_info); 454 MLX5_SET(trans_page_fault_info, info, page_fault_type, 455 pfault->type); 456 MLX5_SET(trans_page_fault_info, info, fault_token, 457 pfault->token); 458 MLX5_SET(trans_page_fault_info, info, wq_number, wq_num); 459 MLX5_SET(trans_page_fault_info, info, error, !!error); 460 } 461 462 err = mlx5_cmd_exec_in(dev->mdev, page_fault_resume, in); 463 if (err) 464 mlx5_ib_err(dev, "Failed to resolve the page fault on WQ 0x%x err %d\n", 465 wq_num, err); 466 } 467 468 static struct mlx5_ib_mr *implicit_get_child_mr(struct mlx5_ib_mr *imr, 469 unsigned long idx) 470 { 471 struct mlx5_ib_dev *dev = mr_to_mdev(imr); 472 struct ib_umem_odp *odp; 473 struct mlx5_ib_mr *mr; 474 struct mlx5_ib_mr *ret; 475 int err; 476 477 odp = ib_umem_odp_alloc_child(to_ib_umem_odp(imr->umem), 478 idx * mlx5_imr_mtt_size, 479 mlx5_imr_mtt_size, &mlx5_mn_ops); 480 if (IS_ERR(odp)) 481 return ERR_CAST(odp); 482 483 mr = mlx5_mr_cache_alloc(dev, imr->access_flags, 484 MLX5_MKC_ACCESS_MODE_MTT, 485 mlx5_imr_mtt_entries); 486 if (IS_ERR(mr)) { 487 ib_umem_odp_release(odp); 488 return mr; 489 } 490 491 mr->access_flags = imr->access_flags; 492 mr->ibmr.pd = imr->ibmr.pd; 493 mr->ibmr.device = &mr_to_mdev(imr)->ib_dev; 494 mr->umem = &odp->umem; 495 mr->ibmr.lkey = mr->mmkey.key; 496 mr->ibmr.rkey = mr->mmkey.key; 497 mr->ibmr.iova = idx * mlx5_imr_mtt_size; 498 mr->parent = imr; 499 odp->private = mr; 500 501 /* 502 * First refcount is owned by the xarray and second refconut 503 * is returned to the caller. 504 */ 505 refcount_set(&mr->mmkey.usecount, 2); 506 507 err = mlx5r_umr_update_xlt(mr, 0, mlx5_imr_mtt_entries, PAGE_SHIFT, 508 MLX5_IB_UPD_XLT_ZAP | MLX5_IB_UPD_XLT_ENABLE, 509 to_mpd(mr->ibmr.pd)->pdn); 510 if (err) { 511 ret = ERR_PTR(err); 512 goto out_mr; 513 } 514 515 xa_lock(&imr->implicit_children); 516 ret = __xa_cmpxchg(&imr->implicit_children, idx, NULL, mr, 517 GFP_KERNEL); 518 if (unlikely(ret)) { 519 if (xa_is_err(ret)) { 520 ret = ERR_PTR(xa_err(ret)); 521 goto out_lock; 522 } 523 /* 524 * Another thread beat us to creating the child mr, use 525 * theirs. 526 */ 527 refcount_inc(&ret->mmkey.usecount); 528 goto out_lock; 529 } 530 531 if (MLX5_CAP_ODP(dev->mdev, mem_page_fault)) { 532 ret = xa_store(&dev->odp_mkeys, mlx5_base_mkey(mr->mmkey.key), 533 &mr->mmkey, GFP_KERNEL); 534 if (xa_is_err(ret)) { 535 ret = ERR_PTR(xa_err(ret)); 536 __xa_erase(&imr->implicit_children, idx); 537 goto out_lock; 538 } 539 mr->mmkey.type = MLX5_MKEY_IMPLICIT_CHILD; 540 } 541 xa_unlock(&imr->implicit_children); 542 mlx5_ib_dbg(mr_to_mdev(imr), "key %x mr %p\n", mr->mmkey.key, mr); 543 return mr; 544 545 out_lock: 546 xa_unlock(&imr->implicit_children); 547 out_mr: 548 mlx5_ib_dereg_mr(&mr->ibmr, NULL); 549 return ret; 550 } 551 552 /* 553 * When using memory scheme ODP, implicit MRs can't use the reserved null mkey 554 * and each implicit MR needs to assign a private null mkey to get the page 555 * faults on. 556 * The null mkey is created with the properties to enable getting the page 557 * fault for every time it is accessed and having all relevant access flags. 558 */ 559 static int alloc_implicit_mr_null_mkey(struct mlx5_ib_dev *dev, 560 struct mlx5_ib_mr *imr, 561 struct mlx5_ib_pd *pd) 562 { 563 size_t inlen = MLX5_ST_SZ_BYTES(create_mkey_in) + 64; 564 void *mkc; 565 u32 *in; 566 int err; 567 568 in = kzalloc(inlen, GFP_KERNEL); 569 if (!in) 570 return -ENOMEM; 571 572 MLX5_SET(create_mkey_in, in, translations_octword_actual_size, 4); 573 MLX5_SET(create_mkey_in, in, pg_access, 1); 574 575 mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry); 576 MLX5_SET(mkc, mkc, a, 1); 577 MLX5_SET(mkc, mkc, rw, 1); 578 MLX5_SET(mkc, mkc, rr, 1); 579 MLX5_SET(mkc, mkc, lw, 1); 580 MLX5_SET(mkc, mkc, lr, 1); 581 MLX5_SET(mkc, mkc, free, 0); 582 MLX5_SET(mkc, mkc, umr_en, 0); 583 MLX5_SET(mkc, mkc, access_mode_1_0, MLX5_MKC_ACCESS_MODE_MTT); 584 585 MLX5_SET(mkc, mkc, translations_octword_size, 4); 586 MLX5_SET(mkc, mkc, log_page_size, 61); 587 MLX5_SET(mkc, mkc, length64, 1); 588 MLX5_SET(mkc, mkc, pd, pd->pdn); 589 MLX5_SET64(mkc, mkc, start_addr, 0); 590 MLX5_SET(mkc, mkc, qpn, 0xffffff); 591 592 err = mlx5_core_create_mkey(dev->mdev, &imr->null_mmkey.key, in, inlen); 593 if (err) 594 goto free_in; 595 596 imr->null_mmkey.type = MLX5_MKEY_NULL; 597 598 free_in: 599 kfree(in); 600 return err; 601 } 602 603 struct mlx5_ib_mr *mlx5_ib_alloc_implicit_mr(struct mlx5_ib_pd *pd, 604 int access_flags) 605 { 606 struct mlx5_ib_dev *dev = to_mdev(pd->ibpd.device); 607 struct ib_umem_odp *umem_odp; 608 struct mlx5_ib_mr *imr; 609 int err; 610 611 if (!mlx5r_umr_can_load_pas(dev, mlx5_imr_mtt_entries * PAGE_SIZE)) 612 return ERR_PTR(-EOPNOTSUPP); 613 614 umem_odp = ib_umem_odp_alloc_implicit(&dev->ib_dev, access_flags); 615 if (IS_ERR(umem_odp)) 616 return ERR_CAST(umem_odp); 617 618 imr = mlx5_mr_cache_alloc(dev, access_flags, MLX5_MKC_ACCESS_MODE_KSM, 619 mlx5_imr_ksm_entries); 620 if (IS_ERR(imr)) { 621 ib_umem_odp_release(umem_odp); 622 return imr; 623 } 624 625 imr->access_flags = access_flags; 626 imr->ibmr.pd = &pd->ibpd; 627 imr->ibmr.iova = 0; 628 imr->umem = &umem_odp->umem; 629 imr->ibmr.lkey = imr->mmkey.key; 630 imr->ibmr.rkey = imr->mmkey.key; 631 imr->ibmr.device = &dev->ib_dev; 632 imr->is_odp_implicit = true; 633 xa_init(&imr->implicit_children); 634 635 if (MLX5_CAP_ODP(dev->mdev, mem_page_fault)) { 636 err = alloc_implicit_mr_null_mkey(dev, imr, pd); 637 if (err) 638 goto out_mr; 639 640 err = mlx5r_store_odp_mkey(dev, &imr->null_mmkey); 641 if (err) 642 goto out_mr; 643 } 644 645 err = mlx5r_umr_update_xlt(imr, 0, 646 mlx5_imr_ksm_entries, 647 mlx5_imr_ksm_page_shift, 648 MLX5_IB_UPD_XLT_INDIRECT | 649 MLX5_IB_UPD_XLT_ZAP | 650 MLX5_IB_UPD_XLT_ENABLE, 651 pd->pdn); 652 if (err) 653 goto out_mr; 654 655 err = mlx5r_store_odp_mkey(dev, &imr->mmkey); 656 if (err) 657 goto out_mr; 658 659 mlx5_ib_dbg(dev, "key %x mr %p\n", imr->mmkey.key, imr); 660 return imr; 661 out_mr: 662 mlx5_ib_err(dev, "Failed to register MKEY %d\n", err); 663 mlx5_ib_dereg_mr(&imr->ibmr, NULL); 664 return ERR_PTR(err); 665 } 666 667 void mlx5_ib_free_odp_mr(struct mlx5_ib_mr *mr) 668 { 669 struct mlx5_ib_mr *mtt; 670 unsigned long idx; 671 672 /* 673 * If this is an implicit MR it is already invalidated so we can just 674 * delete the children mkeys. 675 */ 676 xa_for_each(&mr->implicit_children, idx, mtt) { 677 xa_erase(&mr->implicit_children, idx); 678 mlx5_ib_dereg_mr(&mtt->ibmr, NULL); 679 } 680 681 if (mr->null_mmkey.key) { 682 xa_erase(&mr_to_mdev(mr)->odp_mkeys, 683 mlx5_base_mkey(mr->null_mmkey.key)); 684 685 mlx5_core_destroy_mkey(mr_to_mdev(mr)->mdev, 686 mr->null_mmkey.key); 687 } 688 } 689 690 /* 691 * pdn must be valid only when xlt_flags updates the mkey PD. In this path that 692 * is only MLX5_PF_FLAGS_ENABLE. DOWNGRADE and SNAPSHOT leave the PD masked out. 693 */ 694 #define MLX5_PF_FLAGS_DOWNGRADE BIT(1) 695 #define MLX5_PF_FLAGS_SNAPSHOT BIT(2) 696 #define MLX5_PF_FLAGS_ENABLE BIT(3) 697 static int pagefault_real_mr(struct mlx5_ib_mr *mr, struct ib_umem_odp *odp, 698 u64 user_va, size_t bcnt, u32 *bytes_mapped, 699 u32 flags, u32 pdn) 700 { 701 int page_shift, ret, np; 702 bool downgrade = flags & MLX5_PF_FLAGS_DOWNGRADE; 703 u64 access_mask = 0; 704 u64 start_idx; 705 bool fault = !(flags & MLX5_PF_FLAGS_SNAPSHOT); 706 u32 xlt_flags = MLX5_IB_UPD_XLT_ATOMIC; 707 708 if (flags & MLX5_PF_FLAGS_ENABLE) 709 xlt_flags |= MLX5_IB_UPD_XLT_ENABLE; 710 711 if (flags & MLX5_PF_FLAGS_DOWNGRADE) 712 xlt_flags |= MLX5_IB_UPD_XLT_DOWNGRADE; 713 714 page_shift = odp->page_shift; 715 start_idx = (user_va - ib_umem_start(odp)) >> page_shift; 716 717 if (odp->umem.writable && !downgrade) 718 access_mask |= HMM_PFN_WRITE; 719 720 np = ib_umem_odp_map_dma_and_lock(odp, user_va, bcnt, access_mask, fault); 721 if (np < 0) 722 return np; 723 724 /* 725 * No need to check whether the MTTs really belong to this MR, since 726 * ib_umem_odp_map_dma_and_lock already checks this. 727 */ 728 ret = mlx5r_umr_update_xlt(mr, start_idx, np, page_shift, xlt_flags, 729 pdn); 730 mutex_unlock(&odp->umem_mutex); 731 732 if (ret < 0) { 733 if (ret != -EAGAIN) 734 mlx5_ib_err(mr_to_mdev(mr), 735 "Failed to update mkey page tables\n"); 736 goto out; 737 } 738 739 if (bytes_mapped) { 740 u32 new_mappings = (np << page_shift) - 741 (user_va - round_down(user_va, 1 << page_shift)); 742 743 *bytes_mapped += min_t(u32, new_mappings, bcnt); 744 } 745 746 return np << (page_shift - PAGE_SHIFT); 747 748 out: 749 return ret; 750 } 751 752 static int pagefault_implicit_mr(struct mlx5_ib_mr *imr, 753 struct ib_umem_odp *odp_imr, u64 user_va, 754 size_t bcnt, u32 *bytes_mapped, u32 flags) 755 { 756 unsigned long end_idx = (user_va + bcnt - 1) >> mlx5_imr_mtt_shift; 757 unsigned long upd_start_idx = end_idx + 1; 758 unsigned long upd_len = 0; 759 unsigned long npages = 0; 760 int err; 761 int ret; 762 763 if (unlikely(user_va >= mlx5_imr_ksm_entries * mlx5_imr_mtt_size || 764 mlx5_imr_ksm_entries * mlx5_imr_mtt_size - user_va < bcnt)) 765 return -EFAULT; 766 767 /* Fault each child mr that intersects with our interval. */ 768 while (bcnt) { 769 unsigned long idx = user_va >> mlx5_imr_mtt_shift; 770 struct ib_umem_odp *umem_odp; 771 struct mlx5_ib_mr *mtt; 772 u64 len; 773 774 xa_lock(&imr->implicit_children); 775 mtt = xa_load(&imr->implicit_children, idx); 776 if (unlikely(!mtt)) { 777 xa_unlock(&imr->implicit_children); 778 mtt = implicit_get_child_mr(imr, idx); 779 if (IS_ERR(mtt)) { 780 ret = PTR_ERR(mtt); 781 goto out; 782 } 783 upd_start_idx = min(upd_start_idx, idx); 784 upd_len = idx - upd_start_idx + 1; 785 } else { 786 refcount_inc(&mtt->mmkey.usecount); 787 xa_unlock(&imr->implicit_children); 788 } 789 790 umem_odp = to_ib_umem_odp(mtt->umem); 791 len = min_t(u64, user_va + bcnt, ib_umem_end(umem_odp)) - 792 user_va; 793 794 ret = pagefault_real_mr(mtt, umem_odp, user_va, len, 795 bytes_mapped, flags, 0); 796 797 mlx5r_deref_odp_mkey(&mtt->mmkey); 798 799 if (ret < 0) 800 goto out; 801 user_va += len; 802 bcnt -= len; 803 npages += ret; 804 } 805 806 ret = npages; 807 808 /* 809 * Any time the implicit_children are changed we must perform an 810 * update of the xlt before exiting to ensure the HW and the 811 * implicit_children remains synchronized. 812 */ 813 out: 814 if (likely(!upd_len)) 815 return ret; 816 817 /* 818 * Notice this is not strictly ordered right, the KSM is updated after 819 * the implicit_children is updated, so a parallel page fault could 820 * see a MR that is not yet visible in the KSM. This is similar to a 821 * parallel page fault seeing a MR that is being concurrently removed 822 * from the KSM. Both of these improbable situations are resolved 823 * safely by resuming the HW and then taking another page fault. The 824 * next pagefault handler will see the new information. 825 */ 826 mutex_lock(&odp_imr->umem_mutex); 827 err = mlx5r_umr_update_xlt( 828 imr, upd_start_idx, upd_len, 0, 829 MLX5_IB_UPD_XLT_INDIRECT | MLX5_IB_UPD_XLT_ATOMIC, 0); 830 mutex_unlock(&odp_imr->umem_mutex); 831 if (err) { 832 mlx5_ib_err(mr_to_mdev(imr), "Failed to update PAS\n"); 833 return err; 834 } 835 return ret; 836 } 837 838 static int pagefault_dmabuf_mr(struct mlx5_ib_mr *mr, size_t bcnt, 839 u32 *bytes_mapped, u32 flags, u32 pdn) 840 { 841 struct ib_umem_dmabuf *umem_dmabuf = to_ib_umem_dmabuf(mr->umem); 842 int access_mode = mr->data_direct ? MLX5_MKC_ACCESS_MODE_KSM : 843 MLX5_MKC_ACCESS_MODE_MTT; 844 unsigned int old_page_shift = mr->page_shift; 845 unsigned int page_shift; 846 unsigned long page_size; 847 u32 xlt_flags = 0; 848 int err; 849 850 if (flags & MLX5_PF_FLAGS_ENABLE) 851 xlt_flags |= MLX5_IB_UPD_XLT_ENABLE; 852 853 dma_resv_lock(umem_dmabuf->attach->dmabuf->resv, NULL); 854 err = ib_umem_dmabuf_map_pages(umem_dmabuf); 855 if (err) { 856 dma_resv_unlock(umem_dmabuf->attach->dmabuf->resv); 857 return err; 858 } 859 860 page_size = mlx5_umem_dmabuf_find_best_pgsz(umem_dmabuf, access_mode); 861 if (!page_size) { 862 ib_umem_dmabuf_unmap_pages(umem_dmabuf); 863 err = -EINVAL; 864 } else { 865 page_shift = order_base_2(page_size); 866 if (page_shift != mr->page_shift && mr->dmabuf_faulted) { 867 err = mlx5r_umr_dmabuf_update_pgsz(mr, xlt_flags, pdn, 868 page_shift); 869 } else { 870 mr->page_shift = page_shift; 871 if (mr->data_direct) 872 err = mlx5r_umr_update_data_direct_ksm_pas( 873 mr, xlt_flags); 874 else 875 err = mlx5r_umr_update_mr_pas(mr, xlt_flags, 876 pdn); 877 } 878 } 879 dma_resv_unlock(umem_dmabuf->attach->dmabuf->resv); 880 881 if (err) { 882 mr->page_shift = old_page_shift; 883 return err; 884 } 885 886 mr->dmabuf_faulted = 1; 887 888 if (bytes_mapped) 889 *bytes_mapped += bcnt; 890 891 return ib_umem_num_pages(mr->umem); 892 } 893 894 /* 895 * Returns: 896 * -EFAULT: The io_virt->bcnt is not within the MR, it covers pages that are 897 * not accessible, or the MR is no longer valid. 898 * -EAGAIN/-ENOMEM: The operation should be retried 899 * 900 * -EINVAL/others: General internal malfunction 901 * >0: Number of pages mapped 902 */ 903 static int pagefault_mr(struct mlx5_ib_mr *mr, u64 io_virt, size_t bcnt, 904 u32 *bytes_mapped, u32 flags, bool permissive_fault) 905 { 906 struct ib_umem_odp *odp = to_ib_umem_odp(mr->umem); 907 908 if (unlikely(io_virt < mr->ibmr.iova) && !permissive_fault) 909 return -EFAULT; 910 911 if (mr->umem->is_dmabuf) 912 return pagefault_dmabuf_mr(mr, bcnt, bytes_mapped, flags, 0); 913 914 if (!odp->is_implicit_odp) { 915 u64 offset = io_virt < mr->ibmr.iova ? 0 : io_virt - mr->ibmr.iova; 916 u64 user_va; 917 918 if (check_add_overflow(offset, (u64)odp->umem.address, 919 &user_va)) 920 return -EFAULT; 921 922 if (permissive_fault) { 923 if (user_va < ib_umem_start(odp)) 924 user_va = ib_umem_start(odp); 925 if ((user_va + bcnt) > ib_umem_end(odp)) 926 bcnt = ib_umem_end(odp) - user_va; 927 } else if (unlikely(user_va >= ib_umem_end(odp) || 928 ib_umem_end(odp) - user_va < bcnt)) 929 return -EFAULT; 930 return pagefault_real_mr(mr, odp, user_va, bcnt, bytes_mapped, 931 flags, 0); 932 } 933 return pagefault_implicit_mr(mr, odp, io_virt, bcnt, bytes_mapped, 934 flags); 935 } 936 937 int mlx5_ib_init_odp_mr(struct mlx5_ib_mr *mr, struct ib_pd *pd) 938 { 939 int ret; 940 941 ret = pagefault_real_mr(mr, to_ib_umem_odp(mr->umem), mr->umem->address, 942 mr->umem->length, NULL, 943 MLX5_PF_FLAGS_SNAPSHOT | MLX5_PF_FLAGS_ENABLE, 944 to_mpd(pd)->pdn); 945 return ret >= 0 ? 0 : ret; 946 } 947 948 int mlx5_ib_init_dmabuf_mr(struct mlx5_ib_mr *mr, struct ib_pd *pd) 949 { 950 struct mlx5_ib_dev *dev = mr_to_mdev(mr); 951 u32 pdn; 952 int ret; 953 954 if (mr->data_direct) 955 pdn = dev->ddr.pdn; 956 else 957 pdn = to_mpd(pd)->pdn; 958 959 ret = pagefault_dmabuf_mr(mr, mr->umem->length, NULL, 960 MLX5_PF_FLAGS_ENABLE, pdn); 961 962 return ret >= 0 ? 0 : ret; 963 } 964 965 struct pf_frame { 966 struct pf_frame *next; 967 u32 key; 968 u64 io_virt; 969 size_t bcnt; 970 int depth; 971 }; 972 973 static bool mkey_is_eq(struct mlx5_ib_mkey *mmkey, u32 key) 974 { 975 if (!mmkey) 976 return false; 977 if (mmkey->type == MLX5_MKEY_MW || 978 mmkey->type == MLX5_MKEY_INDIRECT_DEVX) 979 return mlx5_base_mkey(mmkey->key) == mlx5_base_mkey(key); 980 return mmkey->key == key; 981 } 982 983 static struct mlx5_ib_mkey *find_odp_mkey(struct mlx5_ib_dev *dev, u32 key) 984 { 985 struct mlx5_ib_mkey *mmkey; 986 987 xa_lock(&dev->odp_mkeys); 988 mmkey = xa_load(&dev->odp_mkeys, mlx5_base_mkey(key)); 989 if (!mmkey) { 990 mmkey = ERR_PTR(-ENOENT); 991 goto out; 992 } 993 if (!mkey_is_eq(mmkey, key)) { 994 mmkey = ERR_PTR(-EFAULT); 995 goto out; 996 } 997 refcount_inc(&mmkey->usecount); 998 out: 999 xa_unlock(&dev->odp_mkeys); 1000 1001 return mmkey; 1002 } 1003 1004 /* 1005 * Handle a single data segment in a page-fault WQE or RDMA region. 1006 * 1007 * Returns zero on success. The caller may continue to the next data segment. 1008 * Can return the following error codes: 1009 * -EAGAIN to designate a temporary error. The caller will abort handling the 1010 * page fault and resolve it. 1011 * -EFAULT when there's an error mapping the requested pages. The caller will 1012 * abort the page fault handling. 1013 */ 1014 static int pagefault_single_data_segment(struct mlx5_ib_dev *dev, 1015 struct ib_pd *pd, u32 key, 1016 u64 io_virt, size_t bcnt, 1017 u32 *bytes_committed, 1018 u32 *bytes_mapped) 1019 { 1020 int ret, i, outlen, cur_outlen = 0, depth = 0, pages_in_range; 1021 struct pf_frame *head = NULL, *frame; 1022 struct mlx5_ib_mkey *mmkey; 1023 struct mlx5_ib_mr *mr; 1024 struct mlx5_klm *pklm; 1025 u32 *out = NULL; 1026 size_t offset; 1027 1028 io_virt += *bytes_committed; 1029 bcnt -= *bytes_committed; 1030 next_mr: 1031 mmkey = find_odp_mkey(dev, key); 1032 if (IS_ERR(mmkey)) { 1033 ret = PTR_ERR(mmkey); 1034 if (ret == -ENOENT) { 1035 mlx5_ib_dbg( 1036 dev, 1037 "skipping non ODP MR (lkey=0x%06x) in page fault handler.\n", 1038 key); 1039 if (bytes_mapped) 1040 *bytes_mapped += bcnt; 1041 /* 1042 * The user could specify a SGL with multiple lkeys and 1043 * only some of them are ODP. Treat the non-ODP ones as 1044 * fully faulted. 1045 */ 1046 ret = 0; 1047 } 1048 goto end; 1049 } 1050 1051 switch (mmkey->type) { 1052 case MLX5_MKEY_MR: 1053 mr = container_of(mmkey, struct mlx5_ib_mr, mmkey); 1054 1055 pages_in_range = (ALIGN(io_virt + bcnt, PAGE_SIZE) - 1056 (io_virt & PAGE_MASK)) >> 1057 PAGE_SHIFT; 1058 ret = pagefault_mr(mr, io_virt, bcnt, bytes_mapped, 0, false); 1059 if (ret < 0) 1060 goto end; 1061 1062 mlx5_update_odp_stats_with_handled(mr, faults, ret); 1063 1064 if (ret < pages_in_range) { 1065 ret = -EFAULT; 1066 goto end; 1067 } 1068 1069 ret = 0; 1070 break; 1071 1072 case MLX5_MKEY_MW: 1073 case MLX5_MKEY_INDIRECT_DEVX: 1074 if (depth >= MLX5_CAP_GEN(dev->mdev, max_indirection)) { 1075 mlx5_ib_dbg(dev, "indirection level exceeded\n"); 1076 ret = -EFAULT; 1077 goto end; 1078 } 1079 1080 outlen = MLX5_ST_SZ_BYTES(query_mkey_out) + 1081 sizeof(*pklm) * (mmkey->ndescs - 2); 1082 1083 if (outlen > cur_outlen) { 1084 kfree(out); 1085 out = kzalloc(outlen, GFP_KERNEL); 1086 if (!out) { 1087 ret = -ENOMEM; 1088 goto end; 1089 } 1090 cur_outlen = outlen; 1091 } 1092 1093 pklm = (struct mlx5_klm *)MLX5_ADDR_OF(query_mkey_out, out, 1094 bsf0_klm0_pas_mtt0_1); 1095 1096 ret = mlx5_core_query_mkey(dev->mdev, mmkey->key, out, outlen); 1097 if (ret) 1098 goto end; 1099 1100 offset = io_virt - MLX5_GET64(query_mkey_out, out, 1101 memory_key_mkey_entry.start_addr); 1102 1103 for (i = 0; bcnt && i < mmkey->ndescs; i++, pklm++) { 1104 if (offset >= be32_to_cpu(pklm->bcount)) { 1105 offset -= be32_to_cpu(pklm->bcount); 1106 continue; 1107 } 1108 1109 frame = kzalloc_obj(*frame); 1110 if (!frame) { 1111 ret = -ENOMEM; 1112 goto end; 1113 } 1114 1115 frame->key = be32_to_cpu(pklm->key); 1116 frame->io_virt = be64_to_cpu(pklm->va) + offset; 1117 frame->bcnt = min_t(size_t, bcnt, 1118 be32_to_cpu(pklm->bcount) - offset); 1119 frame->depth = depth + 1; 1120 frame->next = head; 1121 head = frame; 1122 1123 bcnt -= frame->bcnt; 1124 offset = 0; 1125 } 1126 break; 1127 1128 default: 1129 mlx5_ib_dbg(dev, "wrong mkey type %d\n", mmkey->type); 1130 ret = -EFAULT; 1131 goto end; 1132 } 1133 1134 if (head) { 1135 frame = head; 1136 head = frame->next; 1137 1138 key = frame->key; 1139 io_virt = frame->io_virt; 1140 bcnt = frame->bcnt; 1141 depth = frame->depth; 1142 kfree(frame); 1143 1144 mlx5r_deref_odp_mkey(mmkey); 1145 goto next_mr; 1146 } 1147 1148 end: 1149 if (!IS_ERR(mmkey)) 1150 mlx5r_deref_odp_mkey(mmkey); 1151 while (head) { 1152 frame = head; 1153 head = frame->next; 1154 kfree(frame); 1155 } 1156 kfree(out); 1157 1158 *bytes_committed = 0; 1159 return ret; 1160 } 1161 1162 /* 1163 * Parse a series of data segments for page fault handling. 1164 * 1165 * @dev: Pointer to mlx5 IB device 1166 * @pfault: contains page fault information. 1167 * @wqe: points at the first data segment in the WQE. 1168 * @wqe_end: points after the end of the WQE. 1169 * @bytes_mapped: receives the number of bytes that the function was able to 1170 * map. This allows the caller to decide intelligently whether 1171 * enough memory was mapped to resolve the page fault 1172 * successfully (e.g. enough for the next MTU, or the entire 1173 * WQE). 1174 * @total_wqe_bytes: receives the total data size of this WQE in bytes (minus 1175 * the committed bytes). 1176 * @receive_queue: receive WQE end of sg list 1177 * 1178 * Returns zero for success or a negative error code. 1179 */ 1180 static int pagefault_data_segments(struct mlx5_ib_dev *dev, 1181 struct mlx5_pagefault *pfault, 1182 void *wqe, 1183 void *wqe_end, u32 *bytes_mapped, 1184 u32 *total_wqe_bytes, bool receive_queue) 1185 { 1186 int ret = 0; 1187 u64 io_virt; 1188 __be32 key; 1189 u32 byte_count; 1190 size_t bcnt; 1191 int inline_segment; 1192 1193 if (bytes_mapped) 1194 *bytes_mapped = 0; 1195 if (total_wqe_bytes) 1196 *total_wqe_bytes = 0; 1197 1198 while (wqe < wqe_end) { 1199 struct mlx5_wqe_data_seg *dseg = wqe; 1200 1201 io_virt = be64_to_cpu(dseg->addr); 1202 key = dseg->lkey; 1203 byte_count = be32_to_cpu(dseg->byte_count); 1204 inline_segment = !!(byte_count & MLX5_INLINE_SEG); 1205 bcnt = byte_count & ~MLX5_INLINE_SEG; 1206 1207 if (inline_segment) { 1208 bcnt = bcnt & MLX5_WQE_INLINE_SEG_BYTE_COUNT_MASK; 1209 wqe += ALIGN(sizeof(struct mlx5_wqe_inline_seg) + bcnt, 1210 16); 1211 } else { 1212 wqe += sizeof(*dseg); 1213 } 1214 1215 /* receive WQE end of sg list. */ 1216 if (receive_queue && bcnt == 0 && 1217 key == dev->mkeys.terminate_scatter_list_mkey && 1218 io_virt == 0) 1219 break; 1220 1221 if (!inline_segment && total_wqe_bytes) { 1222 *total_wqe_bytes += bcnt - min_t(size_t, bcnt, 1223 pfault->bytes_committed); 1224 } 1225 1226 /* A zero length data segment designates a length of 2GB. */ 1227 if (bcnt == 0) 1228 bcnt = 1U << 31; 1229 1230 if (inline_segment || bcnt <= pfault->bytes_committed) { 1231 pfault->bytes_committed -= 1232 min_t(size_t, bcnt, 1233 pfault->bytes_committed); 1234 continue; 1235 } 1236 1237 ret = pagefault_single_data_segment(dev, NULL, be32_to_cpu(key), 1238 io_virt, bcnt, 1239 &pfault->bytes_committed, 1240 bytes_mapped); 1241 if (ret < 0) 1242 break; 1243 } 1244 1245 return ret; 1246 } 1247 1248 /* 1249 * Parse initiator WQE. Advances the wqe pointer to point at the 1250 * scatter-gather list, and set wqe_end to the end of the WQE. 1251 */ 1252 static int mlx5_ib_mr_initiator_pfault_handler( 1253 struct mlx5_ib_dev *dev, struct mlx5_pagefault *pfault, 1254 struct mlx5_ib_qp *qp, void **wqe, void **wqe_end, int wqe_length) 1255 { 1256 struct mlx5_wqe_ctrl_seg *ctrl = *wqe; 1257 u16 wqe_index = pfault->wqe.wqe_index; 1258 struct mlx5_base_av *av; 1259 unsigned ds, opcode; 1260 u32 qpn = qp->trans_qp.base.mqp.qpn; 1261 1262 ds = be32_to_cpu(ctrl->qpn_ds) & MLX5_WQE_CTRL_DS_MASK; 1263 if (ds * MLX5_WQE_DS_UNITS > wqe_length) { 1264 mlx5_ib_err(dev, "Unable to read the complete WQE. ds = 0x%x, ret = 0x%x\n", 1265 ds, wqe_length); 1266 return -EFAULT; 1267 } 1268 1269 if (ds == 0) { 1270 mlx5_ib_err(dev, "Got WQE with zero DS. wqe_index=%x, qpn=%x\n", 1271 wqe_index, qpn); 1272 return -EFAULT; 1273 } 1274 1275 *wqe_end = *wqe + ds * MLX5_WQE_DS_UNITS; 1276 *wqe += sizeof(*ctrl); 1277 1278 opcode = be32_to_cpu(ctrl->opmod_idx_opcode) & 1279 MLX5_WQE_CTRL_OPCODE_MASK; 1280 1281 if (qp->type == IB_QPT_XRC_INI) 1282 *wqe += sizeof(struct mlx5_wqe_xrc_seg); 1283 1284 if (qp->type == IB_QPT_UD || qp->type == MLX5_IB_QPT_DCI) { 1285 av = *wqe; 1286 if (av->dqp_dct & cpu_to_be32(MLX5_EXTENDED_UD_AV)) 1287 *wqe += sizeof(struct mlx5_av); 1288 else 1289 *wqe += sizeof(struct mlx5_base_av); 1290 } 1291 1292 switch (opcode) { 1293 case MLX5_OPCODE_RDMA_WRITE: 1294 case MLX5_OPCODE_RDMA_WRITE_IMM: 1295 case MLX5_OPCODE_RDMA_READ: 1296 *wqe += sizeof(struct mlx5_wqe_raddr_seg); 1297 break; 1298 case MLX5_OPCODE_ATOMIC_CS: 1299 case MLX5_OPCODE_ATOMIC_FA: 1300 *wqe += sizeof(struct mlx5_wqe_raddr_seg); 1301 *wqe += sizeof(struct mlx5_wqe_atomic_seg); 1302 break; 1303 } 1304 1305 return 0; 1306 } 1307 1308 /* 1309 * Parse responder WQE and set wqe_end to the end of the WQE. 1310 */ 1311 static int mlx5_ib_mr_responder_pfault_handler_srq(struct mlx5_ib_dev *dev, 1312 struct mlx5_ib_srq *srq, 1313 void **wqe, void **wqe_end, 1314 int wqe_length) 1315 { 1316 int wqe_size = 1 << srq->msrq.wqe_shift; 1317 1318 if (wqe_size > wqe_length) { 1319 mlx5_ib_err(dev, "Couldn't read all of the receive WQE's content\n"); 1320 return -EFAULT; 1321 } 1322 1323 *wqe_end = *wqe + wqe_size; 1324 *wqe += sizeof(struct mlx5_wqe_srq_next_seg); 1325 1326 return 0; 1327 } 1328 1329 static int mlx5_ib_mr_responder_pfault_handler_rq(struct mlx5_ib_dev *dev, 1330 struct mlx5_ib_qp *qp, 1331 void *wqe, void **wqe_end, 1332 int wqe_length) 1333 { 1334 struct mlx5_ib_wq *wq = &qp->rq; 1335 int wqe_size = 1 << wq->wqe_shift; 1336 1337 if (qp->flags_en & MLX5_QP_FLAG_SIGNATURE) { 1338 mlx5_ib_err(dev, "ODP fault with WQE signatures is not supported\n"); 1339 return -EFAULT; 1340 } 1341 1342 if (wqe_size > wqe_length) { 1343 mlx5_ib_err(dev, "Couldn't read all of the receive WQE's content\n"); 1344 return -EFAULT; 1345 } 1346 1347 *wqe_end = wqe + wqe_size; 1348 1349 return 0; 1350 } 1351 1352 static inline struct mlx5_core_rsc_common *odp_get_rsc(struct mlx5_ib_dev *dev, 1353 u32 wq_num, int pf_type) 1354 { 1355 struct mlx5_core_rsc_common *common = NULL; 1356 struct mlx5_core_srq *srq; 1357 1358 switch (pf_type) { 1359 case MLX5_WQE_PF_TYPE_RMP: 1360 srq = mlx5_cmd_get_srq(dev, wq_num); 1361 if (srq) 1362 common = &srq->common; 1363 break; 1364 case MLX5_WQE_PF_TYPE_REQ_SEND_OR_WRITE: 1365 case MLX5_WQE_PF_TYPE_RESP: 1366 case MLX5_WQE_PF_TYPE_REQ_READ_OR_ATOMIC: 1367 common = mlx5_core_res_hold(dev, wq_num, MLX5_RES_QP); 1368 break; 1369 default: 1370 break; 1371 } 1372 1373 return common; 1374 } 1375 1376 static inline struct mlx5_ib_qp *res_to_qp(struct mlx5_core_rsc_common *res) 1377 { 1378 struct mlx5_core_qp *mqp = (struct mlx5_core_qp *)res; 1379 1380 return to_mibqp(mqp); 1381 } 1382 1383 static inline struct mlx5_ib_srq *res_to_srq(struct mlx5_core_rsc_common *res) 1384 { 1385 struct mlx5_core_srq *msrq = 1386 container_of(res, struct mlx5_core_srq, common); 1387 1388 return to_mibsrq(msrq); 1389 } 1390 1391 static void mlx5_ib_mr_wqe_pfault_handler(struct mlx5_ib_dev *dev, 1392 struct mlx5_pagefault *pfault) 1393 { 1394 bool sq = pfault->type & MLX5_PFAULT_REQUESTOR; 1395 u16 wqe_index = pfault->wqe.wqe_index; 1396 void *wqe, *wqe_start = NULL, *wqe_end = NULL; 1397 u32 bytes_mapped, total_wqe_bytes; 1398 struct mlx5_core_rsc_common *res; 1399 int resume_with_error = 1; 1400 struct mlx5_ib_qp *qp; 1401 size_t bytes_copied; 1402 int ret = 0; 1403 1404 res = odp_get_rsc(dev, pfault->wqe.wq_num, pfault->type); 1405 if (!res) { 1406 mlx5_ib_dbg(dev, "wqe page fault for missing resource %d\n", pfault->wqe.wq_num); 1407 return; 1408 } 1409 1410 if (res->res != MLX5_RES_QP && res->res != MLX5_RES_SRQ && 1411 res->res != MLX5_RES_XSRQ) { 1412 mlx5_ib_err(dev, "wqe page fault for unsupported type %d\n", 1413 pfault->type); 1414 goto resolve_page_fault; 1415 } 1416 1417 wqe_start = (void *)__get_free_page(GFP_KERNEL); 1418 if (!wqe_start) { 1419 mlx5_ib_err(dev, "Error allocating memory for IO page fault handling.\n"); 1420 goto resolve_page_fault; 1421 } 1422 1423 wqe = wqe_start; 1424 qp = (res->res == MLX5_RES_QP) ? res_to_qp(res) : NULL; 1425 if (qp && sq) { 1426 ret = mlx5_ib_read_wqe_sq(qp, wqe_index, wqe, PAGE_SIZE, 1427 &bytes_copied); 1428 if (ret) 1429 goto read_user; 1430 ret = mlx5_ib_mr_initiator_pfault_handler( 1431 dev, pfault, qp, &wqe, &wqe_end, bytes_copied); 1432 } else if (qp && !sq) { 1433 ret = mlx5_ib_read_wqe_rq(qp, wqe_index, wqe, PAGE_SIZE, 1434 &bytes_copied); 1435 if (ret) 1436 goto read_user; 1437 ret = mlx5_ib_mr_responder_pfault_handler_rq( 1438 dev, qp, wqe, &wqe_end, bytes_copied); 1439 } else if (!qp) { 1440 struct mlx5_ib_srq *srq = res_to_srq(res); 1441 1442 ret = mlx5_ib_read_wqe_srq(srq, wqe_index, wqe, PAGE_SIZE, 1443 &bytes_copied); 1444 if (ret) 1445 goto read_user; 1446 ret = mlx5_ib_mr_responder_pfault_handler_srq( 1447 dev, srq, &wqe, &wqe_end, bytes_copied); 1448 } 1449 1450 if (ret < 0 || wqe >= wqe_end) 1451 goto resolve_page_fault; 1452 1453 ret = pagefault_data_segments(dev, pfault, wqe, wqe_end, &bytes_mapped, 1454 &total_wqe_bytes, !sq); 1455 if (ret == -EAGAIN) 1456 goto out; 1457 1458 if (ret < 0 || total_wqe_bytes > bytes_mapped) 1459 goto resolve_page_fault; 1460 1461 out: 1462 ret = 0; 1463 resume_with_error = 0; 1464 1465 read_user: 1466 if (ret) 1467 mlx5_ib_err( 1468 dev, 1469 "Failed reading a WQE following page fault, error %d, wqe_index %x, qpn %llx\n", 1470 ret, wqe_index, pfault->token); 1471 1472 resolve_page_fault: 1473 mlx5_ib_page_fault_resume(dev, pfault, resume_with_error); 1474 mlx5_ib_dbg(dev, "PAGE FAULT completed. QP 0x%x resume_with_error=%d, type: 0x%x\n", 1475 pfault->wqe.wq_num, resume_with_error, 1476 pfault->type); 1477 mlx5_core_res_put(res); 1478 free_page((unsigned long)wqe_start); 1479 } 1480 1481 static void mlx5_ib_mr_rdma_pfault_handler(struct mlx5_ib_dev *dev, 1482 struct mlx5_pagefault *pfault) 1483 { 1484 u64 address; 1485 u32 length; 1486 u32 prefetch_len = pfault->bytes_committed; 1487 int prefetch_activated = 0; 1488 u32 rkey = pfault->rdma.r_key; 1489 int ret; 1490 1491 /* The RDMA responder handler handles the page fault in two parts. 1492 * First it brings the necessary pages for the current packet 1493 * (and uses the pfault context), and then (after resuming the QP) 1494 * prefetches more pages. The second operation cannot use the pfault 1495 * context and therefore uses the dummy_pfault context allocated on 1496 * the stack */ 1497 pfault->rdma.rdma_va += pfault->bytes_committed; 1498 pfault->rdma.rdma_op_len -= min(pfault->bytes_committed, 1499 pfault->rdma.rdma_op_len); 1500 pfault->bytes_committed = 0; 1501 1502 address = pfault->rdma.rdma_va; 1503 length = pfault->rdma.rdma_op_len; 1504 1505 /* For some operations, the hardware cannot tell the exact message 1506 * length, and in those cases it reports zero. Use prefetch 1507 * logic. */ 1508 if (length == 0) { 1509 prefetch_activated = 1; 1510 length = pfault->rdma.packet_size; 1511 prefetch_len = min(MAX_PREFETCH_LEN, prefetch_len); 1512 } 1513 1514 ret = pagefault_single_data_segment(dev, NULL, rkey, address, length, 1515 &pfault->bytes_committed, NULL); 1516 if (ret == -EAGAIN) { 1517 /* We're racing with an invalidation, don't prefetch */ 1518 prefetch_activated = 0; 1519 } else if (ret < 0) { 1520 mlx5_ib_page_fault_resume(dev, pfault, 1); 1521 if (ret != -ENOENT) 1522 mlx5_ib_dbg(dev, "PAGE FAULT error %d. QP 0x%llx, type: 0x%x\n", 1523 ret, pfault->token, pfault->type); 1524 return; 1525 } 1526 1527 mlx5_ib_page_fault_resume(dev, pfault, 0); 1528 mlx5_ib_dbg(dev, "PAGE FAULT completed. QP 0x%llx, type: 0x%x, prefetch_activated: %d\n", 1529 pfault->token, pfault->type, 1530 prefetch_activated); 1531 1532 /* At this point, there might be a new pagefault already arriving in 1533 * the eq, switch to the dummy pagefault for the rest of the 1534 * processing. We're still OK with the objects being alive as the 1535 * work-queue is being fenced. */ 1536 1537 if (prefetch_activated) { 1538 u32 bytes_committed = 0; 1539 1540 ret = pagefault_single_data_segment(dev, NULL, rkey, address, 1541 prefetch_len, 1542 &bytes_committed, NULL); 1543 if (ret < 0 && ret != -EAGAIN) { 1544 mlx5_ib_dbg(dev, "Prefetch failed. ret: %d, QP 0x%llx, address: 0x%.16llx, length = 0x%.16x\n", 1545 ret, pfault->token, address, prefetch_len); 1546 } 1547 } 1548 } 1549 1550 #define MLX5_MEMORY_PAGE_FAULT_FLAGS_LAST BIT(7) 1551 static void mlx5_ib_mr_memory_pfault_handler(struct mlx5_ib_dev *dev, 1552 struct mlx5_pagefault *pfault) 1553 { 1554 u64 prefetch_va = 1555 pfault->memory.va - pfault->memory.prefetch_before_byte_count; 1556 size_t prefetch_size = pfault->memory.prefetch_before_byte_count + 1557 pfault->memory.fault_byte_count + 1558 pfault->memory.prefetch_after_byte_count; 1559 struct mlx5_ib_mkey *mmkey; 1560 struct mlx5_ib_mr *mr, *child_mr; 1561 int ret = 0; 1562 1563 mmkey = find_odp_mkey(dev, pfault->memory.mkey); 1564 if (IS_ERR(mmkey)) 1565 goto err; 1566 1567 switch (mmkey->type) { 1568 case MLX5_MKEY_IMPLICIT_CHILD: 1569 child_mr = container_of(mmkey, struct mlx5_ib_mr, mmkey); 1570 mr = child_mr->parent; 1571 break; 1572 case MLX5_MKEY_NULL: 1573 mr = container_of(mmkey, struct mlx5_ib_mr, null_mmkey); 1574 break; 1575 default: 1576 mr = container_of(mmkey, struct mlx5_ib_mr, mmkey); 1577 break; 1578 } 1579 1580 /* If prefetch fails, handle only demanded page fault */ 1581 ret = pagefault_mr(mr, prefetch_va, prefetch_size, NULL, 0, true); 1582 if (ret < 0) { 1583 ret = pagefault_mr(mr, pfault->memory.va, 1584 pfault->memory.fault_byte_count, NULL, 0, true); 1585 if (ret < 0) 1586 goto err; 1587 } 1588 1589 mlx5_update_odp_stats_with_handled(mr, faults, ret); 1590 mlx5r_deref_odp_mkey(mmkey); 1591 1592 if (pfault->memory.flags & MLX5_MEMORY_PAGE_FAULT_FLAGS_LAST) 1593 mlx5_ib_page_fault_resume(dev, pfault, 0); 1594 1595 mlx5_ib_dbg( 1596 dev, 1597 "PAGE FAULT completed %s. token 0x%llx, mkey: 0x%x, va: 0x%llx, byte_count: 0x%x\n", 1598 pfault->memory.flags & MLX5_MEMORY_PAGE_FAULT_FLAGS_LAST ? 1599 "" : 1600 "without resume cmd", 1601 pfault->token, pfault->memory.mkey, pfault->memory.va, 1602 pfault->memory.fault_byte_count); 1603 1604 return; 1605 1606 err: 1607 if (!IS_ERR(mmkey)) 1608 mlx5r_deref_odp_mkey(mmkey); 1609 mlx5_ib_page_fault_resume(dev, pfault, 1); 1610 mlx5_ib_dbg( 1611 dev, 1612 "PAGE FAULT error. token 0x%llx, mkey: 0x%x, va: 0x%llx, byte_count: 0x%x, err: %d\n", 1613 pfault->token, pfault->memory.mkey, pfault->memory.va, 1614 pfault->memory.fault_byte_count, ret); 1615 } 1616 1617 static void mlx5_ib_pfault(struct mlx5_ib_dev *dev, struct mlx5_pagefault *pfault) 1618 { 1619 u8 event_subtype = pfault->event_subtype; 1620 1621 switch (event_subtype) { 1622 case MLX5_PFAULT_SUBTYPE_WQE: 1623 mlx5_ib_mr_wqe_pfault_handler(dev, pfault); 1624 break; 1625 case MLX5_PFAULT_SUBTYPE_RDMA: 1626 mlx5_ib_mr_rdma_pfault_handler(dev, pfault); 1627 break; 1628 case MLX5_PFAULT_SUBTYPE_MEMORY: 1629 mlx5_ib_mr_memory_pfault_handler(dev, pfault); 1630 break; 1631 default: 1632 mlx5_ib_err(dev, "Invalid page fault event subtype: 0x%x\n", 1633 event_subtype); 1634 mlx5_ib_page_fault_resume(dev, pfault, 1); 1635 } 1636 } 1637 1638 static void mlx5_ib_eqe_pf_action(struct work_struct *work) 1639 { 1640 struct mlx5_pagefault *pfault = container_of(work, 1641 struct mlx5_pagefault, 1642 work); 1643 struct mlx5_ib_pf_eq *eq = pfault->eq; 1644 1645 mlx5_ib_pfault(eq->dev, pfault); 1646 mempool_free(pfault, eq->pool); 1647 } 1648 1649 #define MEMORY_SCHEME_PAGE_FAULT_GRANULARITY 4096 1650 static void mlx5_ib_eq_pf_process(struct mlx5_ib_pf_eq *eq) 1651 { 1652 struct mlx5_eqe_page_fault *pf_eqe; 1653 struct mlx5_pagefault *pfault; 1654 struct mlx5_eqe *eqe; 1655 int cc = 0; 1656 1657 while ((eqe = mlx5_eq_get_eqe(eq->core, cc))) { 1658 pfault = mempool_alloc(eq->pool, GFP_ATOMIC); 1659 if (!pfault) { 1660 schedule_work(&eq->work); 1661 break; 1662 } 1663 1664 pf_eqe = &eqe->data.page_fault; 1665 pfault->event_subtype = eqe->sub_type; 1666 1667 switch (eqe->sub_type) { 1668 case MLX5_PFAULT_SUBTYPE_RDMA: 1669 /* RDMA based event */ 1670 pfault->bytes_committed = 1671 be32_to_cpu(pf_eqe->rdma.bytes_committed); 1672 pfault->type = 1673 be32_to_cpu(pf_eqe->rdma.pftype_token) >> 24; 1674 pfault->token = 1675 be32_to_cpu(pf_eqe->rdma.pftype_token) & 1676 MLX5_24BIT_MASK; 1677 pfault->rdma.r_key = 1678 be32_to_cpu(pf_eqe->rdma.r_key); 1679 pfault->rdma.packet_size = 1680 be16_to_cpu(pf_eqe->rdma.packet_length); 1681 pfault->rdma.rdma_op_len = 1682 be32_to_cpu(pf_eqe->rdma.rdma_op_len); 1683 pfault->rdma.rdma_va = 1684 be64_to_cpu(pf_eqe->rdma.rdma_va); 1685 mlx5_ib_dbg( 1686 eq->dev, 1687 "PAGE_FAULT: subtype: 0x%02x, bytes_committed: 0x%06x, type:0x%x, token: 0x%06llx, r_key: 0x%08x\n", 1688 eqe->sub_type, pfault->bytes_committed, 1689 pfault->type, pfault->token, 1690 pfault->rdma.r_key); 1691 mlx5_ib_dbg(eq->dev, 1692 "PAGE_FAULT: rdma_op_len: 0x%08x, rdma_va: 0x%016llx\n", 1693 pfault->rdma.rdma_op_len, 1694 pfault->rdma.rdma_va); 1695 break; 1696 1697 case MLX5_PFAULT_SUBTYPE_WQE: 1698 /* WQE based event */ 1699 pfault->bytes_committed = 1700 be32_to_cpu(pf_eqe->wqe.bytes_committed); 1701 pfault->type = 1702 (be32_to_cpu(pf_eqe->wqe.pftype_wq) >> 24) & 0x7; 1703 pfault->token = 1704 be32_to_cpu(pf_eqe->wqe.token); 1705 pfault->wqe.wq_num = 1706 be32_to_cpu(pf_eqe->wqe.pftype_wq) & 1707 MLX5_24BIT_MASK; 1708 pfault->wqe.wqe_index = 1709 be16_to_cpu(pf_eqe->wqe.wqe_index); 1710 pfault->wqe.packet_size = 1711 be16_to_cpu(pf_eqe->wqe.packet_length); 1712 mlx5_ib_dbg( 1713 eq->dev, 1714 "PAGE_FAULT: subtype: 0x%02x, bytes_committed: 0x%06x, type:0x%x, token: 0x%06llx, wq_num: 0x%06x, wqe_index: 0x%04x\n", 1715 eqe->sub_type, pfault->bytes_committed, 1716 pfault->type, pfault->token, pfault->wqe.wq_num, 1717 pfault->wqe.wqe_index); 1718 break; 1719 1720 case MLX5_PFAULT_SUBTYPE_MEMORY: 1721 /* Memory based event */ 1722 pfault->bytes_committed = 0; 1723 pfault->token = 1724 be32_to_cpu(pf_eqe->memory.token31_0) | 1725 ((u64)be16_to_cpu(pf_eqe->memory.token47_32) 1726 << 32); 1727 pfault->memory.va = be64_to_cpu(pf_eqe->memory.va); 1728 pfault->memory.mkey = be32_to_cpu(pf_eqe->memory.mkey); 1729 pfault->memory.fault_byte_count = (be32_to_cpu( 1730 pf_eqe->memory.demand_fault_pages) >> 12) * 1731 MEMORY_SCHEME_PAGE_FAULT_GRANULARITY; 1732 pfault->memory.prefetch_before_byte_count = 1733 be16_to_cpu( 1734 pf_eqe->memory.pre_demand_fault_pages) * 1735 MEMORY_SCHEME_PAGE_FAULT_GRANULARITY; 1736 pfault->memory.prefetch_after_byte_count = 1737 be16_to_cpu( 1738 pf_eqe->memory.post_demand_fault_pages) * 1739 MEMORY_SCHEME_PAGE_FAULT_GRANULARITY; 1740 pfault->memory.flags = pf_eqe->memory.flags; 1741 mlx5_ib_dbg( 1742 eq->dev, 1743 "PAGE_FAULT: subtype: 0x%02x, token: 0x%06llx, mkey: 0x%06x, fault_byte_count: 0x%06x, va: 0x%016llx, flags: 0x%02x\n", 1744 eqe->sub_type, pfault->token, 1745 pfault->memory.mkey, 1746 pfault->memory.fault_byte_count, 1747 pfault->memory.va, pfault->memory.flags); 1748 mlx5_ib_dbg( 1749 eq->dev, 1750 "PAGE_FAULT: prefetch size: before: 0x%06x, after 0x%06x\n", 1751 pfault->memory.prefetch_before_byte_count, 1752 pfault->memory.prefetch_after_byte_count); 1753 break; 1754 1755 default: 1756 mlx5_ib_warn(eq->dev, 1757 "Unsupported page fault event sub-type: 0x%02hhx\n", 1758 eqe->sub_type); 1759 /* Unsupported page faults should still be 1760 * resolved by the page fault handler 1761 */ 1762 } 1763 1764 pfault->eq = eq; 1765 INIT_WORK(&pfault->work, mlx5_ib_eqe_pf_action); 1766 queue_work(eq->wq, &pfault->work); 1767 1768 cc = mlx5_eq_update_cc(eq->core, ++cc); 1769 } 1770 1771 mlx5_eq_update_ci(eq->core, cc, 1); 1772 } 1773 1774 static int mlx5_ib_eq_pf_int(struct notifier_block *nb, unsigned long type, 1775 void *data) 1776 { 1777 struct mlx5_ib_pf_eq *eq = 1778 container_of(nb, struct mlx5_ib_pf_eq, irq_nb); 1779 unsigned long flags; 1780 1781 if (spin_trylock_irqsave(&eq->lock, flags)) { 1782 mlx5_ib_eq_pf_process(eq); 1783 spin_unlock_irqrestore(&eq->lock, flags); 1784 } else { 1785 schedule_work(&eq->work); 1786 } 1787 1788 return IRQ_HANDLED; 1789 } 1790 1791 /* mempool_refill() was proposed but unfortunately wasn't accepted 1792 * http://lkml.iu.edu/hypermail/linux/kernel/1512.1/05073.html 1793 * Cheap workaround. 1794 */ 1795 static void mempool_refill(mempool_t *pool) 1796 { 1797 while (pool->curr_nr < pool->min_nr) 1798 mempool_free(mempool_alloc(pool, GFP_KERNEL), pool); 1799 } 1800 1801 static void mlx5_ib_eq_pf_action(struct work_struct *work) 1802 { 1803 struct mlx5_ib_pf_eq *eq = 1804 container_of(work, struct mlx5_ib_pf_eq, work); 1805 1806 mempool_refill(eq->pool); 1807 1808 spin_lock_irq(&eq->lock); 1809 mlx5_ib_eq_pf_process(eq); 1810 spin_unlock_irq(&eq->lock); 1811 } 1812 1813 enum { 1814 MLX5_IB_NUM_PF_EQE = 0x1000, 1815 MLX5_IB_NUM_PF_DRAIN = 64, 1816 }; 1817 1818 int mlx5r_odp_create_eq(struct mlx5_ib_dev *dev, struct mlx5_ib_pf_eq *eq) 1819 { 1820 struct mlx5_eq_param param = {}; 1821 int err = 0; 1822 1823 if (!(dev->odp_caps.general_caps & IB_ODP_SUPPORT)) 1824 return -EOPNOTSUPP; 1825 1826 mutex_lock(&dev->odp_eq_mutex); 1827 if (eq->core) 1828 goto unlock; 1829 INIT_WORK(&eq->work, mlx5_ib_eq_pf_action); 1830 spin_lock_init(&eq->lock); 1831 eq->dev = dev; 1832 1833 eq->pool = mempool_create_kmalloc_pool(MLX5_IB_NUM_PF_DRAIN, 1834 sizeof(struct mlx5_pagefault)); 1835 if (!eq->pool) { 1836 err = -ENOMEM; 1837 goto unlock; 1838 } 1839 1840 eq->wq = alloc_workqueue("mlx5_ib_page_fault", 1841 WQ_HIGHPRI | WQ_UNBOUND | WQ_MEM_RECLAIM, 1842 MLX5_NUM_CMD_EQE); 1843 if (!eq->wq) { 1844 err = -ENOMEM; 1845 goto err_mempool; 1846 } 1847 1848 eq->irq_nb.notifier_call = mlx5_ib_eq_pf_int; 1849 param = (struct mlx5_eq_param) { 1850 .nent = MLX5_IB_NUM_PF_EQE, 1851 }; 1852 param.mask[0] = 1ull << MLX5_EVENT_TYPE_PAGE_FAULT; 1853 eq->core = mlx5_eq_create_generic(dev->mdev, ¶m); 1854 if (IS_ERR(eq->core)) { 1855 err = PTR_ERR(eq->core); 1856 goto err_wq; 1857 } 1858 err = mlx5_eq_enable(dev->mdev, eq->core, &eq->irq_nb); 1859 if (err) { 1860 mlx5_ib_err(dev, "failed to enable odp EQ %d\n", err); 1861 goto err_eq; 1862 } 1863 1864 mutex_unlock(&dev->odp_eq_mutex); 1865 return 0; 1866 err_eq: 1867 mlx5_eq_destroy_generic(dev->mdev, eq->core); 1868 err_wq: 1869 eq->core = NULL; 1870 destroy_workqueue(eq->wq); 1871 err_mempool: 1872 mempool_destroy(eq->pool); 1873 unlock: 1874 mutex_unlock(&dev->odp_eq_mutex); 1875 return err; 1876 } 1877 1878 static int 1879 mlx5_ib_odp_destroy_eq(struct mlx5_ib_dev *dev, struct mlx5_ib_pf_eq *eq) 1880 { 1881 int err; 1882 1883 if (!eq->core) 1884 return 0; 1885 mlx5_eq_disable(dev->mdev, eq->core, &eq->irq_nb); 1886 err = mlx5_eq_destroy_generic(dev->mdev, eq->core); 1887 cancel_work_sync(&eq->work); 1888 destroy_workqueue(eq->wq); 1889 mempool_destroy(eq->pool); 1890 1891 return err; 1892 } 1893 1894 static const struct ib_device_ops mlx5_ib_dev_odp_ops = { 1895 .advise_mr = mlx5_ib_advise_mr, 1896 }; 1897 1898 int mlx5_ib_odp_init_one(struct mlx5_ib_dev *dev) 1899 { 1900 internal_fill_odp_caps(dev); 1901 1902 if (!(dev->odp_caps.general_caps & IB_ODP_SUPPORT)) 1903 return 0; 1904 1905 ib_set_device_ops(&dev->ib_dev, &mlx5_ib_dev_odp_ops); 1906 1907 mutex_init(&dev->odp_eq_mutex); 1908 return 0; 1909 } 1910 1911 void mlx5_ib_odp_cleanup_one(struct mlx5_ib_dev *dev) 1912 { 1913 if (!(dev->odp_caps.general_caps & IB_ODP_SUPPORT)) 1914 return; 1915 1916 mlx5_ib_odp_destroy_eq(dev, &dev->odp_pf_eq); 1917 } 1918 1919 int mlx5_ib_odp_init(void) 1920 { 1921 u32 log_va_pages = ilog2(TASK_SIZE) - PAGE_SHIFT; 1922 u8 mlx5_imr_mtt_bits; 1923 1924 /* 48 is default ARM64 VA space and covers X86 4-level paging which is 47 */ 1925 if (log_va_pages <= 48 - PAGE_SHIFT) 1926 mlx5_imr_mtt_shift = 30; 1927 /* 56 is x86-64, 5-level paging */ 1928 else if (log_va_pages <= 56 - PAGE_SHIFT) 1929 mlx5_imr_mtt_shift = 34; 1930 else 1931 return 0; 1932 1933 mlx5_imr_mtt_size = BIT_ULL(mlx5_imr_mtt_shift); 1934 mlx5_imr_mtt_bits = mlx5_imr_mtt_shift - PAGE_SHIFT; 1935 mlx5_imr_mtt_entries = BIT_ULL(mlx5_imr_mtt_bits); 1936 mlx5_imr_ksm_entries = BIT_ULL(get_order(TASK_SIZE) - 1937 mlx5_imr_mtt_bits); 1938 1939 mlx5_imr_ksm_page_shift = mlx5_imr_mtt_shift; 1940 return 0; 1941 } 1942 1943 struct prefetch_mr_work { 1944 struct work_struct work; 1945 u32 pf_flags; 1946 u32 num_sge; 1947 struct { 1948 u64 io_virt; 1949 struct mlx5_ib_mr *mr; 1950 size_t length; 1951 } frags[]; 1952 }; 1953 1954 static void destroy_prefetch_work(struct prefetch_mr_work *work) 1955 { 1956 u32 i; 1957 1958 for (i = 0; i < work->num_sge; ++i) 1959 mlx5r_deref_odp_mkey(&work->frags[i].mr->mmkey); 1960 1961 kvfree(work); 1962 } 1963 1964 static struct mlx5_ib_mr * 1965 get_prefetchable_mr(struct ib_pd *pd, enum ib_uverbs_advise_mr_advice advice, 1966 u32 lkey) 1967 { 1968 struct mlx5_ib_dev *dev = to_mdev(pd->device); 1969 struct mlx5_ib_mr *mr = NULL; 1970 struct mlx5_ib_mkey *mmkey; 1971 1972 xa_lock(&dev->odp_mkeys); 1973 mmkey = xa_load(&dev->odp_mkeys, mlx5_base_mkey(lkey)); 1974 if (!mmkey || mmkey->key != lkey) { 1975 mr = ERR_PTR(-ENOENT); 1976 goto end; 1977 } 1978 if (mmkey->type != MLX5_MKEY_MR) { 1979 mr = ERR_PTR(-EINVAL); 1980 goto end; 1981 } 1982 1983 mr = container_of(mmkey, struct mlx5_ib_mr, mmkey); 1984 1985 if (mr->ibmr.pd != pd) { 1986 mr = ERR_PTR(-EPERM); 1987 goto end; 1988 } 1989 1990 /* prefetch with write-access must be supported by the MR */ 1991 if (advice == IB_UVERBS_ADVISE_MR_ADVICE_PREFETCH_WRITE && 1992 !mr->umem->writable) { 1993 mr = ERR_PTR(-EPERM); 1994 goto end; 1995 } 1996 1997 refcount_inc(&mmkey->usecount); 1998 end: 1999 xa_unlock(&dev->odp_mkeys); 2000 return mr; 2001 } 2002 2003 static void mlx5_ib_prefetch_mr_work(struct work_struct *w) 2004 { 2005 struct prefetch_mr_work *work = 2006 container_of(w, struct prefetch_mr_work, work); 2007 u32 bytes_mapped = 0; 2008 int ret; 2009 u32 i; 2010 2011 /* We rely on IB/core that work is executed if we have num_sge != 0 only. */ 2012 WARN_ON(!work->num_sge); 2013 for (i = 0; i < work->num_sge; ++i) { 2014 ret = pagefault_mr(work->frags[i].mr, work->frags[i].io_virt, 2015 work->frags[i].length, &bytes_mapped, 2016 work->pf_flags, false); 2017 if (ret <= 0) 2018 continue; 2019 mlx5_update_odp_stats(work->frags[i].mr, prefetch, ret); 2020 } 2021 2022 destroy_prefetch_work(work); 2023 } 2024 2025 static int init_prefetch_work(struct ib_pd *pd, 2026 enum ib_uverbs_advise_mr_advice advice, 2027 u32 pf_flags, struct prefetch_mr_work *work, 2028 struct ib_sge *sg_list, u32 num_sge) 2029 { 2030 u32 i; 2031 2032 INIT_WORK(&work->work, mlx5_ib_prefetch_mr_work); 2033 work->pf_flags = pf_flags; 2034 2035 for (i = 0; i < num_sge; ++i) { 2036 struct mlx5_ib_mr *mr; 2037 2038 mr = get_prefetchable_mr(pd, advice, sg_list[i].lkey); 2039 if (IS_ERR(mr)) { 2040 work->num_sge = i; 2041 return PTR_ERR(mr); 2042 } 2043 work->frags[i].io_virt = sg_list[i].addr; 2044 work->frags[i].length = sg_list[i].length; 2045 work->frags[i].mr = mr; 2046 } 2047 work->num_sge = num_sge; 2048 return 0; 2049 } 2050 2051 static int mlx5_ib_prefetch_sg_list(struct ib_pd *pd, 2052 enum ib_uverbs_advise_mr_advice advice, 2053 u32 pf_flags, struct ib_sge *sg_list, 2054 u32 num_sge) 2055 { 2056 u32 bytes_mapped = 0; 2057 int ret = 0; 2058 u32 i; 2059 2060 for (i = 0; i < num_sge; ++i) { 2061 struct mlx5_ib_mr *mr; 2062 2063 mr = get_prefetchable_mr(pd, advice, sg_list[i].lkey); 2064 if (IS_ERR(mr)) 2065 return PTR_ERR(mr); 2066 ret = pagefault_mr(mr, sg_list[i].addr, sg_list[i].length, 2067 &bytes_mapped, pf_flags, false); 2068 if (ret < 0) { 2069 mlx5r_deref_odp_mkey(&mr->mmkey); 2070 return ret; 2071 } 2072 mlx5_update_odp_stats(mr, prefetch, ret); 2073 mlx5r_deref_odp_mkey(&mr->mmkey); 2074 } 2075 2076 return 0; 2077 } 2078 2079 int mlx5_ib_advise_mr_prefetch(struct ib_pd *pd, 2080 enum ib_uverbs_advise_mr_advice advice, 2081 u32 flags, struct ib_sge *sg_list, u32 num_sge) 2082 { 2083 u32 pf_flags = 0; 2084 struct prefetch_mr_work *work; 2085 int rc; 2086 2087 if (advice == IB_UVERBS_ADVISE_MR_ADVICE_PREFETCH) 2088 pf_flags |= MLX5_PF_FLAGS_DOWNGRADE; 2089 2090 if (advice == IB_UVERBS_ADVISE_MR_ADVICE_PREFETCH_NO_FAULT) 2091 pf_flags |= MLX5_PF_FLAGS_SNAPSHOT; 2092 2093 if (flags & IB_UVERBS_ADVISE_MR_FLAG_FLUSH) 2094 return mlx5_ib_prefetch_sg_list(pd, advice, pf_flags, sg_list, 2095 num_sge); 2096 2097 work = kvzalloc_flex(*work, frags, num_sge); 2098 if (!work) 2099 return -ENOMEM; 2100 2101 rc = init_prefetch_work(pd, advice, pf_flags, work, sg_list, num_sge); 2102 if (rc) { 2103 destroy_prefetch_work(work); 2104 return rc; 2105 } 2106 queue_work(system_dfl_wq, &work->work); 2107 return 0; 2108 } 2109