1 /* 2 * Copyright (c) 2013-2015, Mellanox Technologies. All rights reserved. 3 * 4 * This software is available to you under a choice of one of two 5 * licenses. You may choose to be licensed under the terms of the GNU 6 * General Public License (GPL) Version 2, available from the file 7 * COPYING in the main directory of this source tree, or the 8 * OpenIB.org BSD license below: 9 * 10 * Redistribution and use in source and binary forms, with or 11 * without modification, are permitted provided that the following 12 * conditions are met: 13 * 14 * - Redistributions of source code must retain the above 15 * copyright notice, this list of conditions and the following 16 * disclaimer. 17 * 18 * - Redistributions in binary form must reproduce the above 19 * copyright notice, this list of conditions and the following 20 * disclaimer in the documentation and/or other materials 21 * provided with the distribution. 22 * 23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30 * SOFTWARE. 31 */ 32 33 #include <rdma/ib_umem_odp.h> 34 #include <linux/kernel.h> 35 #include <linux/dma-buf.h> 36 #include <linux/dma-resv.h> 37 #include <linux/hmm.h> 38 #include <linux/hmm-dma.h> 39 #include <linux/pci-p2pdma.h> 40 #include <linux/slab.h> 41 42 #include "mlx5_ib.h" 43 #include "cmd.h" 44 #include "umr.h" 45 #include "qp.h" 46 47 #include <linux/mlx5/eq.h> 48 49 /* Contains the details of a pagefault. */ 50 struct mlx5_pagefault { 51 u32 bytes_committed; 52 u64 token; 53 u8 event_subtype; 54 u8 type; 55 union { 56 /* Initiator or send message responder pagefault details. */ 57 struct { 58 /* Received packet size, only valid for responders. */ 59 u32 packet_size; 60 /* 61 * Number of resource holding WQE, depends on type. 62 */ 63 u32 wq_num; 64 /* 65 * WQE index. Refers to either the send queue or 66 * receive queue, according to event_subtype. 67 */ 68 u16 wqe_index; 69 } wqe; 70 /* RDMA responder pagefault details */ 71 struct { 72 u32 r_key; 73 /* 74 * Received packet size, minimal size page fault 75 * resolution required for forward progress. 76 */ 77 u32 packet_size; 78 u32 rdma_op_len; 79 u64 rdma_va; 80 } rdma; 81 struct { 82 u64 va; 83 u32 mkey; 84 u32 fault_byte_count; 85 u32 prefetch_before_byte_count; 86 u32 prefetch_after_byte_count; 87 u8 flags; 88 } memory; 89 }; 90 91 struct mlx5_ib_pf_eq *eq; 92 struct work_struct work; 93 }; 94 95 #define MAX_PREFETCH_LEN (4*1024*1024U) 96 97 /* Timeout in ms to wait for an active mmu notifier to complete when handling 98 * a pagefault. */ 99 #define MMU_NOTIFIER_TIMEOUT 1000 100 101 static u64 mlx5_imr_ksm_entries; 102 static u64 mlx5_imr_mtt_entries; 103 static u64 mlx5_imr_mtt_size; 104 static u8 mlx5_imr_mtt_shift; 105 static u8 mlx5_imr_ksm_page_shift; 106 107 static void populate_ksm(struct mlx5_ksm *pksm, size_t idx, size_t nentries, 108 struct mlx5_ib_mr *imr, int flags) 109 { 110 struct mlx5_core_dev *dev = mr_to_mdev(imr)->mdev; 111 struct mlx5_ksm *end = pksm + nentries; 112 u64 step = MLX5_CAP_ODP(dev, mem_page_fault) ? mlx5_imr_mtt_size : 0; 113 __be32 key = MLX5_CAP_ODP(dev, mem_page_fault) ? 114 cpu_to_be32(imr->null_mmkey.key) : 115 mr_to_mdev(imr)->mkeys.null_mkey; 116 u64 va = 117 MLX5_CAP_ODP(dev, mem_page_fault) ? idx * mlx5_imr_mtt_size : 0; 118 119 if (flags & MLX5_IB_UPD_XLT_ZAP) { 120 for (; pksm != end; pksm++, idx++, va += step) { 121 pksm->key = key; 122 pksm->va = cpu_to_be64(va); 123 } 124 return; 125 } 126 127 /* 128 * The locking here is pretty subtle. Ideally the implicit_children 129 * xarray would be protected by the umem_mutex, however that is not 130 * possible. Instead this uses a weaker update-then-lock pattern: 131 * 132 * xa_store() 133 * mutex_lock(umem_mutex) 134 * mlx5r_umr_update_xlt() 135 * mutex_unlock(umem_mutex) 136 * destroy lkey 137 * 138 * ie any change the xarray must be followed by the locked update_xlt 139 * before destroying. 140 * 141 * The umem_mutex provides the acquire/release semantic needed to make 142 * the xa_store() visible to a racing thread. 143 */ 144 lockdep_assert_held(&to_ib_umem_odp(imr->umem)->umem_mutex); 145 146 for (; pksm != end; pksm++, idx++, va += step) { 147 struct mlx5_ib_mr *mtt = xa_load(&imr->implicit_children, idx); 148 149 if (mtt) { 150 pksm->key = cpu_to_be32(mtt->ibmr.lkey); 151 pksm->va = cpu_to_be64(idx * mlx5_imr_mtt_size); 152 } else { 153 pksm->key = key; 154 pksm->va = cpu_to_be64(va); 155 } 156 } 157 } 158 159 static int populate_mtt(__be64 *pas, size_t start, size_t nentries, 160 struct mlx5_ib_mr *mr, int flags) 161 { 162 struct ib_umem_odp *odp = to_ib_umem_odp(mr->umem); 163 bool downgrade = flags & MLX5_IB_UPD_XLT_DOWNGRADE; 164 struct pci_p2pdma_map_state p2pdma_state = {}; 165 struct ib_device *dev = odp->umem.ibdev; 166 size_t i; 167 168 if (flags & MLX5_IB_UPD_XLT_ZAP) 169 return 0; 170 171 for (i = 0; i < nentries; i++) { 172 unsigned long pfn = odp->map.pfn_list[start + i]; 173 dma_addr_t dma_addr; 174 175 pfn = odp->map.pfn_list[start + i]; 176 if (!(pfn & HMM_PFN_VALID)) 177 /* ODP initialization */ 178 continue; 179 180 dma_addr = hmm_dma_map_pfn(dev->dma_device, &odp->map, 181 start + i, &p2pdma_state); 182 if (ib_dma_mapping_error(dev, dma_addr)) 183 return -EFAULT; 184 185 dma_addr |= MLX5_IB_MTT_READ; 186 if ((pfn & HMM_PFN_WRITE) && !downgrade) 187 dma_addr |= MLX5_IB_MTT_WRITE; 188 189 pas[i] = cpu_to_be64(dma_addr); 190 odp->npages++; 191 } 192 return 0; 193 } 194 195 int mlx5_odp_populate_xlt(void *xlt, size_t idx, size_t nentries, 196 struct mlx5_ib_mr *mr, int flags) 197 { 198 if (flags & MLX5_IB_UPD_XLT_INDIRECT) { 199 populate_ksm(xlt, idx, nentries, mr, flags); 200 return 0; 201 } else { 202 return populate_mtt(xlt, idx, nentries, mr, flags); 203 } 204 } 205 206 /* 207 * This must be called after the mr has been removed from implicit_children. 208 * NOTE: The MR does not necessarily have to be 209 * empty here, parallel page faults could have raced with the free process and 210 * added pages to it. 211 */ 212 static void free_implicit_child_mr_work(struct work_struct *work) 213 { 214 struct mlx5_ib_mr *mr = 215 container_of(work, struct mlx5_ib_mr, odp_destroy.work); 216 struct mlx5_ib_mr *imr = mr->parent; 217 struct ib_umem_odp *odp_imr = to_ib_umem_odp(imr->umem); 218 struct ib_umem_odp *odp = to_ib_umem_odp(mr->umem); 219 220 mlx5r_deref_wait_odp_mkey(&mr->mmkey); 221 222 mutex_lock(&odp_imr->umem_mutex); 223 mlx5r_umr_update_xlt(mr->parent, 224 ib_umem_start(odp) >> mlx5_imr_mtt_shift, 1, 0, 225 MLX5_IB_UPD_XLT_INDIRECT | MLX5_IB_UPD_XLT_ATOMIC, 226 0); 227 mutex_unlock(&odp_imr->umem_mutex); 228 mlx5_ib_dereg_mr(&mr->ibmr, NULL); 229 230 mlx5r_deref_odp_mkey(&imr->mmkey); 231 } 232 233 static void destroy_unused_implicit_child_mr(struct mlx5_ib_mr *mr) 234 { 235 struct ib_umem_odp *odp = to_ib_umem_odp(mr->umem); 236 unsigned long idx = ib_umem_start(odp) >> mlx5_imr_mtt_shift; 237 struct mlx5_ib_mr *imr = mr->parent; 238 239 /* 240 * If userspace is racing freeing the parent implicit ODP MR then we can 241 * loose the race with parent destruction. In this case 242 * mlx5_ib_free_odp_mr() will free everything in the implicit_children 243 * xarray so NOP is fine. This child MR cannot be destroyed here because 244 * we are under its umem_mutex. 245 */ 246 if (!refcount_inc_not_zero(&imr->mmkey.usecount)) 247 return; 248 249 xa_lock(&imr->implicit_children); 250 if (__xa_cmpxchg(&imr->implicit_children, idx, mr, NULL, GFP_KERNEL) != 251 mr) { 252 xa_unlock(&imr->implicit_children); 253 mlx5r_deref_odp_mkey(&imr->mmkey); 254 return; 255 } 256 257 if (MLX5_CAP_ODP(mr_to_mdev(mr)->mdev, mem_page_fault)) 258 xa_erase(&mr_to_mdev(mr)->odp_mkeys, 259 mlx5_base_mkey(mr->mmkey.key)); 260 xa_unlock(&imr->implicit_children); 261 262 /* Freeing a MR is a sleeping operation, so bounce to a work queue */ 263 INIT_WORK(&mr->odp_destroy.work, free_implicit_child_mr_work); 264 queue_work(system_dfl_wq, &mr->odp_destroy.work); 265 } 266 267 static bool mlx5_ib_invalidate_range(struct mmu_interval_notifier *mni, 268 const struct mmu_notifier_range *range, 269 unsigned long cur_seq) 270 { 271 struct ib_umem_odp *umem_odp = 272 container_of(mni, struct ib_umem_odp, notifier); 273 struct mlx5_ib_mr *mr; 274 const u64 umr_block_mask = MLX5_UMR_MTT_NUM_ENTRIES_ALIGNMENT - 1; 275 u64 idx = 0, blk_start_idx = 0; 276 u64 invalidations = 0; 277 unsigned long start; 278 unsigned long end; 279 int in_block = 0; 280 u64 addr; 281 282 if (!mmu_notifier_range_blockable(range)) 283 return false; 284 285 mutex_lock(&umem_odp->umem_mutex); 286 mmu_interval_set_seq(mni, cur_seq); 287 /* 288 * If npages is zero then umem_odp->private may not be setup yet. This 289 * does not complete until after the first page is mapped for DMA. 290 */ 291 if (!umem_odp->npages) 292 goto out; 293 mr = umem_odp->private; 294 if (!mr) 295 goto out; 296 297 start = max_t(u64, ib_umem_start(umem_odp), range->start); 298 end = min_t(u64, ib_umem_end(umem_odp), range->end); 299 300 /* 301 * Iteration one - zap the HW's MTTs. The notifiers_count ensures that 302 * while we are doing the invalidation, no page fault will attempt to 303 * overwrite the same MTTs. Concurent invalidations might race us, 304 * but they will write 0s as well, so no difference in the end result. 305 */ 306 for (addr = start; addr < end; addr += BIT(umem_odp->page_shift)) { 307 idx = (addr - ib_umem_start(umem_odp)) >> umem_odp->page_shift; 308 /* 309 * Strive to write the MTTs in chunks, but avoid overwriting 310 * non-existing MTTs. The huristic here can be improved to 311 * estimate the cost of another UMR vs. the cost of bigger 312 * UMR. 313 */ 314 if (umem_odp->map.pfn_list[idx] & HMM_PFN_VALID) { 315 if (!in_block) { 316 blk_start_idx = idx; 317 in_block = 1; 318 } 319 } else { 320 u64 umr_offset = idx & umr_block_mask; 321 322 if (in_block && umr_offset == 0) { 323 mlx5r_umr_update_xlt( 324 mr, blk_start_idx, idx - blk_start_idx, 325 0, 326 MLX5_IB_UPD_XLT_ZAP | 327 MLX5_IB_UPD_XLT_ATOMIC, 328 0); 329 in_block = 0; 330 /* Count page invalidations */ 331 invalidations += idx - blk_start_idx + 1; 332 } 333 } 334 } 335 if (in_block) { 336 mlx5r_umr_update_xlt( 337 mr, blk_start_idx, idx - blk_start_idx + 1, 0, 338 MLX5_IB_UPD_XLT_ZAP | MLX5_IB_UPD_XLT_ATOMIC, 0); 339 /* Count page invalidations */ 340 invalidations += idx - blk_start_idx + 1; 341 } 342 343 mlx5_update_odp_stats_with_handled(mr, invalidations, invalidations); 344 345 /* 346 * We are now sure that the device will not access the 347 * memory. We can safely unmap it, and mark it as dirty if 348 * needed. 349 */ 350 351 ib_umem_odp_unmap_dma_pages(umem_odp, start, end); 352 353 if (unlikely(!umem_odp->npages && mr->parent)) 354 destroy_unused_implicit_child_mr(mr); 355 out: 356 mutex_unlock(&umem_odp->umem_mutex); 357 return true; 358 } 359 360 const struct mmu_interval_notifier_ops mlx5_mn_ops = { 361 .invalidate = mlx5_ib_invalidate_range, 362 }; 363 364 static void internal_fill_odp_caps(struct mlx5_ib_dev *dev) 365 { 366 struct ib_odp_caps *caps = &dev->odp_caps; 367 368 memset(caps, 0, sizeof(*caps)); 369 370 if (!MLX5_CAP_GEN(dev->mdev, pg) || !mlx5r_umr_can_load_pas(dev, 0)) 371 return; 372 373 caps->general_caps = IB_ODP_SUPPORT; 374 375 if (MLX5_CAP_GEN(dev->mdev, umr_extended_translation_offset)) 376 dev->odp_max_size = U64_MAX; 377 else 378 dev->odp_max_size = BIT_ULL(MLX5_MAX_UMR_SHIFT + PAGE_SHIFT); 379 380 if (MLX5_CAP_ODP_SCHEME(dev->mdev, ud_odp_caps.send)) 381 caps->per_transport_caps.ud_odp_caps |= IB_ODP_SUPPORT_SEND; 382 383 if (MLX5_CAP_ODP_SCHEME(dev->mdev, ud_odp_caps.srq_receive)) 384 caps->per_transport_caps.ud_odp_caps |= IB_ODP_SUPPORT_SRQ_RECV; 385 386 if (MLX5_CAP_ODP_SCHEME(dev->mdev, rc_odp_caps.send)) 387 caps->per_transport_caps.rc_odp_caps |= IB_ODP_SUPPORT_SEND; 388 389 if (MLX5_CAP_ODP_SCHEME(dev->mdev, rc_odp_caps.receive)) 390 caps->per_transport_caps.rc_odp_caps |= IB_ODP_SUPPORT_RECV; 391 392 if (MLX5_CAP_ODP_SCHEME(dev->mdev, rc_odp_caps.write)) 393 caps->per_transport_caps.rc_odp_caps |= IB_ODP_SUPPORT_WRITE; 394 395 if (MLX5_CAP_ODP_SCHEME(dev->mdev, rc_odp_caps.read)) 396 caps->per_transport_caps.rc_odp_caps |= IB_ODP_SUPPORT_READ; 397 398 if (MLX5_CAP_ODP_SCHEME(dev->mdev, rc_odp_caps.atomic)) 399 caps->per_transport_caps.rc_odp_caps |= IB_ODP_SUPPORT_ATOMIC; 400 401 if (MLX5_CAP_ODP_SCHEME(dev->mdev, rc_odp_caps.srq_receive)) 402 caps->per_transport_caps.rc_odp_caps |= IB_ODP_SUPPORT_SRQ_RECV; 403 404 if (MLX5_CAP_ODP_SCHEME(dev->mdev, xrc_odp_caps.send)) 405 caps->per_transport_caps.xrc_odp_caps |= IB_ODP_SUPPORT_SEND; 406 407 if (MLX5_CAP_ODP_SCHEME(dev->mdev, xrc_odp_caps.receive)) 408 caps->per_transport_caps.xrc_odp_caps |= IB_ODP_SUPPORT_RECV; 409 410 if (MLX5_CAP_ODP_SCHEME(dev->mdev, xrc_odp_caps.write)) 411 caps->per_transport_caps.xrc_odp_caps |= IB_ODP_SUPPORT_WRITE; 412 413 if (MLX5_CAP_ODP_SCHEME(dev->mdev, xrc_odp_caps.read)) 414 caps->per_transport_caps.xrc_odp_caps |= IB_ODP_SUPPORT_READ; 415 416 if (MLX5_CAP_ODP_SCHEME(dev->mdev, xrc_odp_caps.atomic)) 417 caps->per_transport_caps.xrc_odp_caps |= IB_ODP_SUPPORT_ATOMIC; 418 419 if (MLX5_CAP_ODP_SCHEME(dev->mdev, xrc_odp_caps.srq_receive)) 420 caps->per_transport_caps.xrc_odp_caps |= IB_ODP_SUPPORT_SRQ_RECV; 421 422 if (MLX5_CAP_GEN(dev->mdev, fixed_buffer_size) && 423 MLX5_CAP_GEN(dev->mdev, null_mkey) && 424 MLX5_CAP_GEN(dev->mdev, umr_extended_translation_offset) && 425 !MLX5_CAP_GEN(dev->mdev, umr_indirect_mkey_disabled) && 426 mlx5_imr_ksm_entries != 0 && 427 !(mlx5_imr_ksm_page_shift > 428 get_max_log_entity_size_cap(dev, MLX5_MKC_ACCESS_MODE_KSM))) 429 caps->general_caps |= IB_ODP_SUPPORT_IMPLICIT; 430 } 431 432 static void mlx5_ib_page_fault_resume(struct mlx5_ib_dev *dev, 433 struct mlx5_pagefault *pfault, 434 int error) 435 { 436 int wq_num = pfault->event_subtype == MLX5_PFAULT_SUBTYPE_WQE ? 437 pfault->wqe.wq_num : pfault->token; 438 u32 in[MLX5_ST_SZ_DW(page_fault_resume_in)] = {}; 439 void *info; 440 int err; 441 442 MLX5_SET(page_fault_resume_in, in, opcode, MLX5_CMD_OP_PAGE_FAULT_RESUME); 443 444 if (pfault->event_subtype == MLX5_PFAULT_SUBTYPE_MEMORY) { 445 info = MLX5_ADDR_OF(page_fault_resume_in, in, 446 page_fault_info.mem_page_fault_info); 447 MLX5_SET(mem_page_fault_info, info, fault_token_31_0, 448 pfault->token & 0xffffffff); 449 MLX5_SET(mem_page_fault_info, info, fault_token_47_32, 450 (pfault->token >> 32) & 0xffff); 451 MLX5_SET(mem_page_fault_info, info, error, !!error); 452 } else { 453 info = MLX5_ADDR_OF(page_fault_resume_in, in, 454 page_fault_info.trans_page_fault_info); 455 MLX5_SET(trans_page_fault_info, info, page_fault_type, 456 pfault->type); 457 MLX5_SET(trans_page_fault_info, info, fault_token, 458 pfault->token); 459 MLX5_SET(trans_page_fault_info, info, wq_number, wq_num); 460 MLX5_SET(trans_page_fault_info, info, error, !!error); 461 } 462 463 err = mlx5_cmd_exec_in(dev->mdev, page_fault_resume, in); 464 if (err) 465 mlx5_ib_err(dev, "Failed to resolve the page fault on WQ 0x%x err %d\n", 466 wq_num, err); 467 } 468 469 static struct mlx5_ib_mr *implicit_get_child_mr(struct mlx5_ib_mr *imr, 470 unsigned long idx) 471 { 472 struct mlx5_ib_dev *dev = mr_to_mdev(imr); 473 struct ib_umem_odp *odp; 474 struct mlx5_ib_mr *mr; 475 struct mlx5_ib_mr *ret; 476 int err; 477 478 odp = ib_umem_odp_alloc_child(to_ib_umem_odp(imr->umem), 479 idx * mlx5_imr_mtt_size, 480 mlx5_imr_mtt_size, &mlx5_mn_ops); 481 if (IS_ERR(odp)) 482 return ERR_CAST(odp); 483 484 mr = mlx5_mr_cache_alloc(dev, imr->access_flags, 485 MLX5_MKC_ACCESS_MODE_MTT, 486 mlx5_imr_mtt_entries); 487 if (IS_ERR(mr)) { 488 ib_umem_odp_release(odp); 489 return mr; 490 } 491 492 mr->access_flags = imr->access_flags; 493 mr->ibmr.pd = imr->ibmr.pd; 494 mr->ibmr.device = &mr_to_mdev(imr)->ib_dev; 495 mr->umem = &odp->umem; 496 mr->ibmr.lkey = mr->mmkey.key; 497 mr->ibmr.rkey = mr->mmkey.key; 498 mr->ibmr.iova = idx * mlx5_imr_mtt_size; 499 mr->parent = imr; 500 odp->private = mr; 501 502 /* 503 * First refcount is owned by the xarray and second refconut 504 * is returned to the caller. 505 */ 506 refcount_set(&mr->mmkey.usecount, 2); 507 508 err = mlx5r_umr_update_xlt(mr, 0, mlx5_imr_mtt_entries, PAGE_SHIFT, 509 MLX5_IB_UPD_XLT_ZAP | MLX5_IB_UPD_XLT_ENABLE, 510 to_mpd(mr->ibmr.pd)->pdn); 511 if (err) { 512 ret = ERR_PTR(err); 513 goto out_mr; 514 } 515 516 xa_lock(&imr->implicit_children); 517 ret = __xa_cmpxchg(&imr->implicit_children, idx, NULL, mr, 518 GFP_KERNEL); 519 if (unlikely(ret)) { 520 if (xa_is_err(ret)) { 521 ret = ERR_PTR(xa_err(ret)); 522 goto out_lock; 523 } 524 /* 525 * Another thread beat us to creating the child mr, use 526 * theirs. 527 */ 528 refcount_inc(&ret->mmkey.usecount); 529 goto out_lock; 530 } 531 532 if (MLX5_CAP_ODP(dev->mdev, mem_page_fault)) { 533 ret = xa_store(&dev->odp_mkeys, mlx5_base_mkey(mr->mmkey.key), 534 &mr->mmkey, GFP_KERNEL); 535 if (xa_is_err(ret)) { 536 ret = ERR_PTR(xa_err(ret)); 537 __xa_erase(&imr->implicit_children, idx); 538 goto out_lock; 539 } 540 mr->mmkey.type = MLX5_MKEY_IMPLICIT_CHILD; 541 } 542 xa_unlock(&imr->implicit_children); 543 mlx5_ib_dbg(mr_to_mdev(imr), "key %x mr %p\n", mr->mmkey.key, mr); 544 return mr; 545 546 out_lock: 547 xa_unlock(&imr->implicit_children); 548 out_mr: 549 mlx5_ib_dereg_mr(&mr->ibmr, NULL); 550 return ret; 551 } 552 553 /* 554 * When using memory scheme ODP, implicit MRs can't use the reserved null mkey 555 * and each implicit MR needs to assign a private null mkey to get the page 556 * faults on. 557 * The null mkey is created with the properties to enable getting the page 558 * fault for every time it is accessed and having all relevant access flags. 559 */ 560 static int alloc_implicit_mr_null_mkey(struct mlx5_ib_dev *dev, 561 struct mlx5_ib_mr *imr, 562 struct mlx5_ib_pd *pd) 563 { 564 size_t inlen = MLX5_ST_SZ_BYTES(create_mkey_in) + 64; 565 void *mkc; 566 u32 *in; 567 int err; 568 569 in = kzalloc(inlen, GFP_KERNEL); 570 if (!in) 571 return -ENOMEM; 572 573 MLX5_SET(create_mkey_in, in, translations_octword_actual_size, 4); 574 MLX5_SET(create_mkey_in, in, pg_access, 1); 575 576 mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry); 577 MLX5_SET(mkc, mkc, a, 1); 578 MLX5_SET(mkc, mkc, rw, 1); 579 MLX5_SET(mkc, mkc, rr, 1); 580 MLX5_SET(mkc, mkc, lw, 1); 581 MLX5_SET(mkc, mkc, lr, 1); 582 MLX5_SET(mkc, mkc, free, 0); 583 MLX5_SET(mkc, mkc, umr_en, 0); 584 MLX5_SET(mkc, mkc, access_mode_1_0, MLX5_MKC_ACCESS_MODE_MTT); 585 586 MLX5_SET(mkc, mkc, translations_octword_size, 4); 587 MLX5_SET(mkc, mkc, log_page_size, 61); 588 MLX5_SET(mkc, mkc, length64, 1); 589 MLX5_SET(mkc, mkc, pd, pd->pdn); 590 MLX5_SET64(mkc, mkc, start_addr, 0); 591 MLX5_SET(mkc, mkc, qpn, 0xffffff); 592 593 err = mlx5_core_create_mkey(dev->mdev, &imr->null_mmkey.key, in, inlen); 594 if (err) 595 goto free_in; 596 597 imr->null_mmkey.type = MLX5_MKEY_NULL; 598 599 free_in: 600 kfree(in); 601 return err; 602 } 603 604 struct mlx5_ib_mr *mlx5_ib_alloc_implicit_mr(struct mlx5_ib_pd *pd, 605 int access_flags) 606 { 607 struct mlx5_ib_dev *dev = to_mdev(pd->ibpd.device); 608 struct ib_umem_odp *umem_odp; 609 struct mlx5_ib_mr *imr; 610 int err; 611 612 if (!mlx5r_umr_can_load_pas(dev, mlx5_imr_mtt_entries * PAGE_SIZE)) 613 return ERR_PTR(-EOPNOTSUPP); 614 615 umem_odp = ib_umem_odp_alloc_implicit(&dev->ib_dev, access_flags); 616 if (IS_ERR(umem_odp)) 617 return ERR_CAST(umem_odp); 618 619 imr = mlx5_mr_cache_alloc(dev, access_flags, MLX5_MKC_ACCESS_MODE_KSM, 620 mlx5_imr_ksm_entries); 621 if (IS_ERR(imr)) { 622 ib_umem_odp_release(umem_odp); 623 return imr; 624 } 625 626 imr->access_flags = access_flags; 627 imr->ibmr.pd = &pd->ibpd; 628 imr->ibmr.iova = 0; 629 imr->umem = &umem_odp->umem; 630 imr->ibmr.lkey = imr->mmkey.key; 631 imr->ibmr.rkey = imr->mmkey.key; 632 imr->ibmr.device = &dev->ib_dev; 633 imr->is_odp_implicit = true; 634 xa_init(&imr->implicit_children); 635 636 if (MLX5_CAP_ODP(dev->mdev, mem_page_fault)) { 637 err = alloc_implicit_mr_null_mkey(dev, imr, pd); 638 if (err) 639 goto out_mr; 640 641 err = mlx5r_store_odp_mkey(dev, &imr->null_mmkey); 642 if (err) 643 goto out_mr; 644 } 645 646 err = mlx5r_umr_update_xlt(imr, 0, 647 mlx5_imr_ksm_entries, 648 mlx5_imr_ksm_page_shift, 649 MLX5_IB_UPD_XLT_INDIRECT | 650 MLX5_IB_UPD_XLT_ZAP | 651 MLX5_IB_UPD_XLT_ENABLE, 652 pd->pdn); 653 if (err) 654 goto out_mr; 655 656 err = mlx5r_store_odp_mkey(dev, &imr->mmkey); 657 if (err) 658 goto out_mr; 659 660 mlx5_ib_dbg(dev, "key %x mr %p\n", imr->mmkey.key, imr); 661 return imr; 662 out_mr: 663 mlx5_ib_err(dev, "Failed to register MKEY %d\n", err); 664 mlx5_ib_dereg_mr(&imr->ibmr, NULL); 665 return ERR_PTR(err); 666 } 667 668 void mlx5_ib_free_odp_mr(struct mlx5_ib_mr *mr) 669 { 670 struct mlx5_ib_mr *mtt; 671 unsigned long idx; 672 673 /* 674 * If this is an implicit MR it is already invalidated so we can just 675 * delete the children mkeys. 676 */ 677 xa_for_each(&mr->implicit_children, idx, mtt) { 678 xa_erase(&mr->implicit_children, idx); 679 mlx5_ib_dereg_mr(&mtt->ibmr, NULL); 680 } 681 682 if (mr->null_mmkey.key) { 683 xa_erase(&mr_to_mdev(mr)->odp_mkeys, 684 mlx5_base_mkey(mr->null_mmkey.key)); 685 686 mlx5_core_destroy_mkey(mr_to_mdev(mr)->mdev, 687 mr->null_mmkey.key); 688 } 689 } 690 691 /* 692 * pdn must be valid only when xlt_flags updates the mkey PD. In this path that 693 * is only MLX5_PF_FLAGS_ENABLE. DOWNGRADE and SNAPSHOT leave the PD masked out. 694 */ 695 #define MLX5_PF_FLAGS_DOWNGRADE BIT(1) 696 #define MLX5_PF_FLAGS_SNAPSHOT BIT(2) 697 #define MLX5_PF_FLAGS_ENABLE BIT(3) 698 static int pagefault_real_mr(struct mlx5_ib_mr *mr, struct ib_umem_odp *odp, 699 u64 user_va, size_t bcnt, u32 *bytes_mapped, 700 u32 flags, u32 pdn) 701 { 702 int page_shift, ret, np; 703 bool downgrade = flags & MLX5_PF_FLAGS_DOWNGRADE; 704 u64 access_mask = 0; 705 u64 start_idx; 706 bool fault = !(flags & MLX5_PF_FLAGS_SNAPSHOT); 707 u32 xlt_flags = MLX5_IB_UPD_XLT_ATOMIC; 708 709 if (flags & MLX5_PF_FLAGS_ENABLE) 710 xlt_flags |= MLX5_IB_UPD_XLT_ENABLE; 711 712 if (flags & MLX5_PF_FLAGS_DOWNGRADE) 713 xlt_flags |= MLX5_IB_UPD_XLT_DOWNGRADE; 714 715 page_shift = odp->page_shift; 716 start_idx = (user_va - ib_umem_start(odp)) >> page_shift; 717 718 if (odp->umem.writable && !downgrade) 719 access_mask |= HMM_PFN_WRITE; 720 721 np = ib_umem_odp_map_dma_and_lock(odp, user_va, bcnt, access_mask, fault); 722 if (np < 0) 723 return np; 724 725 /* 726 * No need to check whether the MTTs really belong to this MR, since 727 * ib_umem_odp_map_dma_and_lock already checks this. 728 */ 729 ret = mlx5r_umr_update_xlt(mr, start_idx, np, page_shift, xlt_flags, 730 pdn); 731 mutex_unlock(&odp->umem_mutex); 732 733 if (ret < 0) { 734 if (ret != -EAGAIN) 735 mlx5_ib_err(mr_to_mdev(mr), 736 "Failed to update mkey page tables\n"); 737 goto out; 738 } 739 740 if (bytes_mapped) { 741 u32 new_mappings = (np << page_shift) - 742 (user_va - round_down(user_va, 1 << page_shift)); 743 744 *bytes_mapped += min_t(u32, new_mappings, bcnt); 745 } 746 747 return np << (page_shift - PAGE_SHIFT); 748 749 out: 750 return ret; 751 } 752 753 static int pagefault_implicit_mr(struct mlx5_ib_mr *imr, 754 struct ib_umem_odp *odp_imr, u64 user_va, 755 size_t bcnt, u32 *bytes_mapped, u32 flags) 756 { 757 unsigned long end_idx = (user_va + bcnt - 1) >> mlx5_imr_mtt_shift; 758 unsigned long upd_start_idx = end_idx + 1; 759 unsigned long upd_len = 0; 760 unsigned long npages = 0; 761 int err; 762 int ret; 763 764 if (unlikely(user_va >= mlx5_imr_ksm_entries * mlx5_imr_mtt_size || 765 mlx5_imr_ksm_entries * mlx5_imr_mtt_size - user_va < bcnt)) 766 return -EFAULT; 767 768 /* Fault each child mr that intersects with our interval. */ 769 while (bcnt) { 770 unsigned long idx = user_va >> mlx5_imr_mtt_shift; 771 struct ib_umem_odp *umem_odp; 772 struct mlx5_ib_mr *mtt; 773 u64 len; 774 775 xa_lock(&imr->implicit_children); 776 mtt = xa_load(&imr->implicit_children, idx); 777 if (unlikely(!mtt)) { 778 xa_unlock(&imr->implicit_children); 779 mtt = implicit_get_child_mr(imr, idx); 780 if (IS_ERR(mtt)) { 781 ret = PTR_ERR(mtt); 782 goto out; 783 } 784 upd_start_idx = min(upd_start_idx, idx); 785 upd_len = idx - upd_start_idx + 1; 786 } else { 787 refcount_inc(&mtt->mmkey.usecount); 788 xa_unlock(&imr->implicit_children); 789 } 790 791 umem_odp = to_ib_umem_odp(mtt->umem); 792 len = min_t(u64, user_va + bcnt, ib_umem_end(umem_odp)) - 793 user_va; 794 795 ret = pagefault_real_mr(mtt, umem_odp, user_va, len, 796 bytes_mapped, flags, 0); 797 798 mlx5r_deref_odp_mkey(&mtt->mmkey); 799 800 if (ret < 0) 801 goto out; 802 user_va += len; 803 bcnt -= len; 804 npages += ret; 805 } 806 807 ret = npages; 808 809 /* 810 * Any time the implicit_children are changed we must perform an 811 * update of the xlt before exiting to ensure the HW and the 812 * implicit_children remains synchronized. 813 */ 814 out: 815 if (likely(!upd_len)) 816 return ret; 817 818 /* 819 * Notice this is not strictly ordered right, the KSM is updated after 820 * the implicit_children is updated, so a parallel page fault could 821 * see a MR that is not yet visible in the KSM. This is similar to a 822 * parallel page fault seeing a MR that is being concurrently removed 823 * from the KSM. Both of these improbable situations are resolved 824 * safely by resuming the HW and then taking another page fault. The 825 * next pagefault handler will see the new information. 826 */ 827 mutex_lock(&odp_imr->umem_mutex); 828 err = mlx5r_umr_update_xlt( 829 imr, upd_start_idx, upd_len, 0, 830 MLX5_IB_UPD_XLT_INDIRECT | MLX5_IB_UPD_XLT_ATOMIC, 0); 831 mutex_unlock(&odp_imr->umem_mutex); 832 if (err) { 833 mlx5_ib_err(mr_to_mdev(imr), "Failed to update PAS\n"); 834 return err; 835 } 836 return ret; 837 } 838 839 static int pagefault_dmabuf_mr(struct mlx5_ib_mr *mr, size_t bcnt, 840 u32 *bytes_mapped, u32 flags, u32 pdn) 841 { 842 struct ib_umem_dmabuf *umem_dmabuf = to_ib_umem_dmabuf(mr->umem); 843 int access_mode = mr->data_direct ? MLX5_MKC_ACCESS_MODE_KSM : 844 MLX5_MKC_ACCESS_MODE_MTT; 845 unsigned int old_page_shift = mr->page_shift; 846 unsigned int page_shift; 847 unsigned long page_size; 848 u32 xlt_flags = 0; 849 int err; 850 851 if (flags & MLX5_PF_FLAGS_ENABLE) 852 xlt_flags |= MLX5_IB_UPD_XLT_ENABLE; 853 854 dma_resv_lock(umem_dmabuf->attach->dmabuf->resv, NULL); 855 err = ib_umem_dmabuf_map_pages(umem_dmabuf); 856 if (err) { 857 dma_resv_unlock(umem_dmabuf->attach->dmabuf->resv); 858 return err; 859 } 860 861 page_size = mlx5_umem_dmabuf_find_best_pgsz(umem_dmabuf, access_mode); 862 if (!page_size) { 863 ib_umem_dmabuf_unmap_pages(umem_dmabuf); 864 err = -EINVAL; 865 } else { 866 page_shift = order_base_2(page_size); 867 if (page_shift != mr->page_shift && mr->dmabuf_faulted) { 868 err = mlx5r_umr_dmabuf_update_pgsz(mr, xlt_flags, pdn, 869 page_shift); 870 } else { 871 mr->page_shift = page_shift; 872 if (mr->data_direct) 873 err = mlx5r_umr_update_data_direct_ksm_pas( 874 mr, xlt_flags); 875 else 876 err = mlx5r_umr_update_mr_pas(mr, xlt_flags, 877 pdn); 878 } 879 } 880 dma_resv_unlock(umem_dmabuf->attach->dmabuf->resv); 881 882 if (err) { 883 mr->page_shift = old_page_shift; 884 return err; 885 } 886 887 mr->dmabuf_faulted = 1; 888 889 if (bytes_mapped) 890 *bytes_mapped += bcnt; 891 892 return ib_umem_num_pages(mr->umem); 893 } 894 895 /* 896 * Returns: 897 * -EFAULT: The io_virt->bcnt is not within the MR, it covers pages that are 898 * not accessible, or the MR is no longer valid. 899 * -EAGAIN/-ENOMEM: The operation should be retried 900 * 901 * -EINVAL/others: General internal malfunction 902 * >0: Number of pages mapped 903 */ 904 static int pagefault_mr(struct mlx5_ib_mr *mr, u64 io_virt, size_t bcnt, 905 u32 *bytes_mapped, u32 flags, bool permissive_fault) 906 { 907 struct ib_umem_odp *odp = to_ib_umem_odp(mr->umem); 908 909 if (unlikely(io_virt < mr->ibmr.iova) && !permissive_fault) 910 return -EFAULT; 911 912 if (mr->umem->is_dmabuf) 913 return pagefault_dmabuf_mr(mr, bcnt, bytes_mapped, flags, 0); 914 915 if (!odp->is_implicit_odp) { 916 u64 offset = io_virt < mr->ibmr.iova ? 0 : io_virt - mr->ibmr.iova; 917 u64 user_va; 918 919 if (check_add_overflow(offset, (u64)odp->umem.address, 920 &user_va)) 921 return -EFAULT; 922 923 if (permissive_fault) { 924 if (user_va < ib_umem_start(odp)) 925 user_va = ib_umem_start(odp); 926 if ((user_va + bcnt) > ib_umem_end(odp)) 927 bcnt = ib_umem_end(odp) - user_va; 928 } else if (unlikely(user_va >= ib_umem_end(odp) || 929 ib_umem_end(odp) - user_va < bcnt)) 930 return -EFAULT; 931 return pagefault_real_mr(mr, odp, user_va, bcnt, bytes_mapped, 932 flags, 0); 933 } 934 return pagefault_implicit_mr(mr, odp, io_virt, bcnt, bytes_mapped, 935 flags); 936 } 937 938 int mlx5_ib_init_odp_mr(struct mlx5_ib_mr *mr, struct ib_pd *pd) 939 { 940 int ret; 941 942 ret = pagefault_real_mr(mr, to_ib_umem_odp(mr->umem), mr->umem->address, 943 mr->umem->length, NULL, 944 MLX5_PF_FLAGS_SNAPSHOT | MLX5_PF_FLAGS_ENABLE, 945 to_mpd(pd)->pdn); 946 return ret >= 0 ? 0 : ret; 947 } 948 949 int mlx5_ib_init_dmabuf_mr(struct mlx5_ib_mr *mr, struct ib_pd *pd) 950 { 951 struct mlx5_ib_dev *dev = mr_to_mdev(mr); 952 u32 pdn; 953 int ret; 954 955 if (mr->data_direct) 956 pdn = dev->ddr.pdn; 957 else 958 pdn = to_mpd(pd)->pdn; 959 960 ret = pagefault_dmabuf_mr(mr, mr->umem->length, NULL, 961 MLX5_PF_FLAGS_ENABLE, pdn); 962 963 return ret >= 0 ? 0 : ret; 964 } 965 966 struct pf_frame { 967 struct pf_frame *next; 968 u32 key; 969 u64 io_virt; 970 size_t bcnt; 971 int depth; 972 }; 973 974 static bool mkey_is_eq(struct mlx5_ib_mkey *mmkey, u32 key) 975 { 976 if (!mmkey) 977 return false; 978 if (mmkey->type == MLX5_MKEY_MW || 979 mmkey->type == MLX5_MKEY_INDIRECT_DEVX) 980 return mlx5_base_mkey(mmkey->key) == mlx5_base_mkey(key); 981 return mmkey->key == key; 982 } 983 984 static struct mlx5_ib_mkey *find_odp_mkey(struct mlx5_ib_dev *dev, u32 key) 985 { 986 struct mlx5_ib_mkey *mmkey; 987 988 xa_lock(&dev->odp_mkeys); 989 mmkey = xa_load(&dev->odp_mkeys, mlx5_base_mkey(key)); 990 if (!mmkey) { 991 mmkey = ERR_PTR(-ENOENT); 992 goto out; 993 } 994 if (!mkey_is_eq(mmkey, key)) { 995 mmkey = ERR_PTR(-EFAULT); 996 goto out; 997 } 998 refcount_inc(&mmkey->usecount); 999 out: 1000 xa_unlock(&dev->odp_mkeys); 1001 1002 return mmkey; 1003 } 1004 1005 /* 1006 * Handle a single data segment in a page-fault WQE or RDMA region. 1007 * 1008 * Returns zero on success. The caller may continue to the next data segment. 1009 * Can return the following error codes: 1010 * -EAGAIN to designate a temporary error. The caller will abort handling the 1011 * page fault and resolve it. 1012 * -EFAULT when there's an error mapping the requested pages. The caller will 1013 * abort the page fault handling. 1014 */ 1015 static int pagefault_single_data_segment(struct mlx5_ib_dev *dev, 1016 struct ib_pd *pd, u32 key, 1017 u64 io_virt, size_t bcnt, 1018 u32 *bytes_committed, 1019 u32 *bytes_mapped) 1020 { 1021 int ret, i, outlen, cur_outlen = 0, depth = 0, pages_in_range; 1022 struct pf_frame *head = NULL, *frame; 1023 struct mlx5_ib_mkey *mmkey; 1024 struct mlx5_ib_mr *mr; 1025 struct mlx5_klm *pklm; 1026 u32 *out = NULL; 1027 size_t offset; 1028 1029 io_virt += *bytes_committed; 1030 bcnt -= *bytes_committed; 1031 next_mr: 1032 mmkey = find_odp_mkey(dev, key); 1033 if (IS_ERR(mmkey)) { 1034 ret = PTR_ERR(mmkey); 1035 if (ret == -ENOENT) { 1036 mlx5_ib_dbg( 1037 dev, 1038 "skipping non ODP MR (lkey=0x%06x) in page fault handler.\n", 1039 key); 1040 if (bytes_mapped) 1041 *bytes_mapped += bcnt; 1042 /* 1043 * The user could specify a SGL with multiple lkeys and 1044 * only some of them are ODP. Treat the non-ODP ones as 1045 * fully faulted. 1046 */ 1047 ret = 0; 1048 } 1049 goto end; 1050 } 1051 1052 switch (mmkey->type) { 1053 case MLX5_MKEY_MR: 1054 mr = container_of(mmkey, struct mlx5_ib_mr, mmkey); 1055 1056 pages_in_range = (ALIGN(io_virt + bcnt, PAGE_SIZE) - 1057 (io_virt & PAGE_MASK)) >> 1058 PAGE_SHIFT; 1059 ret = pagefault_mr(mr, io_virt, bcnt, bytes_mapped, 0, false); 1060 if (ret < 0) 1061 goto end; 1062 1063 mlx5_update_odp_stats_with_handled(mr, faults, ret); 1064 1065 if (ret < pages_in_range) { 1066 ret = -EFAULT; 1067 goto end; 1068 } 1069 1070 ret = 0; 1071 break; 1072 1073 case MLX5_MKEY_MW: 1074 case MLX5_MKEY_INDIRECT_DEVX: 1075 if (depth >= MLX5_CAP_GEN(dev->mdev, max_indirection)) { 1076 mlx5_ib_dbg(dev, "indirection level exceeded\n"); 1077 ret = -EFAULT; 1078 goto end; 1079 } 1080 1081 outlen = MLX5_ST_SZ_BYTES(query_mkey_out) + 1082 sizeof(*pklm) * (mmkey->ndescs - 2); 1083 1084 if (outlen > cur_outlen) { 1085 kfree(out); 1086 out = kzalloc(outlen, GFP_KERNEL); 1087 if (!out) { 1088 ret = -ENOMEM; 1089 goto end; 1090 } 1091 cur_outlen = outlen; 1092 } 1093 1094 pklm = (struct mlx5_klm *)MLX5_ADDR_OF(query_mkey_out, out, 1095 bsf0_klm0_pas_mtt0_1); 1096 1097 ret = mlx5_core_query_mkey(dev->mdev, mmkey->key, out, outlen); 1098 if (ret) 1099 goto end; 1100 1101 offset = io_virt - MLX5_GET64(query_mkey_out, out, 1102 memory_key_mkey_entry.start_addr); 1103 1104 for (i = 0; bcnt && i < mmkey->ndescs; i++, pklm++) { 1105 if (offset >= be32_to_cpu(pklm->bcount)) { 1106 offset -= be32_to_cpu(pklm->bcount); 1107 continue; 1108 } 1109 1110 frame = kzalloc_obj(*frame); 1111 if (!frame) { 1112 ret = -ENOMEM; 1113 goto end; 1114 } 1115 1116 frame->key = be32_to_cpu(pklm->key); 1117 frame->io_virt = be64_to_cpu(pklm->va) + offset; 1118 frame->bcnt = min_t(size_t, bcnt, 1119 be32_to_cpu(pklm->bcount) - offset); 1120 frame->depth = depth + 1; 1121 frame->next = head; 1122 head = frame; 1123 1124 bcnt -= frame->bcnt; 1125 offset = 0; 1126 } 1127 break; 1128 1129 default: 1130 mlx5_ib_dbg(dev, "wrong mkey type %d\n", mmkey->type); 1131 ret = -EFAULT; 1132 goto end; 1133 } 1134 1135 if (head) { 1136 frame = head; 1137 head = frame->next; 1138 1139 key = frame->key; 1140 io_virt = frame->io_virt; 1141 bcnt = frame->bcnt; 1142 depth = frame->depth; 1143 kfree(frame); 1144 1145 mlx5r_deref_odp_mkey(mmkey); 1146 goto next_mr; 1147 } 1148 1149 end: 1150 if (!IS_ERR(mmkey)) 1151 mlx5r_deref_odp_mkey(mmkey); 1152 while (head) { 1153 frame = head; 1154 head = frame->next; 1155 kfree(frame); 1156 } 1157 kfree(out); 1158 1159 *bytes_committed = 0; 1160 return ret; 1161 } 1162 1163 /* 1164 * Parse a series of data segments for page fault handling. 1165 * 1166 * @dev: Pointer to mlx5 IB device 1167 * @pfault: contains page fault information. 1168 * @wqe: points at the first data segment in the WQE. 1169 * @wqe_end: points after the end of the WQE. 1170 * @bytes_mapped: receives the number of bytes that the function was able to 1171 * map. This allows the caller to decide intelligently whether 1172 * enough memory was mapped to resolve the page fault 1173 * successfully (e.g. enough for the next MTU, or the entire 1174 * WQE). 1175 * @total_wqe_bytes: receives the total data size of this WQE in bytes (minus 1176 * the committed bytes). 1177 * @receive_queue: receive WQE end of sg list 1178 * 1179 * Returns zero for success or a negative error code. 1180 */ 1181 static int pagefault_data_segments(struct mlx5_ib_dev *dev, 1182 struct mlx5_pagefault *pfault, 1183 void *wqe, 1184 void *wqe_end, u32 *bytes_mapped, 1185 u32 *total_wqe_bytes, bool receive_queue) 1186 { 1187 int ret = 0; 1188 u64 io_virt; 1189 __be32 key; 1190 u32 byte_count; 1191 size_t bcnt; 1192 int inline_segment; 1193 1194 if (bytes_mapped) 1195 *bytes_mapped = 0; 1196 if (total_wqe_bytes) 1197 *total_wqe_bytes = 0; 1198 1199 while (wqe < wqe_end) { 1200 struct mlx5_wqe_data_seg *dseg = wqe; 1201 1202 io_virt = be64_to_cpu(dseg->addr); 1203 key = dseg->lkey; 1204 byte_count = be32_to_cpu(dseg->byte_count); 1205 inline_segment = !!(byte_count & MLX5_INLINE_SEG); 1206 bcnt = byte_count & ~MLX5_INLINE_SEG; 1207 1208 if (inline_segment) { 1209 bcnt = bcnt & MLX5_WQE_INLINE_SEG_BYTE_COUNT_MASK; 1210 wqe += ALIGN(sizeof(struct mlx5_wqe_inline_seg) + bcnt, 1211 16); 1212 } else { 1213 wqe += sizeof(*dseg); 1214 } 1215 1216 /* receive WQE end of sg list. */ 1217 if (receive_queue && bcnt == 0 && 1218 key == dev->mkeys.terminate_scatter_list_mkey && 1219 io_virt == 0) 1220 break; 1221 1222 if (!inline_segment && total_wqe_bytes) { 1223 *total_wqe_bytes += bcnt - min_t(size_t, bcnt, 1224 pfault->bytes_committed); 1225 } 1226 1227 /* A zero length data segment designates a length of 2GB. */ 1228 if (bcnt == 0) 1229 bcnt = 1U << 31; 1230 1231 if (inline_segment || bcnt <= pfault->bytes_committed) { 1232 pfault->bytes_committed -= 1233 min_t(size_t, bcnt, 1234 pfault->bytes_committed); 1235 continue; 1236 } 1237 1238 ret = pagefault_single_data_segment(dev, NULL, be32_to_cpu(key), 1239 io_virt, bcnt, 1240 &pfault->bytes_committed, 1241 bytes_mapped); 1242 if (ret < 0) 1243 break; 1244 } 1245 1246 return ret; 1247 } 1248 1249 /* 1250 * Parse initiator WQE. Advances the wqe pointer to point at the 1251 * scatter-gather list, and set wqe_end to the end of the WQE. 1252 */ 1253 static int mlx5_ib_mr_initiator_pfault_handler( 1254 struct mlx5_ib_dev *dev, struct mlx5_pagefault *pfault, 1255 struct mlx5_ib_qp *qp, void **wqe, void **wqe_end, int wqe_length) 1256 { 1257 struct mlx5_wqe_ctrl_seg *ctrl = *wqe; 1258 u16 wqe_index = pfault->wqe.wqe_index; 1259 struct mlx5_base_av *av; 1260 unsigned ds, opcode; 1261 u32 qpn = qp->trans_qp.base.mqp.qpn; 1262 1263 ds = be32_to_cpu(ctrl->qpn_ds) & MLX5_WQE_CTRL_DS_MASK; 1264 if (ds * MLX5_WQE_DS_UNITS > wqe_length) { 1265 mlx5_ib_err(dev, "Unable to read the complete WQE. ds = 0x%x, ret = 0x%x\n", 1266 ds, wqe_length); 1267 return -EFAULT; 1268 } 1269 1270 if (ds == 0) { 1271 mlx5_ib_err(dev, "Got WQE with zero DS. wqe_index=%x, qpn=%x\n", 1272 wqe_index, qpn); 1273 return -EFAULT; 1274 } 1275 1276 *wqe_end = *wqe + ds * MLX5_WQE_DS_UNITS; 1277 *wqe += sizeof(*ctrl); 1278 1279 opcode = be32_to_cpu(ctrl->opmod_idx_opcode) & 1280 MLX5_WQE_CTRL_OPCODE_MASK; 1281 1282 if (qp->type == IB_QPT_XRC_INI) 1283 *wqe += sizeof(struct mlx5_wqe_xrc_seg); 1284 1285 if (qp->type == IB_QPT_UD || qp->type == MLX5_IB_QPT_DCI) { 1286 av = *wqe; 1287 if (av->dqp_dct & cpu_to_be32(MLX5_EXTENDED_UD_AV)) 1288 *wqe += sizeof(struct mlx5_av); 1289 else 1290 *wqe += sizeof(struct mlx5_base_av); 1291 } 1292 1293 switch (opcode) { 1294 case MLX5_OPCODE_RDMA_WRITE: 1295 case MLX5_OPCODE_RDMA_WRITE_IMM: 1296 case MLX5_OPCODE_RDMA_READ: 1297 *wqe += sizeof(struct mlx5_wqe_raddr_seg); 1298 break; 1299 case MLX5_OPCODE_ATOMIC_CS: 1300 case MLX5_OPCODE_ATOMIC_FA: 1301 *wqe += sizeof(struct mlx5_wqe_raddr_seg); 1302 *wqe += sizeof(struct mlx5_wqe_atomic_seg); 1303 break; 1304 } 1305 1306 return 0; 1307 } 1308 1309 /* 1310 * Parse responder WQE and set wqe_end to the end of the WQE. 1311 */ 1312 static int mlx5_ib_mr_responder_pfault_handler_srq(struct mlx5_ib_dev *dev, 1313 struct mlx5_ib_srq *srq, 1314 void **wqe, void **wqe_end, 1315 int wqe_length) 1316 { 1317 int wqe_size = 1 << srq->msrq.wqe_shift; 1318 1319 if (wqe_size > wqe_length) { 1320 mlx5_ib_err(dev, "Couldn't read all of the receive WQE's content\n"); 1321 return -EFAULT; 1322 } 1323 1324 *wqe_end = *wqe + wqe_size; 1325 *wqe += sizeof(struct mlx5_wqe_srq_next_seg); 1326 1327 return 0; 1328 } 1329 1330 static int mlx5_ib_mr_responder_pfault_handler_rq(struct mlx5_ib_dev *dev, 1331 struct mlx5_ib_qp *qp, 1332 void *wqe, void **wqe_end, 1333 int wqe_length) 1334 { 1335 struct mlx5_ib_wq *wq = &qp->rq; 1336 int wqe_size = 1 << wq->wqe_shift; 1337 1338 if (qp->flags_en & MLX5_QP_FLAG_SIGNATURE) { 1339 mlx5_ib_err(dev, "ODP fault with WQE signatures is not supported\n"); 1340 return -EFAULT; 1341 } 1342 1343 if (wqe_size > wqe_length) { 1344 mlx5_ib_err(dev, "Couldn't read all of the receive WQE's content\n"); 1345 return -EFAULT; 1346 } 1347 1348 *wqe_end = wqe + wqe_size; 1349 1350 return 0; 1351 } 1352 1353 static inline struct mlx5_core_rsc_common *odp_get_rsc(struct mlx5_ib_dev *dev, 1354 u32 wq_num, int pf_type) 1355 { 1356 struct mlx5_core_rsc_common *common = NULL; 1357 struct mlx5_core_srq *srq; 1358 1359 switch (pf_type) { 1360 case MLX5_WQE_PF_TYPE_RMP: 1361 srq = mlx5_cmd_get_srq(dev, wq_num); 1362 if (srq) 1363 common = &srq->common; 1364 break; 1365 case MLX5_WQE_PF_TYPE_REQ_SEND_OR_WRITE: 1366 case MLX5_WQE_PF_TYPE_RESP: 1367 case MLX5_WQE_PF_TYPE_REQ_READ_OR_ATOMIC: 1368 common = mlx5_core_res_hold(dev, wq_num, MLX5_RES_QP); 1369 break; 1370 default: 1371 break; 1372 } 1373 1374 return common; 1375 } 1376 1377 static inline struct mlx5_ib_qp *res_to_qp(struct mlx5_core_rsc_common *res) 1378 { 1379 struct mlx5_core_qp *mqp = (struct mlx5_core_qp *)res; 1380 1381 return to_mibqp(mqp); 1382 } 1383 1384 static inline struct mlx5_ib_srq *res_to_srq(struct mlx5_core_rsc_common *res) 1385 { 1386 struct mlx5_core_srq *msrq = 1387 container_of(res, struct mlx5_core_srq, common); 1388 1389 return to_mibsrq(msrq); 1390 } 1391 1392 static void mlx5_ib_mr_wqe_pfault_handler(struct mlx5_ib_dev *dev, 1393 struct mlx5_pagefault *pfault) 1394 { 1395 bool sq = pfault->type & MLX5_PFAULT_REQUESTOR; 1396 u16 wqe_index = pfault->wqe.wqe_index; 1397 void *wqe, *wqe_start = NULL, *wqe_end = NULL; 1398 u32 bytes_mapped, total_wqe_bytes; 1399 struct mlx5_core_rsc_common *res; 1400 int resume_with_error = 1; 1401 struct mlx5_ib_qp *qp; 1402 size_t bytes_copied; 1403 int ret = 0; 1404 1405 res = odp_get_rsc(dev, pfault->wqe.wq_num, pfault->type); 1406 if (!res) { 1407 mlx5_ib_dbg(dev, "wqe page fault for missing resource %d\n", pfault->wqe.wq_num); 1408 return; 1409 } 1410 1411 if (res->res != MLX5_RES_QP && res->res != MLX5_RES_SRQ && 1412 res->res != MLX5_RES_XSRQ) { 1413 mlx5_ib_err(dev, "wqe page fault for unsupported type %d\n", 1414 pfault->type); 1415 goto resolve_page_fault; 1416 } 1417 1418 /* TODO: switch to "fast and as large as possible" allocation helper */ 1419 wqe_start = kmalloc(PAGE_SIZE, GFP_KERNEL); 1420 if (!wqe_start) { 1421 mlx5_ib_err(dev, "Error allocating memory for IO page fault handling.\n"); 1422 goto resolve_page_fault; 1423 } 1424 1425 wqe = wqe_start; 1426 qp = (res->res == MLX5_RES_QP) ? res_to_qp(res) : NULL; 1427 if (qp && sq) { 1428 ret = mlx5_ib_read_wqe_sq(qp, wqe_index, wqe, PAGE_SIZE, 1429 &bytes_copied); 1430 if (ret) 1431 goto read_user; 1432 ret = mlx5_ib_mr_initiator_pfault_handler( 1433 dev, pfault, qp, &wqe, &wqe_end, bytes_copied); 1434 } else if (qp && !sq) { 1435 ret = mlx5_ib_read_wqe_rq(qp, wqe_index, wqe, PAGE_SIZE, 1436 &bytes_copied); 1437 if (ret) 1438 goto read_user; 1439 ret = mlx5_ib_mr_responder_pfault_handler_rq( 1440 dev, qp, wqe, &wqe_end, bytes_copied); 1441 } else if (!qp) { 1442 struct mlx5_ib_srq *srq = res_to_srq(res); 1443 1444 ret = mlx5_ib_read_wqe_srq(srq, wqe_index, wqe, PAGE_SIZE, 1445 &bytes_copied); 1446 if (ret) 1447 goto read_user; 1448 ret = mlx5_ib_mr_responder_pfault_handler_srq( 1449 dev, srq, &wqe, &wqe_end, bytes_copied); 1450 } 1451 1452 if (ret < 0 || wqe >= wqe_end) 1453 goto resolve_page_fault; 1454 1455 ret = pagefault_data_segments(dev, pfault, wqe, wqe_end, &bytes_mapped, 1456 &total_wqe_bytes, !sq); 1457 if (ret == -EAGAIN) 1458 goto out; 1459 1460 if (ret < 0 || total_wqe_bytes > bytes_mapped) 1461 goto resolve_page_fault; 1462 1463 out: 1464 ret = 0; 1465 resume_with_error = 0; 1466 1467 read_user: 1468 if (ret) 1469 mlx5_ib_err( 1470 dev, 1471 "Failed reading a WQE following page fault, error %d, wqe_index %x, qpn %llx\n", 1472 ret, wqe_index, pfault->token); 1473 1474 resolve_page_fault: 1475 mlx5_ib_page_fault_resume(dev, pfault, resume_with_error); 1476 mlx5_ib_dbg(dev, "PAGE FAULT completed. QP 0x%x resume_with_error=%d, type: 0x%x\n", 1477 pfault->wqe.wq_num, resume_with_error, 1478 pfault->type); 1479 mlx5_core_res_put(res); 1480 kfree(wqe_start); 1481 } 1482 1483 static void mlx5_ib_mr_rdma_pfault_handler(struct mlx5_ib_dev *dev, 1484 struct mlx5_pagefault *pfault) 1485 { 1486 u64 address; 1487 u32 length; 1488 u32 prefetch_len = pfault->bytes_committed; 1489 int prefetch_activated = 0; 1490 u32 rkey = pfault->rdma.r_key; 1491 int ret; 1492 1493 /* The RDMA responder handler handles the page fault in two parts. 1494 * First it brings the necessary pages for the current packet 1495 * (and uses the pfault context), and then (after resuming the QP) 1496 * prefetches more pages. The second operation cannot use the pfault 1497 * context and therefore uses the dummy_pfault context allocated on 1498 * the stack */ 1499 pfault->rdma.rdma_va += pfault->bytes_committed; 1500 pfault->rdma.rdma_op_len -= min(pfault->bytes_committed, 1501 pfault->rdma.rdma_op_len); 1502 pfault->bytes_committed = 0; 1503 1504 address = pfault->rdma.rdma_va; 1505 length = pfault->rdma.rdma_op_len; 1506 1507 /* For some operations, the hardware cannot tell the exact message 1508 * length, and in those cases it reports zero. Use prefetch 1509 * logic. */ 1510 if (length == 0) { 1511 prefetch_activated = 1; 1512 length = pfault->rdma.packet_size; 1513 prefetch_len = min(MAX_PREFETCH_LEN, prefetch_len); 1514 } 1515 1516 ret = pagefault_single_data_segment(dev, NULL, rkey, address, length, 1517 &pfault->bytes_committed, NULL); 1518 if (ret == -EAGAIN) { 1519 /* We're racing with an invalidation, don't prefetch */ 1520 prefetch_activated = 0; 1521 } else if (ret < 0) { 1522 mlx5_ib_page_fault_resume(dev, pfault, 1); 1523 if (ret != -ENOENT) 1524 mlx5_ib_dbg(dev, "PAGE FAULT error %d. QP 0x%llx, type: 0x%x\n", 1525 ret, pfault->token, pfault->type); 1526 return; 1527 } 1528 1529 mlx5_ib_page_fault_resume(dev, pfault, 0); 1530 mlx5_ib_dbg(dev, "PAGE FAULT completed. QP 0x%llx, type: 0x%x, prefetch_activated: %d\n", 1531 pfault->token, pfault->type, 1532 prefetch_activated); 1533 1534 /* At this point, there might be a new pagefault already arriving in 1535 * the eq, switch to the dummy pagefault for the rest of the 1536 * processing. We're still OK with the objects being alive as the 1537 * work-queue is being fenced. */ 1538 1539 if (prefetch_activated) { 1540 u32 bytes_committed = 0; 1541 1542 ret = pagefault_single_data_segment(dev, NULL, rkey, address, 1543 prefetch_len, 1544 &bytes_committed, NULL); 1545 if (ret < 0 && ret != -EAGAIN) { 1546 mlx5_ib_dbg(dev, "Prefetch failed. ret: %d, QP 0x%llx, address: 0x%.16llx, length = 0x%.16x\n", 1547 ret, pfault->token, address, prefetch_len); 1548 } 1549 } 1550 } 1551 1552 #define MLX5_MEMORY_PAGE_FAULT_FLAGS_LAST BIT(7) 1553 static void mlx5_ib_mr_memory_pfault_handler(struct mlx5_ib_dev *dev, 1554 struct mlx5_pagefault *pfault) 1555 { 1556 u64 prefetch_va = 1557 pfault->memory.va - pfault->memory.prefetch_before_byte_count; 1558 size_t prefetch_size = pfault->memory.prefetch_before_byte_count + 1559 pfault->memory.fault_byte_count + 1560 pfault->memory.prefetch_after_byte_count; 1561 struct mlx5_ib_mkey *mmkey; 1562 struct mlx5_ib_mr *mr, *child_mr; 1563 int ret = 0; 1564 1565 mmkey = find_odp_mkey(dev, pfault->memory.mkey); 1566 if (IS_ERR(mmkey)) 1567 goto err; 1568 1569 switch (mmkey->type) { 1570 case MLX5_MKEY_IMPLICIT_CHILD: 1571 child_mr = container_of(mmkey, struct mlx5_ib_mr, mmkey); 1572 mr = child_mr->parent; 1573 break; 1574 case MLX5_MKEY_NULL: 1575 mr = container_of(mmkey, struct mlx5_ib_mr, null_mmkey); 1576 break; 1577 default: 1578 mr = container_of(mmkey, struct mlx5_ib_mr, mmkey); 1579 break; 1580 } 1581 1582 /* If prefetch fails, handle only demanded page fault */ 1583 ret = pagefault_mr(mr, prefetch_va, prefetch_size, NULL, 0, true); 1584 if (ret < 0) { 1585 ret = pagefault_mr(mr, pfault->memory.va, 1586 pfault->memory.fault_byte_count, NULL, 0, true); 1587 if (ret < 0) 1588 goto err; 1589 } 1590 1591 mlx5_update_odp_stats_with_handled(mr, faults, ret); 1592 mlx5r_deref_odp_mkey(mmkey); 1593 1594 if (pfault->memory.flags & MLX5_MEMORY_PAGE_FAULT_FLAGS_LAST) 1595 mlx5_ib_page_fault_resume(dev, pfault, 0); 1596 1597 mlx5_ib_dbg( 1598 dev, 1599 "PAGE FAULT completed %s. token 0x%llx, mkey: 0x%x, va: 0x%llx, byte_count: 0x%x\n", 1600 pfault->memory.flags & MLX5_MEMORY_PAGE_FAULT_FLAGS_LAST ? 1601 "" : 1602 "without resume cmd", 1603 pfault->token, pfault->memory.mkey, pfault->memory.va, 1604 pfault->memory.fault_byte_count); 1605 1606 return; 1607 1608 err: 1609 if (!IS_ERR(mmkey)) 1610 mlx5r_deref_odp_mkey(mmkey); 1611 mlx5_ib_page_fault_resume(dev, pfault, 1); 1612 mlx5_ib_dbg( 1613 dev, 1614 "PAGE FAULT error. token 0x%llx, mkey: 0x%x, va: 0x%llx, byte_count: 0x%x, err: %d\n", 1615 pfault->token, pfault->memory.mkey, pfault->memory.va, 1616 pfault->memory.fault_byte_count, ret); 1617 } 1618 1619 static void mlx5_ib_pfault(struct mlx5_ib_dev *dev, struct mlx5_pagefault *pfault) 1620 { 1621 u8 event_subtype = pfault->event_subtype; 1622 1623 switch (event_subtype) { 1624 case MLX5_PFAULT_SUBTYPE_WQE: 1625 mlx5_ib_mr_wqe_pfault_handler(dev, pfault); 1626 break; 1627 case MLX5_PFAULT_SUBTYPE_RDMA: 1628 mlx5_ib_mr_rdma_pfault_handler(dev, pfault); 1629 break; 1630 case MLX5_PFAULT_SUBTYPE_MEMORY: 1631 mlx5_ib_mr_memory_pfault_handler(dev, pfault); 1632 break; 1633 default: 1634 mlx5_ib_err(dev, "Invalid page fault event subtype: 0x%x\n", 1635 event_subtype); 1636 mlx5_ib_page_fault_resume(dev, pfault, 1); 1637 } 1638 } 1639 1640 static void mlx5_ib_eqe_pf_action(struct work_struct *work) 1641 { 1642 struct mlx5_pagefault *pfault = container_of(work, 1643 struct mlx5_pagefault, 1644 work); 1645 struct mlx5_ib_pf_eq *eq = pfault->eq; 1646 1647 mlx5_ib_pfault(eq->dev, pfault); 1648 mempool_free(pfault, eq->pool); 1649 } 1650 1651 #define MEMORY_SCHEME_PAGE_FAULT_GRANULARITY 4096 1652 static void mlx5_ib_eq_pf_process(struct mlx5_ib_pf_eq *eq) 1653 { 1654 struct mlx5_eqe_page_fault *pf_eqe; 1655 struct mlx5_pagefault *pfault; 1656 struct mlx5_eqe *eqe; 1657 int cc = 0; 1658 1659 while ((eqe = mlx5_eq_get_eqe(eq->core, cc))) { 1660 pfault = mempool_alloc(eq->pool, GFP_ATOMIC); 1661 if (!pfault) { 1662 schedule_work(&eq->work); 1663 break; 1664 } 1665 1666 pf_eqe = &eqe->data.page_fault; 1667 pfault->event_subtype = eqe->sub_type; 1668 1669 switch (eqe->sub_type) { 1670 case MLX5_PFAULT_SUBTYPE_RDMA: 1671 /* RDMA based event */ 1672 pfault->bytes_committed = 1673 be32_to_cpu(pf_eqe->rdma.bytes_committed); 1674 pfault->type = 1675 be32_to_cpu(pf_eqe->rdma.pftype_token) >> 24; 1676 pfault->token = 1677 be32_to_cpu(pf_eqe->rdma.pftype_token) & 1678 MLX5_24BIT_MASK; 1679 pfault->rdma.r_key = 1680 be32_to_cpu(pf_eqe->rdma.r_key); 1681 pfault->rdma.packet_size = 1682 be16_to_cpu(pf_eqe->rdma.packet_length); 1683 pfault->rdma.rdma_op_len = 1684 be32_to_cpu(pf_eqe->rdma.rdma_op_len); 1685 pfault->rdma.rdma_va = 1686 be64_to_cpu(pf_eqe->rdma.rdma_va); 1687 mlx5_ib_dbg( 1688 eq->dev, 1689 "PAGE_FAULT: subtype: 0x%02x, bytes_committed: 0x%06x, type:0x%x, token: 0x%06llx, r_key: 0x%08x\n", 1690 eqe->sub_type, pfault->bytes_committed, 1691 pfault->type, pfault->token, 1692 pfault->rdma.r_key); 1693 mlx5_ib_dbg(eq->dev, 1694 "PAGE_FAULT: rdma_op_len: 0x%08x, rdma_va: 0x%016llx\n", 1695 pfault->rdma.rdma_op_len, 1696 pfault->rdma.rdma_va); 1697 break; 1698 1699 case MLX5_PFAULT_SUBTYPE_WQE: 1700 /* WQE based event */ 1701 pfault->bytes_committed = 1702 be32_to_cpu(pf_eqe->wqe.bytes_committed); 1703 pfault->type = 1704 (be32_to_cpu(pf_eqe->wqe.pftype_wq) >> 24) & 0x7; 1705 pfault->token = 1706 be32_to_cpu(pf_eqe->wqe.token); 1707 pfault->wqe.wq_num = 1708 be32_to_cpu(pf_eqe->wqe.pftype_wq) & 1709 MLX5_24BIT_MASK; 1710 pfault->wqe.wqe_index = 1711 be16_to_cpu(pf_eqe->wqe.wqe_index); 1712 pfault->wqe.packet_size = 1713 be16_to_cpu(pf_eqe->wqe.packet_length); 1714 mlx5_ib_dbg( 1715 eq->dev, 1716 "PAGE_FAULT: subtype: 0x%02x, bytes_committed: 0x%06x, type:0x%x, token: 0x%06llx, wq_num: 0x%06x, wqe_index: 0x%04x\n", 1717 eqe->sub_type, pfault->bytes_committed, 1718 pfault->type, pfault->token, pfault->wqe.wq_num, 1719 pfault->wqe.wqe_index); 1720 break; 1721 1722 case MLX5_PFAULT_SUBTYPE_MEMORY: 1723 /* Memory based event */ 1724 pfault->bytes_committed = 0; 1725 pfault->token = 1726 be32_to_cpu(pf_eqe->memory.token31_0) | 1727 ((u64)be16_to_cpu(pf_eqe->memory.token47_32) 1728 << 32); 1729 pfault->memory.va = be64_to_cpu(pf_eqe->memory.va); 1730 pfault->memory.mkey = be32_to_cpu(pf_eqe->memory.mkey); 1731 pfault->memory.fault_byte_count = (be32_to_cpu( 1732 pf_eqe->memory.demand_fault_pages) >> 12) * 1733 MEMORY_SCHEME_PAGE_FAULT_GRANULARITY; 1734 pfault->memory.prefetch_before_byte_count = 1735 be16_to_cpu( 1736 pf_eqe->memory.pre_demand_fault_pages) * 1737 MEMORY_SCHEME_PAGE_FAULT_GRANULARITY; 1738 pfault->memory.prefetch_after_byte_count = 1739 be16_to_cpu( 1740 pf_eqe->memory.post_demand_fault_pages) * 1741 MEMORY_SCHEME_PAGE_FAULT_GRANULARITY; 1742 pfault->memory.flags = pf_eqe->memory.flags; 1743 mlx5_ib_dbg( 1744 eq->dev, 1745 "PAGE_FAULT: subtype: 0x%02x, token: 0x%06llx, mkey: 0x%06x, fault_byte_count: 0x%06x, va: 0x%016llx, flags: 0x%02x\n", 1746 eqe->sub_type, pfault->token, 1747 pfault->memory.mkey, 1748 pfault->memory.fault_byte_count, 1749 pfault->memory.va, pfault->memory.flags); 1750 mlx5_ib_dbg( 1751 eq->dev, 1752 "PAGE_FAULT: prefetch size: before: 0x%06x, after 0x%06x\n", 1753 pfault->memory.prefetch_before_byte_count, 1754 pfault->memory.prefetch_after_byte_count); 1755 break; 1756 1757 default: 1758 mlx5_ib_warn(eq->dev, 1759 "Unsupported page fault event sub-type: 0x%02hhx\n", 1760 eqe->sub_type); 1761 /* Unsupported page faults should still be 1762 * resolved by the page fault handler 1763 */ 1764 } 1765 1766 pfault->eq = eq; 1767 INIT_WORK(&pfault->work, mlx5_ib_eqe_pf_action); 1768 queue_work(eq->wq, &pfault->work); 1769 1770 cc = mlx5_eq_update_cc(eq->core, ++cc); 1771 } 1772 1773 mlx5_eq_update_ci(eq->core, cc, 1); 1774 } 1775 1776 static int mlx5_ib_eq_pf_int(struct notifier_block *nb, unsigned long type, 1777 void *data) 1778 { 1779 struct mlx5_ib_pf_eq *eq = 1780 container_of(nb, struct mlx5_ib_pf_eq, irq_nb); 1781 unsigned long flags; 1782 1783 if (spin_trylock_irqsave(&eq->lock, flags)) { 1784 mlx5_ib_eq_pf_process(eq); 1785 spin_unlock_irqrestore(&eq->lock, flags); 1786 } else { 1787 schedule_work(&eq->work); 1788 } 1789 1790 return IRQ_HANDLED; 1791 } 1792 1793 /* mempool_refill() was proposed but unfortunately wasn't accepted 1794 * http://lkml.iu.edu/hypermail/linux/kernel/1512.1/05073.html 1795 * Cheap workaround. 1796 */ 1797 static void mempool_refill(mempool_t *pool) 1798 { 1799 while (pool->curr_nr < pool->min_nr) 1800 mempool_free(mempool_alloc(pool, GFP_KERNEL), pool); 1801 } 1802 1803 static void mlx5_ib_eq_pf_action(struct work_struct *work) 1804 { 1805 struct mlx5_ib_pf_eq *eq = 1806 container_of(work, struct mlx5_ib_pf_eq, work); 1807 1808 mempool_refill(eq->pool); 1809 1810 spin_lock_irq(&eq->lock); 1811 mlx5_ib_eq_pf_process(eq); 1812 spin_unlock_irq(&eq->lock); 1813 } 1814 1815 enum { 1816 MLX5_IB_NUM_PF_EQE = 0x1000, 1817 MLX5_IB_NUM_PF_DRAIN = 64, 1818 }; 1819 1820 int mlx5r_odp_create_eq(struct mlx5_ib_dev *dev, struct mlx5_ib_pf_eq *eq) 1821 { 1822 struct mlx5_eq_param param = {}; 1823 int err = 0; 1824 1825 if (!(dev->odp_caps.general_caps & IB_ODP_SUPPORT)) 1826 return -EOPNOTSUPP; 1827 1828 mutex_lock(&dev->odp_eq_mutex); 1829 if (eq->core) 1830 goto unlock; 1831 INIT_WORK(&eq->work, mlx5_ib_eq_pf_action); 1832 spin_lock_init(&eq->lock); 1833 eq->dev = dev; 1834 1835 eq->pool = mempool_create_kmalloc_pool(MLX5_IB_NUM_PF_DRAIN, 1836 sizeof(struct mlx5_pagefault)); 1837 if (!eq->pool) { 1838 err = -ENOMEM; 1839 goto unlock; 1840 } 1841 1842 eq->wq = alloc_workqueue("mlx5_ib_page_fault", 1843 WQ_HIGHPRI | WQ_UNBOUND | WQ_MEM_RECLAIM, 1844 MLX5_NUM_CMD_EQE); 1845 if (!eq->wq) { 1846 err = -ENOMEM; 1847 goto err_mempool; 1848 } 1849 1850 eq->irq_nb.notifier_call = mlx5_ib_eq_pf_int; 1851 param = (struct mlx5_eq_param) { 1852 .nent = MLX5_IB_NUM_PF_EQE, 1853 }; 1854 param.mask[0] = 1ull << MLX5_EVENT_TYPE_PAGE_FAULT; 1855 eq->core = mlx5_eq_create_generic(dev->mdev, ¶m); 1856 if (IS_ERR(eq->core)) { 1857 err = PTR_ERR(eq->core); 1858 goto err_wq; 1859 } 1860 err = mlx5_eq_enable(dev->mdev, eq->core, &eq->irq_nb); 1861 if (err) { 1862 mlx5_ib_err(dev, "failed to enable odp EQ %d\n", err); 1863 goto err_eq; 1864 } 1865 1866 mutex_unlock(&dev->odp_eq_mutex); 1867 return 0; 1868 err_eq: 1869 mlx5_eq_destroy_generic(dev->mdev, eq->core); 1870 err_wq: 1871 eq->core = NULL; 1872 destroy_workqueue(eq->wq); 1873 err_mempool: 1874 mempool_destroy(eq->pool); 1875 unlock: 1876 mutex_unlock(&dev->odp_eq_mutex); 1877 return err; 1878 } 1879 1880 static int 1881 mlx5_ib_odp_destroy_eq(struct mlx5_ib_dev *dev, struct mlx5_ib_pf_eq *eq) 1882 { 1883 int err; 1884 1885 if (!eq->core) 1886 return 0; 1887 mlx5_eq_disable(dev->mdev, eq->core, &eq->irq_nb); 1888 err = mlx5_eq_destroy_generic(dev->mdev, eq->core); 1889 cancel_work_sync(&eq->work); 1890 destroy_workqueue(eq->wq); 1891 mempool_destroy(eq->pool); 1892 1893 return err; 1894 } 1895 1896 static const struct ib_device_ops mlx5_ib_dev_odp_ops = { 1897 .advise_mr = mlx5_ib_advise_mr, 1898 }; 1899 1900 int mlx5_ib_odp_init_one(struct mlx5_ib_dev *dev) 1901 { 1902 internal_fill_odp_caps(dev); 1903 1904 if (!(dev->odp_caps.general_caps & IB_ODP_SUPPORT)) 1905 return 0; 1906 1907 ib_set_device_ops(&dev->ib_dev, &mlx5_ib_dev_odp_ops); 1908 1909 mutex_init(&dev->odp_eq_mutex); 1910 return 0; 1911 } 1912 1913 void mlx5_ib_odp_cleanup_one(struct mlx5_ib_dev *dev) 1914 { 1915 if (!(dev->odp_caps.general_caps & IB_ODP_SUPPORT)) 1916 return; 1917 1918 mlx5_ib_odp_destroy_eq(dev, &dev->odp_pf_eq); 1919 } 1920 1921 int mlx5_ib_odp_init(void) 1922 { 1923 u32 log_va_pages = ilog2(TASK_SIZE) - PAGE_SHIFT; 1924 u8 mlx5_imr_mtt_bits; 1925 1926 /* 48 is default ARM64 VA space and covers X86 4-level paging which is 47 */ 1927 if (log_va_pages <= 48 - PAGE_SHIFT) 1928 mlx5_imr_mtt_shift = 30; 1929 /* 56 is x86-64, 5-level paging */ 1930 else if (log_va_pages <= 56 - PAGE_SHIFT) 1931 mlx5_imr_mtt_shift = 34; 1932 else 1933 return 0; 1934 1935 mlx5_imr_mtt_size = BIT_ULL(mlx5_imr_mtt_shift); 1936 mlx5_imr_mtt_bits = mlx5_imr_mtt_shift - PAGE_SHIFT; 1937 mlx5_imr_mtt_entries = BIT_ULL(mlx5_imr_mtt_bits); 1938 mlx5_imr_ksm_entries = BIT_ULL(get_order(TASK_SIZE) - 1939 mlx5_imr_mtt_bits); 1940 1941 mlx5_imr_ksm_page_shift = mlx5_imr_mtt_shift; 1942 return 0; 1943 } 1944 1945 struct prefetch_mr_work { 1946 struct work_struct work; 1947 u32 pf_flags; 1948 u32 num_sge; 1949 struct { 1950 u64 io_virt; 1951 struct mlx5_ib_mr *mr; 1952 size_t length; 1953 } frags[]; 1954 }; 1955 1956 static void destroy_prefetch_work(struct prefetch_mr_work *work) 1957 { 1958 u32 i; 1959 1960 for (i = 0; i < work->num_sge; ++i) 1961 mlx5r_deref_odp_mkey(&work->frags[i].mr->mmkey); 1962 1963 kvfree(work); 1964 } 1965 1966 static struct mlx5_ib_mr * 1967 get_prefetchable_mr(struct ib_pd *pd, enum ib_uverbs_advise_mr_advice advice, 1968 u32 lkey) 1969 { 1970 struct mlx5_ib_dev *dev = to_mdev(pd->device); 1971 struct mlx5_ib_mr *mr = NULL; 1972 struct mlx5_ib_mkey *mmkey; 1973 1974 xa_lock(&dev->odp_mkeys); 1975 mmkey = xa_load(&dev->odp_mkeys, mlx5_base_mkey(lkey)); 1976 if (!mmkey || mmkey->key != lkey) { 1977 mr = ERR_PTR(-ENOENT); 1978 goto end; 1979 } 1980 if (mmkey->type != MLX5_MKEY_MR) { 1981 mr = ERR_PTR(-EINVAL); 1982 goto end; 1983 } 1984 1985 mr = container_of(mmkey, struct mlx5_ib_mr, mmkey); 1986 1987 if (mr->ibmr.pd != pd) { 1988 mr = ERR_PTR(-EPERM); 1989 goto end; 1990 } 1991 1992 /* prefetch with write-access must be supported by the MR */ 1993 if (advice == IB_UVERBS_ADVISE_MR_ADVICE_PREFETCH_WRITE && 1994 !mr->umem->writable) { 1995 mr = ERR_PTR(-EPERM); 1996 goto end; 1997 } 1998 1999 refcount_inc(&mmkey->usecount); 2000 end: 2001 xa_unlock(&dev->odp_mkeys); 2002 return mr; 2003 } 2004 2005 static void mlx5_ib_prefetch_mr_work(struct work_struct *w) 2006 { 2007 struct prefetch_mr_work *work = 2008 container_of(w, struct prefetch_mr_work, work); 2009 u32 bytes_mapped = 0; 2010 int ret; 2011 u32 i; 2012 2013 /* We rely on IB/core that work is executed if we have num_sge != 0 only. */ 2014 WARN_ON(!work->num_sge); 2015 for (i = 0; i < work->num_sge; ++i) { 2016 ret = pagefault_mr(work->frags[i].mr, work->frags[i].io_virt, 2017 work->frags[i].length, &bytes_mapped, 2018 work->pf_flags, false); 2019 if (ret <= 0) 2020 continue; 2021 mlx5_update_odp_stats(work->frags[i].mr, prefetch, ret); 2022 } 2023 2024 destroy_prefetch_work(work); 2025 } 2026 2027 static int init_prefetch_work(struct ib_pd *pd, 2028 enum ib_uverbs_advise_mr_advice advice, 2029 u32 pf_flags, struct prefetch_mr_work *work, 2030 struct ib_sge *sg_list, u32 num_sge) 2031 { 2032 u32 i; 2033 2034 INIT_WORK(&work->work, mlx5_ib_prefetch_mr_work); 2035 work->pf_flags = pf_flags; 2036 2037 for (i = 0; i < num_sge; ++i) { 2038 struct mlx5_ib_mr *mr; 2039 2040 mr = get_prefetchable_mr(pd, advice, sg_list[i].lkey); 2041 if (IS_ERR(mr)) { 2042 work->num_sge = i; 2043 return PTR_ERR(mr); 2044 } 2045 work->frags[i].io_virt = sg_list[i].addr; 2046 work->frags[i].length = sg_list[i].length; 2047 work->frags[i].mr = mr; 2048 } 2049 work->num_sge = num_sge; 2050 return 0; 2051 } 2052 2053 static int mlx5_ib_prefetch_sg_list(struct ib_pd *pd, 2054 enum ib_uverbs_advise_mr_advice advice, 2055 u32 pf_flags, struct ib_sge *sg_list, 2056 u32 num_sge) 2057 { 2058 u32 bytes_mapped = 0; 2059 int ret = 0; 2060 u32 i; 2061 2062 for (i = 0; i < num_sge; ++i) { 2063 struct mlx5_ib_mr *mr; 2064 2065 mr = get_prefetchable_mr(pd, advice, sg_list[i].lkey); 2066 if (IS_ERR(mr)) 2067 return PTR_ERR(mr); 2068 ret = pagefault_mr(mr, sg_list[i].addr, sg_list[i].length, 2069 &bytes_mapped, pf_flags, false); 2070 if (ret < 0) { 2071 mlx5r_deref_odp_mkey(&mr->mmkey); 2072 return ret; 2073 } 2074 mlx5_update_odp_stats(mr, prefetch, ret); 2075 mlx5r_deref_odp_mkey(&mr->mmkey); 2076 } 2077 2078 return 0; 2079 } 2080 2081 int mlx5_ib_advise_mr_prefetch(struct ib_pd *pd, 2082 enum ib_uverbs_advise_mr_advice advice, 2083 u32 flags, struct ib_sge *sg_list, u32 num_sge) 2084 { 2085 u32 pf_flags = 0; 2086 struct prefetch_mr_work *work; 2087 int rc; 2088 2089 if (advice == IB_UVERBS_ADVISE_MR_ADVICE_PREFETCH) 2090 pf_flags |= MLX5_PF_FLAGS_DOWNGRADE; 2091 2092 if (advice == IB_UVERBS_ADVISE_MR_ADVICE_PREFETCH_NO_FAULT) 2093 pf_flags |= MLX5_PF_FLAGS_SNAPSHOT; 2094 2095 if (flags & IB_UVERBS_ADVISE_MR_FLAG_FLUSH) 2096 return mlx5_ib_prefetch_sg_list(pd, advice, pf_flags, sg_list, 2097 num_sge); 2098 2099 work = kvzalloc_flex(*work, frags, num_sge); 2100 if (!work) 2101 return -ENOMEM; 2102 2103 rc = init_prefetch_work(pd, advice, pf_flags, work, sg_list, num_sge); 2104 if (rc) { 2105 destroy_prefetch_work(work); 2106 return rc; 2107 } 2108 queue_work(system_dfl_wq, &work->work); 2109 return 0; 2110 } 2111