1 /* 2 * Copyright (c) 2013-2015, Mellanox Technologies. All rights reserved. 3 * Copyright (c) 2020, Intel Corporation. All rights reserved. 4 * 5 * This software is available to you under a choice of one of two 6 * licenses. You may choose to be licensed under the terms of the GNU 7 * General Public License (GPL) Version 2, available from the file 8 * COPYING in the main directory of this source tree, or the 9 * OpenIB.org BSD license below: 10 * 11 * Redistribution and use in source and binary forms, with or 12 * without modification, are permitted provided that the following 13 * conditions are met: 14 * 15 * - Redistributions of source code must retain the above 16 * copyright notice, this list of conditions and the following 17 * disclaimer. 18 * 19 * - Redistributions in binary form must reproduce the above 20 * copyright notice, this list of conditions and the following 21 * disclaimer in the documentation and/or other materials 22 * provided with the distribution. 23 * 24 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 25 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 26 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 27 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 28 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 29 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 30 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 31 * SOFTWARE. 32 */ 33 34 35 #include <linux/kref.h> 36 #include <linux/random.h> 37 #include <linux/debugfs.h> 38 #include <linux/export.h> 39 #include <linux/delay.h> 40 #include <linux/dma-buf.h> 41 #include <linux/dma-resv.h> 42 #include <rdma/ib_umem.h> 43 #include <rdma/ib_umem_odp.h> 44 #include <rdma/ib_verbs.h> 45 #include "dm.h" 46 #include "mlx5_ib.h" 47 48 /* 49 * We can't use an array for xlt_emergency_page because dma_map_single doesn't 50 * work on kernel modules memory 51 */ 52 void *xlt_emergency_page; 53 static DEFINE_MUTEX(xlt_emergency_page_mutex); 54 55 enum { 56 MAX_PENDING_REG_MR = 8, 57 }; 58 59 #define MLX5_UMR_ALIGN 2048 60 61 static void 62 create_mkey_callback(int status, struct mlx5_async_work *context); 63 static struct mlx5_ib_mr *reg_create(struct ib_pd *pd, struct ib_umem *umem, 64 u64 iova, int access_flags, 65 unsigned int page_size, bool populate); 66 67 static void set_mkc_access_pd_addr_fields(void *mkc, int acc, u64 start_addr, 68 struct ib_pd *pd) 69 { 70 struct mlx5_ib_dev *dev = to_mdev(pd->device); 71 bool ro_pci_enabled = pcie_relaxed_ordering_enabled(dev->mdev->pdev); 72 73 MLX5_SET(mkc, mkc, a, !!(acc & IB_ACCESS_REMOTE_ATOMIC)); 74 MLX5_SET(mkc, mkc, rw, !!(acc & IB_ACCESS_REMOTE_WRITE)); 75 MLX5_SET(mkc, mkc, rr, !!(acc & IB_ACCESS_REMOTE_READ)); 76 MLX5_SET(mkc, mkc, lw, !!(acc & IB_ACCESS_LOCAL_WRITE)); 77 MLX5_SET(mkc, mkc, lr, 1); 78 79 if (MLX5_CAP_GEN(dev->mdev, relaxed_ordering_write)) 80 MLX5_SET(mkc, mkc, relaxed_ordering_write, 81 (acc & IB_ACCESS_RELAXED_ORDERING) && ro_pci_enabled); 82 if (MLX5_CAP_GEN(dev->mdev, relaxed_ordering_read)) 83 MLX5_SET(mkc, mkc, relaxed_ordering_read, 84 (acc & IB_ACCESS_RELAXED_ORDERING) && ro_pci_enabled); 85 86 MLX5_SET(mkc, mkc, pd, to_mpd(pd)->pdn); 87 MLX5_SET(mkc, mkc, qpn, 0xffffff); 88 MLX5_SET64(mkc, mkc, start_addr, start_addr); 89 } 90 91 static void assign_mkey_variant(struct mlx5_ib_dev *dev, 92 struct mlx5_ib_mkey *mkey, u32 *in) 93 { 94 u8 key = atomic_inc_return(&dev->mkey_var); 95 void *mkc; 96 97 mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry); 98 MLX5_SET(mkc, mkc, mkey_7_0, key); 99 mkey->key = key; 100 } 101 102 static int mlx5_ib_create_mkey(struct mlx5_ib_dev *dev, 103 struct mlx5_ib_mkey *mkey, u32 *in, int inlen) 104 { 105 int ret; 106 107 assign_mkey_variant(dev, mkey, in); 108 ret = mlx5_core_create_mkey(dev->mdev, &mkey->key, in, inlen); 109 if (!ret) 110 init_waitqueue_head(&mkey->wait); 111 112 return ret; 113 } 114 115 static int 116 mlx5_ib_create_mkey_cb(struct mlx5_ib_dev *dev, 117 struct mlx5_ib_mkey *mkey, 118 struct mlx5_async_ctx *async_ctx, 119 u32 *in, int inlen, u32 *out, int outlen, 120 struct mlx5_async_work *context) 121 { 122 MLX5_SET(create_mkey_in, in, opcode, MLX5_CMD_OP_CREATE_MKEY); 123 assign_mkey_variant(dev, mkey, in); 124 return mlx5_cmd_exec_cb(async_ctx, in, inlen, out, outlen, 125 create_mkey_callback, context); 126 } 127 128 static int mr_cache_max_order(struct mlx5_ib_dev *dev); 129 static void queue_adjust_cache_locked(struct mlx5_cache_ent *ent); 130 131 static bool umr_can_use_indirect_mkey(struct mlx5_ib_dev *dev) 132 { 133 return !MLX5_CAP_GEN(dev->mdev, umr_indirect_mkey_disabled); 134 } 135 136 static int destroy_mkey(struct mlx5_ib_dev *dev, struct mlx5_ib_mr *mr) 137 { 138 WARN_ON(xa_load(&dev->odp_mkeys, mlx5_base_mkey(mr->mmkey.key))); 139 140 return mlx5_core_destroy_mkey(dev->mdev, mr->mmkey.key); 141 } 142 143 static void create_mkey_callback(int status, struct mlx5_async_work *context) 144 { 145 struct mlx5_ib_mr *mr = 146 container_of(context, struct mlx5_ib_mr, cb_work); 147 struct mlx5_cache_ent *ent = mr->cache_ent; 148 struct mlx5_ib_dev *dev = ent->dev; 149 unsigned long flags; 150 151 if (status) { 152 mlx5_ib_warn(dev, "async reg mr failed. status %d\n", status); 153 kfree(mr); 154 spin_lock_irqsave(&ent->lock, flags); 155 ent->pending--; 156 WRITE_ONCE(dev->fill_delay, 1); 157 spin_unlock_irqrestore(&ent->lock, flags); 158 mod_timer(&dev->delay_timer, jiffies + HZ); 159 return; 160 } 161 162 mr->mmkey.type = MLX5_MKEY_MR; 163 mr->mmkey.key |= mlx5_idx_to_mkey( 164 MLX5_GET(create_mkey_out, mr->out, mkey_index)); 165 init_waitqueue_head(&mr->mmkey.wait); 166 167 WRITE_ONCE(dev->cache.last_add, jiffies); 168 169 spin_lock_irqsave(&ent->lock, flags); 170 list_add_tail(&mr->list, &ent->head); 171 ent->available_mrs++; 172 ent->total_mrs++; 173 /* If we are doing fill_to_high_water then keep going. */ 174 queue_adjust_cache_locked(ent); 175 ent->pending--; 176 spin_unlock_irqrestore(&ent->lock, flags); 177 } 178 179 static struct mlx5_ib_mr *alloc_cache_mr(struct mlx5_cache_ent *ent, void *mkc) 180 { 181 struct mlx5_ib_mr *mr; 182 183 mr = kzalloc(sizeof(*mr), GFP_KERNEL); 184 if (!mr) 185 return NULL; 186 mr->cache_ent = ent; 187 188 set_mkc_access_pd_addr_fields(mkc, 0, 0, ent->dev->umrc.pd); 189 MLX5_SET(mkc, mkc, free, 1); 190 MLX5_SET(mkc, mkc, umr_en, 1); 191 MLX5_SET(mkc, mkc, access_mode_1_0, ent->access_mode & 0x3); 192 MLX5_SET(mkc, mkc, access_mode_4_2, (ent->access_mode >> 2) & 0x7); 193 194 MLX5_SET(mkc, mkc, translations_octword_size, ent->xlt); 195 MLX5_SET(mkc, mkc, log_page_size, ent->page); 196 return mr; 197 } 198 199 /* Asynchronously schedule new MRs to be populated in the cache. */ 200 static int add_keys(struct mlx5_cache_ent *ent, unsigned int num) 201 { 202 size_t inlen = MLX5_ST_SZ_BYTES(create_mkey_in); 203 struct mlx5_ib_mr *mr; 204 void *mkc; 205 u32 *in; 206 int err = 0; 207 int i; 208 209 in = kzalloc(inlen, GFP_KERNEL); 210 if (!in) 211 return -ENOMEM; 212 213 mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry); 214 for (i = 0; i < num; i++) { 215 mr = alloc_cache_mr(ent, mkc); 216 if (!mr) { 217 err = -ENOMEM; 218 break; 219 } 220 spin_lock_irq(&ent->lock); 221 if (ent->pending >= MAX_PENDING_REG_MR) { 222 err = -EAGAIN; 223 spin_unlock_irq(&ent->lock); 224 kfree(mr); 225 break; 226 } 227 ent->pending++; 228 spin_unlock_irq(&ent->lock); 229 err = mlx5_ib_create_mkey_cb(ent->dev, &mr->mmkey, 230 &ent->dev->async_ctx, in, inlen, 231 mr->out, sizeof(mr->out), 232 &mr->cb_work); 233 if (err) { 234 spin_lock_irq(&ent->lock); 235 ent->pending--; 236 spin_unlock_irq(&ent->lock); 237 mlx5_ib_warn(ent->dev, "create mkey failed %d\n", err); 238 kfree(mr); 239 break; 240 } 241 } 242 243 kfree(in); 244 return err; 245 } 246 247 /* Synchronously create a MR in the cache */ 248 static struct mlx5_ib_mr *create_cache_mr(struct mlx5_cache_ent *ent) 249 { 250 size_t inlen = MLX5_ST_SZ_BYTES(create_mkey_in); 251 struct mlx5_ib_mr *mr; 252 void *mkc; 253 u32 *in; 254 int err; 255 256 in = kzalloc(inlen, GFP_KERNEL); 257 if (!in) 258 return ERR_PTR(-ENOMEM); 259 mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry); 260 261 mr = alloc_cache_mr(ent, mkc); 262 if (!mr) { 263 err = -ENOMEM; 264 goto free_in; 265 } 266 267 err = mlx5_core_create_mkey(ent->dev->mdev, &mr->mmkey.key, in, inlen); 268 if (err) 269 goto free_mr; 270 271 init_waitqueue_head(&mr->mmkey.wait); 272 mr->mmkey.type = MLX5_MKEY_MR; 273 WRITE_ONCE(ent->dev->cache.last_add, jiffies); 274 spin_lock_irq(&ent->lock); 275 ent->total_mrs++; 276 spin_unlock_irq(&ent->lock); 277 kfree(in); 278 return mr; 279 free_mr: 280 kfree(mr); 281 free_in: 282 kfree(in); 283 return ERR_PTR(err); 284 } 285 286 static void remove_cache_mr_locked(struct mlx5_cache_ent *ent) 287 { 288 struct mlx5_ib_mr *mr; 289 290 lockdep_assert_held(&ent->lock); 291 if (list_empty(&ent->head)) 292 return; 293 mr = list_first_entry(&ent->head, struct mlx5_ib_mr, list); 294 list_del(&mr->list); 295 ent->available_mrs--; 296 ent->total_mrs--; 297 spin_unlock_irq(&ent->lock); 298 mlx5_core_destroy_mkey(ent->dev->mdev, mr->mmkey.key); 299 kfree(mr); 300 spin_lock_irq(&ent->lock); 301 } 302 303 static int resize_available_mrs(struct mlx5_cache_ent *ent, unsigned int target, 304 bool limit_fill) 305 { 306 int err; 307 308 lockdep_assert_held(&ent->lock); 309 310 while (true) { 311 if (limit_fill) 312 target = ent->limit * 2; 313 if (target == ent->available_mrs + ent->pending) 314 return 0; 315 if (target > ent->available_mrs + ent->pending) { 316 u32 todo = target - (ent->available_mrs + ent->pending); 317 318 spin_unlock_irq(&ent->lock); 319 err = add_keys(ent, todo); 320 if (err == -EAGAIN) 321 usleep_range(3000, 5000); 322 spin_lock_irq(&ent->lock); 323 if (err) { 324 if (err != -EAGAIN) 325 return err; 326 } else 327 return 0; 328 } else { 329 remove_cache_mr_locked(ent); 330 } 331 } 332 } 333 334 static ssize_t size_write(struct file *filp, const char __user *buf, 335 size_t count, loff_t *pos) 336 { 337 struct mlx5_cache_ent *ent = filp->private_data; 338 u32 target; 339 int err; 340 341 err = kstrtou32_from_user(buf, count, 0, &target); 342 if (err) 343 return err; 344 345 /* 346 * Target is the new value of total_mrs the user requests, however we 347 * cannot free MRs that are in use. Compute the target value for 348 * available_mrs. 349 */ 350 spin_lock_irq(&ent->lock); 351 if (target < ent->total_mrs - ent->available_mrs) { 352 err = -EINVAL; 353 goto err_unlock; 354 } 355 target = target - (ent->total_mrs - ent->available_mrs); 356 if (target < ent->limit || target > ent->limit*2) { 357 err = -EINVAL; 358 goto err_unlock; 359 } 360 err = resize_available_mrs(ent, target, false); 361 if (err) 362 goto err_unlock; 363 spin_unlock_irq(&ent->lock); 364 365 return count; 366 367 err_unlock: 368 spin_unlock_irq(&ent->lock); 369 return err; 370 } 371 372 static ssize_t size_read(struct file *filp, char __user *buf, size_t count, 373 loff_t *pos) 374 { 375 struct mlx5_cache_ent *ent = filp->private_data; 376 char lbuf[20]; 377 int err; 378 379 err = snprintf(lbuf, sizeof(lbuf), "%d\n", ent->total_mrs); 380 if (err < 0) 381 return err; 382 383 return simple_read_from_buffer(buf, count, pos, lbuf, err); 384 } 385 386 static const struct file_operations size_fops = { 387 .owner = THIS_MODULE, 388 .open = simple_open, 389 .write = size_write, 390 .read = size_read, 391 }; 392 393 static ssize_t limit_write(struct file *filp, const char __user *buf, 394 size_t count, loff_t *pos) 395 { 396 struct mlx5_cache_ent *ent = filp->private_data; 397 u32 var; 398 int err; 399 400 err = kstrtou32_from_user(buf, count, 0, &var); 401 if (err) 402 return err; 403 404 /* 405 * Upon set we immediately fill the cache to high water mark implied by 406 * the limit. 407 */ 408 spin_lock_irq(&ent->lock); 409 ent->limit = var; 410 err = resize_available_mrs(ent, 0, true); 411 spin_unlock_irq(&ent->lock); 412 if (err) 413 return err; 414 return count; 415 } 416 417 static ssize_t limit_read(struct file *filp, char __user *buf, size_t count, 418 loff_t *pos) 419 { 420 struct mlx5_cache_ent *ent = filp->private_data; 421 char lbuf[20]; 422 int err; 423 424 err = snprintf(lbuf, sizeof(lbuf), "%d\n", ent->limit); 425 if (err < 0) 426 return err; 427 428 return simple_read_from_buffer(buf, count, pos, lbuf, err); 429 } 430 431 static const struct file_operations limit_fops = { 432 .owner = THIS_MODULE, 433 .open = simple_open, 434 .write = limit_write, 435 .read = limit_read, 436 }; 437 438 static bool someone_adding(struct mlx5_mr_cache *cache) 439 { 440 unsigned int i; 441 442 for (i = 0; i < MAX_MR_CACHE_ENTRIES; i++) { 443 struct mlx5_cache_ent *ent = &cache->ent[i]; 444 bool ret; 445 446 spin_lock_irq(&ent->lock); 447 ret = ent->available_mrs < ent->limit; 448 spin_unlock_irq(&ent->lock); 449 if (ret) 450 return true; 451 } 452 return false; 453 } 454 455 /* 456 * Check if the bucket is outside the high/low water mark and schedule an async 457 * update. The cache refill has hysteresis, once the low water mark is hit it is 458 * refilled up to the high mark. 459 */ 460 static void queue_adjust_cache_locked(struct mlx5_cache_ent *ent) 461 { 462 lockdep_assert_held(&ent->lock); 463 464 if (ent->disabled || READ_ONCE(ent->dev->fill_delay)) 465 return; 466 if (ent->available_mrs < ent->limit) { 467 ent->fill_to_high_water = true; 468 queue_work(ent->dev->cache.wq, &ent->work); 469 } else if (ent->fill_to_high_water && 470 ent->available_mrs + ent->pending < 2 * ent->limit) { 471 /* 472 * Once we start populating due to hitting a low water mark 473 * continue until we pass the high water mark. 474 */ 475 queue_work(ent->dev->cache.wq, &ent->work); 476 } else if (ent->available_mrs == 2 * ent->limit) { 477 ent->fill_to_high_water = false; 478 } else if (ent->available_mrs > 2 * ent->limit) { 479 /* Queue deletion of excess entries */ 480 ent->fill_to_high_water = false; 481 if (ent->pending) 482 queue_delayed_work(ent->dev->cache.wq, &ent->dwork, 483 msecs_to_jiffies(1000)); 484 else 485 queue_work(ent->dev->cache.wq, &ent->work); 486 } 487 } 488 489 static void __cache_work_func(struct mlx5_cache_ent *ent) 490 { 491 struct mlx5_ib_dev *dev = ent->dev; 492 struct mlx5_mr_cache *cache = &dev->cache; 493 int err; 494 495 spin_lock_irq(&ent->lock); 496 if (ent->disabled) 497 goto out; 498 499 if (ent->fill_to_high_water && 500 ent->available_mrs + ent->pending < 2 * ent->limit && 501 !READ_ONCE(dev->fill_delay)) { 502 spin_unlock_irq(&ent->lock); 503 err = add_keys(ent, 1); 504 spin_lock_irq(&ent->lock); 505 if (ent->disabled) 506 goto out; 507 if (err) { 508 /* 509 * EAGAIN only happens if pending is positive, so we 510 * will be rescheduled from reg_mr_callback(). The only 511 * failure path here is ENOMEM. 512 */ 513 if (err != -EAGAIN) { 514 mlx5_ib_warn( 515 dev, 516 "command failed order %d, err %d\n", 517 ent->order, err); 518 queue_delayed_work(cache->wq, &ent->dwork, 519 msecs_to_jiffies(1000)); 520 } 521 } 522 } else if (ent->available_mrs > 2 * ent->limit) { 523 bool need_delay; 524 525 /* 526 * The remove_cache_mr() logic is performed as garbage 527 * collection task. Such task is intended to be run when no 528 * other active processes are running. 529 * 530 * The need_resched() will return TRUE if there are user tasks 531 * to be activated in near future. 532 * 533 * In such case, we don't execute remove_cache_mr() and postpone 534 * the garbage collection work to try to run in next cycle, in 535 * order to free CPU resources to other tasks. 536 */ 537 spin_unlock_irq(&ent->lock); 538 need_delay = need_resched() || someone_adding(cache) || 539 !time_after(jiffies, 540 READ_ONCE(cache->last_add) + 300 * HZ); 541 spin_lock_irq(&ent->lock); 542 if (ent->disabled) 543 goto out; 544 if (need_delay) 545 queue_delayed_work(cache->wq, &ent->dwork, 300 * HZ); 546 remove_cache_mr_locked(ent); 547 queue_adjust_cache_locked(ent); 548 } 549 out: 550 spin_unlock_irq(&ent->lock); 551 } 552 553 static void delayed_cache_work_func(struct work_struct *work) 554 { 555 struct mlx5_cache_ent *ent; 556 557 ent = container_of(work, struct mlx5_cache_ent, dwork.work); 558 __cache_work_func(ent); 559 } 560 561 static void cache_work_func(struct work_struct *work) 562 { 563 struct mlx5_cache_ent *ent; 564 565 ent = container_of(work, struct mlx5_cache_ent, work); 566 __cache_work_func(ent); 567 } 568 569 /* Allocate a special entry from the cache */ 570 struct mlx5_ib_mr *mlx5_mr_cache_alloc(struct mlx5_ib_dev *dev, 571 unsigned int entry, int access_flags) 572 { 573 struct mlx5_mr_cache *cache = &dev->cache; 574 struct mlx5_cache_ent *ent; 575 struct mlx5_ib_mr *mr; 576 577 if (WARN_ON(entry <= MR_CACHE_LAST_STD_ENTRY || 578 entry >= ARRAY_SIZE(cache->ent))) 579 return ERR_PTR(-EINVAL); 580 581 /* Matches access in alloc_cache_mr() */ 582 if (!mlx5_ib_can_reconfig_with_umr(dev, 0, access_flags)) 583 return ERR_PTR(-EOPNOTSUPP); 584 585 ent = &cache->ent[entry]; 586 spin_lock_irq(&ent->lock); 587 if (list_empty(&ent->head)) { 588 spin_unlock_irq(&ent->lock); 589 mr = create_cache_mr(ent); 590 if (IS_ERR(mr)) 591 return mr; 592 } else { 593 mr = list_first_entry(&ent->head, struct mlx5_ib_mr, list); 594 list_del(&mr->list); 595 ent->available_mrs--; 596 queue_adjust_cache_locked(ent); 597 spin_unlock_irq(&ent->lock); 598 599 mlx5_clear_mr(mr); 600 } 601 mr->access_flags = access_flags; 602 return mr; 603 } 604 605 /* Return a MR already available in the cache */ 606 static struct mlx5_ib_mr *get_cache_mr(struct mlx5_cache_ent *req_ent) 607 { 608 struct mlx5_ib_mr *mr = NULL; 609 struct mlx5_cache_ent *ent = req_ent; 610 611 spin_lock_irq(&ent->lock); 612 if (!list_empty(&ent->head)) { 613 mr = list_first_entry(&ent->head, struct mlx5_ib_mr, list); 614 list_del(&mr->list); 615 ent->available_mrs--; 616 queue_adjust_cache_locked(ent); 617 spin_unlock_irq(&ent->lock); 618 mlx5_clear_mr(mr); 619 return mr; 620 } 621 queue_adjust_cache_locked(ent); 622 spin_unlock_irq(&ent->lock); 623 req_ent->miss++; 624 return NULL; 625 } 626 627 static void mlx5_mr_cache_free(struct mlx5_ib_dev *dev, struct mlx5_ib_mr *mr) 628 { 629 struct mlx5_cache_ent *ent = mr->cache_ent; 630 631 spin_lock_irq(&ent->lock); 632 list_add_tail(&mr->list, &ent->head); 633 ent->available_mrs++; 634 queue_adjust_cache_locked(ent); 635 spin_unlock_irq(&ent->lock); 636 } 637 638 static void clean_keys(struct mlx5_ib_dev *dev, int c) 639 { 640 struct mlx5_mr_cache *cache = &dev->cache; 641 struct mlx5_cache_ent *ent = &cache->ent[c]; 642 struct mlx5_ib_mr *tmp_mr; 643 struct mlx5_ib_mr *mr; 644 LIST_HEAD(del_list); 645 646 cancel_delayed_work(&ent->dwork); 647 while (1) { 648 spin_lock_irq(&ent->lock); 649 if (list_empty(&ent->head)) { 650 spin_unlock_irq(&ent->lock); 651 break; 652 } 653 mr = list_first_entry(&ent->head, struct mlx5_ib_mr, list); 654 list_move(&mr->list, &del_list); 655 ent->available_mrs--; 656 ent->total_mrs--; 657 spin_unlock_irq(&ent->lock); 658 mlx5_core_destroy_mkey(dev->mdev, mr->mmkey.key); 659 } 660 661 list_for_each_entry_safe(mr, tmp_mr, &del_list, list) { 662 list_del(&mr->list); 663 kfree(mr); 664 } 665 } 666 667 static void mlx5_mr_cache_debugfs_cleanup(struct mlx5_ib_dev *dev) 668 { 669 if (!mlx5_debugfs_root || dev->is_rep) 670 return; 671 672 debugfs_remove_recursive(dev->cache.root); 673 dev->cache.root = NULL; 674 } 675 676 static void mlx5_mr_cache_debugfs_init(struct mlx5_ib_dev *dev) 677 { 678 struct mlx5_mr_cache *cache = &dev->cache; 679 struct mlx5_cache_ent *ent; 680 struct dentry *dir; 681 int i; 682 683 if (!mlx5_debugfs_root || dev->is_rep) 684 return; 685 686 cache->root = debugfs_create_dir("mr_cache", dev->mdev->priv.dbg_root); 687 688 for (i = 0; i < MAX_MR_CACHE_ENTRIES; i++) { 689 ent = &cache->ent[i]; 690 sprintf(ent->name, "%d", ent->order); 691 dir = debugfs_create_dir(ent->name, cache->root); 692 debugfs_create_file("size", 0600, dir, ent, &size_fops); 693 debugfs_create_file("limit", 0600, dir, ent, &limit_fops); 694 debugfs_create_u32("cur", 0400, dir, &ent->available_mrs); 695 debugfs_create_u32("miss", 0600, dir, &ent->miss); 696 } 697 } 698 699 static void delay_time_func(struct timer_list *t) 700 { 701 struct mlx5_ib_dev *dev = from_timer(dev, t, delay_timer); 702 703 WRITE_ONCE(dev->fill_delay, 0); 704 } 705 706 int mlx5_mr_cache_init(struct mlx5_ib_dev *dev) 707 { 708 struct mlx5_mr_cache *cache = &dev->cache; 709 struct mlx5_cache_ent *ent; 710 int i; 711 712 mutex_init(&dev->slow_path_mutex); 713 cache->wq = alloc_ordered_workqueue("mkey_cache", WQ_MEM_RECLAIM); 714 if (!cache->wq) { 715 mlx5_ib_warn(dev, "failed to create work queue\n"); 716 return -ENOMEM; 717 } 718 719 mlx5_cmd_init_async_ctx(dev->mdev, &dev->async_ctx); 720 timer_setup(&dev->delay_timer, delay_time_func, 0); 721 for (i = 0; i < MAX_MR_CACHE_ENTRIES; i++) { 722 ent = &cache->ent[i]; 723 INIT_LIST_HEAD(&ent->head); 724 spin_lock_init(&ent->lock); 725 ent->order = i + 2; 726 ent->dev = dev; 727 ent->limit = 0; 728 729 INIT_WORK(&ent->work, cache_work_func); 730 INIT_DELAYED_WORK(&ent->dwork, delayed_cache_work_func); 731 732 if (i > MR_CACHE_LAST_STD_ENTRY) { 733 mlx5_odp_init_mr_cache_entry(ent); 734 continue; 735 } 736 737 if (ent->order > mr_cache_max_order(dev)) 738 continue; 739 740 ent->page = PAGE_SHIFT; 741 ent->xlt = (1 << ent->order) * sizeof(struct mlx5_mtt) / 742 MLX5_IB_UMR_OCTOWORD; 743 ent->access_mode = MLX5_MKC_ACCESS_MODE_MTT; 744 if ((dev->mdev->profile.mask & MLX5_PROF_MASK_MR_CACHE) && 745 !dev->is_rep && mlx5_core_is_pf(dev->mdev) && 746 mlx5_ib_can_load_pas_with_umr(dev, 0)) 747 ent->limit = dev->mdev->profile.mr_cache[i].limit; 748 else 749 ent->limit = 0; 750 spin_lock_irq(&ent->lock); 751 queue_adjust_cache_locked(ent); 752 spin_unlock_irq(&ent->lock); 753 } 754 755 mlx5_mr_cache_debugfs_init(dev); 756 757 return 0; 758 } 759 760 int mlx5_mr_cache_cleanup(struct mlx5_ib_dev *dev) 761 { 762 unsigned int i; 763 764 if (!dev->cache.wq) 765 return 0; 766 767 for (i = 0; i < MAX_MR_CACHE_ENTRIES; i++) { 768 struct mlx5_cache_ent *ent = &dev->cache.ent[i]; 769 770 spin_lock_irq(&ent->lock); 771 ent->disabled = true; 772 spin_unlock_irq(&ent->lock); 773 cancel_work_sync(&ent->work); 774 cancel_delayed_work_sync(&ent->dwork); 775 } 776 777 mlx5_mr_cache_debugfs_cleanup(dev); 778 mlx5_cmd_cleanup_async_ctx(&dev->async_ctx); 779 780 for (i = 0; i < MAX_MR_CACHE_ENTRIES; i++) 781 clean_keys(dev, i); 782 783 destroy_workqueue(dev->cache.wq); 784 del_timer_sync(&dev->delay_timer); 785 786 return 0; 787 } 788 789 struct ib_mr *mlx5_ib_get_dma_mr(struct ib_pd *pd, int acc) 790 { 791 struct mlx5_ib_dev *dev = to_mdev(pd->device); 792 int inlen = MLX5_ST_SZ_BYTES(create_mkey_in); 793 struct mlx5_ib_mr *mr; 794 void *mkc; 795 u32 *in; 796 int err; 797 798 mr = kzalloc(sizeof(*mr), GFP_KERNEL); 799 if (!mr) 800 return ERR_PTR(-ENOMEM); 801 802 in = kzalloc(inlen, GFP_KERNEL); 803 if (!in) { 804 err = -ENOMEM; 805 goto err_free; 806 } 807 808 mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry); 809 810 MLX5_SET(mkc, mkc, access_mode_1_0, MLX5_MKC_ACCESS_MODE_PA); 811 MLX5_SET(mkc, mkc, length64, 1); 812 set_mkc_access_pd_addr_fields(mkc, acc | IB_ACCESS_RELAXED_ORDERING, 0, 813 pd); 814 815 err = mlx5_ib_create_mkey(dev, &mr->mmkey, in, inlen); 816 if (err) 817 goto err_in; 818 819 kfree(in); 820 mr->mmkey.type = MLX5_MKEY_MR; 821 mr->ibmr.lkey = mr->mmkey.key; 822 mr->ibmr.rkey = mr->mmkey.key; 823 mr->umem = NULL; 824 825 return &mr->ibmr; 826 827 err_in: 828 kfree(in); 829 830 err_free: 831 kfree(mr); 832 833 return ERR_PTR(err); 834 } 835 836 static int get_octo_len(u64 addr, u64 len, int page_shift) 837 { 838 u64 page_size = 1ULL << page_shift; 839 u64 offset; 840 int npages; 841 842 offset = addr & (page_size - 1); 843 npages = ALIGN(len + offset, page_size) >> page_shift; 844 return (npages + 1) / 2; 845 } 846 847 static int mr_cache_max_order(struct mlx5_ib_dev *dev) 848 { 849 if (MLX5_CAP_GEN(dev->mdev, umr_extended_translation_offset)) 850 return MR_CACHE_LAST_STD_ENTRY + 2; 851 return MLX5_MAX_UMR_SHIFT; 852 } 853 854 static void mlx5_ib_umr_done(struct ib_cq *cq, struct ib_wc *wc) 855 { 856 struct mlx5_ib_umr_context *context = 857 container_of(wc->wr_cqe, struct mlx5_ib_umr_context, cqe); 858 859 context->status = wc->status; 860 complete(&context->done); 861 } 862 863 static inline void mlx5_ib_init_umr_context(struct mlx5_ib_umr_context *context) 864 { 865 context->cqe.done = mlx5_ib_umr_done; 866 context->status = -1; 867 init_completion(&context->done); 868 } 869 870 static int mlx5_ib_post_send_wait(struct mlx5_ib_dev *dev, 871 struct mlx5_umr_wr *umrwr) 872 { 873 struct umr_common *umrc = &dev->umrc; 874 const struct ib_send_wr *bad; 875 int err; 876 struct mlx5_ib_umr_context umr_context; 877 878 mlx5_ib_init_umr_context(&umr_context); 879 umrwr->wr.wr_cqe = &umr_context.cqe; 880 881 down(&umrc->sem); 882 err = ib_post_send(umrc->qp, &umrwr->wr, &bad); 883 if (err) { 884 mlx5_ib_warn(dev, "UMR post send failed, err %d\n", err); 885 } else { 886 wait_for_completion(&umr_context.done); 887 if (umr_context.status != IB_WC_SUCCESS) { 888 mlx5_ib_warn(dev, "reg umr failed (%u)\n", 889 umr_context.status); 890 err = -EFAULT; 891 } 892 } 893 up(&umrc->sem); 894 return err; 895 } 896 897 static struct mlx5_cache_ent *mr_cache_ent_from_order(struct mlx5_ib_dev *dev, 898 unsigned int order) 899 { 900 struct mlx5_mr_cache *cache = &dev->cache; 901 902 if (order < cache->ent[0].order) 903 return &cache->ent[0]; 904 order = order - cache->ent[0].order; 905 if (order > MR_CACHE_LAST_STD_ENTRY) 906 return NULL; 907 return &cache->ent[order]; 908 } 909 910 static void set_mr_fields(struct mlx5_ib_dev *dev, struct mlx5_ib_mr *mr, 911 u64 length, int access_flags, u64 iova) 912 { 913 mr->ibmr.lkey = mr->mmkey.key; 914 mr->ibmr.rkey = mr->mmkey.key; 915 mr->ibmr.length = length; 916 mr->ibmr.device = &dev->ib_dev; 917 mr->ibmr.iova = iova; 918 mr->access_flags = access_flags; 919 } 920 921 static unsigned int mlx5_umem_dmabuf_default_pgsz(struct ib_umem *umem, 922 u64 iova) 923 { 924 /* 925 * The alignment of iova has already been checked upon entering 926 * UVERBS_METHOD_REG_DMABUF_MR 927 */ 928 umem->iova = iova; 929 return PAGE_SIZE; 930 } 931 932 static struct mlx5_ib_mr *alloc_cacheable_mr(struct ib_pd *pd, 933 struct ib_umem *umem, u64 iova, 934 int access_flags) 935 { 936 struct mlx5_ib_dev *dev = to_mdev(pd->device); 937 struct mlx5_cache_ent *ent; 938 struct mlx5_ib_mr *mr; 939 unsigned int page_size; 940 941 if (umem->is_dmabuf) 942 page_size = mlx5_umem_dmabuf_default_pgsz(umem, iova); 943 else 944 page_size = mlx5_umem_find_best_pgsz(umem, mkc, log_page_size, 945 0, iova); 946 if (WARN_ON(!page_size)) 947 return ERR_PTR(-EINVAL); 948 ent = mr_cache_ent_from_order( 949 dev, order_base_2(ib_umem_num_dma_blocks(umem, page_size))); 950 /* 951 * Matches access in alloc_cache_mr(). If the MR can't come from the 952 * cache then synchronously create an uncached one. 953 */ 954 if (!ent || ent->limit == 0 || 955 !mlx5_ib_can_reconfig_with_umr(dev, 0, access_flags)) { 956 mutex_lock(&dev->slow_path_mutex); 957 mr = reg_create(pd, umem, iova, access_flags, page_size, false); 958 mutex_unlock(&dev->slow_path_mutex); 959 return mr; 960 } 961 962 mr = get_cache_mr(ent); 963 if (!mr) { 964 mr = create_cache_mr(ent); 965 /* 966 * The above already tried to do the same stuff as reg_create(), 967 * no reason to try it again. 968 */ 969 if (IS_ERR(mr)) 970 return mr; 971 } 972 973 mr->ibmr.pd = pd; 974 mr->umem = umem; 975 mr->page_shift = order_base_2(page_size); 976 set_mr_fields(dev, mr, umem->length, access_flags, iova); 977 978 return mr; 979 } 980 981 #define MLX5_MAX_UMR_CHUNK ((1 << (MLX5_MAX_UMR_SHIFT + 4)) - \ 982 MLX5_UMR_MTT_ALIGNMENT) 983 #define MLX5_SPARE_UMR_CHUNK 0x10000 984 985 /* 986 * Allocate a temporary buffer to hold the per-page information to transfer to 987 * HW. For efficiency this should be as large as it can be, but buffer 988 * allocation failure is not allowed, so try smaller sizes. 989 */ 990 static void *mlx5_ib_alloc_xlt(size_t *nents, size_t ent_size, gfp_t gfp_mask) 991 { 992 const size_t xlt_chunk_align = 993 MLX5_UMR_MTT_ALIGNMENT / ent_size; 994 size_t size; 995 void *res = NULL; 996 997 static_assert(PAGE_SIZE % MLX5_UMR_MTT_ALIGNMENT == 0); 998 999 /* 1000 * MLX5_IB_UPD_XLT_ATOMIC doesn't signal an atomic context just that the 1001 * allocation can't trigger any kind of reclaim. 1002 */ 1003 might_sleep(); 1004 1005 gfp_mask |= __GFP_ZERO | __GFP_NORETRY; 1006 1007 /* 1008 * If the system already has a suitable high order page then just use 1009 * that, but don't try hard to create one. This max is about 1M, so a 1010 * free x86 huge page will satisfy it. 1011 */ 1012 size = min_t(size_t, ent_size * ALIGN(*nents, xlt_chunk_align), 1013 MLX5_MAX_UMR_CHUNK); 1014 *nents = size / ent_size; 1015 res = (void *)__get_free_pages(gfp_mask | __GFP_NOWARN, 1016 get_order(size)); 1017 if (res) 1018 return res; 1019 1020 if (size > MLX5_SPARE_UMR_CHUNK) { 1021 size = MLX5_SPARE_UMR_CHUNK; 1022 *nents = size / ent_size; 1023 res = (void *)__get_free_pages(gfp_mask | __GFP_NOWARN, 1024 get_order(size)); 1025 if (res) 1026 return res; 1027 } 1028 1029 *nents = PAGE_SIZE / ent_size; 1030 res = (void *)__get_free_page(gfp_mask); 1031 if (res) 1032 return res; 1033 1034 mutex_lock(&xlt_emergency_page_mutex); 1035 memset(xlt_emergency_page, 0, PAGE_SIZE); 1036 return xlt_emergency_page; 1037 } 1038 1039 static void mlx5_ib_free_xlt(void *xlt, size_t length) 1040 { 1041 if (xlt == xlt_emergency_page) { 1042 mutex_unlock(&xlt_emergency_page_mutex); 1043 return; 1044 } 1045 1046 free_pages((unsigned long)xlt, get_order(length)); 1047 } 1048 1049 /* 1050 * Create a MLX5_IB_SEND_UMR_UPDATE_XLT work request and XLT buffer ready for 1051 * submission. 1052 */ 1053 static void *mlx5_ib_create_xlt_wr(struct mlx5_ib_mr *mr, 1054 struct mlx5_umr_wr *wr, struct ib_sge *sg, 1055 size_t nents, size_t ent_size, 1056 unsigned int flags) 1057 { 1058 struct mlx5_ib_dev *dev = mr_to_mdev(mr); 1059 struct device *ddev = &dev->mdev->pdev->dev; 1060 dma_addr_t dma; 1061 void *xlt; 1062 1063 xlt = mlx5_ib_alloc_xlt(&nents, ent_size, 1064 flags & MLX5_IB_UPD_XLT_ATOMIC ? GFP_ATOMIC : 1065 GFP_KERNEL); 1066 sg->length = nents * ent_size; 1067 dma = dma_map_single(ddev, xlt, sg->length, DMA_TO_DEVICE); 1068 if (dma_mapping_error(ddev, dma)) { 1069 mlx5_ib_err(dev, "unable to map DMA during XLT update.\n"); 1070 mlx5_ib_free_xlt(xlt, sg->length); 1071 return NULL; 1072 } 1073 sg->addr = dma; 1074 sg->lkey = dev->umrc.pd->local_dma_lkey; 1075 1076 memset(wr, 0, sizeof(*wr)); 1077 wr->wr.send_flags = MLX5_IB_SEND_UMR_UPDATE_XLT; 1078 if (!(flags & MLX5_IB_UPD_XLT_ENABLE)) 1079 wr->wr.send_flags |= MLX5_IB_SEND_UMR_FAIL_IF_FREE; 1080 wr->wr.sg_list = sg; 1081 wr->wr.num_sge = 1; 1082 wr->wr.opcode = MLX5_IB_WR_UMR; 1083 wr->pd = mr->ibmr.pd; 1084 wr->mkey = mr->mmkey.key; 1085 wr->length = mr->ibmr.length; 1086 wr->virt_addr = mr->ibmr.iova; 1087 wr->access_flags = mr->access_flags; 1088 wr->page_shift = mr->page_shift; 1089 wr->xlt_size = sg->length; 1090 return xlt; 1091 } 1092 1093 static void mlx5_ib_unmap_free_xlt(struct mlx5_ib_dev *dev, void *xlt, 1094 struct ib_sge *sg) 1095 { 1096 struct device *ddev = &dev->mdev->pdev->dev; 1097 1098 dma_unmap_single(ddev, sg->addr, sg->length, DMA_TO_DEVICE); 1099 mlx5_ib_free_xlt(xlt, sg->length); 1100 } 1101 1102 static unsigned int xlt_wr_final_send_flags(unsigned int flags) 1103 { 1104 unsigned int res = 0; 1105 1106 if (flags & MLX5_IB_UPD_XLT_ENABLE) 1107 res |= MLX5_IB_SEND_UMR_ENABLE_MR | 1108 MLX5_IB_SEND_UMR_UPDATE_PD_ACCESS | 1109 MLX5_IB_SEND_UMR_UPDATE_TRANSLATION; 1110 if (flags & MLX5_IB_UPD_XLT_PD || flags & MLX5_IB_UPD_XLT_ACCESS) 1111 res |= MLX5_IB_SEND_UMR_UPDATE_PD_ACCESS; 1112 if (flags & MLX5_IB_UPD_XLT_ADDR) 1113 res |= MLX5_IB_SEND_UMR_UPDATE_TRANSLATION; 1114 return res; 1115 } 1116 1117 int mlx5_ib_update_xlt(struct mlx5_ib_mr *mr, u64 idx, int npages, 1118 int page_shift, int flags) 1119 { 1120 struct mlx5_ib_dev *dev = mr_to_mdev(mr); 1121 struct device *ddev = &dev->mdev->pdev->dev; 1122 void *xlt; 1123 struct mlx5_umr_wr wr; 1124 struct ib_sge sg; 1125 int err = 0; 1126 int desc_size = (flags & MLX5_IB_UPD_XLT_INDIRECT) 1127 ? sizeof(struct mlx5_klm) 1128 : sizeof(struct mlx5_mtt); 1129 const int page_align = MLX5_UMR_MTT_ALIGNMENT / desc_size; 1130 const int page_mask = page_align - 1; 1131 size_t pages_mapped = 0; 1132 size_t pages_to_map = 0; 1133 size_t pages_iter; 1134 size_t size_to_map = 0; 1135 size_t orig_sg_length; 1136 1137 if ((flags & MLX5_IB_UPD_XLT_INDIRECT) && 1138 !umr_can_use_indirect_mkey(dev)) 1139 return -EPERM; 1140 1141 if (WARN_ON(!mr->umem->is_odp)) 1142 return -EINVAL; 1143 1144 /* UMR copies MTTs in units of MLX5_UMR_MTT_ALIGNMENT bytes, 1145 * so we need to align the offset and length accordingly 1146 */ 1147 if (idx & page_mask) { 1148 npages += idx & page_mask; 1149 idx &= ~page_mask; 1150 } 1151 pages_to_map = ALIGN(npages, page_align); 1152 1153 xlt = mlx5_ib_create_xlt_wr(mr, &wr, &sg, npages, desc_size, flags); 1154 if (!xlt) 1155 return -ENOMEM; 1156 pages_iter = sg.length / desc_size; 1157 orig_sg_length = sg.length; 1158 1159 if (!(flags & MLX5_IB_UPD_XLT_INDIRECT)) { 1160 struct ib_umem_odp *odp = to_ib_umem_odp(mr->umem); 1161 size_t max_pages = ib_umem_odp_num_pages(odp) - idx; 1162 1163 pages_to_map = min_t(size_t, pages_to_map, max_pages); 1164 } 1165 1166 wr.page_shift = page_shift; 1167 1168 for (pages_mapped = 0; 1169 pages_mapped < pages_to_map && !err; 1170 pages_mapped += pages_iter, idx += pages_iter) { 1171 npages = min_t(int, pages_iter, pages_to_map - pages_mapped); 1172 size_to_map = npages * desc_size; 1173 dma_sync_single_for_cpu(ddev, sg.addr, sg.length, 1174 DMA_TO_DEVICE); 1175 mlx5_odp_populate_xlt(xlt, idx, npages, mr, flags); 1176 dma_sync_single_for_device(ddev, sg.addr, sg.length, 1177 DMA_TO_DEVICE); 1178 1179 sg.length = ALIGN(size_to_map, MLX5_UMR_MTT_ALIGNMENT); 1180 1181 if (pages_mapped + pages_iter >= pages_to_map) 1182 wr.wr.send_flags |= xlt_wr_final_send_flags(flags); 1183 1184 wr.offset = idx * desc_size; 1185 wr.xlt_size = sg.length; 1186 1187 err = mlx5_ib_post_send_wait(dev, &wr); 1188 } 1189 sg.length = orig_sg_length; 1190 mlx5_ib_unmap_free_xlt(dev, xlt, &sg); 1191 return err; 1192 } 1193 1194 /* 1195 * Send the DMA list to the HW for a normal MR using UMR. 1196 * Dmabuf MR is handled in a similar way, except that the MLX5_IB_UPD_XLT_ZAP 1197 * flag may be used. 1198 */ 1199 int mlx5_ib_update_mr_pas(struct mlx5_ib_mr *mr, unsigned int flags) 1200 { 1201 struct mlx5_ib_dev *dev = mr_to_mdev(mr); 1202 struct device *ddev = &dev->mdev->pdev->dev; 1203 struct ib_block_iter biter; 1204 struct mlx5_mtt *cur_mtt; 1205 struct mlx5_umr_wr wr; 1206 size_t orig_sg_length; 1207 struct mlx5_mtt *mtt; 1208 size_t final_size; 1209 struct ib_sge sg; 1210 int err = 0; 1211 1212 if (WARN_ON(mr->umem->is_odp)) 1213 return -EINVAL; 1214 1215 mtt = mlx5_ib_create_xlt_wr(mr, &wr, &sg, 1216 ib_umem_num_dma_blocks(mr->umem, 1217 1 << mr->page_shift), 1218 sizeof(*mtt), flags); 1219 if (!mtt) 1220 return -ENOMEM; 1221 orig_sg_length = sg.length; 1222 1223 cur_mtt = mtt; 1224 rdma_for_each_block (mr->umem->sgt_append.sgt.sgl, &biter, 1225 mr->umem->sgt_append.sgt.nents, 1226 BIT(mr->page_shift)) { 1227 if (cur_mtt == (void *)mtt + sg.length) { 1228 dma_sync_single_for_device(ddev, sg.addr, sg.length, 1229 DMA_TO_DEVICE); 1230 err = mlx5_ib_post_send_wait(dev, &wr); 1231 if (err) 1232 goto err; 1233 dma_sync_single_for_cpu(ddev, sg.addr, sg.length, 1234 DMA_TO_DEVICE); 1235 wr.offset += sg.length; 1236 cur_mtt = mtt; 1237 } 1238 1239 cur_mtt->ptag = 1240 cpu_to_be64(rdma_block_iter_dma_address(&biter) | 1241 MLX5_IB_MTT_PRESENT); 1242 1243 if (mr->umem->is_dmabuf && (flags & MLX5_IB_UPD_XLT_ZAP)) 1244 cur_mtt->ptag = 0; 1245 1246 cur_mtt++; 1247 } 1248 1249 final_size = (void *)cur_mtt - (void *)mtt; 1250 sg.length = ALIGN(final_size, MLX5_UMR_MTT_ALIGNMENT); 1251 memset(cur_mtt, 0, sg.length - final_size); 1252 wr.wr.send_flags |= xlt_wr_final_send_flags(flags); 1253 wr.xlt_size = sg.length; 1254 1255 dma_sync_single_for_device(ddev, sg.addr, sg.length, DMA_TO_DEVICE); 1256 err = mlx5_ib_post_send_wait(dev, &wr); 1257 1258 err: 1259 sg.length = orig_sg_length; 1260 mlx5_ib_unmap_free_xlt(dev, mtt, &sg); 1261 return err; 1262 } 1263 1264 /* 1265 * If ibmr is NULL it will be allocated by reg_create. 1266 * Else, the given ibmr will be used. 1267 */ 1268 static struct mlx5_ib_mr *reg_create(struct ib_pd *pd, struct ib_umem *umem, 1269 u64 iova, int access_flags, 1270 unsigned int page_size, bool populate) 1271 { 1272 struct mlx5_ib_dev *dev = to_mdev(pd->device); 1273 struct mlx5_ib_mr *mr; 1274 __be64 *pas; 1275 void *mkc; 1276 int inlen; 1277 u32 *in; 1278 int err; 1279 bool pg_cap = !!(MLX5_CAP_GEN(dev->mdev, pg)); 1280 1281 if (!page_size) 1282 return ERR_PTR(-EINVAL); 1283 mr = kzalloc(sizeof(*mr), GFP_KERNEL); 1284 if (!mr) 1285 return ERR_PTR(-ENOMEM); 1286 1287 mr->ibmr.pd = pd; 1288 mr->access_flags = access_flags; 1289 mr->page_shift = order_base_2(page_size); 1290 1291 inlen = MLX5_ST_SZ_BYTES(create_mkey_in); 1292 if (populate) 1293 inlen += sizeof(*pas) * 1294 roundup(ib_umem_num_dma_blocks(umem, page_size), 2); 1295 in = kvzalloc(inlen, GFP_KERNEL); 1296 if (!in) { 1297 err = -ENOMEM; 1298 goto err_1; 1299 } 1300 pas = (__be64 *)MLX5_ADDR_OF(create_mkey_in, in, klm_pas_mtt); 1301 if (populate) { 1302 if (WARN_ON(access_flags & IB_ACCESS_ON_DEMAND)) { 1303 err = -EINVAL; 1304 goto err_2; 1305 } 1306 mlx5_ib_populate_pas(umem, 1UL << mr->page_shift, pas, 1307 pg_cap ? MLX5_IB_MTT_PRESENT : 0); 1308 } 1309 1310 /* The pg_access bit allows setting the access flags 1311 * in the page list submitted with the command. */ 1312 MLX5_SET(create_mkey_in, in, pg_access, !!(pg_cap)); 1313 1314 mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry); 1315 set_mkc_access_pd_addr_fields(mkc, access_flags, iova, 1316 populate ? pd : dev->umrc.pd); 1317 MLX5_SET(mkc, mkc, free, !populate); 1318 MLX5_SET(mkc, mkc, access_mode_1_0, MLX5_MKC_ACCESS_MODE_MTT); 1319 MLX5_SET(mkc, mkc, umr_en, 1); 1320 1321 MLX5_SET64(mkc, mkc, len, umem->length); 1322 MLX5_SET(mkc, mkc, bsf_octword_size, 0); 1323 MLX5_SET(mkc, mkc, translations_octword_size, 1324 get_octo_len(iova, umem->length, mr->page_shift)); 1325 MLX5_SET(mkc, mkc, log_page_size, mr->page_shift); 1326 if (populate) { 1327 MLX5_SET(create_mkey_in, in, translations_octword_actual_size, 1328 get_octo_len(iova, umem->length, mr->page_shift)); 1329 } 1330 1331 err = mlx5_ib_create_mkey(dev, &mr->mmkey, in, inlen); 1332 if (err) { 1333 mlx5_ib_warn(dev, "create mkey failed\n"); 1334 goto err_2; 1335 } 1336 mr->mmkey.type = MLX5_MKEY_MR; 1337 mr->umem = umem; 1338 set_mr_fields(dev, mr, umem->length, access_flags, iova); 1339 kvfree(in); 1340 1341 mlx5_ib_dbg(dev, "mkey = 0x%x\n", mr->mmkey.key); 1342 1343 return mr; 1344 1345 err_2: 1346 kvfree(in); 1347 err_1: 1348 kfree(mr); 1349 return ERR_PTR(err); 1350 } 1351 1352 static struct ib_mr *mlx5_ib_get_dm_mr(struct ib_pd *pd, u64 start_addr, 1353 u64 length, int acc, int mode) 1354 { 1355 struct mlx5_ib_dev *dev = to_mdev(pd->device); 1356 int inlen = MLX5_ST_SZ_BYTES(create_mkey_in); 1357 struct mlx5_ib_mr *mr; 1358 void *mkc; 1359 u32 *in; 1360 int err; 1361 1362 mr = kzalloc(sizeof(*mr), GFP_KERNEL); 1363 if (!mr) 1364 return ERR_PTR(-ENOMEM); 1365 1366 in = kzalloc(inlen, GFP_KERNEL); 1367 if (!in) { 1368 err = -ENOMEM; 1369 goto err_free; 1370 } 1371 1372 mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry); 1373 1374 MLX5_SET(mkc, mkc, access_mode_1_0, mode & 0x3); 1375 MLX5_SET(mkc, mkc, access_mode_4_2, (mode >> 2) & 0x7); 1376 MLX5_SET64(mkc, mkc, len, length); 1377 set_mkc_access_pd_addr_fields(mkc, acc, start_addr, pd); 1378 1379 err = mlx5_ib_create_mkey(dev, &mr->mmkey, in, inlen); 1380 if (err) 1381 goto err_in; 1382 1383 kfree(in); 1384 1385 set_mr_fields(dev, mr, length, acc, start_addr); 1386 1387 return &mr->ibmr; 1388 1389 err_in: 1390 kfree(in); 1391 1392 err_free: 1393 kfree(mr); 1394 1395 return ERR_PTR(err); 1396 } 1397 1398 int mlx5_ib_advise_mr(struct ib_pd *pd, 1399 enum ib_uverbs_advise_mr_advice advice, 1400 u32 flags, 1401 struct ib_sge *sg_list, 1402 u32 num_sge, 1403 struct uverbs_attr_bundle *attrs) 1404 { 1405 if (advice != IB_UVERBS_ADVISE_MR_ADVICE_PREFETCH && 1406 advice != IB_UVERBS_ADVISE_MR_ADVICE_PREFETCH_WRITE && 1407 advice != IB_UVERBS_ADVISE_MR_ADVICE_PREFETCH_NO_FAULT) 1408 return -EOPNOTSUPP; 1409 1410 return mlx5_ib_advise_mr_prefetch(pd, advice, flags, 1411 sg_list, num_sge); 1412 } 1413 1414 struct ib_mr *mlx5_ib_reg_dm_mr(struct ib_pd *pd, struct ib_dm *dm, 1415 struct ib_dm_mr_attr *attr, 1416 struct uverbs_attr_bundle *attrs) 1417 { 1418 struct mlx5_ib_dm *mdm = to_mdm(dm); 1419 struct mlx5_core_dev *dev = to_mdev(dm->device)->mdev; 1420 u64 start_addr = mdm->dev_addr + attr->offset; 1421 int mode; 1422 1423 switch (mdm->type) { 1424 case MLX5_IB_UAPI_DM_TYPE_MEMIC: 1425 if (attr->access_flags & ~MLX5_IB_DM_MEMIC_ALLOWED_ACCESS) 1426 return ERR_PTR(-EINVAL); 1427 1428 mode = MLX5_MKC_ACCESS_MODE_MEMIC; 1429 start_addr -= pci_resource_start(dev->pdev, 0); 1430 break; 1431 case MLX5_IB_UAPI_DM_TYPE_STEERING_SW_ICM: 1432 case MLX5_IB_UAPI_DM_TYPE_HEADER_MODIFY_SW_ICM: 1433 if (attr->access_flags & ~MLX5_IB_DM_SW_ICM_ALLOWED_ACCESS) 1434 return ERR_PTR(-EINVAL); 1435 1436 mode = MLX5_MKC_ACCESS_MODE_SW_ICM; 1437 break; 1438 default: 1439 return ERR_PTR(-EINVAL); 1440 } 1441 1442 return mlx5_ib_get_dm_mr(pd, start_addr, attr->length, 1443 attr->access_flags, mode); 1444 } 1445 1446 static struct ib_mr *create_real_mr(struct ib_pd *pd, struct ib_umem *umem, 1447 u64 iova, int access_flags) 1448 { 1449 struct mlx5_ib_dev *dev = to_mdev(pd->device); 1450 struct mlx5_ib_mr *mr = NULL; 1451 bool xlt_with_umr; 1452 int err; 1453 1454 xlt_with_umr = mlx5_ib_can_load_pas_with_umr(dev, umem->length); 1455 if (xlt_with_umr) { 1456 mr = alloc_cacheable_mr(pd, umem, iova, access_flags); 1457 } else { 1458 unsigned int page_size = mlx5_umem_find_best_pgsz( 1459 umem, mkc, log_page_size, 0, iova); 1460 1461 mutex_lock(&dev->slow_path_mutex); 1462 mr = reg_create(pd, umem, iova, access_flags, page_size, true); 1463 mutex_unlock(&dev->slow_path_mutex); 1464 } 1465 if (IS_ERR(mr)) { 1466 ib_umem_release(umem); 1467 return ERR_CAST(mr); 1468 } 1469 1470 mlx5_ib_dbg(dev, "mkey 0x%x\n", mr->mmkey.key); 1471 1472 atomic_add(ib_umem_num_pages(umem), &dev->mdev->priv.reg_pages); 1473 1474 if (xlt_with_umr) { 1475 /* 1476 * If the MR was created with reg_create then it will be 1477 * configured properly but left disabled. It is safe to go ahead 1478 * and configure it again via UMR while enabling it. 1479 */ 1480 err = mlx5_ib_update_mr_pas(mr, MLX5_IB_UPD_XLT_ENABLE); 1481 if (err) { 1482 mlx5_ib_dereg_mr(&mr->ibmr, NULL); 1483 return ERR_PTR(err); 1484 } 1485 } 1486 return &mr->ibmr; 1487 } 1488 1489 static struct ib_mr *create_user_odp_mr(struct ib_pd *pd, u64 start, u64 length, 1490 u64 iova, int access_flags, 1491 struct ib_udata *udata) 1492 { 1493 struct mlx5_ib_dev *dev = to_mdev(pd->device); 1494 struct ib_umem_odp *odp; 1495 struct mlx5_ib_mr *mr; 1496 int err; 1497 1498 if (!IS_ENABLED(CONFIG_INFINIBAND_ON_DEMAND_PAGING)) 1499 return ERR_PTR(-EOPNOTSUPP); 1500 1501 err = mlx5r_odp_create_eq(dev, &dev->odp_pf_eq); 1502 if (err) 1503 return ERR_PTR(err); 1504 if (!start && length == U64_MAX) { 1505 if (iova != 0) 1506 return ERR_PTR(-EINVAL); 1507 if (!(dev->odp_caps.general_caps & IB_ODP_SUPPORT_IMPLICIT)) 1508 return ERR_PTR(-EINVAL); 1509 1510 mr = mlx5_ib_alloc_implicit_mr(to_mpd(pd), access_flags); 1511 if (IS_ERR(mr)) 1512 return ERR_CAST(mr); 1513 return &mr->ibmr; 1514 } 1515 1516 /* ODP requires xlt update via umr to work. */ 1517 if (!mlx5_ib_can_load_pas_with_umr(dev, length)) 1518 return ERR_PTR(-EINVAL); 1519 1520 odp = ib_umem_odp_get(&dev->ib_dev, start, length, access_flags, 1521 &mlx5_mn_ops); 1522 if (IS_ERR(odp)) 1523 return ERR_CAST(odp); 1524 1525 mr = alloc_cacheable_mr(pd, &odp->umem, iova, access_flags); 1526 if (IS_ERR(mr)) { 1527 ib_umem_release(&odp->umem); 1528 return ERR_CAST(mr); 1529 } 1530 xa_init(&mr->implicit_children); 1531 1532 odp->private = mr; 1533 err = mlx5r_store_odp_mkey(dev, &mr->mmkey); 1534 if (err) 1535 goto err_dereg_mr; 1536 1537 err = mlx5_ib_init_odp_mr(mr); 1538 if (err) 1539 goto err_dereg_mr; 1540 return &mr->ibmr; 1541 1542 err_dereg_mr: 1543 mlx5_ib_dereg_mr(&mr->ibmr, NULL); 1544 return ERR_PTR(err); 1545 } 1546 1547 struct ib_mr *mlx5_ib_reg_user_mr(struct ib_pd *pd, u64 start, u64 length, 1548 u64 iova, int access_flags, 1549 struct ib_udata *udata) 1550 { 1551 struct mlx5_ib_dev *dev = to_mdev(pd->device); 1552 struct ib_umem *umem; 1553 1554 if (!IS_ENABLED(CONFIG_INFINIBAND_USER_MEM)) 1555 return ERR_PTR(-EOPNOTSUPP); 1556 1557 mlx5_ib_dbg(dev, "start 0x%llx, iova 0x%llx, length 0x%llx, access_flags 0x%x\n", 1558 start, iova, length, access_flags); 1559 1560 if (access_flags & IB_ACCESS_ON_DEMAND) 1561 return create_user_odp_mr(pd, start, length, iova, access_flags, 1562 udata); 1563 umem = ib_umem_get(&dev->ib_dev, start, length, access_flags); 1564 if (IS_ERR(umem)) 1565 return ERR_CAST(umem); 1566 return create_real_mr(pd, umem, iova, access_flags); 1567 } 1568 1569 static void mlx5_ib_dmabuf_invalidate_cb(struct dma_buf_attachment *attach) 1570 { 1571 struct ib_umem_dmabuf *umem_dmabuf = attach->importer_priv; 1572 struct mlx5_ib_mr *mr = umem_dmabuf->private; 1573 1574 dma_resv_assert_held(umem_dmabuf->attach->dmabuf->resv); 1575 1576 if (!umem_dmabuf->sgt) 1577 return; 1578 1579 mlx5_ib_update_mr_pas(mr, MLX5_IB_UPD_XLT_ZAP); 1580 ib_umem_dmabuf_unmap_pages(umem_dmabuf); 1581 } 1582 1583 static struct dma_buf_attach_ops mlx5_ib_dmabuf_attach_ops = { 1584 .allow_peer2peer = 1, 1585 .move_notify = mlx5_ib_dmabuf_invalidate_cb, 1586 }; 1587 1588 struct ib_mr *mlx5_ib_reg_user_mr_dmabuf(struct ib_pd *pd, u64 offset, 1589 u64 length, u64 virt_addr, 1590 int fd, int access_flags, 1591 struct ib_udata *udata) 1592 { 1593 struct mlx5_ib_dev *dev = to_mdev(pd->device); 1594 struct mlx5_ib_mr *mr = NULL; 1595 struct ib_umem_dmabuf *umem_dmabuf; 1596 int err; 1597 1598 if (!IS_ENABLED(CONFIG_INFINIBAND_USER_MEM) || 1599 !IS_ENABLED(CONFIG_INFINIBAND_ON_DEMAND_PAGING)) 1600 return ERR_PTR(-EOPNOTSUPP); 1601 1602 mlx5_ib_dbg(dev, 1603 "offset 0x%llx, virt_addr 0x%llx, length 0x%llx, fd %d, access_flags 0x%x\n", 1604 offset, virt_addr, length, fd, access_flags); 1605 1606 /* dmabuf requires xlt update via umr to work. */ 1607 if (!mlx5_ib_can_load_pas_with_umr(dev, length)) 1608 return ERR_PTR(-EINVAL); 1609 1610 umem_dmabuf = ib_umem_dmabuf_get(&dev->ib_dev, offset, length, fd, 1611 access_flags, 1612 &mlx5_ib_dmabuf_attach_ops); 1613 if (IS_ERR(umem_dmabuf)) { 1614 mlx5_ib_dbg(dev, "umem_dmabuf get failed (%ld)\n", 1615 PTR_ERR(umem_dmabuf)); 1616 return ERR_CAST(umem_dmabuf); 1617 } 1618 1619 mr = alloc_cacheable_mr(pd, &umem_dmabuf->umem, virt_addr, 1620 access_flags); 1621 if (IS_ERR(mr)) { 1622 ib_umem_release(&umem_dmabuf->umem); 1623 return ERR_CAST(mr); 1624 } 1625 1626 mlx5_ib_dbg(dev, "mkey 0x%x\n", mr->mmkey.key); 1627 1628 atomic_add(ib_umem_num_pages(mr->umem), &dev->mdev->priv.reg_pages); 1629 umem_dmabuf->private = mr; 1630 err = mlx5r_store_odp_mkey(dev, &mr->mmkey); 1631 if (err) 1632 goto err_dereg_mr; 1633 1634 err = mlx5_ib_init_dmabuf_mr(mr); 1635 if (err) 1636 goto err_dereg_mr; 1637 return &mr->ibmr; 1638 1639 err_dereg_mr: 1640 mlx5_ib_dereg_mr(&mr->ibmr, NULL); 1641 return ERR_PTR(err); 1642 } 1643 1644 /** 1645 * revoke_mr - Fence all DMA on the MR 1646 * @mr: The MR to fence 1647 * 1648 * Upon return the NIC will not be doing any DMA to the pages under the MR, 1649 * and any DMA in progress will be completed. Failure of this function 1650 * indicates the HW has failed catastrophically. 1651 */ 1652 static int revoke_mr(struct mlx5_ib_mr *mr) 1653 { 1654 struct mlx5_umr_wr umrwr = {}; 1655 1656 if (mr_to_mdev(mr)->mdev->state == MLX5_DEVICE_STATE_INTERNAL_ERROR) 1657 return 0; 1658 1659 umrwr.wr.send_flags = MLX5_IB_SEND_UMR_DISABLE_MR | 1660 MLX5_IB_SEND_UMR_UPDATE_PD_ACCESS; 1661 umrwr.wr.opcode = MLX5_IB_WR_UMR; 1662 umrwr.pd = mr_to_mdev(mr)->umrc.pd; 1663 umrwr.mkey = mr->mmkey.key; 1664 umrwr.ignore_free_state = 1; 1665 1666 return mlx5_ib_post_send_wait(mr_to_mdev(mr), &umrwr); 1667 } 1668 1669 /* 1670 * True if the change in access flags can be done via UMR, only some access 1671 * flags can be updated. 1672 */ 1673 static bool can_use_umr_rereg_access(struct mlx5_ib_dev *dev, 1674 unsigned int current_access_flags, 1675 unsigned int target_access_flags) 1676 { 1677 unsigned int diffs = current_access_flags ^ target_access_flags; 1678 1679 if (diffs & ~(IB_ACCESS_LOCAL_WRITE | IB_ACCESS_REMOTE_WRITE | 1680 IB_ACCESS_REMOTE_READ | IB_ACCESS_RELAXED_ORDERING)) 1681 return false; 1682 return mlx5_ib_can_reconfig_with_umr(dev, current_access_flags, 1683 target_access_flags); 1684 } 1685 1686 static int umr_rereg_pd_access(struct mlx5_ib_mr *mr, struct ib_pd *pd, 1687 int access_flags) 1688 { 1689 struct mlx5_ib_dev *dev = to_mdev(mr->ibmr.device); 1690 struct mlx5_umr_wr umrwr = { 1691 .wr = { 1692 .send_flags = MLX5_IB_SEND_UMR_FAIL_IF_FREE | 1693 MLX5_IB_SEND_UMR_UPDATE_PD_ACCESS, 1694 .opcode = MLX5_IB_WR_UMR, 1695 }, 1696 .mkey = mr->mmkey.key, 1697 .pd = pd, 1698 .access_flags = access_flags, 1699 }; 1700 int err; 1701 1702 err = mlx5_ib_post_send_wait(dev, &umrwr); 1703 if (err) 1704 return err; 1705 1706 mr->access_flags = access_flags; 1707 return 0; 1708 } 1709 1710 static bool can_use_umr_rereg_pas(struct mlx5_ib_mr *mr, 1711 struct ib_umem *new_umem, 1712 int new_access_flags, u64 iova, 1713 unsigned long *page_size) 1714 { 1715 struct mlx5_ib_dev *dev = to_mdev(mr->ibmr.device); 1716 1717 /* We only track the allocated sizes of MRs from the cache */ 1718 if (!mr->cache_ent) 1719 return false; 1720 if (!mlx5_ib_can_load_pas_with_umr(dev, new_umem->length)) 1721 return false; 1722 1723 *page_size = 1724 mlx5_umem_find_best_pgsz(new_umem, mkc, log_page_size, 0, iova); 1725 if (WARN_ON(!*page_size)) 1726 return false; 1727 return (1ULL << mr->cache_ent->order) >= 1728 ib_umem_num_dma_blocks(new_umem, *page_size); 1729 } 1730 1731 static int umr_rereg_pas(struct mlx5_ib_mr *mr, struct ib_pd *pd, 1732 int access_flags, int flags, struct ib_umem *new_umem, 1733 u64 iova, unsigned long page_size) 1734 { 1735 struct mlx5_ib_dev *dev = to_mdev(mr->ibmr.device); 1736 int upd_flags = MLX5_IB_UPD_XLT_ADDR | MLX5_IB_UPD_XLT_ENABLE; 1737 struct ib_umem *old_umem = mr->umem; 1738 int err; 1739 1740 /* 1741 * To keep everything simple the MR is revoked before we start to mess 1742 * with it. This ensure the change is atomic relative to any use of the 1743 * MR. 1744 */ 1745 err = revoke_mr(mr); 1746 if (err) 1747 return err; 1748 1749 if (flags & IB_MR_REREG_PD) { 1750 mr->ibmr.pd = pd; 1751 upd_flags |= MLX5_IB_UPD_XLT_PD; 1752 } 1753 if (flags & IB_MR_REREG_ACCESS) { 1754 mr->access_flags = access_flags; 1755 upd_flags |= MLX5_IB_UPD_XLT_ACCESS; 1756 } 1757 1758 mr->ibmr.length = new_umem->length; 1759 mr->ibmr.iova = iova; 1760 mr->ibmr.length = new_umem->length; 1761 mr->page_shift = order_base_2(page_size); 1762 mr->umem = new_umem; 1763 err = mlx5_ib_update_mr_pas(mr, upd_flags); 1764 if (err) { 1765 /* 1766 * The MR is revoked at this point so there is no issue to free 1767 * new_umem. 1768 */ 1769 mr->umem = old_umem; 1770 return err; 1771 } 1772 1773 atomic_sub(ib_umem_num_pages(old_umem), &dev->mdev->priv.reg_pages); 1774 ib_umem_release(old_umem); 1775 atomic_add(ib_umem_num_pages(new_umem), &dev->mdev->priv.reg_pages); 1776 return 0; 1777 } 1778 1779 struct ib_mr *mlx5_ib_rereg_user_mr(struct ib_mr *ib_mr, int flags, u64 start, 1780 u64 length, u64 iova, int new_access_flags, 1781 struct ib_pd *new_pd, 1782 struct ib_udata *udata) 1783 { 1784 struct mlx5_ib_dev *dev = to_mdev(ib_mr->device); 1785 struct mlx5_ib_mr *mr = to_mmr(ib_mr); 1786 int err; 1787 1788 if (!IS_ENABLED(CONFIG_INFINIBAND_USER_MEM)) 1789 return ERR_PTR(-EOPNOTSUPP); 1790 1791 mlx5_ib_dbg( 1792 dev, 1793 "start 0x%llx, iova 0x%llx, length 0x%llx, access_flags 0x%x\n", 1794 start, iova, length, new_access_flags); 1795 1796 if (flags & ~(IB_MR_REREG_TRANS | IB_MR_REREG_PD | IB_MR_REREG_ACCESS)) 1797 return ERR_PTR(-EOPNOTSUPP); 1798 1799 if (!(flags & IB_MR_REREG_ACCESS)) 1800 new_access_flags = mr->access_flags; 1801 if (!(flags & IB_MR_REREG_PD)) 1802 new_pd = ib_mr->pd; 1803 1804 if (!(flags & IB_MR_REREG_TRANS)) { 1805 struct ib_umem *umem; 1806 1807 /* Fast path for PD/access change */ 1808 if (can_use_umr_rereg_access(dev, mr->access_flags, 1809 new_access_flags)) { 1810 err = umr_rereg_pd_access(mr, new_pd, new_access_flags); 1811 if (err) 1812 return ERR_PTR(err); 1813 return NULL; 1814 } 1815 /* DM or ODP MR's don't have a normal umem so we can't re-use it */ 1816 if (!mr->umem || is_odp_mr(mr) || is_dmabuf_mr(mr)) 1817 goto recreate; 1818 1819 /* 1820 * Only one active MR can refer to a umem at one time, revoke 1821 * the old MR before assigning the umem to the new one. 1822 */ 1823 err = revoke_mr(mr); 1824 if (err) 1825 return ERR_PTR(err); 1826 umem = mr->umem; 1827 mr->umem = NULL; 1828 atomic_sub(ib_umem_num_pages(umem), &dev->mdev->priv.reg_pages); 1829 1830 return create_real_mr(new_pd, umem, mr->ibmr.iova, 1831 new_access_flags); 1832 } 1833 1834 /* 1835 * DM doesn't have a PAS list so we can't re-use it, odp/dmabuf does 1836 * but the logic around releasing the umem is different 1837 */ 1838 if (!mr->umem || is_odp_mr(mr) || is_dmabuf_mr(mr)) 1839 goto recreate; 1840 1841 if (!(new_access_flags & IB_ACCESS_ON_DEMAND) && 1842 can_use_umr_rereg_access(dev, mr->access_flags, new_access_flags)) { 1843 struct ib_umem *new_umem; 1844 unsigned long page_size; 1845 1846 new_umem = ib_umem_get(&dev->ib_dev, start, length, 1847 new_access_flags); 1848 if (IS_ERR(new_umem)) 1849 return ERR_CAST(new_umem); 1850 1851 /* Fast path for PAS change */ 1852 if (can_use_umr_rereg_pas(mr, new_umem, new_access_flags, iova, 1853 &page_size)) { 1854 err = umr_rereg_pas(mr, new_pd, new_access_flags, flags, 1855 new_umem, iova, page_size); 1856 if (err) { 1857 ib_umem_release(new_umem); 1858 return ERR_PTR(err); 1859 } 1860 return NULL; 1861 } 1862 return create_real_mr(new_pd, new_umem, iova, new_access_flags); 1863 } 1864 1865 /* 1866 * Everything else has no state we can preserve, just create a new MR 1867 * from scratch 1868 */ 1869 recreate: 1870 return mlx5_ib_reg_user_mr(new_pd, start, length, iova, 1871 new_access_flags, udata); 1872 } 1873 1874 static int 1875 mlx5_alloc_priv_descs(struct ib_device *device, 1876 struct mlx5_ib_mr *mr, 1877 int ndescs, 1878 int desc_size) 1879 { 1880 struct mlx5_ib_dev *dev = to_mdev(device); 1881 struct device *ddev = &dev->mdev->pdev->dev; 1882 int size = ndescs * desc_size; 1883 int add_size; 1884 int ret; 1885 1886 add_size = max_t(int, MLX5_UMR_ALIGN - ARCH_KMALLOC_MINALIGN, 0); 1887 1888 mr->descs_alloc = kzalloc(size + add_size, GFP_KERNEL); 1889 if (!mr->descs_alloc) 1890 return -ENOMEM; 1891 1892 mr->descs = PTR_ALIGN(mr->descs_alloc, MLX5_UMR_ALIGN); 1893 1894 mr->desc_map = dma_map_single(ddev, mr->descs, size, DMA_TO_DEVICE); 1895 if (dma_mapping_error(ddev, mr->desc_map)) { 1896 ret = -ENOMEM; 1897 goto err; 1898 } 1899 1900 return 0; 1901 err: 1902 kfree(mr->descs_alloc); 1903 1904 return ret; 1905 } 1906 1907 static void mlx5_free_priv_descs(struct mlx5_ib_mr *mr) 1908 { 1909 struct mlx5_ib_dev *dev = to_mdev(mr->ibmr.device); 1910 int size = mr->max_descs * mr->desc_size; 1911 1912 if (!mr->descs) 1913 return; 1914 1915 dma_unmap_single(&dev->mdev->pdev->dev, mr->desc_map, size, 1916 DMA_TO_DEVICE); 1917 kfree(mr->descs_alloc); 1918 mr->descs = NULL; 1919 } 1920 1921 int mlx5_ib_dereg_mr(struct ib_mr *ibmr, struct ib_udata *udata) 1922 { 1923 struct mlx5_ib_mr *mr = to_mmr(ibmr); 1924 struct mlx5_ib_dev *dev = to_mdev(ibmr->device); 1925 int rc; 1926 1927 /* 1928 * Any async use of the mr must hold the refcount, once the refcount 1929 * goes to zero no other thread, such as ODP page faults, prefetch, any 1930 * UMR activity, etc can touch the mkey. Thus it is safe to destroy it. 1931 */ 1932 if (IS_ENABLED(CONFIG_INFINIBAND_ON_DEMAND_PAGING) && 1933 refcount_read(&mr->mmkey.usecount) != 0 && 1934 xa_erase(&mr_to_mdev(mr)->odp_mkeys, mlx5_base_mkey(mr->mmkey.key))) 1935 mlx5r_deref_wait_odp_mkey(&mr->mmkey); 1936 1937 if (ibmr->type == IB_MR_TYPE_INTEGRITY) { 1938 xa_cmpxchg(&dev->sig_mrs, mlx5_base_mkey(mr->mmkey.key), 1939 mr->sig, NULL, GFP_KERNEL); 1940 1941 if (mr->mtt_mr) { 1942 rc = mlx5_ib_dereg_mr(&mr->mtt_mr->ibmr, NULL); 1943 if (rc) 1944 return rc; 1945 mr->mtt_mr = NULL; 1946 } 1947 if (mr->klm_mr) { 1948 rc = mlx5_ib_dereg_mr(&mr->klm_mr->ibmr, NULL); 1949 if (rc) 1950 return rc; 1951 mr->klm_mr = NULL; 1952 } 1953 1954 if (mlx5_core_destroy_psv(dev->mdev, 1955 mr->sig->psv_memory.psv_idx)) 1956 mlx5_ib_warn(dev, "failed to destroy mem psv %d\n", 1957 mr->sig->psv_memory.psv_idx); 1958 if (mlx5_core_destroy_psv(dev->mdev, mr->sig->psv_wire.psv_idx)) 1959 mlx5_ib_warn(dev, "failed to destroy wire psv %d\n", 1960 mr->sig->psv_wire.psv_idx); 1961 kfree(mr->sig); 1962 mr->sig = NULL; 1963 } 1964 1965 /* Stop DMA */ 1966 if (mr->cache_ent) { 1967 if (revoke_mr(mr)) { 1968 spin_lock_irq(&mr->cache_ent->lock); 1969 mr->cache_ent->total_mrs--; 1970 spin_unlock_irq(&mr->cache_ent->lock); 1971 mr->cache_ent = NULL; 1972 } 1973 } 1974 if (!mr->cache_ent) { 1975 rc = destroy_mkey(to_mdev(mr->ibmr.device), mr); 1976 if (rc) 1977 return rc; 1978 } 1979 1980 if (mr->umem) { 1981 bool is_odp = is_odp_mr(mr); 1982 1983 if (!is_odp) 1984 atomic_sub(ib_umem_num_pages(mr->umem), 1985 &dev->mdev->priv.reg_pages); 1986 ib_umem_release(mr->umem); 1987 if (is_odp) 1988 mlx5_ib_free_odp_mr(mr); 1989 } 1990 1991 if (mr->cache_ent) { 1992 mlx5_mr_cache_free(dev, mr); 1993 } else { 1994 if (!udata) 1995 mlx5_free_priv_descs(mr); 1996 kfree(mr); 1997 } 1998 return 0; 1999 } 2000 2001 static void mlx5_set_umr_free_mkey(struct ib_pd *pd, u32 *in, int ndescs, 2002 int access_mode, int page_shift) 2003 { 2004 void *mkc; 2005 2006 mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry); 2007 2008 /* This is only used from the kernel, so setting the PD is OK. */ 2009 set_mkc_access_pd_addr_fields(mkc, IB_ACCESS_RELAXED_ORDERING, 0, pd); 2010 MLX5_SET(mkc, mkc, free, 1); 2011 MLX5_SET(mkc, mkc, translations_octword_size, ndescs); 2012 MLX5_SET(mkc, mkc, access_mode_1_0, access_mode & 0x3); 2013 MLX5_SET(mkc, mkc, access_mode_4_2, (access_mode >> 2) & 0x7); 2014 MLX5_SET(mkc, mkc, umr_en, 1); 2015 MLX5_SET(mkc, mkc, log_page_size, page_shift); 2016 } 2017 2018 static int _mlx5_alloc_mkey_descs(struct ib_pd *pd, struct mlx5_ib_mr *mr, 2019 int ndescs, int desc_size, int page_shift, 2020 int access_mode, u32 *in, int inlen) 2021 { 2022 struct mlx5_ib_dev *dev = to_mdev(pd->device); 2023 int err; 2024 2025 mr->access_mode = access_mode; 2026 mr->desc_size = desc_size; 2027 mr->max_descs = ndescs; 2028 2029 err = mlx5_alloc_priv_descs(pd->device, mr, ndescs, desc_size); 2030 if (err) 2031 return err; 2032 2033 mlx5_set_umr_free_mkey(pd, in, ndescs, access_mode, page_shift); 2034 2035 err = mlx5_ib_create_mkey(dev, &mr->mmkey, in, inlen); 2036 if (err) 2037 goto err_free_descs; 2038 2039 mr->mmkey.type = MLX5_MKEY_MR; 2040 mr->ibmr.lkey = mr->mmkey.key; 2041 mr->ibmr.rkey = mr->mmkey.key; 2042 2043 return 0; 2044 2045 err_free_descs: 2046 mlx5_free_priv_descs(mr); 2047 return err; 2048 } 2049 2050 static struct mlx5_ib_mr *mlx5_ib_alloc_pi_mr(struct ib_pd *pd, 2051 u32 max_num_sg, u32 max_num_meta_sg, 2052 int desc_size, int access_mode) 2053 { 2054 int inlen = MLX5_ST_SZ_BYTES(create_mkey_in); 2055 int ndescs = ALIGN(max_num_sg + max_num_meta_sg, 4); 2056 int page_shift = 0; 2057 struct mlx5_ib_mr *mr; 2058 u32 *in; 2059 int err; 2060 2061 mr = kzalloc(sizeof(*mr), GFP_KERNEL); 2062 if (!mr) 2063 return ERR_PTR(-ENOMEM); 2064 2065 mr->ibmr.pd = pd; 2066 mr->ibmr.device = pd->device; 2067 2068 in = kzalloc(inlen, GFP_KERNEL); 2069 if (!in) { 2070 err = -ENOMEM; 2071 goto err_free; 2072 } 2073 2074 if (access_mode == MLX5_MKC_ACCESS_MODE_MTT) 2075 page_shift = PAGE_SHIFT; 2076 2077 err = _mlx5_alloc_mkey_descs(pd, mr, ndescs, desc_size, page_shift, 2078 access_mode, in, inlen); 2079 if (err) 2080 goto err_free_in; 2081 2082 kfree(in); 2083 2084 return mr; 2085 2086 err_free_in: 2087 kfree(in); 2088 err_free: 2089 kfree(mr); 2090 return ERR_PTR(err); 2091 } 2092 2093 static int mlx5_alloc_mem_reg_descs(struct ib_pd *pd, struct mlx5_ib_mr *mr, 2094 int ndescs, u32 *in, int inlen) 2095 { 2096 return _mlx5_alloc_mkey_descs(pd, mr, ndescs, sizeof(struct mlx5_mtt), 2097 PAGE_SHIFT, MLX5_MKC_ACCESS_MODE_MTT, in, 2098 inlen); 2099 } 2100 2101 static int mlx5_alloc_sg_gaps_descs(struct ib_pd *pd, struct mlx5_ib_mr *mr, 2102 int ndescs, u32 *in, int inlen) 2103 { 2104 return _mlx5_alloc_mkey_descs(pd, mr, ndescs, sizeof(struct mlx5_klm), 2105 0, MLX5_MKC_ACCESS_MODE_KLMS, in, inlen); 2106 } 2107 2108 static int mlx5_alloc_integrity_descs(struct ib_pd *pd, struct mlx5_ib_mr *mr, 2109 int max_num_sg, int max_num_meta_sg, 2110 u32 *in, int inlen) 2111 { 2112 struct mlx5_ib_dev *dev = to_mdev(pd->device); 2113 u32 psv_index[2]; 2114 void *mkc; 2115 int err; 2116 2117 mr->sig = kzalloc(sizeof(*mr->sig), GFP_KERNEL); 2118 if (!mr->sig) 2119 return -ENOMEM; 2120 2121 /* create mem & wire PSVs */ 2122 err = mlx5_core_create_psv(dev->mdev, to_mpd(pd)->pdn, 2, psv_index); 2123 if (err) 2124 goto err_free_sig; 2125 2126 mr->sig->psv_memory.psv_idx = psv_index[0]; 2127 mr->sig->psv_wire.psv_idx = psv_index[1]; 2128 2129 mr->sig->sig_status_checked = true; 2130 mr->sig->sig_err_exists = false; 2131 /* Next UMR, Arm SIGERR */ 2132 ++mr->sig->sigerr_count; 2133 mr->klm_mr = mlx5_ib_alloc_pi_mr(pd, max_num_sg, max_num_meta_sg, 2134 sizeof(struct mlx5_klm), 2135 MLX5_MKC_ACCESS_MODE_KLMS); 2136 if (IS_ERR(mr->klm_mr)) { 2137 err = PTR_ERR(mr->klm_mr); 2138 goto err_destroy_psv; 2139 } 2140 mr->mtt_mr = mlx5_ib_alloc_pi_mr(pd, max_num_sg, max_num_meta_sg, 2141 sizeof(struct mlx5_mtt), 2142 MLX5_MKC_ACCESS_MODE_MTT); 2143 if (IS_ERR(mr->mtt_mr)) { 2144 err = PTR_ERR(mr->mtt_mr); 2145 goto err_free_klm_mr; 2146 } 2147 2148 /* Set bsf descriptors for mkey */ 2149 mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry); 2150 MLX5_SET(mkc, mkc, bsf_en, 1); 2151 MLX5_SET(mkc, mkc, bsf_octword_size, MLX5_MKEY_BSF_OCTO_SIZE); 2152 2153 err = _mlx5_alloc_mkey_descs(pd, mr, 4, sizeof(struct mlx5_klm), 0, 2154 MLX5_MKC_ACCESS_MODE_KLMS, in, inlen); 2155 if (err) 2156 goto err_free_mtt_mr; 2157 2158 err = xa_err(xa_store(&dev->sig_mrs, mlx5_base_mkey(mr->mmkey.key), 2159 mr->sig, GFP_KERNEL)); 2160 if (err) 2161 goto err_free_descs; 2162 return 0; 2163 2164 err_free_descs: 2165 destroy_mkey(dev, mr); 2166 mlx5_free_priv_descs(mr); 2167 err_free_mtt_mr: 2168 mlx5_ib_dereg_mr(&mr->mtt_mr->ibmr, NULL); 2169 mr->mtt_mr = NULL; 2170 err_free_klm_mr: 2171 mlx5_ib_dereg_mr(&mr->klm_mr->ibmr, NULL); 2172 mr->klm_mr = NULL; 2173 err_destroy_psv: 2174 if (mlx5_core_destroy_psv(dev->mdev, mr->sig->psv_memory.psv_idx)) 2175 mlx5_ib_warn(dev, "failed to destroy mem psv %d\n", 2176 mr->sig->psv_memory.psv_idx); 2177 if (mlx5_core_destroy_psv(dev->mdev, mr->sig->psv_wire.psv_idx)) 2178 mlx5_ib_warn(dev, "failed to destroy wire psv %d\n", 2179 mr->sig->psv_wire.psv_idx); 2180 err_free_sig: 2181 kfree(mr->sig); 2182 2183 return err; 2184 } 2185 2186 static struct ib_mr *__mlx5_ib_alloc_mr(struct ib_pd *pd, 2187 enum ib_mr_type mr_type, u32 max_num_sg, 2188 u32 max_num_meta_sg) 2189 { 2190 struct mlx5_ib_dev *dev = to_mdev(pd->device); 2191 int inlen = MLX5_ST_SZ_BYTES(create_mkey_in); 2192 int ndescs = ALIGN(max_num_sg, 4); 2193 struct mlx5_ib_mr *mr; 2194 u32 *in; 2195 int err; 2196 2197 mr = kzalloc(sizeof(*mr), GFP_KERNEL); 2198 if (!mr) 2199 return ERR_PTR(-ENOMEM); 2200 2201 in = kzalloc(inlen, GFP_KERNEL); 2202 if (!in) { 2203 err = -ENOMEM; 2204 goto err_free; 2205 } 2206 2207 mr->ibmr.device = pd->device; 2208 2209 switch (mr_type) { 2210 case IB_MR_TYPE_MEM_REG: 2211 err = mlx5_alloc_mem_reg_descs(pd, mr, ndescs, in, inlen); 2212 break; 2213 case IB_MR_TYPE_SG_GAPS: 2214 err = mlx5_alloc_sg_gaps_descs(pd, mr, ndescs, in, inlen); 2215 break; 2216 case IB_MR_TYPE_INTEGRITY: 2217 err = mlx5_alloc_integrity_descs(pd, mr, max_num_sg, 2218 max_num_meta_sg, in, inlen); 2219 break; 2220 default: 2221 mlx5_ib_warn(dev, "Invalid mr type %d\n", mr_type); 2222 err = -EINVAL; 2223 } 2224 2225 if (err) 2226 goto err_free_in; 2227 2228 kfree(in); 2229 2230 return &mr->ibmr; 2231 2232 err_free_in: 2233 kfree(in); 2234 err_free: 2235 kfree(mr); 2236 return ERR_PTR(err); 2237 } 2238 2239 struct ib_mr *mlx5_ib_alloc_mr(struct ib_pd *pd, enum ib_mr_type mr_type, 2240 u32 max_num_sg) 2241 { 2242 return __mlx5_ib_alloc_mr(pd, mr_type, max_num_sg, 0); 2243 } 2244 2245 struct ib_mr *mlx5_ib_alloc_mr_integrity(struct ib_pd *pd, 2246 u32 max_num_sg, u32 max_num_meta_sg) 2247 { 2248 return __mlx5_ib_alloc_mr(pd, IB_MR_TYPE_INTEGRITY, max_num_sg, 2249 max_num_meta_sg); 2250 } 2251 2252 int mlx5_ib_alloc_mw(struct ib_mw *ibmw, struct ib_udata *udata) 2253 { 2254 struct mlx5_ib_dev *dev = to_mdev(ibmw->device); 2255 int inlen = MLX5_ST_SZ_BYTES(create_mkey_in); 2256 struct mlx5_ib_mw *mw = to_mmw(ibmw); 2257 unsigned int ndescs; 2258 u32 *in = NULL; 2259 void *mkc; 2260 int err; 2261 struct mlx5_ib_alloc_mw req = {}; 2262 struct { 2263 __u32 comp_mask; 2264 __u32 response_length; 2265 } resp = {}; 2266 2267 err = ib_copy_from_udata(&req, udata, min(udata->inlen, sizeof(req))); 2268 if (err) 2269 return err; 2270 2271 if (req.comp_mask || req.reserved1 || req.reserved2) 2272 return -EOPNOTSUPP; 2273 2274 if (udata->inlen > sizeof(req) && 2275 !ib_is_udata_cleared(udata, sizeof(req), 2276 udata->inlen - sizeof(req))) 2277 return -EOPNOTSUPP; 2278 2279 ndescs = req.num_klms ? roundup(req.num_klms, 4) : roundup(1, 4); 2280 2281 in = kzalloc(inlen, GFP_KERNEL); 2282 if (!in) { 2283 err = -ENOMEM; 2284 goto free; 2285 } 2286 2287 mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry); 2288 2289 MLX5_SET(mkc, mkc, free, 1); 2290 MLX5_SET(mkc, mkc, translations_octword_size, ndescs); 2291 MLX5_SET(mkc, mkc, pd, to_mpd(ibmw->pd)->pdn); 2292 MLX5_SET(mkc, mkc, umr_en, 1); 2293 MLX5_SET(mkc, mkc, lr, 1); 2294 MLX5_SET(mkc, mkc, access_mode_1_0, MLX5_MKC_ACCESS_MODE_KLMS); 2295 MLX5_SET(mkc, mkc, en_rinval, !!((ibmw->type == IB_MW_TYPE_2))); 2296 MLX5_SET(mkc, mkc, qpn, 0xffffff); 2297 2298 err = mlx5_ib_create_mkey(dev, &mw->mmkey, in, inlen); 2299 if (err) 2300 goto free; 2301 2302 mw->mmkey.type = MLX5_MKEY_MW; 2303 ibmw->rkey = mw->mmkey.key; 2304 mw->mmkey.ndescs = ndescs; 2305 2306 resp.response_length = 2307 min(offsetofend(typeof(resp), response_length), udata->outlen); 2308 if (resp.response_length) { 2309 err = ib_copy_to_udata(udata, &resp, resp.response_length); 2310 if (err) 2311 goto free_mkey; 2312 } 2313 2314 if (IS_ENABLED(CONFIG_INFINIBAND_ON_DEMAND_PAGING)) { 2315 err = mlx5r_store_odp_mkey(dev, &mw->mmkey); 2316 if (err) 2317 goto free_mkey; 2318 } 2319 2320 kfree(in); 2321 return 0; 2322 2323 free_mkey: 2324 mlx5_core_destroy_mkey(dev->mdev, mw->mmkey.key); 2325 free: 2326 kfree(in); 2327 return err; 2328 } 2329 2330 int mlx5_ib_dealloc_mw(struct ib_mw *mw) 2331 { 2332 struct mlx5_ib_dev *dev = to_mdev(mw->device); 2333 struct mlx5_ib_mw *mmw = to_mmw(mw); 2334 2335 if (IS_ENABLED(CONFIG_INFINIBAND_ON_DEMAND_PAGING) && 2336 xa_erase(&dev->odp_mkeys, mlx5_base_mkey(mmw->mmkey.key))) 2337 /* 2338 * pagefault_single_data_segment() may be accessing mmw 2339 * if the user bound an ODP MR to this MW. 2340 */ 2341 mlx5r_deref_wait_odp_mkey(&mmw->mmkey); 2342 2343 return mlx5_core_destroy_mkey(dev->mdev, mmw->mmkey.key); 2344 } 2345 2346 int mlx5_ib_check_mr_status(struct ib_mr *ibmr, u32 check_mask, 2347 struct ib_mr_status *mr_status) 2348 { 2349 struct mlx5_ib_mr *mmr = to_mmr(ibmr); 2350 int ret = 0; 2351 2352 if (check_mask & ~IB_MR_CHECK_SIG_STATUS) { 2353 pr_err("Invalid status check mask\n"); 2354 ret = -EINVAL; 2355 goto done; 2356 } 2357 2358 mr_status->fail_status = 0; 2359 if (check_mask & IB_MR_CHECK_SIG_STATUS) { 2360 if (!mmr->sig) { 2361 ret = -EINVAL; 2362 pr_err("signature status check requested on a non-signature enabled MR\n"); 2363 goto done; 2364 } 2365 2366 mmr->sig->sig_status_checked = true; 2367 if (!mmr->sig->sig_err_exists) 2368 goto done; 2369 2370 if (ibmr->lkey == mmr->sig->err_item.key) 2371 memcpy(&mr_status->sig_err, &mmr->sig->err_item, 2372 sizeof(mr_status->sig_err)); 2373 else { 2374 mr_status->sig_err.err_type = IB_SIG_BAD_GUARD; 2375 mr_status->sig_err.sig_err_offset = 0; 2376 mr_status->sig_err.key = mmr->sig->err_item.key; 2377 } 2378 2379 mmr->sig->sig_err_exists = false; 2380 mr_status->fail_status |= IB_MR_CHECK_SIG_STATUS; 2381 } 2382 2383 done: 2384 return ret; 2385 } 2386 2387 static int 2388 mlx5_ib_map_pa_mr_sg_pi(struct ib_mr *ibmr, struct scatterlist *data_sg, 2389 int data_sg_nents, unsigned int *data_sg_offset, 2390 struct scatterlist *meta_sg, int meta_sg_nents, 2391 unsigned int *meta_sg_offset) 2392 { 2393 struct mlx5_ib_mr *mr = to_mmr(ibmr); 2394 unsigned int sg_offset = 0; 2395 int n = 0; 2396 2397 mr->meta_length = 0; 2398 if (data_sg_nents == 1) { 2399 n++; 2400 mr->mmkey.ndescs = 1; 2401 if (data_sg_offset) 2402 sg_offset = *data_sg_offset; 2403 mr->data_length = sg_dma_len(data_sg) - sg_offset; 2404 mr->data_iova = sg_dma_address(data_sg) + sg_offset; 2405 if (meta_sg_nents == 1) { 2406 n++; 2407 mr->meta_ndescs = 1; 2408 if (meta_sg_offset) 2409 sg_offset = *meta_sg_offset; 2410 else 2411 sg_offset = 0; 2412 mr->meta_length = sg_dma_len(meta_sg) - sg_offset; 2413 mr->pi_iova = sg_dma_address(meta_sg) + sg_offset; 2414 } 2415 ibmr->length = mr->data_length + mr->meta_length; 2416 } 2417 2418 return n; 2419 } 2420 2421 static int 2422 mlx5_ib_sg_to_klms(struct mlx5_ib_mr *mr, 2423 struct scatterlist *sgl, 2424 unsigned short sg_nents, 2425 unsigned int *sg_offset_p, 2426 struct scatterlist *meta_sgl, 2427 unsigned short meta_sg_nents, 2428 unsigned int *meta_sg_offset_p) 2429 { 2430 struct scatterlist *sg = sgl; 2431 struct mlx5_klm *klms = mr->descs; 2432 unsigned int sg_offset = sg_offset_p ? *sg_offset_p : 0; 2433 u32 lkey = mr->ibmr.pd->local_dma_lkey; 2434 int i, j = 0; 2435 2436 mr->ibmr.iova = sg_dma_address(sg) + sg_offset; 2437 mr->ibmr.length = 0; 2438 2439 for_each_sg(sgl, sg, sg_nents, i) { 2440 if (unlikely(i >= mr->max_descs)) 2441 break; 2442 klms[i].va = cpu_to_be64(sg_dma_address(sg) + sg_offset); 2443 klms[i].bcount = cpu_to_be32(sg_dma_len(sg) - sg_offset); 2444 klms[i].key = cpu_to_be32(lkey); 2445 mr->ibmr.length += sg_dma_len(sg) - sg_offset; 2446 2447 sg_offset = 0; 2448 } 2449 2450 if (sg_offset_p) 2451 *sg_offset_p = sg_offset; 2452 2453 mr->mmkey.ndescs = i; 2454 mr->data_length = mr->ibmr.length; 2455 2456 if (meta_sg_nents) { 2457 sg = meta_sgl; 2458 sg_offset = meta_sg_offset_p ? *meta_sg_offset_p : 0; 2459 for_each_sg(meta_sgl, sg, meta_sg_nents, j) { 2460 if (unlikely(i + j >= mr->max_descs)) 2461 break; 2462 klms[i + j].va = cpu_to_be64(sg_dma_address(sg) + 2463 sg_offset); 2464 klms[i + j].bcount = cpu_to_be32(sg_dma_len(sg) - 2465 sg_offset); 2466 klms[i + j].key = cpu_to_be32(lkey); 2467 mr->ibmr.length += sg_dma_len(sg) - sg_offset; 2468 2469 sg_offset = 0; 2470 } 2471 if (meta_sg_offset_p) 2472 *meta_sg_offset_p = sg_offset; 2473 2474 mr->meta_ndescs = j; 2475 mr->meta_length = mr->ibmr.length - mr->data_length; 2476 } 2477 2478 return i + j; 2479 } 2480 2481 static int mlx5_set_page(struct ib_mr *ibmr, u64 addr) 2482 { 2483 struct mlx5_ib_mr *mr = to_mmr(ibmr); 2484 __be64 *descs; 2485 2486 if (unlikely(mr->mmkey.ndescs == mr->max_descs)) 2487 return -ENOMEM; 2488 2489 descs = mr->descs; 2490 descs[mr->mmkey.ndescs++] = cpu_to_be64(addr | MLX5_EN_RD | MLX5_EN_WR); 2491 2492 return 0; 2493 } 2494 2495 static int mlx5_set_page_pi(struct ib_mr *ibmr, u64 addr) 2496 { 2497 struct mlx5_ib_mr *mr = to_mmr(ibmr); 2498 __be64 *descs; 2499 2500 if (unlikely(mr->mmkey.ndescs + mr->meta_ndescs == mr->max_descs)) 2501 return -ENOMEM; 2502 2503 descs = mr->descs; 2504 descs[mr->mmkey.ndescs + mr->meta_ndescs++] = 2505 cpu_to_be64(addr | MLX5_EN_RD | MLX5_EN_WR); 2506 2507 return 0; 2508 } 2509 2510 static int 2511 mlx5_ib_map_mtt_mr_sg_pi(struct ib_mr *ibmr, struct scatterlist *data_sg, 2512 int data_sg_nents, unsigned int *data_sg_offset, 2513 struct scatterlist *meta_sg, int meta_sg_nents, 2514 unsigned int *meta_sg_offset) 2515 { 2516 struct mlx5_ib_mr *mr = to_mmr(ibmr); 2517 struct mlx5_ib_mr *pi_mr = mr->mtt_mr; 2518 int n; 2519 2520 pi_mr->mmkey.ndescs = 0; 2521 pi_mr->meta_ndescs = 0; 2522 pi_mr->meta_length = 0; 2523 2524 ib_dma_sync_single_for_cpu(ibmr->device, pi_mr->desc_map, 2525 pi_mr->desc_size * pi_mr->max_descs, 2526 DMA_TO_DEVICE); 2527 2528 pi_mr->ibmr.page_size = ibmr->page_size; 2529 n = ib_sg_to_pages(&pi_mr->ibmr, data_sg, data_sg_nents, data_sg_offset, 2530 mlx5_set_page); 2531 if (n != data_sg_nents) 2532 return n; 2533 2534 pi_mr->data_iova = pi_mr->ibmr.iova; 2535 pi_mr->data_length = pi_mr->ibmr.length; 2536 pi_mr->ibmr.length = pi_mr->data_length; 2537 ibmr->length = pi_mr->data_length; 2538 2539 if (meta_sg_nents) { 2540 u64 page_mask = ~((u64)ibmr->page_size - 1); 2541 u64 iova = pi_mr->data_iova; 2542 2543 n += ib_sg_to_pages(&pi_mr->ibmr, meta_sg, meta_sg_nents, 2544 meta_sg_offset, mlx5_set_page_pi); 2545 2546 pi_mr->meta_length = pi_mr->ibmr.length; 2547 /* 2548 * PI address for the HW is the offset of the metadata address 2549 * relative to the first data page address. 2550 * It equals to first data page address + size of data pages + 2551 * metadata offset at the first metadata page 2552 */ 2553 pi_mr->pi_iova = (iova & page_mask) + 2554 pi_mr->mmkey.ndescs * ibmr->page_size + 2555 (pi_mr->ibmr.iova & ~page_mask); 2556 /* 2557 * In order to use one MTT MR for data and metadata, we register 2558 * also the gaps between the end of the data and the start of 2559 * the metadata (the sig MR will verify that the HW will access 2560 * to right addresses). This mapping is safe because we use 2561 * internal mkey for the registration. 2562 */ 2563 pi_mr->ibmr.length = pi_mr->pi_iova + pi_mr->meta_length - iova; 2564 pi_mr->ibmr.iova = iova; 2565 ibmr->length += pi_mr->meta_length; 2566 } 2567 2568 ib_dma_sync_single_for_device(ibmr->device, pi_mr->desc_map, 2569 pi_mr->desc_size * pi_mr->max_descs, 2570 DMA_TO_DEVICE); 2571 2572 return n; 2573 } 2574 2575 static int 2576 mlx5_ib_map_klm_mr_sg_pi(struct ib_mr *ibmr, struct scatterlist *data_sg, 2577 int data_sg_nents, unsigned int *data_sg_offset, 2578 struct scatterlist *meta_sg, int meta_sg_nents, 2579 unsigned int *meta_sg_offset) 2580 { 2581 struct mlx5_ib_mr *mr = to_mmr(ibmr); 2582 struct mlx5_ib_mr *pi_mr = mr->klm_mr; 2583 int n; 2584 2585 pi_mr->mmkey.ndescs = 0; 2586 pi_mr->meta_ndescs = 0; 2587 pi_mr->meta_length = 0; 2588 2589 ib_dma_sync_single_for_cpu(ibmr->device, pi_mr->desc_map, 2590 pi_mr->desc_size * pi_mr->max_descs, 2591 DMA_TO_DEVICE); 2592 2593 n = mlx5_ib_sg_to_klms(pi_mr, data_sg, data_sg_nents, data_sg_offset, 2594 meta_sg, meta_sg_nents, meta_sg_offset); 2595 2596 ib_dma_sync_single_for_device(ibmr->device, pi_mr->desc_map, 2597 pi_mr->desc_size * pi_mr->max_descs, 2598 DMA_TO_DEVICE); 2599 2600 /* This is zero-based memory region */ 2601 pi_mr->data_iova = 0; 2602 pi_mr->ibmr.iova = 0; 2603 pi_mr->pi_iova = pi_mr->data_length; 2604 ibmr->length = pi_mr->ibmr.length; 2605 2606 return n; 2607 } 2608 2609 int mlx5_ib_map_mr_sg_pi(struct ib_mr *ibmr, struct scatterlist *data_sg, 2610 int data_sg_nents, unsigned int *data_sg_offset, 2611 struct scatterlist *meta_sg, int meta_sg_nents, 2612 unsigned int *meta_sg_offset) 2613 { 2614 struct mlx5_ib_mr *mr = to_mmr(ibmr); 2615 struct mlx5_ib_mr *pi_mr = NULL; 2616 int n; 2617 2618 WARN_ON(ibmr->type != IB_MR_TYPE_INTEGRITY); 2619 2620 mr->mmkey.ndescs = 0; 2621 mr->data_length = 0; 2622 mr->data_iova = 0; 2623 mr->meta_ndescs = 0; 2624 mr->pi_iova = 0; 2625 /* 2626 * As a performance optimization, if possible, there is no need to 2627 * perform UMR operation to register the data/metadata buffers. 2628 * First try to map the sg lists to PA descriptors with local_dma_lkey. 2629 * Fallback to UMR only in case of a failure. 2630 */ 2631 n = mlx5_ib_map_pa_mr_sg_pi(ibmr, data_sg, data_sg_nents, 2632 data_sg_offset, meta_sg, meta_sg_nents, 2633 meta_sg_offset); 2634 if (n == data_sg_nents + meta_sg_nents) 2635 goto out; 2636 /* 2637 * As a performance optimization, if possible, there is no need to map 2638 * the sg lists to KLM descriptors. First try to map the sg lists to MTT 2639 * descriptors and fallback to KLM only in case of a failure. 2640 * It's more efficient for the HW to work with MTT descriptors 2641 * (especially in high load). 2642 * Use KLM (indirect access) only if it's mandatory. 2643 */ 2644 pi_mr = mr->mtt_mr; 2645 n = mlx5_ib_map_mtt_mr_sg_pi(ibmr, data_sg, data_sg_nents, 2646 data_sg_offset, meta_sg, meta_sg_nents, 2647 meta_sg_offset); 2648 if (n == data_sg_nents + meta_sg_nents) 2649 goto out; 2650 2651 pi_mr = mr->klm_mr; 2652 n = mlx5_ib_map_klm_mr_sg_pi(ibmr, data_sg, data_sg_nents, 2653 data_sg_offset, meta_sg, meta_sg_nents, 2654 meta_sg_offset); 2655 if (unlikely(n != data_sg_nents + meta_sg_nents)) 2656 return -ENOMEM; 2657 2658 out: 2659 /* This is zero-based memory region */ 2660 ibmr->iova = 0; 2661 mr->pi_mr = pi_mr; 2662 if (pi_mr) 2663 ibmr->sig_attrs->meta_length = pi_mr->meta_length; 2664 else 2665 ibmr->sig_attrs->meta_length = mr->meta_length; 2666 2667 return 0; 2668 } 2669 2670 int mlx5_ib_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg, int sg_nents, 2671 unsigned int *sg_offset) 2672 { 2673 struct mlx5_ib_mr *mr = to_mmr(ibmr); 2674 int n; 2675 2676 mr->mmkey.ndescs = 0; 2677 2678 ib_dma_sync_single_for_cpu(ibmr->device, mr->desc_map, 2679 mr->desc_size * mr->max_descs, 2680 DMA_TO_DEVICE); 2681 2682 if (mr->access_mode == MLX5_MKC_ACCESS_MODE_KLMS) 2683 n = mlx5_ib_sg_to_klms(mr, sg, sg_nents, sg_offset, NULL, 0, 2684 NULL); 2685 else 2686 n = ib_sg_to_pages(ibmr, sg, sg_nents, sg_offset, 2687 mlx5_set_page); 2688 2689 ib_dma_sync_single_for_device(ibmr->device, mr->desc_map, 2690 mr->desc_size * mr->max_descs, 2691 DMA_TO_DEVICE); 2692 2693 return n; 2694 } 2695