1 /* 2 * Copyright (c) 2013-2015, Mellanox Technologies. All rights reserved. 3 * 4 * This software is available to you under a choice of one of two 5 * licenses. You may choose to be licensed under the terms of the GNU 6 * General Public License (GPL) Version 2, available from the file 7 * COPYING in the main directory of this source tree, or the 8 * OpenIB.org BSD license below: 9 * 10 * Redistribution and use in source and binary forms, with or 11 * without modification, are permitted provided that the following 12 * conditions are met: 13 * 14 * - Redistributions of source code must retain the above 15 * copyright notice, this list of conditions and the following 16 * disclaimer. 17 * 18 * - Redistributions in binary form must reproduce the above 19 * copyright notice, this list of conditions and the following 20 * disclaimer in the documentation and/or other materials 21 * provided with the distribution. 22 * 23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30 * SOFTWARE. 31 */ 32 33 34 #include <linux/kref.h> 35 #include <linux/random.h> 36 #include <linux/debugfs.h> 37 #include <linux/export.h> 38 #include <linux/delay.h> 39 #include <rdma/ib_umem.h> 40 #include <rdma/ib_umem_odp.h> 41 #include <rdma/ib_verbs.h> 42 #include "mlx5_ib.h" 43 44 enum { 45 MAX_PENDING_REG_MR = 8, 46 }; 47 48 #define MLX5_UMR_ALIGN 2048 49 50 static void 51 create_mkey_callback(int status, struct mlx5_async_work *context); 52 53 static void set_mkc_access_pd_addr_fields(void *mkc, int acc, u64 start_addr, 54 struct ib_pd *pd) 55 { 56 struct mlx5_ib_dev *dev = to_mdev(pd->device); 57 58 MLX5_SET(mkc, mkc, a, !!(acc & IB_ACCESS_REMOTE_ATOMIC)); 59 MLX5_SET(mkc, mkc, rw, !!(acc & IB_ACCESS_REMOTE_WRITE)); 60 MLX5_SET(mkc, mkc, rr, !!(acc & IB_ACCESS_REMOTE_READ)); 61 MLX5_SET(mkc, mkc, lw, !!(acc & IB_ACCESS_LOCAL_WRITE)); 62 MLX5_SET(mkc, mkc, lr, 1); 63 64 if (MLX5_CAP_GEN(dev->mdev, relaxed_ordering_write)) 65 MLX5_SET(mkc, mkc, relaxed_ordering_write, 66 !!(acc & IB_ACCESS_RELAXED_ORDERING)); 67 if (MLX5_CAP_GEN(dev->mdev, relaxed_ordering_read)) 68 MLX5_SET(mkc, mkc, relaxed_ordering_read, 69 !!(acc & IB_ACCESS_RELAXED_ORDERING)); 70 71 MLX5_SET(mkc, mkc, pd, to_mpd(pd)->pdn); 72 MLX5_SET(mkc, mkc, qpn, 0xffffff); 73 MLX5_SET64(mkc, mkc, start_addr, start_addr); 74 } 75 76 static void 77 assign_mkey_variant(struct mlx5_ib_dev *dev, struct mlx5_core_mkey *mkey, 78 u32 *in) 79 { 80 u8 key = atomic_inc_return(&dev->mkey_var); 81 void *mkc; 82 83 mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry); 84 MLX5_SET(mkc, mkc, mkey_7_0, key); 85 mkey->key = key; 86 } 87 88 static int 89 mlx5_ib_create_mkey(struct mlx5_ib_dev *dev, struct mlx5_core_mkey *mkey, 90 u32 *in, int inlen) 91 { 92 assign_mkey_variant(dev, mkey, in); 93 return mlx5_core_create_mkey(dev->mdev, mkey, in, inlen); 94 } 95 96 static int 97 mlx5_ib_create_mkey_cb(struct mlx5_ib_dev *dev, 98 struct mlx5_core_mkey *mkey, 99 struct mlx5_async_ctx *async_ctx, 100 u32 *in, int inlen, u32 *out, int outlen, 101 struct mlx5_async_work *context) 102 { 103 MLX5_SET(create_mkey_in, in, opcode, MLX5_CMD_OP_CREATE_MKEY); 104 assign_mkey_variant(dev, mkey, in); 105 return mlx5_cmd_exec_cb(async_ctx, in, inlen, out, outlen, 106 create_mkey_callback, context); 107 } 108 109 static void clean_mr(struct mlx5_ib_dev *dev, struct mlx5_ib_mr *mr); 110 static void dereg_mr(struct mlx5_ib_dev *dev, struct mlx5_ib_mr *mr); 111 static int mr_cache_max_order(struct mlx5_ib_dev *dev); 112 static void queue_adjust_cache_locked(struct mlx5_cache_ent *ent); 113 114 static bool umr_can_use_indirect_mkey(struct mlx5_ib_dev *dev) 115 { 116 return !MLX5_CAP_GEN(dev->mdev, umr_indirect_mkey_disabled); 117 } 118 119 static int destroy_mkey(struct mlx5_ib_dev *dev, struct mlx5_ib_mr *mr) 120 { 121 WARN_ON(xa_load(&dev->odp_mkeys, mlx5_base_mkey(mr->mmkey.key))); 122 123 return mlx5_core_destroy_mkey(dev->mdev, &mr->mmkey); 124 } 125 126 static inline bool mlx5_ib_pas_fits_in_mr(struct mlx5_ib_mr *mr, u64 start, 127 u64 length) 128 { 129 return ((u64)1 << mr->order) * MLX5_ADAPTER_PAGE_SIZE >= 130 length + (start & (MLX5_ADAPTER_PAGE_SIZE - 1)); 131 } 132 133 static void create_mkey_callback(int status, struct mlx5_async_work *context) 134 { 135 struct mlx5_ib_mr *mr = 136 container_of(context, struct mlx5_ib_mr, cb_work); 137 struct mlx5_ib_dev *dev = mr->dev; 138 struct mlx5_cache_ent *ent = mr->cache_ent; 139 unsigned long flags; 140 141 if (status) { 142 mlx5_ib_warn(dev, "async reg mr failed. status %d\n", status); 143 kfree(mr); 144 spin_lock_irqsave(&ent->lock, flags); 145 ent->pending--; 146 WRITE_ONCE(dev->fill_delay, 1); 147 spin_unlock_irqrestore(&ent->lock, flags); 148 mod_timer(&dev->delay_timer, jiffies + HZ); 149 return; 150 } 151 152 mr->mmkey.type = MLX5_MKEY_MR; 153 mr->mmkey.key |= mlx5_idx_to_mkey( 154 MLX5_GET(create_mkey_out, mr->out, mkey_index)); 155 156 WRITE_ONCE(dev->cache.last_add, jiffies); 157 158 spin_lock_irqsave(&ent->lock, flags); 159 list_add_tail(&mr->list, &ent->head); 160 ent->available_mrs++; 161 ent->total_mrs++; 162 /* If we are doing fill_to_high_water then keep going. */ 163 queue_adjust_cache_locked(ent); 164 ent->pending--; 165 spin_unlock_irqrestore(&ent->lock, flags); 166 } 167 168 static struct mlx5_ib_mr *alloc_cache_mr(struct mlx5_cache_ent *ent, void *mkc) 169 { 170 struct mlx5_ib_mr *mr; 171 172 mr = kzalloc(sizeof(*mr), GFP_KERNEL); 173 if (!mr) 174 return NULL; 175 mr->order = ent->order; 176 mr->cache_ent = ent; 177 mr->dev = ent->dev; 178 179 set_mkc_access_pd_addr_fields(mkc, 0, 0, ent->dev->umrc.pd); 180 MLX5_SET(mkc, mkc, free, 1); 181 MLX5_SET(mkc, mkc, umr_en, 1); 182 MLX5_SET(mkc, mkc, access_mode_1_0, ent->access_mode & 0x3); 183 MLX5_SET(mkc, mkc, access_mode_4_2, (ent->access_mode >> 2) & 0x7); 184 185 MLX5_SET(mkc, mkc, translations_octword_size, ent->xlt); 186 MLX5_SET(mkc, mkc, log_page_size, ent->page); 187 return mr; 188 } 189 190 /* Asynchronously schedule new MRs to be populated in the cache. */ 191 static int add_keys(struct mlx5_cache_ent *ent, unsigned int num) 192 { 193 size_t inlen = MLX5_ST_SZ_BYTES(create_mkey_in); 194 struct mlx5_ib_mr *mr; 195 void *mkc; 196 u32 *in; 197 int err = 0; 198 int i; 199 200 in = kzalloc(inlen, GFP_KERNEL); 201 if (!in) 202 return -ENOMEM; 203 204 mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry); 205 for (i = 0; i < num; i++) { 206 mr = alloc_cache_mr(ent, mkc); 207 if (!mr) { 208 err = -ENOMEM; 209 break; 210 } 211 spin_lock_irq(&ent->lock); 212 if (ent->pending >= MAX_PENDING_REG_MR) { 213 err = -EAGAIN; 214 spin_unlock_irq(&ent->lock); 215 kfree(mr); 216 break; 217 } 218 ent->pending++; 219 spin_unlock_irq(&ent->lock); 220 err = mlx5_ib_create_mkey_cb(ent->dev, &mr->mmkey, 221 &ent->dev->async_ctx, in, inlen, 222 mr->out, sizeof(mr->out), 223 &mr->cb_work); 224 if (err) { 225 spin_lock_irq(&ent->lock); 226 ent->pending--; 227 spin_unlock_irq(&ent->lock); 228 mlx5_ib_warn(ent->dev, "create mkey failed %d\n", err); 229 kfree(mr); 230 break; 231 } 232 } 233 234 kfree(in); 235 return err; 236 } 237 238 /* Synchronously create a MR in the cache */ 239 static struct mlx5_ib_mr *create_cache_mr(struct mlx5_cache_ent *ent) 240 { 241 size_t inlen = MLX5_ST_SZ_BYTES(create_mkey_in); 242 struct mlx5_ib_mr *mr; 243 void *mkc; 244 u32 *in; 245 int err; 246 247 in = kzalloc(inlen, GFP_KERNEL); 248 if (!in) 249 return ERR_PTR(-ENOMEM); 250 mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry); 251 252 mr = alloc_cache_mr(ent, mkc); 253 if (!mr) { 254 err = -ENOMEM; 255 goto free_in; 256 } 257 258 err = mlx5_core_create_mkey(ent->dev->mdev, &mr->mmkey, in, inlen); 259 if (err) 260 goto free_mr; 261 262 mr->mmkey.type = MLX5_MKEY_MR; 263 WRITE_ONCE(ent->dev->cache.last_add, jiffies); 264 spin_lock_irq(&ent->lock); 265 ent->total_mrs++; 266 spin_unlock_irq(&ent->lock); 267 kfree(in); 268 return mr; 269 free_mr: 270 kfree(mr); 271 free_in: 272 kfree(in); 273 return ERR_PTR(err); 274 } 275 276 static void remove_cache_mr_locked(struct mlx5_cache_ent *ent) 277 { 278 struct mlx5_ib_mr *mr; 279 280 lockdep_assert_held(&ent->lock); 281 if (list_empty(&ent->head)) 282 return; 283 mr = list_first_entry(&ent->head, struct mlx5_ib_mr, list); 284 list_del(&mr->list); 285 ent->available_mrs--; 286 ent->total_mrs--; 287 spin_unlock_irq(&ent->lock); 288 mlx5_core_destroy_mkey(ent->dev->mdev, &mr->mmkey); 289 kfree(mr); 290 spin_lock_irq(&ent->lock); 291 } 292 293 static int resize_available_mrs(struct mlx5_cache_ent *ent, unsigned int target, 294 bool limit_fill) 295 { 296 int err; 297 298 lockdep_assert_held(&ent->lock); 299 300 while (true) { 301 if (limit_fill) 302 target = ent->limit * 2; 303 if (target == ent->available_mrs + ent->pending) 304 return 0; 305 if (target > ent->available_mrs + ent->pending) { 306 u32 todo = target - (ent->available_mrs + ent->pending); 307 308 spin_unlock_irq(&ent->lock); 309 err = add_keys(ent, todo); 310 if (err == -EAGAIN) 311 usleep_range(3000, 5000); 312 spin_lock_irq(&ent->lock); 313 if (err) { 314 if (err != -EAGAIN) 315 return err; 316 } else 317 return 0; 318 } else { 319 remove_cache_mr_locked(ent); 320 } 321 } 322 } 323 324 static ssize_t size_write(struct file *filp, const char __user *buf, 325 size_t count, loff_t *pos) 326 { 327 struct mlx5_cache_ent *ent = filp->private_data; 328 u32 target; 329 int err; 330 331 err = kstrtou32_from_user(buf, count, 0, &target); 332 if (err) 333 return err; 334 335 /* 336 * Target is the new value of total_mrs the user requests, however we 337 * cannot free MRs that are in use. Compute the target value for 338 * available_mrs. 339 */ 340 spin_lock_irq(&ent->lock); 341 if (target < ent->total_mrs - ent->available_mrs) { 342 err = -EINVAL; 343 goto err_unlock; 344 } 345 target = target - (ent->total_mrs - ent->available_mrs); 346 if (target < ent->limit || target > ent->limit*2) { 347 err = -EINVAL; 348 goto err_unlock; 349 } 350 err = resize_available_mrs(ent, target, false); 351 if (err) 352 goto err_unlock; 353 spin_unlock_irq(&ent->lock); 354 355 return count; 356 357 err_unlock: 358 spin_unlock_irq(&ent->lock); 359 return err; 360 } 361 362 static ssize_t size_read(struct file *filp, char __user *buf, size_t count, 363 loff_t *pos) 364 { 365 struct mlx5_cache_ent *ent = filp->private_data; 366 char lbuf[20]; 367 int err; 368 369 err = snprintf(lbuf, sizeof(lbuf), "%d\n", ent->total_mrs); 370 if (err < 0) 371 return err; 372 373 return simple_read_from_buffer(buf, count, pos, lbuf, err); 374 } 375 376 static const struct file_operations size_fops = { 377 .owner = THIS_MODULE, 378 .open = simple_open, 379 .write = size_write, 380 .read = size_read, 381 }; 382 383 static ssize_t limit_write(struct file *filp, const char __user *buf, 384 size_t count, loff_t *pos) 385 { 386 struct mlx5_cache_ent *ent = filp->private_data; 387 u32 var; 388 int err; 389 390 err = kstrtou32_from_user(buf, count, 0, &var); 391 if (err) 392 return err; 393 394 /* 395 * Upon set we immediately fill the cache to high water mark implied by 396 * the limit. 397 */ 398 spin_lock_irq(&ent->lock); 399 ent->limit = var; 400 err = resize_available_mrs(ent, 0, true); 401 spin_unlock_irq(&ent->lock); 402 if (err) 403 return err; 404 return count; 405 } 406 407 static ssize_t limit_read(struct file *filp, char __user *buf, size_t count, 408 loff_t *pos) 409 { 410 struct mlx5_cache_ent *ent = filp->private_data; 411 char lbuf[20]; 412 int err; 413 414 err = snprintf(lbuf, sizeof(lbuf), "%d\n", ent->limit); 415 if (err < 0) 416 return err; 417 418 return simple_read_from_buffer(buf, count, pos, lbuf, err); 419 } 420 421 static const struct file_operations limit_fops = { 422 .owner = THIS_MODULE, 423 .open = simple_open, 424 .write = limit_write, 425 .read = limit_read, 426 }; 427 428 static bool someone_adding(struct mlx5_mr_cache *cache) 429 { 430 unsigned int i; 431 432 for (i = 0; i < MAX_MR_CACHE_ENTRIES; i++) { 433 struct mlx5_cache_ent *ent = &cache->ent[i]; 434 bool ret; 435 436 spin_lock_irq(&ent->lock); 437 ret = ent->available_mrs < ent->limit; 438 spin_unlock_irq(&ent->lock); 439 if (ret) 440 return true; 441 } 442 return false; 443 } 444 445 /* 446 * Check if the bucket is outside the high/low water mark and schedule an async 447 * update. The cache refill has hysteresis, once the low water mark is hit it is 448 * refilled up to the high mark. 449 */ 450 static void queue_adjust_cache_locked(struct mlx5_cache_ent *ent) 451 { 452 lockdep_assert_held(&ent->lock); 453 454 if (ent->disabled || READ_ONCE(ent->dev->fill_delay)) 455 return; 456 if (ent->available_mrs < ent->limit) { 457 ent->fill_to_high_water = true; 458 queue_work(ent->dev->cache.wq, &ent->work); 459 } else if (ent->fill_to_high_water && 460 ent->available_mrs + ent->pending < 2 * ent->limit) { 461 /* 462 * Once we start populating due to hitting a low water mark 463 * continue until we pass the high water mark. 464 */ 465 queue_work(ent->dev->cache.wq, &ent->work); 466 } else if (ent->available_mrs == 2 * ent->limit) { 467 ent->fill_to_high_water = false; 468 } else if (ent->available_mrs > 2 * ent->limit) { 469 /* Queue deletion of excess entries */ 470 ent->fill_to_high_water = false; 471 if (ent->pending) 472 queue_delayed_work(ent->dev->cache.wq, &ent->dwork, 473 msecs_to_jiffies(1000)); 474 else 475 queue_work(ent->dev->cache.wq, &ent->work); 476 } 477 } 478 479 static void __cache_work_func(struct mlx5_cache_ent *ent) 480 { 481 struct mlx5_ib_dev *dev = ent->dev; 482 struct mlx5_mr_cache *cache = &dev->cache; 483 int err; 484 485 spin_lock_irq(&ent->lock); 486 if (ent->disabled) 487 goto out; 488 489 if (ent->fill_to_high_water && 490 ent->available_mrs + ent->pending < 2 * ent->limit && 491 !READ_ONCE(dev->fill_delay)) { 492 spin_unlock_irq(&ent->lock); 493 err = add_keys(ent, 1); 494 spin_lock_irq(&ent->lock); 495 if (ent->disabled) 496 goto out; 497 if (err) { 498 /* 499 * EAGAIN only happens if pending is positive, so we 500 * will be rescheduled from reg_mr_callback(). The only 501 * failure path here is ENOMEM. 502 */ 503 if (err != -EAGAIN) { 504 mlx5_ib_warn( 505 dev, 506 "command failed order %d, err %d\n", 507 ent->order, err); 508 queue_delayed_work(cache->wq, &ent->dwork, 509 msecs_to_jiffies(1000)); 510 } 511 } 512 } else if (ent->available_mrs > 2 * ent->limit) { 513 bool need_delay; 514 515 /* 516 * The remove_cache_mr() logic is performed as garbage 517 * collection task. Such task is intended to be run when no 518 * other active processes are running. 519 * 520 * The need_resched() will return TRUE if there are user tasks 521 * to be activated in near future. 522 * 523 * In such case, we don't execute remove_cache_mr() and postpone 524 * the garbage collection work to try to run in next cycle, in 525 * order to free CPU resources to other tasks. 526 */ 527 spin_unlock_irq(&ent->lock); 528 need_delay = need_resched() || someone_adding(cache) || 529 time_after(jiffies, 530 READ_ONCE(cache->last_add) + 300 * HZ); 531 spin_lock_irq(&ent->lock); 532 if (ent->disabled) 533 goto out; 534 if (need_delay) 535 queue_delayed_work(cache->wq, &ent->dwork, 300 * HZ); 536 remove_cache_mr_locked(ent); 537 queue_adjust_cache_locked(ent); 538 } 539 out: 540 spin_unlock_irq(&ent->lock); 541 } 542 543 static void delayed_cache_work_func(struct work_struct *work) 544 { 545 struct mlx5_cache_ent *ent; 546 547 ent = container_of(work, struct mlx5_cache_ent, dwork.work); 548 __cache_work_func(ent); 549 } 550 551 static void cache_work_func(struct work_struct *work) 552 { 553 struct mlx5_cache_ent *ent; 554 555 ent = container_of(work, struct mlx5_cache_ent, work); 556 __cache_work_func(ent); 557 } 558 559 /* Allocate a special entry from the cache */ 560 struct mlx5_ib_mr *mlx5_mr_cache_alloc(struct mlx5_ib_dev *dev, 561 unsigned int entry, int access_flags) 562 { 563 struct mlx5_mr_cache *cache = &dev->cache; 564 struct mlx5_cache_ent *ent; 565 struct mlx5_ib_mr *mr; 566 567 if (WARN_ON(entry <= MR_CACHE_LAST_STD_ENTRY || 568 entry >= ARRAY_SIZE(cache->ent))) 569 return ERR_PTR(-EINVAL); 570 571 /* Matches access in alloc_cache_mr() */ 572 if (!mlx5_ib_can_reconfig_with_umr(dev, 0, access_flags)) 573 return ERR_PTR(-EOPNOTSUPP); 574 575 ent = &cache->ent[entry]; 576 spin_lock_irq(&ent->lock); 577 if (list_empty(&ent->head)) { 578 spin_unlock_irq(&ent->lock); 579 mr = create_cache_mr(ent); 580 if (IS_ERR(mr)) 581 return mr; 582 } else { 583 mr = list_first_entry(&ent->head, struct mlx5_ib_mr, list); 584 list_del(&mr->list); 585 ent->available_mrs--; 586 queue_adjust_cache_locked(ent); 587 spin_unlock_irq(&ent->lock); 588 } 589 mr->access_flags = access_flags; 590 return mr; 591 } 592 593 /* Return a MR already available in the cache */ 594 static struct mlx5_ib_mr *get_cache_mr(struct mlx5_cache_ent *req_ent) 595 { 596 struct mlx5_ib_dev *dev = req_ent->dev; 597 struct mlx5_ib_mr *mr = NULL; 598 struct mlx5_cache_ent *ent = req_ent; 599 600 /* Try larger MR pools from the cache to satisfy the allocation */ 601 for (; ent != &dev->cache.ent[MR_CACHE_LAST_STD_ENTRY + 1]; ent++) { 602 mlx5_ib_dbg(dev, "order %u, cache index %zu\n", ent->order, 603 ent - dev->cache.ent); 604 605 spin_lock_irq(&ent->lock); 606 if (!list_empty(&ent->head)) { 607 mr = list_first_entry(&ent->head, struct mlx5_ib_mr, 608 list); 609 list_del(&mr->list); 610 ent->available_mrs--; 611 queue_adjust_cache_locked(ent); 612 spin_unlock_irq(&ent->lock); 613 break; 614 } 615 queue_adjust_cache_locked(ent); 616 spin_unlock_irq(&ent->lock); 617 } 618 619 if (!mr) 620 req_ent->miss++; 621 622 return mr; 623 } 624 625 static void detach_mr_from_cache(struct mlx5_ib_mr *mr) 626 { 627 struct mlx5_cache_ent *ent = mr->cache_ent; 628 629 mr->cache_ent = NULL; 630 spin_lock_irq(&ent->lock); 631 ent->total_mrs--; 632 spin_unlock_irq(&ent->lock); 633 } 634 635 void mlx5_mr_cache_free(struct mlx5_ib_dev *dev, struct mlx5_ib_mr *mr) 636 { 637 struct mlx5_cache_ent *ent = mr->cache_ent; 638 639 if (!ent) 640 return; 641 642 if (mlx5_mr_cache_invalidate(mr)) { 643 detach_mr_from_cache(mr); 644 destroy_mkey(dev, mr); 645 return; 646 } 647 648 spin_lock_irq(&ent->lock); 649 list_add_tail(&mr->list, &ent->head); 650 ent->available_mrs++; 651 queue_adjust_cache_locked(ent); 652 spin_unlock_irq(&ent->lock); 653 } 654 655 static void clean_keys(struct mlx5_ib_dev *dev, int c) 656 { 657 struct mlx5_mr_cache *cache = &dev->cache; 658 struct mlx5_cache_ent *ent = &cache->ent[c]; 659 struct mlx5_ib_mr *tmp_mr; 660 struct mlx5_ib_mr *mr; 661 LIST_HEAD(del_list); 662 663 cancel_delayed_work(&ent->dwork); 664 while (1) { 665 spin_lock_irq(&ent->lock); 666 if (list_empty(&ent->head)) { 667 spin_unlock_irq(&ent->lock); 668 break; 669 } 670 mr = list_first_entry(&ent->head, struct mlx5_ib_mr, list); 671 list_move(&mr->list, &del_list); 672 ent->available_mrs--; 673 ent->total_mrs--; 674 spin_unlock_irq(&ent->lock); 675 mlx5_core_destroy_mkey(dev->mdev, &mr->mmkey); 676 } 677 678 list_for_each_entry_safe(mr, tmp_mr, &del_list, list) { 679 list_del(&mr->list); 680 kfree(mr); 681 } 682 } 683 684 static void mlx5_mr_cache_debugfs_cleanup(struct mlx5_ib_dev *dev) 685 { 686 if (!mlx5_debugfs_root || dev->is_rep) 687 return; 688 689 debugfs_remove_recursive(dev->cache.root); 690 dev->cache.root = NULL; 691 } 692 693 static void mlx5_mr_cache_debugfs_init(struct mlx5_ib_dev *dev) 694 { 695 struct mlx5_mr_cache *cache = &dev->cache; 696 struct mlx5_cache_ent *ent; 697 struct dentry *dir; 698 int i; 699 700 if (!mlx5_debugfs_root || dev->is_rep) 701 return; 702 703 cache->root = debugfs_create_dir("mr_cache", dev->mdev->priv.dbg_root); 704 705 for (i = 0; i < MAX_MR_CACHE_ENTRIES; i++) { 706 ent = &cache->ent[i]; 707 sprintf(ent->name, "%d", ent->order); 708 dir = debugfs_create_dir(ent->name, cache->root); 709 debugfs_create_file("size", 0600, dir, ent, &size_fops); 710 debugfs_create_file("limit", 0600, dir, ent, &limit_fops); 711 debugfs_create_u32("cur", 0400, dir, &ent->available_mrs); 712 debugfs_create_u32("miss", 0600, dir, &ent->miss); 713 } 714 } 715 716 static void delay_time_func(struct timer_list *t) 717 { 718 struct mlx5_ib_dev *dev = from_timer(dev, t, delay_timer); 719 720 WRITE_ONCE(dev->fill_delay, 0); 721 } 722 723 int mlx5_mr_cache_init(struct mlx5_ib_dev *dev) 724 { 725 struct mlx5_mr_cache *cache = &dev->cache; 726 struct mlx5_cache_ent *ent; 727 int i; 728 729 mutex_init(&dev->slow_path_mutex); 730 cache->wq = alloc_ordered_workqueue("mkey_cache", WQ_MEM_RECLAIM); 731 if (!cache->wq) { 732 mlx5_ib_warn(dev, "failed to create work queue\n"); 733 return -ENOMEM; 734 } 735 736 mlx5_cmd_init_async_ctx(dev->mdev, &dev->async_ctx); 737 timer_setup(&dev->delay_timer, delay_time_func, 0); 738 for (i = 0; i < MAX_MR_CACHE_ENTRIES; i++) { 739 ent = &cache->ent[i]; 740 INIT_LIST_HEAD(&ent->head); 741 spin_lock_init(&ent->lock); 742 ent->order = i + 2; 743 ent->dev = dev; 744 ent->limit = 0; 745 746 INIT_WORK(&ent->work, cache_work_func); 747 INIT_DELAYED_WORK(&ent->dwork, delayed_cache_work_func); 748 749 if (i > MR_CACHE_LAST_STD_ENTRY) { 750 mlx5_odp_init_mr_cache_entry(ent); 751 continue; 752 } 753 754 if (ent->order > mr_cache_max_order(dev)) 755 continue; 756 757 ent->page = PAGE_SHIFT; 758 ent->xlt = (1 << ent->order) * sizeof(struct mlx5_mtt) / 759 MLX5_IB_UMR_OCTOWORD; 760 ent->access_mode = MLX5_MKC_ACCESS_MODE_MTT; 761 if ((dev->mdev->profile->mask & MLX5_PROF_MASK_MR_CACHE) && 762 !dev->is_rep && mlx5_core_is_pf(dev->mdev) && 763 mlx5_ib_can_load_pas_with_umr(dev, 0)) 764 ent->limit = dev->mdev->profile->mr_cache[i].limit; 765 else 766 ent->limit = 0; 767 spin_lock_irq(&ent->lock); 768 queue_adjust_cache_locked(ent); 769 spin_unlock_irq(&ent->lock); 770 } 771 772 mlx5_mr_cache_debugfs_init(dev); 773 774 return 0; 775 } 776 777 int mlx5_mr_cache_cleanup(struct mlx5_ib_dev *dev) 778 { 779 unsigned int i; 780 781 if (!dev->cache.wq) 782 return 0; 783 784 for (i = 0; i < MAX_MR_CACHE_ENTRIES; i++) { 785 struct mlx5_cache_ent *ent = &dev->cache.ent[i]; 786 787 spin_lock_irq(&ent->lock); 788 ent->disabled = true; 789 spin_unlock_irq(&ent->lock); 790 cancel_work_sync(&ent->work); 791 cancel_delayed_work_sync(&ent->dwork); 792 } 793 794 mlx5_mr_cache_debugfs_cleanup(dev); 795 mlx5_cmd_cleanup_async_ctx(&dev->async_ctx); 796 797 for (i = 0; i < MAX_MR_CACHE_ENTRIES; i++) 798 clean_keys(dev, i); 799 800 destroy_workqueue(dev->cache.wq); 801 del_timer_sync(&dev->delay_timer); 802 803 return 0; 804 } 805 806 struct ib_mr *mlx5_ib_get_dma_mr(struct ib_pd *pd, int acc) 807 { 808 struct mlx5_ib_dev *dev = to_mdev(pd->device); 809 int inlen = MLX5_ST_SZ_BYTES(create_mkey_in); 810 struct mlx5_ib_mr *mr; 811 void *mkc; 812 u32 *in; 813 int err; 814 815 mr = kzalloc(sizeof(*mr), GFP_KERNEL); 816 if (!mr) 817 return ERR_PTR(-ENOMEM); 818 819 in = kzalloc(inlen, GFP_KERNEL); 820 if (!in) { 821 err = -ENOMEM; 822 goto err_free; 823 } 824 825 mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry); 826 827 MLX5_SET(mkc, mkc, access_mode_1_0, MLX5_MKC_ACCESS_MODE_PA); 828 MLX5_SET(mkc, mkc, length64, 1); 829 set_mkc_access_pd_addr_fields(mkc, acc, 0, pd); 830 831 err = mlx5_ib_create_mkey(dev, &mr->mmkey, in, inlen); 832 if (err) 833 goto err_in; 834 835 kfree(in); 836 mr->mmkey.type = MLX5_MKEY_MR; 837 mr->ibmr.lkey = mr->mmkey.key; 838 mr->ibmr.rkey = mr->mmkey.key; 839 mr->umem = NULL; 840 841 return &mr->ibmr; 842 843 err_in: 844 kfree(in); 845 846 err_free: 847 kfree(mr); 848 849 return ERR_PTR(err); 850 } 851 852 static int get_octo_len(u64 addr, u64 len, int page_shift) 853 { 854 u64 page_size = 1ULL << page_shift; 855 u64 offset; 856 int npages; 857 858 offset = addr & (page_size - 1); 859 npages = ALIGN(len + offset, page_size) >> page_shift; 860 return (npages + 1) / 2; 861 } 862 863 static int mr_cache_max_order(struct mlx5_ib_dev *dev) 864 { 865 if (MLX5_CAP_GEN(dev->mdev, umr_extended_translation_offset)) 866 return MR_CACHE_LAST_STD_ENTRY + 2; 867 return MLX5_MAX_UMR_SHIFT; 868 } 869 870 static int mr_umem_get(struct mlx5_ib_dev *dev, u64 start, u64 length, 871 int access_flags, struct ib_umem **umem, int *npages, 872 int *page_shift, int *ncont, int *order) 873 { 874 struct ib_umem *u; 875 876 *umem = NULL; 877 878 if (access_flags & IB_ACCESS_ON_DEMAND) { 879 struct ib_umem_odp *odp; 880 881 odp = ib_umem_odp_get(&dev->ib_dev, start, length, access_flags, 882 &mlx5_mn_ops); 883 if (IS_ERR(odp)) { 884 mlx5_ib_dbg(dev, "umem get failed (%ld)\n", 885 PTR_ERR(odp)); 886 return PTR_ERR(odp); 887 } 888 889 u = &odp->umem; 890 891 *page_shift = odp->page_shift; 892 *ncont = ib_umem_odp_num_pages(odp); 893 *npages = *ncont << (*page_shift - PAGE_SHIFT); 894 if (order) 895 *order = ilog2(roundup_pow_of_two(*ncont)); 896 } else { 897 u = ib_umem_get(&dev->ib_dev, start, length, access_flags); 898 if (IS_ERR(u)) { 899 mlx5_ib_dbg(dev, "umem get failed (%ld)\n", PTR_ERR(u)); 900 return PTR_ERR(u); 901 } 902 903 mlx5_ib_cont_pages(u, start, MLX5_MKEY_PAGE_SHIFT_MASK, npages, 904 page_shift, ncont, order); 905 } 906 907 if (!*npages) { 908 mlx5_ib_warn(dev, "avoid zero region\n"); 909 ib_umem_release(u); 910 return -EINVAL; 911 } 912 913 *umem = u; 914 915 mlx5_ib_dbg(dev, "npages %d, ncont %d, order %d, page_shift %d\n", 916 *npages, *ncont, *order, *page_shift); 917 918 return 0; 919 } 920 921 static void mlx5_ib_umr_done(struct ib_cq *cq, struct ib_wc *wc) 922 { 923 struct mlx5_ib_umr_context *context = 924 container_of(wc->wr_cqe, struct mlx5_ib_umr_context, cqe); 925 926 context->status = wc->status; 927 complete(&context->done); 928 } 929 930 static inline void mlx5_ib_init_umr_context(struct mlx5_ib_umr_context *context) 931 { 932 context->cqe.done = mlx5_ib_umr_done; 933 context->status = -1; 934 init_completion(&context->done); 935 } 936 937 static int mlx5_ib_post_send_wait(struct mlx5_ib_dev *dev, 938 struct mlx5_umr_wr *umrwr) 939 { 940 struct umr_common *umrc = &dev->umrc; 941 const struct ib_send_wr *bad; 942 int err; 943 struct mlx5_ib_umr_context umr_context; 944 945 mlx5_ib_init_umr_context(&umr_context); 946 umrwr->wr.wr_cqe = &umr_context.cqe; 947 948 down(&umrc->sem); 949 err = ib_post_send(umrc->qp, &umrwr->wr, &bad); 950 if (err) { 951 mlx5_ib_warn(dev, "UMR post send failed, err %d\n", err); 952 } else { 953 wait_for_completion(&umr_context.done); 954 if (umr_context.status != IB_WC_SUCCESS) { 955 mlx5_ib_warn(dev, "reg umr failed (%u)\n", 956 umr_context.status); 957 err = -EFAULT; 958 } 959 } 960 up(&umrc->sem); 961 return err; 962 } 963 964 static struct mlx5_cache_ent *mr_cache_ent_from_order(struct mlx5_ib_dev *dev, 965 unsigned int order) 966 { 967 struct mlx5_mr_cache *cache = &dev->cache; 968 969 if (order < cache->ent[0].order) 970 return &cache->ent[0]; 971 order = order - cache->ent[0].order; 972 if (order > MR_CACHE_LAST_STD_ENTRY) 973 return NULL; 974 return &cache->ent[order]; 975 } 976 977 static struct mlx5_ib_mr * 978 alloc_mr_from_cache(struct ib_pd *pd, struct ib_umem *umem, u64 virt_addr, 979 u64 len, int npages, int page_shift, unsigned int order, 980 int access_flags) 981 { 982 struct mlx5_ib_dev *dev = to_mdev(pd->device); 983 struct mlx5_cache_ent *ent = mr_cache_ent_from_order(dev, order); 984 struct mlx5_ib_mr *mr; 985 986 if (!ent) 987 return ERR_PTR(-E2BIG); 988 989 /* Matches access in alloc_cache_mr() */ 990 if (!mlx5_ib_can_reconfig_with_umr(dev, 0, access_flags)) 991 return ERR_PTR(-EOPNOTSUPP); 992 993 mr = get_cache_mr(ent); 994 if (!mr) { 995 mr = create_cache_mr(ent); 996 if (IS_ERR(mr)) 997 return mr; 998 } 999 1000 mr->ibmr.pd = pd; 1001 mr->umem = umem; 1002 mr->access_flags = access_flags; 1003 mr->desc_size = sizeof(struct mlx5_mtt); 1004 mr->mmkey.iova = virt_addr; 1005 mr->mmkey.size = len; 1006 mr->mmkey.pd = to_mpd(pd)->pdn; 1007 1008 return mr; 1009 } 1010 1011 #define MLX5_MAX_UMR_CHUNK ((1 << (MLX5_MAX_UMR_SHIFT + 4)) - \ 1012 MLX5_UMR_MTT_ALIGNMENT) 1013 #define MLX5_SPARE_UMR_CHUNK 0x10000 1014 1015 int mlx5_ib_update_xlt(struct mlx5_ib_mr *mr, u64 idx, int npages, 1016 int page_shift, int flags) 1017 { 1018 struct mlx5_ib_dev *dev = mr->dev; 1019 struct device *ddev = dev->ib_dev.dev.parent; 1020 int size; 1021 void *xlt; 1022 dma_addr_t dma; 1023 struct mlx5_umr_wr wr; 1024 struct ib_sge sg; 1025 int err = 0; 1026 int desc_size = (flags & MLX5_IB_UPD_XLT_INDIRECT) 1027 ? sizeof(struct mlx5_klm) 1028 : sizeof(struct mlx5_mtt); 1029 const int page_align = MLX5_UMR_MTT_ALIGNMENT / desc_size; 1030 const int page_mask = page_align - 1; 1031 size_t pages_mapped = 0; 1032 size_t pages_to_map = 0; 1033 size_t pages_iter = 0; 1034 size_t size_to_map = 0; 1035 gfp_t gfp; 1036 bool use_emergency_page = false; 1037 1038 if ((flags & MLX5_IB_UPD_XLT_INDIRECT) && 1039 !umr_can_use_indirect_mkey(dev)) 1040 return -EPERM; 1041 1042 /* UMR copies MTTs in units of MLX5_UMR_MTT_ALIGNMENT bytes, 1043 * so we need to align the offset and length accordingly 1044 */ 1045 if (idx & page_mask) { 1046 npages += idx & page_mask; 1047 idx &= ~page_mask; 1048 } 1049 1050 gfp = flags & MLX5_IB_UPD_XLT_ATOMIC ? GFP_ATOMIC : GFP_KERNEL; 1051 gfp |= __GFP_ZERO | __GFP_NOWARN; 1052 1053 pages_to_map = ALIGN(npages, page_align); 1054 size = desc_size * pages_to_map; 1055 size = min_t(int, size, MLX5_MAX_UMR_CHUNK); 1056 1057 xlt = (void *)__get_free_pages(gfp, get_order(size)); 1058 if (!xlt && size > MLX5_SPARE_UMR_CHUNK) { 1059 mlx5_ib_dbg(dev, "Failed to allocate %d bytes of order %d. fallback to spare UMR allocation od %d bytes\n", 1060 size, get_order(size), MLX5_SPARE_UMR_CHUNK); 1061 1062 size = MLX5_SPARE_UMR_CHUNK; 1063 xlt = (void *)__get_free_pages(gfp, get_order(size)); 1064 } 1065 1066 if (!xlt) { 1067 mlx5_ib_warn(dev, "Using XLT emergency buffer\n"); 1068 xlt = (void *)mlx5_ib_get_xlt_emergency_page(); 1069 size = PAGE_SIZE; 1070 memset(xlt, 0, size); 1071 use_emergency_page = true; 1072 } 1073 pages_iter = size / desc_size; 1074 dma = dma_map_single(ddev, xlt, size, DMA_TO_DEVICE); 1075 if (dma_mapping_error(ddev, dma)) { 1076 mlx5_ib_err(dev, "unable to map DMA during XLT update.\n"); 1077 err = -ENOMEM; 1078 goto free_xlt; 1079 } 1080 1081 if (mr->umem->is_odp) { 1082 if (!(flags & MLX5_IB_UPD_XLT_INDIRECT)) { 1083 struct ib_umem_odp *odp = to_ib_umem_odp(mr->umem); 1084 size_t max_pages = ib_umem_odp_num_pages(odp) - idx; 1085 1086 pages_to_map = min_t(size_t, pages_to_map, max_pages); 1087 } 1088 } 1089 1090 sg.addr = dma; 1091 sg.lkey = dev->umrc.pd->local_dma_lkey; 1092 1093 memset(&wr, 0, sizeof(wr)); 1094 wr.wr.send_flags = MLX5_IB_SEND_UMR_UPDATE_XLT; 1095 if (!(flags & MLX5_IB_UPD_XLT_ENABLE)) 1096 wr.wr.send_flags |= MLX5_IB_SEND_UMR_FAIL_IF_FREE; 1097 wr.wr.sg_list = &sg; 1098 wr.wr.num_sge = 1; 1099 wr.wr.opcode = MLX5_IB_WR_UMR; 1100 1101 wr.pd = mr->ibmr.pd; 1102 wr.mkey = mr->mmkey.key; 1103 wr.length = mr->mmkey.size; 1104 wr.virt_addr = mr->mmkey.iova; 1105 wr.access_flags = mr->access_flags; 1106 wr.page_shift = page_shift; 1107 1108 for (pages_mapped = 0; 1109 pages_mapped < pages_to_map && !err; 1110 pages_mapped += pages_iter, idx += pages_iter) { 1111 npages = min_t(int, pages_iter, pages_to_map - pages_mapped); 1112 size_to_map = npages * desc_size; 1113 dma_sync_single_for_cpu(ddev, dma, size, DMA_TO_DEVICE); 1114 if (mr->umem->is_odp) { 1115 mlx5_odp_populate_xlt(xlt, idx, npages, mr, flags); 1116 } else { 1117 __mlx5_ib_populate_pas(dev, mr->umem, page_shift, idx, 1118 npages, xlt, 1119 MLX5_IB_MTT_PRESENT); 1120 /* Clear padding after the pages 1121 * brought from the umem. 1122 */ 1123 memset(xlt + size_to_map, 0, size - size_to_map); 1124 } 1125 dma_sync_single_for_device(ddev, dma, size, DMA_TO_DEVICE); 1126 1127 sg.length = ALIGN(size_to_map, MLX5_UMR_MTT_ALIGNMENT); 1128 1129 if (pages_mapped + pages_iter >= pages_to_map) { 1130 if (flags & MLX5_IB_UPD_XLT_ENABLE) 1131 wr.wr.send_flags |= 1132 MLX5_IB_SEND_UMR_ENABLE_MR | 1133 MLX5_IB_SEND_UMR_UPDATE_PD_ACCESS | 1134 MLX5_IB_SEND_UMR_UPDATE_TRANSLATION; 1135 if (flags & MLX5_IB_UPD_XLT_PD || 1136 flags & MLX5_IB_UPD_XLT_ACCESS) 1137 wr.wr.send_flags |= 1138 MLX5_IB_SEND_UMR_UPDATE_PD_ACCESS; 1139 if (flags & MLX5_IB_UPD_XLT_ADDR) 1140 wr.wr.send_flags |= 1141 MLX5_IB_SEND_UMR_UPDATE_TRANSLATION; 1142 } 1143 1144 wr.offset = idx * desc_size; 1145 wr.xlt_size = sg.length; 1146 1147 err = mlx5_ib_post_send_wait(dev, &wr); 1148 } 1149 dma_unmap_single(ddev, dma, size, DMA_TO_DEVICE); 1150 1151 free_xlt: 1152 if (use_emergency_page) 1153 mlx5_ib_put_xlt_emergency_page(); 1154 else 1155 free_pages((unsigned long)xlt, get_order(size)); 1156 1157 return err; 1158 } 1159 1160 /* 1161 * If ibmr is NULL it will be allocated by reg_create. 1162 * Else, the given ibmr will be used. 1163 */ 1164 static struct mlx5_ib_mr *reg_create(struct ib_mr *ibmr, struct ib_pd *pd, 1165 u64 virt_addr, u64 length, 1166 struct ib_umem *umem, int npages, 1167 int page_shift, int access_flags, 1168 bool populate) 1169 { 1170 struct mlx5_ib_dev *dev = to_mdev(pd->device); 1171 struct mlx5_ib_mr *mr; 1172 __be64 *pas; 1173 void *mkc; 1174 int inlen; 1175 u32 *in; 1176 int err; 1177 bool pg_cap = !!(MLX5_CAP_GEN(dev->mdev, pg)); 1178 1179 mr = ibmr ? to_mmr(ibmr) : kzalloc(sizeof(*mr), GFP_KERNEL); 1180 if (!mr) 1181 return ERR_PTR(-ENOMEM); 1182 1183 mr->ibmr.pd = pd; 1184 mr->access_flags = access_flags; 1185 1186 inlen = MLX5_ST_SZ_BYTES(create_mkey_in); 1187 if (populate) 1188 inlen += sizeof(*pas) * roundup(npages, 2); 1189 in = kvzalloc(inlen, GFP_KERNEL); 1190 if (!in) { 1191 err = -ENOMEM; 1192 goto err_1; 1193 } 1194 pas = (__be64 *)MLX5_ADDR_OF(create_mkey_in, in, klm_pas_mtt); 1195 if (populate) { 1196 if (WARN_ON(access_flags & IB_ACCESS_ON_DEMAND)) { 1197 err = -EINVAL; 1198 goto err_2; 1199 } 1200 mlx5_ib_populate_pas(dev, umem, page_shift, pas, 1201 pg_cap ? MLX5_IB_MTT_PRESENT : 0); 1202 } 1203 1204 /* The pg_access bit allows setting the access flags 1205 * in the page list submitted with the command. */ 1206 MLX5_SET(create_mkey_in, in, pg_access, !!(pg_cap)); 1207 1208 mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry); 1209 set_mkc_access_pd_addr_fields(mkc, access_flags, virt_addr, 1210 populate ? pd : dev->umrc.pd); 1211 MLX5_SET(mkc, mkc, free, !populate); 1212 MLX5_SET(mkc, mkc, access_mode_1_0, MLX5_MKC_ACCESS_MODE_MTT); 1213 MLX5_SET(mkc, mkc, umr_en, 1); 1214 1215 MLX5_SET64(mkc, mkc, len, length); 1216 MLX5_SET(mkc, mkc, bsf_octword_size, 0); 1217 MLX5_SET(mkc, mkc, translations_octword_size, 1218 get_octo_len(virt_addr, length, page_shift)); 1219 MLX5_SET(mkc, mkc, log_page_size, page_shift); 1220 if (populate) { 1221 MLX5_SET(create_mkey_in, in, translations_octword_actual_size, 1222 get_octo_len(virt_addr, length, page_shift)); 1223 } 1224 1225 err = mlx5_ib_create_mkey(dev, &mr->mmkey, in, inlen); 1226 if (err) { 1227 mlx5_ib_warn(dev, "create mkey failed\n"); 1228 goto err_2; 1229 } 1230 mr->mmkey.type = MLX5_MKEY_MR; 1231 mr->desc_size = sizeof(struct mlx5_mtt); 1232 mr->dev = dev; 1233 kvfree(in); 1234 1235 mlx5_ib_dbg(dev, "mkey = 0x%x\n", mr->mmkey.key); 1236 1237 return mr; 1238 1239 err_2: 1240 kvfree(in); 1241 1242 err_1: 1243 if (!ibmr) 1244 kfree(mr); 1245 1246 return ERR_PTR(err); 1247 } 1248 1249 static void set_mr_fields(struct mlx5_ib_dev *dev, struct mlx5_ib_mr *mr, 1250 int npages, u64 length, int access_flags) 1251 { 1252 mr->npages = npages; 1253 atomic_add(npages, &dev->mdev->priv.reg_pages); 1254 mr->ibmr.lkey = mr->mmkey.key; 1255 mr->ibmr.rkey = mr->mmkey.key; 1256 mr->ibmr.length = length; 1257 mr->access_flags = access_flags; 1258 } 1259 1260 static struct ib_mr *mlx5_ib_get_dm_mr(struct ib_pd *pd, u64 start_addr, 1261 u64 length, int acc, int mode) 1262 { 1263 struct mlx5_ib_dev *dev = to_mdev(pd->device); 1264 int inlen = MLX5_ST_SZ_BYTES(create_mkey_in); 1265 struct mlx5_ib_mr *mr; 1266 void *mkc; 1267 u32 *in; 1268 int err; 1269 1270 mr = kzalloc(sizeof(*mr), GFP_KERNEL); 1271 if (!mr) 1272 return ERR_PTR(-ENOMEM); 1273 1274 in = kzalloc(inlen, GFP_KERNEL); 1275 if (!in) { 1276 err = -ENOMEM; 1277 goto err_free; 1278 } 1279 1280 mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry); 1281 1282 MLX5_SET(mkc, mkc, access_mode_1_0, mode & 0x3); 1283 MLX5_SET(mkc, mkc, access_mode_4_2, (mode >> 2) & 0x7); 1284 MLX5_SET64(mkc, mkc, len, length); 1285 set_mkc_access_pd_addr_fields(mkc, acc, start_addr, pd); 1286 1287 err = mlx5_ib_create_mkey(dev, &mr->mmkey, in, inlen); 1288 if (err) 1289 goto err_in; 1290 1291 kfree(in); 1292 1293 mr->umem = NULL; 1294 set_mr_fields(dev, mr, 0, length, acc); 1295 1296 return &mr->ibmr; 1297 1298 err_in: 1299 kfree(in); 1300 1301 err_free: 1302 kfree(mr); 1303 1304 return ERR_PTR(err); 1305 } 1306 1307 int mlx5_ib_advise_mr(struct ib_pd *pd, 1308 enum ib_uverbs_advise_mr_advice advice, 1309 u32 flags, 1310 struct ib_sge *sg_list, 1311 u32 num_sge, 1312 struct uverbs_attr_bundle *attrs) 1313 { 1314 if (advice != IB_UVERBS_ADVISE_MR_ADVICE_PREFETCH && 1315 advice != IB_UVERBS_ADVISE_MR_ADVICE_PREFETCH_WRITE && 1316 advice != IB_UVERBS_ADVISE_MR_ADVICE_PREFETCH_NO_FAULT) 1317 return -EOPNOTSUPP; 1318 1319 return mlx5_ib_advise_mr_prefetch(pd, advice, flags, 1320 sg_list, num_sge); 1321 } 1322 1323 struct ib_mr *mlx5_ib_reg_dm_mr(struct ib_pd *pd, struct ib_dm *dm, 1324 struct ib_dm_mr_attr *attr, 1325 struct uverbs_attr_bundle *attrs) 1326 { 1327 struct mlx5_ib_dm *mdm = to_mdm(dm); 1328 struct mlx5_core_dev *dev = to_mdev(dm->device)->mdev; 1329 u64 start_addr = mdm->dev_addr + attr->offset; 1330 int mode; 1331 1332 switch (mdm->type) { 1333 case MLX5_IB_UAPI_DM_TYPE_MEMIC: 1334 if (attr->access_flags & ~MLX5_IB_DM_MEMIC_ALLOWED_ACCESS) 1335 return ERR_PTR(-EINVAL); 1336 1337 mode = MLX5_MKC_ACCESS_MODE_MEMIC; 1338 start_addr -= pci_resource_start(dev->pdev, 0); 1339 break; 1340 case MLX5_IB_UAPI_DM_TYPE_STEERING_SW_ICM: 1341 case MLX5_IB_UAPI_DM_TYPE_HEADER_MODIFY_SW_ICM: 1342 if (attr->access_flags & ~MLX5_IB_DM_SW_ICM_ALLOWED_ACCESS) 1343 return ERR_PTR(-EINVAL); 1344 1345 mode = MLX5_MKC_ACCESS_MODE_SW_ICM; 1346 break; 1347 default: 1348 return ERR_PTR(-EINVAL); 1349 } 1350 1351 return mlx5_ib_get_dm_mr(pd, start_addr, attr->length, 1352 attr->access_flags, mode); 1353 } 1354 1355 struct ib_mr *mlx5_ib_reg_user_mr(struct ib_pd *pd, u64 start, u64 length, 1356 u64 virt_addr, int access_flags, 1357 struct ib_udata *udata) 1358 { 1359 struct mlx5_ib_dev *dev = to_mdev(pd->device); 1360 struct mlx5_ib_mr *mr = NULL; 1361 bool xlt_with_umr; 1362 struct ib_umem *umem; 1363 int page_shift; 1364 int npages; 1365 int ncont; 1366 int order; 1367 int err; 1368 1369 if (!IS_ENABLED(CONFIG_INFINIBAND_USER_MEM)) 1370 return ERR_PTR(-EOPNOTSUPP); 1371 1372 mlx5_ib_dbg(dev, "start 0x%llx, virt_addr 0x%llx, length 0x%llx, access_flags 0x%x\n", 1373 start, virt_addr, length, access_flags); 1374 1375 xlt_with_umr = mlx5_ib_can_load_pas_with_umr(dev, length); 1376 /* ODP requires xlt update via umr to work. */ 1377 if (!xlt_with_umr && (access_flags & IB_ACCESS_ON_DEMAND)) 1378 return ERR_PTR(-EINVAL); 1379 1380 if (IS_ENABLED(CONFIG_INFINIBAND_ON_DEMAND_PAGING) && !start && 1381 length == U64_MAX) { 1382 if (virt_addr != start) 1383 return ERR_PTR(-EINVAL); 1384 if (!(access_flags & IB_ACCESS_ON_DEMAND) || 1385 !(dev->odp_caps.general_caps & IB_ODP_SUPPORT_IMPLICIT)) 1386 return ERR_PTR(-EINVAL); 1387 1388 mr = mlx5_ib_alloc_implicit_mr(to_mpd(pd), udata, access_flags); 1389 if (IS_ERR(mr)) 1390 return ERR_CAST(mr); 1391 return &mr->ibmr; 1392 } 1393 1394 err = mr_umem_get(dev, start, length, access_flags, &umem, 1395 &npages, &page_shift, &ncont, &order); 1396 1397 if (err < 0) 1398 return ERR_PTR(err); 1399 1400 if (xlt_with_umr) { 1401 mr = alloc_mr_from_cache(pd, umem, virt_addr, length, ncont, 1402 page_shift, order, access_flags); 1403 if (IS_ERR(mr)) 1404 mr = NULL; 1405 } 1406 1407 if (!mr) { 1408 mutex_lock(&dev->slow_path_mutex); 1409 mr = reg_create(NULL, pd, virt_addr, length, umem, ncont, 1410 page_shift, access_flags, !xlt_with_umr); 1411 mutex_unlock(&dev->slow_path_mutex); 1412 } 1413 1414 if (IS_ERR(mr)) { 1415 err = PTR_ERR(mr); 1416 goto error; 1417 } 1418 1419 mlx5_ib_dbg(dev, "mkey 0x%x\n", mr->mmkey.key); 1420 1421 mr->umem = umem; 1422 set_mr_fields(dev, mr, npages, length, access_flags); 1423 1424 if (xlt_with_umr && !(access_flags & IB_ACCESS_ON_DEMAND)) { 1425 /* 1426 * If the MR was created with reg_create then it will be 1427 * configured properly but left disabled. It is safe to go ahead 1428 * and configure it again via UMR while enabling it. 1429 */ 1430 int update_xlt_flags = MLX5_IB_UPD_XLT_ENABLE; 1431 1432 err = mlx5_ib_update_xlt(mr, 0, ncont, page_shift, 1433 update_xlt_flags); 1434 if (err) { 1435 dereg_mr(dev, mr); 1436 return ERR_PTR(err); 1437 } 1438 } 1439 1440 if (is_odp_mr(mr)) { 1441 to_ib_umem_odp(mr->umem)->private = mr; 1442 init_waitqueue_head(&mr->q_deferred_work); 1443 atomic_set(&mr->num_deferred_work, 0); 1444 err = xa_err(xa_store(&dev->odp_mkeys, 1445 mlx5_base_mkey(mr->mmkey.key), &mr->mmkey, 1446 GFP_KERNEL)); 1447 if (err) { 1448 dereg_mr(dev, mr); 1449 return ERR_PTR(err); 1450 } 1451 1452 err = mlx5_ib_init_odp_mr(mr, xlt_with_umr); 1453 if (err) { 1454 dereg_mr(dev, mr); 1455 return ERR_PTR(err); 1456 } 1457 } 1458 1459 return &mr->ibmr; 1460 error: 1461 ib_umem_release(umem); 1462 return ERR_PTR(err); 1463 } 1464 1465 /** 1466 * mlx5_mr_cache_invalidate - Fence all DMA on the MR 1467 * @mr: The MR to fence 1468 * 1469 * Upon return the NIC will not be doing any DMA to the pages under the MR, 1470 * and any DMA inprogress will be completed. Failure of this function 1471 * indicates the HW has failed catastrophically. 1472 */ 1473 int mlx5_mr_cache_invalidate(struct mlx5_ib_mr *mr) 1474 { 1475 struct mlx5_umr_wr umrwr = {}; 1476 1477 if (mr->dev->mdev->state == MLX5_DEVICE_STATE_INTERNAL_ERROR) 1478 return 0; 1479 1480 umrwr.wr.send_flags = MLX5_IB_SEND_UMR_DISABLE_MR | 1481 MLX5_IB_SEND_UMR_UPDATE_PD_ACCESS; 1482 umrwr.wr.opcode = MLX5_IB_WR_UMR; 1483 umrwr.pd = mr->dev->umrc.pd; 1484 umrwr.mkey = mr->mmkey.key; 1485 umrwr.ignore_free_state = 1; 1486 1487 return mlx5_ib_post_send_wait(mr->dev, &umrwr); 1488 } 1489 1490 static int rereg_umr(struct ib_pd *pd, struct mlx5_ib_mr *mr, 1491 int access_flags, int flags) 1492 { 1493 struct mlx5_ib_dev *dev = to_mdev(pd->device); 1494 struct mlx5_umr_wr umrwr = {}; 1495 int err; 1496 1497 umrwr.wr.send_flags = MLX5_IB_SEND_UMR_FAIL_IF_FREE; 1498 1499 umrwr.wr.opcode = MLX5_IB_WR_UMR; 1500 umrwr.mkey = mr->mmkey.key; 1501 1502 if (flags & IB_MR_REREG_PD || flags & IB_MR_REREG_ACCESS) { 1503 umrwr.pd = pd; 1504 umrwr.access_flags = access_flags; 1505 umrwr.wr.send_flags |= MLX5_IB_SEND_UMR_UPDATE_PD_ACCESS; 1506 } 1507 1508 err = mlx5_ib_post_send_wait(dev, &umrwr); 1509 1510 return err; 1511 } 1512 1513 int mlx5_ib_rereg_user_mr(struct ib_mr *ib_mr, int flags, u64 start, 1514 u64 length, u64 virt_addr, int new_access_flags, 1515 struct ib_pd *new_pd, struct ib_udata *udata) 1516 { 1517 struct mlx5_ib_dev *dev = to_mdev(ib_mr->device); 1518 struct mlx5_ib_mr *mr = to_mmr(ib_mr); 1519 struct ib_pd *pd = (flags & IB_MR_REREG_PD) ? new_pd : ib_mr->pd; 1520 int access_flags = flags & IB_MR_REREG_ACCESS ? 1521 new_access_flags : 1522 mr->access_flags; 1523 int page_shift = 0; 1524 int upd_flags = 0; 1525 int npages = 0; 1526 int ncont = 0; 1527 int order = 0; 1528 u64 addr, len; 1529 int err; 1530 1531 mlx5_ib_dbg(dev, "start 0x%llx, virt_addr 0x%llx, length 0x%llx, access_flags 0x%x\n", 1532 start, virt_addr, length, access_flags); 1533 1534 atomic_sub(mr->npages, &dev->mdev->priv.reg_pages); 1535 1536 if (!mr->umem) 1537 return -EINVAL; 1538 1539 if (is_odp_mr(mr)) 1540 return -EOPNOTSUPP; 1541 1542 if (flags & IB_MR_REREG_TRANS) { 1543 addr = virt_addr; 1544 len = length; 1545 } else { 1546 addr = mr->umem->address; 1547 len = mr->umem->length; 1548 } 1549 1550 if (flags != IB_MR_REREG_PD) { 1551 /* 1552 * Replace umem. This needs to be done whether or not UMR is 1553 * used. 1554 */ 1555 flags |= IB_MR_REREG_TRANS; 1556 ib_umem_release(mr->umem); 1557 mr->umem = NULL; 1558 err = mr_umem_get(dev, addr, len, access_flags, &mr->umem, 1559 &npages, &page_shift, &ncont, &order); 1560 if (err) 1561 goto err; 1562 } 1563 1564 if (!mlx5_ib_can_reconfig_with_umr(dev, mr->access_flags, 1565 access_flags) || 1566 !mlx5_ib_can_load_pas_with_umr(dev, len) || 1567 (flags & IB_MR_REREG_TRANS && 1568 !mlx5_ib_pas_fits_in_mr(mr, addr, len))) { 1569 /* 1570 * UMR can't be used - MKey needs to be replaced. 1571 */ 1572 if (mr->cache_ent) 1573 detach_mr_from_cache(mr); 1574 err = destroy_mkey(dev, mr); 1575 if (err) 1576 goto err; 1577 1578 mr = reg_create(ib_mr, pd, addr, len, mr->umem, ncont, 1579 page_shift, access_flags, true); 1580 1581 if (IS_ERR(mr)) { 1582 err = PTR_ERR(mr); 1583 mr = to_mmr(ib_mr); 1584 goto err; 1585 } 1586 } else { 1587 /* 1588 * Send a UMR WQE 1589 */ 1590 mr->ibmr.pd = pd; 1591 mr->access_flags = access_flags; 1592 mr->mmkey.iova = addr; 1593 mr->mmkey.size = len; 1594 mr->mmkey.pd = to_mpd(pd)->pdn; 1595 1596 if (flags & IB_MR_REREG_TRANS) { 1597 upd_flags = MLX5_IB_UPD_XLT_ADDR; 1598 if (flags & IB_MR_REREG_PD) 1599 upd_flags |= MLX5_IB_UPD_XLT_PD; 1600 if (flags & IB_MR_REREG_ACCESS) 1601 upd_flags |= MLX5_IB_UPD_XLT_ACCESS; 1602 err = mlx5_ib_update_xlt(mr, 0, npages, page_shift, 1603 upd_flags); 1604 } else { 1605 err = rereg_umr(pd, mr, access_flags, flags); 1606 } 1607 1608 if (err) 1609 goto err; 1610 } 1611 1612 set_mr_fields(dev, mr, npages, len, access_flags); 1613 1614 return 0; 1615 1616 err: 1617 ib_umem_release(mr->umem); 1618 mr->umem = NULL; 1619 1620 clean_mr(dev, mr); 1621 return err; 1622 } 1623 1624 static int 1625 mlx5_alloc_priv_descs(struct ib_device *device, 1626 struct mlx5_ib_mr *mr, 1627 int ndescs, 1628 int desc_size) 1629 { 1630 int size = ndescs * desc_size; 1631 int add_size; 1632 int ret; 1633 1634 add_size = max_t(int, MLX5_UMR_ALIGN - ARCH_KMALLOC_MINALIGN, 0); 1635 1636 mr->descs_alloc = kzalloc(size + add_size, GFP_KERNEL); 1637 if (!mr->descs_alloc) 1638 return -ENOMEM; 1639 1640 mr->descs = PTR_ALIGN(mr->descs_alloc, MLX5_UMR_ALIGN); 1641 1642 mr->desc_map = dma_map_single(device->dev.parent, mr->descs, 1643 size, DMA_TO_DEVICE); 1644 if (dma_mapping_error(device->dev.parent, mr->desc_map)) { 1645 ret = -ENOMEM; 1646 goto err; 1647 } 1648 1649 return 0; 1650 err: 1651 kfree(mr->descs_alloc); 1652 1653 return ret; 1654 } 1655 1656 static void 1657 mlx5_free_priv_descs(struct mlx5_ib_mr *mr) 1658 { 1659 if (mr->descs) { 1660 struct ib_device *device = mr->ibmr.device; 1661 int size = mr->max_descs * mr->desc_size; 1662 1663 dma_unmap_single(device->dev.parent, mr->desc_map, 1664 size, DMA_TO_DEVICE); 1665 kfree(mr->descs_alloc); 1666 mr->descs = NULL; 1667 } 1668 } 1669 1670 static void clean_mr(struct mlx5_ib_dev *dev, struct mlx5_ib_mr *mr) 1671 { 1672 if (mr->sig) { 1673 if (mlx5_core_destroy_psv(dev->mdev, 1674 mr->sig->psv_memory.psv_idx)) 1675 mlx5_ib_warn(dev, "failed to destroy mem psv %d\n", 1676 mr->sig->psv_memory.psv_idx); 1677 if (mlx5_core_destroy_psv(dev->mdev, 1678 mr->sig->psv_wire.psv_idx)) 1679 mlx5_ib_warn(dev, "failed to destroy wire psv %d\n", 1680 mr->sig->psv_wire.psv_idx); 1681 xa_erase(&dev->sig_mrs, mlx5_base_mkey(mr->mmkey.key)); 1682 kfree(mr->sig); 1683 mr->sig = NULL; 1684 } 1685 1686 if (!mr->cache_ent) { 1687 destroy_mkey(dev, mr); 1688 mlx5_free_priv_descs(mr); 1689 } 1690 } 1691 1692 static void dereg_mr(struct mlx5_ib_dev *dev, struct mlx5_ib_mr *mr) 1693 { 1694 int npages = mr->npages; 1695 struct ib_umem *umem = mr->umem; 1696 1697 /* Stop all DMA */ 1698 if (is_odp_mr(mr)) 1699 mlx5_ib_fence_odp_mr(mr); 1700 else 1701 clean_mr(dev, mr); 1702 1703 if (mr->cache_ent) 1704 mlx5_mr_cache_free(dev, mr); 1705 else 1706 kfree(mr); 1707 1708 ib_umem_release(umem); 1709 atomic_sub(npages, &dev->mdev->priv.reg_pages); 1710 1711 } 1712 1713 int mlx5_ib_dereg_mr(struct ib_mr *ibmr, struct ib_udata *udata) 1714 { 1715 struct mlx5_ib_mr *mmr = to_mmr(ibmr); 1716 1717 if (ibmr->type == IB_MR_TYPE_INTEGRITY) { 1718 dereg_mr(to_mdev(mmr->mtt_mr->ibmr.device), mmr->mtt_mr); 1719 dereg_mr(to_mdev(mmr->klm_mr->ibmr.device), mmr->klm_mr); 1720 } 1721 1722 if (is_odp_mr(mmr) && to_ib_umem_odp(mmr->umem)->is_implicit_odp) { 1723 mlx5_ib_free_implicit_mr(mmr); 1724 return 0; 1725 } 1726 1727 dereg_mr(to_mdev(ibmr->device), mmr); 1728 1729 return 0; 1730 } 1731 1732 static void mlx5_set_umr_free_mkey(struct ib_pd *pd, u32 *in, int ndescs, 1733 int access_mode, int page_shift) 1734 { 1735 void *mkc; 1736 1737 mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry); 1738 1739 /* This is only used from the kernel, so setting the PD is OK. */ 1740 set_mkc_access_pd_addr_fields(mkc, 0, 0, pd); 1741 MLX5_SET(mkc, mkc, free, 1); 1742 MLX5_SET(mkc, mkc, translations_octword_size, ndescs); 1743 MLX5_SET(mkc, mkc, access_mode_1_0, access_mode & 0x3); 1744 MLX5_SET(mkc, mkc, access_mode_4_2, (access_mode >> 2) & 0x7); 1745 MLX5_SET(mkc, mkc, umr_en, 1); 1746 MLX5_SET(mkc, mkc, log_page_size, page_shift); 1747 } 1748 1749 static int _mlx5_alloc_mkey_descs(struct ib_pd *pd, struct mlx5_ib_mr *mr, 1750 int ndescs, int desc_size, int page_shift, 1751 int access_mode, u32 *in, int inlen) 1752 { 1753 struct mlx5_ib_dev *dev = to_mdev(pd->device); 1754 int err; 1755 1756 mr->access_mode = access_mode; 1757 mr->desc_size = desc_size; 1758 mr->max_descs = ndescs; 1759 1760 err = mlx5_alloc_priv_descs(pd->device, mr, ndescs, desc_size); 1761 if (err) 1762 return err; 1763 1764 mlx5_set_umr_free_mkey(pd, in, ndescs, access_mode, page_shift); 1765 1766 err = mlx5_ib_create_mkey(dev, &mr->mmkey, in, inlen); 1767 if (err) 1768 goto err_free_descs; 1769 1770 mr->mmkey.type = MLX5_MKEY_MR; 1771 mr->ibmr.lkey = mr->mmkey.key; 1772 mr->ibmr.rkey = mr->mmkey.key; 1773 1774 return 0; 1775 1776 err_free_descs: 1777 mlx5_free_priv_descs(mr); 1778 return err; 1779 } 1780 1781 static struct mlx5_ib_mr *mlx5_ib_alloc_pi_mr(struct ib_pd *pd, 1782 u32 max_num_sg, u32 max_num_meta_sg, 1783 int desc_size, int access_mode) 1784 { 1785 int inlen = MLX5_ST_SZ_BYTES(create_mkey_in); 1786 int ndescs = ALIGN(max_num_sg + max_num_meta_sg, 4); 1787 int page_shift = 0; 1788 struct mlx5_ib_mr *mr; 1789 u32 *in; 1790 int err; 1791 1792 mr = kzalloc(sizeof(*mr), GFP_KERNEL); 1793 if (!mr) 1794 return ERR_PTR(-ENOMEM); 1795 1796 mr->ibmr.pd = pd; 1797 mr->ibmr.device = pd->device; 1798 1799 in = kzalloc(inlen, GFP_KERNEL); 1800 if (!in) { 1801 err = -ENOMEM; 1802 goto err_free; 1803 } 1804 1805 if (access_mode == MLX5_MKC_ACCESS_MODE_MTT) 1806 page_shift = PAGE_SHIFT; 1807 1808 err = _mlx5_alloc_mkey_descs(pd, mr, ndescs, desc_size, page_shift, 1809 access_mode, in, inlen); 1810 if (err) 1811 goto err_free_in; 1812 1813 mr->umem = NULL; 1814 kfree(in); 1815 1816 return mr; 1817 1818 err_free_in: 1819 kfree(in); 1820 err_free: 1821 kfree(mr); 1822 return ERR_PTR(err); 1823 } 1824 1825 static int mlx5_alloc_mem_reg_descs(struct ib_pd *pd, struct mlx5_ib_mr *mr, 1826 int ndescs, u32 *in, int inlen) 1827 { 1828 return _mlx5_alloc_mkey_descs(pd, mr, ndescs, sizeof(struct mlx5_mtt), 1829 PAGE_SHIFT, MLX5_MKC_ACCESS_MODE_MTT, in, 1830 inlen); 1831 } 1832 1833 static int mlx5_alloc_sg_gaps_descs(struct ib_pd *pd, struct mlx5_ib_mr *mr, 1834 int ndescs, u32 *in, int inlen) 1835 { 1836 return _mlx5_alloc_mkey_descs(pd, mr, ndescs, sizeof(struct mlx5_klm), 1837 0, MLX5_MKC_ACCESS_MODE_KLMS, in, inlen); 1838 } 1839 1840 static int mlx5_alloc_integrity_descs(struct ib_pd *pd, struct mlx5_ib_mr *mr, 1841 int max_num_sg, int max_num_meta_sg, 1842 u32 *in, int inlen) 1843 { 1844 struct mlx5_ib_dev *dev = to_mdev(pd->device); 1845 u32 psv_index[2]; 1846 void *mkc; 1847 int err; 1848 1849 mr->sig = kzalloc(sizeof(*mr->sig), GFP_KERNEL); 1850 if (!mr->sig) 1851 return -ENOMEM; 1852 1853 /* create mem & wire PSVs */ 1854 err = mlx5_core_create_psv(dev->mdev, to_mpd(pd)->pdn, 2, psv_index); 1855 if (err) 1856 goto err_free_sig; 1857 1858 mr->sig->psv_memory.psv_idx = psv_index[0]; 1859 mr->sig->psv_wire.psv_idx = psv_index[1]; 1860 1861 mr->sig->sig_status_checked = true; 1862 mr->sig->sig_err_exists = false; 1863 /* Next UMR, Arm SIGERR */ 1864 ++mr->sig->sigerr_count; 1865 mr->klm_mr = mlx5_ib_alloc_pi_mr(pd, max_num_sg, max_num_meta_sg, 1866 sizeof(struct mlx5_klm), 1867 MLX5_MKC_ACCESS_MODE_KLMS); 1868 if (IS_ERR(mr->klm_mr)) { 1869 err = PTR_ERR(mr->klm_mr); 1870 goto err_destroy_psv; 1871 } 1872 mr->mtt_mr = mlx5_ib_alloc_pi_mr(pd, max_num_sg, max_num_meta_sg, 1873 sizeof(struct mlx5_mtt), 1874 MLX5_MKC_ACCESS_MODE_MTT); 1875 if (IS_ERR(mr->mtt_mr)) { 1876 err = PTR_ERR(mr->mtt_mr); 1877 goto err_free_klm_mr; 1878 } 1879 1880 /* Set bsf descriptors for mkey */ 1881 mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry); 1882 MLX5_SET(mkc, mkc, bsf_en, 1); 1883 MLX5_SET(mkc, mkc, bsf_octword_size, MLX5_MKEY_BSF_OCTO_SIZE); 1884 1885 err = _mlx5_alloc_mkey_descs(pd, mr, 4, sizeof(struct mlx5_klm), 0, 1886 MLX5_MKC_ACCESS_MODE_KLMS, in, inlen); 1887 if (err) 1888 goto err_free_mtt_mr; 1889 1890 err = xa_err(xa_store(&dev->sig_mrs, mlx5_base_mkey(mr->mmkey.key), 1891 mr->sig, GFP_KERNEL)); 1892 if (err) 1893 goto err_free_descs; 1894 return 0; 1895 1896 err_free_descs: 1897 destroy_mkey(dev, mr); 1898 mlx5_free_priv_descs(mr); 1899 err_free_mtt_mr: 1900 dereg_mr(to_mdev(mr->mtt_mr->ibmr.device), mr->mtt_mr); 1901 mr->mtt_mr = NULL; 1902 err_free_klm_mr: 1903 dereg_mr(to_mdev(mr->klm_mr->ibmr.device), mr->klm_mr); 1904 mr->klm_mr = NULL; 1905 err_destroy_psv: 1906 if (mlx5_core_destroy_psv(dev->mdev, mr->sig->psv_memory.psv_idx)) 1907 mlx5_ib_warn(dev, "failed to destroy mem psv %d\n", 1908 mr->sig->psv_memory.psv_idx); 1909 if (mlx5_core_destroy_psv(dev->mdev, mr->sig->psv_wire.psv_idx)) 1910 mlx5_ib_warn(dev, "failed to destroy wire psv %d\n", 1911 mr->sig->psv_wire.psv_idx); 1912 err_free_sig: 1913 kfree(mr->sig); 1914 1915 return err; 1916 } 1917 1918 static struct ib_mr *__mlx5_ib_alloc_mr(struct ib_pd *pd, 1919 enum ib_mr_type mr_type, u32 max_num_sg, 1920 u32 max_num_meta_sg) 1921 { 1922 struct mlx5_ib_dev *dev = to_mdev(pd->device); 1923 int inlen = MLX5_ST_SZ_BYTES(create_mkey_in); 1924 int ndescs = ALIGN(max_num_sg, 4); 1925 struct mlx5_ib_mr *mr; 1926 u32 *in; 1927 int err; 1928 1929 mr = kzalloc(sizeof(*mr), GFP_KERNEL); 1930 if (!mr) 1931 return ERR_PTR(-ENOMEM); 1932 1933 in = kzalloc(inlen, GFP_KERNEL); 1934 if (!in) { 1935 err = -ENOMEM; 1936 goto err_free; 1937 } 1938 1939 mr->ibmr.device = pd->device; 1940 mr->umem = NULL; 1941 1942 switch (mr_type) { 1943 case IB_MR_TYPE_MEM_REG: 1944 err = mlx5_alloc_mem_reg_descs(pd, mr, ndescs, in, inlen); 1945 break; 1946 case IB_MR_TYPE_SG_GAPS: 1947 err = mlx5_alloc_sg_gaps_descs(pd, mr, ndescs, in, inlen); 1948 break; 1949 case IB_MR_TYPE_INTEGRITY: 1950 err = mlx5_alloc_integrity_descs(pd, mr, max_num_sg, 1951 max_num_meta_sg, in, inlen); 1952 break; 1953 default: 1954 mlx5_ib_warn(dev, "Invalid mr type %d\n", mr_type); 1955 err = -EINVAL; 1956 } 1957 1958 if (err) 1959 goto err_free_in; 1960 1961 kfree(in); 1962 1963 return &mr->ibmr; 1964 1965 err_free_in: 1966 kfree(in); 1967 err_free: 1968 kfree(mr); 1969 return ERR_PTR(err); 1970 } 1971 1972 struct ib_mr *mlx5_ib_alloc_mr(struct ib_pd *pd, enum ib_mr_type mr_type, 1973 u32 max_num_sg) 1974 { 1975 return __mlx5_ib_alloc_mr(pd, mr_type, max_num_sg, 0); 1976 } 1977 1978 struct ib_mr *mlx5_ib_alloc_mr_integrity(struct ib_pd *pd, 1979 u32 max_num_sg, u32 max_num_meta_sg) 1980 { 1981 return __mlx5_ib_alloc_mr(pd, IB_MR_TYPE_INTEGRITY, max_num_sg, 1982 max_num_meta_sg); 1983 } 1984 1985 int mlx5_ib_alloc_mw(struct ib_mw *ibmw, struct ib_udata *udata) 1986 { 1987 struct mlx5_ib_dev *dev = to_mdev(ibmw->device); 1988 int inlen = MLX5_ST_SZ_BYTES(create_mkey_in); 1989 struct mlx5_ib_mw *mw = to_mmw(ibmw); 1990 u32 *in = NULL; 1991 void *mkc; 1992 int ndescs; 1993 int err; 1994 struct mlx5_ib_alloc_mw req = {}; 1995 struct { 1996 __u32 comp_mask; 1997 __u32 response_length; 1998 } resp = {}; 1999 2000 err = ib_copy_from_udata(&req, udata, min(udata->inlen, sizeof(req))); 2001 if (err) 2002 return err; 2003 2004 if (req.comp_mask || req.reserved1 || req.reserved2) 2005 return -EOPNOTSUPP; 2006 2007 if (udata->inlen > sizeof(req) && 2008 !ib_is_udata_cleared(udata, sizeof(req), 2009 udata->inlen - sizeof(req))) 2010 return -EOPNOTSUPP; 2011 2012 ndescs = req.num_klms ? roundup(req.num_klms, 4) : roundup(1, 4); 2013 2014 in = kzalloc(inlen, GFP_KERNEL); 2015 if (!in) { 2016 err = -ENOMEM; 2017 goto free; 2018 } 2019 2020 mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry); 2021 2022 MLX5_SET(mkc, mkc, free, 1); 2023 MLX5_SET(mkc, mkc, translations_octword_size, ndescs); 2024 MLX5_SET(mkc, mkc, pd, to_mpd(ibmw->pd)->pdn); 2025 MLX5_SET(mkc, mkc, umr_en, 1); 2026 MLX5_SET(mkc, mkc, lr, 1); 2027 MLX5_SET(mkc, mkc, access_mode_1_0, MLX5_MKC_ACCESS_MODE_KLMS); 2028 MLX5_SET(mkc, mkc, en_rinval, !!((ibmw->type == IB_MW_TYPE_2))); 2029 MLX5_SET(mkc, mkc, qpn, 0xffffff); 2030 2031 err = mlx5_ib_create_mkey(dev, &mw->mmkey, in, inlen); 2032 if (err) 2033 goto free; 2034 2035 mw->mmkey.type = MLX5_MKEY_MW; 2036 ibmw->rkey = mw->mmkey.key; 2037 mw->ndescs = ndescs; 2038 2039 resp.response_length = 2040 min(offsetofend(typeof(resp), response_length), udata->outlen); 2041 if (resp.response_length) { 2042 err = ib_copy_to_udata(udata, &resp, resp.response_length); 2043 if (err) 2044 goto free_mkey; 2045 } 2046 2047 if (IS_ENABLED(CONFIG_INFINIBAND_ON_DEMAND_PAGING)) { 2048 err = xa_err(xa_store(&dev->odp_mkeys, 2049 mlx5_base_mkey(mw->mmkey.key), &mw->mmkey, 2050 GFP_KERNEL)); 2051 if (err) 2052 goto free_mkey; 2053 } 2054 2055 kfree(in); 2056 return 0; 2057 2058 free_mkey: 2059 mlx5_core_destroy_mkey(dev->mdev, &mw->mmkey); 2060 free: 2061 kfree(in); 2062 return err; 2063 } 2064 2065 int mlx5_ib_dealloc_mw(struct ib_mw *mw) 2066 { 2067 struct mlx5_ib_dev *dev = to_mdev(mw->device); 2068 struct mlx5_ib_mw *mmw = to_mmw(mw); 2069 2070 if (IS_ENABLED(CONFIG_INFINIBAND_ON_DEMAND_PAGING)) { 2071 xa_erase(&dev->odp_mkeys, mlx5_base_mkey(mmw->mmkey.key)); 2072 /* 2073 * pagefault_single_data_segment() may be accessing mmw under 2074 * SRCU if the user bound an ODP MR to this MW. 2075 */ 2076 synchronize_srcu(&dev->odp_srcu); 2077 } 2078 2079 return mlx5_core_destroy_mkey(dev->mdev, &mmw->mmkey); 2080 } 2081 2082 int mlx5_ib_check_mr_status(struct ib_mr *ibmr, u32 check_mask, 2083 struct ib_mr_status *mr_status) 2084 { 2085 struct mlx5_ib_mr *mmr = to_mmr(ibmr); 2086 int ret = 0; 2087 2088 if (check_mask & ~IB_MR_CHECK_SIG_STATUS) { 2089 pr_err("Invalid status check mask\n"); 2090 ret = -EINVAL; 2091 goto done; 2092 } 2093 2094 mr_status->fail_status = 0; 2095 if (check_mask & IB_MR_CHECK_SIG_STATUS) { 2096 if (!mmr->sig) { 2097 ret = -EINVAL; 2098 pr_err("signature status check requested on a non-signature enabled MR\n"); 2099 goto done; 2100 } 2101 2102 mmr->sig->sig_status_checked = true; 2103 if (!mmr->sig->sig_err_exists) 2104 goto done; 2105 2106 if (ibmr->lkey == mmr->sig->err_item.key) 2107 memcpy(&mr_status->sig_err, &mmr->sig->err_item, 2108 sizeof(mr_status->sig_err)); 2109 else { 2110 mr_status->sig_err.err_type = IB_SIG_BAD_GUARD; 2111 mr_status->sig_err.sig_err_offset = 0; 2112 mr_status->sig_err.key = mmr->sig->err_item.key; 2113 } 2114 2115 mmr->sig->sig_err_exists = false; 2116 mr_status->fail_status |= IB_MR_CHECK_SIG_STATUS; 2117 } 2118 2119 done: 2120 return ret; 2121 } 2122 2123 static int 2124 mlx5_ib_map_pa_mr_sg_pi(struct ib_mr *ibmr, struct scatterlist *data_sg, 2125 int data_sg_nents, unsigned int *data_sg_offset, 2126 struct scatterlist *meta_sg, int meta_sg_nents, 2127 unsigned int *meta_sg_offset) 2128 { 2129 struct mlx5_ib_mr *mr = to_mmr(ibmr); 2130 unsigned int sg_offset = 0; 2131 int n = 0; 2132 2133 mr->meta_length = 0; 2134 if (data_sg_nents == 1) { 2135 n++; 2136 mr->ndescs = 1; 2137 if (data_sg_offset) 2138 sg_offset = *data_sg_offset; 2139 mr->data_length = sg_dma_len(data_sg) - sg_offset; 2140 mr->data_iova = sg_dma_address(data_sg) + sg_offset; 2141 if (meta_sg_nents == 1) { 2142 n++; 2143 mr->meta_ndescs = 1; 2144 if (meta_sg_offset) 2145 sg_offset = *meta_sg_offset; 2146 else 2147 sg_offset = 0; 2148 mr->meta_length = sg_dma_len(meta_sg) - sg_offset; 2149 mr->pi_iova = sg_dma_address(meta_sg) + sg_offset; 2150 } 2151 ibmr->length = mr->data_length + mr->meta_length; 2152 } 2153 2154 return n; 2155 } 2156 2157 static int 2158 mlx5_ib_sg_to_klms(struct mlx5_ib_mr *mr, 2159 struct scatterlist *sgl, 2160 unsigned short sg_nents, 2161 unsigned int *sg_offset_p, 2162 struct scatterlist *meta_sgl, 2163 unsigned short meta_sg_nents, 2164 unsigned int *meta_sg_offset_p) 2165 { 2166 struct scatterlist *sg = sgl; 2167 struct mlx5_klm *klms = mr->descs; 2168 unsigned int sg_offset = sg_offset_p ? *sg_offset_p : 0; 2169 u32 lkey = mr->ibmr.pd->local_dma_lkey; 2170 int i, j = 0; 2171 2172 mr->ibmr.iova = sg_dma_address(sg) + sg_offset; 2173 mr->ibmr.length = 0; 2174 2175 for_each_sg(sgl, sg, sg_nents, i) { 2176 if (unlikely(i >= mr->max_descs)) 2177 break; 2178 klms[i].va = cpu_to_be64(sg_dma_address(sg) + sg_offset); 2179 klms[i].bcount = cpu_to_be32(sg_dma_len(sg) - sg_offset); 2180 klms[i].key = cpu_to_be32(lkey); 2181 mr->ibmr.length += sg_dma_len(sg) - sg_offset; 2182 2183 sg_offset = 0; 2184 } 2185 2186 if (sg_offset_p) 2187 *sg_offset_p = sg_offset; 2188 2189 mr->ndescs = i; 2190 mr->data_length = mr->ibmr.length; 2191 2192 if (meta_sg_nents) { 2193 sg = meta_sgl; 2194 sg_offset = meta_sg_offset_p ? *meta_sg_offset_p : 0; 2195 for_each_sg(meta_sgl, sg, meta_sg_nents, j) { 2196 if (unlikely(i + j >= mr->max_descs)) 2197 break; 2198 klms[i + j].va = cpu_to_be64(sg_dma_address(sg) + 2199 sg_offset); 2200 klms[i + j].bcount = cpu_to_be32(sg_dma_len(sg) - 2201 sg_offset); 2202 klms[i + j].key = cpu_to_be32(lkey); 2203 mr->ibmr.length += sg_dma_len(sg) - sg_offset; 2204 2205 sg_offset = 0; 2206 } 2207 if (meta_sg_offset_p) 2208 *meta_sg_offset_p = sg_offset; 2209 2210 mr->meta_ndescs = j; 2211 mr->meta_length = mr->ibmr.length - mr->data_length; 2212 } 2213 2214 return i + j; 2215 } 2216 2217 static int mlx5_set_page(struct ib_mr *ibmr, u64 addr) 2218 { 2219 struct mlx5_ib_mr *mr = to_mmr(ibmr); 2220 __be64 *descs; 2221 2222 if (unlikely(mr->ndescs == mr->max_descs)) 2223 return -ENOMEM; 2224 2225 descs = mr->descs; 2226 descs[mr->ndescs++] = cpu_to_be64(addr | MLX5_EN_RD | MLX5_EN_WR); 2227 2228 return 0; 2229 } 2230 2231 static int mlx5_set_page_pi(struct ib_mr *ibmr, u64 addr) 2232 { 2233 struct mlx5_ib_mr *mr = to_mmr(ibmr); 2234 __be64 *descs; 2235 2236 if (unlikely(mr->ndescs + mr->meta_ndescs == mr->max_descs)) 2237 return -ENOMEM; 2238 2239 descs = mr->descs; 2240 descs[mr->ndescs + mr->meta_ndescs++] = 2241 cpu_to_be64(addr | MLX5_EN_RD | MLX5_EN_WR); 2242 2243 return 0; 2244 } 2245 2246 static int 2247 mlx5_ib_map_mtt_mr_sg_pi(struct ib_mr *ibmr, struct scatterlist *data_sg, 2248 int data_sg_nents, unsigned int *data_sg_offset, 2249 struct scatterlist *meta_sg, int meta_sg_nents, 2250 unsigned int *meta_sg_offset) 2251 { 2252 struct mlx5_ib_mr *mr = to_mmr(ibmr); 2253 struct mlx5_ib_mr *pi_mr = mr->mtt_mr; 2254 int n; 2255 2256 pi_mr->ndescs = 0; 2257 pi_mr->meta_ndescs = 0; 2258 pi_mr->meta_length = 0; 2259 2260 ib_dma_sync_single_for_cpu(ibmr->device, pi_mr->desc_map, 2261 pi_mr->desc_size * pi_mr->max_descs, 2262 DMA_TO_DEVICE); 2263 2264 pi_mr->ibmr.page_size = ibmr->page_size; 2265 n = ib_sg_to_pages(&pi_mr->ibmr, data_sg, data_sg_nents, data_sg_offset, 2266 mlx5_set_page); 2267 if (n != data_sg_nents) 2268 return n; 2269 2270 pi_mr->data_iova = pi_mr->ibmr.iova; 2271 pi_mr->data_length = pi_mr->ibmr.length; 2272 pi_mr->ibmr.length = pi_mr->data_length; 2273 ibmr->length = pi_mr->data_length; 2274 2275 if (meta_sg_nents) { 2276 u64 page_mask = ~((u64)ibmr->page_size - 1); 2277 u64 iova = pi_mr->data_iova; 2278 2279 n += ib_sg_to_pages(&pi_mr->ibmr, meta_sg, meta_sg_nents, 2280 meta_sg_offset, mlx5_set_page_pi); 2281 2282 pi_mr->meta_length = pi_mr->ibmr.length; 2283 /* 2284 * PI address for the HW is the offset of the metadata address 2285 * relative to the first data page address. 2286 * It equals to first data page address + size of data pages + 2287 * metadata offset at the first metadata page 2288 */ 2289 pi_mr->pi_iova = (iova & page_mask) + 2290 pi_mr->ndescs * ibmr->page_size + 2291 (pi_mr->ibmr.iova & ~page_mask); 2292 /* 2293 * In order to use one MTT MR for data and metadata, we register 2294 * also the gaps between the end of the data and the start of 2295 * the metadata (the sig MR will verify that the HW will access 2296 * to right addresses). This mapping is safe because we use 2297 * internal mkey for the registration. 2298 */ 2299 pi_mr->ibmr.length = pi_mr->pi_iova + pi_mr->meta_length - iova; 2300 pi_mr->ibmr.iova = iova; 2301 ibmr->length += pi_mr->meta_length; 2302 } 2303 2304 ib_dma_sync_single_for_device(ibmr->device, pi_mr->desc_map, 2305 pi_mr->desc_size * pi_mr->max_descs, 2306 DMA_TO_DEVICE); 2307 2308 return n; 2309 } 2310 2311 static int 2312 mlx5_ib_map_klm_mr_sg_pi(struct ib_mr *ibmr, struct scatterlist *data_sg, 2313 int data_sg_nents, unsigned int *data_sg_offset, 2314 struct scatterlist *meta_sg, int meta_sg_nents, 2315 unsigned int *meta_sg_offset) 2316 { 2317 struct mlx5_ib_mr *mr = to_mmr(ibmr); 2318 struct mlx5_ib_mr *pi_mr = mr->klm_mr; 2319 int n; 2320 2321 pi_mr->ndescs = 0; 2322 pi_mr->meta_ndescs = 0; 2323 pi_mr->meta_length = 0; 2324 2325 ib_dma_sync_single_for_cpu(ibmr->device, pi_mr->desc_map, 2326 pi_mr->desc_size * pi_mr->max_descs, 2327 DMA_TO_DEVICE); 2328 2329 n = mlx5_ib_sg_to_klms(pi_mr, data_sg, data_sg_nents, data_sg_offset, 2330 meta_sg, meta_sg_nents, meta_sg_offset); 2331 2332 ib_dma_sync_single_for_device(ibmr->device, pi_mr->desc_map, 2333 pi_mr->desc_size * pi_mr->max_descs, 2334 DMA_TO_DEVICE); 2335 2336 /* This is zero-based memory region */ 2337 pi_mr->data_iova = 0; 2338 pi_mr->ibmr.iova = 0; 2339 pi_mr->pi_iova = pi_mr->data_length; 2340 ibmr->length = pi_mr->ibmr.length; 2341 2342 return n; 2343 } 2344 2345 int mlx5_ib_map_mr_sg_pi(struct ib_mr *ibmr, struct scatterlist *data_sg, 2346 int data_sg_nents, unsigned int *data_sg_offset, 2347 struct scatterlist *meta_sg, int meta_sg_nents, 2348 unsigned int *meta_sg_offset) 2349 { 2350 struct mlx5_ib_mr *mr = to_mmr(ibmr); 2351 struct mlx5_ib_mr *pi_mr = NULL; 2352 int n; 2353 2354 WARN_ON(ibmr->type != IB_MR_TYPE_INTEGRITY); 2355 2356 mr->ndescs = 0; 2357 mr->data_length = 0; 2358 mr->data_iova = 0; 2359 mr->meta_ndescs = 0; 2360 mr->pi_iova = 0; 2361 /* 2362 * As a performance optimization, if possible, there is no need to 2363 * perform UMR operation to register the data/metadata buffers. 2364 * First try to map the sg lists to PA descriptors with local_dma_lkey. 2365 * Fallback to UMR only in case of a failure. 2366 */ 2367 n = mlx5_ib_map_pa_mr_sg_pi(ibmr, data_sg, data_sg_nents, 2368 data_sg_offset, meta_sg, meta_sg_nents, 2369 meta_sg_offset); 2370 if (n == data_sg_nents + meta_sg_nents) 2371 goto out; 2372 /* 2373 * As a performance optimization, if possible, there is no need to map 2374 * the sg lists to KLM descriptors. First try to map the sg lists to MTT 2375 * descriptors and fallback to KLM only in case of a failure. 2376 * It's more efficient for the HW to work with MTT descriptors 2377 * (especially in high load). 2378 * Use KLM (indirect access) only if it's mandatory. 2379 */ 2380 pi_mr = mr->mtt_mr; 2381 n = mlx5_ib_map_mtt_mr_sg_pi(ibmr, data_sg, data_sg_nents, 2382 data_sg_offset, meta_sg, meta_sg_nents, 2383 meta_sg_offset); 2384 if (n == data_sg_nents + meta_sg_nents) 2385 goto out; 2386 2387 pi_mr = mr->klm_mr; 2388 n = mlx5_ib_map_klm_mr_sg_pi(ibmr, data_sg, data_sg_nents, 2389 data_sg_offset, meta_sg, meta_sg_nents, 2390 meta_sg_offset); 2391 if (unlikely(n != data_sg_nents + meta_sg_nents)) 2392 return -ENOMEM; 2393 2394 out: 2395 /* This is zero-based memory region */ 2396 ibmr->iova = 0; 2397 mr->pi_mr = pi_mr; 2398 if (pi_mr) 2399 ibmr->sig_attrs->meta_length = pi_mr->meta_length; 2400 else 2401 ibmr->sig_attrs->meta_length = mr->meta_length; 2402 2403 return 0; 2404 } 2405 2406 int mlx5_ib_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg, int sg_nents, 2407 unsigned int *sg_offset) 2408 { 2409 struct mlx5_ib_mr *mr = to_mmr(ibmr); 2410 int n; 2411 2412 mr->ndescs = 0; 2413 2414 ib_dma_sync_single_for_cpu(ibmr->device, mr->desc_map, 2415 mr->desc_size * mr->max_descs, 2416 DMA_TO_DEVICE); 2417 2418 if (mr->access_mode == MLX5_MKC_ACCESS_MODE_KLMS) 2419 n = mlx5_ib_sg_to_klms(mr, sg, sg_nents, sg_offset, NULL, 0, 2420 NULL); 2421 else 2422 n = ib_sg_to_pages(ibmr, sg, sg_nents, sg_offset, 2423 mlx5_set_page); 2424 2425 ib_dma_sync_single_for_device(ibmr->device, mr->desc_map, 2426 mr->desc_size * mr->max_descs, 2427 DMA_TO_DEVICE); 2428 2429 return n; 2430 } 2431