1 /* 2 * Copyright (c) 2013-2015, Mellanox Technologies. All rights reserved. 3 * 4 * This software is available to you under a choice of one of two 5 * licenses. You may choose to be licensed under the terms of the GNU 6 * General Public License (GPL) Version 2, available from the file 7 * COPYING in the main directory of this source tree, or the 8 * OpenIB.org BSD license below: 9 * 10 * Redistribution and use in source and binary forms, with or 11 * without modification, are permitted provided that the following 12 * conditions are met: 13 * 14 * - Redistributions of source code must retain the above 15 * copyright notice, this list of conditions and the following 16 * disclaimer. 17 * 18 * - Redistributions in binary form must reproduce the above 19 * copyright notice, this list of conditions and the following 20 * disclaimer in the documentation and/or other materials 21 * provided with the distribution. 22 * 23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30 * SOFTWARE. 31 */ 32 33 34 #include <linux/kref.h> 35 #include <linux/random.h> 36 #include <linux/debugfs.h> 37 #include <linux/export.h> 38 #include <linux/delay.h> 39 #include <rdma/ib_umem.h> 40 #include <rdma/ib_umem_odp.h> 41 #include <rdma/ib_verbs.h> 42 #include "mlx5_ib.h" 43 44 enum { 45 MAX_PENDING_REG_MR = 8, 46 }; 47 48 #define MLX5_UMR_ALIGN 2048 49 50 static void clean_mr(struct mlx5_ib_dev *dev, struct mlx5_ib_mr *mr); 51 static void dereg_mr(struct mlx5_ib_dev *dev, struct mlx5_ib_mr *mr); 52 static int mr_cache_max_order(struct mlx5_ib_dev *dev); 53 static int unreg_umr(struct mlx5_ib_dev *dev, struct mlx5_ib_mr *mr); 54 static bool umr_can_modify_entity_size(struct mlx5_ib_dev *dev) 55 { 56 return !MLX5_CAP_GEN(dev->mdev, umr_modify_entity_size_disabled); 57 } 58 59 static bool umr_can_use_indirect_mkey(struct mlx5_ib_dev *dev) 60 { 61 return !MLX5_CAP_GEN(dev->mdev, umr_indirect_mkey_disabled); 62 } 63 64 static bool use_umr(struct mlx5_ib_dev *dev, int order) 65 { 66 return order <= mr_cache_max_order(dev) && 67 umr_can_modify_entity_size(dev); 68 } 69 70 static int destroy_mkey(struct mlx5_ib_dev *dev, struct mlx5_ib_mr *mr) 71 { 72 int err = mlx5_core_destroy_mkey(dev->mdev, &mr->mmkey); 73 74 if (IS_ENABLED(CONFIG_INFINIBAND_ON_DEMAND_PAGING)) 75 /* Wait until all page fault handlers using the mr complete. */ 76 synchronize_srcu(&dev->mr_srcu); 77 78 return err; 79 } 80 81 static int order2idx(struct mlx5_ib_dev *dev, int order) 82 { 83 struct mlx5_mr_cache *cache = &dev->cache; 84 85 if (order < cache->ent[0].order) 86 return 0; 87 else 88 return order - cache->ent[0].order; 89 } 90 91 static bool use_umr_mtt_update(struct mlx5_ib_mr *mr, u64 start, u64 length) 92 { 93 return ((u64)1 << mr->order) * MLX5_ADAPTER_PAGE_SIZE >= 94 length + (start & (MLX5_ADAPTER_PAGE_SIZE - 1)); 95 } 96 97 static void update_odp_mr(struct mlx5_ib_mr *mr) 98 { 99 if (is_odp_mr(mr)) { 100 /* 101 * This barrier prevents the compiler from moving the 102 * setting of umem->odp_data->private to point to our 103 * MR, before reg_umr finished, to ensure that the MR 104 * initialization have finished before starting to 105 * handle invalidations. 106 */ 107 smp_wmb(); 108 to_ib_umem_odp(mr->umem)->private = mr; 109 /* 110 * Make sure we will see the new 111 * umem->odp_data->private value in the invalidation 112 * routines, before we can get page faults on the 113 * MR. Page faults can happen once we put the MR in 114 * the tree, below this line. Without the barrier, 115 * there can be a fault handling and an invalidation 116 * before umem->odp_data->private == mr is visible to 117 * the invalidation handler. 118 */ 119 smp_wmb(); 120 } 121 } 122 123 static void reg_mr_callback(int status, struct mlx5_async_work *context) 124 { 125 struct mlx5_ib_mr *mr = 126 container_of(context, struct mlx5_ib_mr, cb_work); 127 struct mlx5_ib_dev *dev = mr->dev; 128 struct mlx5_mr_cache *cache = &dev->cache; 129 int c = order2idx(dev, mr->order); 130 struct mlx5_cache_ent *ent = &cache->ent[c]; 131 u8 key; 132 unsigned long flags; 133 struct mlx5_mkey_table *table = &dev->mdev->priv.mkey_table; 134 int err; 135 136 spin_lock_irqsave(&ent->lock, flags); 137 ent->pending--; 138 spin_unlock_irqrestore(&ent->lock, flags); 139 if (status) { 140 mlx5_ib_warn(dev, "async reg mr failed. status %d\n", status); 141 kfree(mr); 142 dev->fill_delay = 1; 143 mod_timer(&dev->delay_timer, jiffies + HZ); 144 return; 145 } 146 147 mr->mmkey.type = MLX5_MKEY_MR; 148 spin_lock_irqsave(&dev->mdev->priv.mkey_lock, flags); 149 key = dev->mdev->priv.mkey_key++; 150 spin_unlock_irqrestore(&dev->mdev->priv.mkey_lock, flags); 151 mr->mmkey.key = mlx5_idx_to_mkey(MLX5_GET(create_mkey_out, mr->out, mkey_index)) | key; 152 153 cache->last_add = jiffies; 154 155 spin_lock_irqsave(&ent->lock, flags); 156 list_add_tail(&mr->list, &ent->head); 157 ent->cur++; 158 ent->size++; 159 spin_unlock_irqrestore(&ent->lock, flags); 160 161 write_lock_irqsave(&table->lock, flags); 162 err = radix_tree_insert(&table->tree, mlx5_base_mkey(mr->mmkey.key), 163 &mr->mmkey); 164 if (err) 165 pr_err("Error inserting to mkey tree. 0x%x\n", -err); 166 write_unlock_irqrestore(&table->lock, flags); 167 168 if (!completion_done(&ent->compl)) 169 complete(&ent->compl); 170 } 171 172 static int add_keys(struct mlx5_ib_dev *dev, int c, int num) 173 { 174 struct mlx5_mr_cache *cache = &dev->cache; 175 struct mlx5_cache_ent *ent = &cache->ent[c]; 176 int inlen = MLX5_ST_SZ_BYTES(create_mkey_in); 177 struct mlx5_ib_mr *mr; 178 void *mkc; 179 u32 *in; 180 int err = 0; 181 int i; 182 183 in = kzalloc(inlen, GFP_KERNEL); 184 if (!in) 185 return -ENOMEM; 186 187 mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry); 188 for (i = 0; i < num; i++) { 189 if (ent->pending >= MAX_PENDING_REG_MR) { 190 err = -EAGAIN; 191 break; 192 } 193 194 mr = kzalloc(sizeof(*mr), GFP_KERNEL); 195 if (!mr) { 196 err = -ENOMEM; 197 break; 198 } 199 mr->order = ent->order; 200 mr->allocated_from_cache = 1; 201 mr->dev = dev; 202 203 MLX5_SET(mkc, mkc, free, 1); 204 MLX5_SET(mkc, mkc, umr_en, 1); 205 MLX5_SET(mkc, mkc, access_mode_1_0, ent->access_mode & 0x3); 206 MLX5_SET(mkc, mkc, access_mode_4_2, 207 (ent->access_mode >> 2) & 0x7); 208 209 MLX5_SET(mkc, mkc, qpn, 0xffffff); 210 MLX5_SET(mkc, mkc, translations_octword_size, ent->xlt); 211 MLX5_SET(mkc, mkc, log_page_size, ent->page); 212 213 spin_lock_irq(&ent->lock); 214 ent->pending++; 215 spin_unlock_irq(&ent->lock); 216 err = mlx5_core_create_mkey_cb(dev->mdev, &mr->mmkey, 217 &dev->async_ctx, in, inlen, 218 mr->out, sizeof(mr->out), 219 reg_mr_callback, &mr->cb_work); 220 if (err) { 221 spin_lock_irq(&ent->lock); 222 ent->pending--; 223 spin_unlock_irq(&ent->lock); 224 mlx5_ib_warn(dev, "create mkey failed %d\n", err); 225 kfree(mr); 226 break; 227 } 228 } 229 230 kfree(in); 231 return err; 232 } 233 234 static void remove_keys(struct mlx5_ib_dev *dev, int c, int num) 235 { 236 struct mlx5_mr_cache *cache = &dev->cache; 237 struct mlx5_cache_ent *ent = &cache->ent[c]; 238 struct mlx5_ib_mr *tmp_mr; 239 struct mlx5_ib_mr *mr; 240 LIST_HEAD(del_list); 241 int i; 242 243 for (i = 0; i < num; i++) { 244 spin_lock_irq(&ent->lock); 245 if (list_empty(&ent->head)) { 246 spin_unlock_irq(&ent->lock); 247 break; 248 } 249 mr = list_first_entry(&ent->head, struct mlx5_ib_mr, list); 250 list_move(&mr->list, &del_list); 251 ent->cur--; 252 ent->size--; 253 spin_unlock_irq(&ent->lock); 254 mlx5_core_destroy_mkey(dev->mdev, &mr->mmkey); 255 } 256 257 if (IS_ENABLED(CONFIG_INFINIBAND_ON_DEMAND_PAGING)) 258 synchronize_srcu(&dev->mr_srcu); 259 260 list_for_each_entry_safe(mr, tmp_mr, &del_list, list) { 261 list_del(&mr->list); 262 kfree(mr); 263 } 264 } 265 266 static ssize_t size_write(struct file *filp, const char __user *buf, 267 size_t count, loff_t *pos) 268 { 269 struct mlx5_cache_ent *ent = filp->private_data; 270 struct mlx5_ib_dev *dev = ent->dev; 271 char lbuf[20] = {0}; 272 u32 var; 273 int err; 274 int c; 275 276 count = min(count, sizeof(lbuf) - 1); 277 if (copy_from_user(lbuf, buf, count)) 278 return -EFAULT; 279 280 c = order2idx(dev, ent->order); 281 282 if (sscanf(lbuf, "%u", &var) != 1) 283 return -EINVAL; 284 285 if (var < ent->limit) 286 return -EINVAL; 287 288 if (var > ent->size) { 289 do { 290 err = add_keys(dev, c, var - ent->size); 291 if (err && err != -EAGAIN) 292 return err; 293 294 usleep_range(3000, 5000); 295 } while (err); 296 } else if (var < ent->size) { 297 remove_keys(dev, c, ent->size - var); 298 } 299 300 return count; 301 } 302 303 static ssize_t size_read(struct file *filp, char __user *buf, size_t count, 304 loff_t *pos) 305 { 306 struct mlx5_cache_ent *ent = filp->private_data; 307 char lbuf[20]; 308 int err; 309 310 err = snprintf(lbuf, sizeof(lbuf), "%d\n", ent->size); 311 if (err < 0) 312 return err; 313 314 return simple_read_from_buffer(buf, count, pos, lbuf, err); 315 } 316 317 static const struct file_operations size_fops = { 318 .owner = THIS_MODULE, 319 .open = simple_open, 320 .write = size_write, 321 .read = size_read, 322 }; 323 324 static ssize_t limit_write(struct file *filp, const char __user *buf, 325 size_t count, loff_t *pos) 326 { 327 struct mlx5_cache_ent *ent = filp->private_data; 328 struct mlx5_ib_dev *dev = ent->dev; 329 char lbuf[20] = {0}; 330 u32 var; 331 int err; 332 int c; 333 334 count = min(count, sizeof(lbuf) - 1); 335 if (copy_from_user(lbuf, buf, count)) 336 return -EFAULT; 337 338 c = order2idx(dev, ent->order); 339 340 if (sscanf(lbuf, "%u", &var) != 1) 341 return -EINVAL; 342 343 if (var > ent->size) 344 return -EINVAL; 345 346 ent->limit = var; 347 348 if (ent->cur < ent->limit) { 349 err = add_keys(dev, c, 2 * ent->limit - ent->cur); 350 if (err) 351 return err; 352 } 353 354 return count; 355 } 356 357 static ssize_t limit_read(struct file *filp, char __user *buf, size_t count, 358 loff_t *pos) 359 { 360 struct mlx5_cache_ent *ent = filp->private_data; 361 char lbuf[20]; 362 int err; 363 364 err = snprintf(lbuf, sizeof(lbuf), "%d\n", ent->limit); 365 if (err < 0) 366 return err; 367 368 return simple_read_from_buffer(buf, count, pos, lbuf, err); 369 } 370 371 static const struct file_operations limit_fops = { 372 .owner = THIS_MODULE, 373 .open = simple_open, 374 .write = limit_write, 375 .read = limit_read, 376 }; 377 378 static int someone_adding(struct mlx5_mr_cache *cache) 379 { 380 int i; 381 382 for (i = 0; i < MAX_MR_CACHE_ENTRIES; i++) { 383 if (cache->ent[i].cur < cache->ent[i].limit) 384 return 1; 385 } 386 387 return 0; 388 } 389 390 static void __cache_work_func(struct mlx5_cache_ent *ent) 391 { 392 struct mlx5_ib_dev *dev = ent->dev; 393 struct mlx5_mr_cache *cache = &dev->cache; 394 int i = order2idx(dev, ent->order); 395 int err; 396 397 if (cache->stopped) 398 return; 399 400 ent = &dev->cache.ent[i]; 401 if (ent->cur < 2 * ent->limit && !dev->fill_delay) { 402 err = add_keys(dev, i, 1); 403 if (ent->cur < 2 * ent->limit) { 404 if (err == -EAGAIN) { 405 mlx5_ib_dbg(dev, "returned eagain, order %d\n", 406 i + 2); 407 queue_delayed_work(cache->wq, &ent->dwork, 408 msecs_to_jiffies(3)); 409 } else if (err) { 410 mlx5_ib_warn(dev, "command failed order %d, err %d\n", 411 i + 2, err); 412 queue_delayed_work(cache->wq, &ent->dwork, 413 msecs_to_jiffies(1000)); 414 } else { 415 queue_work(cache->wq, &ent->work); 416 } 417 } 418 } else if (ent->cur > 2 * ent->limit) { 419 /* 420 * The remove_keys() logic is performed as garbage collection 421 * task. Such task is intended to be run when no other active 422 * processes are running. 423 * 424 * The need_resched() will return TRUE if there are user tasks 425 * to be activated in near future. 426 * 427 * In such case, we don't execute remove_keys() and postpone 428 * the garbage collection work to try to run in next cycle, 429 * in order to free CPU resources to other tasks. 430 */ 431 if (!need_resched() && !someone_adding(cache) && 432 time_after(jiffies, cache->last_add + 300 * HZ)) { 433 remove_keys(dev, i, 1); 434 if (ent->cur > ent->limit) 435 queue_work(cache->wq, &ent->work); 436 } else { 437 queue_delayed_work(cache->wq, &ent->dwork, 300 * HZ); 438 } 439 } 440 } 441 442 static void delayed_cache_work_func(struct work_struct *work) 443 { 444 struct mlx5_cache_ent *ent; 445 446 ent = container_of(work, struct mlx5_cache_ent, dwork.work); 447 __cache_work_func(ent); 448 } 449 450 static void cache_work_func(struct work_struct *work) 451 { 452 struct mlx5_cache_ent *ent; 453 454 ent = container_of(work, struct mlx5_cache_ent, work); 455 __cache_work_func(ent); 456 } 457 458 struct mlx5_ib_mr *mlx5_mr_cache_alloc(struct mlx5_ib_dev *dev, int entry) 459 { 460 struct mlx5_mr_cache *cache = &dev->cache; 461 struct mlx5_cache_ent *ent; 462 struct mlx5_ib_mr *mr; 463 int err; 464 465 if (entry < 0 || entry >= MAX_MR_CACHE_ENTRIES) { 466 mlx5_ib_err(dev, "cache entry %d is out of range\n", entry); 467 return NULL; 468 } 469 470 ent = &cache->ent[entry]; 471 while (1) { 472 spin_lock_irq(&ent->lock); 473 if (list_empty(&ent->head)) { 474 spin_unlock_irq(&ent->lock); 475 476 err = add_keys(dev, entry, 1); 477 if (err && err != -EAGAIN) 478 return ERR_PTR(err); 479 480 wait_for_completion(&ent->compl); 481 } else { 482 mr = list_first_entry(&ent->head, struct mlx5_ib_mr, 483 list); 484 list_del(&mr->list); 485 ent->cur--; 486 spin_unlock_irq(&ent->lock); 487 if (ent->cur < ent->limit) 488 queue_work(cache->wq, &ent->work); 489 return mr; 490 } 491 } 492 } 493 494 static struct mlx5_ib_mr *alloc_cached_mr(struct mlx5_ib_dev *dev, int order) 495 { 496 struct mlx5_mr_cache *cache = &dev->cache; 497 struct mlx5_ib_mr *mr = NULL; 498 struct mlx5_cache_ent *ent; 499 int last_umr_cache_entry; 500 int c; 501 int i; 502 503 c = order2idx(dev, order); 504 last_umr_cache_entry = order2idx(dev, mr_cache_max_order(dev)); 505 if (c < 0 || c > last_umr_cache_entry) { 506 mlx5_ib_warn(dev, "order %d, cache index %d\n", order, c); 507 return NULL; 508 } 509 510 for (i = c; i <= last_umr_cache_entry; i++) { 511 ent = &cache->ent[i]; 512 513 mlx5_ib_dbg(dev, "order %d, cache index %d\n", ent->order, i); 514 515 spin_lock_irq(&ent->lock); 516 if (!list_empty(&ent->head)) { 517 mr = list_first_entry(&ent->head, struct mlx5_ib_mr, 518 list); 519 list_del(&mr->list); 520 ent->cur--; 521 spin_unlock_irq(&ent->lock); 522 if (ent->cur < ent->limit) 523 queue_work(cache->wq, &ent->work); 524 break; 525 } 526 spin_unlock_irq(&ent->lock); 527 528 queue_work(cache->wq, &ent->work); 529 } 530 531 if (!mr) 532 cache->ent[c].miss++; 533 534 return mr; 535 } 536 537 void mlx5_mr_cache_free(struct mlx5_ib_dev *dev, struct mlx5_ib_mr *mr) 538 { 539 struct mlx5_mr_cache *cache = &dev->cache; 540 struct mlx5_cache_ent *ent; 541 int shrink = 0; 542 int c; 543 544 if (!mr->allocated_from_cache) 545 return; 546 547 c = order2idx(dev, mr->order); 548 if (c < 0 || c >= MAX_MR_CACHE_ENTRIES) { 549 mlx5_ib_warn(dev, "order %d, cache index %d\n", mr->order, c); 550 return; 551 } 552 553 if (unreg_umr(dev, mr)) 554 return; 555 556 ent = &cache->ent[c]; 557 spin_lock_irq(&ent->lock); 558 list_add_tail(&mr->list, &ent->head); 559 ent->cur++; 560 if (ent->cur > 2 * ent->limit) 561 shrink = 1; 562 spin_unlock_irq(&ent->lock); 563 564 if (shrink) 565 queue_work(cache->wq, &ent->work); 566 } 567 568 static void clean_keys(struct mlx5_ib_dev *dev, int c) 569 { 570 struct mlx5_mr_cache *cache = &dev->cache; 571 struct mlx5_cache_ent *ent = &cache->ent[c]; 572 struct mlx5_ib_mr *tmp_mr; 573 struct mlx5_ib_mr *mr; 574 LIST_HEAD(del_list); 575 576 cancel_delayed_work(&ent->dwork); 577 while (1) { 578 spin_lock_irq(&ent->lock); 579 if (list_empty(&ent->head)) { 580 spin_unlock_irq(&ent->lock); 581 break; 582 } 583 mr = list_first_entry(&ent->head, struct mlx5_ib_mr, list); 584 list_move(&mr->list, &del_list); 585 ent->cur--; 586 ent->size--; 587 spin_unlock_irq(&ent->lock); 588 mlx5_core_destroy_mkey(dev->mdev, &mr->mmkey); 589 } 590 591 #ifdef CONFIG_INFINIBAND_ON_DEMAND_PAGING 592 synchronize_srcu(&dev->mr_srcu); 593 #endif 594 595 list_for_each_entry_safe(mr, tmp_mr, &del_list, list) { 596 list_del(&mr->list); 597 kfree(mr); 598 } 599 } 600 601 static void mlx5_mr_cache_debugfs_cleanup(struct mlx5_ib_dev *dev) 602 { 603 if (!mlx5_debugfs_root || dev->rep) 604 return; 605 606 debugfs_remove_recursive(dev->cache.root); 607 dev->cache.root = NULL; 608 } 609 610 static void mlx5_mr_cache_debugfs_init(struct mlx5_ib_dev *dev) 611 { 612 struct mlx5_mr_cache *cache = &dev->cache; 613 struct mlx5_cache_ent *ent; 614 struct dentry *dir; 615 int i; 616 617 if (!mlx5_debugfs_root || dev->rep) 618 return; 619 620 cache->root = debugfs_create_dir("mr_cache", dev->mdev->priv.dbg_root); 621 622 for (i = 0; i < MAX_MR_CACHE_ENTRIES; i++) { 623 ent = &cache->ent[i]; 624 sprintf(ent->name, "%d", ent->order); 625 dir = debugfs_create_dir(ent->name, cache->root); 626 debugfs_create_file("size", 0600, dir, ent, &size_fops); 627 debugfs_create_file("limit", 0600, dir, ent, &limit_fops); 628 debugfs_create_u32("cur", 0400, dir, &ent->cur); 629 debugfs_create_u32("miss", 0600, dir, &ent->miss); 630 } 631 } 632 633 static void delay_time_func(struct timer_list *t) 634 { 635 struct mlx5_ib_dev *dev = from_timer(dev, t, delay_timer); 636 637 dev->fill_delay = 0; 638 } 639 640 int mlx5_mr_cache_init(struct mlx5_ib_dev *dev) 641 { 642 struct mlx5_mr_cache *cache = &dev->cache; 643 struct mlx5_cache_ent *ent; 644 int i; 645 646 mutex_init(&dev->slow_path_mutex); 647 cache->wq = alloc_ordered_workqueue("mkey_cache", WQ_MEM_RECLAIM); 648 if (!cache->wq) { 649 mlx5_ib_warn(dev, "failed to create work queue\n"); 650 return -ENOMEM; 651 } 652 653 mlx5_cmd_init_async_ctx(dev->mdev, &dev->async_ctx); 654 timer_setup(&dev->delay_timer, delay_time_func, 0); 655 for (i = 0; i < MAX_MR_CACHE_ENTRIES; i++) { 656 ent = &cache->ent[i]; 657 INIT_LIST_HEAD(&ent->head); 658 spin_lock_init(&ent->lock); 659 ent->order = i + 2; 660 ent->dev = dev; 661 ent->limit = 0; 662 663 init_completion(&ent->compl); 664 INIT_WORK(&ent->work, cache_work_func); 665 INIT_DELAYED_WORK(&ent->dwork, delayed_cache_work_func); 666 667 if (i > MR_CACHE_LAST_STD_ENTRY) { 668 mlx5_odp_init_mr_cache_entry(ent); 669 continue; 670 } 671 672 if (ent->order > mr_cache_max_order(dev)) 673 continue; 674 675 ent->page = PAGE_SHIFT; 676 ent->xlt = (1 << ent->order) * sizeof(struct mlx5_mtt) / 677 MLX5_IB_UMR_OCTOWORD; 678 ent->access_mode = MLX5_MKC_ACCESS_MODE_MTT; 679 if ((dev->mdev->profile->mask & MLX5_PROF_MASK_MR_CACHE) && 680 !dev->rep && 681 mlx5_core_is_pf(dev->mdev)) 682 ent->limit = dev->mdev->profile->mr_cache[i].limit; 683 else 684 ent->limit = 0; 685 queue_work(cache->wq, &ent->work); 686 } 687 688 mlx5_mr_cache_debugfs_init(dev); 689 690 return 0; 691 } 692 693 int mlx5_mr_cache_cleanup(struct mlx5_ib_dev *dev) 694 { 695 int i; 696 697 if (!dev->cache.wq) 698 return 0; 699 700 dev->cache.stopped = 1; 701 flush_workqueue(dev->cache.wq); 702 703 mlx5_mr_cache_debugfs_cleanup(dev); 704 mlx5_cmd_cleanup_async_ctx(&dev->async_ctx); 705 706 for (i = 0; i < MAX_MR_CACHE_ENTRIES; i++) 707 clean_keys(dev, i); 708 709 destroy_workqueue(dev->cache.wq); 710 del_timer_sync(&dev->delay_timer); 711 712 return 0; 713 } 714 715 struct ib_mr *mlx5_ib_get_dma_mr(struct ib_pd *pd, int acc) 716 { 717 struct mlx5_ib_dev *dev = to_mdev(pd->device); 718 int inlen = MLX5_ST_SZ_BYTES(create_mkey_in); 719 struct mlx5_core_dev *mdev = dev->mdev; 720 struct mlx5_ib_mr *mr; 721 void *mkc; 722 u32 *in; 723 int err; 724 725 mr = kzalloc(sizeof(*mr), GFP_KERNEL); 726 if (!mr) 727 return ERR_PTR(-ENOMEM); 728 729 in = kzalloc(inlen, GFP_KERNEL); 730 if (!in) { 731 err = -ENOMEM; 732 goto err_free; 733 } 734 735 mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry); 736 737 MLX5_SET(mkc, mkc, access_mode_1_0, MLX5_MKC_ACCESS_MODE_PA); 738 MLX5_SET(mkc, mkc, a, !!(acc & IB_ACCESS_REMOTE_ATOMIC)); 739 MLX5_SET(mkc, mkc, rw, !!(acc & IB_ACCESS_REMOTE_WRITE)); 740 MLX5_SET(mkc, mkc, rr, !!(acc & IB_ACCESS_REMOTE_READ)); 741 MLX5_SET(mkc, mkc, lw, !!(acc & IB_ACCESS_LOCAL_WRITE)); 742 MLX5_SET(mkc, mkc, lr, 1); 743 744 MLX5_SET(mkc, mkc, length64, 1); 745 MLX5_SET(mkc, mkc, pd, to_mpd(pd)->pdn); 746 MLX5_SET(mkc, mkc, qpn, 0xffffff); 747 MLX5_SET64(mkc, mkc, start_addr, 0); 748 749 err = mlx5_core_create_mkey(mdev, &mr->mmkey, in, inlen); 750 if (err) 751 goto err_in; 752 753 kfree(in); 754 mr->mmkey.type = MLX5_MKEY_MR; 755 mr->ibmr.lkey = mr->mmkey.key; 756 mr->ibmr.rkey = mr->mmkey.key; 757 mr->umem = NULL; 758 759 return &mr->ibmr; 760 761 err_in: 762 kfree(in); 763 764 err_free: 765 kfree(mr); 766 767 return ERR_PTR(err); 768 } 769 770 static int get_octo_len(u64 addr, u64 len, int page_shift) 771 { 772 u64 page_size = 1ULL << page_shift; 773 u64 offset; 774 int npages; 775 776 offset = addr & (page_size - 1); 777 npages = ALIGN(len + offset, page_size) >> page_shift; 778 return (npages + 1) / 2; 779 } 780 781 static int mr_cache_max_order(struct mlx5_ib_dev *dev) 782 { 783 if (MLX5_CAP_GEN(dev->mdev, umr_extended_translation_offset)) 784 return MR_CACHE_LAST_STD_ENTRY + 2; 785 return MLX5_MAX_UMR_SHIFT; 786 } 787 788 static int mr_umem_get(struct mlx5_ib_dev *dev, struct ib_udata *udata, 789 u64 start, u64 length, int access_flags, 790 struct ib_umem **umem, int *npages, int *page_shift, 791 int *ncont, int *order) 792 { 793 struct ib_umem *u; 794 int err; 795 796 *umem = NULL; 797 798 u = ib_umem_get(udata, start, length, access_flags, 0); 799 err = PTR_ERR_OR_ZERO(u); 800 if (err) { 801 mlx5_ib_dbg(dev, "umem get failed (%d)\n", err); 802 return err; 803 } 804 805 mlx5_ib_cont_pages(u, start, MLX5_MKEY_PAGE_SHIFT_MASK, npages, 806 page_shift, ncont, order); 807 if (!*npages) { 808 mlx5_ib_warn(dev, "avoid zero region\n"); 809 ib_umem_release(u); 810 return -EINVAL; 811 } 812 813 *umem = u; 814 815 mlx5_ib_dbg(dev, "npages %d, ncont %d, order %d, page_shift %d\n", 816 *npages, *ncont, *order, *page_shift); 817 818 return 0; 819 } 820 821 static void mlx5_ib_umr_done(struct ib_cq *cq, struct ib_wc *wc) 822 { 823 struct mlx5_ib_umr_context *context = 824 container_of(wc->wr_cqe, struct mlx5_ib_umr_context, cqe); 825 826 context->status = wc->status; 827 complete(&context->done); 828 } 829 830 static inline void mlx5_ib_init_umr_context(struct mlx5_ib_umr_context *context) 831 { 832 context->cqe.done = mlx5_ib_umr_done; 833 context->status = -1; 834 init_completion(&context->done); 835 } 836 837 static int mlx5_ib_post_send_wait(struct mlx5_ib_dev *dev, 838 struct mlx5_umr_wr *umrwr) 839 { 840 struct umr_common *umrc = &dev->umrc; 841 const struct ib_send_wr *bad; 842 int err; 843 struct mlx5_ib_umr_context umr_context; 844 845 mlx5_ib_init_umr_context(&umr_context); 846 umrwr->wr.wr_cqe = &umr_context.cqe; 847 848 down(&umrc->sem); 849 err = ib_post_send(umrc->qp, &umrwr->wr, &bad); 850 if (err) { 851 mlx5_ib_warn(dev, "UMR post send failed, err %d\n", err); 852 } else { 853 wait_for_completion(&umr_context.done); 854 if (umr_context.status != IB_WC_SUCCESS) { 855 mlx5_ib_warn(dev, "reg umr failed (%u)\n", 856 umr_context.status); 857 err = -EFAULT; 858 } 859 } 860 up(&umrc->sem); 861 return err; 862 } 863 864 static struct mlx5_ib_mr *alloc_mr_from_cache( 865 struct ib_pd *pd, struct ib_umem *umem, 866 u64 virt_addr, u64 len, int npages, 867 int page_shift, int order, int access_flags) 868 { 869 struct mlx5_ib_dev *dev = to_mdev(pd->device); 870 struct mlx5_ib_mr *mr; 871 int err = 0; 872 int i; 873 874 for (i = 0; i < 1; i++) { 875 mr = alloc_cached_mr(dev, order); 876 if (mr) 877 break; 878 879 err = add_keys(dev, order2idx(dev, order), 1); 880 if (err && err != -EAGAIN) { 881 mlx5_ib_warn(dev, "add_keys failed, err %d\n", err); 882 break; 883 } 884 } 885 886 if (!mr) 887 return ERR_PTR(-EAGAIN); 888 889 mr->ibmr.pd = pd; 890 mr->umem = umem; 891 mr->access_flags = access_flags; 892 mr->desc_size = sizeof(struct mlx5_mtt); 893 mr->mmkey.iova = virt_addr; 894 mr->mmkey.size = len; 895 mr->mmkey.pd = to_mpd(pd)->pdn; 896 897 return mr; 898 } 899 900 static inline int populate_xlt(struct mlx5_ib_mr *mr, int idx, int npages, 901 void *xlt, int page_shift, size_t size, 902 int flags) 903 { 904 struct mlx5_ib_dev *dev = mr->dev; 905 struct ib_umem *umem = mr->umem; 906 907 if (flags & MLX5_IB_UPD_XLT_INDIRECT) { 908 if (!umr_can_use_indirect_mkey(dev)) 909 return -EPERM; 910 mlx5_odp_populate_klm(xlt, idx, npages, mr, flags); 911 return npages; 912 } 913 914 npages = min_t(size_t, npages, ib_umem_num_pages(umem) - idx); 915 916 if (!(flags & MLX5_IB_UPD_XLT_ZAP)) { 917 __mlx5_ib_populate_pas(dev, umem, page_shift, 918 idx, npages, xlt, 919 MLX5_IB_MTT_PRESENT); 920 /* Clear padding after the pages 921 * brought from the umem. 922 */ 923 memset(xlt + (npages * sizeof(struct mlx5_mtt)), 0, 924 size - npages * sizeof(struct mlx5_mtt)); 925 } 926 927 return npages; 928 } 929 930 #define MLX5_MAX_UMR_CHUNK ((1 << (MLX5_MAX_UMR_SHIFT + 4)) - \ 931 MLX5_UMR_MTT_ALIGNMENT) 932 #define MLX5_SPARE_UMR_CHUNK 0x10000 933 934 int mlx5_ib_update_xlt(struct mlx5_ib_mr *mr, u64 idx, int npages, 935 int page_shift, int flags) 936 { 937 struct mlx5_ib_dev *dev = mr->dev; 938 struct device *ddev = dev->ib_dev.dev.parent; 939 int size; 940 void *xlt; 941 dma_addr_t dma; 942 struct mlx5_umr_wr wr; 943 struct ib_sge sg; 944 int err = 0; 945 int desc_size = (flags & MLX5_IB_UPD_XLT_INDIRECT) 946 ? sizeof(struct mlx5_klm) 947 : sizeof(struct mlx5_mtt); 948 const int page_align = MLX5_UMR_MTT_ALIGNMENT / desc_size; 949 const int page_mask = page_align - 1; 950 size_t pages_mapped = 0; 951 size_t pages_to_map = 0; 952 size_t pages_iter = 0; 953 gfp_t gfp; 954 bool use_emergency_page = false; 955 956 if ((flags & MLX5_IB_UPD_XLT_INDIRECT) && 957 !umr_can_use_indirect_mkey(dev)) 958 return -EPERM; 959 960 /* UMR copies MTTs in units of MLX5_UMR_MTT_ALIGNMENT bytes, 961 * so we need to align the offset and length accordingly 962 */ 963 if (idx & page_mask) { 964 npages += idx & page_mask; 965 idx &= ~page_mask; 966 } 967 968 gfp = flags & MLX5_IB_UPD_XLT_ATOMIC ? GFP_ATOMIC : GFP_KERNEL; 969 gfp |= __GFP_ZERO | __GFP_NOWARN; 970 971 pages_to_map = ALIGN(npages, page_align); 972 size = desc_size * pages_to_map; 973 size = min_t(int, size, MLX5_MAX_UMR_CHUNK); 974 975 xlt = (void *)__get_free_pages(gfp, get_order(size)); 976 if (!xlt && size > MLX5_SPARE_UMR_CHUNK) { 977 mlx5_ib_dbg(dev, "Failed to allocate %d bytes of order %d. fallback to spare UMR allocation od %d bytes\n", 978 size, get_order(size), MLX5_SPARE_UMR_CHUNK); 979 980 size = MLX5_SPARE_UMR_CHUNK; 981 xlt = (void *)__get_free_pages(gfp, get_order(size)); 982 } 983 984 if (!xlt) { 985 mlx5_ib_warn(dev, "Using XLT emergency buffer\n"); 986 xlt = (void *)mlx5_ib_get_xlt_emergency_page(); 987 size = PAGE_SIZE; 988 memset(xlt, 0, size); 989 use_emergency_page = true; 990 } 991 pages_iter = size / desc_size; 992 dma = dma_map_single(ddev, xlt, size, DMA_TO_DEVICE); 993 if (dma_mapping_error(ddev, dma)) { 994 mlx5_ib_err(dev, "unable to map DMA during XLT update.\n"); 995 err = -ENOMEM; 996 goto free_xlt; 997 } 998 999 sg.addr = dma; 1000 sg.lkey = dev->umrc.pd->local_dma_lkey; 1001 1002 memset(&wr, 0, sizeof(wr)); 1003 wr.wr.send_flags = MLX5_IB_SEND_UMR_UPDATE_XLT; 1004 if (!(flags & MLX5_IB_UPD_XLT_ENABLE)) 1005 wr.wr.send_flags |= MLX5_IB_SEND_UMR_FAIL_IF_FREE; 1006 wr.wr.sg_list = &sg; 1007 wr.wr.num_sge = 1; 1008 wr.wr.opcode = MLX5_IB_WR_UMR; 1009 1010 wr.pd = mr->ibmr.pd; 1011 wr.mkey = mr->mmkey.key; 1012 wr.length = mr->mmkey.size; 1013 wr.virt_addr = mr->mmkey.iova; 1014 wr.access_flags = mr->access_flags; 1015 wr.page_shift = page_shift; 1016 1017 for (pages_mapped = 0; 1018 pages_mapped < pages_to_map && !err; 1019 pages_mapped += pages_iter, idx += pages_iter) { 1020 npages = min_t(int, pages_iter, pages_to_map - pages_mapped); 1021 dma_sync_single_for_cpu(ddev, dma, size, DMA_TO_DEVICE); 1022 npages = populate_xlt(mr, idx, npages, xlt, 1023 page_shift, size, flags); 1024 1025 dma_sync_single_for_device(ddev, dma, size, DMA_TO_DEVICE); 1026 1027 sg.length = ALIGN(npages * desc_size, 1028 MLX5_UMR_MTT_ALIGNMENT); 1029 1030 if (pages_mapped + pages_iter >= pages_to_map) { 1031 if (flags & MLX5_IB_UPD_XLT_ENABLE) 1032 wr.wr.send_flags |= 1033 MLX5_IB_SEND_UMR_ENABLE_MR | 1034 MLX5_IB_SEND_UMR_UPDATE_PD_ACCESS | 1035 MLX5_IB_SEND_UMR_UPDATE_TRANSLATION; 1036 if (flags & MLX5_IB_UPD_XLT_PD || 1037 flags & MLX5_IB_UPD_XLT_ACCESS) 1038 wr.wr.send_flags |= 1039 MLX5_IB_SEND_UMR_UPDATE_PD_ACCESS; 1040 if (flags & MLX5_IB_UPD_XLT_ADDR) 1041 wr.wr.send_flags |= 1042 MLX5_IB_SEND_UMR_UPDATE_TRANSLATION; 1043 } 1044 1045 wr.offset = idx * desc_size; 1046 wr.xlt_size = sg.length; 1047 1048 err = mlx5_ib_post_send_wait(dev, &wr); 1049 } 1050 dma_unmap_single(ddev, dma, size, DMA_TO_DEVICE); 1051 1052 free_xlt: 1053 if (use_emergency_page) 1054 mlx5_ib_put_xlt_emergency_page(); 1055 else 1056 free_pages((unsigned long)xlt, get_order(size)); 1057 1058 return err; 1059 } 1060 1061 /* 1062 * If ibmr is NULL it will be allocated by reg_create. 1063 * Else, the given ibmr will be used. 1064 */ 1065 static struct mlx5_ib_mr *reg_create(struct ib_mr *ibmr, struct ib_pd *pd, 1066 u64 virt_addr, u64 length, 1067 struct ib_umem *umem, int npages, 1068 int page_shift, int access_flags, 1069 bool populate) 1070 { 1071 struct mlx5_ib_dev *dev = to_mdev(pd->device); 1072 struct mlx5_ib_mr *mr; 1073 __be64 *pas; 1074 void *mkc; 1075 int inlen; 1076 u32 *in; 1077 int err; 1078 bool pg_cap = !!(MLX5_CAP_GEN(dev->mdev, pg)); 1079 1080 mr = ibmr ? to_mmr(ibmr) : kzalloc(sizeof(*mr), GFP_KERNEL); 1081 if (!mr) 1082 return ERR_PTR(-ENOMEM); 1083 1084 mr->ibmr.pd = pd; 1085 mr->access_flags = access_flags; 1086 1087 inlen = MLX5_ST_SZ_BYTES(create_mkey_in); 1088 if (populate) 1089 inlen += sizeof(*pas) * roundup(npages, 2); 1090 in = kvzalloc(inlen, GFP_KERNEL); 1091 if (!in) { 1092 err = -ENOMEM; 1093 goto err_1; 1094 } 1095 pas = (__be64 *)MLX5_ADDR_OF(create_mkey_in, in, klm_pas_mtt); 1096 if (populate && !(access_flags & IB_ACCESS_ON_DEMAND)) 1097 mlx5_ib_populate_pas(dev, umem, page_shift, pas, 1098 pg_cap ? MLX5_IB_MTT_PRESENT : 0); 1099 1100 /* The pg_access bit allows setting the access flags 1101 * in the page list submitted with the command. */ 1102 MLX5_SET(create_mkey_in, in, pg_access, !!(pg_cap)); 1103 1104 mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry); 1105 MLX5_SET(mkc, mkc, free, !populate); 1106 MLX5_SET(mkc, mkc, access_mode_1_0, MLX5_MKC_ACCESS_MODE_MTT); 1107 MLX5_SET(mkc, mkc, a, !!(access_flags & IB_ACCESS_REMOTE_ATOMIC)); 1108 MLX5_SET(mkc, mkc, rw, !!(access_flags & IB_ACCESS_REMOTE_WRITE)); 1109 MLX5_SET(mkc, mkc, rr, !!(access_flags & IB_ACCESS_REMOTE_READ)); 1110 MLX5_SET(mkc, mkc, lw, !!(access_flags & IB_ACCESS_LOCAL_WRITE)); 1111 MLX5_SET(mkc, mkc, lr, 1); 1112 MLX5_SET(mkc, mkc, umr_en, 1); 1113 1114 MLX5_SET64(mkc, mkc, start_addr, virt_addr); 1115 MLX5_SET64(mkc, mkc, len, length); 1116 MLX5_SET(mkc, mkc, pd, to_mpd(pd)->pdn); 1117 MLX5_SET(mkc, mkc, bsf_octword_size, 0); 1118 MLX5_SET(mkc, mkc, translations_octword_size, 1119 get_octo_len(virt_addr, length, page_shift)); 1120 MLX5_SET(mkc, mkc, log_page_size, page_shift); 1121 MLX5_SET(mkc, mkc, qpn, 0xffffff); 1122 if (populate) { 1123 MLX5_SET(create_mkey_in, in, translations_octword_actual_size, 1124 get_octo_len(virt_addr, length, page_shift)); 1125 } 1126 1127 err = mlx5_core_create_mkey(dev->mdev, &mr->mmkey, in, inlen); 1128 if (err) { 1129 mlx5_ib_warn(dev, "create mkey failed\n"); 1130 goto err_2; 1131 } 1132 mr->mmkey.type = MLX5_MKEY_MR; 1133 mr->desc_size = sizeof(struct mlx5_mtt); 1134 mr->dev = dev; 1135 kvfree(in); 1136 1137 mlx5_ib_dbg(dev, "mkey = 0x%x\n", mr->mmkey.key); 1138 1139 return mr; 1140 1141 err_2: 1142 kvfree(in); 1143 1144 err_1: 1145 if (!ibmr) 1146 kfree(mr); 1147 1148 return ERR_PTR(err); 1149 } 1150 1151 static void set_mr_fields(struct mlx5_ib_dev *dev, struct mlx5_ib_mr *mr, 1152 int npages, u64 length, int access_flags) 1153 { 1154 mr->npages = npages; 1155 atomic_add(npages, &dev->mdev->priv.reg_pages); 1156 mr->ibmr.lkey = mr->mmkey.key; 1157 mr->ibmr.rkey = mr->mmkey.key; 1158 mr->ibmr.length = length; 1159 mr->access_flags = access_flags; 1160 } 1161 1162 static struct ib_mr *mlx5_ib_get_memic_mr(struct ib_pd *pd, u64 memic_addr, 1163 u64 length, int acc) 1164 { 1165 struct mlx5_ib_dev *dev = to_mdev(pd->device); 1166 int inlen = MLX5_ST_SZ_BYTES(create_mkey_in); 1167 struct mlx5_core_dev *mdev = dev->mdev; 1168 struct mlx5_ib_mr *mr; 1169 void *mkc; 1170 u32 *in; 1171 int err; 1172 1173 mr = kzalloc(sizeof(*mr), GFP_KERNEL); 1174 if (!mr) 1175 return ERR_PTR(-ENOMEM); 1176 1177 in = kzalloc(inlen, GFP_KERNEL); 1178 if (!in) { 1179 err = -ENOMEM; 1180 goto err_free; 1181 } 1182 1183 mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry); 1184 1185 MLX5_SET(mkc, mkc, access_mode_1_0, MLX5_MKC_ACCESS_MODE_MEMIC & 0x3); 1186 MLX5_SET(mkc, mkc, access_mode_4_2, 1187 (MLX5_MKC_ACCESS_MODE_MEMIC >> 2) & 0x7); 1188 MLX5_SET(mkc, mkc, a, !!(acc & IB_ACCESS_REMOTE_ATOMIC)); 1189 MLX5_SET(mkc, mkc, rw, !!(acc & IB_ACCESS_REMOTE_WRITE)); 1190 MLX5_SET(mkc, mkc, rr, !!(acc & IB_ACCESS_REMOTE_READ)); 1191 MLX5_SET(mkc, mkc, lw, !!(acc & IB_ACCESS_LOCAL_WRITE)); 1192 MLX5_SET(mkc, mkc, lr, 1); 1193 1194 MLX5_SET64(mkc, mkc, len, length); 1195 MLX5_SET(mkc, mkc, pd, to_mpd(pd)->pdn); 1196 MLX5_SET(mkc, mkc, qpn, 0xffffff); 1197 MLX5_SET64(mkc, mkc, start_addr, 1198 memic_addr - pci_resource_start(dev->mdev->pdev, 0)); 1199 1200 err = mlx5_core_create_mkey(mdev, &mr->mmkey, in, inlen); 1201 if (err) 1202 goto err_in; 1203 1204 kfree(in); 1205 1206 mr->umem = NULL; 1207 set_mr_fields(dev, mr, 0, length, acc); 1208 1209 return &mr->ibmr; 1210 1211 err_in: 1212 kfree(in); 1213 1214 err_free: 1215 kfree(mr); 1216 1217 return ERR_PTR(err); 1218 } 1219 1220 int mlx5_ib_advise_mr(struct ib_pd *pd, 1221 enum ib_uverbs_advise_mr_advice advice, 1222 u32 flags, 1223 struct ib_sge *sg_list, 1224 u32 num_sge, 1225 struct uverbs_attr_bundle *attrs) 1226 { 1227 if (advice != IB_UVERBS_ADVISE_MR_ADVICE_PREFETCH && 1228 advice != IB_UVERBS_ADVISE_MR_ADVICE_PREFETCH_WRITE) 1229 return -EOPNOTSUPP; 1230 1231 return mlx5_ib_advise_mr_prefetch(pd, advice, flags, 1232 sg_list, num_sge); 1233 } 1234 1235 struct ib_mr *mlx5_ib_reg_dm_mr(struct ib_pd *pd, struct ib_dm *dm, 1236 struct ib_dm_mr_attr *attr, 1237 struct uverbs_attr_bundle *attrs) 1238 { 1239 struct mlx5_ib_dm *mdm = to_mdm(dm); 1240 u64 memic_addr; 1241 1242 if (attr->access_flags & ~MLX5_IB_DM_ALLOWED_ACCESS) 1243 return ERR_PTR(-EINVAL); 1244 1245 memic_addr = mdm->dev_addr + attr->offset; 1246 1247 return mlx5_ib_get_memic_mr(pd, memic_addr, attr->length, 1248 attr->access_flags); 1249 } 1250 1251 struct ib_mr *mlx5_ib_reg_user_mr(struct ib_pd *pd, u64 start, u64 length, 1252 u64 virt_addr, int access_flags, 1253 struct ib_udata *udata) 1254 { 1255 struct mlx5_ib_dev *dev = to_mdev(pd->device); 1256 struct mlx5_ib_mr *mr = NULL; 1257 bool populate_mtts = false; 1258 struct ib_umem *umem; 1259 int page_shift; 1260 int npages; 1261 int ncont; 1262 int order; 1263 int err; 1264 1265 if (!IS_ENABLED(CONFIG_INFINIBAND_USER_MEM)) 1266 return ERR_PTR(-EOPNOTSUPP); 1267 1268 mlx5_ib_dbg(dev, "start 0x%llx, virt_addr 0x%llx, length 0x%llx, access_flags 0x%x\n", 1269 start, virt_addr, length, access_flags); 1270 1271 if (IS_ENABLED(CONFIG_INFINIBAND_ON_DEMAND_PAGING) && !start && 1272 length == U64_MAX) { 1273 if (!(access_flags & IB_ACCESS_ON_DEMAND) || 1274 !(dev->odp_caps.general_caps & IB_ODP_SUPPORT_IMPLICIT)) 1275 return ERR_PTR(-EINVAL); 1276 1277 mr = mlx5_ib_alloc_implicit_mr(to_mpd(pd), udata, access_flags); 1278 if (IS_ERR(mr)) 1279 return ERR_CAST(mr); 1280 return &mr->ibmr; 1281 } 1282 1283 err = mr_umem_get(dev, udata, start, length, access_flags, &umem, 1284 &npages, &page_shift, &ncont, &order); 1285 1286 if (err < 0) 1287 return ERR_PTR(err); 1288 1289 if (use_umr(dev, order)) { 1290 mr = alloc_mr_from_cache(pd, umem, virt_addr, length, ncont, 1291 page_shift, order, access_flags); 1292 if (PTR_ERR(mr) == -EAGAIN) { 1293 mlx5_ib_dbg(dev, "cache empty for order %d\n", order); 1294 mr = NULL; 1295 } 1296 populate_mtts = false; 1297 } else if (!MLX5_CAP_GEN(dev->mdev, umr_extended_translation_offset)) { 1298 if (access_flags & IB_ACCESS_ON_DEMAND) { 1299 err = -EINVAL; 1300 pr_err("Got MR registration for ODP MR > 512MB, not supported for Connect-IB\n"); 1301 goto error; 1302 } 1303 populate_mtts = true; 1304 } 1305 1306 if (!mr) { 1307 if (!umr_can_modify_entity_size(dev)) 1308 populate_mtts = true; 1309 mutex_lock(&dev->slow_path_mutex); 1310 mr = reg_create(NULL, pd, virt_addr, length, umem, ncont, 1311 page_shift, access_flags, populate_mtts); 1312 mutex_unlock(&dev->slow_path_mutex); 1313 } 1314 1315 if (IS_ERR(mr)) { 1316 err = PTR_ERR(mr); 1317 goto error; 1318 } 1319 1320 mlx5_ib_dbg(dev, "mkey 0x%x\n", mr->mmkey.key); 1321 1322 mr->umem = umem; 1323 set_mr_fields(dev, mr, npages, length, access_flags); 1324 1325 update_odp_mr(mr); 1326 1327 if (!populate_mtts) { 1328 int update_xlt_flags = MLX5_IB_UPD_XLT_ENABLE; 1329 1330 if (access_flags & IB_ACCESS_ON_DEMAND) 1331 update_xlt_flags |= MLX5_IB_UPD_XLT_ZAP; 1332 1333 err = mlx5_ib_update_xlt(mr, 0, ncont, page_shift, 1334 update_xlt_flags); 1335 1336 if (err) { 1337 dereg_mr(dev, mr); 1338 return ERR_PTR(err); 1339 } 1340 } 1341 1342 if (IS_ENABLED(CONFIG_INFINIBAND_ON_DEMAND_PAGING)) { 1343 mr->live = 1; 1344 atomic_set(&mr->num_pending_prefetch, 0); 1345 } 1346 1347 return &mr->ibmr; 1348 error: 1349 ib_umem_release(umem); 1350 return ERR_PTR(err); 1351 } 1352 1353 static int unreg_umr(struct mlx5_ib_dev *dev, struct mlx5_ib_mr *mr) 1354 { 1355 struct mlx5_core_dev *mdev = dev->mdev; 1356 struct mlx5_umr_wr umrwr = {}; 1357 1358 if (mdev->state == MLX5_DEVICE_STATE_INTERNAL_ERROR) 1359 return 0; 1360 1361 umrwr.wr.send_flags = MLX5_IB_SEND_UMR_DISABLE_MR | 1362 MLX5_IB_SEND_UMR_FAIL_IF_FREE; 1363 umrwr.wr.opcode = MLX5_IB_WR_UMR; 1364 umrwr.mkey = mr->mmkey.key; 1365 1366 return mlx5_ib_post_send_wait(dev, &umrwr); 1367 } 1368 1369 static int rereg_umr(struct ib_pd *pd, struct mlx5_ib_mr *mr, 1370 int access_flags, int flags) 1371 { 1372 struct mlx5_ib_dev *dev = to_mdev(pd->device); 1373 struct mlx5_umr_wr umrwr = {}; 1374 int err; 1375 1376 umrwr.wr.send_flags = MLX5_IB_SEND_UMR_FAIL_IF_FREE; 1377 1378 umrwr.wr.opcode = MLX5_IB_WR_UMR; 1379 umrwr.mkey = mr->mmkey.key; 1380 1381 if (flags & IB_MR_REREG_PD || flags & IB_MR_REREG_ACCESS) { 1382 umrwr.pd = pd; 1383 umrwr.access_flags = access_flags; 1384 umrwr.wr.send_flags |= MLX5_IB_SEND_UMR_UPDATE_PD_ACCESS; 1385 } 1386 1387 err = mlx5_ib_post_send_wait(dev, &umrwr); 1388 1389 return err; 1390 } 1391 1392 int mlx5_ib_rereg_user_mr(struct ib_mr *ib_mr, int flags, u64 start, 1393 u64 length, u64 virt_addr, int new_access_flags, 1394 struct ib_pd *new_pd, struct ib_udata *udata) 1395 { 1396 struct mlx5_ib_dev *dev = to_mdev(ib_mr->device); 1397 struct mlx5_ib_mr *mr = to_mmr(ib_mr); 1398 struct ib_pd *pd = (flags & IB_MR_REREG_PD) ? new_pd : ib_mr->pd; 1399 int access_flags = flags & IB_MR_REREG_ACCESS ? 1400 new_access_flags : 1401 mr->access_flags; 1402 int page_shift = 0; 1403 int upd_flags = 0; 1404 int npages = 0; 1405 int ncont = 0; 1406 int order = 0; 1407 u64 addr, len; 1408 int err; 1409 1410 mlx5_ib_dbg(dev, "start 0x%llx, virt_addr 0x%llx, length 0x%llx, access_flags 0x%x\n", 1411 start, virt_addr, length, access_flags); 1412 1413 atomic_sub(mr->npages, &dev->mdev->priv.reg_pages); 1414 1415 if (!mr->umem) 1416 return -EINVAL; 1417 1418 if (flags & IB_MR_REREG_TRANS) { 1419 addr = virt_addr; 1420 len = length; 1421 } else { 1422 addr = mr->umem->address; 1423 len = mr->umem->length; 1424 } 1425 1426 if (flags != IB_MR_REREG_PD) { 1427 /* 1428 * Replace umem. This needs to be done whether or not UMR is 1429 * used. 1430 */ 1431 flags |= IB_MR_REREG_TRANS; 1432 ib_umem_release(mr->umem); 1433 mr->umem = NULL; 1434 err = mr_umem_get(dev, udata, addr, len, access_flags, 1435 &mr->umem, &npages, &page_shift, &ncont, 1436 &order); 1437 if (err) 1438 goto err; 1439 } 1440 1441 if (flags & IB_MR_REREG_TRANS && !use_umr_mtt_update(mr, addr, len)) { 1442 /* 1443 * UMR can't be used - MKey needs to be replaced. 1444 */ 1445 if (mr->allocated_from_cache) 1446 err = unreg_umr(dev, mr); 1447 else 1448 err = destroy_mkey(dev, mr); 1449 if (err) 1450 goto err; 1451 1452 mr = reg_create(ib_mr, pd, addr, len, mr->umem, ncont, 1453 page_shift, access_flags, true); 1454 1455 if (IS_ERR(mr)) { 1456 err = PTR_ERR(mr); 1457 mr = to_mmr(ib_mr); 1458 goto err; 1459 } 1460 1461 mr->allocated_from_cache = 0; 1462 if (IS_ENABLED(CONFIG_INFINIBAND_ON_DEMAND_PAGING)) 1463 mr->live = 1; 1464 } else { 1465 /* 1466 * Send a UMR WQE 1467 */ 1468 mr->ibmr.pd = pd; 1469 mr->access_flags = access_flags; 1470 mr->mmkey.iova = addr; 1471 mr->mmkey.size = len; 1472 mr->mmkey.pd = to_mpd(pd)->pdn; 1473 1474 if (flags & IB_MR_REREG_TRANS) { 1475 upd_flags = MLX5_IB_UPD_XLT_ADDR; 1476 if (flags & IB_MR_REREG_PD) 1477 upd_flags |= MLX5_IB_UPD_XLT_PD; 1478 if (flags & IB_MR_REREG_ACCESS) 1479 upd_flags |= MLX5_IB_UPD_XLT_ACCESS; 1480 err = mlx5_ib_update_xlt(mr, 0, npages, page_shift, 1481 upd_flags); 1482 } else { 1483 err = rereg_umr(pd, mr, access_flags, flags); 1484 } 1485 1486 if (err) 1487 goto err; 1488 } 1489 1490 set_mr_fields(dev, mr, npages, len, access_flags); 1491 1492 update_odp_mr(mr); 1493 return 0; 1494 1495 err: 1496 if (mr->umem) { 1497 ib_umem_release(mr->umem); 1498 mr->umem = NULL; 1499 } 1500 clean_mr(dev, mr); 1501 return err; 1502 } 1503 1504 static int 1505 mlx5_alloc_priv_descs(struct ib_device *device, 1506 struct mlx5_ib_mr *mr, 1507 int ndescs, 1508 int desc_size) 1509 { 1510 int size = ndescs * desc_size; 1511 int add_size; 1512 int ret; 1513 1514 add_size = max_t(int, MLX5_UMR_ALIGN - ARCH_KMALLOC_MINALIGN, 0); 1515 1516 mr->descs_alloc = kzalloc(size + add_size, GFP_KERNEL); 1517 if (!mr->descs_alloc) 1518 return -ENOMEM; 1519 1520 mr->descs = PTR_ALIGN(mr->descs_alloc, MLX5_UMR_ALIGN); 1521 1522 mr->desc_map = dma_map_single(device->dev.parent, mr->descs, 1523 size, DMA_TO_DEVICE); 1524 if (dma_mapping_error(device->dev.parent, mr->desc_map)) { 1525 ret = -ENOMEM; 1526 goto err; 1527 } 1528 1529 return 0; 1530 err: 1531 kfree(mr->descs_alloc); 1532 1533 return ret; 1534 } 1535 1536 static void 1537 mlx5_free_priv_descs(struct mlx5_ib_mr *mr) 1538 { 1539 if (mr->descs) { 1540 struct ib_device *device = mr->ibmr.device; 1541 int size = mr->max_descs * mr->desc_size; 1542 1543 dma_unmap_single(device->dev.parent, mr->desc_map, 1544 size, DMA_TO_DEVICE); 1545 kfree(mr->descs_alloc); 1546 mr->descs = NULL; 1547 } 1548 } 1549 1550 static void clean_mr(struct mlx5_ib_dev *dev, struct mlx5_ib_mr *mr) 1551 { 1552 int allocated_from_cache = mr->allocated_from_cache; 1553 1554 if (mr->sig) { 1555 if (mlx5_core_destroy_psv(dev->mdev, 1556 mr->sig->psv_memory.psv_idx)) 1557 mlx5_ib_warn(dev, "failed to destroy mem psv %d\n", 1558 mr->sig->psv_memory.psv_idx); 1559 if (mlx5_core_destroy_psv(dev->mdev, 1560 mr->sig->psv_wire.psv_idx)) 1561 mlx5_ib_warn(dev, "failed to destroy wire psv %d\n", 1562 mr->sig->psv_wire.psv_idx); 1563 kfree(mr->sig); 1564 mr->sig = NULL; 1565 } 1566 1567 mlx5_free_priv_descs(mr); 1568 1569 if (!allocated_from_cache) 1570 destroy_mkey(dev, mr); 1571 } 1572 1573 static void dereg_mr(struct mlx5_ib_dev *dev, struct mlx5_ib_mr *mr) 1574 { 1575 int npages = mr->npages; 1576 struct ib_umem *umem = mr->umem; 1577 1578 if (is_odp_mr(mr)) { 1579 struct ib_umem_odp *umem_odp = to_ib_umem_odp(umem); 1580 1581 /* Prevent new page faults and 1582 * prefetch requests from succeeding 1583 */ 1584 mr->live = 0; 1585 1586 /* dequeue pending prefetch requests for the mr */ 1587 if (atomic_read(&mr->num_pending_prefetch)) 1588 flush_workqueue(system_unbound_wq); 1589 WARN_ON(atomic_read(&mr->num_pending_prefetch)); 1590 1591 /* Wait for all running page-fault handlers to finish. */ 1592 synchronize_srcu(&dev->mr_srcu); 1593 /* Destroy all page mappings */ 1594 if (umem_odp->page_list) 1595 mlx5_ib_invalidate_range(umem_odp, ib_umem_start(umem), 1596 ib_umem_end(umem)); 1597 else 1598 mlx5_ib_free_implicit_mr(mr); 1599 /* 1600 * We kill the umem before the MR for ODP, 1601 * so that there will not be any invalidations in 1602 * flight, looking at the *mr struct. 1603 */ 1604 ib_umem_release(umem); 1605 atomic_sub(npages, &dev->mdev->priv.reg_pages); 1606 1607 /* Avoid double-freeing the umem. */ 1608 umem = NULL; 1609 } 1610 1611 clean_mr(dev, mr); 1612 1613 /* 1614 * We should unregister the DMA address from the HCA before 1615 * remove the DMA mapping. 1616 */ 1617 mlx5_mr_cache_free(dev, mr); 1618 if (umem) { 1619 ib_umem_release(umem); 1620 atomic_sub(npages, &dev->mdev->priv.reg_pages); 1621 } 1622 if (!mr->allocated_from_cache) 1623 kfree(mr); 1624 } 1625 1626 int mlx5_ib_dereg_mr(struct ib_mr *ibmr) 1627 { 1628 dereg_mr(to_mdev(ibmr->device), to_mmr(ibmr)); 1629 return 0; 1630 } 1631 1632 struct ib_mr *mlx5_ib_alloc_mr(struct ib_pd *pd, 1633 enum ib_mr_type mr_type, 1634 u32 max_num_sg) 1635 { 1636 struct mlx5_ib_dev *dev = to_mdev(pd->device); 1637 int inlen = MLX5_ST_SZ_BYTES(create_mkey_in); 1638 int ndescs = ALIGN(max_num_sg, 4); 1639 struct mlx5_ib_mr *mr; 1640 void *mkc; 1641 u32 *in; 1642 int err; 1643 1644 mr = kzalloc(sizeof(*mr), GFP_KERNEL); 1645 if (!mr) 1646 return ERR_PTR(-ENOMEM); 1647 1648 in = kzalloc(inlen, GFP_KERNEL); 1649 if (!in) { 1650 err = -ENOMEM; 1651 goto err_free; 1652 } 1653 1654 mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry); 1655 MLX5_SET(mkc, mkc, free, 1); 1656 MLX5_SET(mkc, mkc, translations_octword_size, ndescs); 1657 MLX5_SET(mkc, mkc, qpn, 0xffffff); 1658 MLX5_SET(mkc, mkc, pd, to_mpd(pd)->pdn); 1659 1660 if (mr_type == IB_MR_TYPE_MEM_REG) { 1661 mr->access_mode = MLX5_MKC_ACCESS_MODE_MTT; 1662 MLX5_SET(mkc, mkc, log_page_size, PAGE_SHIFT); 1663 err = mlx5_alloc_priv_descs(pd->device, mr, 1664 ndescs, sizeof(struct mlx5_mtt)); 1665 if (err) 1666 goto err_free_in; 1667 1668 mr->desc_size = sizeof(struct mlx5_mtt); 1669 mr->max_descs = ndescs; 1670 } else if (mr_type == IB_MR_TYPE_SG_GAPS) { 1671 mr->access_mode = MLX5_MKC_ACCESS_MODE_KLMS; 1672 1673 err = mlx5_alloc_priv_descs(pd->device, mr, 1674 ndescs, sizeof(struct mlx5_klm)); 1675 if (err) 1676 goto err_free_in; 1677 mr->desc_size = sizeof(struct mlx5_klm); 1678 mr->max_descs = ndescs; 1679 } else if (mr_type == IB_MR_TYPE_SIGNATURE) { 1680 u32 psv_index[2]; 1681 1682 MLX5_SET(mkc, mkc, bsf_en, 1); 1683 MLX5_SET(mkc, mkc, bsf_octword_size, MLX5_MKEY_BSF_OCTO_SIZE); 1684 mr->sig = kzalloc(sizeof(*mr->sig), GFP_KERNEL); 1685 if (!mr->sig) { 1686 err = -ENOMEM; 1687 goto err_free_in; 1688 } 1689 1690 /* create mem & wire PSVs */ 1691 err = mlx5_core_create_psv(dev->mdev, to_mpd(pd)->pdn, 1692 2, psv_index); 1693 if (err) 1694 goto err_free_sig; 1695 1696 mr->access_mode = MLX5_MKC_ACCESS_MODE_KLMS; 1697 mr->sig->psv_memory.psv_idx = psv_index[0]; 1698 mr->sig->psv_wire.psv_idx = psv_index[1]; 1699 1700 mr->sig->sig_status_checked = true; 1701 mr->sig->sig_err_exists = false; 1702 /* Next UMR, Arm SIGERR */ 1703 ++mr->sig->sigerr_count; 1704 } else { 1705 mlx5_ib_warn(dev, "Invalid mr type %d\n", mr_type); 1706 err = -EINVAL; 1707 goto err_free_in; 1708 } 1709 1710 MLX5_SET(mkc, mkc, access_mode_1_0, mr->access_mode & 0x3); 1711 MLX5_SET(mkc, mkc, access_mode_4_2, (mr->access_mode >> 2) & 0x7); 1712 MLX5_SET(mkc, mkc, umr_en, 1); 1713 1714 mr->ibmr.device = pd->device; 1715 err = mlx5_core_create_mkey(dev->mdev, &mr->mmkey, in, inlen); 1716 if (err) 1717 goto err_destroy_psv; 1718 1719 mr->mmkey.type = MLX5_MKEY_MR; 1720 mr->ibmr.lkey = mr->mmkey.key; 1721 mr->ibmr.rkey = mr->mmkey.key; 1722 mr->umem = NULL; 1723 kfree(in); 1724 1725 return &mr->ibmr; 1726 1727 err_destroy_psv: 1728 if (mr->sig) { 1729 if (mlx5_core_destroy_psv(dev->mdev, 1730 mr->sig->psv_memory.psv_idx)) 1731 mlx5_ib_warn(dev, "failed to destroy mem psv %d\n", 1732 mr->sig->psv_memory.psv_idx); 1733 if (mlx5_core_destroy_psv(dev->mdev, 1734 mr->sig->psv_wire.psv_idx)) 1735 mlx5_ib_warn(dev, "failed to destroy wire psv %d\n", 1736 mr->sig->psv_wire.psv_idx); 1737 } 1738 mlx5_free_priv_descs(mr); 1739 err_free_sig: 1740 kfree(mr->sig); 1741 err_free_in: 1742 kfree(in); 1743 err_free: 1744 kfree(mr); 1745 return ERR_PTR(err); 1746 } 1747 1748 struct ib_mw *mlx5_ib_alloc_mw(struct ib_pd *pd, enum ib_mw_type type, 1749 struct ib_udata *udata) 1750 { 1751 struct mlx5_ib_dev *dev = to_mdev(pd->device); 1752 int inlen = MLX5_ST_SZ_BYTES(create_mkey_in); 1753 struct mlx5_ib_mw *mw = NULL; 1754 u32 *in = NULL; 1755 void *mkc; 1756 int ndescs; 1757 int err; 1758 struct mlx5_ib_alloc_mw req = {}; 1759 struct { 1760 __u32 comp_mask; 1761 __u32 response_length; 1762 } resp = {}; 1763 1764 err = ib_copy_from_udata(&req, udata, min(udata->inlen, sizeof(req))); 1765 if (err) 1766 return ERR_PTR(err); 1767 1768 if (req.comp_mask || req.reserved1 || req.reserved2) 1769 return ERR_PTR(-EOPNOTSUPP); 1770 1771 if (udata->inlen > sizeof(req) && 1772 !ib_is_udata_cleared(udata, sizeof(req), 1773 udata->inlen - sizeof(req))) 1774 return ERR_PTR(-EOPNOTSUPP); 1775 1776 ndescs = req.num_klms ? roundup(req.num_klms, 4) : roundup(1, 4); 1777 1778 mw = kzalloc(sizeof(*mw), GFP_KERNEL); 1779 in = kzalloc(inlen, GFP_KERNEL); 1780 if (!mw || !in) { 1781 err = -ENOMEM; 1782 goto free; 1783 } 1784 1785 mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry); 1786 1787 MLX5_SET(mkc, mkc, free, 1); 1788 MLX5_SET(mkc, mkc, translations_octword_size, ndescs); 1789 MLX5_SET(mkc, mkc, pd, to_mpd(pd)->pdn); 1790 MLX5_SET(mkc, mkc, umr_en, 1); 1791 MLX5_SET(mkc, mkc, lr, 1); 1792 MLX5_SET(mkc, mkc, access_mode_1_0, MLX5_MKC_ACCESS_MODE_KLMS); 1793 MLX5_SET(mkc, mkc, en_rinval, !!((type == IB_MW_TYPE_2))); 1794 MLX5_SET(mkc, mkc, qpn, 0xffffff); 1795 1796 err = mlx5_core_create_mkey(dev->mdev, &mw->mmkey, in, inlen); 1797 if (err) 1798 goto free; 1799 1800 mw->mmkey.type = MLX5_MKEY_MW; 1801 mw->ibmw.rkey = mw->mmkey.key; 1802 mw->ndescs = ndescs; 1803 1804 resp.response_length = min(offsetof(typeof(resp), response_length) + 1805 sizeof(resp.response_length), udata->outlen); 1806 if (resp.response_length) { 1807 err = ib_copy_to_udata(udata, &resp, resp.response_length); 1808 if (err) { 1809 mlx5_core_destroy_mkey(dev->mdev, &mw->mmkey); 1810 goto free; 1811 } 1812 } 1813 1814 kfree(in); 1815 return &mw->ibmw; 1816 1817 free: 1818 kfree(mw); 1819 kfree(in); 1820 return ERR_PTR(err); 1821 } 1822 1823 int mlx5_ib_dealloc_mw(struct ib_mw *mw) 1824 { 1825 struct mlx5_ib_mw *mmw = to_mmw(mw); 1826 int err; 1827 1828 err = mlx5_core_destroy_mkey((to_mdev(mw->device))->mdev, 1829 &mmw->mmkey); 1830 if (!err) 1831 kfree(mmw); 1832 return err; 1833 } 1834 1835 int mlx5_ib_check_mr_status(struct ib_mr *ibmr, u32 check_mask, 1836 struct ib_mr_status *mr_status) 1837 { 1838 struct mlx5_ib_mr *mmr = to_mmr(ibmr); 1839 int ret = 0; 1840 1841 if (check_mask & ~IB_MR_CHECK_SIG_STATUS) { 1842 pr_err("Invalid status check mask\n"); 1843 ret = -EINVAL; 1844 goto done; 1845 } 1846 1847 mr_status->fail_status = 0; 1848 if (check_mask & IB_MR_CHECK_SIG_STATUS) { 1849 if (!mmr->sig) { 1850 ret = -EINVAL; 1851 pr_err("signature status check requested on a non-signature enabled MR\n"); 1852 goto done; 1853 } 1854 1855 mmr->sig->sig_status_checked = true; 1856 if (!mmr->sig->sig_err_exists) 1857 goto done; 1858 1859 if (ibmr->lkey == mmr->sig->err_item.key) 1860 memcpy(&mr_status->sig_err, &mmr->sig->err_item, 1861 sizeof(mr_status->sig_err)); 1862 else { 1863 mr_status->sig_err.err_type = IB_SIG_BAD_GUARD; 1864 mr_status->sig_err.sig_err_offset = 0; 1865 mr_status->sig_err.key = mmr->sig->err_item.key; 1866 } 1867 1868 mmr->sig->sig_err_exists = false; 1869 mr_status->fail_status |= IB_MR_CHECK_SIG_STATUS; 1870 } 1871 1872 done: 1873 return ret; 1874 } 1875 1876 static int 1877 mlx5_ib_sg_to_klms(struct mlx5_ib_mr *mr, 1878 struct scatterlist *sgl, 1879 unsigned short sg_nents, 1880 unsigned int *sg_offset_p) 1881 { 1882 struct scatterlist *sg = sgl; 1883 struct mlx5_klm *klms = mr->descs; 1884 unsigned int sg_offset = sg_offset_p ? *sg_offset_p : 0; 1885 u32 lkey = mr->ibmr.pd->local_dma_lkey; 1886 int i; 1887 1888 mr->ibmr.iova = sg_dma_address(sg) + sg_offset; 1889 mr->ibmr.length = 0; 1890 1891 for_each_sg(sgl, sg, sg_nents, i) { 1892 if (unlikely(i >= mr->max_descs)) 1893 break; 1894 klms[i].va = cpu_to_be64(sg_dma_address(sg) + sg_offset); 1895 klms[i].bcount = cpu_to_be32(sg_dma_len(sg) - sg_offset); 1896 klms[i].key = cpu_to_be32(lkey); 1897 mr->ibmr.length += sg_dma_len(sg) - sg_offset; 1898 1899 sg_offset = 0; 1900 } 1901 mr->ndescs = i; 1902 1903 if (sg_offset_p) 1904 *sg_offset_p = sg_offset; 1905 1906 return i; 1907 } 1908 1909 static int mlx5_set_page(struct ib_mr *ibmr, u64 addr) 1910 { 1911 struct mlx5_ib_mr *mr = to_mmr(ibmr); 1912 __be64 *descs; 1913 1914 if (unlikely(mr->ndescs == mr->max_descs)) 1915 return -ENOMEM; 1916 1917 descs = mr->descs; 1918 descs[mr->ndescs++] = cpu_to_be64(addr | MLX5_EN_RD | MLX5_EN_WR); 1919 1920 return 0; 1921 } 1922 1923 int mlx5_ib_map_mr_sg(struct ib_mr *ibmr, struct scatterlist *sg, int sg_nents, 1924 unsigned int *sg_offset) 1925 { 1926 struct mlx5_ib_mr *mr = to_mmr(ibmr); 1927 int n; 1928 1929 mr->ndescs = 0; 1930 1931 ib_dma_sync_single_for_cpu(ibmr->device, mr->desc_map, 1932 mr->desc_size * mr->max_descs, 1933 DMA_TO_DEVICE); 1934 1935 if (mr->access_mode == MLX5_MKC_ACCESS_MODE_KLMS) 1936 n = mlx5_ib_sg_to_klms(mr, sg, sg_nents, sg_offset); 1937 else 1938 n = ib_sg_to_pages(ibmr, sg, sg_nents, sg_offset, 1939 mlx5_set_page); 1940 1941 ib_dma_sync_single_for_device(ibmr->device, mr->desc_map, 1942 mr->desc_size * mr->max_descs, 1943 DMA_TO_DEVICE); 1944 1945 return n; 1946 } 1947