1 /* 2 * Copyright (c) 2004 Topspin Communications. All rights reserved. 3 * Copyright (c) 2005, 2006, 2007, 2008 Mellanox Technologies. All rights reserved. 4 * Copyright (c) 2006, 2007 Cisco Systems, Inc. All rights reserved. 5 * 6 * This software is available to you under a choice of one of two 7 * licenses. You may choose to be licensed under the terms of the GNU 8 * General Public License (GPL) Version 2, available from the file 9 * COPYING in the main directory of this source tree, or the 10 * OpenIB.org BSD license below: 11 * 12 * Redistribution and use in source and binary forms, with or 13 * without modification, are permitted provided that the following 14 * conditions are met: 15 * 16 * - Redistributions of source code must retain the above 17 * copyright notice, this list of conditions and the following 18 * disclaimer. 19 * 20 * - Redistributions in binary form must reproduce the above 21 * copyright notice, this list of conditions and the following 22 * disclaimer in the documentation and/or other materials 23 * provided with the distribution. 24 * 25 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 26 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 27 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 28 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 29 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 30 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 31 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 32 * SOFTWARE. 33 */ 34 35 #include <linux/init.h> 36 #include <linux/errno.h> 37 #include <linux/export.h> 38 #include <linux/slab.h> 39 #include <linux/kernel.h> 40 #include <linux/vmalloc.h> 41 42 #include <linux/mlx4/cmd.h> 43 44 #include "mlx4.h" 45 #include "icm.h" 46 47 static u32 mlx4_buddy_alloc(struct mlx4_buddy *buddy, int order) 48 { 49 int o; 50 int m; 51 u32 seg; 52 53 spin_lock(&buddy->lock); 54 55 for (o = order; o <= buddy->max_order; ++o) 56 if (buddy->num_free[o]) { 57 m = 1 << (buddy->max_order - o); 58 seg = find_first_bit(buddy->bits[o], m); 59 if (seg < m) 60 goto found; 61 } 62 63 spin_unlock(&buddy->lock); 64 return -1; 65 66 found: 67 clear_bit(seg, buddy->bits[o]); 68 --buddy->num_free[o]; 69 70 while (o > order) { 71 --o; 72 seg <<= 1; 73 set_bit(seg ^ 1, buddy->bits[o]); 74 ++buddy->num_free[o]; 75 } 76 77 spin_unlock(&buddy->lock); 78 79 seg <<= order; 80 81 return seg; 82 } 83 84 static void mlx4_buddy_free(struct mlx4_buddy *buddy, u32 seg, int order) 85 { 86 seg >>= order; 87 88 spin_lock(&buddy->lock); 89 90 while (test_bit(seg ^ 1, buddy->bits[order])) { 91 clear_bit(seg ^ 1, buddy->bits[order]); 92 --buddy->num_free[order]; 93 seg >>= 1; 94 ++order; 95 } 96 97 set_bit(seg, buddy->bits[order]); 98 ++buddy->num_free[order]; 99 100 spin_unlock(&buddy->lock); 101 } 102 103 static int mlx4_buddy_init(struct mlx4_buddy *buddy, int max_order) 104 { 105 int i, s; 106 107 buddy->max_order = max_order; 108 spin_lock_init(&buddy->lock); 109 110 buddy->bits = kcalloc(buddy->max_order + 1, sizeof (long *), 111 GFP_KERNEL); 112 buddy->num_free = kcalloc((buddy->max_order + 1), sizeof *buddy->num_free, 113 GFP_KERNEL); 114 if (!buddy->bits || !buddy->num_free) 115 goto err_out; 116 117 for (i = 0; i <= buddy->max_order; ++i) { 118 s = BITS_TO_LONGS(1 << (buddy->max_order - i)); 119 buddy->bits[i] = kcalloc(s, sizeof (long), GFP_KERNEL | __GFP_NOWARN); 120 if (!buddy->bits[i]) { 121 buddy->bits[i] = vzalloc(s * sizeof(long)); 122 if (!buddy->bits[i]) 123 goto err_out_free; 124 } 125 } 126 127 set_bit(0, buddy->bits[buddy->max_order]); 128 buddy->num_free[buddy->max_order] = 1; 129 130 return 0; 131 132 err_out_free: 133 for (i = 0; i <= buddy->max_order; ++i) 134 if (buddy->bits[i] && is_vmalloc_addr(buddy->bits[i])) 135 vfree(buddy->bits[i]); 136 else 137 kfree(buddy->bits[i]); 138 139 err_out: 140 kfree(buddy->bits); 141 kfree(buddy->num_free); 142 143 return -ENOMEM; 144 } 145 146 static void mlx4_buddy_cleanup(struct mlx4_buddy *buddy) 147 { 148 int i; 149 150 for (i = 0; i <= buddy->max_order; ++i) 151 if (is_vmalloc_addr(buddy->bits[i])) 152 vfree(buddy->bits[i]); 153 else 154 kfree(buddy->bits[i]); 155 156 kfree(buddy->bits); 157 kfree(buddy->num_free); 158 } 159 160 u32 __mlx4_alloc_mtt_range(struct mlx4_dev *dev, int order) 161 { 162 struct mlx4_mr_table *mr_table = &mlx4_priv(dev)->mr_table; 163 u32 seg; 164 int seg_order; 165 u32 offset; 166 167 seg_order = max_t(int, order - log_mtts_per_seg, 0); 168 169 seg = mlx4_buddy_alloc(&mr_table->mtt_buddy, seg_order); 170 if (seg == -1) 171 return -1; 172 173 offset = seg * (1 << log_mtts_per_seg); 174 175 if (mlx4_table_get_range(dev, &mr_table->mtt_table, offset, 176 offset + (1 << order) - 1)) { 177 mlx4_buddy_free(&mr_table->mtt_buddy, seg, seg_order); 178 return -1; 179 } 180 181 return offset; 182 } 183 184 static u32 mlx4_alloc_mtt_range(struct mlx4_dev *dev, int order) 185 { 186 u64 in_param = 0; 187 u64 out_param; 188 int err; 189 190 if (mlx4_is_mfunc(dev)) { 191 set_param_l(&in_param, order); 192 err = mlx4_cmd_imm(dev, in_param, &out_param, RES_MTT, 193 RES_OP_RESERVE_AND_MAP, 194 MLX4_CMD_ALLOC_RES, 195 MLX4_CMD_TIME_CLASS_A, 196 MLX4_CMD_WRAPPED); 197 if (err) 198 return -1; 199 return get_param_l(&out_param); 200 } 201 return __mlx4_alloc_mtt_range(dev, order); 202 } 203 204 int mlx4_mtt_init(struct mlx4_dev *dev, int npages, int page_shift, 205 struct mlx4_mtt *mtt) 206 { 207 int i; 208 209 if (!npages) { 210 mtt->order = -1; 211 mtt->page_shift = MLX4_ICM_PAGE_SHIFT; 212 return 0; 213 } else 214 mtt->page_shift = page_shift; 215 216 for (mtt->order = 0, i = 1; i < npages; i <<= 1) 217 ++mtt->order; 218 219 mtt->offset = mlx4_alloc_mtt_range(dev, mtt->order); 220 if (mtt->offset == -1) 221 return -ENOMEM; 222 223 return 0; 224 } 225 EXPORT_SYMBOL_GPL(mlx4_mtt_init); 226 227 void __mlx4_free_mtt_range(struct mlx4_dev *dev, u32 offset, int order) 228 { 229 u32 first_seg; 230 int seg_order; 231 struct mlx4_mr_table *mr_table = &mlx4_priv(dev)->mr_table; 232 233 seg_order = max_t(int, order - log_mtts_per_seg, 0); 234 first_seg = offset / (1 << log_mtts_per_seg); 235 236 mlx4_buddy_free(&mr_table->mtt_buddy, first_seg, seg_order); 237 mlx4_table_put_range(dev, &mr_table->mtt_table, offset, 238 offset + (1 << order) - 1); 239 } 240 241 static void mlx4_free_mtt_range(struct mlx4_dev *dev, u32 offset, int order) 242 { 243 u64 in_param = 0; 244 int err; 245 246 if (mlx4_is_mfunc(dev)) { 247 set_param_l(&in_param, offset); 248 set_param_h(&in_param, order); 249 err = mlx4_cmd(dev, in_param, RES_MTT, RES_OP_RESERVE_AND_MAP, 250 MLX4_CMD_FREE_RES, 251 MLX4_CMD_TIME_CLASS_A, 252 MLX4_CMD_WRAPPED); 253 if (err) 254 mlx4_warn(dev, "Failed to free mtt range at:" 255 "%d order:%d\n", offset, order); 256 return; 257 } 258 __mlx4_free_mtt_range(dev, offset, order); 259 } 260 261 void mlx4_mtt_cleanup(struct mlx4_dev *dev, struct mlx4_mtt *mtt) 262 { 263 if (mtt->order < 0) 264 return; 265 266 mlx4_free_mtt_range(dev, mtt->offset, mtt->order); 267 } 268 EXPORT_SYMBOL_GPL(mlx4_mtt_cleanup); 269 270 u64 mlx4_mtt_addr(struct mlx4_dev *dev, struct mlx4_mtt *mtt) 271 { 272 return (u64) mtt->offset * dev->caps.mtt_entry_sz; 273 } 274 EXPORT_SYMBOL_GPL(mlx4_mtt_addr); 275 276 static u32 hw_index_to_key(u32 ind) 277 { 278 return (ind >> 24) | (ind << 8); 279 } 280 281 static u32 key_to_hw_index(u32 key) 282 { 283 return (key << 24) | (key >> 8); 284 } 285 286 static int mlx4_SW2HW_MPT(struct mlx4_dev *dev, struct mlx4_cmd_mailbox *mailbox, 287 int mpt_index) 288 { 289 return mlx4_cmd(dev, mailbox->dma, mpt_index, 290 0, MLX4_CMD_SW2HW_MPT, MLX4_CMD_TIME_CLASS_B, 291 MLX4_CMD_WRAPPED); 292 } 293 294 static int mlx4_HW2SW_MPT(struct mlx4_dev *dev, struct mlx4_cmd_mailbox *mailbox, 295 int mpt_index) 296 { 297 return mlx4_cmd_box(dev, 0, mailbox ? mailbox->dma : 0, mpt_index, 298 !mailbox, MLX4_CMD_HW2SW_MPT, 299 MLX4_CMD_TIME_CLASS_B, MLX4_CMD_WRAPPED); 300 } 301 302 static int mlx4_mr_alloc_reserved(struct mlx4_dev *dev, u32 mridx, u32 pd, 303 u64 iova, u64 size, u32 access, int npages, 304 int page_shift, struct mlx4_mr *mr) 305 { 306 mr->iova = iova; 307 mr->size = size; 308 mr->pd = pd; 309 mr->access = access; 310 mr->enabled = MLX4_MPT_DISABLED; 311 mr->key = hw_index_to_key(mridx); 312 313 return mlx4_mtt_init(dev, npages, page_shift, &mr->mtt); 314 } 315 316 static int mlx4_WRITE_MTT(struct mlx4_dev *dev, 317 struct mlx4_cmd_mailbox *mailbox, 318 int num_entries) 319 { 320 return mlx4_cmd(dev, mailbox->dma, num_entries, 0, MLX4_CMD_WRITE_MTT, 321 MLX4_CMD_TIME_CLASS_A, MLX4_CMD_WRAPPED); 322 } 323 324 int __mlx4_mpt_reserve(struct mlx4_dev *dev) 325 { 326 struct mlx4_priv *priv = mlx4_priv(dev); 327 328 return mlx4_bitmap_alloc(&priv->mr_table.mpt_bitmap); 329 } 330 331 static int mlx4_mpt_reserve(struct mlx4_dev *dev) 332 { 333 u64 out_param; 334 335 if (mlx4_is_mfunc(dev)) { 336 if (mlx4_cmd_imm(dev, 0, &out_param, RES_MPT, RES_OP_RESERVE, 337 MLX4_CMD_ALLOC_RES, 338 MLX4_CMD_TIME_CLASS_A, MLX4_CMD_WRAPPED)) 339 return -1; 340 return get_param_l(&out_param); 341 } 342 return __mlx4_mpt_reserve(dev); 343 } 344 345 void __mlx4_mpt_release(struct mlx4_dev *dev, u32 index) 346 { 347 struct mlx4_priv *priv = mlx4_priv(dev); 348 349 mlx4_bitmap_free(&priv->mr_table.mpt_bitmap, index); 350 } 351 352 static void mlx4_mpt_release(struct mlx4_dev *dev, u32 index) 353 { 354 u64 in_param = 0; 355 356 if (mlx4_is_mfunc(dev)) { 357 set_param_l(&in_param, index); 358 if (mlx4_cmd(dev, in_param, RES_MPT, RES_OP_RESERVE, 359 MLX4_CMD_FREE_RES, 360 MLX4_CMD_TIME_CLASS_A, MLX4_CMD_WRAPPED)) 361 mlx4_warn(dev, "Failed to release mr index:%d\n", 362 index); 363 return; 364 } 365 __mlx4_mpt_release(dev, index); 366 } 367 368 int __mlx4_mpt_alloc_icm(struct mlx4_dev *dev, u32 index) 369 { 370 struct mlx4_mr_table *mr_table = &mlx4_priv(dev)->mr_table; 371 372 return mlx4_table_get(dev, &mr_table->dmpt_table, index); 373 } 374 375 static int mlx4_mpt_alloc_icm(struct mlx4_dev *dev, u32 index) 376 { 377 u64 param = 0; 378 379 if (mlx4_is_mfunc(dev)) { 380 set_param_l(¶m, index); 381 return mlx4_cmd_imm(dev, param, ¶m, RES_MPT, RES_OP_MAP_ICM, 382 MLX4_CMD_ALLOC_RES, 383 MLX4_CMD_TIME_CLASS_A, 384 MLX4_CMD_WRAPPED); 385 } 386 return __mlx4_mpt_alloc_icm(dev, index); 387 } 388 389 void __mlx4_mpt_free_icm(struct mlx4_dev *dev, u32 index) 390 { 391 struct mlx4_mr_table *mr_table = &mlx4_priv(dev)->mr_table; 392 393 mlx4_table_put(dev, &mr_table->dmpt_table, index); 394 } 395 396 static void mlx4_mpt_free_icm(struct mlx4_dev *dev, u32 index) 397 { 398 u64 in_param = 0; 399 400 if (mlx4_is_mfunc(dev)) { 401 set_param_l(&in_param, index); 402 if (mlx4_cmd(dev, in_param, RES_MPT, RES_OP_MAP_ICM, 403 MLX4_CMD_FREE_RES, MLX4_CMD_TIME_CLASS_A, 404 MLX4_CMD_WRAPPED)) 405 mlx4_warn(dev, "Failed to free icm of mr index:%d\n", 406 index); 407 return; 408 } 409 return __mlx4_mpt_free_icm(dev, index); 410 } 411 412 int mlx4_mr_alloc(struct mlx4_dev *dev, u32 pd, u64 iova, u64 size, u32 access, 413 int npages, int page_shift, struct mlx4_mr *mr) 414 { 415 u32 index; 416 int err; 417 418 index = mlx4_mpt_reserve(dev); 419 if (index == -1) 420 return -ENOMEM; 421 422 err = mlx4_mr_alloc_reserved(dev, index, pd, iova, size, 423 access, npages, page_shift, mr); 424 if (err) 425 mlx4_mpt_release(dev, index); 426 427 return err; 428 } 429 EXPORT_SYMBOL_GPL(mlx4_mr_alloc); 430 431 static int mlx4_mr_free_reserved(struct mlx4_dev *dev, struct mlx4_mr *mr) 432 { 433 int err; 434 435 if (mr->enabled == MLX4_MPT_EN_HW) { 436 err = mlx4_HW2SW_MPT(dev, NULL, 437 key_to_hw_index(mr->key) & 438 (dev->caps.num_mpts - 1)); 439 if (err) { 440 mlx4_warn(dev, "HW2SW_MPT failed (%d),", err); 441 mlx4_warn(dev, "MR has MWs bound to it.\n"); 442 return err; 443 } 444 445 mr->enabled = MLX4_MPT_EN_SW; 446 } 447 mlx4_mtt_cleanup(dev, &mr->mtt); 448 449 return 0; 450 } 451 452 int mlx4_mr_free(struct mlx4_dev *dev, struct mlx4_mr *mr) 453 { 454 int ret; 455 456 ret = mlx4_mr_free_reserved(dev, mr); 457 if (ret) 458 return ret; 459 if (mr->enabled) 460 mlx4_mpt_free_icm(dev, key_to_hw_index(mr->key)); 461 mlx4_mpt_release(dev, key_to_hw_index(mr->key)); 462 463 return 0; 464 } 465 EXPORT_SYMBOL_GPL(mlx4_mr_free); 466 467 int mlx4_mr_enable(struct mlx4_dev *dev, struct mlx4_mr *mr) 468 { 469 struct mlx4_cmd_mailbox *mailbox; 470 struct mlx4_mpt_entry *mpt_entry; 471 int err; 472 473 err = mlx4_mpt_alloc_icm(dev, key_to_hw_index(mr->key)); 474 if (err) 475 return err; 476 477 mailbox = mlx4_alloc_cmd_mailbox(dev); 478 if (IS_ERR(mailbox)) { 479 err = PTR_ERR(mailbox); 480 goto err_table; 481 } 482 mpt_entry = mailbox->buf; 483 mpt_entry->flags = cpu_to_be32(MLX4_MPT_FLAG_MIO | 484 MLX4_MPT_FLAG_REGION | 485 mr->access); 486 487 mpt_entry->key = cpu_to_be32(key_to_hw_index(mr->key)); 488 mpt_entry->pd_flags = cpu_to_be32(mr->pd | MLX4_MPT_PD_FLAG_EN_INV); 489 mpt_entry->start = cpu_to_be64(mr->iova); 490 mpt_entry->length = cpu_to_be64(mr->size); 491 mpt_entry->entity_size = cpu_to_be32(mr->mtt.page_shift); 492 493 if (mr->mtt.order < 0) { 494 mpt_entry->flags |= cpu_to_be32(MLX4_MPT_FLAG_PHYSICAL); 495 mpt_entry->mtt_addr = 0; 496 } else { 497 mpt_entry->mtt_addr = cpu_to_be64(mlx4_mtt_addr(dev, 498 &mr->mtt)); 499 } 500 501 if (mr->mtt.order >= 0 && mr->mtt.page_shift == 0) { 502 /* fast register MR in free state */ 503 mpt_entry->flags |= cpu_to_be32(MLX4_MPT_FLAG_FREE); 504 mpt_entry->pd_flags |= cpu_to_be32(MLX4_MPT_PD_FLAG_FAST_REG | 505 MLX4_MPT_PD_FLAG_RAE); 506 mpt_entry->mtt_sz = cpu_to_be32(1 << mr->mtt.order); 507 } else { 508 mpt_entry->flags |= cpu_to_be32(MLX4_MPT_FLAG_SW_OWNS); 509 } 510 511 err = mlx4_SW2HW_MPT(dev, mailbox, 512 key_to_hw_index(mr->key) & (dev->caps.num_mpts - 1)); 513 if (err) { 514 mlx4_warn(dev, "SW2HW_MPT failed (%d)\n", err); 515 goto err_cmd; 516 } 517 mr->enabled = MLX4_MPT_EN_HW; 518 519 mlx4_free_cmd_mailbox(dev, mailbox); 520 521 return 0; 522 523 err_cmd: 524 mlx4_free_cmd_mailbox(dev, mailbox); 525 526 err_table: 527 mlx4_mpt_free_icm(dev, key_to_hw_index(mr->key)); 528 return err; 529 } 530 EXPORT_SYMBOL_GPL(mlx4_mr_enable); 531 532 static int mlx4_write_mtt_chunk(struct mlx4_dev *dev, struct mlx4_mtt *mtt, 533 int start_index, int npages, u64 *page_list) 534 { 535 struct mlx4_priv *priv = mlx4_priv(dev); 536 __be64 *mtts; 537 dma_addr_t dma_handle; 538 int i; 539 540 mtts = mlx4_table_find(&priv->mr_table.mtt_table, mtt->offset + 541 start_index, &dma_handle); 542 543 if (!mtts) 544 return -ENOMEM; 545 546 dma_sync_single_for_cpu(&dev->pdev->dev, dma_handle, 547 npages * sizeof (u64), DMA_TO_DEVICE); 548 549 for (i = 0; i < npages; ++i) 550 mtts[i] = cpu_to_be64(page_list[i] | MLX4_MTT_FLAG_PRESENT); 551 552 dma_sync_single_for_device(&dev->pdev->dev, dma_handle, 553 npages * sizeof (u64), DMA_TO_DEVICE); 554 555 return 0; 556 } 557 558 int __mlx4_write_mtt(struct mlx4_dev *dev, struct mlx4_mtt *mtt, 559 int start_index, int npages, u64 *page_list) 560 { 561 int err = 0; 562 int chunk; 563 int mtts_per_page; 564 int max_mtts_first_page; 565 566 /* compute how may mtts fit in the first page */ 567 mtts_per_page = PAGE_SIZE / sizeof(u64); 568 max_mtts_first_page = mtts_per_page - (mtt->offset + start_index) 569 % mtts_per_page; 570 571 chunk = min_t(int, max_mtts_first_page, npages); 572 573 while (npages > 0) { 574 err = mlx4_write_mtt_chunk(dev, mtt, start_index, chunk, page_list); 575 if (err) 576 return err; 577 npages -= chunk; 578 start_index += chunk; 579 page_list += chunk; 580 581 chunk = min_t(int, mtts_per_page, npages); 582 } 583 return err; 584 } 585 586 int mlx4_write_mtt(struct mlx4_dev *dev, struct mlx4_mtt *mtt, 587 int start_index, int npages, u64 *page_list) 588 { 589 struct mlx4_cmd_mailbox *mailbox = NULL; 590 __be64 *inbox = NULL; 591 int chunk; 592 int err = 0; 593 int i; 594 595 if (mtt->order < 0) 596 return -EINVAL; 597 598 if (mlx4_is_mfunc(dev)) { 599 mailbox = mlx4_alloc_cmd_mailbox(dev); 600 if (IS_ERR(mailbox)) 601 return PTR_ERR(mailbox); 602 inbox = mailbox->buf; 603 604 while (npages > 0) { 605 chunk = min_t(int, MLX4_MAILBOX_SIZE / sizeof(u64) - 2, 606 npages); 607 inbox[0] = cpu_to_be64(mtt->offset + start_index); 608 inbox[1] = 0; 609 for (i = 0; i < chunk; ++i) 610 inbox[i + 2] = cpu_to_be64(page_list[i] | 611 MLX4_MTT_FLAG_PRESENT); 612 err = mlx4_WRITE_MTT(dev, mailbox, chunk); 613 if (err) { 614 mlx4_free_cmd_mailbox(dev, mailbox); 615 return err; 616 } 617 618 npages -= chunk; 619 start_index += chunk; 620 page_list += chunk; 621 } 622 mlx4_free_cmd_mailbox(dev, mailbox); 623 return err; 624 } 625 626 return __mlx4_write_mtt(dev, mtt, start_index, npages, page_list); 627 } 628 EXPORT_SYMBOL_GPL(mlx4_write_mtt); 629 630 int mlx4_buf_write_mtt(struct mlx4_dev *dev, struct mlx4_mtt *mtt, 631 struct mlx4_buf *buf) 632 { 633 u64 *page_list; 634 int err; 635 int i; 636 637 page_list = kmalloc(buf->npages * sizeof *page_list, GFP_KERNEL); 638 if (!page_list) 639 return -ENOMEM; 640 641 for (i = 0; i < buf->npages; ++i) 642 if (buf->nbufs == 1) 643 page_list[i] = buf->direct.map + (i << buf->page_shift); 644 else 645 page_list[i] = buf->page_list[i].map; 646 647 err = mlx4_write_mtt(dev, mtt, 0, buf->npages, page_list); 648 649 kfree(page_list); 650 return err; 651 } 652 EXPORT_SYMBOL_GPL(mlx4_buf_write_mtt); 653 654 int mlx4_mw_alloc(struct mlx4_dev *dev, u32 pd, enum mlx4_mw_type type, 655 struct mlx4_mw *mw) 656 { 657 u32 index; 658 659 if ((type == MLX4_MW_TYPE_1 && 660 !(dev->caps.flags & MLX4_DEV_CAP_FLAG_MEM_WINDOW)) || 661 (type == MLX4_MW_TYPE_2 && 662 !(dev->caps.bmme_flags & MLX4_BMME_FLAG_TYPE_2_WIN))) 663 return -ENOTSUPP; 664 665 index = mlx4_mpt_reserve(dev); 666 if (index == -1) 667 return -ENOMEM; 668 669 mw->key = hw_index_to_key(index); 670 mw->pd = pd; 671 mw->type = type; 672 mw->enabled = MLX4_MPT_DISABLED; 673 674 return 0; 675 } 676 EXPORT_SYMBOL_GPL(mlx4_mw_alloc); 677 678 int mlx4_mw_enable(struct mlx4_dev *dev, struct mlx4_mw *mw) 679 { 680 struct mlx4_cmd_mailbox *mailbox; 681 struct mlx4_mpt_entry *mpt_entry; 682 int err; 683 684 err = mlx4_mpt_alloc_icm(dev, key_to_hw_index(mw->key)); 685 if (err) 686 return err; 687 688 mailbox = mlx4_alloc_cmd_mailbox(dev); 689 if (IS_ERR(mailbox)) { 690 err = PTR_ERR(mailbox); 691 goto err_table; 692 } 693 mpt_entry = mailbox->buf; 694 695 /* Note that the MLX4_MPT_FLAG_REGION bit in mpt_entry->flags is turned 696 * off, thus creating a memory window and not a memory region. 697 */ 698 mpt_entry->key = cpu_to_be32(key_to_hw_index(mw->key)); 699 mpt_entry->pd_flags = cpu_to_be32(mw->pd); 700 if (mw->type == MLX4_MW_TYPE_2) { 701 mpt_entry->flags |= cpu_to_be32(MLX4_MPT_FLAG_FREE); 702 mpt_entry->qpn = cpu_to_be32(MLX4_MPT_QP_FLAG_BOUND_QP); 703 mpt_entry->pd_flags |= cpu_to_be32(MLX4_MPT_PD_FLAG_EN_INV); 704 } 705 706 err = mlx4_SW2HW_MPT(dev, mailbox, 707 key_to_hw_index(mw->key) & 708 (dev->caps.num_mpts - 1)); 709 if (err) { 710 mlx4_warn(dev, "SW2HW_MPT failed (%d)\n", err); 711 goto err_cmd; 712 } 713 mw->enabled = MLX4_MPT_EN_HW; 714 715 mlx4_free_cmd_mailbox(dev, mailbox); 716 717 return 0; 718 719 err_cmd: 720 mlx4_free_cmd_mailbox(dev, mailbox); 721 722 err_table: 723 mlx4_mpt_free_icm(dev, key_to_hw_index(mw->key)); 724 return err; 725 } 726 EXPORT_SYMBOL_GPL(mlx4_mw_enable); 727 728 void mlx4_mw_free(struct mlx4_dev *dev, struct mlx4_mw *mw) 729 { 730 int err; 731 732 if (mw->enabled == MLX4_MPT_EN_HW) { 733 err = mlx4_HW2SW_MPT(dev, NULL, 734 key_to_hw_index(mw->key) & 735 (dev->caps.num_mpts - 1)); 736 if (err) 737 mlx4_warn(dev, "xxx HW2SW_MPT failed (%d)\n", err); 738 739 mw->enabled = MLX4_MPT_EN_SW; 740 } 741 if (mw->enabled) 742 mlx4_mpt_free_icm(dev, key_to_hw_index(mw->key)); 743 mlx4_mpt_release(dev, key_to_hw_index(mw->key)); 744 } 745 EXPORT_SYMBOL_GPL(mlx4_mw_free); 746 747 int mlx4_init_mr_table(struct mlx4_dev *dev) 748 { 749 struct mlx4_priv *priv = mlx4_priv(dev); 750 struct mlx4_mr_table *mr_table = &priv->mr_table; 751 int err; 752 753 /* Nothing to do for slaves - all MR handling is forwarded 754 * to the master */ 755 if (mlx4_is_slave(dev)) 756 return 0; 757 758 if (!is_power_of_2(dev->caps.num_mpts)) 759 return -EINVAL; 760 761 err = mlx4_bitmap_init(&mr_table->mpt_bitmap, dev->caps.num_mpts, 762 ~0, dev->caps.reserved_mrws, 0); 763 if (err) 764 return err; 765 766 err = mlx4_buddy_init(&mr_table->mtt_buddy, 767 ilog2((u32)dev->caps.num_mtts / 768 (1 << log_mtts_per_seg))); 769 if (err) 770 goto err_buddy; 771 772 if (dev->caps.reserved_mtts) { 773 priv->reserved_mtts = 774 mlx4_alloc_mtt_range(dev, 775 fls(dev->caps.reserved_mtts - 1)); 776 if (priv->reserved_mtts < 0) { 777 mlx4_warn(dev, "MTT table of order %u is too small.\n", 778 mr_table->mtt_buddy.max_order); 779 err = -ENOMEM; 780 goto err_reserve_mtts; 781 } 782 } 783 784 return 0; 785 786 err_reserve_mtts: 787 mlx4_buddy_cleanup(&mr_table->mtt_buddy); 788 789 err_buddy: 790 mlx4_bitmap_cleanup(&mr_table->mpt_bitmap); 791 792 return err; 793 } 794 795 void mlx4_cleanup_mr_table(struct mlx4_dev *dev) 796 { 797 struct mlx4_priv *priv = mlx4_priv(dev); 798 struct mlx4_mr_table *mr_table = &priv->mr_table; 799 800 if (mlx4_is_slave(dev)) 801 return; 802 if (priv->reserved_mtts >= 0) 803 mlx4_free_mtt_range(dev, priv->reserved_mtts, 804 fls(dev->caps.reserved_mtts - 1)); 805 mlx4_buddy_cleanup(&mr_table->mtt_buddy); 806 mlx4_bitmap_cleanup(&mr_table->mpt_bitmap); 807 } 808 809 static inline int mlx4_check_fmr(struct mlx4_fmr *fmr, u64 *page_list, 810 int npages, u64 iova) 811 { 812 int i, page_mask; 813 814 if (npages > fmr->max_pages) 815 return -EINVAL; 816 817 page_mask = (1 << fmr->page_shift) - 1; 818 819 /* We are getting page lists, so va must be page aligned. */ 820 if (iova & page_mask) 821 return -EINVAL; 822 823 /* Trust the user not to pass misaligned data in page_list */ 824 if (0) 825 for (i = 0; i < npages; ++i) { 826 if (page_list[i] & ~page_mask) 827 return -EINVAL; 828 } 829 830 if (fmr->maps >= fmr->max_maps) 831 return -EINVAL; 832 833 return 0; 834 } 835 836 int mlx4_map_phys_fmr(struct mlx4_dev *dev, struct mlx4_fmr *fmr, u64 *page_list, 837 int npages, u64 iova, u32 *lkey, u32 *rkey) 838 { 839 u32 key; 840 int i, err; 841 842 err = mlx4_check_fmr(fmr, page_list, npages, iova); 843 if (err) 844 return err; 845 846 ++fmr->maps; 847 848 key = key_to_hw_index(fmr->mr.key); 849 key += dev->caps.num_mpts; 850 *lkey = *rkey = fmr->mr.key = hw_index_to_key(key); 851 852 *(u8 *) fmr->mpt = MLX4_MPT_STATUS_SW; 853 854 /* Make sure MPT status is visible before writing MTT entries */ 855 wmb(); 856 857 dma_sync_single_for_cpu(&dev->pdev->dev, fmr->dma_handle, 858 npages * sizeof(u64), DMA_TO_DEVICE); 859 860 for (i = 0; i < npages; ++i) 861 fmr->mtts[i] = cpu_to_be64(page_list[i] | MLX4_MTT_FLAG_PRESENT); 862 863 dma_sync_single_for_device(&dev->pdev->dev, fmr->dma_handle, 864 npages * sizeof(u64), DMA_TO_DEVICE); 865 866 fmr->mpt->key = cpu_to_be32(key); 867 fmr->mpt->lkey = cpu_to_be32(key); 868 fmr->mpt->length = cpu_to_be64(npages * (1ull << fmr->page_shift)); 869 fmr->mpt->start = cpu_to_be64(iova); 870 871 /* Make MTT entries are visible before setting MPT status */ 872 wmb(); 873 874 *(u8 *) fmr->mpt = MLX4_MPT_STATUS_HW; 875 876 /* Make sure MPT status is visible before consumer can use FMR */ 877 wmb(); 878 879 return 0; 880 } 881 EXPORT_SYMBOL_GPL(mlx4_map_phys_fmr); 882 883 int mlx4_fmr_alloc(struct mlx4_dev *dev, u32 pd, u32 access, int max_pages, 884 int max_maps, u8 page_shift, struct mlx4_fmr *fmr) 885 { 886 struct mlx4_priv *priv = mlx4_priv(dev); 887 int err = -ENOMEM; 888 889 if (max_maps > dev->caps.max_fmr_maps) 890 return -EINVAL; 891 892 if (page_shift < (ffs(dev->caps.page_size_cap) - 1) || page_shift >= 32) 893 return -EINVAL; 894 895 /* All MTTs must fit in the same page */ 896 if (max_pages * sizeof *fmr->mtts > PAGE_SIZE) 897 return -EINVAL; 898 899 fmr->page_shift = page_shift; 900 fmr->max_pages = max_pages; 901 fmr->max_maps = max_maps; 902 fmr->maps = 0; 903 904 err = mlx4_mr_alloc(dev, pd, 0, 0, access, max_pages, 905 page_shift, &fmr->mr); 906 if (err) 907 return err; 908 909 fmr->mtts = mlx4_table_find(&priv->mr_table.mtt_table, 910 fmr->mr.mtt.offset, 911 &fmr->dma_handle); 912 913 if (!fmr->mtts) { 914 err = -ENOMEM; 915 goto err_free; 916 } 917 918 return 0; 919 920 err_free: 921 (void) mlx4_mr_free(dev, &fmr->mr); 922 return err; 923 } 924 EXPORT_SYMBOL_GPL(mlx4_fmr_alloc); 925 926 int mlx4_fmr_enable(struct mlx4_dev *dev, struct mlx4_fmr *fmr) 927 { 928 struct mlx4_priv *priv = mlx4_priv(dev); 929 int err; 930 931 err = mlx4_mr_enable(dev, &fmr->mr); 932 if (err) 933 return err; 934 935 fmr->mpt = mlx4_table_find(&priv->mr_table.dmpt_table, 936 key_to_hw_index(fmr->mr.key), NULL); 937 if (!fmr->mpt) 938 return -ENOMEM; 939 940 return 0; 941 } 942 EXPORT_SYMBOL_GPL(mlx4_fmr_enable); 943 944 void mlx4_fmr_unmap(struct mlx4_dev *dev, struct mlx4_fmr *fmr, 945 u32 *lkey, u32 *rkey) 946 { 947 struct mlx4_cmd_mailbox *mailbox; 948 int err; 949 950 if (!fmr->maps) 951 return; 952 953 fmr->maps = 0; 954 955 mailbox = mlx4_alloc_cmd_mailbox(dev); 956 if (IS_ERR(mailbox)) { 957 err = PTR_ERR(mailbox); 958 printk(KERN_WARNING "mlx4_ib: mlx4_alloc_cmd_mailbox" 959 " failed (%d)\n", err); 960 return; 961 } 962 963 err = mlx4_HW2SW_MPT(dev, NULL, 964 key_to_hw_index(fmr->mr.key) & 965 (dev->caps.num_mpts - 1)); 966 mlx4_free_cmd_mailbox(dev, mailbox); 967 if (err) { 968 printk(KERN_WARNING "mlx4_ib: mlx4_HW2SW_MPT failed (%d)\n", 969 err); 970 return; 971 } 972 fmr->mr.enabled = MLX4_MPT_EN_SW; 973 } 974 EXPORT_SYMBOL_GPL(mlx4_fmr_unmap); 975 976 int mlx4_fmr_free(struct mlx4_dev *dev, struct mlx4_fmr *fmr) 977 { 978 int ret; 979 980 if (fmr->maps) 981 return -EBUSY; 982 983 ret = mlx4_mr_free(dev, &fmr->mr); 984 if (ret) 985 return ret; 986 fmr->mr.enabled = MLX4_MPT_DISABLED; 987 988 return 0; 989 } 990 EXPORT_SYMBOL_GPL(mlx4_fmr_free); 991 992 int mlx4_SYNC_TPT(struct mlx4_dev *dev) 993 { 994 return mlx4_cmd(dev, 0, 0, 0, MLX4_CMD_SYNC_TPT, 1000, 995 MLX4_CMD_NATIVE); 996 } 997 EXPORT_SYMBOL_GPL(mlx4_SYNC_TPT); 998