1 /* 2 * Copyright (c) 2004 Topspin Communications. All rights reserved. 3 * Copyright (c) 2005 Mellanox Technologies. All rights reserved. 4 * 5 * This software is available to you under a choice of one of two 6 * licenses. You may choose to be licensed under the terms of the GNU 7 * General Public License (GPL) Version 2, available from the file 8 * COPYING in the main directory of this source tree, or the 9 * OpenIB.org BSD license below: 10 * 11 * Redistribution and use in source and binary forms, with or 12 * without modification, are permitted provided that the following 13 * conditions are met: 14 * 15 * - Redistributions of source code must retain the above 16 * copyright notice, this list of conditions and the following 17 * disclaimer. 18 * 19 * - Redistributions in binary form must reproduce the above 20 * copyright notice, this list of conditions and the following 21 * disclaimer in the documentation and/or other materials 22 * provided with the distribution. 23 * 24 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 25 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 26 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 27 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 28 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 29 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 30 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 31 * SOFTWARE. 32 * 33 * $Id: mthca_mr.c 1349 2004-12-16 21:09:43Z roland $ 34 */ 35 36 #include <linux/slab.h> 37 #include <linux/init.h> 38 #include <linux/errno.h> 39 40 #include "mthca_dev.h" 41 #include "mthca_cmd.h" 42 #include "mthca_memfree.h" 43 44 struct mthca_mtt { 45 struct mthca_buddy *buddy; 46 int order; 47 u32 first_seg; 48 }; 49 50 /* 51 * Must be packed because mtt_seg is 64 bits but only aligned to 32 bits. 52 */ 53 struct mthca_mpt_entry { 54 __be32 flags; 55 __be32 page_size; 56 __be32 key; 57 __be32 pd; 58 __be64 start; 59 __be64 length; 60 __be32 lkey; 61 __be32 window_count; 62 __be32 window_count_limit; 63 __be64 mtt_seg; 64 __be32 mtt_sz; /* Arbel only */ 65 u32 reserved[2]; 66 } __attribute__((packed)); 67 68 #define MTHCA_MPT_FLAG_SW_OWNS (0xfUL << 28) 69 #define MTHCA_MPT_FLAG_MIO (1 << 17) 70 #define MTHCA_MPT_FLAG_BIND_ENABLE (1 << 15) 71 #define MTHCA_MPT_FLAG_PHYSICAL (1 << 9) 72 #define MTHCA_MPT_FLAG_REGION (1 << 8) 73 74 #define MTHCA_MTT_FLAG_PRESENT 1 75 76 #define MTHCA_MPT_STATUS_SW 0xF0 77 #define MTHCA_MPT_STATUS_HW 0x00 78 79 /* 80 * Buddy allocator for MTT segments (currently not very efficient 81 * since it doesn't keep a free list and just searches linearly 82 * through the bitmaps) 83 */ 84 85 static u32 mthca_buddy_alloc(struct mthca_buddy *buddy, int order) 86 { 87 int o; 88 int m; 89 u32 seg; 90 91 spin_lock(&buddy->lock); 92 93 for (o = order; o <= buddy->max_order; ++o) { 94 m = 1 << (buddy->max_order - o); 95 seg = find_first_bit(buddy->bits[o], m); 96 if (seg < m) 97 goto found; 98 } 99 100 spin_unlock(&buddy->lock); 101 return -1; 102 103 found: 104 clear_bit(seg, buddy->bits[o]); 105 106 while (o > order) { 107 --o; 108 seg <<= 1; 109 set_bit(seg ^ 1, buddy->bits[o]); 110 } 111 112 spin_unlock(&buddy->lock); 113 114 seg <<= order; 115 116 return seg; 117 } 118 119 static void mthca_buddy_free(struct mthca_buddy *buddy, u32 seg, int order) 120 { 121 seg >>= order; 122 123 spin_lock(&buddy->lock); 124 125 while (test_bit(seg ^ 1, buddy->bits[order])) { 126 clear_bit(seg ^ 1, buddy->bits[order]); 127 seg >>= 1; 128 ++order; 129 } 130 131 set_bit(seg, buddy->bits[order]); 132 133 spin_unlock(&buddy->lock); 134 } 135 136 static int __devinit mthca_buddy_init(struct mthca_buddy *buddy, int max_order) 137 { 138 int i, s; 139 140 buddy->max_order = max_order; 141 spin_lock_init(&buddy->lock); 142 143 buddy->bits = kzalloc((buddy->max_order + 1) * sizeof (long *), 144 GFP_KERNEL); 145 if (!buddy->bits) 146 goto err_out; 147 148 for (i = 0; i <= buddy->max_order; ++i) { 149 s = BITS_TO_LONGS(1 << (buddy->max_order - i)); 150 buddy->bits[i] = kmalloc(s * sizeof (long), GFP_KERNEL); 151 if (!buddy->bits[i]) 152 goto err_out_free; 153 bitmap_zero(buddy->bits[i], 154 1 << (buddy->max_order - i)); 155 } 156 157 set_bit(0, buddy->bits[buddy->max_order]); 158 159 return 0; 160 161 err_out_free: 162 for (i = 0; i <= buddy->max_order; ++i) 163 kfree(buddy->bits[i]); 164 165 kfree(buddy->bits); 166 167 err_out: 168 return -ENOMEM; 169 } 170 171 static void __devexit mthca_buddy_cleanup(struct mthca_buddy *buddy) 172 { 173 int i; 174 175 for (i = 0; i <= buddy->max_order; ++i) 176 kfree(buddy->bits[i]); 177 178 kfree(buddy->bits); 179 } 180 181 static u32 mthca_alloc_mtt_range(struct mthca_dev *dev, int order, 182 struct mthca_buddy *buddy) 183 { 184 u32 seg = mthca_buddy_alloc(buddy, order); 185 186 if (seg == -1) 187 return -1; 188 189 if (mthca_is_memfree(dev)) 190 if (mthca_table_get_range(dev, dev->mr_table.mtt_table, seg, 191 seg + (1 << order) - 1)) { 192 mthca_buddy_free(buddy, seg, order); 193 seg = -1; 194 } 195 196 return seg; 197 } 198 199 static struct mthca_mtt *__mthca_alloc_mtt(struct mthca_dev *dev, int size, 200 struct mthca_buddy *buddy) 201 { 202 struct mthca_mtt *mtt; 203 int i; 204 205 if (size <= 0) 206 return ERR_PTR(-EINVAL); 207 208 mtt = kmalloc(sizeof *mtt, GFP_KERNEL); 209 if (!mtt) 210 return ERR_PTR(-ENOMEM); 211 212 mtt->buddy = buddy; 213 mtt->order = 0; 214 for (i = MTHCA_MTT_SEG_SIZE / 8; i < size; i <<= 1) 215 ++mtt->order; 216 217 mtt->first_seg = mthca_alloc_mtt_range(dev, mtt->order, buddy); 218 if (mtt->first_seg == -1) { 219 kfree(mtt); 220 return ERR_PTR(-ENOMEM); 221 } 222 223 return mtt; 224 } 225 226 struct mthca_mtt *mthca_alloc_mtt(struct mthca_dev *dev, int size) 227 { 228 return __mthca_alloc_mtt(dev, size, &dev->mr_table.mtt_buddy); 229 } 230 231 void mthca_free_mtt(struct mthca_dev *dev, struct mthca_mtt *mtt) 232 { 233 if (!mtt) 234 return; 235 236 mthca_buddy_free(mtt->buddy, mtt->first_seg, mtt->order); 237 238 mthca_table_put_range(dev, dev->mr_table.mtt_table, 239 mtt->first_seg, 240 mtt->first_seg + (1 << mtt->order) - 1); 241 242 kfree(mtt); 243 } 244 245 int mthca_write_mtt(struct mthca_dev *dev, struct mthca_mtt *mtt, 246 int start_index, u64 *buffer_list, int list_len) 247 { 248 struct mthca_mailbox *mailbox; 249 __be64 *mtt_entry; 250 int err = 0; 251 u8 status; 252 int i; 253 254 mailbox = mthca_alloc_mailbox(dev, GFP_KERNEL); 255 if (IS_ERR(mailbox)) 256 return PTR_ERR(mailbox); 257 mtt_entry = mailbox->buf; 258 259 while (list_len > 0) { 260 mtt_entry[0] = cpu_to_be64(dev->mr_table.mtt_base + 261 mtt->first_seg * MTHCA_MTT_SEG_SIZE + 262 start_index * 8); 263 mtt_entry[1] = 0; 264 for (i = 0; i < list_len && i < MTHCA_MAILBOX_SIZE / 8 - 2; ++i) 265 mtt_entry[i + 2] = cpu_to_be64(buffer_list[i] | 266 MTHCA_MTT_FLAG_PRESENT); 267 268 /* 269 * If we have an odd number of entries to write, add 270 * one more dummy entry for firmware efficiency. 271 */ 272 if (i & 1) 273 mtt_entry[i + 2] = 0; 274 275 err = mthca_WRITE_MTT(dev, mailbox, (i + 1) & ~1, &status); 276 if (err) { 277 mthca_warn(dev, "WRITE_MTT failed (%d)\n", err); 278 goto out; 279 } 280 if (status) { 281 mthca_warn(dev, "WRITE_MTT returned status 0x%02x\n", 282 status); 283 err = -EINVAL; 284 goto out; 285 } 286 287 list_len -= i; 288 start_index += i; 289 buffer_list += i; 290 } 291 292 out: 293 mthca_free_mailbox(dev, mailbox); 294 return err; 295 } 296 297 static inline u32 tavor_hw_index_to_key(u32 ind) 298 { 299 return ind; 300 } 301 302 static inline u32 tavor_key_to_hw_index(u32 key) 303 { 304 return key; 305 } 306 307 static inline u32 arbel_hw_index_to_key(u32 ind) 308 { 309 return (ind >> 24) | (ind << 8); 310 } 311 312 static inline u32 arbel_key_to_hw_index(u32 key) 313 { 314 return (key << 24) | (key >> 8); 315 } 316 317 static inline u32 hw_index_to_key(struct mthca_dev *dev, u32 ind) 318 { 319 if (mthca_is_memfree(dev)) 320 return arbel_hw_index_to_key(ind); 321 else 322 return tavor_hw_index_to_key(ind); 323 } 324 325 static inline u32 key_to_hw_index(struct mthca_dev *dev, u32 key) 326 { 327 if (mthca_is_memfree(dev)) 328 return arbel_key_to_hw_index(key); 329 else 330 return tavor_key_to_hw_index(key); 331 } 332 333 int mthca_mr_alloc(struct mthca_dev *dev, u32 pd, int buffer_size_shift, 334 u64 iova, u64 total_size, u32 access, struct mthca_mr *mr) 335 { 336 struct mthca_mailbox *mailbox; 337 struct mthca_mpt_entry *mpt_entry; 338 u32 key; 339 int i; 340 int err; 341 u8 status; 342 343 might_sleep(); 344 345 WARN_ON(buffer_size_shift >= 32); 346 347 key = mthca_alloc(&dev->mr_table.mpt_alloc); 348 if (key == -1) 349 return -ENOMEM; 350 mr->ibmr.rkey = mr->ibmr.lkey = hw_index_to_key(dev, key); 351 352 if (mthca_is_memfree(dev)) { 353 err = mthca_table_get(dev, dev->mr_table.mpt_table, key); 354 if (err) 355 goto err_out_mpt_free; 356 } 357 358 mailbox = mthca_alloc_mailbox(dev, GFP_KERNEL); 359 if (IS_ERR(mailbox)) { 360 err = PTR_ERR(mailbox); 361 goto err_out_table; 362 } 363 mpt_entry = mailbox->buf; 364 365 mpt_entry->flags = cpu_to_be32(MTHCA_MPT_FLAG_SW_OWNS | 366 MTHCA_MPT_FLAG_MIO | 367 MTHCA_MPT_FLAG_REGION | 368 access); 369 if (!mr->mtt) 370 mpt_entry->flags |= cpu_to_be32(MTHCA_MPT_FLAG_PHYSICAL); 371 372 mpt_entry->page_size = cpu_to_be32(buffer_size_shift - 12); 373 mpt_entry->key = cpu_to_be32(key); 374 mpt_entry->pd = cpu_to_be32(pd); 375 mpt_entry->start = cpu_to_be64(iova); 376 mpt_entry->length = cpu_to_be64(total_size); 377 378 memset(&mpt_entry->lkey, 0, 379 sizeof *mpt_entry - offsetof(struct mthca_mpt_entry, lkey)); 380 381 if (mr->mtt) 382 mpt_entry->mtt_seg = 383 cpu_to_be64(dev->mr_table.mtt_base + 384 mr->mtt->first_seg * MTHCA_MTT_SEG_SIZE); 385 386 if (0) { 387 mthca_dbg(dev, "Dumping MPT entry %08x:\n", mr->ibmr.lkey); 388 for (i = 0; i < sizeof (struct mthca_mpt_entry) / 4; ++i) { 389 if (i % 4 == 0) 390 printk("[%02x] ", i * 4); 391 printk(" %08x", be32_to_cpu(((__be32 *) mpt_entry)[i])); 392 if ((i + 1) % 4 == 0) 393 printk("\n"); 394 } 395 } 396 397 err = mthca_SW2HW_MPT(dev, mailbox, 398 key & (dev->limits.num_mpts - 1), 399 &status); 400 if (err) { 401 mthca_warn(dev, "SW2HW_MPT failed (%d)\n", err); 402 goto err_out_mailbox; 403 } else if (status) { 404 mthca_warn(dev, "SW2HW_MPT returned status 0x%02x\n", 405 status); 406 err = -EINVAL; 407 goto err_out_mailbox; 408 } 409 410 mthca_free_mailbox(dev, mailbox); 411 return err; 412 413 err_out_mailbox: 414 mthca_free_mailbox(dev, mailbox); 415 416 err_out_table: 417 mthca_table_put(dev, dev->mr_table.mpt_table, key); 418 419 err_out_mpt_free: 420 mthca_free(&dev->mr_table.mpt_alloc, key); 421 return err; 422 } 423 424 int mthca_mr_alloc_notrans(struct mthca_dev *dev, u32 pd, 425 u32 access, struct mthca_mr *mr) 426 { 427 mr->mtt = NULL; 428 return mthca_mr_alloc(dev, pd, 12, 0, ~0ULL, access, mr); 429 } 430 431 int mthca_mr_alloc_phys(struct mthca_dev *dev, u32 pd, 432 u64 *buffer_list, int buffer_size_shift, 433 int list_len, u64 iova, u64 total_size, 434 u32 access, struct mthca_mr *mr) 435 { 436 int err; 437 438 mr->mtt = mthca_alloc_mtt(dev, list_len); 439 if (IS_ERR(mr->mtt)) 440 return PTR_ERR(mr->mtt); 441 442 err = mthca_write_mtt(dev, mr->mtt, 0, buffer_list, list_len); 443 if (err) { 444 mthca_free_mtt(dev, mr->mtt); 445 return err; 446 } 447 448 err = mthca_mr_alloc(dev, pd, buffer_size_shift, iova, 449 total_size, access, mr); 450 if (err) 451 mthca_free_mtt(dev, mr->mtt); 452 453 return err; 454 } 455 456 /* Free mr or fmr */ 457 static void mthca_free_region(struct mthca_dev *dev, u32 lkey) 458 { 459 mthca_table_put(dev, dev->mr_table.mpt_table, 460 key_to_hw_index(dev, lkey)); 461 462 mthca_free(&dev->mr_table.mpt_alloc, key_to_hw_index(dev, lkey)); 463 } 464 465 void mthca_free_mr(struct mthca_dev *dev, struct mthca_mr *mr) 466 { 467 int err; 468 u8 status; 469 470 might_sleep(); 471 472 err = mthca_HW2SW_MPT(dev, NULL, 473 key_to_hw_index(dev, mr->ibmr.lkey) & 474 (dev->limits.num_mpts - 1), 475 &status); 476 if (err) 477 mthca_warn(dev, "HW2SW_MPT failed (%d)\n", err); 478 else if (status) 479 mthca_warn(dev, "HW2SW_MPT returned status 0x%02x\n", 480 status); 481 482 mthca_free_region(dev, mr->ibmr.lkey); 483 mthca_free_mtt(dev, mr->mtt); 484 } 485 486 int mthca_fmr_alloc(struct mthca_dev *dev, u32 pd, 487 u32 access, struct mthca_fmr *mr) 488 { 489 struct mthca_mpt_entry *mpt_entry; 490 struct mthca_mailbox *mailbox; 491 u64 mtt_seg; 492 u32 key, idx; 493 u8 status; 494 int list_len = mr->attr.max_pages; 495 int err = -ENOMEM; 496 int i; 497 498 might_sleep(); 499 500 if (mr->attr.page_size < 12 || mr->attr.page_size >= 32) 501 return -EINVAL; 502 503 /* For Arbel, all MTTs must fit in the same page. */ 504 if (mthca_is_memfree(dev) && 505 mr->attr.max_pages * sizeof *mr->mem.arbel.mtts > PAGE_SIZE) 506 return -EINVAL; 507 508 mr->maps = 0; 509 510 key = mthca_alloc(&dev->mr_table.mpt_alloc); 511 if (key == -1) 512 return -ENOMEM; 513 514 idx = key & (dev->limits.num_mpts - 1); 515 mr->ibmr.rkey = mr->ibmr.lkey = hw_index_to_key(dev, key); 516 517 if (mthca_is_memfree(dev)) { 518 err = mthca_table_get(dev, dev->mr_table.mpt_table, key); 519 if (err) 520 goto err_out_mpt_free; 521 522 mr->mem.arbel.mpt = mthca_table_find(dev->mr_table.mpt_table, key); 523 BUG_ON(!mr->mem.arbel.mpt); 524 } else 525 mr->mem.tavor.mpt = dev->mr_table.tavor_fmr.mpt_base + 526 sizeof *(mr->mem.tavor.mpt) * idx; 527 528 mr->mtt = __mthca_alloc_mtt(dev, list_len, dev->mr_table.fmr_mtt_buddy); 529 if (IS_ERR(mr->mtt)) 530 goto err_out_table; 531 532 mtt_seg = mr->mtt->first_seg * MTHCA_MTT_SEG_SIZE; 533 534 if (mthca_is_memfree(dev)) { 535 mr->mem.arbel.mtts = mthca_table_find(dev->mr_table.mtt_table, 536 mr->mtt->first_seg); 537 BUG_ON(!mr->mem.arbel.mtts); 538 } else 539 mr->mem.tavor.mtts = dev->mr_table.tavor_fmr.mtt_base + mtt_seg; 540 541 mailbox = mthca_alloc_mailbox(dev, GFP_KERNEL); 542 if (IS_ERR(mailbox)) 543 goto err_out_free_mtt; 544 545 mpt_entry = mailbox->buf; 546 547 mpt_entry->flags = cpu_to_be32(MTHCA_MPT_FLAG_SW_OWNS | 548 MTHCA_MPT_FLAG_MIO | 549 MTHCA_MPT_FLAG_REGION | 550 access); 551 552 mpt_entry->page_size = cpu_to_be32(mr->attr.page_size - 12); 553 mpt_entry->key = cpu_to_be32(key); 554 mpt_entry->pd = cpu_to_be32(pd); 555 memset(&mpt_entry->start, 0, 556 sizeof *mpt_entry - offsetof(struct mthca_mpt_entry, start)); 557 mpt_entry->mtt_seg = cpu_to_be64(dev->mr_table.mtt_base + mtt_seg); 558 559 if (0) { 560 mthca_dbg(dev, "Dumping MPT entry %08x:\n", mr->ibmr.lkey); 561 for (i = 0; i < sizeof (struct mthca_mpt_entry) / 4; ++i) { 562 if (i % 4 == 0) 563 printk("[%02x] ", i * 4); 564 printk(" %08x", be32_to_cpu(((__be32 *) mpt_entry)[i])); 565 if ((i + 1) % 4 == 0) 566 printk("\n"); 567 } 568 } 569 570 err = mthca_SW2HW_MPT(dev, mailbox, 571 key & (dev->limits.num_mpts - 1), 572 &status); 573 if (err) { 574 mthca_warn(dev, "SW2HW_MPT failed (%d)\n", err); 575 goto err_out_mailbox_free; 576 } 577 if (status) { 578 mthca_warn(dev, "SW2HW_MPT returned status 0x%02x\n", 579 status); 580 err = -EINVAL; 581 goto err_out_mailbox_free; 582 } 583 584 mthca_free_mailbox(dev, mailbox); 585 return 0; 586 587 err_out_mailbox_free: 588 mthca_free_mailbox(dev, mailbox); 589 590 err_out_free_mtt: 591 mthca_free_mtt(dev, mr->mtt); 592 593 err_out_table: 594 mthca_table_put(dev, dev->mr_table.mpt_table, key); 595 596 err_out_mpt_free: 597 mthca_free(&dev->mr_table.mpt_alloc, mr->ibmr.lkey); 598 return err; 599 } 600 601 int mthca_free_fmr(struct mthca_dev *dev, struct mthca_fmr *fmr) 602 { 603 if (fmr->maps) 604 return -EBUSY; 605 606 mthca_free_region(dev, fmr->ibmr.lkey); 607 mthca_free_mtt(dev, fmr->mtt); 608 609 return 0; 610 } 611 612 static inline int mthca_check_fmr(struct mthca_fmr *fmr, u64 *page_list, 613 int list_len, u64 iova) 614 { 615 int i, page_mask; 616 617 if (list_len > fmr->attr.max_pages) 618 return -EINVAL; 619 620 page_mask = (1 << fmr->attr.page_size) - 1; 621 622 /* We are getting page lists, so va must be page aligned. */ 623 if (iova & page_mask) 624 return -EINVAL; 625 626 /* Trust the user not to pass misaligned data in page_list */ 627 if (0) 628 for (i = 0; i < list_len; ++i) { 629 if (page_list[i] & ~page_mask) 630 return -EINVAL; 631 } 632 633 if (fmr->maps >= fmr->attr.max_maps) 634 return -EINVAL; 635 636 return 0; 637 } 638 639 640 int mthca_tavor_map_phys_fmr(struct ib_fmr *ibfmr, u64 *page_list, 641 int list_len, u64 iova) 642 { 643 struct mthca_fmr *fmr = to_mfmr(ibfmr); 644 struct mthca_dev *dev = to_mdev(ibfmr->device); 645 struct mthca_mpt_entry mpt_entry; 646 u32 key; 647 int i, err; 648 649 err = mthca_check_fmr(fmr, page_list, list_len, iova); 650 if (err) 651 return err; 652 653 ++fmr->maps; 654 655 key = tavor_key_to_hw_index(fmr->ibmr.lkey); 656 key += dev->limits.num_mpts; 657 fmr->ibmr.lkey = fmr->ibmr.rkey = tavor_hw_index_to_key(key); 658 659 writeb(MTHCA_MPT_STATUS_SW, fmr->mem.tavor.mpt); 660 661 for (i = 0; i < list_len; ++i) { 662 __be64 mtt_entry = cpu_to_be64(page_list[i] | 663 MTHCA_MTT_FLAG_PRESENT); 664 mthca_write64_raw(mtt_entry, fmr->mem.tavor.mtts + i); 665 } 666 667 mpt_entry.lkey = cpu_to_be32(key); 668 mpt_entry.length = cpu_to_be64(list_len * (1ull << fmr->attr.page_size)); 669 mpt_entry.start = cpu_to_be64(iova); 670 671 __raw_writel((__force u32) mpt_entry.lkey, &fmr->mem.tavor.mpt->key); 672 memcpy_toio(&fmr->mem.tavor.mpt->start, &mpt_entry.start, 673 offsetof(struct mthca_mpt_entry, window_count) - 674 offsetof(struct mthca_mpt_entry, start)); 675 676 writeb(MTHCA_MPT_STATUS_HW, fmr->mem.tavor.mpt); 677 678 return 0; 679 } 680 681 int mthca_arbel_map_phys_fmr(struct ib_fmr *ibfmr, u64 *page_list, 682 int list_len, u64 iova) 683 { 684 struct mthca_fmr *fmr = to_mfmr(ibfmr); 685 struct mthca_dev *dev = to_mdev(ibfmr->device); 686 u32 key; 687 int i, err; 688 689 err = mthca_check_fmr(fmr, page_list, list_len, iova); 690 if (err) 691 return err; 692 693 ++fmr->maps; 694 695 key = arbel_key_to_hw_index(fmr->ibmr.lkey); 696 key += dev->limits.num_mpts; 697 fmr->ibmr.lkey = fmr->ibmr.rkey = arbel_hw_index_to_key(key); 698 699 *(u8 *) fmr->mem.arbel.mpt = MTHCA_MPT_STATUS_SW; 700 701 wmb(); 702 703 for (i = 0; i < list_len; ++i) 704 fmr->mem.arbel.mtts[i] = cpu_to_be64(page_list[i] | 705 MTHCA_MTT_FLAG_PRESENT); 706 707 fmr->mem.arbel.mpt->key = cpu_to_be32(key); 708 fmr->mem.arbel.mpt->lkey = cpu_to_be32(key); 709 fmr->mem.arbel.mpt->length = cpu_to_be64(list_len * (1ull << fmr->attr.page_size)); 710 fmr->mem.arbel.mpt->start = cpu_to_be64(iova); 711 712 wmb(); 713 714 *(u8 *) fmr->mem.arbel.mpt = MTHCA_MPT_STATUS_HW; 715 716 wmb(); 717 718 return 0; 719 } 720 721 void mthca_tavor_fmr_unmap(struct mthca_dev *dev, struct mthca_fmr *fmr) 722 { 723 u32 key; 724 725 if (!fmr->maps) 726 return; 727 728 key = tavor_key_to_hw_index(fmr->ibmr.lkey); 729 key &= dev->limits.num_mpts - 1; 730 fmr->ibmr.lkey = fmr->ibmr.rkey = tavor_hw_index_to_key(key); 731 732 fmr->maps = 0; 733 734 writeb(MTHCA_MPT_STATUS_SW, fmr->mem.tavor.mpt); 735 } 736 737 void mthca_arbel_fmr_unmap(struct mthca_dev *dev, struct mthca_fmr *fmr) 738 { 739 u32 key; 740 741 if (!fmr->maps) 742 return; 743 744 key = arbel_key_to_hw_index(fmr->ibmr.lkey); 745 key &= dev->limits.num_mpts - 1; 746 fmr->ibmr.lkey = fmr->ibmr.rkey = arbel_hw_index_to_key(key); 747 748 fmr->maps = 0; 749 750 *(u8 *) fmr->mem.arbel.mpt = MTHCA_MPT_STATUS_SW; 751 } 752 753 int __devinit mthca_init_mr_table(struct mthca_dev *dev) 754 { 755 int err, i; 756 757 err = mthca_alloc_init(&dev->mr_table.mpt_alloc, 758 dev->limits.num_mpts, 759 ~0, dev->limits.reserved_mrws); 760 if (err) 761 return err; 762 763 if (!mthca_is_memfree(dev) && 764 (dev->mthca_flags & MTHCA_FLAG_DDR_HIDDEN)) 765 dev->limits.fmr_reserved_mtts = 0; 766 else 767 dev->mthca_flags |= MTHCA_FLAG_FMR; 768 769 err = mthca_buddy_init(&dev->mr_table.mtt_buddy, 770 fls(dev->limits.num_mtt_segs - 1)); 771 772 if (err) 773 goto err_mtt_buddy; 774 775 dev->mr_table.tavor_fmr.mpt_base = NULL; 776 dev->mr_table.tavor_fmr.mtt_base = NULL; 777 778 if (dev->limits.fmr_reserved_mtts) { 779 i = fls(dev->limits.fmr_reserved_mtts - 1); 780 781 if (i >= 31) { 782 mthca_warn(dev, "Unable to reserve 2^31 FMR MTTs.\n"); 783 err = -EINVAL; 784 goto err_fmr_mpt; 785 } 786 787 dev->mr_table.tavor_fmr.mpt_base = 788 ioremap(dev->mr_table.mpt_base, 789 (1 << i) * sizeof (struct mthca_mpt_entry)); 790 791 if (!dev->mr_table.tavor_fmr.mpt_base) { 792 mthca_warn(dev, "MPT ioremap for FMR failed.\n"); 793 err = -ENOMEM; 794 goto err_fmr_mpt; 795 } 796 797 dev->mr_table.tavor_fmr.mtt_base = 798 ioremap(dev->mr_table.mtt_base, 799 (1 << i) * MTHCA_MTT_SEG_SIZE); 800 if (!dev->mr_table.tavor_fmr.mtt_base) { 801 mthca_warn(dev, "MTT ioremap for FMR failed.\n"); 802 err = -ENOMEM; 803 goto err_fmr_mtt; 804 } 805 806 err = mthca_buddy_init(&dev->mr_table.tavor_fmr.mtt_buddy, i); 807 if (err) 808 goto err_fmr_mtt_buddy; 809 810 /* Prevent regular MRs from using FMR keys */ 811 err = mthca_buddy_alloc(&dev->mr_table.mtt_buddy, i); 812 if (err) 813 goto err_reserve_fmr; 814 815 dev->mr_table.fmr_mtt_buddy = 816 &dev->mr_table.tavor_fmr.mtt_buddy; 817 } else 818 dev->mr_table.fmr_mtt_buddy = &dev->mr_table.mtt_buddy; 819 820 /* FMR table is always the first, take reserved MTTs out of there */ 821 if (dev->limits.reserved_mtts) { 822 i = fls(dev->limits.reserved_mtts - 1); 823 824 if (mthca_alloc_mtt_range(dev, i, 825 dev->mr_table.fmr_mtt_buddy) == -1) { 826 mthca_warn(dev, "MTT table of order %d is too small.\n", 827 dev->mr_table.fmr_mtt_buddy->max_order); 828 err = -ENOMEM; 829 goto err_reserve_mtts; 830 } 831 } 832 833 return 0; 834 835 err_reserve_mtts: 836 err_reserve_fmr: 837 if (dev->limits.fmr_reserved_mtts) 838 mthca_buddy_cleanup(&dev->mr_table.tavor_fmr.mtt_buddy); 839 840 err_fmr_mtt_buddy: 841 if (dev->mr_table.tavor_fmr.mtt_base) 842 iounmap(dev->mr_table.tavor_fmr.mtt_base); 843 844 err_fmr_mtt: 845 if (dev->mr_table.tavor_fmr.mpt_base) 846 iounmap(dev->mr_table.tavor_fmr.mpt_base); 847 848 err_fmr_mpt: 849 mthca_buddy_cleanup(&dev->mr_table.mtt_buddy); 850 851 err_mtt_buddy: 852 mthca_alloc_cleanup(&dev->mr_table.mpt_alloc); 853 854 return err; 855 } 856 857 void __devexit mthca_cleanup_mr_table(struct mthca_dev *dev) 858 { 859 /* XXX check if any MRs are still allocated? */ 860 if (dev->limits.fmr_reserved_mtts) 861 mthca_buddy_cleanup(&dev->mr_table.tavor_fmr.mtt_buddy); 862 863 mthca_buddy_cleanup(&dev->mr_table.mtt_buddy); 864 865 if (dev->mr_table.tavor_fmr.mtt_base) 866 iounmap(dev->mr_table.tavor_fmr.mtt_base); 867 if (dev->mr_table.tavor_fmr.mpt_base) 868 iounmap(dev->mr_table.tavor_fmr.mpt_base); 869 870 mthca_alloc_cleanup(&dev->mr_table.mpt_alloc); 871 } 872