1 /* 2 * Copyright (C) 2001 Mike Corrigan & Dave Engebretsen, IBM Corporation 3 * 4 * Rewrite, cleanup, new allocation schemes, virtual merging: 5 * Copyright (C) 2004 Olof Johansson, IBM Corporation 6 * and Ben. Herrenschmidt, IBM Corporation 7 * 8 * Dynamic DMA mapping support, bus-independent parts. 9 * 10 * This program is free software; you can redistribute it and/or modify 11 * it under the terms of the GNU General Public License as published by 12 * the Free Software Foundation; either version 2 of the License, or 13 * (at your option) any later version. 14 * 15 * This program is distributed in the hope that it will be useful, 16 * but WITHOUT ANY WARRANTY; without even the implied warranty of 17 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 18 * GNU General Public License for more details. 19 * 20 * You should have received a copy of the GNU General Public License 21 * along with this program; if not, write to the Free Software 22 * Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA 23 */ 24 25 26 #include <linux/init.h> 27 #include <linux/types.h> 28 #include <linux/slab.h> 29 #include <linux/mm.h> 30 #include <linux/spinlock.h> 31 #include <linux/string.h> 32 #include <linux/dma-mapping.h> 33 #include <linux/bitops.h> 34 #include <linux/iommu-helper.h> 35 #include <asm/io.h> 36 #include <asm/prom.h> 37 #include <asm/iommu.h> 38 #include <asm/pci-bridge.h> 39 #include <asm/machdep.h> 40 #include <asm/kdump.h> 41 42 #define DBG(...) 43 44 #ifdef CONFIG_IOMMU_VMERGE 45 static int novmerge = 0; 46 #else 47 static int novmerge = 1; 48 #endif 49 50 static int protect4gb = 1; 51 52 static void __iommu_free(struct iommu_table *, dma_addr_t, unsigned int); 53 54 static int __init setup_protect4gb(char *str) 55 { 56 if (strcmp(str, "on") == 0) 57 protect4gb = 1; 58 else if (strcmp(str, "off") == 0) 59 protect4gb = 0; 60 61 return 1; 62 } 63 64 static int __init setup_iommu(char *str) 65 { 66 if (!strcmp(str, "novmerge")) 67 novmerge = 1; 68 else if (!strcmp(str, "vmerge")) 69 novmerge = 0; 70 return 1; 71 } 72 73 __setup("protect4gb=", setup_protect4gb); 74 __setup("iommu=", setup_iommu); 75 76 static unsigned long iommu_range_alloc(struct device *dev, 77 struct iommu_table *tbl, 78 unsigned long npages, 79 unsigned long *handle, 80 unsigned long mask, 81 unsigned int align_order) 82 { 83 unsigned long n, end, start; 84 unsigned long limit; 85 int largealloc = npages > 15; 86 int pass = 0; 87 unsigned long align_mask; 88 unsigned long boundary_size; 89 90 align_mask = 0xffffffffffffffffl >> (64 - align_order); 91 92 /* This allocator was derived from x86_64's bit string search */ 93 94 /* Sanity check */ 95 if (unlikely(npages == 0)) { 96 if (printk_ratelimit()) 97 WARN_ON(1); 98 return DMA_ERROR_CODE; 99 } 100 101 if (handle && *handle) 102 start = *handle; 103 else 104 start = largealloc ? tbl->it_largehint : tbl->it_hint; 105 106 /* Use only half of the table for small allocs (15 pages or less) */ 107 limit = largealloc ? tbl->it_size : tbl->it_halfpoint; 108 109 if (largealloc && start < tbl->it_halfpoint) 110 start = tbl->it_halfpoint; 111 112 /* The case below can happen if we have a small segment appended 113 * to a large, or when the previous alloc was at the very end of 114 * the available space. If so, go back to the initial start. 115 */ 116 if (start >= limit) 117 start = largealloc ? tbl->it_largehint : tbl->it_hint; 118 119 again: 120 121 if (limit + tbl->it_offset > mask) { 122 limit = mask - tbl->it_offset + 1; 123 /* If we're constrained on address range, first try 124 * at the masked hint to avoid O(n) search complexity, 125 * but on second pass, start at 0. 126 */ 127 if ((start & mask) >= limit || pass > 0) 128 start = 0; 129 else 130 start &= mask; 131 } 132 133 if (dev) 134 boundary_size = ALIGN(dma_get_seg_boundary(dev) + 1, 135 1 << IOMMU_PAGE_SHIFT); 136 else 137 boundary_size = ALIGN(1UL << 32, 1 << IOMMU_PAGE_SHIFT); 138 /* 4GB boundary for iseries_hv_alloc and iseries_hv_map */ 139 140 n = iommu_area_alloc(tbl->it_map, limit, start, npages, 141 tbl->it_offset, boundary_size >> IOMMU_PAGE_SHIFT, 142 align_mask); 143 if (n == -1) { 144 if (likely(pass < 2)) { 145 /* First failure, just rescan the half of the table. 146 * Second failure, rescan the other half of the table. 147 */ 148 start = (largealloc ^ pass) ? tbl->it_halfpoint : 0; 149 limit = pass ? tbl->it_size : limit; 150 pass++; 151 goto again; 152 } else { 153 /* Third failure, give up */ 154 return DMA_ERROR_CODE; 155 } 156 } 157 158 end = n + npages; 159 160 /* Bump the hint to a new block for small allocs. */ 161 if (largealloc) { 162 /* Don't bump to new block to avoid fragmentation */ 163 tbl->it_largehint = end; 164 } else { 165 /* Overflow will be taken care of at the next allocation */ 166 tbl->it_hint = (end + tbl->it_blocksize - 1) & 167 ~(tbl->it_blocksize - 1); 168 } 169 170 /* Update handle for SG allocations */ 171 if (handle) 172 *handle = end; 173 174 return n; 175 } 176 177 static dma_addr_t iommu_alloc(struct device *dev, struct iommu_table *tbl, 178 void *page, unsigned int npages, 179 enum dma_data_direction direction, 180 unsigned long mask, unsigned int align_order, 181 struct dma_attrs *attrs) 182 { 183 unsigned long entry, flags; 184 dma_addr_t ret = DMA_ERROR_CODE; 185 int build_fail; 186 187 spin_lock_irqsave(&(tbl->it_lock), flags); 188 189 entry = iommu_range_alloc(dev, tbl, npages, NULL, mask, align_order); 190 191 if (unlikely(entry == DMA_ERROR_CODE)) { 192 spin_unlock_irqrestore(&(tbl->it_lock), flags); 193 return DMA_ERROR_CODE; 194 } 195 196 entry += tbl->it_offset; /* Offset into real TCE table */ 197 ret = entry << IOMMU_PAGE_SHIFT; /* Set the return dma address */ 198 199 /* Put the TCEs in the HW table */ 200 build_fail = ppc_md.tce_build(tbl, entry, npages, 201 (unsigned long)page & IOMMU_PAGE_MASK, 202 direction, attrs); 203 204 /* ppc_md.tce_build() only returns non-zero for transient errors. 205 * Clean up the table bitmap in this case and return 206 * DMA_ERROR_CODE. For all other errors the functionality is 207 * not altered. 208 */ 209 if (unlikely(build_fail)) { 210 __iommu_free(tbl, ret, npages); 211 212 spin_unlock_irqrestore(&(tbl->it_lock), flags); 213 return DMA_ERROR_CODE; 214 } 215 216 /* Flush/invalidate TLB caches if necessary */ 217 if (ppc_md.tce_flush) 218 ppc_md.tce_flush(tbl); 219 220 spin_unlock_irqrestore(&(tbl->it_lock), flags); 221 222 /* Make sure updates are seen by hardware */ 223 mb(); 224 225 return ret; 226 } 227 228 static void __iommu_free(struct iommu_table *tbl, dma_addr_t dma_addr, 229 unsigned int npages) 230 { 231 unsigned long entry, free_entry; 232 233 entry = dma_addr >> IOMMU_PAGE_SHIFT; 234 free_entry = entry - tbl->it_offset; 235 236 if (((free_entry + npages) > tbl->it_size) || 237 (entry < tbl->it_offset)) { 238 if (printk_ratelimit()) { 239 printk(KERN_INFO "iommu_free: invalid entry\n"); 240 printk(KERN_INFO "\tentry = 0x%lx\n", entry); 241 printk(KERN_INFO "\tdma_addr = 0x%lx\n", (u64)dma_addr); 242 printk(KERN_INFO "\tTable = 0x%lx\n", (u64)tbl); 243 printk(KERN_INFO "\tbus# = 0x%lx\n", (u64)tbl->it_busno); 244 printk(KERN_INFO "\tsize = 0x%lx\n", (u64)tbl->it_size); 245 printk(KERN_INFO "\tstartOff = 0x%lx\n", (u64)tbl->it_offset); 246 printk(KERN_INFO "\tindex = 0x%lx\n", (u64)tbl->it_index); 247 WARN_ON(1); 248 } 249 return; 250 } 251 252 ppc_md.tce_free(tbl, entry, npages); 253 iommu_area_free(tbl->it_map, free_entry, npages); 254 } 255 256 static void iommu_free(struct iommu_table *tbl, dma_addr_t dma_addr, 257 unsigned int npages) 258 { 259 unsigned long flags; 260 261 spin_lock_irqsave(&(tbl->it_lock), flags); 262 263 __iommu_free(tbl, dma_addr, npages); 264 265 /* Make sure TLB cache is flushed if the HW needs it. We do 266 * not do an mb() here on purpose, it is not needed on any of 267 * the current platforms. 268 */ 269 if (ppc_md.tce_flush) 270 ppc_md.tce_flush(tbl); 271 272 spin_unlock_irqrestore(&(tbl->it_lock), flags); 273 } 274 275 int iommu_map_sg(struct device *dev, struct iommu_table *tbl, 276 struct scatterlist *sglist, int nelems, 277 unsigned long mask, enum dma_data_direction direction, 278 struct dma_attrs *attrs) 279 { 280 dma_addr_t dma_next = 0, dma_addr; 281 unsigned long flags; 282 struct scatterlist *s, *outs, *segstart; 283 int outcount, incount, i, build_fail = 0; 284 unsigned int align; 285 unsigned long handle; 286 unsigned int max_seg_size; 287 288 BUG_ON(direction == DMA_NONE); 289 290 if ((nelems == 0) || !tbl) 291 return 0; 292 293 outs = s = segstart = &sglist[0]; 294 outcount = 1; 295 incount = nelems; 296 handle = 0; 297 298 /* Init first segment length for backout at failure */ 299 outs->dma_length = 0; 300 301 DBG("sg mapping %d elements:\n", nelems); 302 303 spin_lock_irqsave(&(tbl->it_lock), flags); 304 305 max_seg_size = dma_get_max_seg_size(dev); 306 for_each_sg(sglist, s, nelems, i) { 307 unsigned long vaddr, npages, entry, slen; 308 309 slen = s->length; 310 /* Sanity check */ 311 if (slen == 0) { 312 dma_next = 0; 313 continue; 314 } 315 /* Allocate iommu entries for that segment */ 316 vaddr = (unsigned long) sg_virt(s); 317 npages = iommu_num_pages(vaddr, slen, IOMMU_PAGE_SIZE); 318 align = 0; 319 if (IOMMU_PAGE_SHIFT < PAGE_SHIFT && slen >= PAGE_SIZE && 320 (vaddr & ~PAGE_MASK) == 0) 321 align = PAGE_SHIFT - IOMMU_PAGE_SHIFT; 322 entry = iommu_range_alloc(dev, tbl, npages, &handle, 323 mask >> IOMMU_PAGE_SHIFT, align); 324 325 DBG(" - vaddr: %lx, size: %lx\n", vaddr, slen); 326 327 /* Handle failure */ 328 if (unlikely(entry == DMA_ERROR_CODE)) { 329 if (printk_ratelimit()) 330 printk(KERN_INFO "iommu_alloc failed, tbl %p vaddr %lx" 331 " npages %lx\n", tbl, vaddr, npages); 332 goto failure; 333 } 334 335 /* Convert entry to a dma_addr_t */ 336 entry += tbl->it_offset; 337 dma_addr = entry << IOMMU_PAGE_SHIFT; 338 dma_addr |= (s->offset & ~IOMMU_PAGE_MASK); 339 340 DBG(" - %lu pages, entry: %lx, dma_addr: %lx\n", 341 npages, entry, dma_addr); 342 343 /* Insert into HW table */ 344 build_fail = ppc_md.tce_build(tbl, entry, npages, 345 vaddr & IOMMU_PAGE_MASK, 346 direction, attrs); 347 if(unlikely(build_fail)) 348 goto failure; 349 350 /* If we are in an open segment, try merging */ 351 if (segstart != s) { 352 DBG(" - trying merge...\n"); 353 /* We cannot merge if: 354 * - allocated dma_addr isn't contiguous to previous allocation 355 */ 356 if (novmerge || (dma_addr != dma_next) || 357 (outs->dma_length + s->length > max_seg_size)) { 358 /* Can't merge: create a new segment */ 359 segstart = s; 360 outcount++; 361 outs = sg_next(outs); 362 DBG(" can't merge, new segment.\n"); 363 } else { 364 outs->dma_length += s->length; 365 DBG(" merged, new len: %ux\n", outs->dma_length); 366 } 367 } 368 369 if (segstart == s) { 370 /* This is a new segment, fill entries */ 371 DBG(" - filling new segment.\n"); 372 outs->dma_address = dma_addr; 373 outs->dma_length = slen; 374 } 375 376 /* Calculate next page pointer for contiguous check */ 377 dma_next = dma_addr + slen; 378 379 DBG(" - dma next is: %lx\n", dma_next); 380 } 381 382 /* Flush/invalidate TLB caches if necessary */ 383 if (ppc_md.tce_flush) 384 ppc_md.tce_flush(tbl); 385 386 spin_unlock_irqrestore(&(tbl->it_lock), flags); 387 388 DBG("mapped %d elements:\n", outcount); 389 390 /* For the sake of iommu_unmap_sg, we clear out the length in the 391 * next entry of the sglist if we didn't fill the list completely 392 */ 393 if (outcount < incount) { 394 outs = sg_next(outs); 395 outs->dma_address = DMA_ERROR_CODE; 396 outs->dma_length = 0; 397 } 398 399 /* Make sure updates are seen by hardware */ 400 mb(); 401 402 return outcount; 403 404 failure: 405 for_each_sg(sglist, s, nelems, i) { 406 if (s->dma_length != 0) { 407 unsigned long vaddr, npages; 408 409 vaddr = s->dma_address & IOMMU_PAGE_MASK; 410 npages = iommu_num_pages(s->dma_address, s->dma_length, 411 IOMMU_PAGE_SIZE); 412 __iommu_free(tbl, vaddr, npages); 413 s->dma_address = DMA_ERROR_CODE; 414 s->dma_length = 0; 415 } 416 if (s == outs) 417 break; 418 } 419 spin_unlock_irqrestore(&(tbl->it_lock), flags); 420 return 0; 421 } 422 423 424 void iommu_unmap_sg(struct iommu_table *tbl, struct scatterlist *sglist, 425 int nelems, enum dma_data_direction direction, 426 struct dma_attrs *attrs) 427 { 428 struct scatterlist *sg; 429 unsigned long flags; 430 431 BUG_ON(direction == DMA_NONE); 432 433 if (!tbl) 434 return; 435 436 spin_lock_irqsave(&(tbl->it_lock), flags); 437 438 sg = sglist; 439 while (nelems--) { 440 unsigned int npages; 441 dma_addr_t dma_handle = sg->dma_address; 442 443 if (sg->dma_length == 0) 444 break; 445 npages = iommu_num_pages(dma_handle, sg->dma_length, 446 IOMMU_PAGE_SIZE); 447 __iommu_free(tbl, dma_handle, npages); 448 sg = sg_next(sg); 449 } 450 451 /* Flush/invalidate TLBs if necessary. As for iommu_free(), we 452 * do not do an mb() here, the affected platforms do not need it 453 * when freeing. 454 */ 455 if (ppc_md.tce_flush) 456 ppc_md.tce_flush(tbl); 457 458 spin_unlock_irqrestore(&(tbl->it_lock), flags); 459 } 460 461 static void iommu_table_clear(struct iommu_table *tbl) 462 { 463 if (!__kdump_flag) { 464 /* Clear the table in case firmware left allocations in it */ 465 ppc_md.tce_free(tbl, tbl->it_offset, tbl->it_size); 466 return; 467 } 468 469 #ifdef CONFIG_CRASH_DUMP 470 if (ppc_md.tce_get) { 471 unsigned long index, tceval, tcecount = 0; 472 473 /* Reserve the existing mappings left by the first kernel. */ 474 for (index = 0; index < tbl->it_size; index++) { 475 tceval = ppc_md.tce_get(tbl, index + tbl->it_offset); 476 /* 477 * Freed TCE entry contains 0x7fffffffffffffff on JS20 478 */ 479 if (tceval && (tceval != 0x7fffffffffffffffUL)) { 480 __set_bit(index, tbl->it_map); 481 tcecount++; 482 } 483 } 484 485 if ((tbl->it_size - tcecount) < KDUMP_MIN_TCE_ENTRIES) { 486 printk(KERN_WARNING "TCE table is full; freeing "); 487 printk(KERN_WARNING "%d entries for the kdump boot\n", 488 KDUMP_MIN_TCE_ENTRIES); 489 for (index = tbl->it_size - KDUMP_MIN_TCE_ENTRIES; 490 index < tbl->it_size; index++) 491 __clear_bit(index, tbl->it_map); 492 } 493 } 494 #endif 495 } 496 497 /* 498 * Build a iommu_table structure. This contains a bit map which 499 * is used to manage allocation of the tce space. 500 */ 501 struct iommu_table *iommu_init_table(struct iommu_table *tbl, int nid) 502 { 503 unsigned long sz; 504 static int welcomed = 0; 505 struct page *page; 506 507 /* Set aside 1/4 of the table for large allocations. */ 508 tbl->it_halfpoint = tbl->it_size * 3 / 4; 509 510 /* number of bytes needed for the bitmap */ 511 sz = (tbl->it_size + 7) >> 3; 512 513 page = alloc_pages_node(nid, GFP_ATOMIC, get_order(sz)); 514 if (!page) 515 panic("iommu_init_table: Can't allocate %ld bytes\n", sz); 516 tbl->it_map = page_address(page); 517 memset(tbl->it_map, 0, sz); 518 519 tbl->it_hint = 0; 520 tbl->it_largehint = tbl->it_halfpoint; 521 spin_lock_init(&tbl->it_lock); 522 523 iommu_table_clear(tbl); 524 525 if (!welcomed) { 526 printk(KERN_INFO "IOMMU table initialized, virtual merging %s\n", 527 novmerge ? "disabled" : "enabled"); 528 welcomed = 1; 529 } 530 531 return tbl; 532 } 533 534 void iommu_free_table(struct iommu_table *tbl, const char *node_name) 535 { 536 unsigned long bitmap_sz, i; 537 unsigned int order; 538 539 if (!tbl || !tbl->it_map) { 540 printk(KERN_ERR "%s: expected TCE map for %s\n", __func__, 541 node_name); 542 return; 543 } 544 545 /* verify that table contains no entries */ 546 /* it_size is in entries, and we're examining 64 at a time */ 547 for (i = 0; i < (tbl->it_size/64); i++) { 548 if (tbl->it_map[i] != 0) { 549 printk(KERN_WARNING "%s: Unexpected TCEs for %s\n", 550 __func__, node_name); 551 break; 552 } 553 } 554 555 /* calculate bitmap size in bytes */ 556 bitmap_sz = (tbl->it_size + 7) / 8; 557 558 /* free bitmap */ 559 order = get_order(bitmap_sz); 560 free_pages((unsigned long) tbl->it_map, order); 561 562 /* free table */ 563 kfree(tbl); 564 } 565 566 /* Creates TCEs for a user provided buffer. The user buffer must be 567 * contiguous real kernel storage (not vmalloc). The address of the buffer 568 * passed here is the kernel (virtual) address of the buffer. The buffer 569 * need not be page aligned, the dma_addr_t returned will point to the same 570 * byte within the page as vaddr. 571 */ 572 dma_addr_t iommu_map_single(struct device *dev, struct iommu_table *tbl, 573 void *vaddr, size_t size, unsigned long mask, 574 enum dma_data_direction direction, struct dma_attrs *attrs) 575 { 576 dma_addr_t dma_handle = DMA_ERROR_CODE; 577 unsigned long uaddr; 578 unsigned int npages, align; 579 580 BUG_ON(direction == DMA_NONE); 581 582 uaddr = (unsigned long)vaddr; 583 npages = iommu_num_pages(uaddr, size, IOMMU_PAGE_SIZE); 584 585 if (tbl) { 586 align = 0; 587 if (IOMMU_PAGE_SHIFT < PAGE_SHIFT && size >= PAGE_SIZE && 588 ((unsigned long)vaddr & ~PAGE_MASK) == 0) 589 align = PAGE_SHIFT - IOMMU_PAGE_SHIFT; 590 591 dma_handle = iommu_alloc(dev, tbl, vaddr, npages, direction, 592 mask >> IOMMU_PAGE_SHIFT, align, 593 attrs); 594 if (dma_handle == DMA_ERROR_CODE) { 595 if (printk_ratelimit()) { 596 printk(KERN_INFO "iommu_alloc failed, " 597 "tbl %p vaddr %p npages %d\n", 598 tbl, vaddr, npages); 599 } 600 } else 601 dma_handle |= (uaddr & ~IOMMU_PAGE_MASK); 602 } 603 604 return dma_handle; 605 } 606 607 void iommu_unmap_single(struct iommu_table *tbl, dma_addr_t dma_handle, 608 size_t size, enum dma_data_direction direction, 609 struct dma_attrs *attrs) 610 { 611 unsigned int npages; 612 613 BUG_ON(direction == DMA_NONE); 614 615 if (tbl) { 616 npages = iommu_num_pages(dma_handle, size, IOMMU_PAGE_SIZE); 617 iommu_free(tbl, dma_handle, npages); 618 } 619 } 620 621 /* Allocates a contiguous real buffer and creates mappings over it. 622 * Returns the virtual address of the buffer and sets dma_handle 623 * to the dma address (mapping) of the first page. 624 */ 625 void *iommu_alloc_coherent(struct device *dev, struct iommu_table *tbl, 626 size_t size, dma_addr_t *dma_handle, 627 unsigned long mask, gfp_t flag, int node) 628 { 629 void *ret = NULL; 630 dma_addr_t mapping; 631 unsigned int order; 632 unsigned int nio_pages, io_order; 633 struct page *page; 634 635 size = PAGE_ALIGN(size); 636 order = get_order(size); 637 638 /* 639 * Client asked for way too much space. This is checked later 640 * anyway. It is easier to debug here for the drivers than in 641 * the tce tables. 642 */ 643 if (order >= IOMAP_MAX_ORDER) { 644 printk("iommu_alloc_consistent size too large: 0x%lx\n", size); 645 return NULL; 646 } 647 648 if (!tbl) 649 return NULL; 650 651 /* Alloc enough pages (and possibly more) */ 652 page = alloc_pages_node(node, flag, order); 653 if (!page) 654 return NULL; 655 ret = page_address(page); 656 memset(ret, 0, size); 657 658 /* Set up tces to cover the allocated range */ 659 nio_pages = size >> IOMMU_PAGE_SHIFT; 660 io_order = get_iommu_order(size); 661 mapping = iommu_alloc(dev, tbl, ret, nio_pages, DMA_BIDIRECTIONAL, 662 mask >> IOMMU_PAGE_SHIFT, io_order, NULL); 663 if (mapping == DMA_ERROR_CODE) { 664 free_pages((unsigned long)ret, order); 665 return NULL; 666 } 667 *dma_handle = mapping; 668 return ret; 669 } 670 671 void iommu_free_coherent(struct iommu_table *tbl, size_t size, 672 void *vaddr, dma_addr_t dma_handle) 673 { 674 if (tbl) { 675 unsigned int nio_pages; 676 677 size = PAGE_ALIGN(size); 678 nio_pages = size >> IOMMU_PAGE_SHIFT; 679 iommu_free(tbl, dma_handle, nio_pages); 680 size = PAGE_ALIGN(size); 681 free_pages((unsigned long)vaddr, get_order(size)); 682 } 683 } 684