1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * linux/arch/arm/mm/dma-mapping.c 4 * 5 * Copyright (C) 2000-2004 Russell King 6 * 7 * DMA uncached mapping support. 8 */ 9 #include <linux/module.h> 10 #include <linux/mm.h> 11 #include <linux/genalloc.h> 12 #include <linux/gfp.h> 13 #include <linux/errno.h> 14 #include <linux/list.h> 15 #include <linux/init.h> 16 #include <linux/device.h> 17 #include <linux/dma-direct.h> 18 #include <linux/dma-map-ops.h> 19 #include <linux/highmem.h> 20 #include <linux/memblock.h> 21 #include <linux/slab.h> 22 #include <linux/iommu.h> 23 #include <linux/io.h> 24 #include <linux/vmalloc.h> 25 #include <linux/sizes.h> 26 #include <linux/cma.h> 27 28 #include <asm/page.h> 29 #include <asm/highmem.h> 30 #include <asm/cacheflush.h> 31 #include <asm/tlbflush.h> 32 #include <asm/mach/arch.h> 33 #include <asm/dma-iommu.h> 34 #include <asm/mach/map.h> 35 #include <asm/system_info.h> 36 #include <asm/xen/xen-ops.h> 37 38 #include "dma.h" 39 #include "mm.h" 40 41 struct arm_dma_alloc_args { 42 struct device *dev; 43 size_t size; 44 gfp_t gfp; 45 pgprot_t prot; 46 const void *caller; 47 bool want_vaddr; 48 int coherent_flag; 49 }; 50 51 struct arm_dma_free_args { 52 struct device *dev; 53 size_t size; 54 void *cpu_addr; 55 struct page *page; 56 bool want_vaddr; 57 }; 58 59 #define NORMAL 0 60 #define COHERENT 1 61 62 struct arm_dma_allocator { 63 void *(*alloc)(struct arm_dma_alloc_args *args, 64 struct page **ret_page); 65 void (*free)(struct arm_dma_free_args *args); 66 }; 67 68 struct arm_dma_buffer { 69 struct list_head list; 70 void *virt; 71 struct arm_dma_allocator *allocator; 72 }; 73 74 static LIST_HEAD(arm_dma_bufs); 75 static DEFINE_SPINLOCK(arm_dma_bufs_lock); 76 77 static struct arm_dma_buffer *arm_dma_buffer_find(void *virt) 78 { 79 struct arm_dma_buffer *buf, *found = NULL; 80 unsigned long flags; 81 82 spin_lock_irqsave(&arm_dma_bufs_lock, flags); 83 list_for_each_entry(buf, &arm_dma_bufs, list) { 84 if (buf->virt == virt) { 85 list_del(&buf->list); 86 found = buf; 87 break; 88 } 89 } 90 spin_unlock_irqrestore(&arm_dma_bufs_lock, flags); 91 return found; 92 } 93 94 /* 95 * The DMA API is built upon the notion of "buffer ownership". A buffer 96 * is either exclusively owned by the CPU (and therefore may be accessed 97 * by it) or exclusively owned by the DMA device. These helper functions 98 * represent the transitions between these two ownership states. 99 * 100 * Note, however, that on later ARMs, this notion does not work due to 101 * speculative prefetches. We model our approach on the assumption that 102 * the CPU does do speculative prefetches, which means we clean caches 103 * before transfers and delay cache invalidation until transfer completion. 104 * 105 */ 106 107 static void __dma_clear_buffer(struct page *page, size_t size, int coherent_flag) 108 { 109 /* 110 * Ensure that the allocated pages are zeroed, and that any data 111 * lurking in the kernel direct-mapped region is invalidated. 112 */ 113 if (PageHighMem(page)) { 114 phys_addr_t base = __pfn_to_phys(page_to_pfn(page)); 115 phys_addr_t end = base + size; 116 while (size > 0) { 117 void *ptr = kmap_atomic(page); 118 memset(ptr, 0, PAGE_SIZE); 119 if (coherent_flag != COHERENT) 120 dmac_flush_range(ptr, ptr + PAGE_SIZE); 121 kunmap_atomic(ptr); 122 page++; 123 size -= PAGE_SIZE; 124 } 125 if (coherent_flag != COHERENT) 126 outer_flush_range(base, end); 127 } else { 128 void *ptr = page_address(page); 129 memset(ptr, 0, size); 130 if (coherent_flag != COHERENT) { 131 dmac_flush_range(ptr, ptr + size); 132 outer_flush_range(__pa(ptr), __pa(ptr) + size); 133 } 134 } 135 } 136 137 /* 138 * Allocate a DMA buffer for 'dev' of size 'size' using the 139 * specified gfp mask. Note that 'size' must be page aligned. 140 */ 141 static struct page *__dma_alloc_buffer(struct device *dev, size_t size, 142 gfp_t gfp, int coherent_flag) 143 { 144 unsigned long order = get_order(size); 145 struct page *page, *p, *e; 146 147 page = alloc_pages(gfp, order); 148 if (!page) 149 return NULL; 150 151 /* 152 * Now split the huge page and free the excess pages 153 */ 154 split_page(page, order); 155 for (p = page + (size >> PAGE_SHIFT), e = page + (1 << order); p < e; p++) 156 __free_page(p); 157 158 __dma_clear_buffer(page, size, coherent_flag); 159 160 return page; 161 } 162 163 /* 164 * Free a DMA buffer. 'size' must be page aligned. 165 */ 166 static void __dma_free_buffer(struct page *page, size_t size) 167 { 168 struct page *e = page + (size >> PAGE_SHIFT); 169 170 while (page < e) { 171 __free_page(page); 172 page++; 173 } 174 } 175 176 static void *__alloc_from_contiguous(struct device *dev, size_t size, 177 pgprot_t prot, struct page **ret_page, 178 const void *caller, bool want_vaddr, 179 int coherent_flag, gfp_t gfp); 180 181 static void *__alloc_remap_buffer(struct device *dev, size_t size, gfp_t gfp, 182 pgprot_t prot, struct page **ret_page, 183 const void *caller, bool want_vaddr); 184 185 #define DEFAULT_DMA_COHERENT_POOL_SIZE SZ_256K 186 static struct gen_pool *atomic_pool __ro_after_init; 187 188 static size_t atomic_pool_size __initdata = DEFAULT_DMA_COHERENT_POOL_SIZE; 189 190 static int __init early_coherent_pool(char *p) 191 { 192 atomic_pool_size = memparse(p, &p); 193 return 0; 194 } 195 early_param("coherent_pool", early_coherent_pool); 196 197 /* 198 * Initialise the coherent pool for atomic allocations. 199 */ 200 static int __init atomic_pool_init(void) 201 { 202 pgprot_t prot = pgprot_dmacoherent(PAGE_KERNEL); 203 gfp_t gfp = GFP_KERNEL | GFP_DMA; 204 struct page *page; 205 void *ptr; 206 207 atomic_pool = gen_pool_create(PAGE_SHIFT, -1); 208 if (!atomic_pool) 209 goto out; 210 /* 211 * The atomic pool is only used for non-coherent allocations 212 * so we must pass NORMAL for coherent_flag. 213 */ 214 if (dev_get_cma_area(NULL)) 215 ptr = __alloc_from_contiguous(NULL, atomic_pool_size, prot, 216 &page, atomic_pool_init, true, NORMAL, 217 GFP_KERNEL); 218 else 219 ptr = __alloc_remap_buffer(NULL, atomic_pool_size, gfp, prot, 220 &page, atomic_pool_init, true); 221 if (ptr) { 222 int ret; 223 224 ret = gen_pool_add_virt(atomic_pool, (unsigned long)ptr, 225 page_to_phys(page), 226 atomic_pool_size, -1); 227 if (ret) 228 goto destroy_genpool; 229 230 gen_pool_set_algo(atomic_pool, 231 gen_pool_first_fit_order_align, 232 NULL); 233 pr_info("DMA: preallocated %zu KiB pool for atomic coherent allocations\n", 234 atomic_pool_size / 1024); 235 return 0; 236 } 237 238 destroy_genpool: 239 gen_pool_destroy(atomic_pool); 240 atomic_pool = NULL; 241 out: 242 pr_err("DMA: failed to allocate %zu KiB pool for atomic coherent allocation\n", 243 atomic_pool_size / 1024); 244 return -ENOMEM; 245 } 246 /* 247 * CMA is activated by core_initcall, so we must be called after it. 248 */ 249 postcore_initcall(atomic_pool_init); 250 251 #ifdef CONFIG_CMA_AREAS 252 struct dma_contig_early_reserve { 253 phys_addr_t base; 254 unsigned long size; 255 }; 256 257 static struct dma_contig_early_reserve dma_mmu_remap[MAX_CMA_AREAS] __initdata; 258 259 static int dma_mmu_remap_num __initdata; 260 261 #ifdef CONFIG_DMA_CMA 262 void __init dma_contiguous_early_fixup(phys_addr_t base, unsigned long size) 263 { 264 dma_mmu_remap[dma_mmu_remap_num].base = base; 265 dma_mmu_remap[dma_mmu_remap_num].size = size; 266 dma_mmu_remap_num++; 267 } 268 #endif 269 270 void __init dma_contiguous_remap(void) 271 { 272 int i; 273 for (i = 0; i < dma_mmu_remap_num; i++) { 274 phys_addr_t start = dma_mmu_remap[i].base; 275 phys_addr_t end = start + dma_mmu_remap[i].size; 276 struct map_desc map; 277 unsigned long addr; 278 279 if (end > arm_lowmem_limit) 280 end = arm_lowmem_limit; 281 if (start >= end) 282 continue; 283 284 map.pfn = __phys_to_pfn(start); 285 map.virtual = __phys_to_virt(start); 286 map.length = end - start; 287 map.type = MT_MEMORY_DMA_READY; 288 289 /* 290 * Clear previous low-memory mapping to ensure that the 291 * TLB does not see any conflicting entries, then flush 292 * the TLB of the old entries before creating new mappings. 293 * 294 * This ensures that any speculatively loaded TLB entries 295 * (even though they may be rare) can not cause any problems, 296 * and ensures that this code is architecturally compliant. 297 */ 298 for (addr = __phys_to_virt(start); addr < __phys_to_virt(end); 299 addr += PMD_SIZE) 300 pmd_clear(pmd_off_k(addr)); 301 302 flush_tlb_kernel_range(__phys_to_virt(start), 303 __phys_to_virt(end)); 304 305 iotable_init(&map, 1); 306 } 307 } 308 #endif 309 310 static int __dma_update_pte(pte_t *pte, unsigned long addr, void *data) 311 { 312 struct page *page = virt_to_page((void *)addr); 313 pgprot_t prot = *(pgprot_t *)data; 314 315 set_pte_ext(pte, mk_pte(page, prot), 0); 316 return 0; 317 } 318 319 static void __dma_remap(struct page *page, size_t size, pgprot_t prot) 320 { 321 unsigned long start = (unsigned long) page_address(page); 322 unsigned end = start + size; 323 324 apply_to_page_range(&init_mm, start, size, __dma_update_pte, &prot); 325 flush_tlb_kernel_range(start, end); 326 } 327 328 static void *__alloc_remap_buffer(struct device *dev, size_t size, gfp_t gfp, 329 pgprot_t prot, struct page **ret_page, 330 const void *caller, bool want_vaddr) 331 { 332 struct page *page; 333 void *ptr = NULL; 334 /* 335 * __alloc_remap_buffer is only called when the device is 336 * non-coherent 337 */ 338 page = __dma_alloc_buffer(dev, size, gfp, NORMAL); 339 if (!page) 340 return NULL; 341 if (!want_vaddr) 342 goto out; 343 344 ptr = dma_common_contiguous_remap(page, size, prot, caller); 345 if (!ptr) { 346 __dma_free_buffer(page, size); 347 return NULL; 348 } 349 350 out: 351 *ret_page = page; 352 return ptr; 353 } 354 355 static void *__alloc_from_pool(size_t size, struct page **ret_page) 356 { 357 unsigned long val; 358 void *ptr = NULL; 359 360 if (!atomic_pool) { 361 WARN(1, "coherent pool not initialised!\n"); 362 return NULL; 363 } 364 365 val = gen_pool_alloc(atomic_pool, size); 366 if (val) { 367 phys_addr_t phys = gen_pool_virt_to_phys(atomic_pool, val); 368 369 *ret_page = phys_to_page(phys); 370 ptr = (void *)val; 371 } 372 373 return ptr; 374 } 375 376 static bool __in_atomic_pool(void *start, size_t size) 377 { 378 return gen_pool_has_addr(atomic_pool, (unsigned long)start, size); 379 } 380 381 static int __free_from_pool(void *start, size_t size) 382 { 383 if (!__in_atomic_pool(start, size)) 384 return 0; 385 386 gen_pool_free(atomic_pool, (unsigned long)start, size); 387 388 return 1; 389 } 390 391 static void *__alloc_from_contiguous(struct device *dev, size_t size, 392 pgprot_t prot, struct page **ret_page, 393 const void *caller, bool want_vaddr, 394 int coherent_flag, gfp_t gfp) 395 { 396 unsigned long order = get_order(size); 397 size_t count = size >> PAGE_SHIFT; 398 struct page *page; 399 void *ptr = NULL; 400 401 page = dma_alloc_from_contiguous(dev, count, order, gfp & __GFP_NOWARN); 402 if (!page) 403 return NULL; 404 405 __dma_clear_buffer(page, size, coherent_flag); 406 407 if (!want_vaddr) 408 goto out; 409 410 if (PageHighMem(page)) { 411 ptr = dma_common_contiguous_remap(page, size, prot, caller); 412 if (!ptr) { 413 dma_release_from_contiguous(dev, page, count); 414 return NULL; 415 } 416 } else { 417 __dma_remap(page, size, prot); 418 ptr = page_address(page); 419 } 420 421 out: 422 *ret_page = page; 423 return ptr; 424 } 425 426 static void __free_from_contiguous(struct device *dev, struct page *page, 427 void *cpu_addr, size_t size, bool want_vaddr) 428 { 429 if (want_vaddr) { 430 if (PageHighMem(page)) 431 dma_common_free_remap(cpu_addr, size); 432 else 433 __dma_remap(page, size, PAGE_KERNEL); 434 } 435 dma_release_from_contiguous(dev, page, size >> PAGE_SHIFT); 436 } 437 438 static inline pgprot_t __get_dma_pgprot(unsigned long attrs, pgprot_t prot) 439 { 440 prot = (attrs & DMA_ATTR_WRITE_COMBINE) ? 441 pgprot_writecombine(prot) : 442 pgprot_dmacoherent(prot); 443 return prot; 444 } 445 446 static void *__alloc_simple_buffer(struct device *dev, size_t size, gfp_t gfp, 447 struct page **ret_page) 448 { 449 struct page *page; 450 /* __alloc_simple_buffer is only called when the device is coherent */ 451 page = __dma_alloc_buffer(dev, size, gfp, COHERENT); 452 if (!page) 453 return NULL; 454 455 *ret_page = page; 456 return page_address(page); 457 } 458 459 static void *simple_allocator_alloc(struct arm_dma_alloc_args *args, 460 struct page **ret_page) 461 { 462 return __alloc_simple_buffer(args->dev, args->size, args->gfp, 463 ret_page); 464 } 465 466 static void simple_allocator_free(struct arm_dma_free_args *args) 467 { 468 __dma_free_buffer(args->page, args->size); 469 } 470 471 static struct arm_dma_allocator simple_allocator = { 472 .alloc = simple_allocator_alloc, 473 .free = simple_allocator_free, 474 }; 475 476 static void *cma_allocator_alloc(struct arm_dma_alloc_args *args, 477 struct page **ret_page) 478 { 479 return __alloc_from_contiguous(args->dev, args->size, args->prot, 480 ret_page, args->caller, 481 args->want_vaddr, args->coherent_flag, 482 args->gfp); 483 } 484 485 static void cma_allocator_free(struct arm_dma_free_args *args) 486 { 487 __free_from_contiguous(args->dev, args->page, args->cpu_addr, 488 args->size, args->want_vaddr); 489 } 490 491 static struct arm_dma_allocator cma_allocator = { 492 .alloc = cma_allocator_alloc, 493 .free = cma_allocator_free, 494 }; 495 496 static void *pool_allocator_alloc(struct arm_dma_alloc_args *args, 497 struct page **ret_page) 498 { 499 return __alloc_from_pool(args->size, ret_page); 500 } 501 502 static void pool_allocator_free(struct arm_dma_free_args *args) 503 { 504 __free_from_pool(args->cpu_addr, args->size); 505 } 506 507 static struct arm_dma_allocator pool_allocator = { 508 .alloc = pool_allocator_alloc, 509 .free = pool_allocator_free, 510 }; 511 512 static void *remap_allocator_alloc(struct arm_dma_alloc_args *args, 513 struct page **ret_page) 514 { 515 return __alloc_remap_buffer(args->dev, args->size, args->gfp, 516 args->prot, ret_page, args->caller, 517 args->want_vaddr); 518 } 519 520 static void remap_allocator_free(struct arm_dma_free_args *args) 521 { 522 if (args->want_vaddr) 523 dma_common_free_remap(args->cpu_addr, args->size); 524 525 __dma_free_buffer(args->page, args->size); 526 } 527 528 static struct arm_dma_allocator remap_allocator = { 529 .alloc = remap_allocator_alloc, 530 .free = remap_allocator_free, 531 }; 532 533 static void *__dma_alloc(struct device *dev, size_t size, dma_addr_t *handle, 534 gfp_t gfp, pgprot_t prot, bool is_coherent, 535 unsigned long attrs, const void *caller) 536 { 537 u64 mask = min_not_zero(dev->coherent_dma_mask, dev->bus_dma_limit); 538 struct page *page = NULL; 539 void *addr; 540 bool allowblock, cma; 541 struct arm_dma_buffer *buf; 542 struct arm_dma_alloc_args args = { 543 .dev = dev, 544 .size = PAGE_ALIGN(size), 545 .gfp = gfp, 546 .prot = prot, 547 .caller = caller, 548 .want_vaddr = ((attrs & DMA_ATTR_NO_KERNEL_MAPPING) == 0), 549 .coherent_flag = is_coherent ? COHERENT : NORMAL, 550 }; 551 552 #ifdef CONFIG_DMA_API_DEBUG 553 u64 limit = (mask + 1) & ~mask; 554 if (limit && size >= limit) { 555 dev_warn(dev, "coherent allocation too big (requested %#x mask %#llx)\n", 556 size, mask); 557 return NULL; 558 } 559 #endif 560 561 buf = kzalloc(sizeof(*buf), 562 gfp & ~(__GFP_DMA | __GFP_DMA32 | __GFP_HIGHMEM)); 563 if (!buf) 564 return NULL; 565 566 if (mask < 0xffffffffULL) 567 gfp |= GFP_DMA; 568 569 args.gfp = gfp; 570 571 *handle = DMA_MAPPING_ERROR; 572 allowblock = gfpflags_allow_blocking(gfp); 573 cma = allowblock ? dev_get_cma_area(dev) : NULL; 574 575 if (cma) 576 buf->allocator = &cma_allocator; 577 else if (is_coherent) 578 buf->allocator = &simple_allocator; 579 else if (allowblock) 580 buf->allocator = &remap_allocator; 581 else 582 buf->allocator = &pool_allocator; 583 584 addr = buf->allocator->alloc(&args, &page); 585 586 if (page) { 587 unsigned long flags; 588 589 *handle = phys_to_dma(dev, page_to_phys(page)); 590 buf->virt = args.want_vaddr ? addr : page; 591 592 spin_lock_irqsave(&arm_dma_bufs_lock, flags); 593 list_add(&buf->list, &arm_dma_bufs); 594 spin_unlock_irqrestore(&arm_dma_bufs_lock, flags); 595 } else { 596 kfree(buf); 597 } 598 599 return args.want_vaddr ? addr : page; 600 } 601 602 /* 603 * Free a buffer as defined by the above mapping. 604 */ 605 static void __arm_dma_free(struct device *dev, size_t size, void *cpu_addr, 606 dma_addr_t handle, unsigned long attrs, 607 bool is_coherent) 608 { 609 struct page *page = phys_to_page(dma_to_phys(dev, handle)); 610 struct arm_dma_buffer *buf; 611 struct arm_dma_free_args args = { 612 .dev = dev, 613 .size = PAGE_ALIGN(size), 614 .cpu_addr = cpu_addr, 615 .page = page, 616 .want_vaddr = ((attrs & DMA_ATTR_NO_KERNEL_MAPPING) == 0), 617 }; 618 619 buf = arm_dma_buffer_find(cpu_addr); 620 if (WARN(!buf, "Freeing invalid buffer %p\n", cpu_addr)) 621 return; 622 623 buf->allocator->free(&args); 624 kfree(buf); 625 } 626 627 static void dma_cache_maint_page(struct page *page, unsigned long offset, 628 size_t size, enum dma_data_direction dir, 629 void (*op)(const void *, size_t, int)) 630 { 631 unsigned long pfn; 632 size_t left = size; 633 634 pfn = page_to_pfn(page) + offset / PAGE_SIZE; 635 offset %= PAGE_SIZE; 636 637 /* 638 * A single sg entry may refer to multiple physically contiguous 639 * pages. But we still need to process highmem pages individually. 640 * If highmem is not configured then the bulk of this loop gets 641 * optimized out. 642 */ 643 do { 644 size_t len = left; 645 void *vaddr; 646 647 page = pfn_to_page(pfn); 648 649 if (PageHighMem(page)) { 650 if (len + offset > PAGE_SIZE) 651 len = PAGE_SIZE - offset; 652 653 if (cache_is_vipt_nonaliasing()) { 654 vaddr = kmap_atomic(page); 655 op(vaddr + offset, len, dir); 656 kunmap_atomic(vaddr); 657 } else { 658 vaddr = kmap_high_get(page); 659 if (vaddr) { 660 op(vaddr + offset, len, dir); 661 kunmap_high(page); 662 } 663 } 664 } else { 665 vaddr = page_address(page) + offset; 666 op(vaddr, len, dir); 667 } 668 offset = 0; 669 pfn++; 670 left -= len; 671 } while (left); 672 } 673 674 /* 675 * Make an area consistent for devices. 676 * Note: Drivers should NOT use this function directly. 677 * Use the driver DMA support - see dma-mapping.h (dma_sync_*) 678 */ 679 static void __dma_page_cpu_to_dev(struct page *page, unsigned long off, 680 size_t size, enum dma_data_direction dir) 681 { 682 phys_addr_t paddr; 683 684 dma_cache_maint_page(page, off, size, dir, dmac_map_area); 685 686 paddr = page_to_phys(page) + off; 687 if (dir == DMA_FROM_DEVICE) { 688 outer_inv_range(paddr, paddr + size); 689 } else { 690 outer_clean_range(paddr, paddr + size); 691 } 692 /* FIXME: non-speculating: flush on bidirectional mappings? */ 693 } 694 695 static void __dma_page_dev_to_cpu(struct page *page, unsigned long off, 696 size_t size, enum dma_data_direction dir) 697 { 698 phys_addr_t paddr = page_to_phys(page) + off; 699 700 /* FIXME: non-speculating: not required */ 701 /* in any case, don't bother invalidating if DMA to device */ 702 if (dir != DMA_TO_DEVICE) { 703 outer_inv_range(paddr, paddr + size); 704 705 dma_cache_maint_page(page, off, size, dir, dmac_unmap_area); 706 } 707 708 /* 709 * Mark the D-cache clean for these pages to avoid extra flushing. 710 */ 711 if (dir != DMA_TO_DEVICE && size >= PAGE_SIZE) { 712 unsigned long pfn; 713 size_t left = size; 714 715 pfn = page_to_pfn(page) + off / PAGE_SIZE; 716 off %= PAGE_SIZE; 717 if (off) { 718 pfn++; 719 left -= PAGE_SIZE - off; 720 } 721 while (left >= PAGE_SIZE) { 722 page = pfn_to_page(pfn++); 723 set_bit(PG_dcache_clean, &page->flags); 724 left -= PAGE_SIZE; 725 } 726 } 727 } 728 729 #ifdef CONFIG_ARM_DMA_USE_IOMMU 730 731 static int __dma_info_to_prot(enum dma_data_direction dir, unsigned long attrs) 732 { 733 int prot = 0; 734 735 if (attrs & DMA_ATTR_PRIVILEGED) 736 prot |= IOMMU_PRIV; 737 738 switch (dir) { 739 case DMA_BIDIRECTIONAL: 740 return prot | IOMMU_READ | IOMMU_WRITE; 741 case DMA_TO_DEVICE: 742 return prot | IOMMU_READ; 743 case DMA_FROM_DEVICE: 744 return prot | IOMMU_WRITE; 745 default: 746 return prot; 747 } 748 } 749 750 /* IOMMU */ 751 752 static int extend_iommu_mapping(struct dma_iommu_mapping *mapping); 753 754 static inline dma_addr_t __alloc_iova(struct dma_iommu_mapping *mapping, 755 size_t size) 756 { 757 unsigned int order = get_order(size); 758 unsigned int align = 0; 759 unsigned int count, start; 760 size_t mapping_size = mapping->bits << PAGE_SHIFT; 761 unsigned long flags; 762 dma_addr_t iova; 763 int i; 764 765 if (order > CONFIG_ARM_DMA_IOMMU_ALIGNMENT) 766 order = CONFIG_ARM_DMA_IOMMU_ALIGNMENT; 767 768 count = PAGE_ALIGN(size) >> PAGE_SHIFT; 769 align = (1 << order) - 1; 770 771 spin_lock_irqsave(&mapping->lock, flags); 772 for (i = 0; i < mapping->nr_bitmaps; i++) { 773 start = bitmap_find_next_zero_area(mapping->bitmaps[i], 774 mapping->bits, 0, count, align); 775 776 if (start > mapping->bits) 777 continue; 778 779 bitmap_set(mapping->bitmaps[i], start, count); 780 break; 781 } 782 783 /* 784 * No unused range found. Try to extend the existing mapping 785 * and perform a second attempt to reserve an IO virtual 786 * address range of size bytes. 787 */ 788 if (i == mapping->nr_bitmaps) { 789 if (extend_iommu_mapping(mapping)) { 790 spin_unlock_irqrestore(&mapping->lock, flags); 791 return DMA_MAPPING_ERROR; 792 } 793 794 start = bitmap_find_next_zero_area(mapping->bitmaps[i], 795 mapping->bits, 0, count, align); 796 797 if (start > mapping->bits) { 798 spin_unlock_irqrestore(&mapping->lock, flags); 799 return DMA_MAPPING_ERROR; 800 } 801 802 bitmap_set(mapping->bitmaps[i], start, count); 803 } 804 spin_unlock_irqrestore(&mapping->lock, flags); 805 806 iova = mapping->base + (mapping_size * i); 807 iova += start << PAGE_SHIFT; 808 809 return iova; 810 } 811 812 static inline void __free_iova(struct dma_iommu_mapping *mapping, 813 dma_addr_t addr, size_t size) 814 { 815 unsigned int start, count; 816 size_t mapping_size = mapping->bits << PAGE_SHIFT; 817 unsigned long flags; 818 dma_addr_t bitmap_base; 819 u32 bitmap_index; 820 821 if (!size) 822 return; 823 824 bitmap_index = (u32) (addr - mapping->base) / (u32) mapping_size; 825 BUG_ON(addr < mapping->base || bitmap_index > mapping->extensions); 826 827 bitmap_base = mapping->base + mapping_size * bitmap_index; 828 829 start = (addr - bitmap_base) >> PAGE_SHIFT; 830 831 if (addr + size > bitmap_base + mapping_size) { 832 /* 833 * The address range to be freed reaches into the iova 834 * range of the next bitmap. This should not happen as 835 * we don't allow this in __alloc_iova (at the 836 * moment). 837 */ 838 BUG(); 839 } else 840 count = size >> PAGE_SHIFT; 841 842 spin_lock_irqsave(&mapping->lock, flags); 843 bitmap_clear(mapping->bitmaps[bitmap_index], start, count); 844 spin_unlock_irqrestore(&mapping->lock, flags); 845 } 846 847 /* We'll try 2M, 1M, 64K, and finally 4K; array must end with 0! */ 848 static const int iommu_order_array[] = { 9, 8, 4, 0 }; 849 850 static struct page **__iommu_alloc_buffer(struct device *dev, size_t size, 851 gfp_t gfp, unsigned long attrs, 852 int coherent_flag) 853 { 854 struct page **pages; 855 int count = size >> PAGE_SHIFT; 856 int array_size = count * sizeof(struct page *); 857 int i = 0; 858 int order_idx = 0; 859 860 if (array_size <= PAGE_SIZE) 861 pages = kzalloc(array_size, GFP_KERNEL); 862 else 863 pages = vzalloc(array_size); 864 if (!pages) 865 return NULL; 866 867 if (attrs & DMA_ATTR_FORCE_CONTIGUOUS) 868 { 869 unsigned long order = get_order(size); 870 struct page *page; 871 872 page = dma_alloc_from_contiguous(dev, count, order, 873 gfp & __GFP_NOWARN); 874 if (!page) 875 goto error; 876 877 __dma_clear_buffer(page, size, coherent_flag); 878 879 for (i = 0; i < count; i++) 880 pages[i] = page + i; 881 882 return pages; 883 } 884 885 /* Go straight to 4K chunks if caller says it's OK. */ 886 if (attrs & DMA_ATTR_ALLOC_SINGLE_PAGES) 887 order_idx = ARRAY_SIZE(iommu_order_array) - 1; 888 889 /* 890 * IOMMU can map any pages, so himem can also be used here 891 */ 892 gfp |= __GFP_NOWARN | __GFP_HIGHMEM; 893 894 while (count) { 895 int j, order; 896 897 order = iommu_order_array[order_idx]; 898 899 /* Drop down when we get small */ 900 if (__fls(count) < order) { 901 order_idx++; 902 continue; 903 } 904 905 if (order) { 906 /* See if it's easy to allocate a high-order chunk */ 907 pages[i] = alloc_pages(gfp | __GFP_NORETRY, order); 908 909 /* Go down a notch at first sign of pressure */ 910 if (!pages[i]) { 911 order_idx++; 912 continue; 913 } 914 } else { 915 pages[i] = alloc_pages(gfp, 0); 916 if (!pages[i]) 917 goto error; 918 } 919 920 if (order) { 921 split_page(pages[i], order); 922 j = 1 << order; 923 while (--j) 924 pages[i + j] = pages[i] + j; 925 } 926 927 __dma_clear_buffer(pages[i], PAGE_SIZE << order, coherent_flag); 928 i += 1 << order; 929 count -= 1 << order; 930 } 931 932 return pages; 933 error: 934 while (i--) 935 if (pages[i]) 936 __free_pages(pages[i], 0); 937 kvfree(pages); 938 return NULL; 939 } 940 941 static int __iommu_free_buffer(struct device *dev, struct page **pages, 942 size_t size, unsigned long attrs) 943 { 944 int count = size >> PAGE_SHIFT; 945 int i; 946 947 if (attrs & DMA_ATTR_FORCE_CONTIGUOUS) { 948 dma_release_from_contiguous(dev, pages[0], count); 949 } else { 950 for (i = 0; i < count; i++) 951 if (pages[i]) 952 __free_pages(pages[i], 0); 953 } 954 955 kvfree(pages); 956 return 0; 957 } 958 959 /* 960 * Create a mapping in device IO address space for specified pages 961 */ 962 static dma_addr_t 963 __iommu_create_mapping(struct device *dev, struct page **pages, size_t size, 964 unsigned long attrs) 965 { 966 struct dma_iommu_mapping *mapping = to_dma_iommu_mapping(dev); 967 unsigned int count = PAGE_ALIGN(size) >> PAGE_SHIFT; 968 dma_addr_t dma_addr, iova; 969 int i; 970 971 dma_addr = __alloc_iova(mapping, size); 972 if (dma_addr == DMA_MAPPING_ERROR) 973 return dma_addr; 974 975 iova = dma_addr; 976 for (i = 0; i < count; ) { 977 int ret; 978 979 unsigned int next_pfn = page_to_pfn(pages[i]) + 1; 980 phys_addr_t phys = page_to_phys(pages[i]); 981 unsigned int len, j; 982 983 for (j = i + 1; j < count; j++, next_pfn++) 984 if (page_to_pfn(pages[j]) != next_pfn) 985 break; 986 987 len = (j - i) << PAGE_SHIFT; 988 ret = iommu_map(mapping->domain, iova, phys, len, 989 __dma_info_to_prot(DMA_BIDIRECTIONAL, attrs), 990 GFP_KERNEL); 991 if (ret < 0) 992 goto fail; 993 iova += len; 994 i = j; 995 } 996 return dma_addr; 997 fail: 998 iommu_unmap(mapping->domain, dma_addr, iova-dma_addr); 999 __free_iova(mapping, dma_addr, size); 1000 return DMA_MAPPING_ERROR; 1001 } 1002 1003 static int __iommu_remove_mapping(struct device *dev, dma_addr_t iova, size_t size) 1004 { 1005 struct dma_iommu_mapping *mapping = to_dma_iommu_mapping(dev); 1006 1007 /* 1008 * add optional in-page offset from iova to size and align 1009 * result to page size 1010 */ 1011 size = PAGE_ALIGN((iova & ~PAGE_MASK) + size); 1012 iova &= PAGE_MASK; 1013 1014 iommu_unmap(mapping->domain, iova, size); 1015 __free_iova(mapping, iova, size); 1016 return 0; 1017 } 1018 1019 static struct page **__atomic_get_pages(void *addr) 1020 { 1021 struct page *page; 1022 phys_addr_t phys; 1023 1024 phys = gen_pool_virt_to_phys(atomic_pool, (unsigned long)addr); 1025 page = phys_to_page(phys); 1026 1027 return (struct page **)page; 1028 } 1029 1030 static struct page **__iommu_get_pages(void *cpu_addr, unsigned long attrs) 1031 { 1032 if (__in_atomic_pool(cpu_addr, PAGE_SIZE)) 1033 return __atomic_get_pages(cpu_addr); 1034 1035 if (attrs & DMA_ATTR_NO_KERNEL_MAPPING) 1036 return cpu_addr; 1037 1038 return dma_common_find_pages(cpu_addr); 1039 } 1040 1041 static void *__iommu_alloc_simple(struct device *dev, size_t size, gfp_t gfp, 1042 dma_addr_t *handle, int coherent_flag, 1043 unsigned long attrs) 1044 { 1045 struct page *page; 1046 void *addr; 1047 1048 if (coherent_flag == COHERENT) 1049 addr = __alloc_simple_buffer(dev, size, gfp, &page); 1050 else 1051 addr = __alloc_from_pool(size, &page); 1052 if (!addr) 1053 return NULL; 1054 1055 *handle = __iommu_create_mapping(dev, &page, size, attrs); 1056 if (*handle == DMA_MAPPING_ERROR) 1057 goto err_mapping; 1058 1059 return addr; 1060 1061 err_mapping: 1062 __free_from_pool(addr, size); 1063 return NULL; 1064 } 1065 1066 static void __iommu_free_atomic(struct device *dev, void *cpu_addr, 1067 dma_addr_t handle, size_t size, int coherent_flag) 1068 { 1069 __iommu_remove_mapping(dev, handle, size); 1070 if (coherent_flag == COHERENT) 1071 __dma_free_buffer(virt_to_page(cpu_addr), size); 1072 else 1073 __free_from_pool(cpu_addr, size); 1074 } 1075 1076 static void *arm_iommu_alloc_attrs(struct device *dev, size_t size, 1077 dma_addr_t *handle, gfp_t gfp, unsigned long attrs) 1078 { 1079 pgprot_t prot = __get_dma_pgprot(attrs, PAGE_KERNEL); 1080 struct page **pages; 1081 void *addr = NULL; 1082 int coherent_flag = dev->dma_coherent ? COHERENT : NORMAL; 1083 1084 *handle = DMA_MAPPING_ERROR; 1085 size = PAGE_ALIGN(size); 1086 1087 if (coherent_flag == COHERENT || !gfpflags_allow_blocking(gfp)) 1088 return __iommu_alloc_simple(dev, size, gfp, handle, 1089 coherent_flag, attrs); 1090 1091 pages = __iommu_alloc_buffer(dev, size, gfp, attrs, coherent_flag); 1092 if (!pages) 1093 return NULL; 1094 1095 *handle = __iommu_create_mapping(dev, pages, size, attrs); 1096 if (*handle == DMA_MAPPING_ERROR) 1097 goto err_buffer; 1098 1099 if (attrs & DMA_ATTR_NO_KERNEL_MAPPING) 1100 return pages; 1101 1102 addr = dma_common_pages_remap(pages, size, prot, 1103 __builtin_return_address(0)); 1104 if (!addr) 1105 goto err_mapping; 1106 1107 return addr; 1108 1109 err_mapping: 1110 __iommu_remove_mapping(dev, *handle, size); 1111 err_buffer: 1112 __iommu_free_buffer(dev, pages, size, attrs); 1113 return NULL; 1114 } 1115 1116 static int arm_iommu_mmap_attrs(struct device *dev, struct vm_area_struct *vma, 1117 void *cpu_addr, dma_addr_t dma_addr, size_t size, 1118 unsigned long attrs) 1119 { 1120 struct page **pages = __iommu_get_pages(cpu_addr, attrs); 1121 unsigned long nr_pages = PAGE_ALIGN(size) >> PAGE_SHIFT; 1122 int err; 1123 1124 if (!pages) 1125 return -ENXIO; 1126 1127 if (vma->vm_pgoff >= nr_pages) 1128 return -ENXIO; 1129 1130 if (!dev->dma_coherent) 1131 vma->vm_page_prot = __get_dma_pgprot(attrs, vma->vm_page_prot); 1132 1133 err = vm_map_pages(vma, pages, nr_pages); 1134 if (err) 1135 pr_err("Remapping memory failed: %d\n", err); 1136 1137 return err; 1138 } 1139 1140 /* 1141 * free a page as defined by the above mapping. 1142 * Must not be called with IRQs disabled. 1143 */ 1144 static void arm_iommu_free_attrs(struct device *dev, size_t size, void *cpu_addr, 1145 dma_addr_t handle, unsigned long attrs) 1146 { 1147 int coherent_flag = dev->dma_coherent ? COHERENT : NORMAL; 1148 struct page **pages; 1149 size = PAGE_ALIGN(size); 1150 1151 if (coherent_flag == COHERENT || __in_atomic_pool(cpu_addr, size)) { 1152 __iommu_free_atomic(dev, cpu_addr, handle, size, coherent_flag); 1153 return; 1154 } 1155 1156 pages = __iommu_get_pages(cpu_addr, attrs); 1157 if (!pages) { 1158 WARN(1, "trying to free invalid coherent area: %p\n", cpu_addr); 1159 return; 1160 } 1161 1162 if ((attrs & DMA_ATTR_NO_KERNEL_MAPPING) == 0) 1163 dma_common_free_remap(cpu_addr, size); 1164 1165 __iommu_remove_mapping(dev, handle, size); 1166 __iommu_free_buffer(dev, pages, size, attrs); 1167 } 1168 1169 static int arm_iommu_get_sgtable(struct device *dev, struct sg_table *sgt, 1170 void *cpu_addr, dma_addr_t dma_addr, 1171 size_t size, unsigned long attrs) 1172 { 1173 unsigned int count = PAGE_ALIGN(size) >> PAGE_SHIFT; 1174 struct page **pages = __iommu_get_pages(cpu_addr, attrs); 1175 1176 if (!pages) 1177 return -ENXIO; 1178 1179 return sg_alloc_table_from_pages(sgt, pages, count, 0, size, 1180 GFP_KERNEL); 1181 } 1182 1183 /* 1184 * Map a part of the scatter-gather list into contiguous io address space 1185 */ 1186 static int __map_sg_chunk(struct device *dev, struct scatterlist *sg, 1187 size_t size, dma_addr_t *handle, 1188 enum dma_data_direction dir, unsigned long attrs) 1189 { 1190 struct dma_iommu_mapping *mapping = to_dma_iommu_mapping(dev); 1191 dma_addr_t iova, iova_base; 1192 int ret = 0; 1193 unsigned int count; 1194 struct scatterlist *s; 1195 int prot; 1196 1197 size = PAGE_ALIGN(size); 1198 *handle = DMA_MAPPING_ERROR; 1199 1200 iova_base = iova = __alloc_iova(mapping, size); 1201 if (iova == DMA_MAPPING_ERROR) 1202 return -ENOMEM; 1203 1204 for (count = 0, s = sg; count < (size >> PAGE_SHIFT); s = sg_next(s)) { 1205 phys_addr_t phys = page_to_phys(sg_page(s)); 1206 unsigned int len = PAGE_ALIGN(s->offset + s->length); 1207 1208 if (!dev->dma_coherent && !(attrs & DMA_ATTR_SKIP_CPU_SYNC)) 1209 __dma_page_cpu_to_dev(sg_page(s), s->offset, s->length, dir); 1210 1211 prot = __dma_info_to_prot(dir, attrs); 1212 1213 ret = iommu_map(mapping->domain, iova, phys, len, prot, 1214 GFP_KERNEL); 1215 if (ret < 0) 1216 goto fail; 1217 count += len >> PAGE_SHIFT; 1218 iova += len; 1219 } 1220 *handle = iova_base; 1221 1222 return 0; 1223 fail: 1224 iommu_unmap(mapping->domain, iova_base, count * PAGE_SIZE); 1225 __free_iova(mapping, iova_base, size); 1226 return ret; 1227 } 1228 1229 /** 1230 * arm_iommu_map_sg - map a set of SG buffers for streaming mode DMA 1231 * @dev: valid struct device pointer 1232 * @sg: list of buffers 1233 * @nents: number of buffers to map 1234 * @dir: DMA transfer direction 1235 * 1236 * Map a set of buffers described by scatterlist in streaming mode for DMA. 1237 * The scatter gather list elements are merged together (if possible) and 1238 * tagged with the appropriate dma address and length. They are obtained via 1239 * sg_dma_{address,length}. 1240 */ 1241 static int arm_iommu_map_sg(struct device *dev, struct scatterlist *sg, 1242 int nents, enum dma_data_direction dir, unsigned long attrs) 1243 { 1244 struct scatterlist *s = sg, *dma = sg, *start = sg; 1245 int i, count = 0, ret; 1246 unsigned int offset = s->offset; 1247 unsigned int size = s->offset + s->length; 1248 unsigned int max = dma_get_max_seg_size(dev); 1249 1250 for (i = 1; i < nents; i++) { 1251 s = sg_next(s); 1252 1253 s->dma_length = 0; 1254 1255 if (s->offset || (size & ~PAGE_MASK) || size + s->length > max) { 1256 ret = __map_sg_chunk(dev, start, size, 1257 &dma->dma_address, dir, attrs); 1258 if (ret < 0) 1259 goto bad_mapping; 1260 1261 dma->dma_address += offset; 1262 dma->dma_length = size - offset; 1263 1264 size = offset = s->offset; 1265 start = s; 1266 dma = sg_next(dma); 1267 count += 1; 1268 } 1269 size += s->length; 1270 } 1271 ret = __map_sg_chunk(dev, start, size, &dma->dma_address, dir, attrs); 1272 if (ret < 0) 1273 goto bad_mapping; 1274 1275 dma->dma_address += offset; 1276 dma->dma_length = size - offset; 1277 1278 return count+1; 1279 1280 bad_mapping: 1281 for_each_sg(sg, s, count, i) 1282 __iommu_remove_mapping(dev, sg_dma_address(s), sg_dma_len(s)); 1283 if (ret == -ENOMEM) 1284 return ret; 1285 return -EINVAL; 1286 } 1287 1288 /** 1289 * arm_iommu_unmap_sg - unmap a set of SG buffers mapped by dma_map_sg 1290 * @dev: valid struct device pointer 1291 * @sg: list of buffers 1292 * @nents: number of buffers to unmap (same as was passed to dma_map_sg) 1293 * @dir: DMA transfer direction (same as was passed to dma_map_sg) 1294 * 1295 * Unmap a set of streaming mode DMA translations. Again, CPU access 1296 * rules concerning calls here are the same as for dma_unmap_single(). 1297 */ 1298 static void arm_iommu_unmap_sg(struct device *dev, 1299 struct scatterlist *sg, int nents, 1300 enum dma_data_direction dir, 1301 unsigned long attrs) 1302 { 1303 struct scatterlist *s; 1304 int i; 1305 1306 for_each_sg(sg, s, nents, i) { 1307 if (sg_dma_len(s)) 1308 __iommu_remove_mapping(dev, sg_dma_address(s), 1309 sg_dma_len(s)); 1310 if (!dev->dma_coherent && !(attrs & DMA_ATTR_SKIP_CPU_SYNC)) 1311 __dma_page_dev_to_cpu(sg_page(s), s->offset, 1312 s->length, dir); 1313 } 1314 } 1315 1316 /** 1317 * arm_iommu_sync_sg_for_cpu 1318 * @dev: valid struct device pointer 1319 * @sg: list of buffers 1320 * @nents: number of buffers to map (returned from dma_map_sg) 1321 * @dir: DMA transfer direction (same as was passed to dma_map_sg) 1322 */ 1323 static void arm_iommu_sync_sg_for_cpu(struct device *dev, 1324 struct scatterlist *sg, 1325 int nents, enum dma_data_direction dir) 1326 { 1327 struct scatterlist *s; 1328 int i; 1329 1330 if (dev->dma_coherent) 1331 return; 1332 1333 for_each_sg(sg, s, nents, i) 1334 __dma_page_dev_to_cpu(sg_page(s), s->offset, s->length, dir); 1335 1336 } 1337 1338 /** 1339 * arm_iommu_sync_sg_for_device 1340 * @dev: valid struct device pointer 1341 * @sg: list of buffers 1342 * @nents: number of buffers to map (returned from dma_map_sg) 1343 * @dir: DMA transfer direction (same as was passed to dma_map_sg) 1344 */ 1345 static void arm_iommu_sync_sg_for_device(struct device *dev, 1346 struct scatterlist *sg, 1347 int nents, enum dma_data_direction dir) 1348 { 1349 struct scatterlist *s; 1350 int i; 1351 1352 if (dev->dma_coherent) 1353 return; 1354 1355 for_each_sg(sg, s, nents, i) 1356 __dma_page_cpu_to_dev(sg_page(s), s->offset, s->length, dir); 1357 } 1358 1359 /** 1360 * arm_iommu_map_page 1361 * @dev: valid struct device pointer 1362 * @page: page that buffer resides in 1363 * @offset: offset into page for start of buffer 1364 * @size: size of buffer to map 1365 * @dir: DMA transfer direction 1366 * 1367 * IOMMU aware version of arm_dma_map_page() 1368 */ 1369 static dma_addr_t arm_iommu_map_page(struct device *dev, struct page *page, 1370 unsigned long offset, size_t size, enum dma_data_direction dir, 1371 unsigned long attrs) 1372 { 1373 struct dma_iommu_mapping *mapping = to_dma_iommu_mapping(dev); 1374 dma_addr_t dma_addr; 1375 int ret, prot, len = PAGE_ALIGN(size + offset); 1376 1377 if (!dev->dma_coherent && !(attrs & DMA_ATTR_SKIP_CPU_SYNC)) 1378 __dma_page_cpu_to_dev(page, offset, size, dir); 1379 1380 dma_addr = __alloc_iova(mapping, len); 1381 if (dma_addr == DMA_MAPPING_ERROR) 1382 return dma_addr; 1383 1384 prot = __dma_info_to_prot(dir, attrs); 1385 1386 ret = iommu_map(mapping->domain, dma_addr, page_to_phys(page), len, 1387 prot, GFP_KERNEL); 1388 if (ret < 0) 1389 goto fail; 1390 1391 return dma_addr + offset; 1392 fail: 1393 __free_iova(mapping, dma_addr, len); 1394 return DMA_MAPPING_ERROR; 1395 } 1396 1397 /** 1398 * arm_iommu_unmap_page 1399 * @dev: valid struct device pointer 1400 * @handle: DMA address of buffer 1401 * @size: size of buffer (same as passed to dma_map_page) 1402 * @dir: DMA transfer direction (same as passed to dma_map_page) 1403 * 1404 * IOMMU aware version of arm_dma_unmap_page() 1405 */ 1406 static void arm_iommu_unmap_page(struct device *dev, dma_addr_t handle, 1407 size_t size, enum dma_data_direction dir, unsigned long attrs) 1408 { 1409 struct dma_iommu_mapping *mapping = to_dma_iommu_mapping(dev); 1410 dma_addr_t iova = handle & PAGE_MASK; 1411 struct page *page; 1412 int offset = handle & ~PAGE_MASK; 1413 int len = PAGE_ALIGN(size + offset); 1414 1415 if (!iova) 1416 return; 1417 1418 if (!dev->dma_coherent && !(attrs & DMA_ATTR_SKIP_CPU_SYNC)) { 1419 page = phys_to_page(iommu_iova_to_phys(mapping->domain, iova)); 1420 __dma_page_dev_to_cpu(page, offset, size, dir); 1421 } 1422 1423 iommu_unmap(mapping->domain, iova, len); 1424 __free_iova(mapping, iova, len); 1425 } 1426 1427 /** 1428 * arm_iommu_map_resource - map a device resource for DMA 1429 * @dev: valid struct device pointer 1430 * @phys_addr: physical address of resource 1431 * @size: size of resource to map 1432 * @dir: DMA transfer direction 1433 */ 1434 static dma_addr_t arm_iommu_map_resource(struct device *dev, 1435 phys_addr_t phys_addr, size_t size, 1436 enum dma_data_direction dir, unsigned long attrs) 1437 { 1438 struct dma_iommu_mapping *mapping = to_dma_iommu_mapping(dev); 1439 dma_addr_t dma_addr; 1440 int ret, prot; 1441 phys_addr_t addr = phys_addr & PAGE_MASK; 1442 unsigned int offset = phys_addr & ~PAGE_MASK; 1443 size_t len = PAGE_ALIGN(size + offset); 1444 1445 dma_addr = __alloc_iova(mapping, len); 1446 if (dma_addr == DMA_MAPPING_ERROR) 1447 return dma_addr; 1448 1449 prot = __dma_info_to_prot(dir, attrs) | IOMMU_MMIO; 1450 1451 ret = iommu_map(mapping->domain, dma_addr, addr, len, prot, GFP_KERNEL); 1452 if (ret < 0) 1453 goto fail; 1454 1455 return dma_addr + offset; 1456 fail: 1457 __free_iova(mapping, dma_addr, len); 1458 return DMA_MAPPING_ERROR; 1459 } 1460 1461 /** 1462 * arm_iommu_unmap_resource - unmap a device DMA resource 1463 * @dev: valid struct device pointer 1464 * @dma_handle: DMA address to resource 1465 * @size: size of resource to map 1466 * @dir: DMA transfer direction 1467 */ 1468 static void arm_iommu_unmap_resource(struct device *dev, dma_addr_t dma_handle, 1469 size_t size, enum dma_data_direction dir, 1470 unsigned long attrs) 1471 { 1472 struct dma_iommu_mapping *mapping = to_dma_iommu_mapping(dev); 1473 dma_addr_t iova = dma_handle & PAGE_MASK; 1474 unsigned int offset = dma_handle & ~PAGE_MASK; 1475 size_t len = PAGE_ALIGN(size + offset); 1476 1477 if (!iova) 1478 return; 1479 1480 iommu_unmap(mapping->domain, iova, len); 1481 __free_iova(mapping, iova, len); 1482 } 1483 1484 static void arm_iommu_sync_single_for_cpu(struct device *dev, 1485 dma_addr_t handle, size_t size, enum dma_data_direction dir) 1486 { 1487 struct dma_iommu_mapping *mapping = to_dma_iommu_mapping(dev); 1488 dma_addr_t iova = handle & PAGE_MASK; 1489 struct page *page; 1490 unsigned int offset = handle & ~PAGE_MASK; 1491 1492 if (dev->dma_coherent || !iova) 1493 return; 1494 1495 page = phys_to_page(iommu_iova_to_phys(mapping->domain, iova)); 1496 __dma_page_dev_to_cpu(page, offset, size, dir); 1497 } 1498 1499 static void arm_iommu_sync_single_for_device(struct device *dev, 1500 dma_addr_t handle, size_t size, enum dma_data_direction dir) 1501 { 1502 struct dma_iommu_mapping *mapping = to_dma_iommu_mapping(dev); 1503 dma_addr_t iova = handle & PAGE_MASK; 1504 struct page *page; 1505 unsigned int offset = handle & ~PAGE_MASK; 1506 1507 if (dev->dma_coherent || !iova) 1508 return; 1509 1510 page = phys_to_page(iommu_iova_to_phys(mapping->domain, iova)); 1511 __dma_page_cpu_to_dev(page, offset, size, dir); 1512 } 1513 1514 static const struct dma_map_ops iommu_ops = { 1515 .alloc = arm_iommu_alloc_attrs, 1516 .free = arm_iommu_free_attrs, 1517 .mmap = arm_iommu_mmap_attrs, 1518 .get_sgtable = arm_iommu_get_sgtable, 1519 1520 .map_page = arm_iommu_map_page, 1521 .unmap_page = arm_iommu_unmap_page, 1522 .sync_single_for_cpu = arm_iommu_sync_single_for_cpu, 1523 .sync_single_for_device = arm_iommu_sync_single_for_device, 1524 1525 .map_sg = arm_iommu_map_sg, 1526 .unmap_sg = arm_iommu_unmap_sg, 1527 .sync_sg_for_cpu = arm_iommu_sync_sg_for_cpu, 1528 .sync_sg_for_device = arm_iommu_sync_sg_for_device, 1529 1530 .map_resource = arm_iommu_map_resource, 1531 .unmap_resource = arm_iommu_unmap_resource, 1532 }; 1533 1534 /** 1535 * arm_iommu_create_mapping 1536 * @bus: pointer to the bus holding the client device (for IOMMU calls) 1537 * @base: start address of the valid IO address space 1538 * @size: maximum size of the valid IO address space 1539 * 1540 * Creates a mapping structure which holds information about used/unused 1541 * IO address ranges, which is required to perform memory allocation and 1542 * mapping with IOMMU aware functions. 1543 * 1544 * The client device need to be attached to the mapping with 1545 * arm_iommu_attach_device function. 1546 */ 1547 struct dma_iommu_mapping * 1548 arm_iommu_create_mapping(const struct bus_type *bus, dma_addr_t base, u64 size) 1549 { 1550 unsigned int bits = size >> PAGE_SHIFT; 1551 unsigned int bitmap_size = BITS_TO_LONGS(bits) * sizeof(long); 1552 struct dma_iommu_mapping *mapping; 1553 int extensions = 1; 1554 int err = -ENOMEM; 1555 1556 /* currently only 32-bit DMA address space is supported */ 1557 if (size > DMA_BIT_MASK(32) + 1) 1558 return ERR_PTR(-ERANGE); 1559 1560 if (!bitmap_size) 1561 return ERR_PTR(-EINVAL); 1562 1563 if (bitmap_size > PAGE_SIZE) { 1564 extensions = bitmap_size / PAGE_SIZE; 1565 bitmap_size = PAGE_SIZE; 1566 } 1567 1568 mapping = kzalloc(sizeof(struct dma_iommu_mapping), GFP_KERNEL); 1569 if (!mapping) 1570 goto err; 1571 1572 mapping->bitmap_size = bitmap_size; 1573 mapping->bitmaps = kcalloc(extensions, sizeof(unsigned long *), 1574 GFP_KERNEL); 1575 if (!mapping->bitmaps) 1576 goto err2; 1577 1578 mapping->bitmaps[0] = kzalloc(bitmap_size, GFP_KERNEL); 1579 if (!mapping->bitmaps[0]) 1580 goto err3; 1581 1582 mapping->nr_bitmaps = 1; 1583 mapping->extensions = extensions; 1584 mapping->base = base; 1585 mapping->bits = BITS_PER_BYTE * bitmap_size; 1586 1587 spin_lock_init(&mapping->lock); 1588 1589 mapping->domain = iommu_domain_alloc(bus); 1590 if (!mapping->domain) 1591 goto err4; 1592 1593 kref_init(&mapping->kref); 1594 return mapping; 1595 err4: 1596 kfree(mapping->bitmaps[0]); 1597 err3: 1598 kfree(mapping->bitmaps); 1599 err2: 1600 kfree(mapping); 1601 err: 1602 return ERR_PTR(err); 1603 } 1604 EXPORT_SYMBOL_GPL(arm_iommu_create_mapping); 1605 1606 static void release_iommu_mapping(struct kref *kref) 1607 { 1608 int i; 1609 struct dma_iommu_mapping *mapping = 1610 container_of(kref, struct dma_iommu_mapping, kref); 1611 1612 iommu_domain_free(mapping->domain); 1613 for (i = 0; i < mapping->nr_bitmaps; i++) 1614 kfree(mapping->bitmaps[i]); 1615 kfree(mapping->bitmaps); 1616 kfree(mapping); 1617 } 1618 1619 static int extend_iommu_mapping(struct dma_iommu_mapping *mapping) 1620 { 1621 int next_bitmap; 1622 1623 if (mapping->nr_bitmaps >= mapping->extensions) 1624 return -EINVAL; 1625 1626 next_bitmap = mapping->nr_bitmaps; 1627 mapping->bitmaps[next_bitmap] = kzalloc(mapping->bitmap_size, 1628 GFP_ATOMIC); 1629 if (!mapping->bitmaps[next_bitmap]) 1630 return -ENOMEM; 1631 1632 mapping->nr_bitmaps++; 1633 1634 return 0; 1635 } 1636 1637 void arm_iommu_release_mapping(struct dma_iommu_mapping *mapping) 1638 { 1639 if (mapping) 1640 kref_put(&mapping->kref, release_iommu_mapping); 1641 } 1642 EXPORT_SYMBOL_GPL(arm_iommu_release_mapping); 1643 1644 static int __arm_iommu_attach_device(struct device *dev, 1645 struct dma_iommu_mapping *mapping) 1646 { 1647 int err; 1648 1649 err = iommu_attach_device(mapping->domain, dev); 1650 if (err) 1651 return err; 1652 1653 kref_get(&mapping->kref); 1654 to_dma_iommu_mapping(dev) = mapping; 1655 1656 pr_debug("Attached IOMMU controller to %s device.\n", dev_name(dev)); 1657 return 0; 1658 } 1659 1660 /** 1661 * arm_iommu_attach_device 1662 * @dev: valid struct device pointer 1663 * @mapping: io address space mapping structure (returned from 1664 * arm_iommu_create_mapping) 1665 * 1666 * Attaches specified io address space mapping to the provided device. 1667 * This replaces the dma operations (dma_map_ops pointer) with the 1668 * IOMMU aware version. 1669 * 1670 * More than one client might be attached to the same io address space 1671 * mapping. 1672 */ 1673 int arm_iommu_attach_device(struct device *dev, 1674 struct dma_iommu_mapping *mapping) 1675 { 1676 int err; 1677 1678 err = __arm_iommu_attach_device(dev, mapping); 1679 if (err) 1680 return err; 1681 1682 set_dma_ops(dev, &iommu_ops); 1683 return 0; 1684 } 1685 EXPORT_SYMBOL_GPL(arm_iommu_attach_device); 1686 1687 /** 1688 * arm_iommu_detach_device 1689 * @dev: valid struct device pointer 1690 * 1691 * Detaches the provided device from a previously attached map. 1692 * This overwrites the dma_ops pointer with appropriate non-IOMMU ops. 1693 */ 1694 void arm_iommu_detach_device(struct device *dev) 1695 { 1696 struct dma_iommu_mapping *mapping; 1697 1698 mapping = to_dma_iommu_mapping(dev); 1699 if (!mapping) { 1700 dev_warn(dev, "Not attached\n"); 1701 return; 1702 } 1703 1704 iommu_detach_device(mapping->domain, dev); 1705 kref_put(&mapping->kref, release_iommu_mapping); 1706 to_dma_iommu_mapping(dev) = NULL; 1707 set_dma_ops(dev, NULL); 1708 1709 pr_debug("Detached IOMMU controller from %s device.\n", dev_name(dev)); 1710 } 1711 EXPORT_SYMBOL_GPL(arm_iommu_detach_device); 1712 1713 static void arm_setup_iommu_dma_ops(struct device *dev, u64 dma_base, u64 size, 1714 const struct iommu_ops *iommu, bool coherent) 1715 { 1716 struct dma_iommu_mapping *mapping; 1717 1718 mapping = arm_iommu_create_mapping(dev->bus, dma_base, size); 1719 if (IS_ERR(mapping)) { 1720 pr_warn("Failed to create %llu-byte IOMMU mapping for device %s\n", 1721 size, dev_name(dev)); 1722 return; 1723 } 1724 1725 if (__arm_iommu_attach_device(dev, mapping)) { 1726 pr_warn("Failed to attached device %s to IOMMU_mapping\n", 1727 dev_name(dev)); 1728 arm_iommu_release_mapping(mapping); 1729 return; 1730 } 1731 1732 set_dma_ops(dev, &iommu_ops); 1733 } 1734 1735 static void arm_teardown_iommu_dma_ops(struct device *dev) 1736 { 1737 struct dma_iommu_mapping *mapping = to_dma_iommu_mapping(dev); 1738 1739 if (!mapping) 1740 return; 1741 1742 arm_iommu_detach_device(dev); 1743 arm_iommu_release_mapping(mapping); 1744 } 1745 1746 #else 1747 1748 static void arm_setup_iommu_dma_ops(struct device *dev, u64 dma_base, u64 size, 1749 const struct iommu_ops *iommu, bool coherent) 1750 { 1751 } 1752 1753 static void arm_teardown_iommu_dma_ops(struct device *dev) { } 1754 1755 #endif /* CONFIG_ARM_DMA_USE_IOMMU */ 1756 1757 void arch_setup_dma_ops(struct device *dev, u64 dma_base, u64 size, 1758 const struct iommu_ops *iommu, bool coherent) 1759 { 1760 /* 1761 * Due to legacy code that sets the ->dma_coherent flag from a bus 1762 * notifier we can't just assign coherent to the ->dma_coherent flag 1763 * here, but instead have to make sure we only set but never clear it 1764 * for now. 1765 */ 1766 if (coherent) 1767 dev->dma_coherent = true; 1768 1769 /* 1770 * Don't override the dma_ops if they have already been set. Ideally 1771 * this should be the only location where dma_ops are set, remove this 1772 * check when all other callers of set_dma_ops will have disappeared. 1773 */ 1774 if (dev->dma_ops) 1775 return; 1776 1777 if (iommu) 1778 arm_setup_iommu_dma_ops(dev, dma_base, size, iommu, coherent); 1779 1780 xen_setup_dma_ops(dev); 1781 dev->archdata.dma_ops_setup = true; 1782 } 1783 1784 void arch_teardown_dma_ops(struct device *dev) 1785 { 1786 if (!dev->archdata.dma_ops_setup) 1787 return; 1788 1789 arm_teardown_iommu_dma_ops(dev); 1790 /* Let arch_setup_dma_ops() start again from scratch upon re-probe */ 1791 set_dma_ops(dev, NULL); 1792 } 1793 1794 void arch_sync_dma_for_device(phys_addr_t paddr, size_t size, 1795 enum dma_data_direction dir) 1796 { 1797 __dma_page_cpu_to_dev(phys_to_page(paddr), paddr & (PAGE_SIZE - 1), 1798 size, dir); 1799 } 1800 1801 void arch_sync_dma_for_cpu(phys_addr_t paddr, size_t size, 1802 enum dma_data_direction dir) 1803 { 1804 __dma_page_dev_to_cpu(phys_to_page(paddr), paddr & (PAGE_SIZE - 1), 1805 size, dir); 1806 } 1807 1808 void *arch_dma_alloc(struct device *dev, size_t size, dma_addr_t *dma_handle, 1809 gfp_t gfp, unsigned long attrs) 1810 { 1811 return __dma_alloc(dev, size, dma_handle, gfp, 1812 __get_dma_pgprot(attrs, PAGE_KERNEL), false, 1813 attrs, __builtin_return_address(0)); 1814 } 1815 1816 void arch_dma_free(struct device *dev, size_t size, void *cpu_addr, 1817 dma_addr_t dma_handle, unsigned long attrs) 1818 { 1819 __arm_dma_free(dev, size, cpu_addr, dma_handle, attrs, false); 1820 } 1821