1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * linux/arch/arm/mm/dma-mapping.c 4 * 5 * Copyright (C) 2000-2004 Russell King 6 * 7 * DMA uncached mapping support. 8 */ 9 #include <linux/module.h> 10 #include <linux/mm.h> 11 #include <linux/genalloc.h> 12 #include <linux/gfp.h> 13 #include <linux/errno.h> 14 #include <linux/list.h> 15 #include <linux/init.h> 16 #include <linux/device.h> 17 #include <linux/dma-direct.h> 18 #include <linux/dma-map-ops.h> 19 #include <linux/highmem.h> 20 #include <linux/memblock.h> 21 #include <linux/slab.h> 22 #include <linux/iommu.h> 23 #include <linux/io.h> 24 #include <linux/vmalloc.h> 25 #include <linux/sizes.h> 26 #include <linux/cma.h> 27 28 #include <asm/memory.h> 29 #include <asm/highmem.h> 30 #include <asm/cacheflush.h> 31 #include <asm/tlbflush.h> 32 #include <asm/mach/arch.h> 33 #include <asm/dma-iommu.h> 34 #include <asm/mach/map.h> 35 #include <asm/system_info.h> 36 #include <asm/xen/xen-ops.h> 37 38 #include "dma.h" 39 #include "mm.h" 40 41 struct arm_dma_alloc_args { 42 struct device *dev; 43 size_t size; 44 gfp_t gfp; 45 pgprot_t prot; 46 const void *caller; 47 bool want_vaddr; 48 int coherent_flag; 49 }; 50 51 struct arm_dma_free_args { 52 struct device *dev; 53 size_t size; 54 void *cpu_addr; 55 struct page *page; 56 bool want_vaddr; 57 }; 58 59 #define NORMAL 0 60 #define COHERENT 1 61 62 struct arm_dma_allocator { 63 void *(*alloc)(struct arm_dma_alloc_args *args, 64 struct page **ret_page); 65 void (*free)(struct arm_dma_free_args *args); 66 }; 67 68 struct arm_dma_buffer { 69 struct list_head list; 70 void *virt; 71 struct arm_dma_allocator *allocator; 72 }; 73 74 static LIST_HEAD(arm_dma_bufs); 75 static DEFINE_SPINLOCK(arm_dma_bufs_lock); 76 77 static struct arm_dma_buffer *arm_dma_buffer_find(void *virt) 78 { 79 struct arm_dma_buffer *buf, *found = NULL; 80 unsigned long flags; 81 82 spin_lock_irqsave(&arm_dma_bufs_lock, flags); 83 list_for_each_entry(buf, &arm_dma_bufs, list) { 84 if (buf->virt == virt) { 85 list_del(&buf->list); 86 found = buf; 87 break; 88 } 89 } 90 spin_unlock_irqrestore(&arm_dma_bufs_lock, flags); 91 return found; 92 } 93 94 /* 95 * The DMA API is built upon the notion of "buffer ownership". A buffer 96 * is either exclusively owned by the CPU (and therefore may be accessed 97 * by it) or exclusively owned by the DMA device. These helper functions 98 * represent the transitions between these two ownership states. 99 * 100 * Note, however, that on later ARMs, this notion does not work due to 101 * speculative prefetches. We model our approach on the assumption that 102 * the CPU does do speculative prefetches, which means we clean caches 103 * before transfers and delay cache invalidation until transfer completion. 104 * 105 */ 106 107 static void __dma_clear_buffer(struct page *page, size_t size, int coherent_flag) 108 { 109 /* 110 * Ensure that the allocated pages are zeroed, and that any data 111 * lurking in the kernel direct-mapped region is invalidated. 112 */ 113 if (PageHighMem(page)) { 114 phys_addr_t base = __pfn_to_phys(page_to_pfn(page)); 115 phys_addr_t end = base + size; 116 while (size > 0) { 117 void *ptr = kmap_atomic(page); 118 memset(ptr, 0, PAGE_SIZE); 119 if (coherent_flag != COHERENT) 120 dmac_flush_range(ptr, ptr + PAGE_SIZE); 121 kunmap_atomic(ptr); 122 page++; 123 size -= PAGE_SIZE; 124 } 125 if (coherent_flag != COHERENT) 126 outer_flush_range(base, end); 127 } else { 128 void *ptr = page_address(page); 129 memset(ptr, 0, size); 130 if (coherent_flag != COHERENT) { 131 dmac_flush_range(ptr, ptr + size); 132 outer_flush_range(__pa(ptr), __pa(ptr) + size); 133 } 134 } 135 } 136 137 /* 138 * Allocate a DMA buffer for 'dev' of size 'size' using the 139 * specified gfp mask. Note that 'size' must be page aligned. 140 */ 141 static struct page *__dma_alloc_buffer(struct device *dev, size_t size, 142 gfp_t gfp, int coherent_flag) 143 { 144 unsigned long order = get_order(size); 145 struct page *page, *p, *e; 146 147 page = alloc_pages(gfp, order); 148 if (!page) 149 return NULL; 150 151 /* 152 * Now split the huge page and free the excess pages 153 */ 154 split_page(page, order); 155 for (p = page + (size >> PAGE_SHIFT), e = page + (1 << order); p < e; p++) 156 __free_page(p); 157 158 __dma_clear_buffer(page, size, coherent_flag); 159 160 return page; 161 } 162 163 /* 164 * Free a DMA buffer. 'size' must be page aligned. 165 */ 166 static void __dma_free_buffer(struct page *page, size_t size) 167 { 168 struct page *e = page + (size >> PAGE_SHIFT); 169 170 while (page < e) { 171 __free_page(page); 172 page++; 173 } 174 } 175 176 static void *__alloc_from_contiguous(struct device *dev, size_t size, 177 pgprot_t prot, struct page **ret_page, 178 const void *caller, bool want_vaddr, 179 int coherent_flag, gfp_t gfp); 180 181 static void *__alloc_remap_buffer(struct device *dev, size_t size, gfp_t gfp, 182 pgprot_t prot, struct page **ret_page, 183 const void *caller, bool want_vaddr); 184 185 #define DEFAULT_DMA_COHERENT_POOL_SIZE SZ_256K 186 static struct gen_pool *atomic_pool __ro_after_init; 187 188 static size_t atomic_pool_size __initdata = DEFAULT_DMA_COHERENT_POOL_SIZE; 189 190 static int __init early_coherent_pool(char *p) 191 { 192 atomic_pool_size = memparse(p, &p); 193 return 0; 194 } 195 early_param("coherent_pool", early_coherent_pool); 196 197 /* 198 * Initialise the coherent pool for atomic allocations. 199 */ 200 static int __init atomic_pool_init(void) 201 { 202 pgprot_t prot = pgprot_dmacoherent(PAGE_KERNEL); 203 gfp_t gfp = GFP_KERNEL | GFP_DMA; 204 struct page *page; 205 void *ptr; 206 207 atomic_pool = gen_pool_create(PAGE_SHIFT, -1); 208 if (!atomic_pool) 209 goto out; 210 /* 211 * The atomic pool is only used for non-coherent allocations 212 * so we must pass NORMAL for coherent_flag. 213 */ 214 if (dev_get_cma_area(NULL)) 215 ptr = __alloc_from_contiguous(NULL, atomic_pool_size, prot, 216 &page, atomic_pool_init, true, NORMAL, 217 GFP_KERNEL); 218 else 219 ptr = __alloc_remap_buffer(NULL, atomic_pool_size, gfp, prot, 220 &page, atomic_pool_init, true); 221 if (ptr) { 222 int ret; 223 224 ret = gen_pool_add_virt(atomic_pool, (unsigned long)ptr, 225 page_to_phys(page), 226 atomic_pool_size, -1); 227 if (ret) 228 goto destroy_genpool; 229 230 gen_pool_set_algo(atomic_pool, 231 gen_pool_first_fit_order_align, 232 NULL); 233 pr_info("DMA: preallocated %zu KiB pool for atomic coherent allocations\n", 234 atomic_pool_size / 1024); 235 return 0; 236 } 237 238 destroy_genpool: 239 gen_pool_destroy(atomic_pool); 240 atomic_pool = NULL; 241 out: 242 pr_err("DMA: failed to allocate %zu KiB pool for atomic coherent allocation\n", 243 atomic_pool_size / 1024); 244 return -ENOMEM; 245 } 246 /* 247 * CMA is activated by core_initcall, so we must be called after it. 248 */ 249 postcore_initcall(atomic_pool_init); 250 251 #ifdef CONFIG_CMA_AREAS 252 struct dma_contig_early_reserve { 253 phys_addr_t base; 254 unsigned long size; 255 }; 256 257 static struct dma_contig_early_reserve dma_mmu_remap[MAX_CMA_AREAS] __initdata; 258 259 static int dma_mmu_remap_num __initdata; 260 261 void __init dma_contiguous_early_fixup(phys_addr_t base, unsigned long size) 262 { 263 dma_mmu_remap[dma_mmu_remap_num].base = base; 264 dma_mmu_remap[dma_mmu_remap_num].size = size; 265 dma_mmu_remap_num++; 266 } 267 268 void __init dma_contiguous_remap(void) 269 { 270 int i; 271 for (i = 0; i < dma_mmu_remap_num; i++) { 272 phys_addr_t start = dma_mmu_remap[i].base; 273 phys_addr_t end = start + dma_mmu_remap[i].size; 274 struct map_desc map; 275 unsigned long addr; 276 277 if (end > arm_lowmem_limit) 278 end = arm_lowmem_limit; 279 if (start >= end) 280 continue; 281 282 map.pfn = __phys_to_pfn(start); 283 map.virtual = __phys_to_virt(start); 284 map.length = end - start; 285 map.type = MT_MEMORY_DMA_READY; 286 287 /* 288 * Clear previous low-memory mapping to ensure that the 289 * TLB does not see any conflicting entries, then flush 290 * the TLB of the old entries before creating new mappings. 291 * 292 * This ensures that any speculatively loaded TLB entries 293 * (even though they may be rare) can not cause any problems, 294 * and ensures that this code is architecturally compliant. 295 */ 296 for (addr = __phys_to_virt(start); addr < __phys_to_virt(end); 297 addr += PMD_SIZE) 298 pmd_clear(pmd_off_k(addr)); 299 300 flush_tlb_kernel_range(__phys_to_virt(start), 301 __phys_to_virt(end)); 302 303 iotable_init(&map, 1); 304 } 305 } 306 #endif 307 308 static int __dma_update_pte(pte_t *pte, unsigned long addr, void *data) 309 { 310 struct page *page = virt_to_page((void *)addr); 311 pgprot_t prot = *(pgprot_t *)data; 312 313 set_pte_ext(pte, mk_pte(page, prot), 0); 314 return 0; 315 } 316 317 static void __dma_remap(struct page *page, size_t size, pgprot_t prot) 318 { 319 unsigned long start = (unsigned long) page_address(page); 320 unsigned end = start + size; 321 322 apply_to_page_range(&init_mm, start, size, __dma_update_pte, &prot); 323 flush_tlb_kernel_range(start, end); 324 } 325 326 static void *__alloc_remap_buffer(struct device *dev, size_t size, gfp_t gfp, 327 pgprot_t prot, struct page **ret_page, 328 const void *caller, bool want_vaddr) 329 { 330 struct page *page; 331 void *ptr = NULL; 332 /* 333 * __alloc_remap_buffer is only called when the device is 334 * non-coherent 335 */ 336 page = __dma_alloc_buffer(dev, size, gfp, NORMAL); 337 if (!page) 338 return NULL; 339 if (!want_vaddr) 340 goto out; 341 342 ptr = dma_common_contiguous_remap(page, size, prot, caller); 343 if (!ptr) { 344 __dma_free_buffer(page, size); 345 return NULL; 346 } 347 348 out: 349 *ret_page = page; 350 return ptr; 351 } 352 353 static void *__alloc_from_pool(size_t size, struct page **ret_page) 354 { 355 unsigned long val; 356 void *ptr = NULL; 357 358 if (!atomic_pool) { 359 WARN(1, "coherent pool not initialised!\n"); 360 return NULL; 361 } 362 363 val = gen_pool_alloc(atomic_pool, size); 364 if (val) { 365 phys_addr_t phys = gen_pool_virt_to_phys(atomic_pool, val); 366 367 *ret_page = phys_to_page(phys); 368 ptr = (void *)val; 369 } 370 371 return ptr; 372 } 373 374 static bool __in_atomic_pool(void *start, size_t size) 375 { 376 return gen_pool_has_addr(atomic_pool, (unsigned long)start, size); 377 } 378 379 static int __free_from_pool(void *start, size_t size) 380 { 381 if (!__in_atomic_pool(start, size)) 382 return 0; 383 384 gen_pool_free(atomic_pool, (unsigned long)start, size); 385 386 return 1; 387 } 388 389 static void *__alloc_from_contiguous(struct device *dev, size_t size, 390 pgprot_t prot, struct page **ret_page, 391 const void *caller, bool want_vaddr, 392 int coherent_flag, gfp_t gfp) 393 { 394 unsigned long order = get_order(size); 395 size_t count = size >> PAGE_SHIFT; 396 struct page *page; 397 void *ptr = NULL; 398 399 page = dma_alloc_from_contiguous(dev, count, order, gfp & __GFP_NOWARN); 400 if (!page) 401 return NULL; 402 403 __dma_clear_buffer(page, size, coherent_flag); 404 405 if (!want_vaddr) 406 goto out; 407 408 if (PageHighMem(page)) { 409 ptr = dma_common_contiguous_remap(page, size, prot, caller); 410 if (!ptr) { 411 dma_release_from_contiguous(dev, page, count); 412 return NULL; 413 } 414 } else { 415 __dma_remap(page, size, prot); 416 ptr = page_address(page); 417 } 418 419 out: 420 *ret_page = page; 421 return ptr; 422 } 423 424 static void __free_from_contiguous(struct device *dev, struct page *page, 425 void *cpu_addr, size_t size, bool want_vaddr) 426 { 427 if (want_vaddr) { 428 if (PageHighMem(page)) 429 dma_common_free_remap(cpu_addr, size); 430 else 431 __dma_remap(page, size, PAGE_KERNEL); 432 } 433 dma_release_from_contiguous(dev, page, size >> PAGE_SHIFT); 434 } 435 436 static inline pgprot_t __get_dma_pgprot(unsigned long attrs, pgprot_t prot) 437 { 438 prot = (attrs & DMA_ATTR_WRITE_COMBINE) ? 439 pgprot_writecombine(prot) : 440 pgprot_dmacoherent(prot); 441 return prot; 442 } 443 444 static void *__alloc_simple_buffer(struct device *dev, size_t size, gfp_t gfp, 445 struct page **ret_page) 446 { 447 struct page *page; 448 /* __alloc_simple_buffer is only called when the device is coherent */ 449 page = __dma_alloc_buffer(dev, size, gfp, COHERENT); 450 if (!page) 451 return NULL; 452 453 *ret_page = page; 454 return page_address(page); 455 } 456 457 static void *simple_allocator_alloc(struct arm_dma_alloc_args *args, 458 struct page **ret_page) 459 { 460 return __alloc_simple_buffer(args->dev, args->size, args->gfp, 461 ret_page); 462 } 463 464 static void simple_allocator_free(struct arm_dma_free_args *args) 465 { 466 __dma_free_buffer(args->page, args->size); 467 } 468 469 static struct arm_dma_allocator simple_allocator = { 470 .alloc = simple_allocator_alloc, 471 .free = simple_allocator_free, 472 }; 473 474 static void *cma_allocator_alloc(struct arm_dma_alloc_args *args, 475 struct page **ret_page) 476 { 477 return __alloc_from_contiguous(args->dev, args->size, args->prot, 478 ret_page, args->caller, 479 args->want_vaddr, args->coherent_flag, 480 args->gfp); 481 } 482 483 static void cma_allocator_free(struct arm_dma_free_args *args) 484 { 485 __free_from_contiguous(args->dev, args->page, args->cpu_addr, 486 args->size, args->want_vaddr); 487 } 488 489 static struct arm_dma_allocator cma_allocator = { 490 .alloc = cma_allocator_alloc, 491 .free = cma_allocator_free, 492 }; 493 494 static void *pool_allocator_alloc(struct arm_dma_alloc_args *args, 495 struct page **ret_page) 496 { 497 return __alloc_from_pool(args->size, ret_page); 498 } 499 500 static void pool_allocator_free(struct arm_dma_free_args *args) 501 { 502 __free_from_pool(args->cpu_addr, args->size); 503 } 504 505 static struct arm_dma_allocator pool_allocator = { 506 .alloc = pool_allocator_alloc, 507 .free = pool_allocator_free, 508 }; 509 510 static void *remap_allocator_alloc(struct arm_dma_alloc_args *args, 511 struct page **ret_page) 512 { 513 return __alloc_remap_buffer(args->dev, args->size, args->gfp, 514 args->prot, ret_page, args->caller, 515 args->want_vaddr); 516 } 517 518 static void remap_allocator_free(struct arm_dma_free_args *args) 519 { 520 if (args->want_vaddr) 521 dma_common_free_remap(args->cpu_addr, args->size); 522 523 __dma_free_buffer(args->page, args->size); 524 } 525 526 static struct arm_dma_allocator remap_allocator = { 527 .alloc = remap_allocator_alloc, 528 .free = remap_allocator_free, 529 }; 530 531 static void *__dma_alloc(struct device *dev, size_t size, dma_addr_t *handle, 532 gfp_t gfp, pgprot_t prot, bool is_coherent, 533 unsigned long attrs, const void *caller) 534 { 535 u64 mask = min_not_zero(dev->coherent_dma_mask, dev->bus_dma_limit); 536 struct page *page = NULL; 537 void *addr; 538 bool allowblock, cma; 539 struct arm_dma_buffer *buf; 540 struct arm_dma_alloc_args args = { 541 .dev = dev, 542 .size = PAGE_ALIGN(size), 543 .gfp = gfp, 544 .prot = prot, 545 .caller = caller, 546 .want_vaddr = ((attrs & DMA_ATTR_NO_KERNEL_MAPPING) == 0), 547 .coherent_flag = is_coherent ? COHERENT : NORMAL, 548 }; 549 550 #ifdef CONFIG_DMA_API_DEBUG 551 u64 limit = (mask + 1) & ~mask; 552 if (limit && size >= limit) { 553 dev_warn(dev, "coherent allocation too big (requested %#x mask %#llx)\n", 554 size, mask); 555 return NULL; 556 } 557 #endif 558 559 buf = kzalloc(sizeof(*buf), 560 gfp & ~(__GFP_DMA | __GFP_DMA32 | __GFP_HIGHMEM)); 561 if (!buf) 562 return NULL; 563 564 if (mask < 0xffffffffULL) 565 gfp |= GFP_DMA; 566 567 args.gfp = gfp; 568 569 *handle = DMA_MAPPING_ERROR; 570 allowblock = gfpflags_allow_blocking(gfp); 571 cma = allowblock ? dev_get_cma_area(dev) : NULL; 572 573 if (cma) 574 buf->allocator = &cma_allocator; 575 else if (is_coherent) 576 buf->allocator = &simple_allocator; 577 else if (allowblock) 578 buf->allocator = &remap_allocator; 579 else 580 buf->allocator = &pool_allocator; 581 582 addr = buf->allocator->alloc(&args, &page); 583 584 if (page) { 585 unsigned long flags; 586 587 *handle = phys_to_dma(dev, page_to_phys(page)); 588 buf->virt = args.want_vaddr ? addr : page; 589 590 spin_lock_irqsave(&arm_dma_bufs_lock, flags); 591 list_add(&buf->list, &arm_dma_bufs); 592 spin_unlock_irqrestore(&arm_dma_bufs_lock, flags); 593 } else { 594 kfree(buf); 595 } 596 597 return args.want_vaddr ? addr : page; 598 } 599 600 /* 601 * Free a buffer as defined by the above mapping. 602 */ 603 static void __arm_dma_free(struct device *dev, size_t size, void *cpu_addr, 604 dma_addr_t handle, unsigned long attrs, 605 bool is_coherent) 606 { 607 struct page *page = phys_to_page(dma_to_phys(dev, handle)); 608 struct arm_dma_buffer *buf; 609 struct arm_dma_free_args args = { 610 .dev = dev, 611 .size = PAGE_ALIGN(size), 612 .cpu_addr = cpu_addr, 613 .page = page, 614 .want_vaddr = ((attrs & DMA_ATTR_NO_KERNEL_MAPPING) == 0), 615 }; 616 617 buf = arm_dma_buffer_find(cpu_addr); 618 if (WARN(!buf, "Freeing invalid buffer %p\n", cpu_addr)) 619 return; 620 621 buf->allocator->free(&args); 622 kfree(buf); 623 } 624 625 static void dma_cache_maint_page(struct page *page, unsigned long offset, 626 size_t size, enum dma_data_direction dir, 627 void (*op)(const void *, size_t, int)) 628 { 629 unsigned long pfn; 630 size_t left = size; 631 632 pfn = page_to_pfn(page) + offset / PAGE_SIZE; 633 offset %= PAGE_SIZE; 634 635 /* 636 * A single sg entry may refer to multiple physically contiguous 637 * pages. But we still need to process highmem pages individually. 638 * If highmem is not configured then the bulk of this loop gets 639 * optimized out. 640 */ 641 do { 642 size_t len = left; 643 void *vaddr; 644 645 page = pfn_to_page(pfn); 646 647 if (PageHighMem(page)) { 648 if (len + offset > PAGE_SIZE) 649 len = PAGE_SIZE - offset; 650 651 if (cache_is_vipt_nonaliasing()) { 652 vaddr = kmap_atomic(page); 653 op(vaddr + offset, len, dir); 654 kunmap_atomic(vaddr); 655 } else { 656 vaddr = kmap_high_get(page); 657 if (vaddr) { 658 op(vaddr + offset, len, dir); 659 kunmap_high(page); 660 } 661 } 662 } else { 663 vaddr = page_address(page) + offset; 664 op(vaddr, len, dir); 665 } 666 offset = 0; 667 pfn++; 668 left -= len; 669 } while (left); 670 } 671 672 /* 673 * Make an area consistent for devices. 674 * Note: Drivers should NOT use this function directly. 675 * Use the driver DMA support - see dma-mapping.h (dma_sync_*) 676 */ 677 static void __dma_page_cpu_to_dev(struct page *page, unsigned long off, 678 size_t size, enum dma_data_direction dir) 679 { 680 phys_addr_t paddr; 681 682 dma_cache_maint_page(page, off, size, dir, dmac_map_area); 683 684 paddr = page_to_phys(page) + off; 685 if (dir == DMA_FROM_DEVICE) { 686 outer_inv_range(paddr, paddr + size); 687 } else { 688 outer_clean_range(paddr, paddr + size); 689 } 690 /* FIXME: non-speculating: flush on bidirectional mappings? */ 691 } 692 693 static void __dma_page_dev_to_cpu(struct page *page, unsigned long off, 694 size_t size, enum dma_data_direction dir) 695 { 696 phys_addr_t paddr = page_to_phys(page) + off; 697 698 /* FIXME: non-speculating: not required */ 699 /* in any case, don't bother invalidating if DMA to device */ 700 if (dir != DMA_TO_DEVICE) { 701 outer_inv_range(paddr, paddr + size); 702 703 dma_cache_maint_page(page, off, size, dir, dmac_unmap_area); 704 } 705 706 /* 707 * Mark the D-cache clean for these pages to avoid extra flushing. 708 */ 709 if (dir != DMA_TO_DEVICE && size >= PAGE_SIZE) { 710 unsigned long pfn; 711 size_t left = size; 712 713 pfn = page_to_pfn(page) + off / PAGE_SIZE; 714 off %= PAGE_SIZE; 715 if (off) { 716 pfn++; 717 left -= PAGE_SIZE - off; 718 } 719 while (left >= PAGE_SIZE) { 720 page = pfn_to_page(pfn++); 721 set_bit(PG_dcache_clean, &page->flags); 722 left -= PAGE_SIZE; 723 } 724 } 725 } 726 727 #ifdef CONFIG_ARM_DMA_USE_IOMMU 728 729 static int __dma_info_to_prot(enum dma_data_direction dir, unsigned long attrs) 730 { 731 int prot = 0; 732 733 if (attrs & DMA_ATTR_PRIVILEGED) 734 prot |= IOMMU_PRIV; 735 736 switch (dir) { 737 case DMA_BIDIRECTIONAL: 738 return prot | IOMMU_READ | IOMMU_WRITE; 739 case DMA_TO_DEVICE: 740 return prot | IOMMU_READ; 741 case DMA_FROM_DEVICE: 742 return prot | IOMMU_WRITE; 743 default: 744 return prot; 745 } 746 } 747 748 /* IOMMU */ 749 750 static int extend_iommu_mapping(struct dma_iommu_mapping *mapping); 751 752 static inline dma_addr_t __alloc_iova(struct dma_iommu_mapping *mapping, 753 size_t size) 754 { 755 unsigned int order = get_order(size); 756 unsigned int align = 0; 757 unsigned int count, start; 758 size_t mapping_size = mapping->bits << PAGE_SHIFT; 759 unsigned long flags; 760 dma_addr_t iova; 761 int i; 762 763 if (order > CONFIG_ARM_DMA_IOMMU_ALIGNMENT) 764 order = CONFIG_ARM_DMA_IOMMU_ALIGNMENT; 765 766 count = PAGE_ALIGN(size) >> PAGE_SHIFT; 767 align = (1 << order) - 1; 768 769 spin_lock_irqsave(&mapping->lock, flags); 770 for (i = 0; i < mapping->nr_bitmaps; i++) { 771 start = bitmap_find_next_zero_area(mapping->bitmaps[i], 772 mapping->bits, 0, count, align); 773 774 if (start > mapping->bits) 775 continue; 776 777 bitmap_set(mapping->bitmaps[i], start, count); 778 break; 779 } 780 781 /* 782 * No unused range found. Try to extend the existing mapping 783 * and perform a second attempt to reserve an IO virtual 784 * address range of size bytes. 785 */ 786 if (i == mapping->nr_bitmaps) { 787 if (extend_iommu_mapping(mapping)) { 788 spin_unlock_irqrestore(&mapping->lock, flags); 789 return DMA_MAPPING_ERROR; 790 } 791 792 start = bitmap_find_next_zero_area(mapping->bitmaps[i], 793 mapping->bits, 0, count, align); 794 795 if (start > mapping->bits) { 796 spin_unlock_irqrestore(&mapping->lock, flags); 797 return DMA_MAPPING_ERROR; 798 } 799 800 bitmap_set(mapping->bitmaps[i], start, count); 801 } 802 spin_unlock_irqrestore(&mapping->lock, flags); 803 804 iova = mapping->base + (mapping_size * i); 805 iova += start << PAGE_SHIFT; 806 807 return iova; 808 } 809 810 static inline void __free_iova(struct dma_iommu_mapping *mapping, 811 dma_addr_t addr, size_t size) 812 { 813 unsigned int start, count; 814 size_t mapping_size = mapping->bits << PAGE_SHIFT; 815 unsigned long flags; 816 dma_addr_t bitmap_base; 817 u32 bitmap_index; 818 819 if (!size) 820 return; 821 822 bitmap_index = (u32) (addr - mapping->base) / (u32) mapping_size; 823 BUG_ON(addr < mapping->base || bitmap_index > mapping->extensions); 824 825 bitmap_base = mapping->base + mapping_size * bitmap_index; 826 827 start = (addr - bitmap_base) >> PAGE_SHIFT; 828 829 if (addr + size > bitmap_base + mapping_size) { 830 /* 831 * The address range to be freed reaches into the iova 832 * range of the next bitmap. This should not happen as 833 * we don't allow this in __alloc_iova (at the 834 * moment). 835 */ 836 BUG(); 837 } else 838 count = size >> PAGE_SHIFT; 839 840 spin_lock_irqsave(&mapping->lock, flags); 841 bitmap_clear(mapping->bitmaps[bitmap_index], start, count); 842 spin_unlock_irqrestore(&mapping->lock, flags); 843 } 844 845 /* We'll try 2M, 1M, 64K, and finally 4K; array must end with 0! */ 846 static const int iommu_order_array[] = { 9, 8, 4, 0 }; 847 848 static struct page **__iommu_alloc_buffer(struct device *dev, size_t size, 849 gfp_t gfp, unsigned long attrs, 850 int coherent_flag) 851 { 852 struct page **pages; 853 int count = size >> PAGE_SHIFT; 854 int array_size = count * sizeof(struct page *); 855 int i = 0; 856 int order_idx = 0; 857 858 if (array_size <= PAGE_SIZE) 859 pages = kzalloc(array_size, GFP_KERNEL); 860 else 861 pages = vzalloc(array_size); 862 if (!pages) 863 return NULL; 864 865 if (attrs & DMA_ATTR_FORCE_CONTIGUOUS) 866 { 867 unsigned long order = get_order(size); 868 struct page *page; 869 870 page = dma_alloc_from_contiguous(dev, count, order, 871 gfp & __GFP_NOWARN); 872 if (!page) 873 goto error; 874 875 __dma_clear_buffer(page, size, coherent_flag); 876 877 for (i = 0; i < count; i++) 878 pages[i] = page + i; 879 880 return pages; 881 } 882 883 /* Go straight to 4K chunks if caller says it's OK. */ 884 if (attrs & DMA_ATTR_ALLOC_SINGLE_PAGES) 885 order_idx = ARRAY_SIZE(iommu_order_array) - 1; 886 887 /* 888 * IOMMU can map any pages, so himem can also be used here 889 */ 890 gfp |= __GFP_NOWARN | __GFP_HIGHMEM; 891 892 while (count) { 893 int j, order; 894 895 order = iommu_order_array[order_idx]; 896 897 /* Drop down when we get small */ 898 if (__fls(count) < order) { 899 order_idx++; 900 continue; 901 } 902 903 if (order) { 904 /* See if it's easy to allocate a high-order chunk */ 905 pages[i] = alloc_pages(gfp | __GFP_NORETRY, order); 906 907 /* Go down a notch at first sign of pressure */ 908 if (!pages[i]) { 909 order_idx++; 910 continue; 911 } 912 } else { 913 pages[i] = alloc_pages(gfp, 0); 914 if (!pages[i]) 915 goto error; 916 } 917 918 if (order) { 919 split_page(pages[i], order); 920 j = 1 << order; 921 while (--j) 922 pages[i + j] = pages[i] + j; 923 } 924 925 __dma_clear_buffer(pages[i], PAGE_SIZE << order, coherent_flag); 926 i += 1 << order; 927 count -= 1 << order; 928 } 929 930 return pages; 931 error: 932 while (i--) 933 if (pages[i]) 934 __free_pages(pages[i], 0); 935 kvfree(pages); 936 return NULL; 937 } 938 939 static int __iommu_free_buffer(struct device *dev, struct page **pages, 940 size_t size, unsigned long attrs) 941 { 942 int count = size >> PAGE_SHIFT; 943 int i; 944 945 if (attrs & DMA_ATTR_FORCE_CONTIGUOUS) { 946 dma_release_from_contiguous(dev, pages[0], count); 947 } else { 948 for (i = 0; i < count; i++) 949 if (pages[i]) 950 __free_pages(pages[i], 0); 951 } 952 953 kvfree(pages); 954 return 0; 955 } 956 957 /* 958 * Create a mapping in device IO address space for specified pages 959 */ 960 static dma_addr_t 961 __iommu_create_mapping(struct device *dev, struct page **pages, size_t size, 962 unsigned long attrs) 963 { 964 struct dma_iommu_mapping *mapping = to_dma_iommu_mapping(dev); 965 unsigned int count = PAGE_ALIGN(size) >> PAGE_SHIFT; 966 dma_addr_t dma_addr, iova; 967 int i; 968 969 dma_addr = __alloc_iova(mapping, size); 970 if (dma_addr == DMA_MAPPING_ERROR) 971 return dma_addr; 972 973 iova = dma_addr; 974 for (i = 0; i < count; ) { 975 int ret; 976 977 unsigned int next_pfn = page_to_pfn(pages[i]) + 1; 978 phys_addr_t phys = page_to_phys(pages[i]); 979 unsigned int len, j; 980 981 for (j = i + 1; j < count; j++, next_pfn++) 982 if (page_to_pfn(pages[j]) != next_pfn) 983 break; 984 985 len = (j - i) << PAGE_SHIFT; 986 ret = iommu_map(mapping->domain, iova, phys, len, 987 __dma_info_to_prot(DMA_BIDIRECTIONAL, attrs)); 988 if (ret < 0) 989 goto fail; 990 iova += len; 991 i = j; 992 } 993 return dma_addr; 994 fail: 995 iommu_unmap(mapping->domain, dma_addr, iova-dma_addr); 996 __free_iova(mapping, dma_addr, size); 997 return DMA_MAPPING_ERROR; 998 } 999 1000 static int __iommu_remove_mapping(struct device *dev, dma_addr_t iova, size_t size) 1001 { 1002 struct dma_iommu_mapping *mapping = to_dma_iommu_mapping(dev); 1003 1004 /* 1005 * add optional in-page offset from iova to size and align 1006 * result to page size 1007 */ 1008 size = PAGE_ALIGN((iova & ~PAGE_MASK) + size); 1009 iova &= PAGE_MASK; 1010 1011 iommu_unmap(mapping->domain, iova, size); 1012 __free_iova(mapping, iova, size); 1013 return 0; 1014 } 1015 1016 static struct page **__atomic_get_pages(void *addr) 1017 { 1018 struct page *page; 1019 phys_addr_t phys; 1020 1021 phys = gen_pool_virt_to_phys(atomic_pool, (unsigned long)addr); 1022 page = phys_to_page(phys); 1023 1024 return (struct page **)page; 1025 } 1026 1027 static struct page **__iommu_get_pages(void *cpu_addr, unsigned long attrs) 1028 { 1029 if (__in_atomic_pool(cpu_addr, PAGE_SIZE)) 1030 return __atomic_get_pages(cpu_addr); 1031 1032 if (attrs & DMA_ATTR_NO_KERNEL_MAPPING) 1033 return cpu_addr; 1034 1035 return dma_common_find_pages(cpu_addr); 1036 } 1037 1038 static void *__iommu_alloc_simple(struct device *dev, size_t size, gfp_t gfp, 1039 dma_addr_t *handle, int coherent_flag, 1040 unsigned long attrs) 1041 { 1042 struct page *page; 1043 void *addr; 1044 1045 if (coherent_flag == COHERENT) 1046 addr = __alloc_simple_buffer(dev, size, gfp, &page); 1047 else 1048 addr = __alloc_from_pool(size, &page); 1049 if (!addr) 1050 return NULL; 1051 1052 *handle = __iommu_create_mapping(dev, &page, size, attrs); 1053 if (*handle == DMA_MAPPING_ERROR) 1054 goto err_mapping; 1055 1056 return addr; 1057 1058 err_mapping: 1059 __free_from_pool(addr, size); 1060 return NULL; 1061 } 1062 1063 static void __iommu_free_atomic(struct device *dev, void *cpu_addr, 1064 dma_addr_t handle, size_t size, int coherent_flag) 1065 { 1066 __iommu_remove_mapping(dev, handle, size); 1067 if (coherent_flag == COHERENT) 1068 __dma_free_buffer(virt_to_page(cpu_addr), size); 1069 else 1070 __free_from_pool(cpu_addr, size); 1071 } 1072 1073 static void *arm_iommu_alloc_attrs(struct device *dev, size_t size, 1074 dma_addr_t *handle, gfp_t gfp, unsigned long attrs) 1075 { 1076 pgprot_t prot = __get_dma_pgprot(attrs, PAGE_KERNEL); 1077 struct page **pages; 1078 void *addr = NULL; 1079 int coherent_flag = dev->dma_coherent ? COHERENT : NORMAL; 1080 1081 *handle = DMA_MAPPING_ERROR; 1082 size = PAGE_ALIGN(size); 1083 1084 if (coherent_flag == COHERENT || !gfpflags_allow_blocking(gfp)) 1085 return __iommu_alloc_simple(dev, size, gfp, handle, 1086 coherent_flag, attrs); 1087 1088 pages = __iommu_alloc_buffer(dev, size, gfp, attrs, coherent_flag); 1089 if (!pages) 1090 return NULL; 1091 1092 *handle = __iommu_create_mapping(dev, pages, size, attrs); 1093 if (*handle == DMA_MAPPING_ERROR) 1094 goto err_buffer; 1095 1096 if (attrs & DMA_ATTR_NO_KERNEL_MAPPING) 1097 return pages; 1098 1099 addr = dma_common_pages_remap(pages, size, prot, 1100 __builtin_return_address(0)); 1101 if (!addr) 1102 goto err_mapping; 1103 1104 return addr; 1105 1106 err_mapping: 1107 __iommu_remove_mapping(dev, *handle, size); 1108 err_buffer: 1109 __iommu_free_buffer(dev, pages, size, attrs); 1110 return NULL; 1111 } 1112 1113 static int arm_iommu_mmap_attrs(struct device *dev, struct vm_area_struct *vma, 1114 void *cpu_addr, dma_addr_t dma_addr, size_t size, 1115 unsigned long attrs) 1116 { 1117 struct page **pages = __iommu_get_pages(cpu_addr, attrs); 1118 unsigned long nr_pages = PAGE_ALIGN(size) >> PAGE_SHIFT; 1119 int err; 1120 1121 if (!pages) 1122 return -ENXIO; 1123 1124 if (vma->vm_pgoff >= nr_pages) 1125 return -ENXIO; 1126 1127 if (!dev->dma_coherent) 1128 vma->vm_page_prot = __get_dma_pgprot(attrs, vma->vm_page_prot); 1129 1130 err = vm_map_pages(vma, pages, nr_pages); 1131 if (err) 1132 pr_err("Remapping memory failed: %d\n", err); 1133 1134 return err; 1135 } 1136 1137 /* 1138 * free a page as defined by the above mapping. 1139 * Must not be called with IRQs disabled. 1140 */ 1141 static void arm_iommu_free_attrs(struct device *dev, size_t size, void *cpu_addr, 1142 dma_addr_t handle, unsigned long attrs) 1143 { 1144 int coherent_flag = dev->dma_coherent ? COHERENT : NORMAL; 1145 struct page **pages; 1146 size = PAGE_ALIGN(size); 1147 1148 if (coherent_flag == COHERENT || __in_atomic_pool(cpu_addr, size)) { 1149 __iommu_free_atomic(dev, cpu_addr, handle, size, coherent_flag); 1150 return; 1151 } 1152 1153 pages = __iommu_get_pages(cpu_addr, attrs); 1154 if (!pages) { 1155 WARN(1, "trying to free invalid coherent area: %p\n", cpu_addr); 1156 return; 1157 } 1158 1159 if ((attrs & DMA_ATTR_NO_KERNEL_MAPPING) == 0) 1160 dma_common_free_remap(cpu_addr, size); 1161 1162 __iommu_remove_mapping(dev, handle, size); 1163 __iommu_free_buffer(dev, pages, size, attrs); 1164 } 1165 1166 static int arm_iommu_get_sgtable(struct device *dev, struct sg_table *sgt, 1167 void *cpu_addr, dma_addr_t dma_addr, 1168 size_t size, unsigned long attrs) 1169 { 1170 unsigned int count = PAGE_ALIGN(size) >> PAGE_SHIFT; 1171 struct page **pages = __iommu_get_pages(cpu_addr, attrs); 1172 1173 if (!pages) 1174 return -ENXIO; 1175 1176 return sg_alloc_table_from_pages(sgt, pages, count, 0, size, 1177 GFP_KERNEL); 1178 } 1179 1180 /* 1181 * Map a part of the scatter-gather list into contiguous io address space 1182 */ 1183 static int __map_sg_chunk(struct device *dev, struct scatterlist *sg, 1184 size_t size, dma_addr_t *handle, 1185 enum dma_data_direction dir, unsigned long attrs) 1186 { 1187 struct dma_iommu_mapping *mapping = to_dma_iommu_mapping(dev); 1188 dma_addr_t iova, iova_base; 1189 int ret = 0; 1190 unsigned int count; 1191 struct scatterlist *s; 1192 int prot; 1193 1194 size = PAGE_ALIGN(size); 1195 *handle = DMA_MAPPING_ERROR; 1196 1197 iova_base = iova = __alloc_iova(mapping, size); 1198 if (iova == DMA_MAPPING_ERROR) 1199 return -ENOMEM; 1200 1201 for (count = 0, s = sg; count < (size >> PAGE_SHIFT); s = sg_next(s)) { 1202 phys_addr_t phys = page_to_phys(sg_page(s)); 1203 unsigned int len = PAGE_ALIGN(s->offset + s->length); 1204 1205 if (!dev->dma_coherent && !(attrs & DMA_ATTR_SKIP_CPU_SYNC)) 1206 __dma_page_cpu_to_dev(sg_page(s), s->offset, s->length, dir); 1207 1208 prot = __dma_info_to_prot(dir, attrs); 1209 1210 ret = iommu_map(mapping->domain, iova, phys, len, prot); 1211 if (ret < 0) 1212 goto fail; 1213 count += len >> PAGE_SHIFT; 1214 iova += len; 1215 } 1216 *handle = iova_base; 1217 1218 return 0; 1219 fail: 1220 iommu_unmap(mapping->domain, iova_base, count * PAGE_SIZE); 1221 __free_iova(mapping, iova_base, size); 1222 return ret; 1223 } 1224 1225 /** 1226 * arm_iommu_map_sg - map a set of SG buffers for streaming mode DMA 1227 * @dev: valid struct device pointer 1228 * @sg: list of buffers 1229 * @nents: number of buffers to map 1230 * @dir: DMA transfer direction 1231 * 1232 * Map a set of buffers described by scatterlist in streaming mode for DMA. 1233 * The scatter gather list elements are merged together (if possible) and 1234 * tagged with the appropriate dma address and length. They are obtained via 1235 * sg_dma_{address,length}. 1236 */ 1237 static int arm_iommu_map_sg(struct device *dev, struct scatterlist *sg, 1238 int nents, enum dma_data_direction dir, unsigned long attrs) 1239 { 1240 struct scatterlist *s = sg, *dma = sg, *start = sg; 1241 int i, count = 0, ret; 1242 unsigned int offset = s->offset; 1243 unsigned int size = s->offset + s->length; 1244 unsigned int max = dma_get_max_seg_size(dev); 1245 1246 for (i = 1; i < nents; i++) { 1247 s = sg_next(s); 1248 1249 s->dma_length = 0; 1250 1251 if (s->offset || (size & ~PAGE_MASK) || size + s->length > max) { 1252 ret = __map_sg_chunk(dev, start, size, 1253 &dma->dma_address, dir, attrs); 1254 if (ret < 0) 1255 goto bad_mapping; 1256 1257 dma->dma_address += offset; 1258 dma->dma_length = size - offset; 1259 1260 size = offset = s->offset; 1261 start = s; 1262 dma = sg_next(dma); 1263 count += 1; 1264 } 1265 size += s->length; 1266 } 1267 ret = __map_sg_chunk(dev, start, size, &dma->dma_address, dir, attrs); 1268 if (ret < 0) 1269 goto bad_mapping; 1270 1271 dma->dma_address += offset; 1272 dma->dma_length = size - offset; 1273 1274 return count+1; 1275 1276 bad_mapping: 1277 for_each_sg(sg, s, count, i) 1278 __iommu_remove_mapping(dev, sg_dma_address(s), sg_dma_len(s)); 1279 if (ret == -ENOMEM) 1280 return ret; 1281 return -EINVAL; 1282 } 1283 1284 /** 1285 * arm_iommu_unmap_sg - unmap a set of SG buffers mapped by dma_map_sg 1286 * @dev: valid struct device pointer 1287 * @sg: list of buffers 1288 * @nents: number of buffers to unmap (same as was passed to dma_map_sg) 1289 * @dir: DMA transfer direction (same as was passed to dma_map_sg) 1290 * 1291 * Unmap a set of streaming mode DMA translations. Again, CPU access 1292 * rules concerning calls here are the same as for dma_unmap_single(). 1293 */ 1294 static void arm_iommu_unmap_sg(struct device *dev, 1295 struct scatterlist *sg, int nents, 1296 enum dma_data_direction dir, 1297 unsigned long attrs) 1298 { 1299 struct scatterlist *s; 1300 int i; 1301 1302 for_each_sg(sg, s, nents, i) { 1303 if (sg_dma_len(s)) 1304 __iommu_remove_mapping(dev, sg_dma_address(s), 1305 sg_dma_len(s)); 1306 if (!dev->dma_coherent && !(attrs & DMA_ATTR_SKIP_CPU_SYNC)) 1307 __dma_page_dev_to_cpu(sg_page(s), s->offset, 1308 s->length, dir); 1309 } 1310 } 1311 1312 /** 1313 * arm_iommu_sync_sg_for_cpu 1314 * @dev: valid struct device pointer 1315 * @sg: list of buffers 1316 * @nents: number of buffers to map (returned from dma_map_sg) 1317 * @dir: DMA transfer direction (same as was passed to dma_map_sg) 1318 */ 1319 static void arm_iommu_sync_sg_for_cpu(struct device *dev, 1320 struct scatterlist *sg, 1321 int nents, enum dma_data_direction dir) 1322 { 1323 struct scatterlist *s; 1324 int i; 1325 1326 if (dev->dma_coherent) 1327 return; 1328 1329 for_each_sg(sg, s, nents, i) 1330 __dma_page_dev_to_cpu(sg_page(s), s->offset, s->length, dir); 1331 1332 } 1333 1334 /** 1335 * arm_iommu_sync_sg_for_device 1336 * @dev: valid struct device pointer 1337 * @sg: list of buffers 1338 * @nents: number of buffers to map (returned from dma_map_sg) 1339 * @dir: DMA transfer direction (same as was passed to dma_map_sg) 1340 */ 1341 static void arm_iommu_sync_sg_for_device(struct device *dev, 1342 struct scatterlist *sg, 1343 int nents, enum dma_data_direction dir) 1344 { 1345 struct scatterlist *s; 1346 int i; 1347 1348 if (dev->dma_coherent) 1349 return; 1350 1351 for_each_sg(sg, s, nents, i) 1352 __dma_page_cpu_to_dev(sg_page(s), s->offset, s->length, dir); 1353 } 1354 1355 /** 1356 * arm_iommu_map_page 1357 * @dev: valid struct device pointer 1358 * @page: page that buffer resides in 1359 * @offset: offset into page for start of buffer 1360 * @size: size of buffer to map 1361 * @dir: DMA transfer direction 1362 * 1363 * IOMMU aware version of arm_dma_map_page() 1364 */ 1365 static dma_addr_t arm_iommu_map_page(struct device *dev, struct page *page, 1366 unsigned long offset, size_t size, enum dma_data_direction dir, 1367 unsigned long attrs) 1368 { 1369 struct dma_iommu_mapping *mapping = to_dma_iommu_mapping(dev); 1370 dma_addr_t dma_addr; 1371 int ret, prot, len = PAGE_ALIGN(size + offset); 1372 1373 if (!dev->dma_coherent && !(attrs & DMA_ATTR_SKIP_CPU_SYNC)) 1374 __dma_page_cpu_to_dev(page, offset, size, dir); 1375 1376 dma_addr = __alloc_iova(mapping, len); 1377 if (dma_addr == DMA_MAPPING_ERROR) 1378 return dma_addr; 1379 1380 prot = __dma_info_to_prot(dir, attrs); 1381 1382 ret = iommu_map(mapping->domain, dma_addr, page_to_phys(page), len, prot); 1383 if (ret < 0) 1384 goto fail; 1385 1386 return dma_addr + offset; 1387 fail: 1388 __free_iova(mapping, dma_addr, len); 1389 return DMA_MAPPING_ERROR; 1390 } 1391 1392 /** 1393 * arm_iommu_unmap_page 1394 * @dev: valid struct device pointer 1395 * @handle: DMA address of buffer 1396 * @size: size of buffer (same as passed to dma_map_page) 1397 * @dir: DMA transfer direction (same as passed to dma_map_page) 1398 * 1399 * IOMMU aware version of arm_dma_unmap_page() 1400 */ 1401 static void arm_iommu_unmap_page(struct device *dev, dma_addr_t handle, 1402 size_t size, enum dma_data_direction dir, unsigned long attrs) 1403 { 1404 struct dma_iommu_mapping *mapping = to_dma_iommu_mapping(dev); 1405 dma_addr_t iova = handle & PAGE_MASK; 1406 struct page *page; 1407 int offset = handle & ~PAGE_MASK; 1408 int len = PAGE_ALIGN(size + offset); 1409 1410 if (!iova) 1411 return; 1412 1413 if (!dev->dma_coherent && !(attrs & DMA_ATTR_SKIP_CPU_SYNC)) { 1414 page = phys_to_page(iommu_iova_to_phys(mapping->domain, iova)); 1415 __dma_page_dev_to_cpu(page, offset, size, dir); 1416 } 1417 1418 iommu_unmap(mapping->domain, iova, len); 1419 __free_iova(mapping, iova, len); 1420 } 1421 1422 /** 1423 * arm_iommu_map_resource - map a device resource for DMA 1424 * @dev: valid struct device pointer 1425 * @phys_addr: physical address of resource 1426 * @size: size of resource to map 1427 * @dir: DMA transfer direction 1428 */ 1429 static dma_addr_t arm_iommu_map_resource(struct device *dev, 1430 phys_addr_t phys_addr, size_t size, 1431 enum dma_data_direction dir, unsigned long attrs) 1432 { 1433 struct dma_iommu_mapping *mapping = to_dma_iommu_mapping(dev); 1434 dma_addr_t dma_addr; 1435 int ret, prot; 1436 phys_addr_t addr = phys_addr & PAGE_MASK; 1437 unsigned int offset = phys_addr & ~PAGE_MASK; 1438 size_t len = PAGE_ALIGN(size + offset); 1439 1440 dma_addr = __alloc_iova(mapping, len); 1441 if (dma_addr == DMA_MAPPING_ERROR) 1442 return dma_addr; 1443 1444 prot = __dma_info_to_prot(dir, attrs) | IOMMU_MMIO; 1445 1446 ret = iommu_map(mapping->domain, dma_addr, addr, len, prot); 1447 if (ret < 0) 1448 goto fail; 1449 1450 return dma_addr + offset; 1451 fail: 1452 __free_iova(mapping, dma_addr, len); 1453 return DMA_MAPPING_ERROR; 1454 } 1455 1456 /** 1457 * arm_iommu_unmap_resource - unmap a device DMA resource 1458 * @dev: valid struct device pointer 1459 * @dma_handle: DMA address to resource 1460 * @size: size of resource to map 1461 * @dir: DMA transfer direction 1462 */ 1463 static void arm_iommu_unmap_resource(struct device *dev, dma_addr_t dma_handle, 1464 size_t size, enum dma_data_direction dir, 1465 unsigned long attrs) 1466 { 1467 struct dma_iommu_mapping *mapping = to_dma_iommu_mapping(dev); 1468 dma_addr_t iova = dma_handle & PAGE_MASK; 1469 unsigned int offset = dma_handle & ~PAGE_MASK; 1470 size_t len = PAGE_ALIGN(size + offset); 1471 1472 if (!iova) 1473 return; 1474 1475 iommu_unmap(mapping->domain, iova, len); 1476 __free_iova(mapping, iova, len); 1477 } 1478 1479 static void arm_iommu_sync_single_for_cpu(struct device *dev, 1480 dma_addr_t handle, size_t size, enum dma_data_direction dir) 1481 { 1482 struct dma_iommu_mapping *mapping = to_dma_iommu_mapping(dev); 1483 dma_addr_t iova = handle & PAGE_MASK; 1484 struct page *page; 1485 unsigned int offset = handle & ~PAGE_MASK; 1486 1487 if (dev->dma_coherent || !iova) 1488 return; 1489 1490 page = phys_to_page(iommu_iova_to_phys(mapping->domain, iova)); 1491 __dma_page_dev_to_cpu(page, offset, size, dir); 1492 } 1493 1494 static void arm_iommu_sync_single_for_device(struct device *dev, 1495 dma_addr_t handle, size_t size, enum dma_data_direction dir) 1496 { 1497 struct dma_iommu_mapping *mapping = to_dma_iommu_mapping(dev); 1498 dma_addr_t iova = handle & PAGE_MASK; 1499 struct page *page; 1500 unsigned int offset = handle & ~PAGE_MASK; 1501 1502 if (dev->dma_coherent || !iova) 1503 return; 1504 1505 page = phys_to_page(iommu_iova_to_phys(mapping->domain, iova)); 1506 __dma_page_cpu_to_dev(page, offset, size, dir); 1507 } 1508 1509 static const struct dma_map_ops iommu_ops = { 1510 .alloc = arm_iommu_alloc_attrs, 1511 .free = arm_iommu_free_attrs, 1512 .mmap = arm_iommu_mmap_attrs, 1513 .get_sgtable = arm_iommu_get_sgtable, 1514 1515 .map_page = arm_iommu_map_page, 1516 .unmap_page = arm_iommu_unmap_page, 1517 .sync_single_for_cpu = arm_iommu_sync_single_for_cpu, 1518 .sync_single_for_device = arm_iommu_sync_single_for_device, 1519 1520 .map_sg = arm_iommu_map_sg, 1521 .unmap_sg = arm_iommu_unmap_sg, 1522 .sync_sg_for_cpu = arm_iommu_sync_sg_for_cpu, 1523 .sync_sg_for_device = arm_iommu_sync_sg_for_device, 1524 1525 .map_resource = arm_iommu_map_resource, 1526 .unmap_resource = arm_iommu_unmap_resource, 1527 }; 1528 1529 /** 1530 * arm_iommu_create_mapping 1531 * @bus: pointer to the bus holding the client device (for IOMMU calls) 1532 * @base: start address of the valid IO address space 1533 * @size: maximum size of the valid IO address space 1534 * 1535 * Creates a mapping structure which holds information about used/unused 1536 * IO address ranges, which is required to perform memory allocation and 1537 * mapping with IOMMU aware functions. 1538 * 1539 * The client device need to be attached to the mapping with 1540 * arm_iommu_attach_device function. 1541 */ 1542 struct dma_iommu_mapping * 1543 arm_iommu_create_mapping(struct bus_type *bus, dma_addr_t base, u64 size) 1544 { 1545 unsigned int bits = size >> PAGE_SHIFT; 1546 unsigned int bitmap_size = BITS_TO_LONGS(bits) * sizeof(long); 1547 struct dma_iommu_mapping *mapping; 1548 int extensions = 1; 1549 int err = -ENOMEM; 1550 1551 /* currently only 32-bit DMA address space is supported */ 1552 if (size > DMA_BIT_MASK(32) + 1) 1553 return ERR_PTR(-ERANGE); 1554 1555 if (!bitmap_size) 1556 return ERR_PTR(-EINVAL); 1557 1558 if (bitmap_size > PAGE_SIZE) { 1559 extensions = bitmap_size / PAGE_SIZE; 1560 bitmap_size = PAGE_SIZE; 1561 } 1562 1563 mapping = kzalloc(sizeof(struct dma_iommu_mapping), GFP_KERNEL); 1564 if (!mapping) 1565 goto err; 1566 1567 mapping->bitmap_size = bitmap_size; 1568 mapping->bitmaps = kcalloc(extensions, sizeof(unsigned long *), 1569 GFP_KERNEL); 1570 if (!mapping->bitmaps) 1571 goto err2; 1572 1573 mapping->bitmaps[0] = kzalloc(bitmap_size, GFP_KERNEL); 1574 if (!mapping->bitmaps[0]) 1575 goto err3; 1576 1577 mapping->nr_bitmaps = 1; 1578 mapping->extensions = extensions; 1579 mapping->base = base; 1580 mapping->bits = BITS_PER_BYTE * bitmap_size; 1581 1582 spin_lock_init(&mapping->lock); 1583 1584 mapping->domain = iommu_domain_alloc(bus); 1585 if (!mapping->domain) 1586 goto err4; 1587 1588 kref_init(&mapping->kref); 1589 return mapping; 1590 err4: 1591 kfree(mapping->bitmaps[0]); 1592 err3: 1593 kfree(mapping->bitmaps); 1594 err2: 1595 kfree(mapping); 1596 err: 1597 return ERR_PTR(err); 1598 } 1599 EXPORT_SYMBOL_GPL(arm_iommu_create_mapping); 1600 1601 static void release_iommu_mapping(struct kref *kref) 1602 { 1603 int i; 1604 struct dma_iommu_mapping *mapping = 1605 container_of(kref, struct dma_iommu_mapping, kref); 1606 1607 iommu_domain_free(mapping->domain); 1608 for (i = 0; i < mapping->nr_bitmaps; i++) 1609 kfree(mapping->bitmaps[i]); 1610 kfree(mapping->bitmaps); 1611 kfree(mapping); 1612 } 1613 1614 static int extend_iommu_mapping(struct dma_iommu_mapping *mapping) 1615 { 1616 int next_bitmap; 1617 1618 if (mapping->nr_bitmaps >= mapping->extensions) 1619 return -EINVAL; 1620 1621 next_bitmap = mapping->nr_bitmaps; 1622 mapping->bitmaps[next_bitmap] = kzalloc(mapping->bitmap_size, 1623 GFP_ATOMIC); 1624 if (!mapping->bitmaps[next_bitmap]) 1625 return -ENOMEM; 1626 1627 mapping->nr_bitmaps++; 1628 1629 return 0; 1630 } 1631 1632 void arm_iommu_release_mapping(struct dma_iommu_mapping *mapping) 1633 { 1634 if (mapping) 1635 kref_put(&mapping->kref, release_iommu_mapping); 1636 } 1637 EXPORT_SYMBOL_GPL(arm_iommu_release_mapping); 1638 1639 static int __arm_iommu_attach_device(struct device *dev, 1640 struct dma_iommu_mapping *mapping) 1641 { 1642 int err; 1643 1644 err = iommu_attach_device(mapping->domain, dev); 1645 if (err) 1646 return err; 1647 1648 kref_get(&mapping->kref); 1649 to_dma_iommu_mapping(dev) = mapping; 1650 1651 pr_debug("Attached IOMMU controller to %s device.\n", dev_name(dev)); 1652 return 0; 1653 } 1654 1655 /** 1656 * arm_iommu_attach_device 1657 * @dev: valid struct device pointer 1658 * @mapping: io address space mapping structure (returned from 1659 * arm_iommu_create_mapping) 1660 * 1661 * Attaches specified io address space mapping to the provided device. 1662 * This replaces the dma operations (dma_map_ops pointer) with the 1663 * IOMMU aware version. 1664 * 1665 * More than one client might be attached to the same io address space 1666 * mapping. 1667 */ 1668 int arm_iommu_attach_device(struct device *dev, 1669 struct dma_iommu_mapping *mapping) 1670 { 1671 int err; 1672 1673 err = __arm_iommu_attach_device(dev, mapping); 1674 if (err) 1675 return err; 1676 1677 set_dma_ops(dev, &iommu_ops); 1678 return 0; 1679 } 1680 EXPORT_SYMBOL_GPL(arm_iommu_attach_device); 1681 1682 /** 1683 * arm_iommu_detach_device 1684 * @dev: valid struct device pointer 1685 * 1686 * Detaches the provided device from a previously attached map. 1687 * This overwrites the dma_ops pointer with appropriate non-IOMMU ops. 1688 */ 1689 void arm_iommu_detach_device(struct device *dev) 1690 { 1691 struct dma_iommu_mapping *mapping; 1692 1693 mapping = to_dma_iommu_mapping(dev); 1694 if (!mapping) { 1695 dev_warn(dev, "Not attached\n"); 1696 return; 1697 } 1698 1699 iommu_detach_device(mapping->domain, dev); 1700 kref_put(&mapping->kref, release_iommu_mapping); 1701 to_dma_iommu_mapping(dev) = NULL; 1702 set_dma_ops(dev, NULL); 1703 1704 pr_debug("Detached IOMMU controller from %s device.\n", dev_name(dev)); 1705 } 1706 EXPORT_SYMBOL_GPL(arm_iommu_detach_device); 1707 1708 static void arm_setup_iommu_dma_ops(struct device *dev, u64 dma_base, u64 size, 1709 const struct iommu_ops *iommu, bool coherent) 1710 { 1711 struct dma_iommu_mapping *mapping; 1712 1713 mapping = arm_iommu_create_mapping(dev->bus, dma_base, size); 1714 if (IS_ERR(mapping)) { 1715 pr_warn("Failed to create %llu-byte IOMMU mapping for device %s\n", 1716 size, dev_name(dev)); 1717 return; 1718 } 1719 1720 if (__arm_iommu_attach_device(dev, mapping)) { 1721 pr_warn("Failed to attached device %s to IOMMU_mapping\n", 1722 dev_name(dev)); 1723 arm_iommu_release_mapping(mapping); 1724 return; 1725 } 1726 1727 set_dma_ops(dev, &iommu_ops); 1728 } 1729 1730 static void arm_teardown_iommu_dma_ops(struct device *dev) 1731 { 1732 struct dma_iommu_mapping *mapping = to_dma_iommu_mapping(dev); 1733 1734 if (!mapping) 1735 return; 1736 1737 arm_iommu_detach_device(dev); 1738 arm_iommu_release_mapping(mapping); 1739 } 1740 1741 #else 1742 1743 static void arm_setup_iommu_dma_ops(struct device *dev, u64 dma_base, u64 size, 1744 const struct iommu_ops *iommu, bool coherent) 1745 { 1746 } 1747 1748 static void arm_teardown_iommu_dma_ops(struct device *dev) { } 1749 1750 #endif /* CONFIG_ARM_DMA_USE_IOMMU */ 1751 1752 void arch_setup_dma_ops(struct device *dev, u64 dma_base, u64 size, 1753 const struct iommu_ops *iommu, bool coherent) 1754 { 1755 /* 1756 * Due to legacy code that sets the ->dma_coherent flag from a bus 1757 * notifier we can't just assign coherent to the ->dma_coherent flag 1758 * here, but instead have to make sure we only set but never clear it 1759 * for now. 1760 */ 1761 if (coherent) 1762 dev->dma_coherent = true; 1763 1764 /* 1765 * Don't override the dma_ops if they have already been set. Ideally 1766 * this should be the only location where dma_ops are set, remove this 1767 * check when all other callers of set_dma_ops will have disappeared. 1768 */ 1769 if (dev->dma_ops) 1770 return; 1771 1772 if (iommu) 1773 arm_setup_iommu_dma_ops(dev, dma_base, size, iommu, coherent); 1774 1775 xen_setup_dma_ops(dev); 1776 dev->archdata.dma_ops_setup = true; 1777 } 1778 1779 void arch_teardown_dma_ops(struct device *dev) 1780 { 1781 if (!dev->archdata.dma_ops_setup) 1782 return; 1783 1784 arm_teardown_iommu_dma_ops(dev); 1785 /* Let arch_setup_dma_ops() start again from scratch upon re-probe */ 1786 set_dma_ops(dev, NULL); 1787 } 1788 1789 void arch_sync_dma_for_device(phys_addr_t paddr, size_t size, 1790 enum dma_data_direction dir) 1791 { 1792 __dma_page_cpu_to_dev(phys_to_page(paddr), paddr & (PAGE_SIZE - 1), 1793 size, dir); 1794 } 1795 1796 void arch_sync_dma_for_cpu(phys_addr_t paddr, size_t size, 1797 enum dma_data_direction dir) 1798 { 1799 __dma_page_dev_to_cpu(phys_to_page(paddr), paddr & (PAGE_SIZE - 1), 1800 size, dir); 1801 } 1802 1803 void *arch_dma_alloc(struct device *dev, size_t size, dma_addr_t *dma_handle, 1804 gfp_t gfp, unsigned long attrs) 1805 { 1806 return __dma_alloc(dev, size, dma_handle, gfp, 1807 __get_dma_pgprot(attrs, PAGE_KERNEL), false, 1808 attrs, __builtin_return_address(0)); 1809 } 1810 1811 void arch_dma_free(struct device *dev, size_t size, void *cpu_addr, 1812 dma_addr_t dma_handle, unsigned long attrs) 1813 { 1814 __arm_dma_free(dev, size, cpu_addr, dma_handle, attrs, false); 1815 } 1816