1 /* 2 * linux/arch/arm/mm/dma-mapping.c 3 * 4 * Copyright (C) 2000-2004 Russell King 5 * 6 * This program is free software; you can redistribute it and/or modify 7 * it under the terms of the GNU General Public License version 2 as 8 * published by the Free Software Foundation. 9 * 10 * DMA uncached mapping support. 11 */ 12 #include <linux/bootmem.h> 13 #include <linux/module.h> 14 #include <linux/mm.h> 15 #include <linux/genalloc.h> 16 #include <linux/gfp.h> 17 #include <linux/errno.h> 18 #include <linux/list.h> 19 #include <linux/init.h> 20 #include <linux/device.h> 21 #include <linux/dma-mapping.h> 22 #include <linux/dma-contiguous.h> 23 #include <linux/highmem.h> 24 #include <linux/memblock.h> 25 #include <linux/slab.h> 26 #include <linux/iommu.h> 27 #include <linux/io.h> 28 #include <linux/vmalloc.h> 29 #include <linux/sizes.h> 30 #include <linux/cma.h> 31 32 #include <asm/memory.h> 33 #include <asm/highmem.h> 34 #include <asm/cacheflush.h> 35 #include <asm/tlbflush.h> 36 #include <asm/mach/arch.h> 37 #include <asm/dma-iommu.h> 38 #include <asm/mach/map.h> 39 #include <asm/system_info.h> 40 #include <asm/dma-contiguous.h> 41 42 #include "dma.h" 43 #include "mm.h" 44 45 struct arm_dma_alloc_args { 46 struct device *dev; 47 size_t size; 48 gfp_t gfp; 49 pgprot_t prot; 50 const void *caller; 51 bool want_vaddr; 52 int coherent_flag; 53 }; 54 55 struct arm_dma_free_args { 56 struct device *dev; 57 size_t size; 58 void *cpu_addr; 59 struct page *page; 60 bool want_vaddr; 61 }; 62 63 #define NORMAL 0 64 #define COHERENT 1 65 66 struct arm_dma_allocator { 67 void *(*alloc)(struct arm_dma_alloc_args *args, 68 struct page **ret_page); 69 void (*free)(struct arm_dma_free_args *args); 70 }; 71 72 struct arm_dma_buffer { 73 struct list_head list; 74 void *virt; 75 struct arm_dma_allocator *allocator; 76 }; 77 78 static LIST_HEAD(arm_dma_bufs); 79 static DEFINE_SPINLOCK(arm_dma_bufs_lock); 80 81 static struct arm_dma_buffer *arm_dma_buffer_find(void *virt) 82 { 83 struct arm_dma_buffer *buf, *found = NULL; 84 unsigned long flags; 85 86 spin_lock_irqsave(&arm_dma_bufs_lock, flags); 87 list_for_each_entry(buf, &arm_dma_bufs, list) { 88 if (buf->virt == virt) { 89 list_del(&buf->list); 90 found = buf; 91 break; 92 } 93 } 94 spin_unlock_irqrestore(&arm_dma_bufs_lock, flags); 95 return found; 96 } 97 98 /* 99 * The DMA API is built upon the notion of "buffer ownership". A buffer 100 * is either exclusively owned by the CPU (and therefore may be accessed 101 * by it) or exclusively owned by the DMA device. These helper functions 102 * represent the transitions between these two ownership states. 103 * 104 * Note, however, that on later ARMs, this notion does not work due to 105 * speculative prefetches. We model our approach on the assumption that 106 * the CPU does do speculative prefetches, which means we clean caches 107 * before transfers and delay cache invalidation until transfer completion. 108 * 109 */ 110 static void __dma_page_cpu_to_dev(struct page *, unsigned long, 111 size_t, enum dma_data_direction); 112 static void __dma_page_dev_to_cpu(struct page *, unsigned long, 113 size_t, enum dma_data_direction); 114 115 /** 116 * arm_dma_map_page - map a portion of a page for streaming DMA 117 * @dev: valid struct device pointer, or NULL for ISA and EISA-like devices 118 * @page: page that buffer resides in 119 * @offset: offset into page for start of buffer 120 * @size: size of buffer to map 121 * @dir: DMA transfer direction 122 * 123 * Ensure that any data held in the cache is appropriately discarded 124 * or written back. 125 * 126 * The device owns this memory once this call has completed. The CPU 127 * can regain ownership by calling dma_unmap_page(). 128 */ 129 static dma_addr_t arm_dma_map_page(struct device *dev, struct page *page, 130 unsigned long offset, size_t size, enum dma_data_direction dir, 131 unsigned long attrs) 132 { 133 if ((attrs & DMA_ATTR_SKIP_CPU_SYNC) == 0) 134 __dma_page_cpu_to_dev(page, offset, size, dir); 135 return pfn_to_dma(dev, page_to_pfn(page)) + offset; 136 } 137 138 static dma_addr_t arm_coherent_dma_map_page(struct device *dev, struct page *page, 139 unsigned long offset, size_t size, enum dma_data_direction dir, 140 unsigned long attrs) 141 { 142 return pfn_to_dma(dev, page_to_pfn(page)) + offset; 143 } 144 145 /** 146 * arm_dma_unmap_page - unmap a buffer previously mapped through dma_map_page() 147 * @dev: valid struct device pointer, or NULL for ISA and EISA-like devices 148 * @handle: DMA address of buffer 149 * @size: size of buffer (same as passed to dma_map_page) 150 * @dir: DMA transfer direction (same as passed to dma_map_page) 151 * 152 * Unmap a page streaming mode DMA translation. The handle and size 153 * must match what was provided in the previous dma_map_page() call. 154 * All other usages are undefined. 155 * 156 * After this call, reads by the CPU to the buffer are guaranteed to see 157 * whatever the device wrote there. 158 */ 159 static void arm_dma_unmap_page(struct device *dev, dma_addr_t handle, 160 size_t size, enum dma_data_direction dir, unsigned long attrs) 161 { 162 if ((attrs & DMA_ATTR_SKIP_CPU_SYNC) == 0) 163 __dma_page_dev_to_cpu(pfn_to_page(dma_to_pfn(dev, handle)), 164 handle & ~PAGE_MASK, size, dir); 165 } 166 167 static void arm_dma_sync_single_for_cpu(struct device *dev, 168 dma_addr_t handle, size_t size, enum dma_data_direction dir) 169 { 170 unsigned int offset = handle & (PAGE_SIZE - 1); 171 struct page *page = pfn_to_page(dma_to_pfn(dev, handle-offset)); 172 __dma_page_dev_to_cpu(page, offset, size, dir); 173 } 174 175 static void arm_dma_sync_single_for_device(struct device *dev, 176 dma_addr_t handle, size_t size, enum dma_data_direction dir) 177 { 178 unsigned int offset = handle & (PAGE_SIZE - 1); 179 struct page *page = pfn_to_page(dma_to_pfn(dev, handle-offset)); 180 __dma_page_cpu_to_dev(page, offset, size, dir); 181 } 182 183 struct dma_map_ops arm_dma_ops = { 184 .alloc = arm_dma_alloc, 185 .free = arm_dma_free, 186 .mmap = arm_dma_mmap, 187 .get_sgtable = arm_dma_get_sgtable, 188 .map_page = arm_dma_map_page, 189 .unmap_page = arm_dma_unmap_page, 190 .map_sg = arm_dma_map_sg, 191 .unmap_sg = arm_dma_unmap_sg, 192 .sync_single_for_cpu = arm_dma_sync_single_for_cpu, 193 .sync_single_for_device = arm_dma_sync_single_for_device, 194 .sync_sg_for_cpu = arm_dma_sync_sg_for_cpu, 195 .sync_sg_for_device = arm_dma_sync_sg_for_device, 196 }; 197 EXPORT_SYMBOL(arm_dma_ops); 198 199 static void *arm_coherent_dma_alloc(struct device *dev, size_t size, 200 dma_addr_t *handle, gfp_t gfp, unsigned long attrs); 201 static void arm_coherent_dma_free(struct device *dev, size_t size, void *cpu_addr, 202 dma_addr_t handle, unsigned long attrs); 203 static int arm_coherent_dma_mmap(struct device *dev, struct vm_area_struct *vma, 204 void *cpu_addr, dma_addr_t dma_addr, size_t size, 205 unsigned long attrs); 206 207 struct dma_map_ops arm_coherent_dma_ops = { 208 .alloc = arm_coherent_dma_alloc, 209 .free = arm_coherent_dma_free, 210 .mmap = arm_coherent_dma_mmap, 211 .get_sgtable = arm_dma_get_sgtable, 212 .map_page = arm_coherent_dma_map_page, 213 .map_sg = arm_dma_map_sg, 214 }; 215 EXPORT_SYMBOL(arm_coherent_dma_ops); 216 217 static int __dma_supported(struct device *dev, u64 mask, bool warn) 218 { 219 unsigned long max_dma_pfn; 220 221 /* 222 * If the mask allows for more memory than we can address, 223 * and we actually have that much memory, then we must 224 * indicate that DMA to this device is not supported. 225 */ 226 if (sizeof(mask) != sizeof(dma_addr_t) && 227 mask > (dma_addr_t)~0 && 228 dma_to_pfn(dev, ~0) < max_pfn - 1) { 229 if (warn) { 230 dev_warn(dev, "Coherent DMA mask %#llx is larger than dma_addr_t allows\n", 231 mask); 232 dev_warn(dev, "Driver did not use or check the return value from dma_set_coherent_mask()?\n"); 233 } 234 return 0; 235 } 236 237 max_dma_pfn = min(max_pfn, arm_dma_pfn_limit); 238 239 /* 240 * Translate the device's DMA mask to a PFN limit. This 241 * PFN number includes the page which we can DMA to. 242 */ 243 if (dma_to_pfn(dev, mask) < max_dma_pfn) { 244 if (warn) 245 dev_warn(dev, "Coherent DMA mask %#llx (pfn %#lx-%#lx) covers a smaller range of system memory than the DMA zone pfn 0x0-%#lx\n", 246 mask, 247 dma_to_pfn(dev, 0), dma_to_pfn(dev, mask) + 1, 248 max_dma_pfn + 1); 249 return 0; 250 } 251 252 return 1; 253 } 254 255 static u64 get_coherent_dma_mask(struct device *dev) 256 { 257 u64 mask = (u64)DMA_BIT_MASK(32); 258 259 if (dev) { 260 mask = dev->coherent_dma_mask; 261 262 /* 263 * Sanity check the DMA mask - it must be non-zero, and 264 * must be able to be satisfied by a DMA allocation. 265 */ 266 if (mask == 0) { 267 dev_warn(dev, "coherent DMA mask is unset\n"); 268 return 0; 269 } 270 271 if (!__dma_supported(dev, mask, true)) 272 return 0; 273 } 274 275 return mask; 276 } 277 278 static void __dma_clear_buffer(struct page *page, size_t size, int coherent_flag) 279 { 280 /* 281 * Ensure that the allocated pages are zeroed, and that any data 282 * lurking in the kernel direct-mapped region is invalidated. 283 */ 284 if (PageHighMem(page)) { 285 phys_addr_t base = __pfn_to_phys(page_to_pfn(page)); 286 phys_addr_t end = base + size; 287 while (size > 0) { 288 void *ptr = kmap_atomic(page); 289 memset(ptr, 0, PAGE_SIZE); 290 if (coherent_flag != COHERENT) 291 dmac_flush_range(ptr, ptr + PAGE_SIZE); 292 kunmap_atomic(ptr); 293 page++; 294 size -= PAGE_SIZE; 295 } 296 if (coherent_flag != COHERENT) 297 outer_flush_range(base, end); 298 } else { 299 void *ptr = page_address(page); 300 memset(ptr, 0, size); 301 if (coherent_flag != COHERENT) { 302 dmac_flush_range(ptr, ptr + size); 303 outer_flush_range(__pa(ptr), __pa(ptr) + size); 304 } 305 } 306 } 307 308 /* 309 * Allocate a DMA buffer for 'dev' of size 'size' using the 310 * specified gfp mask. Note that 'size' must be page aligned. 311 */ 312 static struct page *__dma_alloc_buffer(struct device *dev, size_t size, 313 gfp_t gfp, int coherent_flag) 314 { 315 unsigned long order = get_order(size); 316 struct page *page, *p, *e; 317 318 page = alloc_pages(gfp, order); 319 if (!page) 320 return NULL; 321 322 /* 323 * Now split the huge page and free the excess pages 324 */ 325 split_page(page, order); 326 for (p = page + (size >> PAGE_SHIFT), e = page + (1 << order); p < e; p++) 327 __free_page(p); 328 329 __dma_clear_buffer(page, size, coherent_flag); 330 331 return page; 332 } 333 334 /* 335 * Free a DMA buffer. 'size' must be page aligned. 336 */ 337 static void __dma_free_buffer(struct page *page, size_t size) 338 { 339 struct page *e = page + (size >> PAGE_SHIFT); 340 341 while (page < e) { 342 __free_page(page); 343 page++; 344 } 345 } 346 347 #ifdef CONFIG_MMU 348 349 static void *__alloc_from_contiguous(struct device *dev, size_t size, 350 pgprot_t prot, struct page **ret_page, 351 const void *caller, bool want_vaddr, 352 int coherent_flag); 353 354 static void *__alloc_remap_buffer(struct device *dev, size_t size, gfp_t gfp, 355 pgprot_t prot, struct page **ret_page, 356 const void *caller, bool want_vaddr); 357 358 static void * 359 __dma_alloc_remap(struct page *page, size_t size, gfp_t gfp, pgprot_t prot, 360 const void *caller) 361 { 362 /* 363 * DMA allocation can be mapped to user space, so lets 364 * set VM_USERMAP flags too. 365 */ 366 return dma_common_contiguous_remap(page, size, 367 VM_ARM_DMA_CONSISTENT | VM_USERMAP, 368 prot, caller); 369 } 370 371 static void __dma_free_remap(void *cpu_addr, size_t size) 372 { 373 dma_common_free_remap(cpu_addr, size, 374 VM_ARM_DMA_CONSISTENT | VM_USERMAP); 375 } 376 377 #define DEFAULT_DMA_COHERENT_POOL_SIZE SZ_256K 378 static struct gen_pool *atomic_pool; 379 380 static size_t atomic_pool_size = DEFAULT_DMA_COHERENT_POOL_SIZE; 381 382 static int __init early_coherent_pool(char *p) 383 { 384 atomic_pool_size = memparse(p, &p); 385 return 0; 386 } 387 early_param("coherent_pool", early_coherent_pool); 388 389 void __init init_dma_coherent_pool_size(unsigned long size) 390 { 391 /* 392 * Catch any attempt to set the pool size too late. 393 */ 394 BUG_ON(atomic_pool); 395 396 /* 397 * Set architecture specific coherent pool size only if 398 * it has not been changed by kernel command line parameter. 399 */ 400 if (atomic_pool_size == DEFAULT_DMA_COHERENT_POOL_SIZE) 401 atomic_pool_size = size; 402 } 403 404 /* 405 * Initialise the coherent pool for atomic allocations. 406 */ 407 static int __init atomic_pool_init(void) 408 { 409 pgprot_t prot = pgprot_dmacoherent(PAGE_KERNEL); 410 gfp_t gfp = GFP_KERNEL | GFP_DMA; 411 struct page *page; 412 void *ptr; 413 414 atomic_pool = gen_pool_create(PAGE_SHIFT, -1); 415 if (!atomic_pool) 416 goto out; 417 /* 418 * The atomic pool is only used for non-coherent allocations 419 * so we must pass NORMAL for coherent_flag. 420 */ 421 if (dev_get_cma_area(NULL)) 422 ptr = __alloc_from_contiguous(NULL, atomic_pool_size, prot, 423 &page, atomic_pool_init, true, NORMAL); 424 else 425 ptr = __alloc_remap_buffer(NULL, atomic_pool_size, gfp, prot, 426 &page, atomic_pool_init, true); 427 if (ptr) { 428 int ret; 429 430 ret = gen_pool_add_virt(atomic_pool, (unsigned long)ptr, 431 page_to_phys(page), 432 atomic_pool_size, -1); 433 if (ret) 434 goto destroy_genpool; 435 436 gen_pool_set_algo(atomic_pool, 437 gen_pool_first_fit_order_align, 438 (void *)PAGE_SHIFT); 439 pr_info("DMA: preallocated %zu KiB pool for atomic coherent allocations\n", 440 atomic_pool_size / 1024); 441 return 0; 442 } 443 444 destroy_genpool: 445 gen_pool_destroy(atomic_pool); 446 atomic_pool = NULL; 447 out: 448 pr_err("DMA: failed to allocate %zu KiB pool for atomic coherent allocation\n", 449 atomic_pool_size / 1024); 450 return -ENOMEM; 451 } 452 /* 453 * CMA is activated by core_initcall, so we must be called after it. 454 */ 455 postcore_initcall(atomic_pool_init); 456 457 struct dma_contig_early_reserve { 458 phys_addr_t base; 459 unsigned long size; 460 }; 461 462 static struct dma_contig_early_reserve dma_mmu_remap[MAX_CMA_AREAS] __initdata; 463 464 static int dma_mmu_remap_num __initdata; 465 466 void __init dma_contiguous_early_fixup(phys_addr_t base, unsigned long size) 467 { 468 dma_mmu_remap[dma_mmu_remap_num].base = base; 469 dma_mmu_remap[dma_mmu_remap_num].size = size; 470 dma_mmu_remap_num++; 471 } 472 473 void __init dma_contiguous_remap(void) 474 { 475 int i; 476 for (i = 0; i < dma_mmu_remap_num; i++) { 477 phys_addr_t start = dma_mmu_remap[i].base; 478 phys_addr_t end = start + dma_mmu_remap[i].size; 479 struct map_desc map; 480 unsigned long addr; 481 482 if (end > arm_lowmem_limit) 483 end = arm_lowmem_limit; 484 if (start >= end) 485 continue; 486 487 map.pfn = __phys_to_pfn(start); 488 map.virtual = __phys_to_virt(start); 489 map.length = end - start; 490 map.type = MT_MEMORY_DMA_READY; 491 492 /* 493 * Clear previous low-memory mapping to ensure that the 494 * TLB does not see any conflicting entries, then flush 495 * the TLB of the old entries before creating new mappings. 496 * 497 * This ensures that any speculatively loaded TLB entries 498 * (even though they may be rare) can not cause any problems, 499 * and ensures that this code is architecturally compliant. 500 */ 501 for (addr = __phys_to_virt(start); addr < __phys_to_virt(end); 502 addr += PMD_SIZE) 503 pmd_clear(pmd_off_k(addr)); 504 505 flush_tlb_kernel_range(__phys_to_virt(start), 506 __phys_to_virt(end)); 507 508 iotable_init(&map, 1); 509 } 510 } 511 512 static int __dma_update_pte(pte_t *pte, pgtable_t token, unsigned long addr, 513 void *data) 514 { 515 struct page *page = virt_to_page(addr); 516 pgprot_t prot = *(pgprot_t *)data; 517 518 set_pte_ext(pte, mk_pte(page, prot), 0); 519 return 0; 520 } 521 522 static void __dma_remap(struct page *page, size_t size, pgprot_t prot) 523 { 524 unsigned long start = (unsigned long) page_address(page); 525 unsigned end = start + size; 526 527 apply_to_page_range(&init_mm, start, size, __dma_update_pte, &prot); 528 flush_tlb_kernel_range(start, end); 529 } 530 531 static void *__alloc_remap_buffer(struct device *dev, size_t size, gfp_t gfp, 532 pgprot_t prot, struct page **ret_page, 533 const void *caller, bool want_vaddr) 534 { 535 struct page *page; 536 void *ptr = NULL; 537 /* 538 * __alloc_remap_buffer is only called when the device is 539 * non-coherent 540 */ 541 page = __dma_alloc_buffer(dev, size, gfp, NORMAL); 542 if (!page) 543 return NULL; 544 if (!want_vaddr) 545 goto out; 546 547 ptr = __dma_alloc_remap(page, size, gfp, prot, caller); 548 if (!ptr) { 549 __dma_free_buffer(page, size); 550 return NULL; 551 } 552 553 out: 554 *ret_page = page; 555 return ptr; 556 } 557 558 static void *__alloc_from_pool(size_t size, struct page **ret_page) 559 { 560 unsigned long val; 561 void *ptr = NULL; 562 563 if (!atomic_pool) { 564 WARN(1, "coherent pool not initialised!\n"); 565 return NULL; 566 } 567 568 val = gen_pool_alloc(atomic_pool, size); 569 if (val) { 570 phys_addr_t phys = gen_pool_virt_to_phys(atomic_pool, val); 571 572 *ret_page = phys_to_page(phys); 573 ptr = (void *)val; 574 } 575 576 return ptr; 577 } 578 579 static bool __in_atomic_pool(void *start, size_t size) 580 { 581 return addr_in_gen_pool(atomic_pool, (unsigned long)start, size); 582 } 583 584 static int __free_from_pool(void *start, size_t size) 585 { 586 if (!__in_atomic_pool(start, size)) 587 return 0; 588 589 gen_pool_free(atomic_pool, (unsigned long)start, size); 590 591 return 1; 592 } 593 594 static void *__alloc_from_contiguous(struct device *dev, size_t size, 595 pgprot_t prot, struct page **ret_page, 596 const void *caller, bool want_vaddr, 597 int coherent_flag) 598 { 599 unsigned long order = get_order(size); 600 size_t count = size >> PAGE_SHIFT; 601 struct page *page; 602 void *ptr = NULL; 603 604 page = dma_alloc_from_contiguous(dev, count, order); 605 if (!page) 606 return NULL; 607 608 __dma_clear_buffer(page, size, coherent_flag); 609 610 if (!want_vaddr) 611 goto out; 612 613 if (PageHighMem(page)) { 614 ptr = __dma_alloc_remap(page, size, GFP_KERNEL, prot, caller); 615 if (!ptr) { 616 dma_release_from_contiguous(dev, page, count); 617 return NULL; 618 } 619 } else { 620 __dma_remap(page, size, prot); 621 ptr = page_address(page); 622 } 623 624 out: 625 *ret_page = page; 626 return ptr; 627 } 628 629 static void __free_from_contiguous(struct device *dev, struct page *page, 630 void *cpu_addr, size_t size, bool want_vaddr) 631 { 632 if (want_vaddr) { 633 if (PageHighMem(page)) 634 __dma_free_remap(cpu_addr, size); 635 else 636 __dma_remap(page, size, PAGE_KERNEL); 637 } 638 dma_release_from_contiguous(dev, page, size >> PAGE_SHIFT); 639 } 640 641 static inline pgprot_t __get_dma_pgprot(unsigned long attrs, pgprot_t prot) 642 { 643 prot = (attrs & DMA_ATTR_WRITE_COMBINE) ? 644 pgprot_writecombine(prot) : 645 pgprot_dmacoherent(prot); 646 return prot; 647 } 648 649 #define nommu() 0 650 651 #else /* !CONFIG_MMU */ 652 653 #define nommu() 1 654 655 #define __get_dma_pgprot(attrs, prot) __pgprot(0) 656 #define __alloc_remap_buffer(dev, size, gfp, prot, ret, c, wv) NULL 657 #define __alloc_from_pool(size, ret_page) NULL 658 #define __alloc_from_contiguous(dev, size, prot, ret, c, wv, coherent_flag) NULL 659 #define __free_from_pool(cpu_addr, size) do { } while (0) 660 #define __free_from_contiguous(dev, page, cpu_addr, size, wv) do { } while (0) 661 #define __dma_free_remap(cpu_addr, size) do { } while (0) 662 663 #endif /* CONFIG_MMU */ 664 665 static void *__alloc_simple_buffer(struct device *dev, size_t size, gfp_t gfp, 666 struct page **ret_page) 667 { 668 struct page *page; 669 /* __alloc_simple_buffer is only called when the device is coherent */ 670 page = __dma_alloc_buffer(dev, size, gfp, COHERENT); 671 if (!page) 672 return NULL; 673 674 *ret_page = page; 675 return page_address(page); 676 } 677 678 static void *simple_allocator_alloc(struct arm_dma_alloc_args *args, 679 struct page **ret_page) 680 { 681 return __alloc_simple_buffer(args->dev, args->size, args->gfp, 682 ret_page); 683 } 684 685 static void simple_allocator_free(struct arm_dma_free_args *args) 686 { 687 __dma_free_buffer(args->page, args->size); 688 } 689 690 static struct arm_dma_allocator simple_allocator = { 691 .alloc = simple_allocator_alloc, 692 .free = simple_allocator_free, 693 }; 694 695 static void *cma_allocator_alloc(struct arm_dma_alloc_args *args, 696 struct page **ret_page) 697 { 698 return __alloc_from_contiguous(args->dev, args->size, args->prot, 699 ret_page, args->caller, 700 args->want_vaddr, args->coherent_flag); 701 } 702 703 static void cma_allocator_free(struct arm_dma_free_args *args) 704 { 705 __free_from_contiguous(args->dev, args->page, args->cpu_addr, 706 args->size, args->want_vaddr); 707 } 708 709 static struct arm_dma_allocator cma_allocator = { 710 .alloc = cma_allocator_alloc, 711 .free = cma_allocator_free, 712 }; 713 714 static void *pool_allocator_alloc(struct arm_dma_alloc_args *args, 715 struct page **ret_page) 716 { 717 return __alloc_from_pool(args->size, ret_page); 718 } 719 720 static void pool_allocator_free(struct arm_dma_free_args *args) 721 { 722 __free_from_pool(args->cpu_addr, args->size); 723 } 724 725 static struct arm_dma_allocator pool_allocator = { 726 .alloc = pool_allocator_alloc, 727 .free = pool_allocator_free, 728 }; 729 730 static void *remap_allocator_alloc(struct arm_dma_alloc_args *args, 731 struct page **ret_page) 732 { 733 return __alloc_remap_buffer(args->dev, args->size, args->gfp, 734 args->prot, ret_page, args->caller, 735 args->want_vaddr); 736 } 737 738 static void remap_allocator_free(struct arm_dma_free_args *args) 739 { 740 if (args->want_vaddr) 741 __dma_free_remap(args->cpu_addr, args->size); 742 743 __dma_free_buffer(args->page, args->size); 744 } 745 746 static struct arm_dma_allocator remap_allocator = { 747 .alloc = remap_allocator_alloc, 748 .free = remap_allocator_free, 749 }; 750 751 static void *__dma_alloc(struct device *dev, size_t size, dma_addr_t *handle, 752 gfp_t gfp, pgprot_t prot, bool is_coherent, 753 unsigned long attrs, const void *caller) 754 { 755 u64 mask = get_coherent_dma_mask(dev); 756 struct page *page = NULL; 757 void *addr; 758 bool allowblock, cma; 759 struct arm_dma_buffer *buf; 760 struct arm_dma_alloc_args args = { 761 .dev = dev, 762 .size = PAGE_ALIGN(size), 763 .gfp = gfp, 764 .prot = prot, 765 .caller = caller, 766 .want_vaddr = ((attrs & DMA_ATTR_NO_KERNEL_MAPPING) == 0), 767 .coherent_flag = is_coherent ? COHERENT : NORMAL, 768 }; 769 770 #ifdef CONFIG_DMA_API_DEBUG 771 u64 limit = (mask + 1) & ~mask; 772 if (limit && size >= limit) { 773 dev_warn(dev, "coherent allocation too big (requested %#x mask %#llx)\n", 774 size, mask); 775 return NULL; 776 } 777 #endif 778 779 if (!mask) 780 return NULL; 781 782 buf = kzalloc(sizeof(*buf), 783 gfp & ~(__GFP_DMA | __GFP_DMA32 | __GFP_HIGHMEM)); 784 if (!buf) 785 return NULL; 786 787 if (mask < 0xffffffffULL) 788 gfp |= GFP_DMA; 789 790 /* 791 * Following is a work-around (a.k.a. hack) to prevent pages 792 * with __GFP_COMP being passed to split_page() which cannot 793 * handle them. The real problem is that this flag probably 794 * should be 0 on ARM as it is not supported on this 795 * platform; see CONFIG_HUGETLBFS. 796 */ 797 gfp &= ~(__GFP_COMP); 798 args.gfp = gfp; 799 800 *handle = DMA_ERROR_CODE; 801 allowblock = gfpflags_allow_blocking(gfp); 802 cma = allowblock ? dev_get_cma_area(dev) : false; 803 804 if (cma) 805 buf->allocator = &cma_allocator; 806 else if (nommu() || is_coherent) 807 buf->allocator = &simple_allocator; 808 else if (allowblock) 809 buf->allocator = &remap_allocator; 810 else 811 buf->allocator = &pool_allocator; 812 813 addr = buf->allocator->alloc(&args, &page); 814 815 if (page) { 816 unsigned long flags; 817 818 *handle = pfn_to_dma(dev, page_to_pfn(page)); 819 buf->virt = args.want_vaddr ? addr : page; 820 821 spin_lock_irqsave(&arm_dma_bufs_lock, flags); 822 list_add(&buf->list, &arm_dma_bufs); 823 spin_unlock_irqrestore(&arm_dma_bufs_lock, flags); 824 } else { 825 kfree(buf); 826 } 827 828 return args.want_vaddr ? addr : page; 829 } 830 831 /* 832 * Allocate DMA-coherent memory space and return both the kernel remapped 833 * virtual and bus address for that space. 834 */ 835 void *arm_dma_alloc(struct device *dev, size_t size, dma_addr_t *handle, 836 gfp_t gfp, unsigned long attrs) 837 { 838 pgprot_t prot = __get_dma_pgprot(attrs, PAGE_KERNEL); 839 840 return __dma_alloc(dev, size, handle, gfp, prot, false, 841 attrs, __builtin_return_address(0)); 842 } 843 844 static void *arm_coherent_dma_alloc(struct device *dev, size_t size, 845 dma_addr_t *handle, gfp_t gfp, unsigned long attrs) 846 { 847 return __dma_alloc(dev, size, handle, gfp, PAGE_KERNEL, true, 848 attrs, __builtin_return_address(0)); 849 } 850 851 static int __arm_dma_mmap(struct device *dev, struct vm_area_struct *vma, 852 void *cpu_addr, dma_addr_t dma_addr, size_t size, 853 unsigned long attrs) 854 { 855 int ret = -ENXIO; 856 #ifdef CONFIG_MMU 857 unsigned long nr_vma_pages = (vma->vm_end - vma->vm_start) >> PAGE_SHIFT; 858 unsigned long nr_pages = PAGE_ALIGN(size) >> PAGE_SHIFT; 859 unsigned long pfn = dma_to_pfn(dev, dma_addr); 860 unsigned long off = vma->vm_pgoff; 861 862 if (dma_mmap_from_coherent(dev, vma, cpu_addr, size, &ret)) 863 return ret; 864 865 if (off < nr_pages && nr_vma_pages <= (nr_pages - off)) { 866 ret = remap_pfn_range(vma, vma->vm_start, 867 pfn + off, 868 vma->vm_end - vma->vm_start, 869 vma->vm_page_prot); 870 } 871 #else 872 ret = vm_iomap_memory(vma, vma->vm_start, 873 (vma->vm_end - vma->vm_start)); 874 #endif /* CONFIG_MMU */ 875 876 return ret; 877 } 878 879 /* 880 * Create userspace mapping for the DMA-coherent memory. 881 */ 882 static int arm_coherent_dma_mmap(struct device *dev, struct vm_area_struct *vma, 883 void *cpu_addr, dma_addr_t dma_addr, size_t size, 884 unsigned long attrs) 885 { 886 return __arm_dma_mmap(dev, vma, cpu_addr, dma_addr, size, attrs); 887 } 888 889 int arm_dma_mmap(struct device *dev, struct vm_area_struct *vma, 890 void *cpu_addr, dma_addr_t dma_addr, size_t size, 891 unsigned long attrs) 892 { 893 #ifdef CONFIG_MMU 894 vma->vm_page_prot = __get_dma_pgprot(attrs, vma->vm_page_prot); 895 #endif /* CONFIG_MMU */ 896 return __arm_dma_mmap(dev, vma, cpu_addr, dma_addr, size, attrs); 897 } 898 899 /* 900 * Free a buffer as defined by the above mapping. 901 */ 902 static void __arm_dma_free(struct device *dev, size_t size, void *cpu_addr, 903 dma_addr_t handle, unsigned long attrs, 904 bool is_coherent) 905 { 906 struct page *page = pfn_to_page(dma_to_pfn(dev, handle)); 907 struct arm_dma_buffer *buf; 908 struct arm_dma_free_args args = { 909 .dev = dev, 910 .size = PAGE_ALIGN(size), 911 .cpu_addr = cpu_addr, 912 .page = page, 913 .want_vaddr = ((attrs & DMA_ATTR_NO_KERNEL_MAPPING) == 0), 914 }; 915 916 buf = arm_dma_buffer_find(cpu_addr); 917 if (WARN(!buf, "Freeing invalid buffer %p\n", cpu_addr)) 918 return; 919 920 buf->allocator->free(&args); 921 kfree(buf); 922 } 923 924 void arm_dma_free(struct device *dev, size_t size, void *cpu_addr, 925 dma_addr_t handle, unsigned long attrs) 926 { 927 __arm_dma_free(dev, size, cpu_addr, handle, attrs, false); 928 } 929 930 static void arm_coherent_dma_free(struct device *dev, size_t size, void *cpu_addr, 931 dma_addr_t handle, unsigned long attrs) 932 { 933 __arm_dma_free(dev, size, cpu_addr, handle, attrs, true); 934 } 935 936 int arm_dma_get_sgtable(struct device *dev, struct sg_table *sgt, 937 void *cpu_addr, dma_addr_t handle, size_t size, 938 unsigned long attrs) 939 { 940 struct page *page = pfn_to_page(dma_to_pfn(dev, handle)); 941 int ret; 942 943 ret = sg_alloc_table(sgt, 1, GFP_KERNEL); 944 if (unlikely(ret)) 945 return ret; 946 947 sg_set_page(sgt->sgl, page, PAGE_ALIGN(size), 0); 948 return 0; 949 } 950 951 static void dma_cache_maint_page(struct page *page, unsigned long offset, 952 size_t size, enum dma_data_direction dir, 953 void (*op)(const void *, size_t, int)) 954 { 955 unsigned long pfn; 956 size_t left = size; 957 958 pfn = page_to_pfn(page) + offset / PAGE_SIZE; 959 offset %= PAGE_SIZE; 960 961 /* 962 * A single sg entry may refer to multiple physically contiguous 963 * pages. But we still need to process highmem pages individually. 964 * If highmem is not configured then the bulk of this loop gets 965 * optimized out. 966 */ 967 do { 968 size_t len = left; 969 void *vaddr; 970 971 page = pfn_to_page(pfn); 972 973 if (PageHighMem(page)) { 974 if (len + offset > PAGE_SIZE) 975 len = PAGE_SIZE - offset; 976 977 if (cache_is_vipt_nonaliasing()) { 978 vaddr = kmap_atomic(page); 979 op(vaddr + offset, len, dir); 980 kunmap_atomic(vaddr); 981 } else { 982 vaddr = kmap_high_get(page); 983 if (vaddr) { 984 op(vaddr + offset, len, dir); 985 kunmap_high(page); 986 } 987 } 988 } else { 989 vaddr = page_address(page) + offset; 990 op(vaddr, len, dir); 991 } 992 offset = 0; 993 pfn++; 994 left -= len; 995 } while (left); 996 } 997 998 /* 999 * Make an area consistent for devices. 1000 * Note: Drivers should NOT use this function directly, as it will break 1001 * platforms with CONFIG_DMABOUNCE. 1002 * Use the driver DMA support - see dma-mapping.h (dma_sync_*) 1003 */ 1004 static void __dma_page_cpu_to_dev(struct page *page, unsigned long off, 1005 size_t size, enum dma_data_direction dir) 1006 { 1007 phys_addr_t paddr; 1008 1009 dma_cache_maint_page(page, off, size, dir, dmac_map_area); 1010 1011 paddr = page_to_phys(page) + off; 1012 if (dir == DMA_FROM_DEVICE) { 1013 outer_inv_range(paddr, paddr + size); 1014 } else { 1015 outer_clean_range(paddr, paddr + size); 1016 } 1017 /* FIXME: non-speculating: flush on bidirectional mappings? */ 1018 } 1019 1020 static void __dma_page_dev_to_cpu(struct page *page, unsigned long off, 1021 size_t size, enum dma_data_direction dir) 1022 { 1023 phys_addr_t paddr = page_to_phys(page) + off; 1024 1025 /* FIXME: non-speculating: not required */ 1026 /* in any case, don't bother invalidating if DMA to device */ 1027 if (dir != DMA_TO_DEVICE) { 1028 outer_inv_range(paddr, paddr + size); 1029 1030 dma_cache_maint_page(page, off, size, dir, dmac_unmap_area); 1031 } 1032 1033 /* 1034 * Mark the D-cache clean for these pages to avoid extra flushing. 1035 */ 1036 if (dir != DMA_TO_DEVICE && size >= PAGE_SIZE) { 1037 unsigned long pfn; 1038 size_t left = size; 1039 1040 pfn = page_to_pfn(page) + off / PAGE_SIZE; 1041 off %= PAGE_SIZE; 1042 if (off) { 1043 pfn++; 1044 left -= PAGE_SIZE - off; 1045 } 1046 while (left >= PAGE_SIZE) { 1047 page = pfn_to_page(pfn++); 1048 set_bit(PG_dcache_clean, &page->flags); 1049 left -= PAGE_SIZE; 1050 } 1051 } 1052 } 1053 1054 /** 1055 * arm_dma_map_sg - map a set of SG buffers for streaming mode DMA 1056 * @dev: valid struct device pointer, or NULL for ISA and EISA-like devices 1057 * @sg: list of buffers 1058 * @nents: number of buffers to map 1059 * @dir: DMA transfer direction 1060 * 1061 * Map a set of buffers described by scatterlist in streaming mode for DMA. 1062 * This is the scatter-gather version of the dma_map_single interface. 1063 * Here the scatter gather list elements are each tagged with the 1064 * appropriate dma address and length. They are obtained via 1065 * sg_dma_{address,length}. 1066 * 1067 * Device ownership issues as mentioned for dma_map_single are the same 1068 * here. 1069 */ 1070 int arm_dma_map_sg(struct device *dev, struct scatterlist *sg, int nents, 1071 enum dma_data_direction dir, unsigned long attrs) 1072 { 1073 struct dma_map_ops *ops = get_dma_ops(dev); 1074 struct scatterlist *s; 1075 int i, j; 1076 1077 for_each_sg(sg, s, nents, i) { 1078 #ifdef CONFIG_NEED_SG_DMA_LENGTH 1079 s->dma_length = s->length; 1080 #endif 1081 s->dma_address = ops->map_page(dev, sg_page(s), s->offset, 1082 s->length, dir, attrs); 1083 if (dma_mapping_error(dev, s->dma_address)) 1084 goto bad_mapping; 1085 } 1086 return nents; 1087 1088 bad_mapping: 1089 for_each_sg(sg, s, i, j) 1090 ops->unmap_page(dev, sg_dma_address(s), sg_dma_len(s), dir, attrs); 1091 return 0; 1092 } 1093 1094 /** 1095 * arm_dma_unmap_sg - unmap a set of SG buffers mapped by dma_map_sg 1096 * @dev: valid struct device pointer, or NULL for ISA and EISA-like devices 1097 * @sg: list of buffers 1098 * @nents: number of buffers to unmap (same as was passed to dma_map_sg) 1099 * @dir: DMA transfer direction (same as was passed to dma_map_sg) 1100 * 1101 * Unmap a set of streaming mode DMA translations. Again, CPU access 1102 * rules concerning calls here are the same as for dma_unmap_single(). 1103 */ 1104 void arm_dma_unmap_sg(struct device *dev, struct scatterlist *sg, int nents, 1105 enum dma_data_direction dir, unsigned long attrs) 1106 { 1107 struct dma_map_ops *ops = get_dma_ops(dev); 1108 struct scatterlist *s; 1109 1110 int i; 1111 1112 for_each_sg(sg, s, nents, i) 1113 ops->unmap_page(dev, sg_dma_address(s), sg_dma_len(s), dir, attrs); 1114 } 1115 1116 /** 1117 * arm_dma_sync_sg_for_cpu 1118 * @dev: valid struct device pointer, or NULL for ISA and EISA-like devices 1119 * @sg: list of buffers 1120 * @nents: number of buffers to map (returned from dma_map_sg) 1121 * @dir: DMA transfer direction (same as was passed to dma_map_sg) 1122 */ 1123 void arm_dma_sync_sg_for_cpu(struct device *dev, struct scatterlist *sg, 1124 int nents, enum dma_data_direction dir) 1125 { 1126 struct dma_map_ops *ops = get_dma_ops(dev); 1127 struct scatterlist *s; 1128 int i; 1129 1130 for_each_sg(sg, s, nents, i) 1131 ops->sync_single_for_cpu(dev, sg_dma_address(s), s->length, 1132 dir); 1133 } 1134 1135 /** 1136 * arm_dma_sync_sg_for_device 1137 * @dev: valid struct device pointer, or NULL for ISA and EISA-like devices 1138 * @sg: list of buffers 1139 * @nents: number of buffers to map (returned from dma_map_sg) 1140 * @dir: DMA transfer direction (same as was passed to dma_map_sg) 1141 */ 1142 void arm_dma_sync_sg_for_device(struct device *dev, struct scatterlist *sg, 1143 int nents, enum dma_data_direction dir) 1144 { 1145 struct dma_map_ops *ops = get_dma_ops(dev); 1146 struct scatterlist *s; 1147 int i; 1148 1149 for_each_sg(sg, s, nents, i) 1150 ops->sync_single_for_device(dev, sg_dma_address(s), s->length, 1151 dir); 1152 } 1153 1154 /* 1155 * Return whether the given device DMA address mask can be supported 1156 * properly. For example, if your device can only drive the low 24-bits 1157 * during bus mastering, then you would pass 0x00ffffff as the mask 1158 * to this function. 1159 */ 1160 int dma_supported(struct device *dev, u64 mask) 1161 { 1162 return __dma_supported(dev, mask, false); 1163 } 1164 EXPORT_SYMBOL(dma_supported); 1165 1166 #define PREALLOC_DMA_DEBUG_ENTRIES 4096 1167 1168 static int __init dma_debug_do_init(void) 1169 { 1170 dma_debug_init(PREALLOC_DMA_DEBUG_ENTRIES); 1171 return 0; 1172 } 1173 core_initcall(dma_debug_do_init); 1174 1175 #ifdef CONFIG_ARM_DMA_USE_IOMMU 1176 1177 /* IOMMU */ 1178 1179 static int extend_iommu_mapping(struct dma_iommu_mapping *mapping); 1180 1181 static inline dma_addr_t __alloc_iova(struct dma_iommu_mapping *mapping, 1182 size_t size) 1183 { 1184 unsigned int order = get_order(size); 1185 unsigned int align = 0; 1186 unsigned int count, start; 1187 size_t mapping_size = mapping->bits << PAGE_SHIFT; 1188 unsigned long flags; 1189 dma_addr_t iova; 1190 int i; 1191 1192 if (order > CONFIG_ARM_DMA_IOMMU_ALIGNMENT) 1193 order = CONFIG_ARM_DMA_IOMMU_ALIGNMENT; 1194 1195 count = PAGE_ALIGN(size) >> PAGE_SHIFT; 1196 align = (1 << order) - 1; 1197 1198 spin_lock_irqsave(&mapping->lock, flags); 1199 for (i = 0; i < mapping->nr_bitmaps; i++) { 1200 start = bitmap_find_next_zero_area(mapping->bitmaps[i], 1201 mapping->bits, 0, count, align); 1202 1203 if (start > mapping->bits) 1204 continue; 1205 1206 bitmap_set(mapping->bitmaps[i], start, count); 1207 break; 1208 } 1209 1210 /* 1211 * No unused range found. Try to extend the existing mapping 1212 * and perform a second attempt to reserve an IO virtual 1213 * address range of size bytes. 1214 */ 1215 if (i == mapping->nr_bitmaps) { 1216 if (extend_iommu_mapping(mapping)) { 1217 spin_unlock_irqrestore(&mapping->lock, flags); 1218 return DMA_ERROR_CODE; 1219 } 1220 1221 start = bitmap_find_next_zero_area(mapping->bitmaps[i], 1222 mapping->bits, 0, count, align); 1223 1224 if (start > mapping->bits) { 1225 spin_unlock_irqrestore(&mapping->lock, flags); 1226 return DMA_ERROR_CODE; 1227 } 1228 1229 bitmap_set(mapping->bitmaps[i], start, count); 1230 } 1231 spin_unlock_irqrestore(&mapping->lock, flags); 1232 1233 iova = mapping->base + (mapping_size * i); 1234 iova += start << PAGE_SHIFT; 1235 1236 return iova; 1237 } 1238 1239 static inline void __free_iova(struct dma_iommu_mapping *mapping, 1240 dma_addr_t addr, size_t size) 1241 { 1242 unsigned int start, count; 1243 size_t mapping_size = mapping->bits << PAGE_SHIFT; 1244 unsigned long flags; 1245 dma_addr_t bitmap_base; 1246 u32 bitmap_index; 1247 1248 if (!size) 1249 return; 1250 1251 bitmap_index = (u32) (addr - mapping->base) / (u32) mapping_size; 1252 BUG_ON(addr < mapping->base || bitmap_index > mapping->extensions); 1253 1254 bitmap_base = mapping->base + mapping_size * bitmap_index; 1255 1256 start = (addr - bitmap_base) >> PAGE_SHIFT; 1257 1258 if (addr + size > bitmap_base + mapping_size) { 1259 /* 1260 * The address range to be freed reaches into the iova 1261 * range of the next bitmap. This should not happen as 1262 * we don't allow this in __alloc_iova (at the 1263 * moment). 1264 */ 1265 BUG(); 1266 } else 1267 count = size >> PAGE_SHIFT; 1268 1269 spin_lock_irqsave(&mapping->lock, flags); 1270 bitmap_clear(mapping->bitmaps[bitmap_index], start, count); 1271 spin_unlock_irqrestore(&mapping->lock, flags); 1272 } 1273 1274 /* We'll try 2M, 1M, 64K, and finally 4K; array must end with 0! */ 1275 static const int iommu_order_array[] = { 9, 8, 4, 0 }; 1276 1277 static struct page **__iommu_alloc_buffer(struct device *dev, size_t size, 1278 gfp_t gfp, unsigned long attrs, 1279 int coherent_flag) 1280 { 1281 struct page **pages; 1282 int count = size >> PAGE_SHIFT; 1283 int array_size = count * sizeof(struct page *); 1284 int i = 0; 1285 int order_idx = 0; 1286 1287 if (array_size <= PAGE_SIZE) 1288 pages = kzalloc(array_size, GFP_KERNEL); 1289 else 1290 pages = vzalloc(array_size); 1291 if (!pages) 1292 return NULL; 1293 1294 if (attrs & DMA_ATTR_FORCE_CONTIGUOUS) 1295 { 1296 unsigned long order = get_order(size); 1297 struct page *page; 1298 1299 page = dma_alloc_from_contiguous(dev, count, order); 1300 if (!page) 1301 goto error; 1302 1303 __dma_clear_buffer(page, size, coherent_flag); 1304 1305 for (i = 0; i < count; i++) 1306 pages[i] = page + i; 1307 1308 return pages; 1309 } 1310 1311 /* Go straight to 4K chunks if caller says it's OK. */ 1312 if (attrs & DMA_ATTR_ALLOC_SINGLE_PAGES) 1313 order_idx = ARRAY_SIZE(iommu_order_array) - 1; 1314 1315 /* 1316 * IOMMU can map any pages, so himem can also be used here 1317 */ 1318 gfp |= __GFP_NOWARN | __GFP_HIGHMEM; 1319 1320 while (count) { 1321 int j, order; 1322 1323 order = iommu_order_array[order_idx]; 1324 1325 /* Drop down when we get small */ 1326 if (__fls(count) < order) { 1327 order_idx++; 1328 continue; 1329 } 1330 1331 if (order) { 1332 /* See if it's easy to allocate a high-order chunk */ 1333 pages[i] = alloc_pages(gfp | __GFP_NORETRY, order); 1334 1335 /* Go down a notch at first sign of pressure */ 1336 if (!pages[i]) { 1337 order_idx++; 1338 continue; 1339 } 1340 } else { 1341 pages[i] = alloc_pages(gfp, 0); 1342 if (!pages[i]) 1343 goto error; 1344 } 1345 1346 if (order) { 1347 split_page(pages[i], order); 1348 j = 1 << order; 1349 while (--j) 1350 pages[i + j] = pages[i] + j; 1351 } 1352 1353 __dma_clear_buffer(pages[i], PAGE_SIZE << order, coherent_flag); 1354 i += 1 << order; 1355 count -= 1 << order; 1356 } 1357 1358 return pages; 1359 error: 1360 while (i--) 1361 if (pages[i]) 1362 __free_pages(pages[i], 0); 1363 kvfree(pages); 1364 return NULL; 1365 } 1366 1367 static int __iommu_free_buffer(struct device *dev, struct page **pages, 1368 size_t size, unsigned long attrs) 1369 { 1370 int count = size >> PAGE_SHIFT; 1371 int i; 1372 1373 if (attrs & DMA_ATTR_FORCE_CONTIGUOUS) { 1374 dma_release_from_contiguous(dev, pages[0], count); 1375 } else { 1376 for (i = 0; i < count; i++) 1377 if (pages[i]) 1378 __free_pages(pages[i], 0); 1379 } 1380 1381 kvfree(pages); 1382 return 0; 1383 } 1384 1385 /* 1386 * Create a CPU mapping for a specified pages 1387 */ 1388 static void * 1389 __iommu_alloc_remap(struct page **pages, size_t size, gfp_t gfp, pgprot_t prot, 1390 const void *caller) 1391 { 1392 return dma_common_pages_remap(pages, size, 1393 VM_ARM_DMA_CONSISTENT | VM_USERMAP, prot, caller); 1394 } 1395 1396 /* 1397 * Create a mapping in device IO address space for specified pages 1398 */ 1399 static dma_addr_t 1400 __iommu_create_mapping(struct device *dev, struct page **pages, size_t size) 1401 { 1402 struct dma_iommu_mapping *mapping = to_dma_iommu_mapping(dev); 1403 unsigned int count = PAGE_ALIGN(size) >> PAGE_SHIFT; 1404 dma_addr_t dma_addr, iova; 1405 int i; 1406 1407 dma_addr = __alloc_iova(mapping, size); 1408 if (dma_addr == DMA_ERROR_CODE) 1409 return dma_addr; 1410 1411 iova = dma_addr; 1412 for (i = 0; i < count; ) { 1413 int ret; 1414 1415 unsigned int next_pfn = page_to_pfn(pages[i]) + 1; 1416 phys_addr_t phys = page_to_phys(pages[i]); 1417 unsigned int len, j; 1418 1419 for (j = i + 1; j < count; j++, next_pfn++) 1420 if (page_to_pfn(pages[j]) != next_pfn) 1421 break; 1422 1423 len = (j - i) << PAGE_SHIFT; 1424 ret = iommu_map(mapping->domain, iova, phys, len, 1425 IOMMU_READ|IOMMU_WRITE); 1426 if (ret < 0) 1427 goto fail; 1428 iova += len; 1429 i = j; 1430 } 1431 return dma_addr; 1432 fail: 1433 iommu_unmap(mapping->domain, dma_addr, iova-dma_addr); 1434 __free_iova(mapping, dma_addr, size); 1435 return DMA_ERROR_CODE; 1436 } 1437 1438 static int __iommu_remove_mapping(struct device *dev, dma_addr_t iova, size_t size) 1439 { 1440 struct dma_iommu_mapping *mapping = to_dma_iommu_mapping(dev); 1441 1442 /* 1443 * add optional in-page offset from iova to size and align 1444 * result to page size 1445 */ 1446 size = PAGE_ALIGN((iova & ~PAGE_MASK) + size); 1447 iova &= PAGE_MASK; 1448 1449 iommu_unmap(mapping->domain, iova, size); 1450 __free_iova(mapping, iova, size); 1451 return 0; 1452 } 1453 1454 static struct page **__atomic_get_pages(void *addr) 1455 { 1456 struct page *page; 1457 phys_addr_t phys; 1458 1459 phys = gen_pool_virt_to_phys(atomic_pool, (unsigned long)addr); 1460 page = phys_to_page(phys); 1461 1462 return (struct page **)page; 1463 } 1464 1465 static struct page **__iommu_get_pages(void *cpu_addr, unsigned long attrs) 1466 { 1467 struct vm_struct *area; 1468 1469 if (__in_atomic_pool(cpu_addr, PAGE_SIZE)) 1470 return __atomic_get_pages(cpu_addr); 1471 1472 if (attrs & DMA_ATTR_NO_KERNEL_MAPPING) 1473 return cpu_addr; 1474 1475 area = find_vm_area(cpu_addr); 1476 if (area && (area->flags & VM_ARM_DMA_CONSISTENT)) 1477 return area->pages; 1478 return NULL; 1479 } 1480 1481 static void *__iommu_alloc_simple(struct device *dev, size_t size, gfp_t gfp, 1482 dma_addr_t *handle, int coherent_flag) 1483 { 1484 struct page *page; 1485 void *addr; 1486 1487 if (coherent_flag == COHERENT) 1488 addr = __alloc_simple_buffer(dev, size, gfp, &page); 1489 else 1490 addr = __alloc_from_pool(size, &page); 1491 if (!addr) 1492 return NULL; 1493 1494 *handle = __iommu_create_mapping(dev, &page, size); 1495 if (*handle == DMA_ERROR_CODE) 1496 goto err_mapping; 1497 1498 return addr; 1499 1500 err_mapping: 1501 __free_from_pool(addr, size); 1502 return NULL; 1503 } 1504 1505 static void __iommu_free_atomic(struct device *dev, void *cpu_addr, 1506 dma_addr_t handle, size_t size, int coherent_flag) 1507 { 1508 __iommu_remove_mapping(dev, handle, size); 1509 if (coherent_flag == COHERENT) 1510 __dma_free_buffer(virt_to_page(cpu_addr), size); 1511 else 1512 __free_from_pool(cpu_addr, size); 1513 } 1514 1515 static void *__arm_iommu_alloc_attrs(struct device *dev, size_t size, 1516 dma_addr_t *handle, gfp_t gfp, unsigned long attrs, 1517 int coherent_flag) 1518 { 1519 pgprot_t prot = __get_dma_pgprot(attrs, PAGE_KERNEL); 1520 struct page **pages; 1521 void *addr = NULL; 1522 1523 *handle = DMA_ERROR_CODE; 1524 size = PAGE_ALIGN(size); 1525 1526 if (coherent_flag == COHERENT || !gfpflags_allow_blocking(gfp)) 1527 return __iommu_alloc_simple(dev, size, gfp, handle, 1528 coherent_flag); 1529 1530 /* 1531 * Following is a work-around (a.k.a. hack) to prevent pages 1532 * with __GFP_COMP being passed to split_page() which cannot 1533 * handle them. The real problem is that this flag probably 1534 * should be 0 on ARM as it is not supported on this 1535 * platform; see CONFIG_HUGETLBFS. 1536 */ 1537 gfp &= ~(__GFP_COMP); 1538 1539 pages = __iommu_alloc_buffer(dev, size, gfp, attrs, coherent_flag); 1540 if (!pages) 1541 return NULL; 1542 1543 *handle = __iommu_create_mapping(dev, pages, size); 1544 if (*handle == DMA_ERROR_CODE) 1545 goto err_buffer; 1546 1547 if (attrs & DMA_ATTR_NO_KERNEL_MAPPING) 1548 return pages; 1549 1550 addr = __iommu_alloc_remap(pages, size, gfp, prot, 1551 __builtin_return_address(0)); 1552 if (!addr) 1553 goto err_mapping; 1554 1555 return addr; 1556 1557 err_mapping: 1558 __iommu_remove_mapping(dev, *handle, size); 1559 err_buffer: 1560 __iommu_free_buffer(dev, pages, size, attrs); 1561 return NULL; 1562 } 1563 1564 static void *arm_iommu_alloc_attrs(struct device *dev, size_t size, 1565 dma_addr_t *handle, gfp_t gfp, unsigned long attrs) 1566 { 1567 return __arm_iommu_alloc_attrs(dev, size, handle, gfp, attrs, NORMAL); 1568 } 1569 1570 static void *arm_coherent_iommu_alloc_attrs(struct device *dev, size_t size, 1571 dma_addr_t *handle, gfp_t gfp, unsigned long attrs) 1572 { 1573 return __arm_iommu_alloc_attrs(dev, size, handle, gfp, attrs, COHERENT); 1574 } 1575 1576 static int __arm_iommu_mmap_attrs(struct device *dev, struct vm_area_struct *vma, 1577 void *cpu_addr, dma_addr_t dma_addr, size_t size, 1578 unsigned long attrs) 1579 { 1580 unsigned long uaddr = vma->vm_start; 1581 unsigned long usize = vma->vm_end - vma->vm_start; 1582 struct page **pages = __iommu_get_pages(cpu_addr, attrs); 1583 unsigned long nr_pages = PAGE_ALIGN(size) >> PAGE_SHIFT; 1584 unsigned long off = vma->vm_pgoff; 1585 1586 if (!pages) 1587 return -ENXIO; 1588 1589 if (off >= nr_pages || (usize >> PAGE_SHIFT) > nr_pages - off) 1590 return -ENXIO; 1591 1592 pages += off; 1593 1594 do { 1595 int ret = vm_insert_page(vma, uaddr, *pages++); 1596 if (ret) { 1597 pr_err("Remapping memory failed: %d\n", ret); 1598 return ret; 1599 } 1600 uaddr += PAGE_SIZE; 1601 usize -= PAGE_SIZE; 1602 } while (usize > 0); 1603 1604 return 0; 1605 } 1606 static int arm_iommu_mmap_attrs(struct device *dev, 1607 struct vm_area_struct *vma, void *cpu_addr, 1608 dma_addr_t dma_addr, size_t size, unsigned long attrs) 1609 { 1610 vma->vm_page_prot = __get_dma_pgprot(attrs, vma->vm_page_prot); 1611 1612 return __arm_iommu_mmap_attrs(dev, vma, cpu_addr, dma_addr, size, attrs); 1613 } 1614 1615 static int arm_coherent_iommu_mmap_attrs(struct device *dev, 1616 struct vm_area_struct *vma, void *cpu_addr, 1617 dma_addr_t dma_addr, size_t size, unsigned long attrs) 1618 { 1619 return __arm_iommu_mmap_attrs(dev, vma, cpu_addr, dma_addr, size, attrs); 1620 } 1621 1622 /* 1623 * free a page as defined by the above mapping. 1624 * Must not be called with IRQs disabled. 1625 */ 1626 void __arm_iommu_free_attrs(struct device *dev, size_t size, void *cpu_addr, 1627 dma_addr_t handle, unsigned long attrs, int coherent_flag) 1628 { 1629 struct page **pages; 1630 size = PAGE_ALIGN(size); 1631 1632 if (coherent_flag == COHERENT || __in_atomic_pool(cpu_addr, size)) { 1633 __iommu_free_atomic(dev, cpu_addr, handle, size, coherent_flag); 1634 return; 1635 } 1636 1637 pages = __iommu_get_pages(cpu_addr, attrs); 1638 if (!pages) { 1639 WARN(1, "trying to free invalid coherent area: %p\n", cpu_addr); 1640 return; 1641 } 1642 1643 if ((attrs & DMA_ATTR_NO_KERNEL_MAPPING) == 0) { 1644 dma_common_free_remap(cpu_addr, size, 1645 VM_ARM_DMA_CONSISTENT | VM_USERMAP); 1646 } 1647 1648 __iommu_remove_mapping(dev, handle, size); 1649 __iommu_free_buffer(dev, pages, size, attrs); 1650 } 1651 1652 void arm_iommu_free_attrs(struct device *dev, size_t size, 1653 void *cpu_addr, dma_addr_t handle, unsigned long attrs) 1654 { 1655 __arm_iommu_free_attrs(dev, size, cpu_addr, handle, attrs, NORMAL); 1656 } 1657 1658 void arm_coherent_iommu_free_attrs(struct device *dev, size_t size, 1659 void *cpu_addr, dma_addr_t handle, unsigned long attrs) 1660 { 1661 __arm_iommu_free_attrs(dev, size, cpu_addr, handle, attrs, COHERENT); 1662 } 1663 1664 static int arm_iommu_get_sgtable(struct device *dev, struct sg_table *sgt, 1665 void *cpu_addr, dma_addr_t dma_addr, 1666 size_t size, unsigned long attrs) 1667 { 1668 unsigned int count = PAGE_ALIGN(size) >> PAGE_SHIFT; 1669 struct page **pages = __iommu_get_pages(cpu_addr, attrs); 1670 1671 if (!pages) 1672 return -ENXIO; 1673 1674 return sg_alloc_table_from_pages(sgt, pages, count, 0, size, 1675 GFP_KERNEL); 1676 } 1677 1678 static int __dma_direction_to_prot(enum dma_data_direction dir) 1679 { 1680 int prot; 1681 1682 switch (dir) { 1683 case DMA_BIDIRECTIONAL: 1684 prot = IOMMU_READ | IOMMU_WRITE; 1685 break; 1686 case DMA_TO_DEVICE: 1687 prot = IOMMU_READ; 1688 break; 1689 case DMA_FROM_DEVICE: 1690 prot = IOMMU_WRITE; 1691 break; 1692 default: 1693 prot = 0; 1694 } 1695 1696 return prot; 1697 } 1698 1699 /* 1700 * Map a part of the scatter-gather list into contiguous io address space 1701 */ 1702 static int __map_sg_chunk(struct device *dev, struct scatterlist *sg, 1703 size_t size, dma_addr_t *handle, 1704 enum dma_data_direction dir, unsigned long attrs, 1705 bool is_coherent) 1706 { 1707 struct dma_iommu_mapping *mapping = to_dma_iommu_mapping(dev); 1708 dma_addr_t iova, iova_base; 1709 int ret = 0; 1710 unsigned int count; 1711 struct scatterlist *s; 1712 int prot; 1713 1714 size = PAGE_ALIGN(size); 1715 *handle = DMA_ERROR_CODE; 1716 1717 iova_base = iova = __alloc_iova(mapping, size); 1718 if (iova == DMA_ERROR_CODE) 1719 return -ENOMEM; 1720 1721 for (count = 0, s = sg; count < (size >> PAGE_SHIFT); s = sg_next(s)) { 1722 phys_addr_t phys = page_to_phys(sg_page(s)); 1723 unsigned int len = PAGE_ALIGN(s->offset + s->length); 1724 1725 if (!is_coherent && (attrs & DMA_ATTR_SKIP_CPU_SYNC) == 0) 1726 __dma_page_cpu_to_dev(sg_page(s), s->offset, s->length, dir); 1727 1728 prot = __dma_direction_to_prot(dir); 1729 1730 ret = iommu_map(mapping->domain, iova, phys, len, prot); 1731 if (ret < 0) 1732 goto fail; 1733 count += len >> PAGE_SHIFT; 1734 iova += len; 1735 } 1736 *handle = iova_base; 1737 1738 return 0; 1739 fail: 1740 iommu_unmap(mapping->domain, iova_base, count * PAGE_SIZE); 1741 __free_iova(mapping, iova_base, size); 1742 return ret; 1743 } 1744 1745 static int __iommu_map_sg(struct device *dev, struct scatterlist *sg, int nents, 1746 enum dma_data_direction dir, unsigned long attrs, 1747 bool is_coherent) 1748 { 1749 struct scatterlist *s = sg, *dma = sg, *start = sg; 1750 int i, count = 0; 1751 unsigned int offset = s->offset; 1752 unsigned int size = s->offset + s->length; 1753 unsigned int max = dma_get_max_seg_size(dev); 1754 1755 for (i = 1; i < nents; i++) { 1756 s = sg_next(s); 1757 1758 s->dma_address = DMA_ERROR_CODE; 1759 s->dma_length = 0; 1760 1761 if (s->offset || (size & ~PAGE_MASK) || size + s->length > max) { 1762 if (__map_sg_chunk(dev, start, size, &dma->dma_address, 1763 dir, attrs, is_coherent) < 0) 1764 goto bad_mapping; 1765 1766 dma->dma_address += offset; 1767 dma->dma_length = size - offset; 1768 1769 size = offset = s->offset; 1770 start = s; 1771 dma = sg_next(dma); 1772 count += 1; 1773 } 1774 size += s->length; 1775 } 1776 if (__map_sg_chunk(dev, start, size, &dma->dma_address, dir, attrs, 1777 is_coherent) < 0) 1778 goto bad_mapping; 1779 1780 dma->dma_address += offset; 1781 dma->dma_length = size - offset; 1782 1783 return count+1; 1784 1785 bad_mapping: 1786 for_each_sg(sg, s, count, i) 1787 __iommu_remove_mapping(dev, sg_dma_address(s), sg_dma_len(s)); 1788 return 0; 1789 } 1790 1791 /** 1792 * arm_coherent_iommu_map_sg - map a set of SG buffers for streaming mode DMA 1793 * @dev: valid struct device pointer 1794 * @sg: list of buffers 1795 * @nents: number of buffers to map 1796 * @dir: DMA transfer direction 1797 * 1798 * Map a set of i/o coherent buffers described by scatterlist in streaming 1799 * mode for DMA. The scatter gather list elements are merged together (if 1800 * possible) and tagged with the appropriate dma address and length. They are 1801 * obtained via sg_dma_{address,length}. 1802 */ 1803 int arm_coherent_iommu_map_sg(struct device *dev, struct scatterlist *sg, 1804 int nents, enum dma_data_direction dir, unsigned long attrs) 1805 { 1806 return __iommu_map_sg(dev, sg, nents, dir, attrs, true); 1807 } 1808 1809 /** 1810 * arm_iommu_map_sg - map a set of SG buffers for streaming mode DMA 1811 * @dev: valid struct device pointer 1812 * @sg: list of buffers 1813 * @nents: number of buffers to map 1814 * @dir: DMA transfer direction 1815 * 1816 * Map a set of buffers described by scatterlist in streaming mode for DMA. 1817 * The scatter gather list elements are merged together (if possible) and 1818 * tagged with the appropriate dma address and length. They are obtained via 1819 * sg_dma_{address,length}. 1820 */ 1821 int arm_iommu_map_sg(struct device *dev, struct scatterlist *sg, 1822 int nents, enum dma_data_direction dir, unsigned long attrs) 1823 { 1824 return __iommu_map_sg(dev, sg, nents, dir, attrs, false); 1825 } 1826 1827 static void __iommu_unmap_sg(struct device *dev, struct scatterlist *sg, 1828 int nents, enum dma_data_direction dir, 1829 unsigned long attrs, bool is_coherent) 1830 { 1831 struct scatterlist *s; 1832 int i; 1833 1834 for_each_sg(sg, s, nents, i) { 1835 if (sg_dma_len(s)) 1836 __iommu_remove_mapping(dev, sg_dma_address(s), 1837 sg_dma_len(s)); 1838 if (!is_coherent && (attrs & DMA_ATTR_SKIP_CPU_SYNC) == 0) 1839 __dma_page_dev_to_cpu(sg_page(s), s->offset, 1840 s->length, dir); 1841 } 1842 } 1843 1844 /** 1845 * arm_coherent_iommu_unmap_sg - unmap a set of SG buffers mapped by dma_map_sg 1846 * @dev: valid struct device pointer 1847 * @sg: list of buffers 1848 * @nents: number of buffers to unmap (same as was passed to dma_map_sg) 1849 * @dir: DMA transfer direction (same as was passed to dma_map_sg) 1850 * 1851 * Unmap a set of streaming mode DMA translations. Again, CPU access 1852 * rules concerning calls here are the same as for dma_unmap_single(). 1853 */ 1854 void arm_coherent_iommu_unmap_sg(struct device *dev, struct scatterlist *sg, 1855 int nents, enum dma_data_direction dir, 1856 unsigned long attrs) 1857 { 1858 __iommu_unmap_sg(dev, sg, nents, dir, attrs, true); 1859 } 1860 1861 /** 1862 * arm_iommu_unmap_sg - unmap a set of SG buffers mapped by dma_map_sg 1863 * @dev: valid struct device pointer 1864 * @sg: list of buffers 1865 * @nents: number of buffers to unmap (same as was passed to dma_map_sg) 1866 * @dir: DMA transfer direction (same as was passed to dma_map_sg) 1867 * 1868 * Unmap a set of streaming mode DMA translations. Again, CPU access 1869 * rules concerning calls here are the same as for dma_unmap_single(). 1870 */ 1871 void arm_iommu_unmap_sg(struct device *dev, struct scatterlist *sg, int nents, 1872 enum dma_data_direction dir, 1873 unsigned long attrs) 1874 { 1875 __iommu_unmap_sg(dev, sg, nents, dir, attrs, false); 1876 } 1877 1878 /** 1879 * arm_iommu_sync_sg_for_cpu 1880 * @dev: valid struct device pointer 1881 * @sg: list of buffers 1882 * @nents: number of buffers to map (returned from dma_map_sg) 1883 * @dir: DMA transfer direction (same as was passed to dma_map_sg) 1884 */ 1885 void arm_iommu_sync_sg_for_cpu(struct device *dev, struct scatterlist *sg, 1886 int nents, enum dma_data_direction dir) 1887 { 1888 struct scatterlist *s; 1889 int i; 1890 1891 for_each_sg(sg, s, nents, i) 1892 __dma_page_dev_to_cpu(sg_page(s), s->offset, s->length, dir); 1893 1894 } 1895 1896 /** 1897 * arm_iommu_sync_sg_for_device 1898 * @dev: valid struct device pointer 1899 * @sg: list of buffers 1900 * @nents: number of buffers to map (returned from dma_map_sg) 1901 * @dir: DMA transfer direction (same as was passed to dma_map_sg) 1902 */ 1903 void arm_iommu_sync_sg_for_device(struct device *dev, struct scatterlist *sg, 1904 int nents, enum dma_data_direction dir) 1905 { 1906 struct scatterlist *s; 1907 int i; 1908 1909 for_each_sg(sg, s, nents, i) 1910 __dma_page_cpu_to_dev(sg_page(s), s->offset, s->length, dir); 1911 } 1912 1913 1914 /** 1915 * arm_coherent_iommu_map_page 1916 * @dev: valid struct device pointer 1917 * @page: page that buffer resides in 1918 * @offset: offset into page for start of buffer 1919 * @size: size of buffer to map 1920 * @dir: DMA transfer direction 1921 * 1922 * Coherent IOMMU aware version of arm_dma_map_page() 1923 */ 1924 static dma_addr_t arm_coherent_iommu_map_page(struct device *dev, struct page *page, 1925 unsigned long offset, size_t size, enum dma_data_direction dir, 1926 unsigned long attrs) 1927 { 1928 struct dma_iommu_mapping *mapping = to_dma_iommu_mapping(dev); 1929 dma_addr_t dma_addr; 1930 int ret, prot, len = PAGE_ALIGN(size + offset); 1931 1932 dma_addr = __alloc_iova(mapping, len); 1933 if (dma_addr == DMA_ERROR_CODE) 1934 return dma_addr; 1935 1936 prot = __dma_direction_to_prot(dir); 1937 1938 ret = iommu_map(mapping->domain, dma_addr, page_to_phys(page), len, prot); 1939 if (ret < 0) 1940 goto fail; 1941 1942 return dma_addr + offset; 1943 fail: 1944 __free_iova(mapping, dma_addr, len); 1945 return DMA_ERROR_CODE; 1946 } 1947 1948 /** 1949 * arm_iommu_map_page 1950 * @dev: valid struct device pointer 1951 * @page: page that buffer resides in 1952 * @offset: offset into page for start of buffer 1953 * @size: size of buffer to map 1954 * @dir: DMA transfer direction 1955 * 1956 * IOMMU aware version of arm_dma_map_page() 1957 */ 1958 static dma_addr_t arm_iommu_map_page(struct device *dev, struct page *page, 1959 unsigned long offset, size_t size, enum dma_data_direction dir, 1960 unsigned long attrs) 1961 { 1962 if ((attrs & DMA_ATTR_SKIP_CPU_SYNC) == 0) 1963 __dma_page_cpu_to_dev(page, offset, size, dir); 1964 1965 return arm_coherent_iommu_map_page(dev, page, offset, size, dir, attrs); 1966 } 1967 1968 /** 1969 * arm_coherent_iommu_unmap_page 1970 * @dev: valid struct device pointer 1971 * @handle: DMA address of buffer 1972 * @size: size of buffer (same as passed to dma_map_page) 1973 * @dir: DMA transfer direction (same as passed to dma_map_page) 1974 * 1975 * Coherent IOMMU aware version of arm_dma_unmap_page() 1976 */ 1977 static void arm_coherent_iommu_unmap_page(struct device *dev, dma_addr_t handle, 1978 size_t size, enum dma_data_direction dir, unsigned long attrs) 1979 { 1980 struct dma_iommu_mapping *mapping = to_dma_iommu_mapping(dev); 1981 dma_addr_t iova = handle & PAGE_MASK; 1982 int offset = handle & ~PAGE_MASK; 1983 int len = PAGE_ALIGN(size + offset); 1984 1985 if (!iova) 1986 return; 1987 1988 iommu_unmap(mapping->domain, iova, len); 1989 __free_iova(mapping, iova, len); 1990 } 1991 1992 /** 1993 * arm_iommu_unmap_page 1994 * @dev: valid struct device pointer 1995 * @handle: DMA address of buffer 1996 * @size: size of buffer (same as passed to dma_map_page) 1997 * @dir: DMA transfer direction (same as passed to dma_map_page) 1998 * 1999 * IOMMU aware version of arm_dma_unmap_page() 2000 */ 2001 static void arm_iommu_unmap_page(struct device *dev, dma_addr_t handle, 2002 size_t size, enum dma_data_direction dir, unsigned long attrs) 2003 { 2004 struct dma_iommu_mapping *mapping = to_dma_iommu_mapping(dev); 2005 dma_addr_t iova = handle & PAGE_MASK; 2006 struct page *page = phys_to_page(iommu_iova_to_phys(mapping->domain, iova)); 2007 int offset = handle & ~PAGE_MASK; 2008 int len = PAGE_ALIGN(size + offset); 2009 2010 if (!iova) 2011 return; 2012 2013 if ((attrs & DMA_ATTR_SKIP_CPU_SYNC) == 0) 2014 __dma_page_dev_to_cpu(page, offset, size, dir); 2015 2016 iommu_unmap(mapping->domain, iova, len); 2017 __free_iova(mapping, iova, len); 2018 } 2019 2020 /** 2021 * arm_iommu_map_resource - map a device resource for DMA 2022 * @dev: valid struct device pointer 2023 * @phys_addr: physical address of resource 2024 * @size: size of resource to map 2025 * @dir: DMA transfer direction 2026 */ 2027 static dma_addr_t arm_iommu_map_resource(struct device *dev, 2028 phys_addr_t phys_addr, size_t size, 2029 enum dma_data_direction dir, unsigned long attrs) 2030 { 2031 struct dma_iommu_mapping *mapping = to_dma_iommu_mapping(dev); 2032 dma_addr_t dma_addr; 2033 int ret, prot; 2034 phys_addr_t addr = phys_addr & PAGE_MASK; 2035 unsigned int offset = phys_addr & ~PAGE_MASK; 2036 size_t len = PAGE_ALIGN(size + offset); 2037 2038 dma_addr = __alloc_iova(mapping, len); 2039 if (dma_addr == DMA_ERROR_CODE) 2040 return dma_addr; 2041 2042 prot = __dma_direction_to_prot(dir) | IOMMU_MMIO; 2043 2044 ret = iommu_map(mapping->domain, dma_addr, addr, len, prot); 2045 if (ret < 0) 2046 goto fail; 2047 2048 return dma_addr + offset; 2049 fail: 2050 __free_iova(mapping, dma_addr, len); 2051 return DMA_ERROR_CODE; 2052 } 2053 2054 /** 2055 * arm_iommu_unmap_resource - unmap a device DMA resource 2056 * @dev: valid struct device pointer 2057 * @dma_handle: DMA address to resource 2058 * @size: size of resource to map 2059 * @dir: DMA transfer direction 2060 */ 2061 static void arm_iommu_unmap_resource(struct device *dev, dma_addr_t dma_handle, 2062 size_t size, enum dma_data_direction dir, 2063 unsigned long attrs) 2064 { 2065 struct dma_iommu_mapping *mapping = to_dma_iommu_mapping(dev); 2066 dma_addr_t iova = dma_handle & PAGE_MASK; 2067 unsigned int offset = dma_handle & ~PAGE_MASK; 2068 size_t len = PAGE_ALIGN(size + offset); 2069 2070 if (!iova) 2071 return; 2072 2073 iommu_unmap(mapping->domain, iova, len); 2074 __free_iova(mapping, iova, len); 2075 } 2076 2077 static void arm_iommu_sync_single_for_cpu(struct device *dev, 2078 dma_addr_t handle, size_t size, enum dma_data_direction dir) 2079 { 2080 struct dma_iommu_mapping *mapping = to_dma_iommu_mapping(dev); 2081 dma_addr_t iova = handle & PAGE_MASK; 2082 struct page *page = phys_to_page(iommu_iova_to_phys(mapping->domain, iova)); 2083 unsigned int offset = handle & ~PAGE_MASK; 2084 2085 if (!iova) 2086 return; 2087 2088 __dma_page_dev_to_cpu(page, offset, size, dir); 2089 } 2090 2091 static void arm_iommu_sync_single_for_device(struct device *dev, 2092 dma_addr_t handle, size_t size, enum dma_data_direction dir) 2093 { 2094 struct dma_iommu_mapping *mapping = to_dma_iommu_mapping(dev); 2095 dma_addr_t iova = handle & PAGE_MASK; 2096 struct page *page = phys_to_page(iommu_iova_to_phys(mapping->domain, iova)); 2097 unsigned int offset = handle & ~PAGE_MASK; 2098 2099 if (!iova) 2100 return; 2101 2102 __dma_page_cpu_to_dev(page, offset, size, dir); 2103 } 2104 2105 struct dma_map_ops iommu_ops = { 2106 .alloc = arm_iommu_alloc_attrs, 2107 .free = arm_iommu_free_attrs, 2108 .mmap = arm_iommu_mmap_attrs, 2109 .get_sgtable = arm_iommu_get_sgtable, 2110 2111 .map_page = arm_iommu_map_page, 2112 .unmap_page = arm_iommu_unmap_page, 2113 .sync_single_for_cpu = arm_iommu_sync_single_for_cpu, 2114 .sync_single_for_device = arm_iommu_sync_single_for_device, 2115 2116 .map_sg = arm_iommu_map_sg, 2117 .unmap_sg = arm_iommu_unmap_sg, 2118 .sync_sg_for_cpu = arm_iommu_sync_sg_for_cpu, 2119 .sync_sg_for_device = arm_iommu_sync_sg_for_device, 2120 2121 .map_resource = arm_iommu_map_resource, 2122 .unmap_resource = arm_iommu_unmap_resource, 2123 }; 2124 2125 struct dma_map_ops iommu_coherent_ops = { 2126 .alloc = arm_coherent_iommu_alloc_attrs, 2127 .free = arm_coherent_iommu_free_attrs, 2128 .mmap = arm_coherent_iommu_mmap_attrs, 2129 .get_sgtable = arm_iommu_get_sgtable, 2130 2131 .map_page = arm_coherent_iommu_map_page, 2132 .unmap_page = arm_coherent_iommu_unmap_page, 2133 2134 .map_sg = arm_coherent_iommu_map_sg, 2135 .unmap_sg = arm_coherent_iommu_unmap_sg, 2136 2137 .map_resource = arm_iommu_map_resource, 2138 .unmap_resource = arm_iommu_unmap_resource, 2139 }; 2140 2141 /** 2142 * arm_iommu_create_mapping 2143 * @bus: pointer to the bus holding the client device (for IOMMU calls) 2144 * @base: start address of the valid IO address space 2145 * @size: maximum size of the valid IO address space 2146 * 2147 * Creates a mapping structure which holds information about used/unused 2148 * IO address ranges, which is required to perform memory allocation and 2149 * mapping with IOMMU aware functions. 2150 * 2151 * The client device need to be attached to the mapping with 2152 * arm_iommu_attach_device function. 2153 */ 2154 struct dma_iommu_mapping * 2155 arm_iommu_create_mapping(struct bus_type *bus, dma_addr_t base, u64 size) 2156 { 2157 unsigned int bits = size >> PAGE_SHIFT; 2158 unsigned int bitmap_size = BITS_TO_LONGS(bits) * sizeof(long); 2159 struct dma_iommu_mapping *mapping; 2160 int extensions = 1; 2161 int err = -ENOMEM; 2162 2163 /* currently only 32-bit DMA address space is supported */ 2164 if (size > DMA_BIT_MASK(32) + 1) 2165 return ERR_PTR(-ERANGE); 2166 2167 if (!bitmap_size) 2168 return ERR_PTR(-EINVAL); 2169 2170 if (bitmap_size > PAGE_SIZE) { 2171 extensions = bitmap_size / PAGE_SIZE; 2172 bitmap_size = PAGE_SIZE; 2173 } 2174 2175 mapping = kzalloc(sizeof(struct dma_iommu_mapping), GFP_KERNEL); 2176 if (!mapping) 2177 goto err; 2178 2179 mapping->bitmap_size = bitmap_size; 2180 mapping->bitmaps = kzalloc(extensions * sizeof(unsigned long *), 2181 GFP_KERNEL); 2182 if (!mapping->bitmaps) 2183 goto err2; 2184 2185 mapping->bitmaps[0] = kzalloc(bitmap_size, GFP_KERNEL); 2186 if (!mapping->bitmaps[0]) 2187 goto err3; 2188 2189 mapping->nr_bitmaps = 1; 2190 mapping->extensions = extensions; 2191 mapping->base = base; 2192 mapping->bits = BITS_PER_BYTE * bitmap_size; 2193 2194 spin_lock_init(&mapping->lock); 2195 2196 mapping->domain = iommu_domain_alloc(bus); 2197 if (!mapping->domain) 2198 goto err4; 2199 2200 kref_init(&mapping->kref); 2201 return mapping; 2202 err4: 2203 kfree(mapping->bitmaps[0]); 2204 err3: 2205 kfree(mapping->bitmaps); 2206 err2: 2207 kfree(mapping); 2208 err: 2209 return ERR_PTR(err); 2210 } 2211 EXPORT_SYMBOL_GPL(arm_iommu_create_mapping); 2212 2213 static void release_iommu_mapping(struct kref *kref) 2214 { 2215 int i; 2216 struct dma_iommu_mapping *mapping = 2217 container_of(kref, struct dma_iommu_mapping, kref); 2218 2219 iommu_domain_free(mapping->domain); 2220 for (i = 0; i < mapping->nr_bitmaps; i++) 2221 kfree(mapping->bitmaps[i]); 2222 kfree(mapping->bitmaps); 2223 kfree(mapping); 2224 } 2225 2226 static int extend_iommu_mapping(struct dma_iommu_mapping *mapping) 2227 { 2228 int next_bitmap; 2229 2230 if (mapping->nr_bitmaps >= mapping->extensions) 2231 return -EINVAL; 2232 2233 next_bitmap = mapping->nr_bitmaps; 2234 mapping->bitmaps[next_bitmap] = kzalloc(mapping->bitmap_size, 2235 GFP_ATOMIC); 2236 if (!mapping->bitmaps[next_bitmap]) 2237 return -ENOMEM; 2238 2239 mapping->nr_bitmaps++; 2240 2241 return 0; 2242 } 2243 2244 void arm_iommu_release_mapping(struct dma_iommu_mapping *mapping) 2245 { 2246 if (mapping) 2247 kref_put(&mapping->kref, release_iommu_mapping); 2248 } 2249 EXPORT_SYMBOL_GPL(arm_iommu_release_mapping); 2250 2251 static int __arm_iommu_attach_device(struct device *dev, 2252 struct dma_iommu_mapping *mapping) 2253 { 2254 int err; 2255 2256 err = iommu_attach_device(mapping->domain, dev); 2257 if (err) 2258 return err; 2259 2260 kref_get(&mapping->kref); 2261 to_dma_iommu_mapping(dev) = mapping; 2262 2263 pr_debug("Attached IOMMU controller to %s device.\n", dev_name(dev)); 2264 return 0; 2265 } 2266 2267 /** 2268 * arm_iommu_attach_device 2269 * @dev: valid struct device pointer 2270 * @mapping: io address space mapping structure (returned from 2271 * arm_iommu_create_mapping) 2272 * 2273 * Attaches specified io address space mapping to the provided device. 2274 * This replaces the dma operations (dma_map_ops pointer) with the 2275 * IOMMU aware version. 2276 * 2277 * More than one client might be attached to the same io address space 2278 * mapping. 2279 */ 2280 int arm_iommu_attach_device(struct device *dev, 2281 struct dma_iommu_mapping *mapping) 2282 { 2283 int err; 2284 2285 err = __arm_iommu_attach_device(dev, mapping); 2286 if (err) 2287 return err; 2288 2289 set_dma_ops(dev, &iommu_ops); 2290 return 0; 2291 } 2292 EXPORT_SYMBOL_GPL(arm_iommu_attach_device); 2293 2294 static void __arm_iommu_detach_device(struct device *dev) 2295 { 2296 struct dma_iommu_mapping *mapping; 2297 2298 mapping = to_dma_iommu_mapping(dev); 2299 if (!mapping) { 2300 dev_warn(dev, "Not attached\n"); 2301 return; 2302 } 2303 2304 iommu_detach_device(mapping->domain, dev); 2305 kref_put(&mapping->kref, release_iommu_mapping); 2306 to_dma_iommu_mapping(dev) = NULL; 2307 2308 pr_debug("Detached IOMMU controller from %s device.\n", dev_name(dev)); 2309 } 2310 2311 /** 2312 * arm_iommu_detach_device 2313 * @dev: valid struct device pointer 2314 * 2315 * Detaches the provided device from a previously attached map. 2316 * This voids the dma operations (dma_map_ops pointer) 2317 */ 2318 void arm_iommu_detach_device(struct device *dev) 2319 { 2320 __arm_iommu_detach_device(dev); 2321 set_dma_ops(dev, NULL); 2322 } 2323 EXPORT_SYMBOL_GPL(arm_iommu_detach_device); 2324 2325 static struct dma_map_ops *arm_get_iommu_dma_map_ops(bool coherent) 2326 { 2327 return coherent ? &iommu_coherent_ops : &iommu_ops; 2328 } 2329 2330 static bool arm_setup_iommu_dma_ops(struct device *dev, u64 dma_base, u64 size, 2331 const struct iommu_ops *iommu) 2332 { 2333 struct dma_iommu_mapping *mapping; 2334 2335 if (!iommu) 2336 return false; 2337 2338 mapping = arm_iommu_create_mapping(dev->bus, dma_base, size); 2339 if (IS_ERR(mapping)) { 2340 pr_warn("Failed to create %llu-byte IOMMU mapping for device %s\n", 2341 size, dev_name(dev)); 2342 return false; 2343 } 2344 2345 if (__arm_iommu_attach_device(dev, mapping)) { 2346 pr_warn("Failed to attached device %s to IOMMU_mapping\n", 2347 dev_name(dev)); 2348 arm_iommu_release_mapping(mapping); 2349 return false; 2350 } 2351 2352 return true; 2353 } 2354 2355 static void arm_teardown_iommu_dma_ops(struct device *dev) 2356 { 2357 struct dma_iommu_mapping *mapping = to_dma_iommu_mapping(dev); 2358 2359 if (!mapping) 2360 return; 2361 2362 __arm_iommu_detach_device(dev); 2363 arm_iommu_release_mapping(mapping); 2364 } 2365 2366 #else 2367 2368 static bool arm_setup_iommu_dma_ops(struct device *dev, u64 dma_base, u64 size, 2369 const struct iommu_ops *iommu) 2370 { 2371 return false; 2372 } 2373 2374 static void arm_teardown_iommu_dma_ops(struct device *dev) { } 2375 2376 #define arm_get_iommu_dma_map_ops arm_get_dma_map_ops 2377 2378 #endif /* CONFIG_ARM_DMA_USE_IOMMU */ 2379 2380 static struct dma_map_ops *arm_get_dma_map_ops(bool coherent) 2381 { 2382 return coherent ? &arm_coherent_dma_ops : &arm_dma_ops; 2383 } 2384 2385 void arch_setup_dma_ops(struct device *dev, u64 dma_base, u64 size, 2386 const struct iommu_ops *iommu, bool coherent) 2387 { 2388 struct dma_map_ops *dma_ops; 2389 2390 dev->archdata.dma_coherent = coherent; 2391 if (arm_setup_iommu_dma_ops(dev, dma_base, size, iommu)) 2392 dma_ops = arm_get_iommu_dma_map_ops(coherent); 2393 else 2394 dma_ops = arm_get_dma_map_ops(coherent); 2395 2396 set_dma_ops(dev, dma_ops); 2397 } 2398 2399 void arch_teardown_dma_ops(struct device *dev) 2400 { 2401 arm_teardown_iommu_dma_ops(dev); 2402 } 2403