1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * arch-independent dma-mapping routines 4 * 5 * Copyright (c) 2006 SUSE Linux Products GmbH 6 * Copyright (c) 2006 Tejun Heo <teheo@suse.de> 7 */ 8 #include <linux/memblock.h> /* for max_pfn */ 9 #include <linux/acpi.h> 10 #include <linux/dma-map-ops.h> 11 #include <linux/export.h> 12 #include <linux/gfp.h> 13 #include <linux/of_device.h> 14 #include <linux/slab.h> 15 #include <linux/vmalloc.h> 16 #include "debug.h" 17 #include "direct.h" 18 19 bool dma_default_coherent; 20 21 /* 22 * Managed DMA API 23 */ 24 struct dma_devres { 25 size_t size; 26 void *vaddr; 27 dma_addr_t dma_handle; 28 unsigned long attrs; 29 }; 30 31 static void dmam_release(struct device *dev, void *res) 32 { 33 struct dma_devres *this = res; 34 35 dma_free_attrs(dev, this->size, this->vaddr, this->dma_handle, 36 this->attrs); 37 } 38 39 static int dmam_match(struct device *dev, void *res, void *match_data) 40 { 41 struct dma_devres *this = res, *match = match_data; 42 43 if (this->vaddr == match->vaddr) { 44 WARN_ON(this->size != match->size || 45 this->dma_handle != match->dma_handle); 46 return 1; 47 } 48 return 0; 49 } 50 51 /** 52 * dmam_free_coherent - Managed dma_free_coherent() 53 * @dev: Device to free coherent memory for 54 * @size: Size of allocation 55 * @vaddr: Virtual address of the memory to free 56 * @dma_handle: DMA handle of the memory to free 57 * 58 * Managed dma_free_coherent(). 59 */ 60 void dmam_free_coherent(struct device *dev, size_t size, void *vaddr, 61 dma_addr_t dma_handle) 62 { 63 struct dma_devres match_data = { size, vaddr, dma_handle }; 64 65 dma_free_coherent(dev, size, vaddr, dma_handle); 66 WARN_ON(devres_destroy(dev, dmam_release, dmam_match, &match_data)); 67 } 68 EXPORT_SYMBOL(dmam_free_coherent); 69 70 /** 71 * dmam_alloc_attrs - Managed dma_alloc_attrs() 72 * @dev: Device to allocate non_coherent memory for 73 * @size: Size of allocation 74 * @dma_handle: Out argument for allocated DMA handle 75 * @gfp: Allocation flags 76 * @attrs: Flags in the DMA_ATTR_* namespace. 77 * 78 * Managed dma_alloc_attrs(). Memory allocated using this function will be 79 * automatically released on driver detach. 80 * 81 * RETURNS: 82 * Pointer to allocated memory on success, NULL on failure. 83 */ 84 void *dmam_alloc_attrs(struct device *dev, size_t size, dma_addr_t *dma_handle, 85 gfp_t gfp, unsigned long attrs) 86 { 87 struct dma_devres *dr; 88 void *vaddr; 89 90 dr = devres_alloc(dmam_release, sizeof(*dr), gfp); 91 if (!dr) 92 return NULL; 93 94 vaddr = dma_alloc_attrs(dev, size, dma_handle, gfp, attrs); 95 if (!vaddr) { 96 devres_free(dr); 97 return NULL; 98 } 99 100 dr->vaddr = vaddr; 101 dr->dma_handle = *dma_handle; 102 dr->size = size; 103 dr->attrs = attrs; 104 105 devres_add(dev, dr); 106 107 return vaddr; 108 } 109 EXPORT_SYMBOL(dmam_alloc_attrs); 110 111 static bool dma_go_direct(struct device *dev, dma_addr_t mask, 112 const struct dma_map_ops *ops) 113 { 114 if (likely(!ops)) 115 return true; 116 #ifdef CONFIG_DMA_OPS_BYPASS 117 if (dev->dma_ops_bypass) 118 return min_not_zero(mask, dev->bus_dma_limit) >= 119 dma_direct_get_required_mask(dev); 120 #endif 121 return false; 122 } 123 124 125 /* 126 * Check if the devices uses a direct mapping for streaming DMA operations. 127 * This allows IOMMU drivers to set a bypass mode if the DMA mask is large 128 * enough. 129 */ 130 static inline bool dma_alloc_direct(struct device *dev, 131 const struct dma_map_ops *ops) 132 { 133 return dma_go_direct(dev, dev->coherent_dma_mask, ops); 134 } 135 136 static inline bool dma_map_direct(struct device *dev, 137 const struct dma_map_ops *ops) 138 { 139 return dma_go_direct(dev, *dev->dma_mask, ops); 140 } 141 142 dma_addr_t dma_map_page_attrs(struct device *dev, struct page *page, 143 size_t offset, size_t size, enum dma_data_direction dir, 144 unsigned long attrs) 145 { 146 const struct dma_map_ops *ops = get_dma_ops(dev); 147 dma_addr_t addr; 148 149 BUG_ON(!valid_dma_direction(dir)); 150 151 if (WARN_ON_ONCE(!dev->dma_mask)) 152 return DMA_MAPPING_ERROR; 153 154 if (dma_map_direct(dev, ops) || 155 arch_dma_map_page_direct(dev, page_to_phys(page) + offset + size)) 156 addr = dma_direct_map_page(dev, page, offset, size, dir, attrs); 157 else 158 addr = ops->map_page(dev, page, offset, size, dir, attrs); 159 debug_dma_map_page(dev, page, offset, size, dir, addr, attrs); 160 161 return addr; 162 } 163 EXPORT_SYMBOL(dma_map_page_attrs); 164 165 void dma_unmap_page_attrs(struct device *dev, dma_addr_t addr, size_t size, 166 enum dma_data_direction dir, unsigned long attrs) 167 { 168 const struct dma_map_ops *ops = get_dma_ops(dev); 169 170 BUG_ON(!valid_dma_direction(dir)); 171 if (dma_map_direct(dev, ops) || 172 arch_dma_unmap_page_direct(dev, addr + size)) 173 dma_direct_unmap_page(dev, addr, size, dir, attrs); 174 else if (ops->unmap_page) 175 ops->unmap_page(dev, addr, size, dir, attrs); 176 debug_dma_unmap_page(dev, addr, size, dir); 177 } 178 EXPORT_SYMBOL(dma_unmap_page_attrs); 179 180 static int __dma_map_sg_attrs(struct device *dev, struct scatterlist *sg, 181 int nents, enum dma_data_direction dir, unsigned long attrs) 182 { 183 const struct dma_map_ops *ops = get_dma_ops(dev); 184 int ents; 185 186 BUG_ON(!valid_dma_direction(dir)); 187 188 if (WARN_ON_ONCE(!dev->dma_mask)) 189 return 0; 190 191 if (dma_map_direct(dev, ops) || 192 arch_dma_map_sg_direct(dev, sg, nents)) 193 ents = dma_direct_map_sg(dev, sg, nents, dir, attrs); 194 else 195 ents = ops->map_sg(dev, sg, nents, dir, attrs); 196 197 if (ents > 0) 198 debug_dma_map_sg(dev, sg, nents, ents, dir, attrs); 199 else if (WARN_ON_ONCE(ents != -EINVAL && ents != -ENOMEM && 200 ents != -EIO)) 201 return -EIO; 202 203 return ents; 204 } 205 206 /** 207 * dma_map_sg_attrs - Map the given buffer for DMA 208 * @dev: The device for which to perform the DMA operation 209 * @sg: The sg_table object describing the buffer 210 * @nents: Number of entries to map 211 * @dir: DMA direction 212 * @attrs: Optional DMA attributes for the map operation 213 * 214 * Maps a buffer described by a scatterlist passed in the sg argument with 215 * nents segments for the @dir DMA operation by the @dev device. 216 * 217 * Returns the number of mapped entries (which can be less than nents) 218 * on success. Zero is returned for any error. 219 * 220 * dma_unmap_sg_attrs() should be used to unmap the buffer with the 221 * original sg and original nents (not the value returned by this funciton). 222 */ 223 unsigned int dma_map_sg_attrs(struct device *dev, struct scatterlist *sg, 224 int nents, enum dma_data_direction dir, unsigned long attrs) 225 { 226 int ret; 227 228 ret = __dma_map_sg_attrs(dev, sg, nents, dir, attrs); 229 if (ret < 0) 230 return 0; 231 return ret; 232 } 233 EXPORT_SYMBOL(dma_map_sg_attrs); 234 235 /** 236 * dma_map_sgtable - Map the given buffer for DMA 237 * @dev: The device for which to perform the DMA operation 238 * @sgt: The sg_table object describing the buffer 239 * @dir: DMA direction 240 * @attrs: Optional DMA attributes for the map operation 241 * 242 * Maps a buffer described by a scatterlist stored in the given sg_table 243 * object for the @dir DMA operation by the @dev device. After success, the 244 * ownership for the buffer is transferred to the DMA domain. One has to 245 * call dma_sync_sgtable_for_cpu() or dma_unmap_sgtable() to move the 246 * ownership of the buffer back to the CPU domain before touching the 247 * buffer by the CPU. 248 * 249 * Returns 0 on success or a negative error code on error. The following 250 * error codes are supported with the given meaning: 251 * 252 * -EINVAL An invalid argument, unaligned access or other error 253 * in usage. Will not succeed if retried. 254 * -ENOMEM Insufficient resources (like memory or IOVA space) to 255 * complete the mapping. Should succeed if retried later. 256 * -EIO Legacy error code with an unknown meaning. eg. this is 257 * returned if a lower level call returned DMA_MAPPING_ERROR. 258 */ 259 int dma_map_sgtable(struct device *dev, struct sg_table *sgt, 260 enum dma_data_direction dir, unsigned long attrs) 261 { 262 int nents; 263 264 nents = __dma_map_sg_attrs(dev, sgt->sgl, sgt->orig_nents, dir, attrs); 265 if (nents < 0) 266 return nents; 267 sgt->nents = nents; 268 return 0; 269 } 270 EXPORT_SYMBOL_GPL(dma_map_sgtable); 271 272 void dma_unmap_sg_attrs(struct device *dev, struct scatterlist *sg, 273 int nents, enum dma_data_direction dir, 274 unsigned long attrs) 275 { 276 const struct dma_map_ops *ops = get_dma_ops(dev); 277 278 BUG_ON(!valid_dma_direction(dir)); 279 debug_dma_unmap_sg(dev, sg, nents, dir); 280 if (dma_map_direct(dev, ops) || 281 arch_dma_unmap_sg_direct(dev, sg, nents)) 282 dma_direct_unmap_sg(dev, sg, nents, dir, attrs); 283 else if (ops->unmap_sg) 284 ops->unmap_sg(dev, sg, nents, dir, attrs); 285 } 286 EXPORT_SYMBOL(dma_unmap_sg_attrs); 287 288 dma_addr_t dma_map_resource(struct device *dev, phys_addr_t phys_addr, 289 size_t size, enum dma_data_direction dir, unsigned long attrs) 290 { 291 const struct dma_map_ops *ops = get_dma_ops(dev); 292 dma_addr_t addr = DMA_MAPPING_ERROR; 293 294 BUG_ON(!valid_dma_direction(dir)); 295 296 if (WARN_ON_ONCE(!dev->dma_mask)) 297 return DMA_MAPPING_ERROR; 298 299 if (dma_map_direct(dev, ops)) 300 addr = dma_direct_map_resource(dev, phys_addr, size, dir, attrs); 301 else if (ops->map_resource) 302 addr = ops->map_resource(dev, phys_addr, size, dir, attrs); 303 304 debug_dma_map_resource(dev, phys_addr, size, dir, addr, attrs); 305 return addr; 306 } 307 EXPORT_SYMBOL(dma_map_resource); 308 309 void dma_unmap_resource(struct device *dev, dma_addr_t addr, size_t size, 310 enum dma_data_direction dir, unsigned long attrs) 311 { 312 const struct dma_map_ops *ops = get_dma_ops(dev); 313 314 BUG_ON(!valid_dma_direction(dir)); 315 if (!dma_map_direct(dev, ops) && ops->unmap_resource) 316 ops->unmap_resource(dev, addr, size, dir, attrs); 317 debug_dma_unmap_resource(dev, addr, size, dir); 318 } 319 EXPORT_SYMBOL(dma_unmap_resource); 320 321 void dma_sync_single_for_cpu(struct device *dev, dma_addr_t addr, size_t size, 322 enum dma_data_direction dir) 323 { 324 const struct dma_map_ops *ops = get_dma_ops(dev); 325 326 BUG_ON(!valid_dma_direction(dir)); 327 if (dma_map_direct(dev, ops)) 328 dma_direct_sync_single_for_cpu(dev, addr, size, dir); 329 else if (ops->sync_single_for_cpu) 330 ops->sync_single_for_cpu(dev, addr, size, dir); 331 debug_dma_sync_single_for_cpu(dev, addr, size, dir); 332 } 333 EXPORT_SYMBOL(dma_sync_single_for_cpu); 334 335 void dma_sync_single_for_device(struct device *dev, dma_addr_t addr, 336 size_t size, enum dma_data_direction dir) 337 { 338 const struct dma_map_ops *ops = get_dma_ops(dev); 339 340 BUG_ON(!valid_dma_direction(dir)); 341 if (dma_map_direct(dev, ops)) 342 dma_direct_sync_single_for_device(dev, addr, size, dir); 343 else if (ops->sync_single_for_device) 344 ops->sync_single_for_device(dev, addr, size, dir); 345 debug_dma_sync_single_for_device(dev, addr, size, dir); 346 } 347 EXPORT_SYMBOL(dma_sync_single_for_device); 348 349 void dma_sync_sg_for_cpu(struct device *dev, struct scatterlist *sg, 350 int nelems, enum dma_data_direction dir) 351 { 352 const struct dma_map_ops *ops = get_dma_ops(dev); 353 354 BUG_ON(!valid_dma_direction(dir)); 355 if (dma_map_direct(dev, ops)) 356 dma_direct_sync_sg_for_cpu(dev, sg, nelems, dir); 357 else if (ops->sync_sg_for_cpu) 358 ops->sync_sg_for_cpu(dev, sg, nelems, dir); 359 debug_dma_sync_sg_for_cpu(dev, sg, nelems, dir); 360 } 361 EXPORT_SYMBOL(dma_sync_sg_for_cpu); 362 363 void dma_sync_sg_for_device(struct device *dev, struct scatterlist *sg, 364 int nelems, enum dma_data_direction dir) 365 { 366 const struct dma_map_ops *ops = get_dma_ops(dev); 367 368 BUG_ON(!valid_dma_direction(dir)); 369 if (dma_map_direct(dev, ops)) 370 dma_direct_sync_sg_for_device(dev, sg, nelems, dir); 371 else if (ops->sync_sg_for_device) 372 ops->sync_sg_for_device(dev, sg, nelems, dir); 373 debug_dma_sync_sg_for_device(dev, sg, nelems, dir); 374 } 375 EXPORT_SYMBOL(dma_sync_sg_for_device); 376 377 /* 378 * The whole dma_get_sgtable() idea is fundamentally unsafe - it seems 379 * that the intention is to allow exporting memory allocated via the 380 * coherent DMA APIs through the dma_buf API, which only accepts a 381 * scattertable. This presents a couple of problems: 382 * 1. Not all memory allocated via the coherent DMA APIs is backed by 383 * a struct page 384 * 2. Passing coherent DMA memory into the streaming APIs is not allowed 385 * as we will try to flush the memory through a different alias to that 386 * actually being used (and the flushes are redundant.) 387 */ 388 int dma_get_sgtable_attrs(struct device *dev, struct sg_table *sgt, 389 void *cpu_addr, dma_addr_t dma_addr, size_t size, 390 unsigned long attrs) 391 { 392 const struct dma_map_ops *ops = get_dma_ops(dev); 393 394 if (dma_alloc_direct(dev, ops)) 395 return dma_direct_get_sgtable(dev, sgt, cpu_addr, dma_addr, 396 size, attrs); 397 if (!ops->get_sgtable) 398 return -ENXIO; 399 return ops->get_sgtable(dev, sgt, cpu_addr, dma_addr, size, attrs); 400 } 401 EXPORT_SYMBOL(dma_get_sgtable_attrs); 402 403 #ifdef CONFIG_MMU 404 /* 405 * Return the page attributes used for mapping dma_alloc_* memory, either in 406 * kernel space if remapping is needed, or to userspace through dma_mmap_*. 407 */ 408 pgprot_t dma_pgprot(struct device *dev, pgprot_t prot, unsigned long attrs) 409 { 410 if (force_dma_unencrypted(dev)) 411 prot = pgprot_decrypted(prot); 412 if (dev_is_dma_coherent(dev)) 413 return prot; 414 #ifdef CONFIG_ARCH_HAS_DMA_WRITE_COMBINE 415 if (attrs & DMA_ATTR_WRITE_COMBINE) 416 return pgprot_writecombine(prot); 417 #endif 418 return pgprot_dmacoherent(prot); 419 } 420 #endif /* CONFIG_MMU */ 421 422 /** 423 * dma_can_mmap - check if a given device supports dma_mmap_* 424 * @dev: device to check 425 * 426 * Returns %true if @dev supports dma_mmap_coherent() and dma_mmap_attrs() to 427 * map DMA allocations to userspace. 428 */ 429 bool dma_can_mmap(struct device *dev) 430 { 431 const struct dma_map_ops *ops = get_dma_ops(dev); 432 433 if (dma_alloc_direct(dev, ops)) 434 return dma_direct_can_mmap(dev); 435 return ops->mmap != NULL; 436 } 437 EXPORT_SYMBOL_GPL(dma_can_mmap); 438 439 /** 440 * dma_mmap_attrs - map a coherent DMA allocation into user space 441 * @dev: valid struct device pointer, or NULL for ISA and EISA-like devices 442 * @vma: vm_area_struct describing requested user mapping 443 * @cpu_addr: kernel CPU-view address returned from dma_alloc_attrs 444 * @dma_addr: device-view address returned from dma_alloc_attrs 445 * @size: size of memory originally requested in dma_alloc_attrs 446 * @attrs: attributes of mapping properties requested in dma_alloc_attrs 447 * 448 * Map a coherent DMA buffer previously allocated by dma_alloc_attrs into user 449 * space. The coherent DMA buffer must not be freed by the driver until the 450 * user space mapping has been released. 451 */ 452 int dma_mmap_attrs(struct device *dev, struct vm_area_struct *vma, 453 void *cpu_addr, dma_addr_t dma_addr, size_t size, 454 unsigned long attrs) 455 { 456 const struct dma_map_ops *ops = get_dma_ops(dev); 457 458 if (dma_alloc_direct(dev, ops)) 459 return dma_direct_mmap(dev, vma, cpu_addr, dma_addr, size, 460 attrs); 461 if (!ops->mmap) 462 return -ENXIO; 463 return ops->mmap(dev, vma, cpu_addr, dma_addr, size, attrs); 464 } 465 EXPORT_SYMBOL(dma_mmap_attrs); 466 467 u64 dma_get_required_mask(struct device *dev) 468 { 469 const struct dma_map_ops *ops = get_dma_ops(dev); 470 471 if (dma_alloc_direct(dev, ops)) 472 return dma_direct_get_required_mask(dev); 473 if (ops->get_required_mask) 474 return ops->get_required_mask(dev); 475 476 /* 477 * We require every DMA ops implementation to at least support a 32-bit 478 * DMA mask (and use bounce buffering if that isn't supported in 479 * hardware). As the direct mapping code has its own routine to 480 * actually report an optimal mask we default to 32-bit here as that 481 * is the right thing for most IOMMUs, and at least not actively 482 * harmful in general. 483 */ 484 return DMA_BIT_MASK(32); 485 } 486 EXPORT_SYMBOL_GPL(dma_get_required_mask); 487 488 void *dma_alloc_attrs(struct device *dev, size_t size, dma_addr_t *dma_handle, 489 gfp_t flag, unsigned long attrs) 490 { 491 const struct dma_map_ops *ops = get_dma_ops(dev); 492 void *cpu_addr; 493 494 WARN_ON_ONCE(!dev->coherent_dma_mask); 495 496 if (dma_alloc_from_dev_coherent(dev, size, dma_handle, &cpu_addr)) 497 return cpu_addr; 498 499 /* let the implementation decide on the zone to allocate from: */ 500 flag &= ~(__GFP_DMA | __GFP_DMA32 | __GFP_HIGHMEM); 501 502 if (dma_alloc_direct(dev, ops)) 503 cpu_addr = dma_direct_alloc(dev, size, dma_handle, flag, attrs); 504 else if (ops->alloc) 505 cpu_addr = ops->alloc(dev, size, dma_handle, flag, attrs); 506 else 507 return NULL; 508 509 debug_dma_alloc_coherent(dev, size, *dma_handle, cpu_addr, attrs); 510 return cpu_addr; 511 } 512 EXPORT_SYMBOL(dma_alloc_attrs); 513 514 void dma_free_attrs(struct device *dev, size_t size, void *cpu_addr, 515 dma_addr_t dma_handle, unsigned long attrs) 516 { 517 const struct dma_map_ops *ops = get_dma_ops(dev); 518 519 if (dma_release_from_dev_coherent(dev, get_order(size), cpu_addr)) 520 return; 521 /* 522 * On non-coherent platforms which implement DMA-coherent buffers via 523 * non-cacheable remaps, ops->free() may call vunmap(). Thus getting 524 * this far in IRQ context is a) at risk of a BUG_ON() or trying to 525 * sleep on some machines, and b) an indication that the driver is 526 * probably misusing the coherent API anyway. 527 */ 528 WARN_ON(irqs_disabled()); 529 530 if (!cpu_addr) 531 return; 532 533 debug_dma_free_coherent(dev, size, cpu_addr, dma_handle); 534 if (dma_alloc_direct(dev, ops)) 535 dma_direct_free(dev, size, cpu_addr, dma_handle, attrs); 536 else if (ops->free) 537 ops->free(dev, size, cpu_addr, dma_handle, attrs); 538 } 539 EXPORT_SYMBOL(dma_free_attrs); 540 541 static struct page *__dma_alloc_pages(struct device *dev, size_t size, 542 dma_addr_t *dma_handle, enum dma_data_direction dir, gfp_t gfp) 543 { 544 const struct dma_map_ops *ops = get_dma_ops(dev); 545 546 if (WARN_ON_ONCE(!dev->coherent_dma_mask)) 547 return NULL; 548 if (WARN_ON_ONCE(gfp & (__GFP_DMA | __GFP_DMA32 | __GFP_HIGHMEM))) 549 return NULL; 550 551 size = PAGE_ALIGN(size); 552 if (dma_alloc_direct(dev, ops)) 553 return dma_direct_alloc_pages(dev, size, dma_handle, dir, gfp); 554 if (!ops->alloc_pages) 555 return NULL; 556 return ops->alloc_pages(dev, size, dma_handle, dir, gfp); 557 } 558 559 struct page *dma_alloc_pages(struct device *dev, size_t size, 560 dma_addr_t *dma_handle, enum dma_data_direction dir, gfp_t gfp) 561 { 562 struct page *page = __dma_alloc_pages(dev, size, dma_handle, dir, gfp); 563 564 if (page) 565 debug_dma_map_page(dev, page, 0, size, dir, *dma_handle, 0); 566 return page; 567 } 568 EXPORT_SYMBOL_GPL(dma_alloc_pages); 569 570 static void __dma_free_pages(struct device *dev, size_t size, struct page *page, 571 dma_addr_t dma_handle, enum dma_data_direction dir) 572 { 573 const struct dma_map_ops *ops = get_dma_ops(dev); 574 575 size = PAGE_ALIGN(size); 576 if (dma_alloc_direct(dev, ops)) 577 dma_direct_free_pages(dev, size, page, dma_handle, dir); 578 else if (ops->free_pages) 579 ops->free_pages(dev, size, page, dma_handle, dir); 580 } 581 582 void dma_free_pages(struct device *dev, size_t size, struct page *page, 583 dma_addr_t dma_handle, enum dma_data_direction dir) 584 { 585 debug_dma_unmap_page(dev, dma_handle, size, dir); 586 __dma_free_pages(dev, size, page, dma_handle, dir); 587 } 588 EXPORT_SYMBOL_GPL(dma_free_pages); 589 590 int dma_mmap_pages(struct device *dev, struct vm_area_struct *vma, 591 size_t size, struct page *page) 592 { 593 unsigned long count = PAGE_ALIGN(size) >> PAGE_SHIFT; 594 595 if (vma->vm_pgoff >= count || vma_pages(vma) > count - vma->vm_pgoff) 596 return -ENXIO; 597 return remap_pfn_range(vma, vma->vm_start, 598 page_to_pfn(page) + vma->vm_pgoff, 599 vma_pages(vma) << PAGE_SHIFT, vma->vm_page_prot); 600 } 601 EXPORT_SYMBOL_GPL(dma_mmap_pages); 602 603 static struct sg_table *alloc_single_sgt(struct device *dev, size_t size, 604 enum dma_data_direction dir, gfp_t gfp) 605 { 606 struct sg_table *sgt; 607 struct page *page; 608 609 sgt = kmalloc(sizeof(*sgt), gfp); 610 if (!sgt) 611 return NULL; 612 if (sg_alloc_table(sgt, 1, gfp)) 613 goto out_free_sgt; 614 page = __dma_alloc_pages(dev, size, &sgt->sgl->dma_address, dir, gfp); 615 if (!page) 616 goto out_free_table; 617 sg_set_page(sgt->sgl, page, PAGE_ALIGN(size), 0); 618 sg_dma_len(sgt->sgl) = sgt->sgl->length; 619 return sgt; 620 out_free_table: 621 sg_free_table(sgt); 622 out_free_sgt: 623 kfree(sgt); 624 return NULL; 625 } 626 627 struct sg_table *dma_alloc_noncontiguous(struct device *dev, size_t size, 628 enum dma_data_direction dir, gfp_t gfp, unsigned long attrs) 629 { 630 const struct dma_map_ops *ops = get_dma_ops(dev); 631 struct sg_table *sgt; 632 633 if (WARN_ON_ONCE(attrs & ~DMA_ATTR_ALLOC_SINGLE_PAGES)) 634 return NULL; 635 636 if (ops && ops->alloc_noncontiguous) 637 sgt = ops->alloc_noncontiguous(dev, size, dir, gfp, attrs); 638 else 639 sgt = alloc_single_sgt(dev, size, dir, gfp); 640 641 if (sgt) { 642 sgt->nents = 1; 643 debug_dma_map_sg(dev, sgt->sgl, sgt->orig_nents, 1, dir, attrs); 644 } 645 return sgt; 646 } 647 EXPORT_SYMBOL_GPL(dma_alloc_noncontiguous); 648 649 static void free_single_sgt(struct device *dev, size_t size, 650 struct sg_table *sgt, enum dma_data_direction dir) 651 { 652 __dma_free_pages(dev, size, sg_page(sgt->sgl), sgt->sgl->dma_address, 653 dir); 654 sg_free_table(sgt); 655 kfree(sgt); 656 } 657 658 void dma_free_noncontiguous(struct device *dev, size_t size, 659 struct sg_table *sgt, enum dma_data_direction dir) 660 { 661 const struct dma_map_ops *ops = get_dma_ops(dev); 662 663 debug_dma_unmap_sg(dev, sgt->sgl, sgt->orig_nents, dir); 664 if (ops && ops->free_noncontiguous) 665 ops->free_noncontiguous(dev, size, sgt, dir); 666 else 667 free_single_sgt(dev, size, sgt, dir); 668 } 669 EXPORT_SYMBOL_GPL(dma_free_noncontiguous); 670 671 void *dma_vmap_noncontiguous(struct device *dev, size_t size, 672 struct sg_table *sgt) 673 { 674 const struct dma_map_ops *ops = get_dma_ops(dev); 675 unsigned long count = PAGE_ALIGN(size) >> PAGE_SHIFT; 676 677 if (ops && ops->alloc_noncontiguous) 678 return vmap(sgt_handle(sgt)->pages, count, VM_MAP, PAGE_KERNEL); 679 return page_address(sg_page(sgt->sgl)); 680 } 681 EXPORT_SYMBOL_GPL(dma_vmap_noncontiguous); 682 683 void dma_vunmap_noncontiguous(struct device *dev, void *vaddr) 684 { 685 const struct dma_map_ops *ops = get_dma_ops(dev); 686 687 if (ops && ops->alloc_noncontiguous) 688 vunmap(vaddr); 689 } 690 EXPORT_SYMBOL_GPL(dma_vunmap_noncontiguous); 691 692 int dma_mmap_noncontiguous(struct device *dev, struct vm_area_struct *vma, 693 size_t size, struct sg_table *sgt) 694 { 695 const struct dma_map_ops *ops = get_dma_ops(dev); 696 697 if (ops && ops->alloc_noncontiguous) { 698 unsigned long count = PAGE_ALIGN(size) >> PAGE_SHIFT; 699 700 if (vma->vm_pgoff >= count || 701 vma_pages(vma) > count - vma->vm_pgoff) 702 return -ENXIO; 703 return vm_map_pages(vma, sgt_handle(sgt)->pages, count); 704 } 705 return dma_mmap_pages(dev, vma, size, sg_page(sgt->sgl)); 706 } 707 EXPORT_SYMBOL_GPL(dma_mmap_noncontiguous); 708 709 int dma_supported(struct device *dev, u64 mask) 710 { 711 const struct dma_map_ops *ops = get_dma_ops(dev); 712 713 /* 714 * ->dma_supported sets the bypass flag, so we must always call 715 * into the method here unless the device is truly direct mapped. 716 */ 717 if (!ops) 718 return dma_direct_supported(dev, mask); 719 if (!ops->dma_supported) 720 return 1; 721 return ops->dma_supported(dev, mask); 722 } 723 EXPORT_SYMBOL(dma_supported); 724 725 #ifdef CONFIG_ARCH_HAS_DMA_SET_MASK 726 void arch_dma_set_mask(struct device *dev, u64 mask); 727 #else 728 #define arch_dma_set_mask(dev, mask) do { } while (0) 729 #endif 730 731 int dma_set_mask(struct device *dev, u64 mask) 732 { 733 /* 734 * Truncate the mask to the actually supported dma_addr_t width to 735 * avoid generating unsupportable addresses. 736 */ 737 mask = (dma_addr_t)mask; 738 739 if (!dev->dma_mask || !dma_supported(dev, mask)) 740 return -EIO; 741 742 arch_dma_set_mask(dev, mask); 743 *dev->dma_mask = mask; 744 return 0; 745 } 746 EXPORT_SYMBOL(dma_set_mask); 747 748 #ifndef CONFIG_ARCH_HAS_DMA_SET_COHERENT_MASK 749 int dma_set_coherent_mask(struct device *dev, u64 mask) 750 { 751 /* 752 * Truncate the mask to the actually supported dma_addr_t width to 753 * avoid generating unsupportable addresses. 754 */ 755 mask = (dma_addr_t)mask; 756 757 if (!dma_supported(dev, mask)) 758 return -EIO; 759 760 dev->coherent_dma_mask = mask; 761 return 0; 762 } 763 EXPORT_SYMBOL(dma_set_coherent_mask); 764 #endif 765 766 size_t dma_max_mapping_size(struct device *dev) 767 { 768 const struct dma_map_ops *ops = get_dma_ops(dev); 769 size_t size = SIZE_MAX; 770 771 if (dma_map_direct(dev, ops)) 772 size = dma_direct_max_mapping_size(dev); 773 else if (ops && ops->max_mapping_size) 774 size = ops->max_mapping_size(dev); 775 776 return size; 777 } 778 EXPORT_SYMBOL_GPL(dma_max_mapping_size); 779 780 bool dma_need_sync(struct device *dev, dma_addr_t dma_addr) 781 { 782 const struct dma_map_ops *ops = get_dma_ops(dev); 783 784 if (dma_map_direct(dev, ops)) 785 return dma_direct_need_sync(dev, dma_addr); 786 return ops->sync_single_for_cpu || ops->sync_single_for_device; 787 } 788 EXPORT_SYMBOL_GPL(dma_need_sync); 789 790 unsigned long dma_get_merge_boundary(struct device *dev) 791 { 792 const struct dma_map_ops *ops = get_dma_ops(dev); 793 794 if (!ops || !ops->get_merge_boundary) 795 return 0; /* can't merge */ 796 797 return ops->get_merge_boundary(dev); 798 } 799 EXPORT_SYMBOL_GPL(dma_get_merge_boundary); 800