1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * arch-independent dma-mapping routines 4 * 5 * Copyright (c) 2006 SUSE Linux Products GmbH 6 * Copyright (c) 2006 Tejun Heo <teheo@suse.de> 7 */ 8 #include <linux/memblock.h> /* for max_pfn */ 9 #include <linux/acpi.h> 10 #include <linux/dma-map-ops.h> 11 #include <linux/export.h> 12 #include <linux/gfp.h> 13 #include <linux/kmsan.h> 14 #include <linux/of_device.h> 15 #include <linux/slab.h> 16 #include <linux/vmalloc.h> 17 #include "debug.h" 18 #include "direct.h" 19 20 #if defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_DEVICE) || \ 21 defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_CPU) || \ 22 defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_CPU_ALL) 23 bool dma_default_coherent = IS_ENABLED(CONFIG_ARCH_DMA_DEFAULT_COHERENT); 24 #endif 25 26 /* 27 * Managed DMA API 28 */ 29 struct dma_devres { 30 size_t size; 31 void *vaddr; 32 dma_addr_t dma_handle; 33 unsigned long attrs; 34 }; 35 36 static void dmam_release(struct device *dev, void *res) 37 { 38 struct dma_devres *this = res; 39 40 dma_free_attrs(dev, this->size, this->vaddr, this->dma_handle, 41 this->attrs); 42 } 43 44 static int dmam_match(struct device *dev, void *res, void *match_data) 45 { 46 struct dma_devres *this = res, *match = match_data; 47 48 if (this->vaddr == match->vaddr) { 49 WARN_ON(this->size != match->size || 50 this->dma_handle != match->dma_handle); 51 return 1; 52 } 53 return 0; 54 } 55 56 /** 57 * dmam_free_coherent - Managed dma_free_coherent() 58 * @dev: Device to free coherent memory for 59 * @size: Size of allocation 60 * @vaddr: Virtual address of the memory to free 61 * @dma_handle: DMA handle of the memory to free 62 * 63 * Managed dma_free_coherent(). 64 */ 65 void dmam_free_coherent(struct device *dev, size_t size, void *vaddr, 66 dma_addr_t dma_handle) 67 { 68 struct dma_devres match_data = { size, vaddr, dma_handle }; 69 70 WARN_ON(devres_destroy(dev, dmam_release, dmam_match, &match_data)); 71 dma_free_coherent(dev, size, vaddr, dma_handle); 72 } 73 EXPORT_SYMBOL(dmam_free_coherent); 74 75 /** 76 * dmam_alloc_attrs - Managed dma_alloc_attrs() 77 * @dev: Device to allocate non_coherent memory for 78 * @size: Size of allocation 79 * @dma_handle: Out argument for allocated DMA handle 80 * @gfp: Allocation flags 81 * @attrs: Flags in the DMA_ATTR_* namespace. 82 * 83 * Managed dma_alloc_attrs(). Memory allocated using this function will be 84 * automatically released on driver detach. 85 * 86 * RETURNS: 87 * Pointer to allocated memory on success, NULL on failure. 88 */ 89 void *dmam_alloc_attrs(struct device *dev, size_t size, dma_addr_t *dma_handle, 90 gfp_t gfp, unsigned long attrs) 91 { 92 struct dma_devres *dr; 93 void *vaddr; 94 95 dr = devres_alloc(dmam_release, sizeof(*dr), gfp); 96 if (!dr) 97 return NULL; 98 99 vaddr = dma_alloc_attrs(dev, size, dma_handle, gfp, attrs); 100 if (!vaddr) { 101 devres_free(dr); 102 return NULL; 103 } 104 105 dr->vaddr = vaddr; 106 dr->dma_handle = *dma_handle; 107 dr->size = size; 108 dr->attrs = attrs; 109 110 devres_add(dev, dr); 111 112 return vaddr; 113 } 114 EXPORT_SYMBOL(dmam_alloc_attrs); 115 116 static bool dma_go_direct(struct device *dev, dma_addr_t mask, 117 const struct dma_map_ops *ops) 118 { 119 if (likely(!ops)) 120 return true; 121 #ifdef CONFIG_DMA_OPS_BYPASS 122 if (dev->dma_ops_bypass) 123 return min_not_zero(mask, dev->bus_dma_limit) >= 124 dma_direct_get_required_mask(dev); 125 #endif 126 return false; 127 } 128 129 130 /* 131 * Check if the devices uses a direct mapping for streaming DMA operations. 132 * This allows IOMMU drivers to set a bypass mode if the DMA mask is large 133 * enough. 134 */ 135 static inline bool dma_alloc_direct(struct device *dev, 136 const struct dma_map_ops *ops) 137 { 138 return dma_go_direct(dev, dev->coherent_dma_mask, ops); 139 } 140 141 static inline bool dma_map_direct(struct device *dev, 142 const struct dma_map_ops *ops) 143 { 144 return dma_go_direct(dev, *dev->dma_mask, ops); 145 } 146 147 dma_addr_t dma_map_page_attrs(struct device *dev, struct page *page, 148 size_t offset, size_t size, enum dma_data_direction dir, 149 unsigned long attrs) 150 { 151 const struct dma_map_ops *ops = get_dma_ops(dev); 152 dma_addr_t addr; 153 154 BUG_ON(!valid_dma_direction(dir)); 155 156 if (WARN_ON_ONCE(!dev->dma_mask)) 157 return DMA_MAPPING_ERROR; 158 159 if (dma_map_direct(dev, ops) || 160 arch_dma_map_page_direct(dev, page_to_phys(page) + offset + size)) 161 addr = dma_direct_map_page(dev, page, offset, size, dir, attrs); 162 else 163 addr = ops->map_page(dev, page, offset, size, dir, attrs); 164 kmsan_handle_dma(page, offset, size, dir); 165 debug_dma_map_page(dev, page, offset, size, dir, addr, attrs); 166 167 return addr; 168 } 169 EXPORT_SYMBOL(dma_map_page_attrs); 170 171 void dma_unmap_page_attrs(struct device *dev, dma_addr_t addr, size_t size, 172 enum dma_data_direction dir, unsigned long attrs) 173 { 174 const struct dma_map_ops *ops = get_dma_ops(dev); 175 176 BUG_ON(!valid_dma_direction(dir)); 177 if (dma_map_direct(dev, ops) || 178 arch_dma_unmap_page_direct(dev, addr + size)) 179 dma_direct_unmap_page(dev, addr, size, dir, attrs); 180 else if (ops->unmap_page) 181 ops->unmap_page(dev, addr, size, dir, attrs); 182 debug_dma_unmap_page(dev, addr, size, dir); 183 } 184 EXPORT_SYMBOL(dma_unmap_page_attrs); 185 186 static int __dma_map_sg_attrs(struct device *dev, struct scatterlist *sg, 187 int nents, enum dma_data_direction dir, unsigned long attrs) 188 { 189 const struct dma_map_ops *ops = get_dma_ops(dev); 190 int ents; 191 192 BUG_ON(!valid_dma_direction(dir)); 193 194 if (WARN_ON_ONCE(!dev->dma_mask)) 195 return 0; 196 197 if (dma_map_direct(dev, ops) || 198 arch_dma_map_sg_direct(dev, sg, nents)) 199 ents = dma_direct_map_sg(dev, sg, nents, dir, attrs); 200 else 201 ents = ops->map_sg(dev, sg, nents, dir, attrs); 202 203 if (ents > 0) { 204 kmsan_handle_dma_sg(sg, nents, dir); 205 debug_dma_map_sg(dev, sg, nents, ents, dir, attrs); 206 } else if (WARN_ON_ONCE(ents != -EINVAL && ents != -ENOMEM && 207 ents != -EIO && ents != -EREMOTEIO)) { 208 return -EIO; 209 } 210 211 return ents; 212 } 213 214 /** 215 * dma_map_sg_attrs - Map the given buffer for DMA 216 * @dev: The device for which to perform the DMA operation 217 * @sg: The sg_table object describing the buffer 218 * @nents: Number of entries to map 219 * @dir: DMA direction 220 * @attrs: Optional DMA attributes for the map operation 221 * 222 * Maps a buffer described by a scatterlist passed in the sg argument with 223 * nents segments for the @dir DMA operation by the @dev device. 224 * 225 * Returns the number of mapped entries (which can be less than nents) 226 * on success. Zero is returned for any error. 227 * 228 * dma_unmap_sg_attrs() should be used to unmap the buffer with the 229 * original sg and original nents (not the value returned by this funciton). 230 */ 231 unsigned int dma_map_sg_attrs(struct device *dev, struct scatterlist *sg, 232 int nents, enum dma_data_direction dir, unsigned long attrs) 233 { 234 int ret; 235 236 ret = __dma_map_sg_attrs(dev, sg, nents, dir, attrs); 237 if (ret < 0) 238 return 0; 239 return ret; 240 } 241 EXPORT_SYMBOL(dma_map_sg_attrs); 242 243 /** 244 * dma_map_sgtable - Map the given buffer for DMA 245 * @dev: The device for which to perform the DMA operation 246 * @sgt: The sg_table object describing the buffer 247 * @dir: DMA direction 248 * @attrs: Optional DMA attributes for the map operation 249 * 250 * Maps a buffer described by a scatterlist stored in the given sg_table 251 * object for the @dir DMA operation by the @dev device. After success, the 252 * ownership for the buffer is transferred to the DMA domain. One has to 253 * call dma_sync_sgtable_for_cpu() or dma_unmap_sgtable() to move the 254 * ownership of the buffer back to the CPU domain before touching the 255 * buffer by the CPU. 256 * 257 * Returns 0 on success or a negative error code on error. The following 258 * error codes are supported with the given meaning: 259 * 260 * -EINVAL An invalid argument, unaligned access or other error 261 * in usage. Will not succeed if retried. 262 * -ENOMEM Insufficient resources (like memory or IOVA space) to 263 * complete the mapping. Should succeed if retried later. 264 * -EIO Legacy error code with an unknown meaning. eg. this is 265 * returned if a lower level call returned 266 * DMA_MAPPING_ERROR. 267 * -EREMOTEIO The DMA device cannot access P2PDMA memory specified 268 * in the sg_table. This will not succeed if retried. 269 */ 270 int dma_map_sgtable(struct device *dev, struct sg_table *sgt, 271 enum dma_data_direction dir, unsigned long attrs) 272 { 273 int nents; 274 275 nents = __dma_map_sg_attrs(dev, sgt->sgl, sgt->orig_nents, dir, attrs); 276 if (nents < 0) 277 return nents; 278 sgt->nents = nents; 279 return 0; 280 } 281 EXPORT_SYMBOL_GPL(dma_map_sgtable); 282 283 void dma_unmap_sg_attrs(struct device *dev, struct scatterlist *sg, 284 int nents, enum dma_data_direction dir, 285 unsigned long attrs) 286 { 287 const struct dma_map_ops *ops = get_dma_ops(dev); 288 289 BUG_ON(!valid_dma_direction(dir)); 290 debug_dma_unmap_sg(dev, sg, nents, dir); 291 if (dma_map_direct(dev, ops) || 292 arch_dma_unmap_sg_direct(dev, sg, nents)) 293 dma_direct_unmap_sg(dev, sg, nents, dir, attrs); 294 else if (ops->unmap_sg) 295 ops->unmap_sg(dev, sg, nents, dir, attrs); 296 } 297 EXPORT_SYMBOL(dma_unmap_sg_attrs); 298 299 dma_addr_t dma_map_resource(struct device *dev, phys_addr_t phys_addr, 300 size_t size, enum dma_data_direction dir, unsigned long attrs) 301 { 302 const struct dma_map_ops *ops = get_dma_ops(dev); 303 dma_addr_t addr = DMA_MAPPING_ERROR; 304 305 BUG_ON(!valid_dma_direction(dir)); 306 307 if (WARN_ON_ONCE(!dev->dma_mask)) 308 return DMA_MAPPING_ERROR; 309 310 if (dma_map_direct(dev, ops)) 311 addr = dma_direct_map_resource(dev, phys_addr, size, dir, attrs); 312 else if (ops->map_resource) 313 addr = ops->map_resource(dev, phys_addr, size, dir, attrs); 314 315 debug_dma_map_resource(dev, phys_addr, size, dir, addr, attrs); 316 return addr; 317 } 318 EXPORT_SYMBOL(dma_map_resource); 319 320 void dma_unmap_resource(struct device *dev, dma_addr_t addr, size_t size, 321 enum dma_data_direction dir, unsigned long attrs) 322 { 323 const struct dma_map_ops *ops = get_dma_ops(dev); 324 325 BUG_ON(!valid_dma_direction(dir)); 326 if (!dma_map_direct(dev, ops) && ops->unmap_resource) 327 ops->unmap_resource(dev, addr, size, dir, attrs); 328 debug_dma_unmap_resource(dev, addr, size, dir); 329 } 330 EXPORT_SYMBOL(dma_unmap_resource); 331 332 #ifdef CONFIG_DMA_NEED_SYNC 333 void __dma_sync_single_for_cpu(struct device *dev, dma_addr_t addr, size_t size, 334 enum dma_data_direction dir) 335 { 336 const struct dma_map_ops *ops = get_dma_ops(dev); 337 338 BUG_ON(!valid_dma_direction(dir)); 339 if (dma_map_direct(dev, ops)) 340 dma_direct_sync_single_for_cpu(dev, addr, size, dir); 341 else if (ops->sync_single_for_cpu) 342 ops->sync_single_for_cpu(dev, addr, size, dir); 343 debug_dma_sync_single_for_cpu(dev, addr, size, dir); 344 } 345 EXPORT_SYMBOL(__dma_sync_single_for_cpu); 346 347 void __dma_sync_single_for_device(struct device *dev, dma_addr_t addr, 348 size_t size, enum dma_data_direction dir) 349 { 350 const struct dma_map_ops *ops = get_dma_ops(dev); 351 352 BUG_ON(!valid_dma_direction(dir)); 353 if (dma_map_direct(dev, ops)) 354 dma_direct_sync_single_for_device(dev, addr, size, dir); 355 else if (ops->sync_single_for_device) 356 ops->sync_single_for_device(dev, addr, size, dir); 357 debug_dma_sync_single_for_device(dev, addr, size, dir); 358 } 359 EXPORT_SYMBOL(__dma_sync_single_for_device); 360 361 void __dma_sync_sg_for_cpu(struct device *dev, struct scatterlist *sg, 362 int nelems, enum dma_data_direction dir) 363 { 364 const struct dma_map_ops *ops = get_dma_ops(dev); 365 366 BUG_ON(!valid_dma_direction(dir)); 367 if (dma_map_direct(dev, ops)) 368 dma_direct_sync_sg_for_cpu(dev, sg, nelems, dir); 369 else if (ops->sync_sg_for_cpu) 370 ops->sync_sg_for_cpu(dev, sg, nelems, dir); 371 debug_dma_sync_sg_for_cpu(dev, sg, nelems, dir); 372 } 373 EXPORT_SYMBOL(__dma_sync_sg_for_cpu); 374 375 void __dma_sync_sg_for_device(struct device *dev, struct scatterlist *sg, 376 int nelems, enum dma_data_direction dir) 377 { 378 const struct dma_map_ops *ops = get_dma_ops(dev); 379 380 BUG_ON(!valid_dma_direction(dir)); 381 if (dma_map_direct(dev, ops)) 382 dma_direct_sync_sg_for_device(dev, sg, nelems, dir); 383 else if (ops->sync_sg_for_device) 384 ops->sync_sg_for_device(dev, sg, nelems, dir); 385 debug_dma_sync_sg_for_device(dev, sg, nelems, dir); 386 } 387 EXPORT_SYMBOL(__dma_sync_sg_for_device); 388 389 bool __dma_need_sync(struct device *dev, dma_addr_t dma_addr) 390 { 391 const struct dma_map_ops *ops = get_dma_ops(dev); 392 393 if (dma_map_direct(dev, ops)) 394 /* 395 * dma_skip_sync could've been reset on first SWIOTLB buffer 396 * mapping, but @dma_addr is not necessary an SWIOTLB buffer. 397 * In this case, fall back to more granular check. 398 */ 399 return dma_direct_need_sync(dev, dma_addr); 400 return true; 401 } 402 EXPORT_SYMBOL_GPL(__dma_need_sync); 403 404 static void dma_setup_need_sync(struct device *dev) 405 { 406 const struct dma_map_ops *ops = get_dma_ops(dev); 407 408 if (dma_map_direct(dev, ops) || (ops->flags & DMA_F_CAN_SKIP_SYNC)) 409 /* 410 * dma_skip_sync will be reset to %false on first SWIOTLB buffer 411 * mapping, if any. During the device initialization, it's 412 * enough to check only for the DMA coherence. 413 */ 414 dev->dma_skip_sync = dev_is_dma_coherent(dev); 415 else if (!ops->sync_single_for_device && !ops->sync_single_for_cpu && 416 !ops->sync_sg_for_device && !ops->sync_sg_for_cpu) 417 /* 418 * Synchronization is not possible when none of DMA sync ops 419 * is set. 420 */ 421 dev->dma_skip_sync = true; 422 else 423 dev->dma_skip_sync = false; 424 } 425 #else /* !CONFIG_DMA_NEED_SYNC */ 426 static inline void dma_setup_need_sync(struct device *dev) { } 427 #endif /* !CONFIG_DMA_NEED_SYNC */ 428 429 /* 430 * The whole dma_get_sgtable() idea is fundamentally unsafe - it seems 431 * that the intention is to allow exporting memory allocated via the 432 * coherent DMA APIs through the dma_buf API, which only accepts a 433 * scattertable. This presents a couple of problems: 434 * 1. Not all memory allocated via the coherent DMA APIs is backed by 435 * a struct page 436 * 2. Passing coherent DMA memory into the streaming APIs is not allowed 437 * as we will try to flush the memory through a different alias to that 438 * actually being used (and the flushes are redundant.) 439 */ 440 int dma_get_sgtable_attrs(struct device *dev, struct sg_table *sgt, 441 void *cpu_addr, dma_addr_t dma_addr, size_t size, 442 unsigned long attrs) 443 { 444 const struct dma_map_ops *ops = get_dma_ops(dev); 445 446 if (dma_alloc_direct(dev, ops)) 447 return dma_direct_get_sgtable(dev, sgt, cpu_addr, dma_addr, 448 size, attrs); 449 if (!ops->get_sgtable) 450 return -ENXIO; 451 return ops->get_sgtable(dev, sgt, cpu_addr, dma_addr, size, attrs); 452 } 453 EXPORT_SYMBOL(dma_get_sgtable_attrs); 454 455 #ifdef CONFIG_MMU 456 /* 457 * Return the page attributes used for mapping dma_alloc_* memory, either in 458 * kernel space if remapping is needed, or to userspace through dma_mmap_*. 459 */ 460 pgprot_t dma_pgprot(struct device *dev, pgprot_t prot, unsigned long attrs) 461 { 462 if (dev_is_dma_coherent(dev)) 463 return prot; 464 #ifdef CONFIG_ARCH_HAS_DMA_WRITE_COMBINE 465 if (attrs & DMA_ATTR_WRITE_COMBINE) 466 return pgprot_writecombine(prot); 467 #endif 468 return pgprot_dmacoherent(prot); 469 } 470 #endif /* CONFIG_MMU */ 471 472 /** 473 * dma_can_mmap - check if a given device supports dma_mmap_* 474 * @dev: device to check 475 * 476 * Returns %true if @dev supports dma_mmap_coherent() and dma_mmap_attrs() to 477 * map DMA allocations to userspace. 478 */ 479 bool dma_can_mmap(struct device *dev) 480 { 481 const struct dma_map_ops *ops = get_dma_ops(dev); 482 483 if (dma_alloc_direct(dev, ops)) 484 return dma_direct_can_mmap(dev); 485 return ops->mmap != NULL; 486 } 487 EXPORT_SYMBOL_GPL(dma_can_mmap); 488 489 /** 490 * dma_mmap_attrs - map a coherent DMA allocation into user space 491 * @dev: valid struct device pointer, or NULL for ISA and EISA-like devices 492 * @vma: vm_area_struct describing requested user mapping 493 * @cpu_addr: kernel CPU-view address returned from dma_alloc_attrs 494 * @dma_addr: device-view address returned from dma_alloc_attrs 495 * @size: size of memory originally requested in dma_alloc_attrs 496 * @attrs: attributes of mapping properties requested in dma_alloc_attrs 497 * 498 * Map a coherent DMA buffer previously allocated by dma_alloc_attrs into user 499 * space. The coherent DMA buffer must not be freed by the driver until the 500 * user space mapping has been released. 501 */ 502 int dma_mmap_attrs(struct device *dev, struct vm_area_struct *vma, 503 void *cpu_addr, dma_addr_t dma_addr, size_t size, 504 unsigned long attrs) 505 { 506 const struct dma_map_ops *ops = get_dma_ops(dev); 507 508 if (dma_alloc_direct(dev, ops)) 509 return dma_direct_mmap(dev, vma, cpu_addr, dma_addr, size, 510 attrs); 511 if (!ops->mmap) 512 return -ENXIO; 513 return ops->mmap(dev, vma, cpu_addr, dma_addr, size, attrs); 514 } 515 EXPORT_SYMBOL(dma_mmap_attrs); 516 517 u64 dma_get_required_mask(struct device *dev) 518 { 519 const struct dma_map_ops *ops = get_dma_ops(dev); 520 521 if (dma_alloc_direct(dev, ops)) 522 return dma_direct_get_required_mask(dev); 523 if (ops->get_required_mask) 524 return ops->get_required_mask(dev); 525 526 /* 527 * We require every DMA ops implementation to at least support a 32-bit 528 * DMA mask (and use bounce buffering if that isn't supported in 529 * hardware). As the direct mapping code has its own routine to 530 * actually report an optimal mask we default to 32-bit here as that 531 * is the right thing for most IOMMUs, and at least not actively 532 * harmful in general. 533 */ 534 return DMA_BIT_MASK(32); 535 } 536 EXPORT_SYMBOL_GPL(dma_get_required_mask); 537 538 void *dma_alloc_attrs(struct device *dev, size_t size, dma_addr_t *dma_handle, 539 gfp_t flag, unsigned long attrs) 540 { 541 const struct dma_map_ops *ops = get_dma_ops(dev); 542 void *cpu_addr; 543 544 WARN_ON_ONCE(!dev->coherent_dma_mask); 545 546 /* 547 * DMA allocations can never be turned back into a page pointer, so 548 * requesting compound pages doesn't make sense (and can't even be 549 * supported at all by various backends). 550 */ 551 if (WARN_ON_ONCE(flag & __GFP_COMP)) 552 return NULL; 553 554 if (dma_alloc_from_dev_coherent(dev, size, dma_handle, &cpu_addr)) 555 return cpu_addr; 556 557 /* let the implementation decide on the zone to allocate from: */ 558 flag &= ~(__GFP_DMA | __GFP_DMA32 | __GFP_HIGHMEM); 559 560 if (dma_alloc_direct(dev, ops)) 561 cpu_addr = dma_direct_alloc(dev, size, dma_handle, flag, attrs); 562 else if (ops->alloc) 563 cpu_addr = ops->alloc(dev, size, dma_handle, flag, attrs); 564 else 565 return NULL; 566 567 debug_dma_alloc_coherent(dev, size, *dma_handle, cpu_addr, attrs); 568 return cpu_addr; 569 } 570 EXPORT_SYMBOL(dma_alloc_attrs); 571 572 void dma_free_attrs(struct device *dev, size_t size, void *cpu_addr, 573 dma_addr_t dma_handle, unsigned long attrs) 574 { 575 const struct dma_map_ops *ops = get_dma_ops(dev); 576 577 if (dma_release_from_dev_coherent(dev, get_order(size), cpu_addr)) 578 return; 579 /* 580 * On non-coherent platforms which implement DMA-coherent buffers via 581 * non-cacheable remaps, ops->free() may call vunmap(). Thus getting 582 * this far in IRQ context is a) at risk of a BUG_ON() or trying to 583 * sleep on some machines, and b) an indication that the driver is 584 * probably misusing the coherent API anyway. 585 */ 586 WARN_ON(irqs_disabled()); 587 588 if (!cpu_addr) 589 return; 590 591 debug_dma_free_coherent(dev, size, cpu_addr, dma_handle); 592 if (dma_alloc_direct(dev, ops)) 593 dma_direct_free(dev, size, cpu_addr, dma_handle, attrs); 594 else if (ops->free) 595 ops->free(dev, size, cpu_addr, dma_handle, attrs); 596 } 597 EXPORT_SYMBOL(dma_free_attrs); 598 599 static struct page *__dma_alloc_pages(struct device *dev, size_t size, 600 dma_addr_t *dma_handle, enum dma_data_direction dir, gfp_t gfp) 601 { 602 const struct dma_map_ops *ops = get_dma_ops(dev); 603 604 if (WARN_ON_ONCE(!dev->coherent_dma_mask)) 605 return NULL; 606 if (WARN_ON_ONCE(gfp & (__GFP_DMA | __GFP_DMA32 | __GFP_HIGHMEM))) 607 return NULL; 608 if (WARN_ON_ONCE(gfp & __GFP_COMP)) 609 return NULL; 610 611 size = PAGE_ALIGN(size); 612 if (dma_alloc_direct(dev, ops)) 613 return dma_direct_alloc_pages(dev, size, dma_handle, dir, gfp); 614 if (!ops->alloc_pages_op) 615 return NULL; 616 return ops->alloc_pages_op(dev, size, dma_handle, dir, gfp); 617 } 618 619 struct page *dma_alloc_pages(struct device *dev, size_t size, 620 dma_addr_t *dma_handle, enum dma_data_direction dir, gfp_t gfp) 621 { 622 struct page *page = __dma_alloc_pages(dev, size, dma_handle, dir, gfp); 623 624 if (page) 625 debug_dma_map_page(dev, page, 0, size, dir, *dma_handle, 0); 626 return page; 627 } 628 EXPORT_SYMBOL_GPL(dma_alloc_pages); 629 630 static void __dma_free_pages(struct device *dev, size_t size, struct page *page, 631 dma_addr_t dma_handle, enum dma_data_direction dir) 632 { 633 const struct dma_map_ops *ops = get_dma_ops(dev); 634 635 size = PAGE_ALIGN(size); 636 if (dma_alloc_direct(dev, ops)) 637 dma_direct_free_pages(dev, size, page, dma_handle, dir); 638 else if (ops->free_pages) 639 ops->free_pages(dev, size, page, dma_handle, dir); 640 } 641 642 void dma_free_pages(struct device *dev, size_t size, struct page *page, 643 dma_addr_t dma_handle, enum dma_data_direction dir) 644 { 645 debug_dma_unmap_page(dev, dma_handle, size, dir); 646 __dma_free_pages(dev, size, page, dma_handle, dir); 647 } 648 EXPORT_SYMBOL_GPL(dma_free_pages); 649 650 int dma_mmap_pages(struct device *dev, struct vm_area_struct *vma, 651 size_t size, struct page *page) 652 { 653 unsigned long count = PAGE_ALIGN(size) >> PAGE_SHIFT; 654 655 if (vma->vm_pgoff >= count || vma_pages(vma) > count - vma->vm_pgoff) 656 return -ENXIO; 657 return remap_pfn_range(vma, vma->vm_start, 658 page_to_pfn(page) + vma->vm_pgoff, 659 vma_pages(vma) << PAGE_SHIFT, vma->vm_page_prot); 660 } 661 EXPORT_SYMBOL_GPL(dma_mmap_pages); 662 663 static struct sg_table *alloc_single_sgt(struct device *dev, size_t size, 664 enum dma_data_direction dir, gfp_t gfp) 665 { 666 struct sg_table *sgt; 667 struct page *page; 668 669 sgt = kmalloc(sizeof(*sgt), gfp); 670 if (!sgt) 671 return NULL; 672 if (sg_alloc_table(sgt, 1, gfp)) 673 goto out_free_sgt; 674 page = __dma_alloc_pages(dev, size, &sgt->sgl->dma_address, dir, gfp); 675 if (!page) 676 goto out_free_table; 677 sg_set_page(sgt->sgl, page, PAGE_ALIGN(size), 0); 678 sg_dma_len(sgt->sgl) = sgt->sgl->length; 679 return sgt; 680 out_free_table: 681 sg_free_table(sgt); 682 out_free_sgt: 683 kfree(sgt); 684 return NULL; 685 } 686 687 struct sg_table *dma_alloc_noncontiguous(struct device *dev, size_t size, 688 enum dma_data_direction dir, gfp_t gfp, unsigned long attrs) 689 { 690 const struct dma_map_ops *ops = get_dma_ops(dev); 691 struct sg_table *sgt; 692 693 if (WARN_ON_ONCE(attrs & ~DMA_ATTR_ALLOC_SINGLE_PAGES)) 694 return NULL; 695 if (WARN_ON_ONCE(gfp & __GFP_COMP)) 696 return NULL; 697 698 if (ops && ops->alloc_noncontiguous) 699 sgt = ops->alloc_noncontiguous(dev, size, dir, gfp, attrs); 700 else 701 sgt = alloc_single_sgt(dev, size, dir, gfp); 702 703 if (sgt) { 704 sgt->nents = 1; 705 debug_dma_map_sg(dev, sgt->sgl, sgt->orig_nents, 1, dir, attrs); 706 } 707 return sgt; 708 } 709 EXPORT_SYMBOL_GPL(dma_alloc_noncontiguous); 710 711 static void free_single_sgt(struct device *dev, size_t size, 712 struct sg_table *sgt, enum dma_data_direction dir) 713 { 714 __dma_free_pages(dev, size, sg_page(sgt->sgl), sgt->sgl->dma_address, 715 dir); 716 sg_free_table(sgt); 717 kfree(sgt); 718 } 719 720 void dma_free_noncontiguous(struct device *dev, size_t size, 721 struct sg_table *sgt, enum dma_data_direction dir) 722 { 723 const struct dma_map_ops *ops = get_dma_ops(dev); 724 725 debug_dma_unmap_sg(dev, sgt->sgl, sgt->orig_nents, dir); 726 if (ops && ops->free_noncontiguous) 727 ops->free_noncontiguous(dev, size, sgt, dir); 728 else 729 free_single_sgt(dev, size, sgt, dir); 730 } 731 EXPORT_SYMBOL_GPL(dma_free_noncontiguous); 732 733 void *dma_vmap_noncontiguous(struct device *dev, size_t size, 734 struct sg_table *sgt) 735 { 736 const struct dma_map_ops *ops = get_dma_ops(dev); 737 unsigned long count = PAGE_ALIGN(size) >> PAGE_SHIFT; 738 739 if (ops && ops->alloc_noncontiguous) 740 return vmap(sgt_handle(sgt)->pages, count, VM_MAP, PAGE_KERNEL); 741 return page_address(sg_page(sgt->sgl)); 742 } 743 EXPORT_SYMBOL_GPL(dma_vmap_noncontiguous); 744 745 void dma_vunmap_noncontiguous(struct device *dev, void *vaddr) 746 { 747 const struct dma_map_ops *ops = get_dma_ops(dev); 748 749 if (ops && ops->alloc_noncontiguous) 750 vunmap(vaddr); 751 } 752 EXPORT_SYMBOL_GPL(dma_vunmap_noncontiguous); 753 754 int dma_mmap_noncontiguous(struct device *dev, struct vm_area_struct *vma, 755 size_t size, struct sg_table *sgt) 756 { 757 const struct dma_map_ops *ops = get_dma_ops(dev); 758 759 if (ops && ops->alloc_noncontiguous) { 760 unsigned long count = PAGE_ALIGN(size) >> PAGE_SHIFT; 761 762 if (vma->vm_pgoff >= count || 763 vma_pages(vma) > count - vma->vm_pgoff) 764 return -ENXIO; 765 return vm_map_pages(vma, sgt_handle(sgt)->pages, count); 766 } 767 return dma_mmap_pages(dev, vma, size, sg_page(sgt->sgl)); 768 } 769 EXPORT_SYMBOL_GPL(dma_mmap_noncontiguous); 770 771 static int dma_supported(struct device *dev, u64 mask) 772 { 773 const struct dma_map_ops *ops = get_dma_ops(dev); 774 775 /* 776 * ->dma_supported sets the bypass flag, so we must always call 777 * into the method here unless the device is truly direct mapped. 778 */ 779 if (!ops) 780 return dma_direct_supported(dev, mask); 781 if (!ops->dma_supported) 782 return 1; 783 return ops->dma_supported(dev, mask); 784 } 785 786 bool dma_pci_p2pdma_supported(struct device *dev) 787 { 788 const struct dma_map_ops *ops = get_dma_ops(dev); 789 790 /* if ops is not set, dma direct will be used which supports P2PDMA */ 791 if (!ops) 792 return true; 793 794 /* 795 * Note: dma_ops_bypass is not checked here because P2PDMA should 796 * not be used with dma mapping ops that do not have support even 797 * if the specific device is bypassing them. 798 */ 799 800 return ops->flags & DMA_F_PCI_P2PDMA_SUPPORTED; 801 } 802 EXPORT_SYMBOL_GPL(dma_pci_p2pdma_supported); 803 804 int dma_set_mask(struct device *dev, u64 mask) 805 { 806 /* 807 * Truncate the mask to the actually supported dma_addr_t width to 808 * avoid generating unsupportable addresses. 809 */ 810 mask = (dma_addr_t)mask; 811 812 if (!dev->dma_mask || !dma_supported(dev, mask)) 813 return -EIO; 814 815 arch_dma_set_mask(dev, mask); 816 *dev->dma_mask = mask; 817 dma_setup_need_sync(dev); 818 819 return 0; 820 } 821 EXPORT_SYMBOL(dma_set_mask); 822 823 int dma_set_coherent_mask(struct device *dev, u64 mask) 824 { 825 /* 826 * Truncate the mask to the actually supported dma_addr_t width to 827 * avoid generating unsupportable addresses. 828 */ 829 mask = (dma_addr_t)mask; 830 831 if (!dma_supported(dev, mask)) 832 return -EIO; 833 834 dev->coherent_dma_mask = mask; 835 return 0; 836 } 837 EXPORT_SYMBOL(dma_set_coherent_mask); 838 839 /** 840 * dma_addressing_limited - return if the device is addressing limited 841 * @dev: device to check 842 * 843 * Return %true if the devices DMA mask is too small to address all memory in 844 * the system, else %false. Lack of addressing bits is the prime reason for 845 * bounce buffering, but might not be the only one. 846 */ 847 bool dma_addressing_limited(struct device *dev) 848 { 849 const struct dma_map_ops *ops = get_dma_ops(dev); 850 851 if (min_not_zero(dma_get_mask(dev), dev->bus_dma_limit) < 852 dma_get_required_mask(dev)) 853 return true; 854 855 if (unlikely(ops)) 856 return false; 857 return !dma_direct_all_ram_mapped(dev); 858 } 859 EXPORT_SYMBOL_GPL(dma_addressing_limited); 860 861 size_t dma_max_mapping_size(struct device *dev) 862 { 863 const struct dma_map_ops *ops = get_dma_ops(dev); 864 size_t size = SIZE_MAX; 865 866 if (dma_map_direct(dev, ops)) 867 size = dma_direct_max_mapping_size(dev); 868 else if (ops && ops->max_mapping_size) 869 size = ops->max_mapping_size(dev); 870 871 return size; 872 } 873 EXPORT_SYMBOL_GPL(dma_max_mapping_size); 874 875 size_t dma_opt_mapping_size(struct device *dev) 876 { 877 const struct dma_map_ops *ops = get_dma_ops(dev); 878 size_t size = SIZE_MAX; 879 880 if (ops && ops->opt_mapping_size) 881 size = ops->opt_mapping_size(); 882 883 return min(dma_max_mapping_size(dev), size); 884 } 885 EXPORT_SYMBOL_GPL(dma_opt_mapping_size); 886 887 unsigned long dma_get_merge_boundary(struct device *dev) 888 { 889 const struct dma_map_ops *ops = get_dma_ops(dev); 890 891 if (!ops || !ops->get_merge_boundary) 892 return 0; /* can't merge */ 893 894 return ops->get_merge_boundary(dev); 895 } 896 EXPORT_SYMBOL_GPL(dma_get_merge_boundary); 897