1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * arch-independent dma-mapping routines 4 * 5 * Copyright (c) 2006 SUSE Linux Products GmbH 6 * Copyright (c) 2006 Tejun Heo <teheo@suse.de> 7 */ 8 #include <linux/memblock.h> /* for max_pfn */ 9 #include <linux/acpi.h> 10 #include <linux/dma-map-ops.h> 11 #include <linux/export.h> 12 #include <linux/gfp.h> 13 #include <linux/iommu-dma.h> 14 #include <linux/kmsan.h> 15 #include <linux/of_device.h> 16 #include <linux/slab.h> 17 #include <linux/vmalloc.h> 18 #include "debug.h" 19 #include "direct.h" 20 21 #define CREATE_TRACE_POINTS 22 #include <trace/events/dma.h> 23 24 #if defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_DEVICE) || \ 25 defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_CPU) || \ 26 defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_CPU_ALL) 27 bool dma_default_coherent = IS_ENABLED(CONFIG_ARCH_DMA_DEFAULT_COHERENT); 28 #endif 29 30 /* 31 * Managed DMA API 32 */ 33 struct dma_devres { 34 size_t size; 35 void *vaddr; 36 dma_addr_t dma_handle; 37 unsigned long attrs; 38 }; 39 40 static void dmam_release(struct device *dev, void *res) 41 { 42 struct dma_devres *this = res; 43 44 dma_free_attrs(dev, this->size, this->vaddr, this->dma_handle, 45 this->attrs); 46 } 47 48 static int dmam_match(struct device *dev, void *res, void *match_data) 49 { 50 struct dma_devres *this = res, *match = match_data; 51 52 if (this->vaddr == match->vaddr) { 53 WARN_ON(this->size != match->size || 54 this->dma_handle != match->dma_handle); 55 return 1; 56 } 57 return 0; 58 } 59 60 /** 61 * dmam_free_coherent - Managed dma_free_coherent() 62 * @dev: Device to free coherent memory for 63 * @size: Size of allocation 64 * @vaddr: Virtual address of the memory to free 65 * @dma_handle: DMA handle of the memory to free 66 * 67 * Managed dma_free_coherent(). 68 */ 69 void dmam_free_coherent(struct device *dev, size_t size, void *vaddr, 70 dma_addr_t dma_handle) 71 { 72 struct dma_devres match_data = { size, vaddr, dma_handle }; 73 74 WARN_ON(devres_destroy(dev, dmam_release, dmam_match, &match_data)); 75 dma_free_coherent(dev, size, vaddr, dma_handle); 76 } 77 EXPORT_SYMBOL(dmam_free_coherent); 78 79 /** 80 * dmam_alloc_attrs - Managed dma_alloc_attrs() 81 * @dev: Device to allocate non_coherent memory for 82 * @size: Size of allocation 83 * @dma_handle: Out argument for allocated DMA handle 84 * @gfp: Allocation flags 85 * @attrs: Flags in the DMA_ATTR_* namespace. 86 * 87 * Managed dma_alloc_attrs(). Memory allocated using this function will be 88 * automatically released on driver detach. 89 * 90 * RETURNS: 91 * Pointer to allocated memory on success, NULL on failure. 92 */ 93 void *dmam_alloc_attrs(struct device *dev, size_t size, dma_addr_t *dma_handle, 94 gfp_t gfp, unsigned long attrs) 95 { 96 struct dma_devres *dr; 97 void *vaddr; 98 99 dr = devres_alloc(dmam_release, sizeof(*dr), gfp); 100 if (!dr) 101 return NULL; 102 103 vaddr = dma_alloc_attrs(dev, size, dma_handle, gfp, attrs); 104 if (!vaddr) { 105 devres_free(dr); 106 return NULL; 107 } 108 109 dr->vaddr = vaddr; 110 dr->dma_handle = *dma_handle; 111 dr->size = size; 112 dr->attrs = attrs; 113 114 devres_add(dev, dr); 115 116 return vaddr; 117 } 118 EXPORT_SYMBOL(dmam_alloc_attrs); 119 120 static bool dma_go_direct(struct device *dev, dma_addr_t mask, 121 const struct dma_map_ops *ops) 122 { 123 if (use_dma_iommu(dev)) 124 return false; 125 126 if (likely(!ops)) 127 return true; 128 129 #ifdef CONFIG_DMA_OPS_BYPASS 130 if (dev->dma_ops_bypass) 131 return min_not_zero(mask, dev->bus_dma_limit) >= 132 dma_direct_get_required_mask(dev); 133 #endif 134 return false; 135 } 136 137 138 /* 139 * Check if the devices uses a direct mapping for streaming DMA operations. 140 * This allows IOMMU drivers to set a bypass mode if the DMA mask is large 141 * enough. 142 */ 143 static inline bool dma_alloc_direct(struct device *dev, 144 const struct dma_map_ops *ops) 145 { 146 return dma_go_direct(dev, dev->coherent_dma_mask, ops); 147 } 148 149 static inline bool dma_map_direct(struct device *dev, 150 const struct dma_map_ops *ops) 151 { 152 return dma_go_direct(dev, *dev->dma_mask, ops); 153 } 154 155 dma_addr_t dma_map_page_attrs(struct device *dev, struct page *page, 156 size_t offset, size_t size, enum dma_data_direction dir, 157 unsigned long attrs) 158 { 159 const struct dma_map_ops *ops = get_dma_ops(dev); 160 dma_addr_t addr; 161 162 BUG_ON(!valid_dma_direction(dir)); 163 164 if (WARN_ON_ONCE(!dev->dma_mask)) 165 return DMA_MAPPING_ERROR; 166 167 if (dma_map_direct(dev, ops) || 168 arch_dma_map_page_direct(dev, page_to_phys(page) + offset + size)) 169 addr = dma_direct_map_page(dev, page, offset, size, dir, attrs); 170 else if (use_dma_iommu(dev)) 171 addr = iommu_dma_map_page(dev, page, offset, size, dir, attrs); 172 else 173 addr = ops->map_page(dev, page, offset, size, dir, attrs); 174 kmsan_handle_dma(page, offset, size, dir); 175 trace_dma_map_page(dev, page_to_phys(page) + offset, addr, size, dir, 176 attrs); 177 debug_dma_map_page(dev, page, offset, size, dir, addr, attrs); 178 179 return addr; 180 } 181 EXPORT_SYMBOL(dma_map_page_attrs); 182 183 void dma_unmap_page_attrs(struct device *dev, dma_addr_t addr, size_t size, 184 enum dma_data_direction dir, unsigned long attrs) 185 { 186 const struct dma_map_ops *ops = get_dma_ops(dev); 187 188 BUG_ON(!valid_dma_direction(dir)); 189 if (dma_map_direct(dev, ops) || 190 arch_dma_unmap_page_direct(dev, addr + size)) 191 dma_direct_unmap_page(dev, addr, size, dir, attrs); 192 else if (use_dma_iommu(dev)) 193 iommu_dma_unmap_page(dev, addr, size, dir, attrs); 194 else 195 ops->unmap_page(dev, addr, size, dir, attrs); 196 trace_dma_unmap_page(dev, addr, size, dir, attrs); 197 debug_dma_unmap_page(dev, addr, size, dir); 198 } 199 EXPORT_SYMBOL(dma_unmap_page_attrs); 200 201 static int __dma_map_sg_attrs(struct device *dev, struct scatterlist *sg, 202 int nents, enum dma_data_direction dir, unsigned long attrs) 203 { 204 const struct dma_map_ops *ops = get_dma_ops(dev); 205 int ents; 206 207 BUG_ON(!valid_dma_direction(dir)); 208 209 if (WARN_ON_ONCE(!dev->dma_mask)) 210 return 0; 211 212 if (dma_map_direct(dev, ops) || 213 arch_dma_map_sg_direct(dev, sg, nents)) 214 ents = dma_direct_map_sg(dev, sg, nents, dir, attrs); 215 else if (use_dma_iommu(dev)) 216 ents = iommu_dma_map_sg(dev, sg, nents, dir, attrs); 217 else 218 ents = ops->map_sg(dev, sg, nents, dir, attrs); 219 220 if (ents > 0) { 221 kmsan_handle_dma_sg(sg, nents, dir); 222 trace_dma_map_sg(dev, sg, nents, ents, dir, attrs); 223 debug_dma_map_sg(dev, sg, nents, ents, dir, attrs); 224 } else if (WARN_ON_ONCE(ents != -EINVAL && ents != -ENOMEM && 225 ents != -EIO && ents != -EREMOTEIO)) { 226 return -EIO; 227 } 228 229 return ents; 230 } 231 232 /** 233 * dma_map_sg_attrs - Map the given buffer for DMA 234 * @dev: The device for which to perform the DMA operation 235 * @sg: The sg_table object describing the buffer 236 * @nents: Number of entries to map 237 * @dir: DMA direction 238 * @attrs: Optional DMA attributes for the map operation 239 * 240 * Maps a buffer described by a scatterlist passed in the sg argument with 241 * nents segments for the @dir DMA operation by the @dev device. 242 * 243 * Returns the number of mapped entries (which can be less than nents) 244 * on success. Zero is returned for any error. 245 * 246 * dma_unmap_sg_attrs() should be used to unmap the buffer with the 247 * original sg and original nents (not the value returned by this funciton). 248 */ 249 unsigned int dma_map_sg_attrs(struct device *dev, struct scatterlist *sg, 250 int nents, enum dma_data_direction dir, unsigned long attrs) 251 { 252 int ret; 253 254 ret = __dma_map_sg_attrs(dev, sg, nents, dir, attrs); 255 if (ret < 0) 256 return 0; 257 return ret; 258 } 259 EXPORT_SYMBOL(dma_map_sg_attrs); 260 261 /** 262 * dma_map_sgtable - Map the given buffer for DMA 263 * @dev: The device for which to perform the DMA operation 264 * @sgt: The sg_table object describing the buffer 265 * @dir: DMA direction 266 * @attrs: Optional DMA attributes for the map operation 267 * 268 * Maps a buffer described by a scatterlist stored in the given sg_table 269 * object for the @dir DMA operation by the @dev device. After success, the 270 * ownership for the buffer is transferred to the DMA domain. One has to 271 * call dma_sync_sgtable_for_cpu() or dma_unmap_sgtable() to move the 272 * ownership of the buffer back to the CPU domain before touching the 273 * buffer by the CPU. 274 * 275 * Returns 0 on success or a negative error code on error. The following 276 * error codes are supported with the given meaning: 277 * 278 * -EINVAL An invalid argument, unaligned access or other error 279 * in usage. Will not succeed if retried. 280 * -ENOMEM Insufficient resources (like memory or IOVA space) to 281 * complete the mapping. Should succeed if retried later. 282 * -EIO Legacy error code with an unknown meaning. eg. this is 283 * returned if a lower level call returned 284 * DMA_MAPPING_ERROR. 285 * -EREMOTEIO The DMA device cannot access P2PDMA memory specified 286 * in the sg_table. This will not succeed if retried. 287 */ 288 int dma_map_sgtable(struct device *dev, struct sg_table *sgt, 289 enum dma_data_direction dir, unsigned long attrs) 290 { 291 int nents; 292 293 nents = __dma_map_sg_attrs(dev, sgt->sgl, sgt->orig_nents, dir, attrs); 294 if (nents < 0) 295 return nents; 296 sgt->nents = nents; 297 return 0; 298 } 299 EXPORT_SYMBOL_GPL(dma_map_sgtable); 300 301 void dma_unmap_sg_attrs(struct device *dev, struct scatterlist *sg, 302 int nents, enum dma_data_direction dir, 303 unsigned long attrs) 304 { 305 const struct dma_map_ops *ops = get_dma_ops(dev); 306 307 BUG_ON(!valid_dma_direction(dir)); 308 trace_dma_unmap_sg(dev, sg, nents, dir, attrs); 309 debug_dma_unmap_sg(dev, sg, nents, dir); 310 if (dma_map_direct(dev, ops) || 311 arch_dma_unmap_sg_direct(dev, sg, nents)) 312 dma_direct_unmap_sg(dev, sg, nents, dir, attrs); 313 else if (use_dma_iommu(dev)) 314 iommu_dma_unmap_sg(dev, sg, nents, dir, attrs); 315 else if (ops->unmap_sg) 316 ops->unmap_sg(dev, sg, nents, dir, attrs); 317 } 318 EXPORT_SYMBOL(dma_unmap_sg_attrs); 319 320 dma_addr_t dma_map_resource(struct device *dev, phys_addr_t phys_addr, 321 size_t size, enum dma_data_direction dir, unsigned long attrs) 322 { 323 const struct dma_map_ops *ops = get_dma_ops(dev); 324 dma_addr_t addr = DMA_MAPPING_ERROR; 325 326 BUG_ON(!valid_dma_direction(dir)); 327 328 if (WARN_ON_ONCE(!dev->dma_mask)) 329 return DMA_MAPPING_ERROR; 330 331 if (dma_map_direct(dev, ops)) 332 addr = dma_direct_map_resource(dev, phys_addr, size, dir, attrs); 333 else if (use_dma_iommu(dev)) 334 addr = iommu_dma_map_resource(dev, phys_addr, size, dir, attrs); 335 else if (ops->map_resource) 336 addr = ops->map_resource(dev, phys_addr, size, dir, attrs); 337 338 trace_dma_map_resource(dev, phys_addr, addr, size, dir, attrs); 339 debug_dma_map_resource(dev, phys_addr, size, dir, addr, attrs); 340 return addr; 341 } 342 EXPORT_SYMBOL(dma_map_resource); 343 344 void dma_unmap_resource(struct device *dev, dma_addr_t addr, size_t size, 345 enum dma_data_direction dir, unsigned long attrs) 346 { 347 const struct dma_map_ops *ops = get_dma_ops(dev); 348 349 BUG_ON(!valid_dma_direction(dir)); 350 if (dma_map_direct(dev, ops)) 351 ; /* nothing to do: uncached and no swiotlb */ 352 else if (use_dma_iommu(dev)) 353 iommu_dma_unmap_resource(dev, addr, size, dir, attrs); 354 else if (ops->unmap_resource) 355 ops->unmap_resource(dev, addr, size, dir, attrs); 356 trace_dma_unmap_resource(dev, addr, size, dir, attrs); 357 debug_dma_unmap_resource(dev, addr, size, dir); 358 } 359 EXPORT_SYMBOL(dma_unmap_resource); 360 361 #ifdef CONFIG_DMA_NEED_SYNC 362 void __dma_sync_single_for_cpu(struct device *dev, dma_addr_t addr, size_t size, 363 enum dma_data_direction dir) 364 { 365 const struct dma_map_ops *ops = get_dma_ops(dev); 366 367 BUG_ON(!valid_dma_direction(dir)); 368 if (dma_map_direct(dev, ops)) 369 dma_direct_sync_single_for_cpu(dev, addr, size, dir); 370 else if (use_dma_iommu(dev)) 371 iommu_dma_sync_single_for_cpu(dev, addr, size, dir); 372 else if (ops->sync_single_for_cpu) 373 ops->sync_single_for_cpu(dev, addr, size, dir); 374 trace_dma_sync_single_for_cpu(dev, addr, size, dir); 375 debug_dma_sync_single_for_cpu(dev, addr, size, dir); 376 } 377 EXPORT_SYMBOL(__dma_sync_single_for_cpu); 378 379 void __dma_sync_single_for_device(struct device *dev, dma_addr_t addr, 380 size_t size, enum dma_data_direction dir) 381 { 382 const struct dma_map_ops *ops = get_dma_ops(dev); 383 384 BUG_ON(!valid_dma_direction(dir)); 385 if (dma_map_direct(dev, ops)) 386 dma_direct_sync_single_for_device(dev, addr, size, dir); 387 else if (use_dma_iommu(dev)) 388 iommu_dma_sync_single_for_device(dev, addr, size, dir); 389 else if (ops->sync_single_for_device) 390 ops->sync_single_for_device(dev, addr, size, dir); 391 trace_dma_sync_single_for_device(dev, addr, size, dir); 392 debug_dma_sync_single_for_device(dev, addr, size, dir); 393 } 394 EXPORT_SYMBOL(__dma_sync_single_for_device); 395 396 void __dma_sync_sg_for_cpu(struct device *dev, struct scatterlist *sg, 397 int nelems, enum dma_data_direction dir) 398 { 399 const struct dma_map_ops *ops = get_dma_ops(dev); 400 401 BUG_ON(!valid_dma_direction(dir)); 402 if (dma_map_direct(dev, ops)) 403 dma_direct_sync_sg_for_cpu(dev, sg, nelems, dir); 404 else if (use_dma_iommu(dev)) 405 iommu_dma_sync_sg_for_cpu(dev, sg, nelems, dir); 406 else if (ops->sync_sg_for_cpu) 407 ops->sync_sg_for_cpu(dev, sg, nelems, dir); 408 trace_dma_sync_sg_for_cpu(dev, sg, nelems, dir); 409 debug_dma_sync_sg_for_cpu(dev, sg, nelems, dir); 410 } 411 EXPORT_SYMBOL(__dma_sync_sg_for_cpu); 412 413 void __dma_sync_sg_for_device(struct device *dev, struct scatterlist *sg, 414 int nelems, enum dma_data_direction dir) 415 { 416 const struct dma_map_ops *ops = get_dma_ops(dev); 417 418 BUG_ON(!valid_dma_direction(dir)); 419 if (dma_map_direct(dev, ops)) 420 dma_direct_sync_sg_for_device(dev, sg, nelems, dir); 421 else if (use_dma_iommu(dev)) 422 iommu_dma_sync_sg_for_device(dev, sg, nelems, dir); 423 else if (ops->sync_sg_for_device) 424 ops->sync_sg_for_device(dev, sg, nelems, dir); 425 trace_dma_sync_sg_for_device(dev, sg, nelems, dir); 426 debug_dma_sync_sg_for_device(dev, sg, nelems, dir); 427 } 428 EXPORT_SYMBOL(__dma_sync_sg_for_device); 429 430 bool __dma_need_sync(struct device *dev, dma_addr_t dma_addr) 431 { 432 const struct dma_map_ops *ops = get_dma_ops(dev); 433 434 if (dma_map_direct(dev, ops)) 435 /* 436 * dma_skip_sync could've been reset on first SWIOTLB buffer 437 * mapping, but @dma_addr is not necessary an SWIOTLB buffer. 438 * In this case, fall back to more granular check. 439 */ 440 return dma_direct_need_sync(dev, dma_addr); 441 return true; 442 } 443 EXPORT_SYMBOL_GPL(__dma_need_sync); 444 445 static void dma_setup_need_sync(struct device *dev) 446 { 447 const struct dma_map_ops *ops = get_dma_ops(dev); 448 449 if (dma_map_direct(dev, ops) || use_dma_iommu(dev)) 450 /* 451 * dma_skip_sync will be reset to %false on first SWIOTLB buffer 452 * mapping, if any. During the device initialization, it's 453 * enough to check only for the DMA coherence. 454 */ 455 dev->dma_skip_sync = dev_is_dma_coherent(dev); 456 else if (!ops->sync_single_for_device && !ops->sync_single_for_cpu && 457 !ops->sync_sg_for_device && !ops->sync_sg_for_cpu) 458 /* 459 * Synchronization is not possible when none of DMA sync ops 460 * is set. 461 */ 462 dev->dma_skip_sync = true; 463 else 464 dev->dma_skip_sync = false; 465 } 466 #else /* !CONFIG_DMA_NEED_SYNC */ 467 static inline void dma_setup_need_sync(struct device *dev) { } 468 #endif /* !CONFIG_DMA_NEED_SYNC */ 469 470 /* 471 * The whole dma_get_sgtable() idea is fundamentally unsafe - it seems 472 * that the intention is to allow exporting memory allocated via the 473 * coherent DMA APIs through the dma_buf API, which only accepts a 474 * scattertable. This presents a couple of problems: 475 * 1. Not all memory allocated via the coherent DMA APIs is backed by 476 * a struct page 477 * 2. Passing coherent DMA memory into the streaming APIs is not allowed 478 * as we will try to flush the memory through a different alias to that 479 * actually being used (and the flushes are redundant.) 480 */ 481 int dma_get_sgtable_attrs(struct device *dev, struct sg_table *sgt, 482 void *cpu_addr, dma_addr_t dma_addr, size_t size, 483 unsigned long attrs) 484 { 485 const struct dma_map_ops *ops = get_dma_ops(dev); 486 487 if (dma_alloc_direct(dev, ops)) 488 return dma_direct_get_sgtable(dev, sgt, cpu_addr, dma_addr, 489 size, attrs); 490 if (use_dma_iommu(dev)) 491 return iommu_dma_get_sgtable(dev, sgt, cpu_addr, dma_addr, 492 size, attrs); 493 if (!ops->get_sgtable) 494 return -ENXIO; 495 return ops->get_sgtable(dev, sgt, cpu_addr, dma_addr, size, attrs); 496 } 497 EXPORT_SYMBOL(dma_get_sgtable_attrs); 498 499 #ifdef CONFIG_MMU 500 /* 501 * Return the page attributes used for mapping dma_alloc_* memory, either in 502 * kernel space if remapping is needed, or to userspace through dma_mmap_*. 503 */ 504 pgprot_t dma_pgprot(struct device *dev, pgprot_t prot, unsigned long attrs) 505 { 506 if (dev_is_dma_coherent(dev)) 507 return prot; 508 #ifdef CONFIG_ARCH_HAS_DMA_WRITE_COMBINE 509 if (attrs & DMA_ATTR_WRITE_COMBINE) 510 return pgprot_writecombine(prot); 511 #endif 512 return pgprot_dmacoherent(prot); 513 } 514 #endif /* CONFIG_MMU */ 515 516 /** 517 * dma_can_mmap - check if a given device supports dma_mmap_* 518 * @dev: device to check 519 * 520 * Returns %true if @dev supports dma_mmap_coherent() and dma_mmap_attrs() to 521 * map DMA allocations to userspace. 522 */ 523 bool dma_can_mmap(struct device *dev) 524 { 525 const struct dma_map_ops *ops = get_dma_ops(dev); 526 527 if (dma_alloc_direct(dev, ops)) 528 return dma_direct_can_mmap(dev); 529 if (use_dma_iommu(dev)) 530 return true; 531 return ops->mmap != NULL; 532 } 533 EXPORT_SYMBOL_GPL(dma_can_mmap); 534 535 /** 536 * dma_mmap_attrs - map a coherent DMA allocation into user space 537 * @dev: valid struct device pointer, or NULL for ISA and EISA-like devices 538 * @vma: vm_area_struct describing requested user mapping 539 * @cpu_addr: kernel CPU-view address returned from dma_alloc_attrs 540 * @dma_addr: device-view address returned from dma_alloc_attrs 541 * @size: size of memory originally requested in dma_alloc_attrs 542 * @attrs: attributes of mapping properties requested in dma_alloc_attrs 543 * 544 * Map a coherent DMA buffer previously allocated by dma_alloc_attrs into user 545 * space. The coherent DMA buffer must not be freed by the driver until the 546 * user space mapping has been released. 547 */ 548 int dma_mmap_attrs(struct device *dev, struct vm_area_struct *vma, 549 void *cpu_addr, dma_addr_t dma_addr, size_t size, 550 unsigned long attrs) 551 { 552 const struct dma_map_ops *ops = get_dma_ops(dev); 553 554 if (dma_alloc_direct(dev, ops)) 555 return dma_direct_mmap(dev, vma, cpu_addr, dma_addr, size, 556 attrs); 557 if (use_dma_iommu(dev)) 558 return iommu_dma_mmap(dev, vma, cpu_addr, dma_addr, size, 559 attrs); 560 if (!ops->mmap) 561 return -ENXIO; 562 return ops->mmap(dev, vma, cpu_addr, dma_addr, size, attrs); 563 } 564 EXPORT_SYMBOL(dma_mmap_attrs); 565 566 u64 dma_get_required_mask(struct device *dev) 567 { 568 const struct dma_map_ops *ops = get_dma_ops(dev); 569 570 if (dma_alloc_direct(dev, ops)) 571 return dma_direct_get_required_mask(dev); 572 573 if (use_dma_iommu(dev)) 574 return DMA_BIT_MASK(32); 575 576 if (ops->get_required_mask) 577 return ops->get_required_mask(dev); 578 579 /* 580 * We require every DMA ops implementation to at least support a 32-bit 581 * DMA mask (and use bounce buffering if that isn't supported in 582 * hardware). As the direct mapping code has its own routine to 583 * actually report an optimal mask we default to 32-bit here as that 584 * is the right thing for most IOMMUs, and at least not actively 585 * harmful in general. 586 */ 587 return DMA_BIT_MASK(32); 588 } 589 EXPORT_SYMBOL_GPL(dma_get_required_mask); 590 591 void *dma_alloc_attrs(struct device *dev, size_t size, dma_addr_t *dma_handle, 592 gfp_t flag, unsigned long attrs) 593 { 594 const struct dma_map_ops *ops = get_dma_ops(dev); 595 void *cpu_addr; 596 597 WARN_ON_ONCE(!dev->coherent_dma_mask); 598 599 /* 600 * DMA allocations can never be turned back into a page pointer, so 601 * requesting compound pages doesn't make sense (and can't even be 602 * supported at all by various backends). 603 */ 604 if (WARN_ON_ONCE(flag & __GFP_COMP)) 605 return NULL; 606 607 if (dma_alloc_from_dev_coherent(dev, size, dma_handle, &cpu_addr)) 608 return cpu_addr; 609 610 /* let the implementation decide on the zone to allocate from: */ 611 flag &= ~(__GFP_DMA | __GFP_DMA32 | __GFP_HIGHMEM); 612 613 if (dma_alloc_direct(dev, ops)) 614 cpu_addr = dma_direct_alloc(dev, size, dma_handle, flag, attrs); 615 else if (use_dma_iommu(dev)) 616 cpu_addr = iommu_dma_alloc(dev, size, dma_handle, flag, attrs); 617 else if (ops->alloc) 618 cpu_addr = ops->alloc(dev, size, dma_handle, flag, attrs); 619 else 620 return NULL; 621 622 trace_dma_alloc(dev, cpu_addr, *dma_handle, size, flag, attrs); 623 debug_dma_alloc_coherent(dev, size, *dma_handle, cpu_addr, attrs); 624 return cpu_addr; 625 } 626 EXPORT_SYMBOL(dma_alloc_attrs); 627 628 void dma_free_attrs(struct device *dev, size_t size, void *cpu_addr, 629 dma_addr_t dma_handle, unsigned long attrs) 630 { 631 const struct dma_map_ops *ops = get_dma_ops(dev); 632 633 if (dma_release_from_dev_coherent(dev, get_order(size), cpu_addr)) 634 return; 635 /* 636 * On non-coherent platforms which implement DMA-coherent buffers via 637 * non-cacheable remaps, ops->free() may call vunmap(). Thus getting 638 * this far in IRQ context is a) at risk of a BUG_ON() or trying to 639 * sleep on some machines, and b) an indication that the driver is 640 * probably misusing the coherent API anyway. 641 */ 642 WARN_ON(irqs_disabled()); 643 644 if (!cpu_addr) 645 return; 646 647 trace_dma_free(dev, cpu_addr, dma_handle, size, attrs); 648 debug_dma_free_coherent(dev, size, cpu_addr, dma_handle); 649 if (dma_alloc_direct(dev, ops)) 650 dma_direct_free(dev, size, cpu_addr, dma_handle, attrs); 651 else if (use_dma_iommu(dev)) 652 iommu_dma_free(dev, size, cpu_addr, dma_handle, attrs); 653 else if (ops->free) 654 ops->free(dev, size, cpu_addr, dma_handle, attrs); 655 } 656 EXPORT_SYMBOL(dma_free_attrs); 657 658 static struct page *__dma_alloc_pages(struct device *dev, size_t size, 659 dma_addr_t *dma_handle, enum dma_data_direction dir, gfp_t gfp) 660 { 661 const struct dma_map_ops *ops = get_dma_ops(dev); 662 663 if (WARN_ON_ONCE(!dev->coherent_dma_mask)) 664 return NULL; 665 if (WARN_ON_ONCE(gfp & (__GFP_DMA | __GFP_DMA32 | __GFP_HIGHMEM))) 666 return NULL; 667 if (WARN_ON_ONCE(gfp & __GFP_COMP)) 668 return NULL; 669 670 size = PAGE_ALIGN(size); 671 if (dma_alloc_direct(dev, ops)) 672 return dma_direct_alloc_pages(dev, size, dma_handle, dir, gfp); 673 if (use_dma_iommu(dev)) 674 return dma_common_alloc_pages(dev, size, dma_handle, dir, gfp); 675 if (!ops->alloc_pages_op) 676 return NULL; 677 return ops->alloc_pages_op(dev, size, dma_handle, dir, gfp); 678 } 679 680 struct page *dma_alloc_pages(struct device *dev, size_t size, 681 dma_addr_t *dma_handle, enum dma_data_direction dir, gfp_t gfp) 682 { 683 struct page *page = __dma_alloc_pages(dev, size, dma_handle, dir, gfp); 684 685 if (page) { 686 trace_dma_map_page(dev, page_to_phys(page), *dma_handle, size, 687 dir, 0); 688 debug_dma_map_page(dev, page, 0, size, dir, *dma_handle, 0); 689 } 690 return page; 691 } 692 EXPORT_SYMBOL_GPL(dma_alloc_pages); 693 694 static void __dma_free_pages(struct device *dev, size_t size, struct page *page, 695 dma_addr_t dma_handle, enum dma_data_direction dir) 696 { 697 const struct dma_map_ops *ops = get_dma_ops(dev); 698 699 size = PAGE_ALIGN(size); 700 if (dma_alloc_direct(dev, ops)) 701 dma_direct_free_pages(dev, size, page, dma_handle, dir); 702 else if (use_dma_iommu(dev)) 703 dma_common_free_pages(dev, size, page, dma_handle, dir); 704 else if (ops->free_pages) 705 ops->free_pages(dev, size, page, dma_handle, dir); 706 } 707 708 void dma_free_pages(struct device *dev, size_t size, struct page *page, 709 dma_addr_t dma_handle, enum dma_data_direction dir) 710 { 711 trace_dma_unmap_page(dev, dma_handle, size, dir, 0); 712 debug_dma_unmap_page(dev, dma_handle, size, dir); 713 __dma_free_pages(dev, size, page, dma_handle, dir); 714 } 715 EXPORT_SYMBOL_GPL(dma_free_pages); 716 717 int dma_mmap_pages(struct device *dev, struct vm_area_struct *vma, 718 size_t size, struct page *page) 719 { 720 unsigned long count = PAGE_ALIGN(size) >> PAGE_SHIFT; 721 722 if (vma->vm_pgoff >= count || vma_pages(vma) > count - vma->vm_pgoff) 723 return -ENXIO; 724 return remap_pfn_range(vma, vma->vm_start, 725 page_to_pfn(page) + vma->vm_pgoff, 726 vma_pages(vma) << PAGE_SHIFT, vma->vm_page_prot); 727 } 728 EXPORT_SYMBOL_GPL(dma_mmap_pages); 729 730 static struct sg_table *alloc_single_sgt(struct device *dev, size_t size, 731 enum dma_data_direction dir, gfp_t gfp) 732 { 733 struct sg_table *sgt; 734 struct page *page; 735 736 sgt = kmalloc(sizeof(*sgt), gfp); 737 if (!sgt) 738 return NULL; 739 if (sg_alloc_table(sgt, 1, gfp)) 740 goto out_free_sgt; 741 page = __dma_alloc_pages(dev, size, &sgt->sgl->dma_address, dir, gfp); 742 if (!page) 743 goto out_free_table; 744 sg_set_page(sgt->sgl, page, PAGE_ALIGN(size), 0); 745 sg_dma_len(sgt->sgl) = sgt->sgl->length; 746 return sgt; 747 out_free_table: 748 sg_free_table(sgt); 749 out_free_sgt: 750 kfree(sgt); 751 return NULL; 752 } 753 754 struct sg_table *dma_alloc_noncontiguous(struct device *dev, size_t size, 755 enum dma_data_direction dir, gfp_t gfp, unsigned long attrs) 756 { 757 struct sg_table *sgt; 758 759 if (WARN_ON_ONCE(attrs & ~DMA_ATTR_ALLOC_SINGLE_PAGES)) 760 return NULL; 761 if (WARN_ON_ONCE(gfp & __GFP_COMP)) 762 return NULL; 763 764 if (use_dma_iommu(dev)) 765 sgt = iommu_dma_alloc_noncontiguous(dev, size, dir, gfp, attrs); 766 else 767 sgt = alloc_single_sgt(dev, size, dir, gfp); 768 769 if (sgt) { 770 sgt->nents = 1; 771 trace_dma_map_sg(dev, sgt->sgl, sgt->orig_nents, 1, dir, attrs); 772 debug_dma_map_sg(dev, sgt->sgl, sgt->orig_nents, 1, dir, attrs); 773 } 774 return sgt; 775 } 776 EXPORT_SYMBOL_GPL(dma_alloc_noncontiguous); 777 778 static void free_single_sgt(struct device *dev, size_t size, 779 struct sg_table *sgt, enum dma_data_direction dir) 780 { 781 __dma_free_pages(dev, size, sg_page(sgt->sgl), sgt->sgl->dma_address, 782 dir); 783 sg_free_table(sgt); 784 kfree(sgt); 785 } 786 787 void dma_free_noncontiguous(struct device *dev, size_t size, 788 struct sg_table *sgt, enum dma_data_direction dir) 789 { 790 trace_dma_unmap_sg(dev, sgt->sgl, sgt->orig_nents, dir, 0); 791 debug_dma_unmap_sg(dev, sgt->sgl, sgt->orig_nents, dir); 792 793 if (use_dma_iommu(dev)) 794 iommu_dma_free_noncontiguous(dev, size, sgt, dir); 795 else 796 free_single_sgt(dev, size, sgt, dir); 797 } 798 EXPORT_SYMBOL_GPL(dma_free_noncontiguous); 799 800 void *dma_vmap_noncontiguous(struct device *dev, size_t size, 801 struct sg_table *sgt) 802 { 803 804 if (use_dma_iommu(dev)) 805 return iommu_dma_vmap_noncontiguous(dev, size, sgt); 806 807 return page_address(sg_page(sgt->sgl)); 808 } 809 EXPORT_SYMBOL_GPL(dma_vmap_noncontiguous); 810 811 void dma_vunmap_noncontiguous(struct device *dev, void *vaddr) 812 { 813 if (use_dma_iommu(dev)) 814 iommu_dma_vunmap_noncontiguous(dev, vaddr); 815 } 816 EXPORT_SYMBOL_GPL(dma_vunmap_noncontiguous); 817 818 int dma_mmap_noncontiguous(struct device *dev, struct vm_area_struct *vma, 819 size_t size, struct sg_table *sgt) 820 { 821 if (use_dma_iommu(dev)) 822 return iommu_dma_mmap_noncontiguous(dev, vma, size, sgt); 823 return dma_mmap_pages(dev, vma, size, sg_page(sgt->sgl)); 824 } 825 EXPORT_SYMBOL_GPL(dma_mmap_noncontiguous); 826 827 static int dma_supported(struct device *dev, u64 mask) 828 { 829 const struct dma_map_ops *ops = get_dma_ops(dev); 830 831 if (use_dma_iommu(dev)) { 832 if (WARN_ON(ops)) 833 return false; 834 return true; 835 } 836 837 /* 838 * ->dma_supported sets and clears the bypass flag, so ignore it here 839 * and always call into the method if there is one. 840 */ 841 if (ops) { 842 if (!ops->dma_supported) 843 return true; 844 return ops->dma_supported(dev, mask); 845 } 846 847 return dma_direct_supported(dev, mask); 848 } 849 850 bool dma_pci_p2pdma_supported(struct device *dev) 851 { 852 const struct dma_map_ops *ops = get_dma_ops(dev); 853 854 /* 855 * Note: dma_ops_bypass is not checked here because P2PDMA should 856 * not be used with dma mapping ops that do not have support even 857 * if the specific device is bypassing them. 858 */ 859 860 /* if ops is not set, dma direct and default IOMMU support P2PDMA */ 861 return !ops; 862 } 863 EXPORT_SYMBOL_GPL(dma_pci_p2pdma_supported); 864 865 int dma_set_mask(struct device *dev, u64 mask) 866 { 867 /* 868 * Truncate the mask to the actually supported dma_addr_t width to 869 * avoid generating unsupportable addresses. 870 */ 871 mask = (dma_addr_t)mask; 872 873 if (!dev->dma_mask || !dma_supported(dev, mask)) 874 return -EIO; 875 876 arch_dma_set_mask(dev, mask); 877 *dev->dma_mask = mask; 878 dma_setup_need_sync(dev); 879 880 return 0; 881 } 882 EXPORT_SYMBOL(dma_set_mask); 883 884 int dma_set_coherent_mask(struct device *dev, u64 mask) 885 { 886 /* 887 * Truncate the mask to the actually supported dma_addr_t width to 888 * avoid generating unsupportable addresses. 889 */ 890 mask = (dma_addr_t)mask; 891 892 if (!dma_supported(dev, mask)) 893 return -EIO; 894 895 dev->coherent_dma_mask = mask; 896 return 0; 897 } 898 EXPORT_SYMBOL(dma_set_coherent_mask); 899 900 /** 901 * dma_addressing_limited - return if the device is addressing limited 902 * @dev: device to check 903 * 904 * Return %true if the devices DMA mask is too small to address all memory in 905 * the system, else %false. Lack of addressing bits is the prime reason for 906 * bounce buffering, but might not be the only one. 907 */ 908 bool dma_addressing_limited(struct device *dev) 909 { 910 const struct dma_map_ops *ops = get_dma_ops(dev); 911 912 if (min_not_zero(dma_get_mask(dev), dev->bus_dma_limit) < 913 dma_get_required_mask(dev)) 914 return true; 915 916 if (unlikely(ops) || use_dma_iommu(dev)) 917 return false; 918 return !dma_direct_all_ram_mapped(dev); 919 } 920 EXPORT_SYMBOL_GPL(dma_addressing_limited); 921 922 size_t dma_max_mapping_size(struct device *dev) 923 { 924 const struct dma_map_ops *ops = get_dma_ops(dev); 925 size_t size = SIZE_MAX; 926 927 if (dma_map_direct(dev, ops)) 928 size = dma_direct_max_mapping_size(dev); 929 else if (use_dma_iommu(dev)) 930 size = iommu_dma_max_mapping_size(dev); 931 else if (ops && ops->max_mapping_size) 932 size = ops->max_mapping_size(dev); 933 934 return size; 935 } 936 EXPORT_SYMBOL_GPL(dma_max_mapping_size); 937 938 size_t dma_opt_mapping_size(struct device *dev) 939 { 940 const struct dma_map_ops *ops = get_dma_ops(dev); 941 size_t size = SIZE_MAX; 942 943 if (use_dma_iommu(dev)) 944 size = iommu_dma_opt_mapping_size(); 945 else if (ops && ops->opt_mapping_size) 946 size = ops->opt_mapping_size(); 947 948 return min(dma_max_mapping_size(dev), size); 949 } 950 EXPORT_SYMBOL_GPL(dma_opt_mapping_size); 951 952 unsigned long dma_get_merge_boundary(struct device *dev) 953 { 954 const struct dma_map_ops *ops = get_dma_ops(dev); 955 956 if (use_dma_iommu(dev)) 957 return iommu_dma_get_merge_boundary(dev); 958 959 if (!ops || !ops->get_merge_boundary) 960 return 0; /* can't merge */ 961 962 return ops->get_merge_boundary(dev); 963 } 964 EXPORT_SYMBOL_GPL(dma_get_merge_boundary); 965