1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * arch-independent dma-mapping routines 4 * 5 * Copyright (c) 2006 SUSE Linux Products GmbH 6 * Copyright (c) 2006 Tejun Heo <teheo@suse.de> 7 */ 8 #include <linux/memblock.h> /* for max_pfn */ 9 #include <linux/acpi.h> 10 #include <linux/dma-map-ops.h> 11 #include <linux/export.h> 12 #include <linux/gfp.h> 13 #include <linux/iommu-dma.h> 14 #include <linux/kmsan.h> 15 #include <linux/of_device.h> 16 #include <linux/slab.h> 17 #include <linux/vmalloc.h> 18 #include "debug.h" 19 #include "direct.h" 20 21 #define CREATE_TRACE_POINTS 22 #include <trace/events/dma.h> 23 24 #if defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_DEVICE) || \ 25 defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_CPU) || \ 26 defined(CONFIG_ARCH_HAS_SYNC_DMA_FOR_CPU_ALL) 27 bool dma_default_coherent = IS_ENABLED(CONFIG_ARCH_DMA_DEFAULT_COHERENT); 28 #endif 29 30 /* 31 * Managed DMA API 32 */ 33 struct dma_devres { 34 size_t size; 35 void *vaddr; 36 dma_addr_t dma_handle; 37 unsigned long attrs; 38 }; 39 40 static void dmam_release(struct device *dev, void *res) 41 { 42 struct dma_devres *this = res; 43 44 dma_free_attrs(dev, this->size, this->vaddr, this->dma_handle, 45 this->attrs); 46 } 47 48 static int dmam_match(struct device *dev, void *res, void *match_data) 49 { 50 struct dma_devres *this = res, *match = match_data; 51 52 if (this->vaddr == match->vaddr) { 53 WARN_ON(this->size != match->size || 54 this->dma_handle != match->dma_handle); 55 return 1; 56 } 57 return 0; 58 } 59 60 /** 61 * dmam_free_coherent - Managed dma_free_coherent() 62 * @dev: Device to free coherent memory for 63 * @size: Size of allocation 64 * @vaddr: Virtual address of the memory to free 65 * @dma_handle: DMA handle of the memory to free 66 * 67 * Managed dma_free_coherent(). 68 */ 69 void dmam_free_coherent(struct device *dev, size_t size, void *vaddr, 70 dma_addr_t dma_handle) 71 { 72 struct dma_devres match_data = { size, vaddr, dma_handle }; 73 74 WARN_ON(devres_destroy(dev, dmam_release, dmam_match, &match_data)); 75 dma_free_coherent(dev, size, vaddr, dma_handle); 76 } 77 EXPORT_SYMBOL(dmam_free_coherent); 78 79 /** 80 * dmam_alloc_attrs - Managed dma_alloc_attrs() 81 * @dev: Device to allocate non_coherent memory for 82 * @size: Size of allocation 83 * @dma_handle: Out argument for allocated DMA handle 84 * @gfp: Allocation flags 85 * @attrs: Flags in the DMA_ATTR_* namespace. 86 * 87 * Managed dma_alloc_attrs(). Memory allocated using this function will be 88 * automatically released on driver detach. 89 * 90 * RETURNS: 91 * Pointer to allocated memory on success, NULL on failure. 92 */ 93 void *dmam_alloc_attrs(struct device *dev, size_t size, dma_addr_t *dma_handle, 94 gfp_t gfp, unsigned long attrs) 95 { 96 struct dma_devres *dr; 97 void *vaddr; 98 99 dr = devres_alloc(dmam_release, sizeof(*dr), gfp); 100 if (!dr) 101 return NULL; 102 103 vaddr = dma_alloc_attrs(dev, size, dma_handle, gfp, attrs); 104 if (!vaddr) { 105 devres_free(dr); 106 return NULL; 107 } 108 109 dr->vaddr = vaddr; 110 dr->dma_handle = *dma_handle; 111 dr->size = size; 112 dr->attrs = attrs; 113 114 devres_add(dev, dr); 115 116 return vaddr; 117 } 118 EXPORT_SYMBOL(dmam_alloc_attrs); 119 120 static bool dma_go_direct(struct device *dev, dma_addr_t mask, 121 const struct dma_map_ops *ops) 122 { 123 if (use_dma_iommu(dev)) 124 return false; 125 126 if (likely(!ops)) 127 return true; 128 129 if (IS_ENABLED(CONFIG_DMA_OPS_BYPASS) && dev_dma_ops_bypass(dev)) 130 return min_not_zero(mask, dev->bus_dma_limit) >= 131 dma_direct_get_required_mask(dev); 132 return false; 133 } 134 135 136 /* 137 * Check if the devices uses a direct mapping for streaming DMA operations. 138 * This allows IOMMU drivers to set a bypass mode if the DMA mask is large 139 * enough. 140 */ 141 static inline bool dma_alloc_direct(struct device *dev, 142 const struct dma_map_ops *ops) 143 { 144 return dma_go_direct(dev, dev->coherent_dma_mask, ops); 145 } 146 147 static inline bool dma_map_direct(struct device *dev, 148 const struct dma_map_ops *ops) 149 { 150 return dma_go_direct(dev, *dev->dma_mask, ops); 151 } 152 153 dma_addr_t dma_map_phys(struct device *dev, phys_addr_t phys, size_t size, 154 enum dma_data_direction dir, unsigned long attrs) 155 { 156 const struct dma_map_ops *ops = get_dma_ops(dev); 157 bool is_mmio = attrs & DMA_ATTR_MMIO; 158 bool is_cc_shared = attrs & DMA_ATTR_CC_SHARED; 159 dma_addr_t addr = DMA_MAPPING_ERROR; 160 161 BUG_ON(!valid_dma_direction(dir)); 162 163 if (WARN_ON_ONCE(!dev->dma_mask)) 164 return DMA_MAPPING_ERROR; 165 166 if (!dev_is_dma_coherent(dev) && (attrs & DMA_ATTR_REQUIRE_COHERENT)) 167 return DMA_MAPPING_ERROR; 168 169 if (dma_map_direct(dev, ops) || 170 (!is_mmio && !is_cc_shared && 171 arch_dma_map_phys_direct(dev, phys + size))) 172 addr = dma_direct_map_phys(dev, phys, size, dir, attrs, true); 173 else if (is_cc_shared) 174 return DMA_MAPPING_ERROR; 175 else if (use_dma_iommu(dev)) 176 addr = iommu_dma_map_phys(dev, phys, size, dir, attrs); 177 else if (ops->map_phys) 178 addr = ops->map_phys(dev, phys, size, dir, attrs); 179 180 if (!is_mmio) 181 kmsan_handle_dma(phys, size, dir); 182 trace_dma_map_phys(dev, phys, addr, size, dir, attrs); 183 debug_dma_map_phys(dev, phys, size, dir, addr, attrs); 184 185 return addr; 186 } 187 EXPORT_SYMBOL_GPL(dma_map_phys); 188 189 dma_addr_t dma_map_page_attrs(struct device *dev, struct page *page, 190 size_t offset, size_t size, enum dma_data_direction dir, 191 unsigned long attrs) 192 { 193 phys_addr_t phys = page_to_phys(page) + offset; 194 195 if (unlikely(attrs & DMA_ATTR_MMIO)) 196 return DMA_MAPPING_ERROR; 197 198 if (IS_ENABLED(CONFIG_DMA_API_DEBUG) && 199 WARN_ON_ONCE(is_zone_device_page(page))) 200 return DMA_MAPPING_ERROR; 201 202 return dma_map_phys(dev, phys, size, dir, attrs); 203 } 204 EXPORT_SYMBOL(dma_map_page_attrs); 205 206 void dma_unmap_phys(struct device *dev, dma_addr_t addr, size_t size, 207 enum dma_data_direction dir, unsigned long attrs) 208 { 209 const struct dma_map_ops *ops = get_dma_ops(dev); 210 bool is_mmio = attrs & DMA_ATTR_MMIO; 211 bool is_cc_shared = attrs & DMA_ATTR_CC_SHARED; 212 213 BUG_ON(!valid_dma_direction(dir)); 214 215 if (dma_map_direct(dev, ops) || 216 (!is_mmio && !is_cc_shared && 217 arch_dma_unmap_phys_direct(dev, addr + size))) 218 dma_direct_unmap_phys(dev, addr, size, dir, attrs, true); 219 else if (is_cc_shared) 220 return; 221 else if (use_dma_iommu(dev)) 222 iommu_dma_unmap_phys(dev, addr, size, dir, attrs); 223 else if (ops->unmap_phys) 224 ops->unmap_phys(dev, addr, size, dir, attrs); 225 trace_dma_unmap_phys(dev, addr, size, dir, attrs); 226 debug_dma_unmap_phys(dev, addr, size, dir, attrs); 227 } 228 EXPORT_SYMBOL_GPL(dma_unmap_phys); 229 230 void dma_unmap_page_attrs(struct device *dev, dma_addr_t addr, size_t size, 231 enum dma_data_direction dir, unsigned long attrs) 232 { 233 if (unlikely(attrs & DMA_ATTR_MMIO)) 234 return; 235 236 dma_unmap_phys(dev, addr, size, dir, attrs); 237 } 238 EXPORT_SYMBOL(dma_unmap_page_attrs); 239 240 static int __dma_map_sg_attrs(struct device *dev, struct scatterlist *sg, 241 int nents, enum dma_data_direction dir, unsigned long attrs) 242 { 243 const struct dma_map_ops *ops = get_dma_ops(dev); 244 int ents; 245 246 BUG_ON(!valid_dma_direction(dir)); 247 248 if (!dev_is_dma_coherent(dev) && (attrs & DMA_ATTR_REQUIRE_COHERENT)) 249 return -EOPNOTSUPP; 250 251 if (WARN_ON_ONCE(!dev->dma_mask)) 252 return 0; 253 254 if (dma_map_direct(dev, ops) || 255 arch_dma_map_sg_direct(dev, sg, nents)) 256 ents = dma_direct_map_sg(dev, sg, nents, dir, attrs); 257 else if (use_dma_iommu(dev)) 258 ents = iommu_dma_map_sg(dev, sg, nents, dir, attrs); 259 else 260 ents = ops->map_sg(dev, sg, nents, dir, attrs); 261 262 if (ents > 0) { 263 kmsan_handle_dma_sg(sg, nents, dir); 264 trace_dma_map_sg(dev, sg, nents, ents, dir, attrs); 265 debug_dma_map_sg(dev, sg, nents, ents, dir, attrs); 266 } else if (WARN_ON_ONCE(ents != -EINVAL && ents != -ENOMEM && 267 ents != -EIO && ents != -EREMOTEIO)) { 268 trace_dma_map_sg_err(dev, sg, nents, ents, dir, attrs); 269 return -EIO; 270 } 271 272 return ents; 273 } 274 275 /** 276 * dma_map_sg_attrs - Map the given buffer for DMA 277 * @dev: The device for which to perform the DMA operation 278 * @sg: The sg_table object describing the buffer 279 * @nents: Number of entries to map 280 * @dir: DMA direction 281 * @attrs: Optional DMA attributes for the map operation 282 * 283 * Maps a buffer described by a scatterlist passed in the sg argument with 284 * nents segments for the @dir DMA operation by the @dev device. 285 * 286 * Returns the number of mapped entries (which can be less than nents) 287 * on success. Zero is returned for any error. 288 * 289 * dma_unmap_sg_attrs() should be used to unmap the buffer with the 290 * original sg and original nents (not the value returned by this funciton). 291 */ 292 unsigned int dma_map_sg_attrs(struct device *dev, struct scatterlist *sg, 293 int nents, enum dma_data_direction dir, unsigned long attrs) 294 { 295 int ret; 296 297 ret = __dma_map_sg_attrs(dev, sg, nents, dir, attrs); 298 if (ret < 0) 299 return 0; 300 return ret; 301 } 302 EXPORT_SYMBOL(dma_map_sg_attrs); 303 304 /** 305 * dma_map_sgtable - Map the given buffer for DMA 306 * @dev: The device for which to perform the DMA operation 307 * @sgt: The sg_table object describing the buffer 308 * @dir: DMA direction 309 * @attrs: Optional DMA attributes for the map operation 310 * 311 * Maps a buffer described by a scatterlist stored in the given sg_table 312 * object for the @dir DMA operation by the @dev device. After success, the 313 * ownership for the buffer is transferred to the DMA domain. One has to 314 * call dma_sync_sgtable_for_cpu() or dma_unmap_sgtable() to move the 315 * ownership of the buffer back to the CPU domain before touching the 316 * buffer by the CPU. 317 * 318 * Returns 0 on success or a negative error code on error. The following 319 * error codes are supported with the given meaning: 320 * 321 * -EINVAL An invalid argument, unaligned access or other error 322 * in usage. Will not succeed if retried. 323 * -ENOMEM Insufficient resources (like memory or IOVA space) to 324 * complete the mapping. Should succeed if retried later. 325 * -EIO Legacy error code with an unknown meaning. eg. this is 326 * returned if a lower level call returned 327 * DMA_MAPPING_ERROR. 328 * -EREMOTEIO The DMA device cannot access P2PDMA memory specified 329 * in the sg_table. This will not succeed if retried. 330 */ 331 int dma_map_sgtable(struct device *dev, struct sg_table *sgt, 332 enum dma_data_direction dir, unsigned long attrs) 333 { 334 int nents; 335 336 nents = __dma_map_sg_attrs(dev, sgt->sgl, sgt->orig_nents, dir, attrs); 337 if (nents < 0) 338 return nents; 339 sgt->nents = nents; 340 return 0; 341 } 342 EXPORT_SYMBOL_GPL(dma_map_sgtable); 343 344 void dma_unmap_sg_attrs(struct device *dev, struct scatterlist *sg, 345 int nents, enum dma_data_direction dir, 346 unsigned long attrs) 347 { 348 const struct dma_map_ops *ops = get_dma_ops(dev); 349 350 BUG_ON(!valid_dma_direction(dir)); 351 trace_dma_unmap_sg(dev, sg, nents, dir, attrs); 352 debug_dma_unmap_sg(dev, sg, nents, dir, attrs); 353 if (dma_map_direct(dev, ops) || 354 arch_dma_unmap_sg_direct(dev, sg, nents)) 355 dma_direct_unmap_sg(dev, sg, nents, dir, attrs); 356 else if (use_dma_iommu(dev)) 357 iommu_dma_unmap_sg(dev, sg, nents, dir, attrs); 358 else if (ops->unmap_sg) 359 ops->unmap_sg(dev, sg, nents, dir, attrs); 360 } 361 EXPORT_SYMBOL(dma_unmap_sg_attrs); 362 363 dma_addr_t dma_map_resource(struct device *dev, phys_addr_t phys_addr, 364 size_t size, enum dma_data_direction dir, unsigned long attrs) 365 { 366 return dma_map_phys(dev, phys_addr, size, dir, attrs | DMA_ATTR_MMIO); 367 } 368 EXPORT_SYMBOL(dma_map_resource); 369 370 void dma_unmap_resource(struct device *dev, dma_addr_t addr, size_t size, 371 enum dma_data_direction dir, unsigned long attrs) 372 { 373 dma_unmap_phys(dev, addr, size, dir, attrs | DMA_ATTR_MMIO); 374 } 375 EXPORT_SYMBOL(dma_unmap_resource); 376 377 #ifdef CONFIG_DMA_NEED_SYNC 378 void __dma_sync_single_for_cpu(struct device *dev, dma_addr_t addr, size_t size, 379 enum dma_data_direction dir) 380 { 381 const struct dma_map_ops *ops = get_dma_ops(dev); 382 383 BUG_ON(!valid_dma_direction(dir)); 384 if (dma_map_direct(dev, ops)) 385 dma_direct_sync_single_for_cpu(dev, addr, size, dir, true); 386 else if (use_dma_iommu(dev)) 387 iommu_dma_sync_single_for_cpu(dev, addr, size, dir); 388 else if (ops->sync_single_for_cpu) 389 ops->sync_single_for_cpu(dev, addr, size, dir); 390 trace_dma_sync_single_for_cpu(dev, addr, size, dir); 391 debug_dma_sync_single_for_cpu(dev, addr, size, dir); 392 } 393 EXPORT_SYMBOL(__dma_sync_single_for_cpu); 394 395 void __dma_sync_single_for_device(struct device *dev, dma_addr_t addr, 396 size_t size, enum dma_data_direction dir) 397 { 398 const struct dma_map_ops *ops = get_dma_ops(dev); 399 400 BUG_ON(!valid_dma_direction(dir)); 401 if (dma_map_direct(dev, ops)) 402 dma_direct_sync_single_for_device(dev, addr, size, dir); 403 else if (use_dma_iommu(dev)) 404 iommu_dma_sync_single_for_device(dev, addr, size, dir); 405 else if (ops->sync_single_for_device) 406 ops->sync_single_for_device(dev, addr, size, dir); 407 trace_dma_sync_single_for_device(dev, addr, size, dir); 408 debug_dma_sync_single_for_device(dev, addr, size, dir); 409 } 410 EXPORT_SYMBOL(__dma_sync_single_for_device); 411 412 void __dma_sync_sg_for_cpu(struct device *dev, struct scatterlist *sg, 413 int nelems, enum dma_data_direction dir) 414 { 415 const struct dma_map_ops *ops = get_dma_ops(dev); 416 417 BUG_ON(!valid_dma_direction(dir)); 418 if (dma_map_direct(dev, ops)) 419 dma_direct_sync_sg_for_cpu(dev, sg, nelems, dir); 420 else if (use_dma_iommu(dev)) 421 iommu_dma_sync_sg_for_cpu(dev, sg, nelems, dir); 422 else if (ops->sync_sg_for_cpu) 423 ops->sync_sg_for_cpu(dev, sg, nelems, dir); 424 trace_dma_sync_sg_for_cpu(dev, sg, nelems, dir); 425 debug_dma_sync_sg_for_cpu(dev, sg, nelems, dir); 426 } 427 EXPORT_SYMBOL(__dma_sync_sg_for_cpu); 428 429 void __dma_sync_sg_for_device(struct device *dev, struct scatterlist *sg, 430 int nelems, enum dma_data_direction dir) 431 { 432 const struct dma_map_ops *ops = get_dma_ops(dev); 433 434 BUG_ON(!valid_dma_direction(dir)); 435 if (dma_map_direct(dev, ops)) 436 dma_direct_sync_sg_for_device(dev, sg, nelems, dir); 437 else if (use_dma_iommu(dev)) 438 iommu_dma_sync_sg_for_device(dev, sg, nelems, dir); 439 else if (ops->sync_sg_for_device) 440 ops->sync_sg_for_device(dev, sg, nelems, dir); 441 trace_dma_sync_sg_for_device(dev, sg, nelems, dir); 442 debug_dma_sync_sg_for_device(dev, sg, nelems, dir); 443 } 444 EXPORT_SYMBOL(__dma_sync_sg_for_device); 445 446 bool __dma_need_sync(struct device *dev, dma_addr_t dma_addr) 447 { 448 const struct dma_map_ops *ops = get_dma_ops(dev); 449 450 if (dma_map_direct(dev, ops)) 451 /* 452 * dma_skip_sync could've been reset on first SWIOTLB buffer 453 * mapping, but @dma_addr is not necessary an SWIOTLB buffer. 454 * In this case, fall back to more granular check. 455 */ 456 return dma_direct_need_sync(dev, dma_addr); 457 return true; 458 } 459 EXPORT_SYMBOL_GPL(__dma_need_sync); 460 461 /** 462 * dma_need_unmap - does this device need dma_unmap_* operations 463 * @dev: device to check 464 * 465 * If this function returns %false, drivers can skip calling dma_unmap_* after 466 * finishing an I/O. This function must be called after all mappings that might 467 * need to be unmapped have been performed. 468 */ 469 bool dma_need_unmap(struct device *dev) 470 { 471 if (!dma_map_direct(dev, get_dma_ops(dev))) 472 return true; 473 if (!dev_dma_skip_sync(dev)) 474 return true; 475 return IS_ENABLED(CONFIG_DMA_API_DEBUG); 476 } 477 EXPORT_SYMBOL_GPL(dma_need_unmap); 478 479 static void dma_setup_need_sync(struct device *dev) 480 { 481 const struct dma_map_ops *ops = get_dma_ops(dev); 482 483 if (dma_map_direct(dev, ops) || use_dma_iommu(dev)) 484 /* 485 * dma_skip_sync will be reset to %false on first SWIOTLB buffer 486 * mapping, if any. During the device initialization, it's 487 * enough to check only for the DMA coherence. 488 */ 489 dev_assign_dma_skip_sync(dev, dev_is_dma_coherent(dev)); 490 else if (!ops->sync_single_for_device && !ops->sync_single_for_cpu && 491 !ops->sync_sg_for_device && !ops->sync_sg_for_cpu) 492 /* 493 * Synchronization is not possible when none of DMA sync ops 494 * is set. 495 */ 496 dev_set_dma_skip_sync(dev); 497 else 498 dev_clear_dma_skip_sync(dev); 499 } 500 #else /* !CONFIG_DMA_NEED_SYNC */ 501 static inline void dma_setup_need_sync(struct device *dev) { } 502 #endif /* !CONFIG_DMA_NEED_SYNC */ 503 504 /* 505 * The whole dma_get_sgtable() idea is fundamentally unsafe - it seems 506 * that the intention is to allow exporting memory allocated via the 507 * coherent DMA APIs through the dma_buf API, which only accepts a 508 * scattertable. This presents a couple of problems: 509 * 1. Not all memory allocated via the coherent DMA APIs is backed by 510 * a struct page 511 * 2. Passing coherent DMA memory into the streaming APIs is not allowed 512 * as we will try to flush the memory through a different alias to that 513 * actually being used (and the flushes are redundant.) 514 */ 515 int dma_get_sgtable_attrs(struct device *dev, struct sg_table *sgt, 516 void *cpu_addr, dma_addr_t dma_addr, size_t size, 517 unsigned long attrs) 518 { 519 const struct dma_map_ops *ops = get_dma_ops(dev); 520 521 if (dma_alloc_direct(dev, ops)) 522 return dma_direct_get_sgtable(dev, sgt, cpu_addr, dma_addr, 523 size, attrs); 524 if (use_dma_iommu(dev)) 525 return iommu_dma_get_sgtable(dev, sgt, cpu_addr, dma_addr, 526 size, attrs); 527 if (!ops->get_sgtable) 528 return -ENXIO; 529 return ops->get_sgtable(dev, sgt, cpu_addr, dma_addr, size, attrs); 530 } 531 EXPORT_SYMBOL(dma_get_sgtable_attrs); 532 533 #ifdef CONFIG_MMU 534 /* 535 * Return the page attributes used for mapping dma_alloc_* memory, either in 536 * kernel space if remapping is needed, or to userspace through dma_mmap_*. 537 */ 538 pgprot_t dma_pgprot(struct device *dev, pgprot_t prot, unsigned long attrs) 539 { 540 pgprot_t dma_prot; 541 542 if (dev_is_dma_coherent(dev)) 543 dma_prot = prot; 544 #ifdef CONFIG_ARCH_HAS_DMA_WRITE_COMBINE 545 else if (attrs & DMA_ATTR_WRITE_COMBINE) 546 dma_prot = pgprot_writecombine(prot); 547 #endif 548 else 549 dma_prot = pgprot_dmacoherent(prot); 550 551 if (attrs & (DMA_ATTR_CC_SHARED | __DMA_ATTR_ALLOC_CC_SHARED)) 552 return pgprot_decrypted(dma_prot); 553 else 554 return pgprot_encrypted(dma_prot); 555 } 556 #endif /* CONFIG_MMU */ 557 558 /** 559 * dma_can_mmap - check if a given device supports dma_mmap_* 560 * @dev: device to check 561 * 562 * Returns %true if @dev supports dma_mmap_coherent() and dma_mmap_attrs() to 563 * map DMA allocations to userspace. 564 */ 565 bool dma_can_mmap(struct device *dev) 566 { 567 const struct dma_map_ops *ops = get_dma_ops(dev); 568 569 if (dma_alloc_direct(dev, ops)) 570 return dma_direct_can_mmap(dev); 571 if (use_dma_iommu(dev)) 572 return true; 573 return ops->mmap != NULL; 574 } 575 EXPORT_SYMBOL_GPL(dma_can_mmap); 576 577 /** 578 * dma_mmap_attrs - map a coherent DMA allocation into user space 579 * @dev: valid struct device pointer, or NULL for ISA and EISA-like devices 580 * @vma: vm_area_struct describing requested user mapping 581 * @cpu_addr: kernel CPU-view address returned from dma_alloc_attrs 582 * @dma_addr: device-view address returned from dma_alloc_attrs 583 * @size: size of memory originally requested in dma_alloc_attrs 584 * @attrs: attributes of mapping properties requested in dma_alloc_attrs 585 * 586 * Map a coherent DMA buffer previously allocated by dma_alloc_attrs into user 587 * space. The coherent DMA buffer must not be freed by the driver until the 588 * user space mapping has been released. 589 */ 590 int dma_mmap_attrs(struct device *dev, struct vm_area_struct *vma, 591 void *cpu_addr, dma_addr_t dma_addr, size_t size, 592 unsigned long attrs) 593 { 594 const struct dma_map_ops *ops = get_dma_ops(dev); 595 596 if (dma_alloc_direct(dev, ops)) 597 return dma_direct_mmap(dev, vma, cpu_addr, dma_addr, size, 598 attrs); 599 if (use_dma_iommu(dev)) 600 return iommu_dma_mmap(dev, vma, cpu_addr, dma_addr, size, 601 attrs); 602 if (!ops->mmap) 603 return -ENXIO; 604 return ops->mmap(dev, vma, cpu_addr, dma_addr, size, attrs); 605 } 606 EXPORT_SYMBOL(dma_mmap_attrs); 607 608 u64 dma_get_required_mask(struct device *dev) 609 { 610 const struct dma_map_ops *ops = get_dma_ops(dev); 611 612 if (dma_alloc_direct(dev, ops)) 613 return dma_direct_get_required_mask(dev); 614 615 if (use_dma_iommu(dev)) 616 return DMA_BIT_MASK(32); 617 618 if (ops->get_required_mask) 619 return ops->get_required_mask(dev); 620 621 /* 622 * We require every DMA ops implementation to at least support a 32-bit 623 * DMA mask (and use bounce buffering if that isn't supported in 624 * hardware). As the direct mapping code has its own routine to 625 * actually report an optimal mask we default to 32-bit here as that 626 * is the right thing for most IOMMUs, and at least not actively 627 * harmful in general. 628 */ 629 return DMA_BIT_MASK(32); 630 } 631 EXPORT_SYMBOL_GPL(dma_get_required_mask); 632 633 void *dma_alloc_attrs(struct device *dev, size_t size, dma_addr_t *dma_handle, 634 gfp_t flag, unsigned long attrs) 635 { 636 const struct dma_map_ops *ops = get_dma_ops(dev); 637 void *cpu_addr; 638 639 WARN_ON_ONCE(!dev->coherent_dma_mask); 640 641 /* 642 * DMA allocations can never be turned back into a page pointer, so 643 * requesting compound pages doesn't make sense (and can't even be 644 * supported at all by various backends). 645 */ 646 if (WARN_ON_ONCE(flag & __GFP_COMP)) 647 return NULL; 648 649 if (attrs & (DMA_ATTR_CC_SHARED | __DMA_ATTR_ALLOC_CC_SHARED)) { 650 trace_dma_alloc(dev, NULL, 0, size, DMA_BIDIRECTIONAL, flag, 651 attrs); 652 return NULL; 653 } 654 655 if (force_dma_unencrypted(dev)) 656 attrs |= __DMA_ATTR_ALLOC_CC_SHARED; 657 658 if (dma_alloc_from_dev_coherent(dev, size, dma_handle, &cpu_addr)) { 659 trace_dma_alloc(dev, cpu_addr, *dma_handle, size, 660 DMA_BIDIRECTIONAL, flag, attrs); 661 return cpu_addr; 662 } 663 664 /* let the implementation decide on the zone to allocate from: */ 665 flag &= ~(__GFP_DMA | __GFP_DMA32 | __GFP_HIGHMEM); 666 667 if (dma_alloc_direct(dev, ops) || arch_dma_alloc_direct(dev)) { 668 cpu_addr = dma_direct_alloc(dev, size, dma_handle, flag, attrs); 669 } else if (use_dma_iommu(dev)) { 670 cpu_addr = iommu_dma_alloc(dev, size, dma_handle, flag, attrs); 671 } else if (ops->alloc) { 672 cpu_addr = ops->alloc(dev, size, dma_handle, flag, attrs); 673 } else { 674 trace_dma_alloc(dev, NULL, 0, size, DMA_BIDIRECTIONAL, flag, 675 attrs); 676 return NULL; 677 } 678 679 trace_dma_alloc(dev, cpu_addr, *dma_handle, size, DMA_BIDIRECTIONAL, 680 flag, attrs); 681 debug_dma_alloc_coherent(dev, size, *dma_handle, cpu_addr, attrs); 682 return cpu_addr; 683 } 684 EXPORT_SYMBOL(dma_alloc_attrs); 685 686 void dma_free_attrs(struct device *dev, size_t size, void *cpu_addr, 687 dma_addr_t dma_handle, unsigned long attrs) 688 { 689 const struct dma_map_ops *ops = get_dma_ops(dev); 690 691 if (dma_release_from_dev_coherent(dev, get_order(size), cpu_addr)) 692 return; 693 /* 694 * On non-coherent platforms which implement DMA-coherent buffers via 695 * non-cacheable remaps, ops->free() may call vunmap(). Thus getting 696 * this far in IRQ context is a) at risk of a BUG_ON() or trying to 697 * sleep on some machines, and b) an indication that the driver is 698 * probably misusing the coherent API anyway. 699 */ 700 WARN_ON(irqs_disabled()); 701 702 trace_dma_free(dev, cpu_addr, dma_handle, size, DMA_BIDIRECTIONAL, 703 attrs); 704 if (!cpu_addr) 705 return; 706 707 debug_dma_free_coherent(dev, size, cpu_addr, dma_handle, attrs); 708 if (dma_alloc_direct(dev, ops) || arch_dma_free_direct(dev, dma_handle)) 709 dma_direct_free(dev, size, cpu_addr, dma_handle, attrs); 710 else if (use_dma_iommu(dev)) 711 iommu_dma_free(dev, size, cpu_addr, dma_handle, attrs); 712 else if (ops->free) 713 ops->free(dev, size, cpu_addr, dma_handle, attrs); 714 } 715 EXPORT_SYMBOL(dma_free_attrs); 716 717 static struct page *__dma_alloc_pages(struct device *dev, size_t size, 718 dma_addr_t *dma_handle, enum dma_data_direction dir, gfp_t gfp) 719 { 720 const struct dma_map_ops *ops = get_dma_ops(dev); 721 722 if (WARN_ON_ONCE(!dev->coherent_dma_mask)) 723 return NULL; 724 if (WARN_ON_ONCE(gfp & (__GFP_DMA | __GFP_DMA32 | __GFP_HIGHMEM))) 725 return NULL; 726 if (WARN_ON_ONCE(gfp & __GFP_COMP)) 727 return NULL; 728 729 size = PAGE_ALIGN(size); 730 if (dma_alloc_direct(dev, ops)) 731 return dma_direct_alloc_pages(dev, size, dma_handle, dir, gfp); 732 if (use_dma_iommu(dev)) 733 return dma_common_alloc_pages(dev, size, dma_handle, dir, gfp); 734 if (!ops->alloc_pages_op) 735 return NULL; 736 return ops->alloc_pages_op(dev, size, dma_handle, dir, gfp); 737 } 738 739 struct page *dma_alloc_pages(struct device *dev, size_t size, 740 dma_addr_t *dma_handle, enum dma_data_direction dir, gfp_t gfp) 741 { 742 struct page *page = __dma_alloc_pages(dev, size, dma_handle, dir, gfp); 743 744 if (page) { 745 trace_dma_alloc_pages(dev, page_to_virt(page), *dma_handle, 746 size, dir, gfp, 0); 747 debug_dma_alloc_pages(dev, page, size, dir, *dma_handle); 748 } else { 749 trace_dma_alloc_pages(dev, NULL, 0, size, dir, gfp, 0); 750 } 751 return page; 752 } 753 EXPORT_SYMBOL_GPL(dma_alloc_pages); 754 755 static void __dma_free_pages(struct device *dev, size_t size, struct page *page, 756 dma_addr_t dma_handle, enum dma_data_direction dir) 757 { 758 const struct dma_map_ops *ops = get_dma_ops(dev); 759 760 size = PAGE_ALIGN(size); 761 if (dma_alloc_direct(dev, ops)) 762 dma_direct_free_pages(dev, size, page, dma_handle, dir); 763 else if (use_dma_iommu(dev)) 764 dma_common_free_pages(dev, size, page, dma_handle, dir); 765 else if (ops->free_pages) 766 ops->free_pages(dev, size, page, dma_handle, dir); 767 } 768 769 void dma_free_pages(struct device *dev, size_t size, struct page *page, 770 dma_addr_t dma_handle, enum dma_data_direction dir) 771 { 772 trace_dma_free_pages(dev, page_to_virt(page), dma_handle, size, dir, 0); 773 debug_dma_free_pages(dev, page, size, dir, dma_handle); 774 __dma_free_pages(dev, size, page, dma_handle, dir); 775 } 776 EXPORT_SYMBOL_GPL(dma_free_pages); 777 778 int dma_mmap_pages(struct device *dev, struct vm_area_struct *vma, 779 size_t size, struct page *page) 780 { 781 const pgoff_t pgoff_start = vma_start_pgoff(vma); 782 const pgoff_t pgoff_end = vma_end_pgoff(vma); 783 const unsigned long count = PAGE_ALIGN(size) >> PAGE_SHIFT; 784 785 if (pgoff_start >= count || pgoff_end > count) 786 return -ENXIO; 787 return remap_pfn_range(vma, vma->vm_start, 788 page_to_pfn(page) + pgoff_start, 789 vma_pages(vma) << PAGE_SHIFT, vma->vm_page_prot); 790 } 791 EXPORT_SYMBOL_GPL(dma_mmap_pages); 792 793 static struct sg_table *alloc_single_sgt(struct device *dev, size_t size, 794 enum dma_data_direction dir, gfp_t gfp) 795 { 796 struct sg_table *sgt; 797 struct page *page; 798 799 sgt = kmalloc_obj(*sgt, gfp); 800 if (!sgt) 801 return NULL; 802 if (sg_alloc_table(sgt, 1, gfp)) 803 goto out_free_sgt; 804 page = __dma_alloc_pages(dev, size, &sgt->sgl->dma_address, dir, gfp); 805 if (!page) 806 goto out_free_table; 807 sg_set_page(sgt->sgl, page, PAGE_ALIGN(size), 0); 808 sg_dma_len(sgt->sgl) = sgt->sgl->length; 809 return sgt; 810 out_free_table: 811 sg_free_table(sgt); 812 out_free_sgt: 813 kfree(sgt); 814 return NULL; 815 } 816 817 struct sg_table *dma_alloc_noncontiguous(struct device *dev, size_t size, 818 enum dma_data_direction dir, gfp_t gfp, unsigned long attrs) 819 { 820 struct sg_table *sgt; 821 822 if (WARN_ON_ONCE(attrs & ~DMA_ATTR_ALLOC_SINGLE_PAGES)) 823 return NULL; 824 if (WARN_ON_ONCE(gfp & __GFP_COMP)) 825 return NULL; 826 827 if (use_dma_iommu(dev)) 828 sgt = iommu_dma_alloc_noncontiguous(dev, size, dir, gfp, attrs); 829 else 830 sgt = alloc_single_sgt(dev, size, dir, gfp); 831 832 if (sgt) { 833 sgt->nents = 1; 834 trace_dma_alloc_sgt(dev, sgt, size, dir, gfp, attrs); 835 debug_dma_map_sg(dev, sgt->sgl, sgt->orig_nents, 1, dir, attrs); 836 } else { 837 trace_dma_alloc_sgt_err(dev, NULL, 0, size, dir, gfp, attrs); 838 } 839 return sgt; 840 } 841 EXPORT_SYMBOL_GPL(dma_alloc_noncontiguous); 842 843 static void free_single_sgt(struct device *dev, size_t size, 844 struct sg_table *sgt, enum dma_data_direction dir) 845 { 846 __dma_free_pages(dev, size, sg_page(sgt->sgl), sgt->sgl->dma_address, 847 dir); 848 sg_free_table(sgt); 849 kfree(sgt); 850 } 851 852 void dma_free_noncontiguous(struct device *dev, size_t size, 853 struct sg_table *sgt, enum dma_data_direction dir) 854 { 855 trace_dma_free_sgt(dev, sgt, size, dir); 856 debug_dma_unmap_sg(dev, sgt->sgl, sgt->orig_nents, dir, 0); 857 858 if (use_dma_iommu(dev)) 859 iommu_dma_free_noncontiguous(dev, size, sgt, dir); 860 else 861 free_single_sgt(dev, size, sgt, dir); 862 } 863 EXPORT_SYMBOL_GPL(dma_free_noncontiguous); 864 865 void *dma_vmap_noncontiguous(struct device *dev, size_t size, 866 struct sg_table *sgt) 867 { 868 869 if (use_dma_iommu(dev)) 870 return iommu_dma_vmap_noncontiguous(dev, size, sgt); 871 872 return page_address(sg_page(sgt->sgl)); 873 } 874 EXPORT_SYMBOL_GPL(dma_vmap_noncontiguous); 875 876 void dma_vunmap_noncontiguous(struct device *dev, void *vaddr) 877 { 878 if (use_dma_iommu(dev)) 879 iommu_dma_vunmap_noncontiguous(dev, vaddr); 880 } 881 EXPORT_SYMBOL_GPL(dma_vunmap_noncontiguous); 882 883 int dma_mmap_noncontiguous(struct device *dev, struct vm_area_struct *vma, 884 size_t size, struct sg_table *sgt) 885 { 886 if (use_dma_iommu(dev)) 887 return iommu_dma_mmap_noncontiguous(dev, vma, size, sgt); 888 return dma_mmap_pages(dev, vma, size, sg_page(sgt->sgl)); 889 } 890 EXPORT_SYMBOL_GPL(dma_mmap_noncontiguous); 891 892 static int dma_supported(struct device *dev, u64 mask) 893 { 894 const struct dma_map_ops *ops = get_dma_ops(dev); 895 896 if (use_dma_iommu(dev)) { 897 if (WARN_ON(ops)) 898 return false; 899 return true; 900 } 901 902 /* 903 * ->dma_supported sets and clears the bypass flag, so ignore it here 904 * and always call into the method if there is one. 905 */ 906 if (ops) { 907 if (!ops->dma_supported) 908 return true; 909 return ops->dma_supported(dev, mask); 910 } 911 912 return dma_direct_supported(dev, mask); 913 } 914 915 bool dma_pci_p2pdma_supported(struct device *dev) 916 { 917 const struct dma_map_ops *ops = get_dma_ops(dev); 918 919 /* 920 * Note: dma_ops_bypass is not checked here because P2PDMA should 921 * not be used with dma mapping ops that do not have support even 922 * if the specific device is bypassing them. 923 */ 924 925 /* if ops is not set, dma direct and default IOMMU support P2PDMA */ 926 return !ops; 927 } 928 EXPORT_SYMBOL_GPL(dma_pci_p2pdma_supported); 929 930 int dma_set_mask(struct device *dev, u64 mask) 931 { 932 /* 933 * Truncate the mask to the actually supported dma_addr_t width to 934 * avoid generating unsupportable addresses. 935 */ 936 mask = (dma_addr_t)mask; 937 938 if (!dev->dma_mask || !dma_supported(dev, mask)) 939 return -EIO; 940 941 arch_dma_set_mask(dev, mask); 942 *dev->dma_mask = mask; 943 dma_setup_need_sync(dev); 944 945 return 0; 946 } 947 EXPORT_SYMBOL(dma_set_mask); 948 949 int dma_set_coherent_mask(struct device *dev, u64 mask) 950 { 951 /* 952 * Truncate the mask to the actually supported dma_addr_t width to 953 * avoid generating unsupportable addresses. 954 */ 955 mask = (dma_addr_t)mask; 956 957 if (!dma_supported(dev, mask)) 958 return -EIO; 959 960 dev->coherent_dma_mask = mask; 961 return 0; 962 } 963 EXPORT_SYMBOL(dma_set_coherent_mask); 964 965 static bool __dma_addressing_limited(struct device *dev) 966 { 967 const struct dma_map_ops *ops = get_dma_ops(dev); 968 969 if (min_not_zero(dma_get_mask(dev), dev->bus_dma_limit) < 970 dma_get_required_mask(dev)) 971 return true; 972 973 if (unlikely(ops) || use_dma_iommu(dev)) 974 return false; 975 return !dma_direct_all_ram_mapped(dev); 976 } 977 978 /** 979 * dma_addressing_limited - return if the device is addressing limited 980 * @dev: device to check 981 * 982 * Return %true if the devices DMA mask is too small to address all memory in 983 * the system, else %false. Lack of addressing bits is the prime reason for 984 * bounce buffering, but might not be the only one. 985 */ 986 bool dma_addressing_limited(struct device *dev) 987 { 988 if (!__dma_addressing_limited(dev)) 989 return false; 990 991 dev_dbg(dev, "device is DMA addressing limited\n"); 992 return true; 993 } 994 EXPORT_SYMBOL_GPL(dma_addressing_limited); 995 996 size_t dma_max_mapping_size(struct device *dev) 997 { 998 const struct dma_map_ops *ops = get_dma_ops(dev); 999 size_t size = SIZE_MAX; 1000 1001 if (!dev->dma_mask) 1002 return 0; 1003 1004 if (dma_map_direct(dev, ops)) 1005 size = dma_direct_max_mapping_size(dev); 1006 else if (use_dma_iommu(dev)) 1007 size = iommu_dma_max_mapping_size(dev); 1008 else if (ops && ops->max_mapping_size) 1009 size = ops->max_mapping_size(dev); 1010 1011 return size; 1012 } 1013 EXPORT_SYMBOL_GPL(dma_max_mapping_size); 1014 1015 size_t dma_opt_mapping_size(struct device *dev) 1016 { 1017 const struct dma_map_ops *ops = get_dma_ops(dev); 1018 size_t size = SIZE_MAX; 1019 1020 if (use_dma_iommu(dev)) 1021 size = iommu_dma_opt_mapping_size(); 1022 else if (ops && ops->opt_mapping_size) 1023 size = ops->opt_mapping_size(); 1024 1025 return min(dma_max_mapping_size(dev), size); 1026 } 1027 EXPORT_SYMBOL_GPL(dma_opt_mapping_size); 1028 1029 unsigned long dma_get_merge_boundary(struct device *dev) 1030 { 1031 const struct dma_map_ops *ops = get_dma_ops(dev); 1032 1033 if (use_dma_iommu(dev)) 1034 return iommu_dma_get_merge_boundary(dev); 1035 1036 if (!ops || !ops->get_merge_boundary) 1037 return 0; /* can't merge */ 1038 1039 return ops->get_merge_boundary(dev); 1040 } 1041 EXPORT_SYMBOL_GPL(dma_get_merge_boundary); 1042