1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * arch-independent dma-mapping routines 4 * 5 * Copyright (c) 2006 SUSE Linux Products GmbH 6 * Copyright (c) 2006 Tejun Heo <teheo@suse.de> 7 */ 8 #include <linux/memblock.h> /* for max_pfn */ 9 #include <linux/acpi.h> 10 #include <linux/dma-direct.h> 11 #include <linux/dma-mapping.h> 12 #include <linux/export.h> 13 #include <linux/gfp.h> 14 #include <linux/of_device.h> 15 #include <linux/slab.h> 16 #include <linux/vmalloc.h> 17 18 /* 19 * Managed DMA API 20 */ 21 struct dma_devres { 22 size_t size; 23 void *vaddr; 24 dma_addr_t dma_handle; 25 unsigned long attrs; 26 }; 27 28 static void dmam_release(struct device *dev, void *res) 29 { 30 struct dma_devres *this = res; 31 32 dma_free_attrs(dev, this->size, this->vaddr, this->dma_handle, 33 this->attrs); 34 } 35 36 static int dmam_match(struct device *dev, void *res, void *match_data) 37 { 38 struct dma_devres *this = res, *match = match_data; 39 40 if (this->vaddr == match->vaddr) { 41 WARN_ON(this->size != match->size || 42 this->dma_handle != match->dma_handle); 43 return 1; 44 } 45 return 0; 46 } 47 48 /** 49 * dmam_free_coherent - Managed dma_free_coherent() 50 * @dev: Device to free coherent memory for 51 * @size: Size of allocation 52 * @vaddr: Virtual address of the memory to free 53 * @dma_handle: DMA handle of the memory to free 54 * 55 * Managed dma_free_coherent(). 56 */ 57 void dmam_free_coherent(struct device *dev, size_t size, void *vaddr, 58 dma_addr_t dma_handle) 59 { 60 struct dma_devres match_data = { size, vaddr, dma_handle }; 61 62 dma_free_coherent(dev, size, vaddr, dma_handle); 63 WARN_ON(devres_destroy(dev, dmam_release, dmam_match, &match_data)); 64 } 65 EXPORT_SYMBOL(dmam_free_coherent); 66 67 /** 68 * dmam_alloc_attrs - Managed dma_alloc_attrs() 69 * @dev: Device to allocate non_coherent memory for 70 * @size: Size of allocation 71 * @dma_handle: Out argument for allocated DMA handle 72 * @gfp: Allocation flags 73 * @attrs: Flags in the DMA_ATTR_* namespace. 74 * 75 * Managed dma_alloc_attrs(). Memory allocated using this function will be 76 * automatically released on driver detach. 77 * 78 * RETURNS: 79 * Pointer to allocated memory on success, NULL on failure. 80 */ 81 void *dmam_alloc_attrs(struct device *dev, size_t size, dma_addr_t *dma_handle, 82 gfp_t gfp, unsigned long attrs) 83 { 84 struct dma_devres *dr; 85 void *vaddr; 86 87 dr = devres_alloc(dmam_release, sizeof(*dr), gfp); 88 if (!dr) 89 return NULL; 90 91 vaddr = dma_alloc_attrs(dev, size, dma_handle, gfp, attrs); 92 if (!vaddr) { 93 devres_free(dr); 94 return NULL; 95 } 96 97 dr->vaddr = vaddr; 98 dr->dma_handle = *dma_handle; 99 dr->size = size; 100 dr->attrs = attrs; 101 102 devres_add(dev, dr); 103 104 return vaddr; 105 } 106 EXPORT_SYMBOL(dmam_alloc_attrs); 107 108 static bool dma_go_direct(struct device *dev, dma_addr_t mask, 109 const struct dma_map_ops *ops) 110 { 111 if (likely(!ops)) 112 return true; 113 #ifdef CONFIG_DMA_OPS_BYPASS 114 if (dev->dma_ops_bypass) 115 return min_not_zero(mask, dev->bus_dma_limit) >= 116 dma_direct_get_required_mask(dev); 117 #endif 118 return false; 119 } 120 121 122 /* 123 * Check if the devices uses a direct mapping for streaming DMA operations. 124 * This allows IOMMU drivers to set a bypass mode if the DMA mask is large 125 * enough. 126 */ 127 static inline bool dma_alloc_direct(struct device *dev, 128 const struct dma_map_ops *ops) 129 { 130 return dma_go_direct(dev, dev->coherent_dma_mask, ops); 131 } 132 133 static inline bool dma_map_direct(struct device *dev, 134 const struct dma_map_ops *ops) 135 { 136 return dma_go_direct(dev, *dev->dma_mask, ops); 137 } 138 139 dma_addr_t dma_map_page_attrs(struct device *dev, struct page *page, 140 size_t offset, size_t size, enum dma_data_direction dir, 141 unsigned long attrs) 142 { 143 const struct dma_map_ops *ops = get_dma_ops(dev); 144 dma_addr_t addr; 145 146 BUG_ON(!valid_dma_direction(dir)); 147 148 if (WARN_ON_ONCE(!dev->dma_mask)) 149 return DMA_MAPPING_ERROR; 150 151 if (dma_map_direct(dev, ops)) 152 addr = dma_direct_map_page(dev, page, offset, size, dir, attrs); 153 else 154 addr = ops->map_page(dev, page, offset, size, dir, attrs); 155 debug_dma_map_page(dev, page, offset, size, dir, addr); 156 157 return addr; 158 } 159 EXPORT_SYMBOL(dma_map_page_attrs); 160 161 void dma_unmap_page_attrs(struct device *dev, dma_addr_t addr, size_t size, 162 enum dma_data_direction dir, unsigned long attrs) 163 { 164 const struct dma_map_ops *ops = get_dma_ops(dev); 165 166 BUG_ON(!valid_dma_direction(dir)); 167 if (dma_map_direct(dev, ops)) 168 dma_direct_unmap_page(dev, addr, size, dir, attrs); 169 else if (ops->unmap_page) 170 ops->unmap_page(dev, addr, size, dir, attrs); 171 debug_dma_unmap_page(dev, addr, size, dir); 172 } 173 EXPORT_SYMBOL(dma_unmap_page_attrs); 174 175 /* 176 * dma_maps_sg_attrs returns 0 on error and > 0 on success. 177 * It should never return a value < 0. 178 */ 179 int dma_map_sg_attrs(struct device *dev, struct scatterlist *sg, int nents, 180 enum dma_data_direction dir, unsigned long attrs) 181 { 182 const struct dma_map_ops *ops = get_dma_ops(dev); 183 int ents; 184 185 BUG_ON(!valid_dma_direction(dir)); 186 187 if (WARN_ON_ONCE(!dev->dma_mask)) 188 return 0; 189 190 if (dma_map_direct(dev, ops)) 191 ents = dma_direct_map_sg(dev, sg, nents, dir, attrs); 192 else 193 ents = ops->map_sg(dev, sg, nents, dir, attrs); 194 BUG_ON(ents < 0); 195 debug_dma_map_sg(dev, sg, nents, ents, dir); 196 197 return ents; 198 } 199 EXPORT_SYMBOL(dma_map_sg_attrs); 200 201 void dma_unmap_sg_attrs(struct device *dev, struct scatterlist *sg, 202 int nents, enum dma_data_direction dir, 203 unsigned long attrs) 204 { 205 const struct dma_map_ops *ops = get_dma_ops(dev); 206 207 BUG_ON(!valid_dma_direction(dir)); 208 debug_dma_unmap_sg(dev, sg, nents, dir); 209 if (dma_map_direct(dev, ops)) 210 dma_direct_unmap_sg(dev, sg, nents, dir, attrs); 211 else if (ops->unmap_sg) 212 ops->unmap_sg(dev, sg, nents, dir, attrs); 213 } 214 EXPORT_SYMBOL(dma_unmap_sg_attrs); 215 216 dma_addr_t dma_map_resource(struct device *dev, phys_addr_t phys_addr, 217 size_t size, enum dma_data_direction dir, unsigned long attrs) 218 { 219 const struct dma_map_ops *ops = get_dma_ops(dev); 220 dma_addr_t addr = DMA_MAPPING_ERROR; 221 222 BUG_ON(!valid_dma_direction(dir)); 223 224 if (WARN_ON_ONCE(!dev->dma_mask)) 225 return DMA_MAPPING_ERROR; 226 227 /* Don't allow RAM to be mapped */ 228 if (WARN_ON_ONCE(pfn_valid(PHYS_PFN(phys_addr)))) 229 return DMA_MAPPING_ERROR; 230 231 if (dma_map_direct(dev, ops)) 232 addr = dma_direct_map_resource(dev, phys_addr, size, dir, attrs); 233 else if (ops->map_resource) 234 addr = ops->map_resource(dev, phys_addr, size, dir, attrs); 235 236 debug_dma_map_resource(dev, phys_addr, size, dir, addr); 237 return addr; 238 } 239 EXPORT_SYMBOL(dma_map_resource); 240 241 void dma_unmap_resource(struct device *dev, dma_addr_t addr, size_t size, 242 enum dma_data_direction dir, unsigned long attrs) 243 { 244 const struct dma_map_ops *ops = get_dma_ops(dev); 245 246 BUG_ON(!valid_dma_direction(dir)); 247 if (!dma_map_direct(dev, ops) && ops->unmap_resource) 248 ops->unmap_resource(dev, addr, size, dir, attrs); 249 debug_dma_unmap_resource(dev, addr, size, dir); 250 } 251 EXPORT_SYMBOL(dma_unmap_resource); 252 253 void dma_sync_single_for_cpu(struct device *dev, dma_addr_t addr, size_t size, 254 enum dma_data_direction dir) 255 { 256 const struct dma_map_ops *ops = get_dma_ops(dev); 257 258 BUG_ON(!valid_dma_direction(dir)); 259 if (dma_map_direct(dev, ops)) 260 dma_direct_sync_single_for_cpu(dev, addr, size, dir); 261 else if (ops->sync_single_for_cpu) 262 ops->sync_single_for_cpu(dev, addr, size, dir); 263 debug_dma_sync_single_for_cpu(dev, addr, size, dir); 264 } 265 EXPORT_SYMBOL(dma_sync_single_for_cpu); 266 267 void dma_sync_single_for_device(struct device *dev, dma_addr_t addr, 268 size_t size, enum dma_data_direction dir) 269 { 270 const struct dma_map_ops *ops = get_dma_ops(dev); 271 272 BUG_ON(!valid_dma_direction(dir)); 273 if (dma_map_direct(dev, ops)) 274 dma_direct_sync_single_for_device(dev, addr, size, dir); 275 else if (ops->sync_single_for_device) 276 ops->sync_single_for_device(dev, addr, size, dir); 277 debug_dma_sync_single_for_device(dev, addr, size, dir); 278 } 279 EXPORT_SYMBOL(dma_sync_single_for_device); 280 281 void dma_sync_sg_for_cpu(struct device *dev, struct scatterlist *sg, 282 int nelems, enum dma_data_direction dir) 283 { 284 const struct dma_map_ops *ops = get_dma_ops(dev); 285 286 BUG_ON(!valid_dma_direction(dir)); 287 if (dma_map_direct(dev, ops)) 288 dma_direct_sync_sg_for_cpu(dev, sg, nelems, dir); 289 else if (ops->sync_sg_for_cpu) 290 ops->sync_sg_for_cpu(dev, sg, nelems, dir); 291 debug_dma_sync_sg_for_cpu(dev, sg, nelems, dir); 292 } 293 EXPORT_SYMBOL(dma_sync_sg_for_cpu); 294 295 void dma_sync_sg_for_device(struct device *dev, struct scatterlist *sg, 296 int nelems, enum dma_data_direction dir) 297 { 298 const struct dma_map_ops *ops = get_dma_ops(dev); 299 300 BUG_ON(!valid_dma_direction(dir)); 301 if (dma_map_direct(dev, ops)) 302 dma_direct_sync_sg_for_device(dev, sg, nelems, dir); 303 else if (ops->sync_sg_for_device) 304 ops->sync_sg_for_device(dev, sg, nelems, dir); 305 debug_dma_sync_sg_for_device(dev, sg, nelems, dir); 306 } 307 EXPORT_SYMBOL(dma_sync_sg_for_device); 308 309 /* 310 * The whole dma_get_sgtable() idea is fundamentally unsafe - it seems 311 * that the intention is to allow exporting memory allocated via the 312 * coherent DMA APIs through the dma_buf API, which only accepts a 313 * scattertable. This presents a couple of problems: 314 * 1. Not all memory allocated via the coherent DMA APIs is backed by 315 * a struct page 316 * 2. Passing coherent DMA memory into the streaming APIs is not allowed 317 * as we will try to flush the memory through a different alias to that 318 * actually being used (and the flushes are redundant.) 319 */ 320 int dma_get_sgtable_attrs(struct device *dev, struct sg_table *sgt, 321 void *cpu_addr, dma_addr_t dma_addr, size_t size, 322 unsigned long attrs) 323 { 324 const struct dma_map_ops *ops = get_dma_ops(dev); 325 326 if (dma_alloc_direct(dev, ops)) 327 return dma_direct_get_sgtable(dev, sgt, cpu_addr, dma_addr, 328 size, attrs); 329 if (!ops->get_sgtable) 330 return -ENXIO; 331 return ops->get_sgtable(dev, sgt, cpu_addr, dma_addr, size, attrs); 332 } 333 EXPORT_SYMBOL(dma_get_sgtable_attrs); 334 335 #ifdef CONFIG_MMU 336 /* 337 * Return the page attributes used for mapping dma_alloc_* memory, either in 338 * kernel space if remapping is needed, or to userspace through dma_mmap_*. 339 */ 340 pgprot_t dma_pgprot(struct device *dev, pgprot_t prot, unsigned long attrs) 341 { 342 if (force_dma_unencrypted(dev)) 343 prot = pgprot_decrypted(prot); 344 if (dev_is_dma_coherent(dev) || 345 (IS_ENABLED(CONFIG_DMA_NONCOHERENT_CACHE_SYNC) && 346 (attrs & DMA_ATTR_NON_CONSISTENT))) 347 return prot; 348 #ifdef CONFIG_ARCH_HAS_DMA_WRITE_COMBINE 349 if (attrs & DMA_ATTR_WRITE_COMBINE) 350 return pgprot_writecombine(prot); 351 #endif 352 return pgprot_dmacoherent(prot); 353 } 354 #endif /* CONFIG_MMU */ 355 356 /** 357 * dma_can_mmap - check if a given device supports dma_mmap_* 358 * @dev: device to check 359 * 360 * Returns %true if @dev supports dma_mmap_coherent() and dma_mmap_attrs() to 361 * map DMA allocations to userspace. 362 */ 363 bool dma_can_mmap(struct device *dev) 364 { 365 const struct dma_map_ops *ops = get_dma_ops(dev); 366 367 if (dma_alloc_direct(dev, ops)) 368 return dma_direct_can_mmap(dev); 369 return ops->mmap != NULL; 370 } 371 EXPORT_SYMBOL_GPL(dma_can_mmap); 372 373 /** 374 * dma_mmap_attrs - map a coherent DMA allocation into user space 375 * @dev: valid struct device pointer, or NULL for ISA and EISA-like devices 376 * @vma: vm_area_struct describing requested user mapping 377 * @cpu_addr: kernel CPU-view address returned from dma_alloc_attrs 378 * @dma_addr: device-view address returned from dma_alloc_attrs 379 * @size: size of memory originally requested in dma_alloc_attrs 380 * @attrs: attributes of mapping properties requested in dma_alloc_attrs 381 * 382 * Map a coherent DMA buffer previously allocated by dma_alloc_attrs into user 383 * space. The coherent DMA buffer must not be freed by the driver until the 384 * user space mapping has been released. 385 */ 386 int dma_mmap_attrs(struct device *dev, struct vm_area_struct *vma, 387 void *cpu_addr, dma_addr_t dma_addr, size_t size, 388 unsigned long attrs) 389 { 390 const struct dma_map_ops *ops = get_dma_ops(dev); 391 392 if (dma_alloc_direct(dev, ops)) 393 return dma_direct_mmap(dev, vma, cpu_addr, dma_addr, size, 394 attrs); 395 if (!ops->mmap) 396 return -ENXIO; 397 return ops->mmap(dev, vma, cpu_addr, dma_addr, size, attrs); 398 } 399 EXPORT_SYMBOL(dma_mmap_attrs); 400 401 u64 dma_get_required_mask(struct device *dev) 402 { 403 const struct dma_map_ops *ops = get_dma_ops(dev); 404 405 if (dma_alloc_direct(dev, ops)) 406 return dma_direct_get_required_mask(dev); 407 if (ops->get_required_mask) 408 return ops->get_required_mask(dev); 409 410 /* 411 * We require every DMA ops implementation to at least support a 32-bit 412 * DMA mask (and use bounce buffering if that isn't supported in 413 * hardware). As the direct mapping code has its own routine to 414 * actually report an optimal mask we default to 32-bit here as that 415 * is the right thing for most IOMMUs, and at least not actively 416 * harmful in general. 417 */ 418 return DMA_BIT_MASK(32); 419 } 420 EXPORT_SYMBOL_GPL(dma_get_required_mask); 421 422 void *dma_alloc_attrs(struct device *dev, size_t size, dma_addr_t *dma_handle, 423 gfp_t flag, unsigned long attrs) 424 { 425 const struct dma_map_ops *ops = get_dma_ops(dev); 426 void *cpu_addr; 427 428 WARN_ON_ONCE(!dev->coherent_dma_mask); 429 430 if (dma_alloc_from_dev_coherent(dev, size, dma_handle, &cpu_addr)) 431 return cpu_addr; 432 433 /* let the implementation decide on the zone to allocate from: */ 434 flag &= ~(__GFP_DMA | __GFP_DMA32 | __GFP_HIGHMEM); 435 436 if (dma_alloc_direct(dev, ops)) 437 cpu_addr = dma_direct_alloc(dev, size, dma_handle, flag, attrs); 438 else if (ops->alloc) 439 cpu_addr = ops->alloc(dev, size, dma_handle, flag, attrs); 440 else 441 return NULL; 442 443 debug_dma_alloc_coherent(dev, size, *dma_handle, cpu_addr); 444 return cpu_addr; 445 } 446 EXPORT_SYMBOL(dma_alloc_attrs); 447 448 void dma_free_attrs(struct device *dev, size_t size, void *cpu_addr, 449 dma_addr_t dma_handle, unsigned long attrs) 450 { 451 const struct dma_map_ops *ops = get_dma_ops(dev); 452 453 if (dma_release_from_dev_coherent(dev, get_order(size), cpu_addr)) 454 return; 455 /* 456 * On non-coherent platforms which implement DMA-coherent buffers via 457 * non-cacheable remaps, ops->free() may call vunmap(). Thus getting 458 * this far in IRQ context is a) at risk of a BUG_ON() or trying to 459 * sleep on some machines, and b) an indication that the driver is 460 * probably misusing the coherent API anyway. 461 */ 462 WARN_ON(irqs_disabled()); 463 464 if (!cpu_addr) 465 return; 466 467 debug_dma_free_coherent(dev, size, cpu_addr, dma_handle); 468 if (dma_alloc_direct(dev, ops)) 469 dma_direct_free(dev, size, cpu_addr, dma_handle, attrs); 470 else if (ops->free) 471 ops->free(dev, size, cpu_addr, dma_handle, attrs); 472 } 473 EXPORT_SYMBOL(dma_free_attrs); 474 475 int dma_supported(struct device *dev, u64 mask) 476 { 477 const struct dma_map_ops *ops = get_dma_ops(dev); 478 479 /* 480 * ->dma_supported sets the bypass flag, so we must always call 481 * into the method here unless the device is truly direct mapped. 482 */ 483 if (!ops) 484 return dma_direct_supported(dev, mask); 485 if (!ops->dma_supported) 486 return 1; 487 return ops->dma_supported(dev, mask); 488 } 489 EXPORT_SYMBOL(dma_supported); 490 491 #ifdef CONFIG_ARCH_HAS_DMA_SET_MASK 492 void arch_dma_set_mask(struct device *dev, u64 mask); 493 #else 494 #define arch_dma_set_mask(dev, mask) do { } while (0) 495 #endif 496 497 int dma_set_mask(struct device *dev, u64 mask) 498 { 499 /* 500 * Truncate the mask to the actually supported dma_addr_t width to 501 * avoid generating unsupportable addresses. 502 */ 503 mask = (dma_addr_t)mask; 504 505 if (!dev->dma_mask || !dma_supported(dev, mask)) 506 return -EIO; 507 508 arch_dma_set_mask(dev, mask); 509 *dev->dma_mask = mask; 510 return 0; 511 } 512 EXPORT_SYMBOL(dma_set_mask); 513 514 #ifndef CONFIG_ARCH_HAS_DMA_SET_COHERENT_MASK 515 int dma_set_coherent_mask(struct device *dev, u64 mask) 516 { 517 /* 518 * Truncate the mask to the actually supported dma_addr_t width to 519 * avoid generating unsupportable addresses. 520 */ 521 mask = (dma_addr_t)mask; 522 523 if (!dma_supported(dev, mask)) 524 return -EIO; 525 526 dev->coherent_dma_mask = mask; 527 return 0; 528 } 529 EXPORT_SYMBOL(dma_set_coherent_mask); 530 #endif 531 532 void dma_cache_sync(struct device *dev, void *vaddr, size_t size, 533 enum dma_data_direction dir) 534 { 535 const struct dma_map_ops *ops = get_dma_ops(dev); 536 537 BUG_ON(!valid_dma_direction(dir)); 538 539 if (dma_alloc_direct(dev, ops)) 540 arch_dma_cache_sync(dev, vaddr, size, dir); 541 else if (ops->cache_sync) 542 ops->cache_sync(dev, vaddr, size, dir); 543 } 544 EXPORT_SYMBOL(dma_cache_sync); 545 546 size_t dma_max_mapping_size(struct device *dev) 547 { 548 const struct dma_map_ops *ops = get_dma_ops(dev); 549 size_t size = SIZE_MAX; 550 551 if (dma_map_direct(dev, ops)) 552 size = dma_direct_max_mapping_size(dev); 553 else if (ops && ops->max_mapping_size) 554 size = ops->max_mapping_size(dev); 555 556 return size; 557 } 558 EXPORT_SYMBOL_GPL(dma_max_mapping_size); 559 560 bool dma_need_sync(struct device *dev, dma_addr_t dma_addr) 561 { 562 const struct dma_map_ops *ops = get_dma_ops(dev); 563 564 if (dma_map_direct(dev, ops)) 565 return dma_direct_need_sync(dev, dma_addr); 566 return ops->sync_single_for_cpu || ops->sync_single_for_device; 567 } 568 EXPORT_SYMBOL_GPL(dma_need_sync); 569 570 unsigned long dma_get_merge_boundary(struct device *dev) 571 { 572 const struct dma_map_ops *ops = get_dma_ops(dev); 573 574 if (!ops || !ops->get_merge_boundary) 575 return 0; /* can't merge */ 576 577 return ops->get_merge_boundary(dev); 578 } 579 EXPORT_SYMBOL_GPL(dma_get_merge_boundary); 580