1 /* SPDX-License-Identifier: GPL-2.0 */ 2 #ifndef _LINUX_DMA_MAPPING_H 3 #define _LINUX_DMA_MAPPING_H 4 5 #include <linux/device.h> 6 #include <linux/err.h> 7 #include <linux/dma-direction.h> 8 #include <linux/scatterlist.h> 9 #include <linux/bug.h> 10 #include <linux/cache.h> 11 12 /* 13 * List of possible attributes associated with a DMA mapping. The semantics 14 * of each attribute should be defined in Documentation/core-api/dma-attributes.rst. 15 */ 16 17 /* 18 * DMA_ATTR_WEAK_ORDERING: Specifies that reads and writes to the mapping 19 * may be weakly ordered, that is that reads and writes may pass each other. 20 */ 21 #define DMA_ATTR_WEAK_ORDERING (1UL << 1) 22 /* 23 * DMA_ATTR_WRITE_COMBINE: Specifies that writes to the mapping may be 24 * buffered to improve performance. 25 */ 26 #define DMA_ATTR_WRITE_COMBINE (1UL << 2) 27 /* 28 * DMA_ATTR_NO_KERNEL_MAPPING: Lets the platform to avoid creating a kernel 29 * virtual mapping for the allocated buffer. 30 */ 31 #define DMA_ATTR_NO_KERNEL_MAPPING (1UL << 4) 32 /* 33 * DMA_ATTR_SKIP_CPU_SYNC: Allows platform code to skip synchronization of 34 * the CPU cache for the given buffer assuming that it has been already 35 * transferred to 'device' domain. 36 */ 37 #define DMA_ATTR_SKIP_CPU_SYNC (1UL << 5) 38 /* 39 * DMA_ATTR_FORCE_CONTIGUOUS: Forces contiguous allocation of the buffer 40 * in physical memory. 41 */ 42 #define DMA_ATTR_FORCE_CONTIGUOUS (1UL << 6) 43 /* 44 * DMA_ATTR_ALLOC_SINGLE_PAGES: This is a hint to the DMA-mapping subsystem 45 * that it's probably not worth the time to try to allocate memory to in a way 46 * that gives better TLB efficiency. 47 */ 48 #define DMA_ATTR_ALLOC_SINGLE_PAGES (1UL << 7) 49 /* 50 * DMA_ATTR_NO_WARN: This tells the DMA-mapping subsystem to suppress 51 * allocation failure reports (similarly to __GFP_NOWARN). 52 */ 53 #define DMA_ATTR_NO_WARN (1UL << 8) 54 55 /* 56 * DMA_ATTR_PRIVILEGED: used to indicate that the buffer is fully 57 * accessible at an elevated privilege level (and ideally inaccessible or 58 * at least read-only at lesser-privileged levels). 59 */ 60 #define DMA_ATTR_PRIVILEGED (1UL << 9) 61 62 /* 63 * DMA_ATTR_MMIO - Indicates memory-mapped I/O (MMIO) region for DMA mapping 64 * 65 * This attribute indicates the physical address is not normal system 66 * memory. It may not be used with kmap*()/phys_to_virt()/phys_to_page() 67 * functions, it may not be cacheable, and access using CPU load/store 68 * instructions may not be allowed. 69 * 70 * Usually this will be used to describe MMIO addresses, or other non-cacheable 71 * register addresses. When DMA mapping this sort of address we call 72 * the operation Peer to Peer as a one device is DMA'ing to another device. 73 * For PCI devices the p2pdma APIs must be used to determine if DMA_ATTR_MMIO 74 * is appropriate. 75 * 76 * For architectures that require cache flushing for DMA coherence 77 * DMA_ATTR_MMIO will not perform any cache flushing. The address 78 * provided must never be mapped cacheable into the CPU. 79 */ 80 #define DMA_ATTR_MMIO (1UL << 10) 81 82 /* 83 * DMA_ATTR_DEBUGGING_IGNORE_CACHELINES: Indicates the CPU cache line can be 84 * overlapped. All mappings sharing a cacheline must have this attribute for 85 * this to be considered safe. 86 */ 87 #define DMA_ATTR_DEBUGGING_IGNORE_CACHELINES (1UL << 11) 88 89 /* 90 * DMA_ATTR_REQUIRE_COHERENT: Indicates that DMA coherency is required. 91 * All mappings that carry this attribute can't work with SWIOTLB and cache 92 * flushing. 93 */ 94 #define DMA_ATTR_REQUIRE_COHERENT (1UL << 12) 95 /* 96 * DMA_ATTR_CC_SHARED: Indicates the DMA mapping is shared (decrypted) for 97 * confidential computing guests. For normal system memory the caller must have 98 * called set_memory_decrypted(), and pgprot_decrypted must be used when 99 * creating CPU PTEs for the mapping. The same shared semantic may be passed 100 * to the vIOMMU when it sets up the IOPTE. For MMIO use together with 101 * DMA_ATTR_MMIO to indicate shared MMIO. Unless DMA_ATTR_MMIO is provided 102 * a struct page is required. 103 */ 104 #define DMA_ATTR_CC_SHARED (1UL << 13) 105 106 /* 107 * __DMA_ATTR_ALLOC_CC_SHARED: Internal DMA-mapping attribute used by 108 * allocation paths that create shared (decrypted) backing pages for 109 * confidential computing guests. Drivers must not pass this attribute to 110 * dma_alloc_attrs(). 111 */ 112 #define __DMA_ATTR_ALLOC_CC_SHARED (1UL << 14) 113 114 /* 115 * A dma_addr_t can hold any valid DMA or bus address for the platform. It can 116 * be given to a device to use as a DMA source or target. It is specific to a 117 * given device and there may be a translation between the CPU physical address 118 * space and the bus address space. 119 * 120 * DMA_MAPPING_ERROR is the magic error code if a mapping failed. It should not 121 * be used directly in drivers, but checked for using dma_mapping_error() 122 * instead. 123 */ 124 #define DMA_MAPPING_ERROR (~(dma_addr_t)0) 125 126 #define DMA_BIT_MASK(n) GENMASK_ULL((n) - 1, 0) 127 128 struct dma_iova_state { 129 dma_addr_t addr; 130 u64 __size; 131 }; 132 133 /* 134 * Use the high bit to mark if we used swiotlb for one or more ranges. 135 */ 136 #define DMA_IOVA_USE_SWIOTLB (1ULL << 63) 137 138 static inline size_t dma_iova_size(struct dma_iova_state *state) 139 { 140 /* Casting is needed for 32-bits systems */ 141 return (size_t)(state->__size & ~DMA_IOVA_USE_SWIOTLB); 142 } 143 144 #ifdef CONFIG_DMA_API_DEBUG 145 void debug_dma_mapping_error(struct device *dev, dma_addr_t dma_addr); 146 void debug_dma_map_single(struct device *dev, const void *addr, 147 unsigned long len); 148 #else 149 static inline void debug_dma_mapping_error(struct device *dev, 150 dma_addr_t dma_addr) 151 { 152 } 153 static inline void debug_dma_map_single(struct device *dev, const void *addr, 154 unsigned long len) 155 { 156 } 157 #endif /* CONFIG_DMA_API_DEBUG */ 158 159 #ifdef CONFIG_HAS_DMA 160 static inline int dma_mapping_error(struct device *dev, dma_addr_t dma_addr) 161 { 162 debug_dma_mapping_error(dev, dma_addr); 163 164 if (unlikely(dma_addr == DMA_MAPPING_ERROR)) 165 return -ENOMEM; 166 return 0; 167 } 168 169 dma_addr_t dma_map_page_attrs(struct device *dev, struct page *page, 170 size_t offset, size_t size, enum dma_data_direction dir, 171 unsigned long attrs); 172 void dma_unmap_page_attrs(struct device *dev, dma_addr_t addr, size_t size, 173 enum dma_data_direction dir, unsigned long attrs); 174 dma_addr_t dma_map_phys(struct device *dev, phys_addr_t phys, size_t size, 175 enum dma_data_direction dir, unsigned long attrs); 176 void dma_unmap_phys(struct device *dev, dma_addr_t addr, size_t size, 177 enum dma_data_direction dir, unsigned long attrs); 178 unsigned int dma_map_sg_attrs(struct device *dev, struct scatterlist *sg, 179 int nents, enum dma_data_direction dir, unsigned long attrs); 180 void dma_unmap_sg_attrs(struct device *dev, struct scatterlist *sg, 181 int nents, enum dma_data_direction dir, 182 unsigned long attrs); 183 int dma_map_sgtable(struct device *dev, struct sg_table *sgt, 184 enum dma_data_direction dir, unsigned long attrs); 185 dma_addr_t dma_map_resource(struct device *dev, phys_addr_t phys_addr, 186 size_t size, enum dma_data_direction dir, unsigned long attrs); 187 void dma_unmap_resource(struct device *dev, dma_addr_t addr, size_t size, 188 enum dma_data_direction dir, unsigned long attrs); 189 void *dma_alloc_attrs(struct device *dev, size_t size, dma_addr_t *dma_handle, 190 gfp_t flag, unsigned long attrs); 191 void dma_free_attrs(struct device *dev, size_t size, void *cpu_addr, 192 dma_addr_t dma_handle, unsigned long attrs); 193 void *dmam_alloc_attrs(struct device *dev, size_t size, dma_addr_t *dma_handle, 194 gfp_t gfp, unsigned long attrs); 195 void dmam_free_coherent(struct device *dev, size_t size, void *vaddr, 196 dma_addr_t dma_handle); 197 int dma_get_sgtable_attrs(struct device *dev, struct sg_table *sgt, 198 void *cpu_addr, dma_addr_t dma_addr, size_t size, 199 unsigned long attrs); 200 int dma_mmap_attrs(struct device *dev, struct vm_area_struct *vma, 201 void *cpu_addr, dma_addr_t dma_addr, size_t size, 202 unsigned long attrs); 203 bool dma_can_mmap(struct device *dev); 204 bool dma_pci_p2pdma_supported(struct device *dev); 205 int dma_set_mask(struct device *dev, u64 mask); 206 int dma_set_coherent_mask(struct device *dev, u64 mask); 207 u64 dma_get_required_mask(struct device *dev); 208 bool dma_addressing_limited(struct device *dev); 209 size_t dma_max_mapping_size(struct device *dev); 210 size_t dma_max_opt_mapping_size(struct device *dev); 211 unsigned long dma_get_merge_boundary(struct device *dev); 212 struct sg_table *dma_alloc_noncontiguous(struct device *dev, size_t size, 213 enum dma_data_direction dir, gfp_t gfp, unsigned long attrs); 214 void dma_free_noncontiguous(struct device *dev, size_t size, 215 struct sg_table *sgt, enum dma_data_direction dir); 216 void *dma_vmap_noncontiguous(struct device *dev, size_t size, 217 struct sg_table *sgt); 218 void dma_vunmap_noncontiguous(struct device *dev, void *vaddr); 219 int dma_mmap_noncontiguous(struct device *dev, struct vm_area_struct *vma, 220 size_t size, struct sg_table *sgt); 221 #else /* CONFIG_HAS_DMA */ 222 static inline dma_addr_t dma_map_page_attrs(struct device *dev, 223 struct page *page, size_t offset, size_t size, 224 enum dma_data_direction dir, unsigned long attrs) 225 { 226 return DMA_MAPPING_ERROR; 227 } 228 static inline void dma_unmap_page_attrs(struct device *dev, dma_addr_t addr, 229 size_t size, enum dma_data_direction dir, unsigned long attrs) 230 { 231 } 232 static inline dma_addr_t dma_map_phys(struct device *dev, phys_addr_t phys, 233 size_t size, enum dma_data_direction dir, unsigned long attrs) 234 { 235 return DMA_MAPPING_ERROR; 236 } 237 static inline void dma_unmap_phys(struct device *dev, dma_addr_t addr, 238 size_t size, enum dma_data_direction dir, unsigned long attrs) 239 { 240 } 241 static inline unsigned int dma_map_sg_attrs(struct device *dev, 242 struct scatterlist *sg, int nents, enum dma_data_direction dir, 243 unsigned long attrs) 244 { 245 return 0; 246 } 247 static inline void dma_unmap_sg_attrs(struct device *dev, 248 struct scatterlist *sg, int nents, enum dma_data_direction dir, 249 unsigned long attrs) 250 { 251 } 252 static inline int dma_map_sgtable(struct device *dev, struct sg_table *sgt, 253 enum dma_data_direction dir, unsigned long attrs) 254 { 255 return -EOPNOTSUPP; 256 } 257 static inline dma_addr_t dma_map_resource(struct device *dev, 258 phys_addr_t phys_addr, size_t size, enum dma_data_direction dir, 259 unsigned long attrs) 260 { 261 return DMA_MAPPING_ERROR; 262 } 263 static inline void dma_unmap_resource(struct device *dev, dma_addr_t addr, 264 size_t size, enum dma_data_direction dir, unsigned long attrs) 265 { 266 } 267 static inline int dma_mapping_error(struct device *dev, dma_addr_t dma_addr) 268 { 269 return -ENOMEM; 270 } 271 static inline void *dma_alloc_attrs(struct device *dev, size_t size, 272 dma_addr_t *dma_handle, gfp_t flag, unsigned long attrs) 273 { 274 return NULL; 275 } 276 static inline void dma_free_attrs(struct device *dev, size_t size, 277 void *cpu_addr, dma_addr_t dma_handle, unsigned long attrs) 278 { 279 } 280 static inline void *dmam_alloc_attrs(struct device *dev, size_t size, 281 dma_addr_t *dma_handle, gfp_t gfp, unsigned long attrs) 282 { 283 return NULL; 284 } 285 static inline void dmam_free_coherent(struct device *dev, size_t size, 286 void *vaddr, dma_addr_t dma_handle) 287 { 288 } 289 static inline int dma_get_sgtable_attrs(struct device *dev, 290 struct sg_table *sgt, void *cpu_addr, dma_addr_t dma_addr, 291 size_t size, unsigned long attrs) 292 { 293 return -ENXIO; 294 } 295 static inline int dma_mmap_attrs(struct device *dev, struct vm_area_struct *vma, 296 void *cpu_addr, dma_addr_t dma_addr, size_t size, 297 unsigned long attrs) 298 { 299 return -ENXIO; 300 } 301 static inline bool dma_can_mmap(struct device *dev) 302 { 303 return false; 304 } 305 static inline bool dma_pci_p2pdma_supported(struct device *dev) 306 { 307 return false; 308 } 309 static inline int dma_set_mask(struct device *dev, u64 mask) 310 { 311 return -EIO; 312 } 313 static inline int dma_set_coherent_mask(struct device *dev, u64 mask) 314 { 315 return -EIO; 316 } 317 static inline u64 dma_get_required_mask(struct device *dev) 318 { 319 return 0; 320 } 321 static inline bool dma_addressing_limited(struct device *dev) 322 { 323 return false; 324 } 325 static inline size_t dma_max_mapping_size(struct device *dev) 326 { 327 return 0; 328 } 329 static inline size_t dma_max_opt_mapping_size(struct device *dev) 330 { 331 return 0; 332 } 333 static inline unsigned long dma_get_merge_boundary(struct device *dev) 334 { 335 return 0; 336 } 337 static inline struct sg_table *dma_alloc_noncontiguous(struct device *dev, 338 size_t size, enum dma_data_direction dir, gfp_t gfp, 339 unsigned long attrs) 340 { 341 return NULL; 342 } 343 static inline void dma_free_noncontiguous(struct device *dev, size_t size, 344 struct sg_table *sgt, enum dma_data_direction dir) 345 { 346 } 347 static inline void *dma_vmap_noncontiguous(struct device *dev, size_t size, 348 struct sg_table *sgt) 349 { 350 return NULL; 351 } 352 static inline void dma_vunmap_noncontiguous(struct device *dev, void *vaddr) 353 { 354 } 355 static inline int dma_mmap_noncontiguous(struct device *dev, 356 struct vm_area_struct *vma, size_t size, struct sg_table *sgt) 357 { 358 return -EINVAL; 359 } 360 #endif /* CONFIG_HAS_DMA */ 361 362 #ifdef CONFIG_IOMMU_DMA 363 /** 364 * dma_use_iova - check if the IOVA API is used for this state 365 * @state: IOVA state 366 * 367 * Return %true if the DMA transfers uses the dma_iova_*() calls or %false if 368 * they can't be used. 369 */ 370 static inline bool dma_use_iova(struct dma_iova_state *state) 371 { 372 return state->__size != 0; 373 } 374 375 bool dma_iova_try_alloc(struct device *dev, struct dma_iova_state *state, 376 phys_addr_t phys, size_t size); 377 void dma_iova_free(struct device *dev, struct dma_iova_state *state); 378 void dma_iova_destroy(struct device *dev, struct dma_iova_state *state, 379 size_t mapped_len, enum dma_data_direction dir, 380 unsigned long attrs); 381 int dma_iova_sync(struct device *dev, struct dma_iova_state *state, 382 size_t offset, size_t size); 383 int dma_iova_link(struct device *dev, struct dma_iova_state *state, 384 phys_addr_t phys, size_t offset, size_t size, 385 enum dma_data_direction dir, unsigned long attrs); 386 void dma_iova_unlink(struct device *dev, struct dma_iova_state *state, 387 size_t offset, size_t size, enum dma_data_direction dir, 388 unsigned long attrs); 389 #else /* CONFIG_IOMMU_DMA */ 390 static inline bool dma_use_iova(struct dma_iova_state *state) 391 { 392 return false; 393 } 394 static inline bool dma_iova_try_alloc(struct device *dev, 395 struct dma_iova_state *state, phys_addr_t phys, size_t size) 396 { 397 return false; 398 } 399 static inline void dma_iova_free(struct device *dev, 400 struct dma_iova_state *state) 401 { 402 } 403 static inline void dma_iova_destroy(struct device *dev, 404 struct dma_iova_state *state, size_t mapped_len, 405 enum dma_data_direction dir, unsigned long attrs) 406 { 407 } 408 static inline int dma_iova_sync(struct device *dev, 409 struct dma_iova_state *state, size_t offset, size_t size) 410 { 411 return -EOPNOTSUPP; 412 } 413 static inline int dma_iova_link(struct device *dev, 414 struct dma_iova_state *state, phys_addr_t phys, size_t offset, 415 size_t size, enum dma_data_direction dir, unsigned long attrs) 416 { 417 return -EOPNOTSUPP; 418 } 419 static inline void dma_iova_unlink(struct device *dev, 420 struct dma_iova_state *state, size_t offset, size_t size, 421 enum dma_data_direction dir, unsigned long attrs) 422 { 423 } 424 #endif /* CONFIG_IOMMU_DMA */ 425 426 #if defined(CONFIG_HAS_DMA) && defined(CONFIG_DMA_NEED_SYNC) 427 void __dma_sync_single_for_cpu(struct device *dev, dma_addr_t addr, size_t size, 428 enum dma_data_direction dir); 429 void __dma_sync_single_for_device(struct device *dev, dma_addr_t addr, 430 size_t size, enum dma_data_direction dir); 431 void __dma_sync_sg_for_cpu(struct device *dev, struct scatterlist *sg, 432 int nelems, enum dma_data_direction dir); 433 void __dma_sync_sg_for_device(struct device *dev, struct scatterlist *sg, 434 int nelems, enum dma_data_direction dir); 435 bool __dma_need_sync(struct device *dev, dma_addr_t dma_addr); 436 437 static inline bool dma_dev_need_sync(const struct device *dev) 438 { 439 /* Always call DMA sync operations when debugging is enabled */ 440 return !dev_dma_skip_sync(dev) || IS_ENABLED(CONFIG_DMA_API_DEBUG); 441 } 442 443 static inline void dma_sync_single_for_cpu(struct device *dev, dma_addr_t addr, 444 size_t size, enum dma_data_direction dir) 445 { 446 if (dma_dev_need_sync(dev)) 447 __dma_sync_single_for_cpu(dev, addr, size, dir); 448 } 449 450 static inline void dma_sync_single_for_device(struct device *dev, 451 dma_addr_t addr, size_t size, enum dma_data_direction dir) 452 { 453 if (dma_dev_need_sync(dev)) 454 __dma_sync_single_for_device(dev, addr, size, dir); 455 } 456 457 static inline void dma_sync_sg_for_cpu(struct device *dev, 458 struct scatterlist *sg, int nelems, enum dma_data_direction dir) 459 { 460 if (dma_dev_need_sync(dev)) 461 __dma_sync_sg_for_cpu(dev, sg, nelems, dir); 462 } 463 464 static inline void dma_sync_sg_for_device(struct device *dev, 465 struct scatterlist *sg, int nelems, enum dma_data_direction dir) 466 { 467 if (dma_dev_need_sync(dev)) 468 __dma_sync_sg_for_device(dev, sg, nelems, dir); 469 } 470 471 static inline bool dma_need_sync(struct device *dev, dma_addr_t dma_addr) 472 { 473 return dma_dev_need_sync(dev) ? __dma_need_sync(dev, dma_addr) : false; 474 } 475 bool dma_need_unmap(struct device *dev); 476 #else /* !CONFIG_HAS_DMA || !CONFIG_DMA_NEED_SYNC */ 477 static inline bool dma_dev_need_sync(const struct device *dev) 478 { 479 return false; 480 } 481 static inline void dma_sync_single_for_cpu(struct device *dev, dma_addr_t addr, 482 size_t size, enum dma_data_direction dir) 483 { 484 } 485 static inline void dma_sync_single_for_device(struct device *dev, 486 dma_addr_t addr, size_t size, enum dma_data_direction dir) 487 { 488 } 489 static inline void dma_sync_sg_for_cpu(struct device *dev, 490 struct scatterlist *sg, int nelems, enum dma_data_direction dir) 491 { 492 } 493 static inline void dma_sync_sg_for_device(struct device *dev, 494 struct scatterlist *sg, int nelems, enum dma_data_direction dir) 495 { 496 } 497 static inline bool dma_need_sync(struct device *dev, dma_addr_t dma_addr) 498 { 499 return false; 500 } 501 static inline bool dma_need_unmap(struct device *dev) 502 { 503 return false; 504 } 505 #endif /* !CONFIG_HAS_DMA || !CONFIG_DMA_NEED_SYNC */ 506 507 struct page *dma_alloc_pages(struct device *dev, size_t size, 508 dma_addr_t *dma_handle, enum dma_data_direction dir, gfp_t gfp); 509 void dma_free_pages(struct device *dev, size_t size, struct page *page, 510 dma_addr_t dma_handle, enum dma_data_direction dir); 511 int dma_mmap_pages(struct device *dev, struct vm_area_struct *vma, 512 size_t size, struct page *page); 513 514 static inline void *dma_alloc_noncoherent(struct device *dev, size_t size, 515 dma_addr_t *dma_handle, enum dma_data_direction dir, gfp_t gfp) 516 { 517 struct page *page = dma_alloc_pages(dev, size, dma_handle, dir, gfp); 518 return page ? page_address(page) : NULL; 519 } 520 521 static inline void dma_free_noncoherent(struct device *dev, size_t size, 522 void *vaddr, dma_addr_t dma_handle, enum dma_data_direction dir) 523 { 524 dma_free_pages(dev, size, virt_to_page(vaddr), dma_handle, dir); 525 } 526 527 static inline dma_addr_t dma_map_single_attrs(struct device *dev, void *ptr, 528 size_t size, enum dma_data_direction dir, unsigned long attrs) 529 { 530 /* DMA must never operate on areas that might be remapped. */ 531 if (dev_WARN_ONCE(dev, is_vmalloc_addr(ptr), 532 "rejecting DMA map of vmalloc memory\n")) 533 return DMA_MAPPING_ERROR; 534 debug_dma_map_single(dev, ptr, size); 535 return dma_map_page_attrs(dev, virt_to_page(ptr), offset_in_page(ptr), 536 size, dir, attrs); 537 } 538 539 static inline void dma_unmap_single_attrs(struct device *dev, dma_addr_t addr, 540 size_t size, enum dma_data_direction dir, unsigned long attrs) 541 { 542 return dma_unmap_page_attrs(dev, addr, size, dir, attrs); 543 } 544 545 static inline void dma_sync_single_range_for_cpu(struct device *dev, 546 dma_addr_t addr, unsigned long offset, size_t size, 547 enum dma_data_direction dir) 548 { 549 return dma_sync_single_for_cpu(dev, addr + offset, size, dir); 550 } 551 552 static inline void dma_sync_single_range_for_device(struct device *dev, 553 dma_addr_t addr, unsigned long offset, size_t size, 554 enum dma_data_direction dir) 555 { 556 return dma_sync_single_for_device(dev, addr + offset, size, dir); 557 } 558 559 /** 560 * dma_unmap_sgtable - Unmap the given buffer for DMA 561 * @dev: The device for which to perform the DMA operation 562 * @sgt: The sg_table object describing the buffer 563 * @dir: DMA direction 564 * @attrs: Optional DMA attributes for the unmap operation 565 * 566 * Unmaps a buffer described by a scatterlist stored in the given sg_table 567 * object for the @dir DMA operation by the @dev device. After this function 568 * the ownership of the buffer is transferred back to the CPU domain. 569 */ 570 static inline void dma_unmap_sgtable(struct device *dev, struct sg_table *sgt, 571 enum dma_data_direction dir, unsigned long attrs) 572 { 573 dma_unmap_sg_attrs(dev, sgt->sgl, sgt->orig_nents, dir, attrs); 574 } 575 576 /** 577 * dma_sync_sgtable_for_cpu - Synchronize the given buffer for CPU access 578 * @dev: The device for which to perform the DMA operation 579 * @sgt: The sg_table object describing the buffer 580 * @dir: DMA direction 581 * 582 * Performs the needed cache synchronization and moves the ownership of the 583 * buffer back to the CPU domain, so it is safe to perform any access to it 584 * by the CPU. Before doing any further DMA operations, one has to transfer 585 * the ownership of the buffer back to the DMA domain by calling the 586 * dma_sync_sgtable_for_device(). 587 */ 588 static inline void dma_sync_sgtable_for_cpu(struct device *dev, 589 struct sg_table *sgt, enum dma_data_direction dir) 590 { 591 dma_sync_sg_for_cpu(dev, sgt->sgl, sgt->orig_nents, dir); 592 } 593 594 /** 595 * dma_sync_sgtable_for_device - Synchronize the given buffer for DMA 596 * @dev: The device for which to perform the DMA operation 597 * @sgt: The sg_table object describing the buffer 598 * @dir: DMA direction 599 * 600 * Performs the needed cache synchronization and moves the ownership of the 601 * buffer back to the DMA domain, so it is safe to perform the DMA operation. 602 * Once finished, one has to call dma_sync_sgtable_for_cpu() or 603 * dma_unmap_sgtable(). 604 */ 605 static inline void dma_sync_sgtable_for_device(struct device *dev, 606 struct sg_table *sgt, enum dma_data_direction dir) 607 { 608 dma_sync_sg_for_device(dev, sgt->sgl, sgt->orig_nents, dir); 609 } 610 611 #define dma_map_single(d, a, s, r) dma_map_single_attrs(d, a, s, r, 0) 612 #define dma_unmap_single(d, a, s, r) dma_unmap_single_attrs(d, a, s, r, 0) 613 #define dma_map_sg(d, s, n, r) dma_map_sg_attrs(d, s, n, r, 0) 614 #define dma_unmap_sg(d, s, n, r) dma_unmap_sg_attrs(d, s, n, r, 0) 615 #define dma_map_page(d, p, o, s, r) dma_map_page_attrs(d, p, o, s, r, 0) 616 #define dma_unmap_page(d, a, s, r) dma_unmap_page_attrs(d, a, s, r, 0) 617 #define dma_get_sgtable(d, t, v, h, s) dma_get_sgtable_attrs(d, t, v, h, s, 0) 618 #define dma_mmap_coherent(d, v, c, h, s) dma_mmap_attrs(d, v, c, h, s, 0) 619 620 bool dma_coherent_ok(struct device *dev, phys_addr_t phys, size_t size); 621 622 static inline void *dma_alloc_coherent(struct device *dev, size_t size, 623 dma_addr_t *dma_handle, gfp_t gfp) 624 { 625 return dma_alloc_attrs(dev, size, dma_handle, gfp, 626 (gfp & __GFP_NOWARN) ? DMA_ATTR_NO_WARN : 0); 627 } 628 629 static inline void dma_free_coherent(struct device *dev, size_t size, 630 void *cpu_addr, dma_addr_t dma_handle) 631 { 632 return dma_free_attrs(dev, size, cpu_addr, dma_handle, 0); 633 } 634 635 636 static inline u64 dma_get_mask(struct device *dev) 637 { 638 if (dev->dma_mask && *dev->dma_mask) 639 return *dev->dma_mask; 640 return DMA_BIT_MASK(32); 641 } 642 643 /* 644 * Set both the DMA mask and the coherent DMA mask to the same thing. 645 * Note that we don't check the return value from dma_set_coherent_mask() 646 * as the DMA API guarantees that the coherent DMA mask can be set to 647 * the same or smaller than the streaming DMA mask. 648 */ 649 static inline int dma_set_mask_and_coherent(struct device *dev, u64 mask) 650 { 651 int rc = dma_set_mask(dev, mask); 652 if (rc == 0) 653 dma_set_coherent_mask(dev, mask); 654 return rc; 655 } 656 657 /* 658 * Similar to the above, except it deals with the case where the device 659 * does not have dev->dma_mask appropriately setup. 660 */ 661 static inline int dma_coerce_mask_and_coherent(struct device *dev, u64 mask) 662 { 663 dev->dma_mask = &dev->coherent_dma_mask; 664 return dma_set_mask_and_coherent(dev, mask); 665 } 666 667 static inline unsigned int dma_get_max_seg_size(struct device *dev) 668 { 669 if (dev->dma_parms && dev->dma_parms->max_segment_size) 670 return dev->dma_parms->max_segment_size; 671 return SZ_64K; 672 } 673 674 static inline void dma_set_max_seg_size(struct device *dev, unsigned int size) 675 { 676 if (WARN_ON_ONCE(!dev->dma_parms)) 677 return; 678 dev->dma_parms->max_segment_size = size; 679 } 680 681 static inline unsigned long dma_get_seg_boundary(struct device *dev) 682 { 683 if (dev->dma_parms && dev->dma_parms->segment_boundary_mask) 684 return dev->dma_parms->segment_boundary_mask; 685 return ULONG_MAX; 686 } 687 688 /** 689 * dma_get_seg_boundary_nr_pages - return the segment boundary in "page" units 690 * @dev: device to guery the boundary for 691 * @page_shift: ilog() of the IOMMU page size 692 * 693 * Return the segment boundary in IOMMU page units (which may be different from 694 * the CPU page size) for the passed in device. 695 * 696 * If @dev is NULL a boundary of U32_MAX is assumed, this case is just for 697 * non-DMA API callers. 698 */ 699 static inline unsigned long dma_get_seg_boundary_nr_pages(struct device *dev, 700 unsigned int page_shift) 701 { 702 if (!dev) 703 return (U32_MAX >> page_shift) + 1; 704 return (dma_get_seg_boundary(dev) >> page_shift) + 1; 705 } 706 707 static inline void dma_set_seg_boundary(struct device *dev, unsigned long mask) 708 { 709 if (WARN_ON_ONCE(!dev->dma_parms)) 710 return; 711 dev->dma_parms->segment_boundary_mask = mask; 712 } 713 714 static inline unsigned int dma_get_min_align_mask(struct device *dev) 715 { 716 if (dev->dma_parms) 717 return dev->dma_parms->min_align_mask; 718 return 0; 719 } 720 721 static inline void dma_set_min_align_mask(struct device *dev, 722 unsigned int min_align_mask) 723 { 724 if (WARN_ON_ONCE(!dev->dma_parms)) 725 return; 726 dev->dma_parms->min_align_mask = min_align_mask; 727 } 728 729 #ifndef dma_get_cache_alignment 730 static inline int dma_get_cache_alignment(void) 731 { 732 #ifdef ARCH_HAS_DMA_MINALIGN 733 return ARCH_DMA_MINALIGN; 734 #endif 735 return 1; 736 } 737 #endif 738 739 #ifdef ARCH_HAS_DMA_MINALIGN 740 #define ____dma_from_device_aligned __aligned(ARCH_DMA_MINALIGN) 741 #else 742 #define ____dma_from_device_aligned 743 #endif 744 /* Mark start of DMA buffer */ 745 #define __dma_from_device_group_begin(GROUP) \ 746 __cacheline_group_begin(GROUP) ____dma_from_device_aligned 747 /* Mark end of DMA buffer */ 748 #define __dma_from_device_group_end(GROUP) \ 749 __cacheline_group_end(GROUP) ____dma_from_device_aligned 750 751 static inline void *dmam_alloc_coherent(struct device *dev, size_t size, 752 dma_addr_t *dma_handle, gfp_t gfp) 753 { 754 return dmam_alloc_attrs(dev, size, dma_handle, gfp, 755 (gfp & __GFP_NOWARN) ? DMA_ATTR_NO_WARN : 0); 756 } 757 758 static inline void *dma_alloc_wc(struct device *dev, size_t size, 759 dma_addr_t *dma_addr, gfp_t gfp) 760 { 761 unsigned long attrs = DMA_ATTR_WRITE_COMBINE; 762 763 if (gfp & __GFP_NOWARN) 764 attrs |= DMA_ATTR_NO_WARN; 765 766 return dma_alloc_attrs(dev, size, dma_addr, gfp, attrs); 767 } 768 769 static inline void dma_free_wc(struct device *dev, size_t size, 770 void *cpu_addr, dma_addr_t dma_addr) 771 { 772 return dma_free_attrs(dev, size, cpu_addr, dma_addr, 773 DMA_ATTR_WRITE_COMBINE); 774 } 775 776 static inline int dma_mmap_wc(struct device *dev, 777 struct vm_area_struct *vma, 778 void *cpu_addr, dma_addr_t dma_addr, 779 size_t size) 780 { 781 return dma_mmap_attrs(dev, vma, cpu_addr, dma_addr, size, 782 DMA_ATTR_WRITE_COMBINE); 783 } 784 785 #ifdef CONFIG_NEED_DMA_MAP_STATE 786 #define DEFINE_DMA_UNMAP_ADDR(ADDR_NAME) dma_addr_t ADDR_NAME 787 #define DEFINE_DMA_UNMAP_LEN(LEN_NAME) __u32 LEN_NAME 788 #define dma_unmap_addr(PTR, ADDR_NAME) ((PTR)->ADDR_NAME) 789 #define dma_unmap_addr_set(PTR, ADDR_NAME, VAL) (((PTR)->ADDR_NAME) = (VAL)) 790 #define dma_unmap_len(PTR, LEN_NAME) ((PTR)->LEN_NAME) 791 #define dma_unmap_len_set(PTR, LEN_NAME, VAL) (((PTR)->LEN_NAME) = (VAL)) 792 #else 793 #define DEFINE_DMA_UNMAP_ADDR(ADDR_NAME) 794 #define DEFINE_DMA_UNMAP_LEN(LEN_NAME) 795 #define dma_unmap_addr(PTR, ADDR_NAME) \ 796 ({ typeof(PTR) __p __maybe_unused = PTR; 0; }) 797 #define dma_unmap_addr_set(PTR, ADDR_NAME, VAL) \ 798 do { typeof(PTR) __p __maybe_unused = PTR; } while (0) 799 #define dma_unmap_len(PTR, LEN_NAME) \ 800 ({ typeof(PTR) __p __maybe_unused = PTR; 0; }) 801 #define dma_unmap_len_set(PTR, LEN_NAME, VAL) \ 802 do { typeof(PTR) __p __maybe_unused = PTR; } while (0) 803 #endif 804 805 #endif /* _LINUX_DMA_MAPPING_H */ 806