34 	size_t		size;  member
44 	dma_free_attrs(dev, this->size, this->vaddr, this->dma_handle,  in dmam_release()
53 		WARN_ON(this->size != match->size ||  in dmam_match()
63  * @size: Size of allocation
69 void dmam_free_coherent(struct device *dev, size_t size, void *vaddr,  in dmam_free_coherent()  argument
72 	struct dma_devres match_data = { size, vaddr, dma_handle };  in dmam_free_coherent()
75 	dma_free_coherent(dev, size, vaddr, dma_handle);  in dmam_free_coherent()
82  * @size: Size of allocation
93 void *dmam_alloc_attrs(struct device *dev, size_t size, dma_addr_t *dma_handle,  in dmam_alloc_attrs()  argument
103 	vaddr = dma_alloc_attrs(dev, size, dma_handle, gfp, attrs);  in dmam_alloc_attrs()
111 	dr->size = size;  in dmam_alloc_attrs()
155 dma_addr_t dma_map_phys(struct device *dev, phys_addr_t phys, size_t size,  in dma_map_phys()  argument
168 	    (!is_mmio && arch_dma_map_phys_direct(dev, phys + size)))  in dma_map_phys()
169 		addr = dma_direct_map_phys(dev, phys, size, dir, attrs);  in dma_map_phys()
171 		addr = iommu_dma_map_phys(dev, phys, size, dir, attrs);  in dma_map_phys()
173 		addr = ops->map_phys(dev, phys, size, dir, attrs);  in dma_map_phys()
176 		kmsan_handle_dma(phys, size, dir);  in dma_map_phys()
177 	trace_dma_map_phys(dev, phys, addr, size, dir, attrs);  in dma_map_phys()
178 	debug_dma_map_phys(dev, phys, size, dir, addr, attrs);  in dma_map_phys()
185 		size_t offset, size_t size, enum dma_data_direction dir,  in dma_map_page_attrs()  argument
197 	return dma_map_phys(dev, phys, size, dir, attrs);  in dma_map_page_attrs()
201 void dma_unmap_phys(struct device *dev, dma_addr_t addr, size_t size,  in dma_unmap_phys()  argument
209 	    (!is_mmio && arch_dma_unmap_phys_direct(dev, addr + size)))  in dma_unmap_phys()
210 		dma_direct_unmap_phys(dev, addr, size, dir, attrs);  in dma_unmap_phys()
212 		iommu_dma_unmap_phys(dev, addr, size, dir, attrs);  in dma_unmap_phys()
214 		ops->unmap_phys(dev, addr, size, dir, attrs);  in dma_unmap_phys()
215 	trace_dma_unmap_phys(dev, addr, size, dir, attrs);  in dma_unmap_phys()
216 	debug_dma_unmap_phys(dev, addr, size, dir);  in dma_unmap_phys()
220 void dma_unmap_page_attrs(struct device *dev, dma_addr_t addr, size_t size,  in dma_unmap_page_attrs()  argument
226 	dma_unmap_phys(dev, addr, size, dir, attrs);  in dma_unmap_page_attrs()
351 		size_t size, enum dma_data_direction dir, unsigned long attrs)  in dma_map_resource()  argument
357 	return dma_map_phys(dev, phys_addr, size, dir, attrs | DMA_ATTR_MMIO);  in dma_map_resource()
361 void dma_unmap_resource(struct device *dev, dma_addr_t addr, size_t size,  in dma_unmap_resource()  argument
364 	dma_unmap_phys(dev, addr, size, dir, attrs | DMA_ATTR_MMIO);  in dma_unmap_resource()
369 void __dma_sync_single_for_cpu(struct device *dev, dma_addr_t addr, size_t size,  in __dma_sync_single_for_cpu()  argument
376 		dma_direct_sync_single_for_cpu(dev, addr, size, dir);  in __dma_sync_single_for_cpu()
378 		iommu_dma_sync_single_for_cpu(dev, addr, size, dir);  in __dma_sync_single_for_cpu()
380 		ops->sync_single_for_cpu(dev, addr, size, dir);  in __dma_sync_single_for_cpu()
381 	trace_dma_sync_single_for_cpu(dev, addr, size, dir);  in __dma_sync_single_for_cpu()
382 	debug_dma_sync_single_for_cpu(dev, addr, size, dir);  in __dma_sync_single_for_cpu()
387 		size_t size, enum dma_data_direction dir)  in __dma_sync_single_for_device()  argument
393 		dma_direct_sync_single_for_device(dev, addr, size, dir);  in __dma_sync_single_for_device()
395 		iommu_dma_sync_single_for_device(dev, addr, size, dir);  in __dma_sync_single_for_device()
397 		ops->sync_single_for_device(dev, addr, size, dir);  in __dma_sync_single_for_device()
398 	trace_dma_sync_single_for_device(dev, addr, size, dir);  in __dma_sync_single_for_device()
399 	debug_dma_sync_single_for_device(dev, addr, size, dir);  in __dma_sync_single_for_device()
507 		void *cpu_addr, dma_addr_t dma_addr, size_t size,  in dma_get_sgtable_attrs()  argument
514 				size, attrs);  in dma_get_sgtable_attrs()
517 				size, attrs);  in dma_get_sgtable_attrs()
520 	return ops->get_sgtable(dev, sgt, cpu_addr, dma_addr, size, attrs);  in dma_get_sgtable_attrs()
566  * @size: size of memory originally requested in dma_alloc_attrs
574 		void *cpu_addr, dma_addr_t dma_addr, size_t size,  in dma_mmap_attrs()  argument
580 		return dma_direct_mmap(dev, vma, cpu_addr, dma_addr, size,  in dma_mmap_attrs()
583 		return iommu_dma_mmap(dev, vma, cpu_addr, dma_addr, size,  in dma_mmap_attrs()
587 	return ops->mmap(dev, vma, cpu_addr, dma_addr, size, attrs);  in dma_mmap_attrs()
616 void *dma_alloc_attrs(struct device *dev, size_t size, dma_addr_t *dma_handle,  in dma_alloc_attrs()  argument
632 	if (dma_alloc_from_dev_coherent(dev, size, dma_handle, &cpu_addr)) {  in dma_alloc_attrs()
633 		trace_dma_alloc(dev, cpu_addr, *dma_handle, size,  in dma_alloc_attrs()
642 		cpu_addr = dma_direct_alloc(dev, size, dma_handle, flag, attrs);  in dma_alloc_attrs()
644 		cpu_addr = iommu_dma_alloc(dev, size, dma_handle, flag, attrs);  in dma_alloc_attrs()
646 		cpu_addr = ops->alloc(dev, size, dma_handle, flag, attrs);  in dma_alloc_attrs()
648 		trace_dma_alloc(dev, NULL, 0, size, DMA_BIDIRECTIONAL, flag,  in dma_alloc_attrs()
653 	trace_dma_alloc(dev, cpu_addr, *dma_handle, size, DMA_BIDIRECTIONAL,  in dma_alloc_attrs()
655 	debug_dma_alloc_coherent(dev, size, *dma_handle, cpu_addr, attrs);  in dma_alloc_attrs()
660 void dma_free_attrs(struct device *dev, size_t size, void *cpu_addr,  in dma_free_attrs()  argument
665 	if (dma_release_from_dev_coherent(dev, get_order(size), cpu_addr))  in dma_free_attrs()
676 	trace_dma_free(dev, cpu_addr, dma_handle, size, DMA_BIDIRECTIONAL,  in dma_free_attrs()
681 	debug_dma_free_coherent(dev, size, cpu_addr, dma_handle);  in dma_free_attrs()
683 		dma_direct_free(dev, size, cpu_addr, dma_handle, attrs);  in dma_free_attrs()
685 		iommu_dma_free(dev, size, cpu_addr, dma_handle, attrs);  in dma_free_attrs()
687 		ops->free(dev, size, cpu_addr, dma_handle, attrs);  in dma_free_attrs()
691 static struct page *__dma_alloc_pages(struct device *dev, size_t size,  in __dma_alloc_pages()  argument
703 	size = PAGE_ALIGN(size);  in __dma_alloc_pages()
705 		return dma_direct_alloc_pages(dev, size, dma_handle, dir, gfp);  in __dma_alloc_pages()
707 		return dma_common_alloc_pages(dev, size, dma_handle, dir, gfp);  in __dma_alloc_pages()
710 	return ops->alloc_pages_op(dev, size, dma_handle, dir, gfp);  in __dma_alloc_pages()
713 struct page *dma_alloc_pages(struct device *dev, size_t size,  in dma_alloc_pages()  argument
716 	struct page *page = __dma_alloc_pages(dev, size, dma_handle, dir, gfp);  in dma_alloc_pages()
720 				      size, dir, gfp, 0);  in dma_alloc_pages()
721 		debug_dma_alloc_pages(dev, page, size, dir, *dma_handle, 0);  in dma_alloc_pages()
723 		trace_dma_alloc_pages(dev, NULL, 0, size, dir, gfp, 0);  in dma_alloc_pages()
729 static void __dma_free_pages(struct device *dev, size_t size, struct page *page,  in __dma_free_pages()  argument
734 	size = PAGE_ALIGN(size);  in __dma_free_pages()
736 		dma_direct_free_pages(dev, size, page, dma_handle, dir);  in __dma_free_pages()
738 		dma_common_free_pages(dev, size, page, dma_handle, dir);  in __dma_free_pages()
740 		ops->free_pages(dev, size, page, dma_handle, dir);  in __dma_free_pages()
743 void dma_free_pages(struct device *dev, size_t size, struct page *page,  in dma_free_pages()  argument
746 	trace_dma_free_pages(dev, page_to_virt(page), dma_handle, size, dir, 0);  in dma_free_pages()
747 	debug_dma_free_pages(dev, page, size, dir, dma_handle);  in dma_free_pages()
748 	__dma_free_pages(dev, size, page, dma_handle, dir);  in dma_free_pages()
753 		size_t size, struct page *page)  in dma_mmap_pages()  argument
755 	unsigned long count = PAGE_ALIGN(size) >> PAGE_SHIFT;  in dma_mmap_pages()
765 static struct sg_table *alloc_single_sgt(struct device *dev, size_t size,  in alloc_single_sgt()  argument
776 	page = __dma_alloc_pages(dev, size, &sgt->sgl->dma_address, dir, gfp);  in alloc_single_sgt()
779 	sg_set_page(sgt->sgl, page, PAGE_ALIGN(size), 0);  in alloc_single_sgt()
789 struct sg_table *dma_alloc_noncontiguous(struct device *dev, size_t size,  in dma_alloc_noncontiguous()  argument
800 		sgt = iommu_dma_alloc_noncontiguous(dev, size, dir, gfp, attrs);  in dma_alloc_noncontiguous()
802 		sgt = alloc_single_sgt(dev, size, dir, gfp);  in dma_alloc_noncontiguous()
806 		trace_dma_alloc_sgt(dev, sgt, size, dir, gfp, attrs);  in dma_alloc_noncontiguous()
809 		trace_dma_alloc_sgt_err(dev, NULL, 0, size, dir, gfp, attrs);  in dma_alloc_noncontiguous()
815 static void free_single_sgt(struct device *dev, size_t size,  in free_single_sgt()  argument
818 	__dma_free_pages(dev, size, sg_page(sgt->sgl), sgt->sgl->dma_address,  in free_single_sgt()
824 void dma_free_noncontiguous(struct device *dev, size_t size,  in dma_free_noncontiguous()  argument
827 	trace_dma_free_sgt(dev, sgt, size, dir);  in dma_free_noncontiguous()
831 		iommu_dma_free_noncontiguous(dev, size, sgt, dir);  in dma_free_noncontiguous()
833 		free_single_sgt(dev, size, sgt, dir);  in dma_free_noncontiguous()
837 void *dma_vmap_noncontiguous(struct device *dev, size_t size,  in dma_vmap_noncontiguous()  argument
842 		return iommu_dma_vmap_noncontiguous(dev, size, sgt);  in dma_vmap_noncontiguous()
856 		size_t size, struct sg_table *sgt)  in dma_mmap_noncontiguous()  argument
859 		return iommu_dma_mmap_noncontiguous(dev, vma, size, sgt);  in dma_mmap_noncontiguous()
860 	return dma_mmap_pages(dev, vma, size, sg_page(sgt->sgl));  in dma_mmap_noncontiguous()
971 	size_t size = SIZE_MAX;  in dma_max_mapping_size()  local
974 		size = dma_direct_max_mapping_size(dev);  in dma_max_mapping_size()
976 		size = iommu_dma_max_mapping_size(dev);  in dma_max_mapping_size()
978 		size = ops->max_mapping_size(dev);  in dma_max_mapping_size()
980 	return size;  in dma_max_mapping_size()
987 	size_t size = SIZE_MAX;  in dma_opt_mapping_size()  local
990 		size = iommu_dma_opt_mapping_size();  in dma_opt_mapping_size()
992 		size = ops->opt_mapping_size();  in dma_opt_mapping_size()
994 	return min(dma_max_mapping_size(dev), size);  in dma_opt_mapping_size()