34 	size_t		size;  member
44 	dma_free_attrs(dev, this->size, this->vaddr, this->dma_handle,  in dmam_release()
53 		WARN_ON(this->size != match->size ||  in dmam_match()
69 void dmam_free_coherent(struct device *dev, size_t size, void *vaddr,  in dmam_free_coherent()  argument
72 	struct dma_devres match_data = { size, vaddr, dma_handle };  in dmam_free_coherent()
75 	dma_free_coherent(dev, size, vaddr, dma_handle);  in dmam_free_coherent()
93 void *dmam_alloc_attrs(struct device *dev, size_t size, dma_addr_t *dma_handle,  in dmam_alloc_attrs()  argument
103 	vaddr = dma_alloc_attrs(dev, size, dma_handle, gfp, attrs);  in dmam_alloc_attrs()
111 	dr->size = size;  in dmam_alloc_attrs()
153 dma_addr_t dma_map_phys(struct device *dev, phys_addr_t phys, size_t size,  in dma_map_phys()  argument
171 	     arch_dma_map_phys_direct(dev, phys + size)))  in dma_map_phys()
172 		addr = dma_direct_map_phys(dev, phys, size, dir, attrs, true);  in dma_map_phys()
176 		addr = iommu_dma_map_phys(dev, phys, size, dir, attrs);  in dma_map_phys()
178 		addr = ops->map_phys(dev, phys, size, dir, attrs);  in dma_map_phys()
181 		kmsan_handle_dma(phys, size, dir);  in dma_map_phys()
182 	trace_dma_map_phys(dev, phys, addr, size, dir, attrs);  in dma_map_phys()
183 	debug_dma_map_phys(dev, phys, size, dir, addr, attrs);  in dma_map_phys()
190 		size_t offset, size_t size, enum dma_data_direction dir,  in dma_map_page_attrs()  argument
202 	return dma_map_phys(dev, phys, size, dir, attrs);  in dma_map_page_attrs()
206 void dma_unmap_phys(struct device *dev, dma_addr_t addr, size_t size,  in dma_unmap_phys()  argument
217 	     arch_dma_unmap_phys_direct(dev, addr + size)))  in dma_unmap_phys()
218 		dma_direct_unmap_phys(dev, addr, size, dir, attrs, true);  in dma_unmap_phys()
222 		iommu_dma_unmap_phys(dev, addr, size, dir, attrs);  in dma_unmap_phys()
224 		ops->unmap_phys(dev, addr, size, dir, attrs);  in dma_unmap_phys()
225 	trace_dma_unmap_phys(dev, addr, size, dir, attrs);  in dma_unmap_phys()
226 	debug_dma_unmap_phys(dev, addr, size, dir, attrs);  in dma_unmap_phys()
230 void dma_unmap_page_attrs(struct device *dev, dma_addr_t addr, size_t size,  in dma_unmap_page_attrs()  argument
236 	dma_unmap_phys(dev, addr, size, dir, attrs);  in dma_unmap_page_attrs()
364 		size_t size, enum dma_data_direction dir, unsigned long attrs)  in dma_map_resource()  argument
366 	return dma_map_phys(dev, phys_addr, size, dir, attrs | DMA_ATTR_MMIO);  in dma_map_resource()
370 void dma_unmap_resource(struct device *dev, dma_addr_t addr, size_t size,  in dma_unmap_resource()  argument
373 	dma_unmap_phys(dev, addr, size, dir, attrs | DMA_ATTR_MMIO);  in dma_unmap_resource()
378 void __dma_sync_single_for_cpu(struct device *dev, dma_addr_t addr, size_t size,  in __dma_sync_single_for_cpu()  argument
385 		dma_direct_sync_single_for_cpu(dev, addr, size, dir, true);  in __dma_sync_single_for_cpu()
387 		iommu_dma_sync_single_for_cpu(dev, addr, size, dir);  in __dma_sync_single_for_cpu()
389 		ops->sync_single_for_cpu(dev, addr, size, dir);  in __dma_sync_single_for_cpu()
390 	trace_dma_sync_single_for_cpu(dev, addr, size, dir);  in __dma_sync_single_for_cpu()
391 	debug_dma_sync_single_for_cpu(dev, addr, size, dir);  in __dma_sync_single_for_cpu()
396 		size_t size, enum dma_data_direction dir)  in __dma_sync_single_for_device()  argument
402 		dma_direct_sync_single_for_device(dev, addr, size, dir);  in __dma_sync_single_for_device()
404 		iommu_dma_sync_single_for_device(dev, addr, size, dir);  in __dma_sync_single_for_device()
406 		ops->sync_single_for_device(dev, addr, size, dir);  in __dma_sync_single_for_device()
407 	trace_dma_sync_single_for_device(dev, addr, size, dir);  in __dma_sync_single_for_device()
408 	debug_dma_sync_single_for_device(dev, addr, size, dir);  in __dma_sync_single_for_device()
516 		void *cpu_addr, dma_addr_t dma_addr, size_t size,  in dma_get_sgtable_attrs()  argument
523 				size, attrs);  in dma_get_sgtable_attrs()
526 				size, attrs);  in dma_get_sgtable_attrs()
529 	return ops->get_sgtable(dev, sgt, cpu_addr, dma_addr, size, attrs);  in dma_get_sgtable_attrs()
591 		void *cpu_addr, dma_addr_t dma_addr, size_t size,  in dma_mmap_attrs()  argument
597 		return dma_direct_mmap(dev, vma, cpu_addr, dma_addr, size,  in dma_mmap_attrs()
600 		return iommu_dma_mmap(dev, vma, cpu_addr, dma_addr, size,  in dma_mmap_attrs()
604 	return ops->mmap(dev, vma, cpu_addr, dma_addr, size, attrs);  in dma_mmap_attrs()
633 void *dma_alloc_attrs(struct device *dev, size_t size, dma_addr_t *dma_handle,  in dma_alloc_attrs()  argument
650 		trace_dma_alloc(dev, NULL, 0, size, DMA_BIDIRECTIONAL, flag,  in dma_alloc_attrs()
658 	if (dma_alloc_from_dev_coherent(dev, size, dma_handle, &cpu_addr)) {  in dma_alloc_attrs()
659 		trace_dma_alloc(dev, cpu_addr, *dma_handle, size,  in dma_alloc_attrs()
668 		cpu_addr = dma_direct_alloc(dev, size, dma_handle, flag, attrs);  in dma_alloc_attrs()
670 		cpu_addr = iommu_dma_alloc(dev, size, dma_handle, flag, attrs);  in dma_alloc_attrs()
672 		cpu_addr = ops->alloc(dev, size, dma_handle, flag, attrs);  in dma_alloc_attrs()
674 		trace_dma_alloc(dev, NULL, 0, size, DMA_BIDIRECTIONAL, flag,  in dma_alloc_attrs()
679 	trace_dma_alloc(dev, cpu_addr, *dma_handle, size, DMA_BIDIRECTIONAL,  in dma_alloc_attrs()
681 	debug_dma_alloc_coherent(dev, size, *dma_handle, cpu_addr, attrs);  in dma_alloc_attrs()
686 void dma_free_attrs(struct device *dev, size_t size, void *cpu_addr,  in dma_free_attrs()  argument
691 	if (dma_release_from_dev_coherent(dev, get_order(size), cpu_addr))  in dma_free_attrs()
702 	trace_dma_free(dev, cpu_addr, dma_handle, size, DMA_BIDIRECTIONAL,  in dma_free_attrs()
707 	debug_dma_free_coherent(dev, size, cpu_addr, dma_handle, attrs);  in dma_free_attrs()
709 		dma_direct_free(dev, size, cpu_addr, dma_handle, attrs);  in dma_free_attrs()
711 		iommu_dma_free(dev, size, cpu_addr, dma_handle, attrs);  in dma_free_attrs()
713 		ops->free(dev, size, cpu_addr, dma_handle, attrs);  in dma_free_attrs()
717 static struct page *__dma_alloc_pages(struct device *dev, size_t size,  in __dma_alloc_pages()  argument
729 	size = PAGE_ALIGN(size);  in __dma_alloc_pages()
731 		return dma_direct_alloc_pages(dev, size, dma_handle, dir, gfp);  in __dma_alloc_pages()
733 		return dma_common_alloc_pages(dev, size, dma_handle, dir, gfp);  in __dma_alloc_pages()
736 	return ops->alloc_pages_op(dev, size, dma_handle, dir, gfp);  in __dma_alloc_pages()
739 struct page *dma_alloc_pages(struct device *dev, size_t size,  in dma_alloc_pages()  argument
742 	struct page *page = __dma_alloc_pages(dev, size, dma_handle, dir, gfp);  in dma_alloc_pages()
746 				      size, dir, gfp, 0);  in dma_alloc_pages()
747 		debug_dma_alloc_pages(dev, page, size, dir, *dma_handle);  in dma_alloc_pages()
749 		trace_dma_alloc_pages(dev, NULL, 0, size, dir, gfp, 0);  in dma_alloc_pages()
755 static void __dma_free_pages(struct device *dev, size_t size, struct page *page,  in __dma_free_pages()  argument
760 	size = PAGE_ALIGN(size);  in __dma_free_pages()
762 		dma_direct_free_pages(dev, size, page, dma_handle, dir);  in __dma_free_pages()
764 		dma_common_free_pages(dev, size, page, dma_handle, dir);  in __dma_free_pages()
766 		ops->free_pages(dev, size, page, dma_handle, dir);  in __dma_free_pages()
769 void dma_free_pages(struct device *dev, size_t size, struct page *page,  in dma_free_pages()  argument
772 	trace_dma_free_pages(dev, page_to_virt(page), dma_handle, size, dir, 0);  in dma_free_pages()
773 	debug_dma_free_pages(dev, page, size, dir, dma_handle);  in dma_free_pages()
774 	__dma_free_pages(dev, size, page, dma_handle, dir);  in dma_free_pages()
779 		size_t size, struct page *page)  in dma_mmap_pages()  argument
783 	const unsigned long count = PAGE_ALIGN(size) >> PAGE_SHIFT;  in dma_mmap_pages()
793 static struct sg_table *alloc_single_sgt(struct device *dev, size_t size,  in alloc_single_sgt()  argument
804 	page = __dma_alloc_pages(dev, size, &sgt->sgl->dma_address, dir, gfp);  in alloc_single_sgt()
807 	sg_set_page(sgt->sgl, page, PAGE_ALIGN(size), 0);  in alloc_single_sgt()
817 struct sg_table *dma_alloc_noncontiguous(struct device *dev, size_t size,  in dma_alloc_noncontiguous()  argument
828 		sgt = iommu_dma_alloc_noncontiguous(dev, size, dir, gfp, attrs);  in dma_alloc_noncontiguous()
830 		sgt = alloc_single_sgt(dev, size, dir, gfp);  in dma_alloc_noncontiguous()
834 		trace_dma_alloc_sgt(dev, sgt, size, dir, gfp, attrs);  in dma_alloc_noncontiguous()
837 		trace_dma_alloc_sgt_err(dev, NULL, 0, size, dir, gfp, attrs);  in dma_alloc_noncontiguous()
843 static void free_single_sgt(struct device *dev, size_t size,  in free_single_sgt()  argument
846 	__dma_free_pages(dev, size, sg_page(sgt->sgl), sgt->sgl->dma_address,  in free_single_sgt()
852 void dma_free_noncontiguous(struct device *dev, size_t size,  in dma_free_noncontiguous()  argument
855 	trace_dma_free_sgt(dev, sgt, size, dir);  in dma_free_noncontiguous()
859 		iommu_dma_free_noncontiguous(dev, size, sgt, dir);  in dma_free_noncontiguous()
861 		free_single_sgt(dev, size, sgt, dir);  in dma_free_noncontiguous()
865 void *dma_vmap_noncontiguous(struct device *dev, size_t size,  in dma_vmap_noncontiguous()  argument
870 		return iommu_dma_vmap_noncontiguous(dev, size, sgt);  in dma_vmap_noncontiguous()
884 		size_t size, struct sg_table *sgt)  in dma_mmap_noncontiguous()  argument
887 		return iommu_dma_mmap_noncontiguous(dev, vma, size, sgt);  in dma_mmap_noncontiguous()
888 	return dma_mmap_pages(dev, vma, size, sg_page(sgt->sgl));  in dma_mmap_noncontiguous()
999 	size_t size = SIZE_MAX;  in dma_max_mapping_size()  local
1005 		size = dma_direct_max_mapping_size(dev);  in dma_max_mapping_size()
1007 		size = iommu_dma_max_mapping_size(dev);  in dma_max_mapping_size()
1009 		size = ops->max_mapping_size(dev);  in dma_max_mapping_size()
1011 	return size;  in dma_max_mapping_size()
1018 	size_t size = SIZE_MAX;  in dma_opt_mapping_size()  local
1021 		size = iommu_dma_opt_mapping_size();  in dma_opt_mapping_size()
1023 		size = ops->opt_mapping_size();  in dma_opt_mapping_size()
1025 	return min(dma_max_mapping_size(dev), size);  in dma_opt_mapping_size()