xref: /linux/include/linux/dma-mapping.h (revision 3fd6c59042dbba50391e30862beac979491145fe)
1 /* SPDX-License-Identifier: GPL-2.0 */
2 #ifndef _LINUX_DMA_MAPPING_H
3 #define _LINUX_DMA_MAPPING_H
4 
5 #include <linux/device.h>
6 #include <linux/err.h>
7 #include <linux/dma-direction.h>
8 #include <linux/scatterlist.h>
9 #include <linux/bug.h>
10 
11 /**
12  * List of possible attributes associated with a DMA mapping. The semantics
13  * of each attribute should be defined in Documentation/core-api/dma-attributes.rst.
14  */
15 
16 /*
17  * DMA_ATTR_WEAK_ORDERING: Specifies that reads and writes to the mapping
18  * may be weakly ordered, that is that reads and writes may pass each other.
19  */
20 #define DMA_ATTR_WEAK_ORDERING		(1UL << 1)
21 /*
22  * DMA_ATTR_WRITE_COMBINE: Specifies that writes to the mapping may be
23  * buffered to improve performance.
24  */
25 #define DMA_ATTR_WRITE_COMBINE		(1UL << 2)
26 /*
27  * DMA_ATTR_NO_KERNEL_MAPPING: Lets the platform to avoid creating a kernel
28  * virtual mapping for the allocated buffer.
29  */
30 #define DMA_ATTR_NO_KERNEL_MAPPING	(1UL << 4)
31 /*
32  * DMA_ATTR_SKIP_CPU_SYNC: Allows platform code to skip synchronization of
33  * the CPU cache for the given buffer assuming that it has been already
34  * transferred to 'device' domain.
35  */
36 #define DMA_ATTR_SKIP_CPU_SYNC		(1UL << 5)
37 /*
38  * DMA_ATTR_FORCE_CONTIGUOUS: Forces contiguous allocation of the buffer
39  * in physical memory.
40  */
41 #define DMA_ATTR_FORCE_CONTIGUOUS	(1UL << 6)
42 /*
43  * DMA_ATTR_ALLOC_SINGLE_PAGES: This is a hint to the DMA-mapping subsystem
44  * that it's probably not worth the time to try to allocate memory to in a way
45  * that gives better TLB efficiency.
46  */
47 #define DMA_ATTR_ALLOC_SINGLE_PAGES	(1UL << 7)
48 /*
49  * DMA_ATTR_NO_WARN: This tells the DMA-mapping subsystem to suppress
50  * allocation failure reports (similarly to __GFP_NOWARN).
51  */
52 #define DMA_ATTR_NO_WARN	(1UL << 8)
53 
54 /*
55  * DMA_ATTR_PRIVILEGED: used to indicate that the buffer is fully
56  * accessible at an elevated privilege level (and ideally inaccessible or
57  * at least read-only at lesser-privileged levels).
58  */
59 #define DMA_ATTR_PRIVILEGED		(1UL << 9)
60 
61 /*
62  * A dma_addr_t can hold any valid DMA or bus address for the platform.  It can
63  * be given to a device to use as a DMA source or target.  It is specific to a
64  * given device and there may be a translation between the CPU physical address
65  * space and the bus address space.
66  *
67  * DMA_MAPPING_ERROR is the magic error code if a mapping failed.  It should not
68  * be used directly in drivers, but checked for using dma_mapping_error()
69  * instead.
70  */
71 #define DMA_MAPPING_ERROR		(~(dma_addr_t)0)
72 
73 #define DMA_BIT_MASK(n)	(((n) == 64) ? ~0ULL : ((1ULL<<(n))-1))
74 
75 #ifdef CONFIG_DMA_API_DEBUG
76 void debug_dma_mapping_error(struct device *dev, dma_addr_t dma_addr);
77 void debug_dma_map_single(struct device *dev, const void *addr,
78 		unsigned long len);
79 #else
debug_dma_mapping_error(struct device * dev,dma_addr_t dma_addr)80 static inline void debug_dma_mapping_error(struct device *dev,
81 		dma_addr_t dma_addr)
82 {
83 }
debug_dma_map_single(struct device * dev,const void * addr,unsigned long len)84 static inline void debug_dma_map_single(struct device *dev, const void *addr,
85 		unsigned long len)
86 {
87 }
88 #endif /* CONFIG_DMA_API_DEBUG */
89 
90 #ifdef CONFIG_HAS_DMA
dma_mapping_error(struct device * dev,dma_addr_t dma_addr)91 static inline int dma_mapping_error(struct device *dev, dma_addr_t dma_addr)
92 {
93 	debug_dma_mapping_error(dev, dma_addr);
94 
95 	if (unlikely(dma_addr == DMA_MAPPING_ERROR))
96 		return -ENOMEM;
97 	return 0;
98 }
99 
100 dma_addr_t dma_map_page_attrs(struct device *dev, struct page *page,
101 		size_t offset, size_t size, enum dma_data_direction dir,
102 		unsigned long attrs);
103 void dma_unmap_page_attrs(struct device *dev, dma_addr_t addr, size_t size,
104 		enum dma_data_direction dir, unsigned long attrs);
105 unsigned int dma_map_sg_attrs(struct device *dev, struct scatterlist *sg,
106 		int nents, enum dma_data_direction dir, unsigned long attrs);
107 void dma_unmap_sg_attrs(struct device *dev, struct scatterlist *sg,
108 				      int nents, enum dma_data_direction dir,
109 				      unsigned long attrs);
110 int dma_map_sgtable(struct device *dev, struct sg_table *sgt,
111 		enum dma_data_direction dir, unsigned long attrs);
112 dma_addr_t dma_map_resource(struct device *dev, phys_addr_t phys_addr,
113 		size_t size, enum dma_data_direction dir, unsigned long attrs);
114 void dma_unmap_resource(struct device *dev, dma_addr_t addr, size_t size,
115 		enum dma_data_direction dir, unsigned long attrs);
116 void *dma_alloc_attrs(struct device *dev, size_t size, dma_addr_t *dma_handle,
117 		gfp_t flag, unsigned long attrs);
118 void dma_free_attrs(struct device *dev, size_t size, void *cpu_addr,
119 		dma_addr_t dma_handle, unsigned long attrs);
120 void *dmam_alloc_attrs(struct device *dev, size_t size, dma_addr_t *dma_handle,
121 		gfp_t gfp, unsigned long attrs);
122 void dmam_free_coherent(struct device *dev, size_t size, void *vaddr,
123 		dma_addr_t dma_handle);
124 int dma_get_sgtable_attrs(struct device *dev, struct sg_table *sgt,
125 		void *cpu_addr, dma_addr_t dma_addr, size_t size,
126 		unsigned long attrs);
127 int dma_mmap_attrs(struct device *dev, struct vm_area_struct *vma,
128 		void *cpu_addr, dma_addr_t dma_addr, size_t size,
129 		unsigned long attrs);
130 bool dma_can_mmap(struct device *dev);
131 bool dma_pci_p2pdma_supported(struct device *dev);
132 int dma_set_mask(struct device *dev, u64 mask);
133 int dma_set_coherent_mask(struct device *dev, u64 mask);
134 u64 dma_get_required_mask(struct device *dev);
135 bool dma_addressing_limited(struct device *dev);
136 size_t dma_max_mapping_size(struct device *dev);
137 size_t dma_opt_mapping_size(struct device *dev);
138 unsigned long dma_get_merge_boundary(struct device *dev);
139 struct sg_table *dma_alloc_noncontiguous(struct device *dev, size_t size,
140 		enum dma_data_direction dir, gfp_t gfp, unsigned long attrs);
141 void dma_free_noncontiguous(struct device *dev, size_t size,
142 		struct sg_table *sgt, enum dma_data_direction dir);
143 void *dma_vmap_noncontiguous(struct device *dev, size_t size,
144 		struct sg_table *sgt);
145 void dma_vunmap_noncontiguous(struct device *dev, void *vaddr);
146 int dma_mmap_noncontiguous(struct device *dev, struct vm_area_struct *vma,
147 		size_t size, struct sg_table *sgt);
148 #else /* CONFIG_HAS_DMA */
dma_map_page_attrs(struct device * dev,struct page * page,size_t offset,size_t size,enum dma_data_direction dir,unsigned long attrs)149 static inline dma_addr_t dma_map_page_attrs(struct device *dev,
150 		struct page *page, size_t offset, size_t size,
151 		enum dma_data_direction dir, unsigned long attrs)
152 {
153 	return DMA_MAPPING_ERROR;
154 }
dma_unmap_page_attrs(struct device * dev,dma_addr_t addr,size_t size,enum dma_data_direction dir,unsigned long attrs)155 static inline void dma_unmap_page_attrs(struct device *dev, dma_addr_t addr,
156 		size_t size, enum dma_data_direction dir, unsigned long attrs)
157 {
158 }
dma_map_sg_attrs(struct device * dev,struct scatterlist * sg,int nents,enum dma_data_direction dir,unsigned long attrs)159 static inline unsigned int dma_map_sg_attrs(struct device *dev,
160 		struct scatterlist *sg, int nents, enum dma_data_direction dir,
161 		unsigned long attrs)
162 {
163 	return 0;
164 }
dma_unmap_sg_attrs(struct device * dev,struct scatterlist * sg,int nents,enum dma_data_direction dir,unsigned long attrs)165 static inline void dma_unmap_sg_attrs(struct device *dev,
166 		struct scatterlist *sg, int nents, enum dma_data_direction dir,
167 		unsigned long attrs)
168 {
169 }
dma_map_sgtable(struct device * dev,struct sg_table * sgt,enum dma_data_direction dir,unsigned long attrs)170 static inline int dma_map_sgtable(struct device *dev, struct sg_table *sgt,
171 		enum dma_data_direction dir, unsigned long attrs)
172 {
173 	return -EOPNOTSUPP;
174 }
dma_map_resource(struct device * dev,phys_addr_t phys_addr,size_t size,enum dma_data_direction dir,unsigned long attrs)175 static inline dma_addr_t dma_map_resource(struct device *dev,
176 		phys_addr_t phys_addr, size_t size, enum dma_data_direction dir,
177 		unsigned long attrs)
178 {
179 	return DMA_MAPPING_ERROR;
180 }
dma_unmap_resource(struct device * dev,dma_addr_t addr,size_t size,enum dma_data_direction dir,unsigned long attrs)181 static inline void dma_unmap_resource(struct device *dev, dma_addr_t addr,
182 		size_t size, enum dma_data_direction dir, unsigned long attrs)
183 {
184 }
dma_mapping_error(struct device * dev,dma_addr_t dma_addr)185 static inline int dma_mapping_error(struct device *dev, dma_addr_t dma_addr)
186 {
187 	return -ENOMEM;
188 }
dma_alloc_attrs(struct device * dev,size_t size,dma_addr_t * dma_handle,gfp_t flag,unsigned long attrs)189 static inline void *dma_alloc_attrs(struct device *dev, size_t size,
190 		dma_addr_t *dma_handle, gfp_t flag, unsigned long attrs)
191 {
192 	return NULL;
193 }
dma_free_attrs(struct device * dev,size_t size,void * cpu_addr,dma_addr_t dma_handle,unsigned long attrs)194 static void dma_free_attrs(struct device *dev, size_t size, void *cpu_addr,
195 		dma_addr_t dma_handle, unsigned long attrs)
196 {
197 }
dmam_alloc_attrs(struct device * dev,size_t size,dma_addr_t * dma_handle,gfp_t gfp,unsigned long attrs)198 static inline void *dmam_alloc_attrs(struct device *dev, size_t size,
199 		dma_addr_t *dma_handle, gfp_t gfp, unsigned long attrs)
200 {
201 	return NULL;
202 }
dmam_free_coherent(struct device * dev,size_t size,void * vaddr,dma_addr_t dma_handle)203 static inline void dmam_free_coherent(struct device *dev, size_t size,
204 		void *vaddr, dma_addr_t dma_handle)
205 {
206 }
dma_get_sgtable_attrs(struct device * dev,struct sg_table * sgt,void * cpu_addr,dma_addr_t dma_addr,size_t size,unsigned long attrs)207 static inline int dma_get_sgtable_attrs(struct device *dev,
208 		struct sg_table *sgt, void *cpu_addr, dma_addr_t dma_addr,
209 		size_t size, unsigned long attrs)
210 {
211 	return -ENXIO;
212 }
dma_mmap_attrs(struct device * dev,struct vm_area_struct * vma,void * cpu_addr,dma_addr_t dma_addr,size_t size,unsigned long attrs)213 static inline int dma_mmap_attrs(struct device *dev, struct vm_area_struct *vma,
214 		void *cpu_addr, dma_addr_t dma_addr, size_t size,
215 		unsigned long attrs)
216 {
217 	return -ENXIO;
218 }
dma_can_mmap(struct device * dev)219 static inline bool dma_can_mmap(struct device *dev)
220 {
221 	return false;
222 }
dma_pci_p2pdma_supported(struct device * dev)223 static inline bool dma_pci_p2pdma_supported(struct device *dev)
224 {
225 	return false;
226 }
dma_set_mask(struct device * dev,u64 mask)227 static inline int dma_set_mask(struct device *dev, u64 mask)
228 {
229 	return -EIO;
230 }
dma_set_coherent_mask(struct device * dev,u64 mask)231 static inline int dma_set_coherent_mask(struct device *dev, u64 mask)
232 {
233 	return -EIO;
234 }
dma_get_required_mask(struct device * dev)235 static inline u64 dma_get_required_mask(struct device *dev)
236 {
237 	return 0;
238 }
dma_addressing_limited(struct device * dev)239 static inline bool dma_addressing_limited(struct device *dev)
240 {
241 	return false;
242 }
dma_max_mapping_size(struct device * dev)243 static inline size_t dma_max_mapping_size(struct device *dev)
244 {
245 	return 0;
246 }
dma_opt_mapping_size(struct device * dev)247 static inline size_t dma_opt_mapping_size(struct device *dev)
248 {
249 	return 0;
250 }
dma_get_merge_boundary(struct device * dev)251 static inline unsigned long dma_get_merge_boundary(struct device *dev)
252 {
253 	return 0;
254 }
dma_alloc_noncontiguous(struct device * dev,size_t size,enum dma_data_direction dir,gfp_t gfp,unsigned long attrs)255 static inline struct sg_table *dma_alloc_noncontiguous(struct device *dev,
256 		size_t size, enum dma_data_direction dir, gfp_t gfp,
257 		unsigned long attrs)
258 {
259 	return NULL;
260 }
dma_free_noncontiguous(struct device * dev,size_t size,struct sg_table * sgt,enum dma_data_direction dir)261 static inline void dma_free_noncontiguous(struct device *dev, size_t size,
262 		struct sg_table *sgt, enum dma_data_direction dir)
263 {
264 }
dma_vmap_noncontiguous(struct device * dev,size_t size,struct sg_table * sgt)265 static inline void *dma_vmap_noncontiguous(struct device *dev, size_t size,
266 		struct sg_table *sgt)
267 {
268 	return NULL;
269 }
dma_vunmap_noncontiguous(struct device * dev,void * vaddr)270 static inline void dma_vunmap_noncontiguous(struct device *dev, void *vaddr)
271 {
272 }
dma_mmap_noncontiguous(struct device * dev,struct vm_area_struct * vma,size_t size,struct sg_table * sgt)273 static inline int dma_mmap_noncontiguous(struct device *dev,
274 		struct vm_area_struct *vma, size_t size, struct sg_table *sgt)
275 {
276 	return -EINVAL;
277 }
278 #endif /* CONFIG_HAS_DMA */
279 
280 #if defined(CONFIG_HAS_DMA) && defined(CONFIG_DMA_NEED_SYNC)
281 void __dma_sync_single_for_cpu(struct device *dev, dma_addr_t addr, size_t size,
282 		enum dma_data_direction dir);
283 void __dma_sync_single_for_device(struct device *dev, dma_addr_t addr,
284 		size_t size, enum dma_data_direction dir);
285 void __dma_sync_sg_for_cpu(struct device *dev, struct scatterlist *sg,
286 		int nelems, enum dma_data_direction dir);
287 void __dma_sync_sg_for_device(struct device *dev, struct scatterlist *sg,
288 		int nelems, enum dma_data_direction dir);
289 bool __dma_need_sync(struct device *dev, dma_addr_t dma_addr);
290 
dma_dev_need_sync(const struct device * dev)291 static inline bool dma_dev_need_sync(const struct device *dev)
292 {
293 	/* Always call DMA sync operations when debugging is enabled */
294 	return !dev->dma_skip_sync || IS_ENABLED(CONFIG_DMA_API_DEBUG);
295 }
296 
dma_sync_single_for_cpu(struct device * dev,dma_addr_t addr,size_t size,enum dma_data_direction dir)297 static inline void dma_sync_single_for_cpu(struct device *dev, dma_addr_t addr,
298 		size_t size, enum dma_data_direction dir)
299 {
300 	if (dma_dev_need_sync(dev))
301 		__dma_sync_single_for_cpu(dev, addr, size, dir);
302 }
303 
dma_sync_single_for_device(struct device * dev,dma_addr_t addr,size_t size,enum dma_data_direction dir)304 static inline void dma_sync_single_for_device(struct device *dev,
305 		dma_addr_t addr, size_t size, enum dma_data_direction dir)
306 {
307 	if (dma_dev_need_sync(dev))
308 		__dma_sync_single_for_device(dev, addr, size, dir);
309 }
310 
dma_sync_sg_for_cpu(struct device * dev,struct scatterlist * sg,int nelems,enum dma_data_direction dir)311 static inline void dma_sync_sg_for_cpu(struct device *dev,
312 		struct scatterlist *sg, int nelems, enum dma_data_direction dir)
313 {
314 	if (dma_dev_need_sync(dev))
315 		__dma_sync_sg_for_cpu(dev, sg, nelems, dir);
316 }
317 
dma_sync_sg_for_device(struct device * dev,struct scatterlist * sg,int nelems,enum dma_data_direction dir)318 static inline void dma_sync_sg_for_device(struct device *dev,
319 		struct scatterlist *sg, int nelems, enum dma_data_direction dir)
320 {
321 	if (dma_dev_need_sync(dev))
322 		__dma_sync_sg_for_device(dev, sg, nelems, dir);
323 }
324 
dma_need_sync(struct device * dev,dma_addr_t dma_addr)325 static inline bool dma_need_sync(struct device *dev, dma_addr_t dma_addr)
326 {
327 	return dma_dev_need_sync(dev) ? __dma_need_sync(dev, dma_addr) : false;
328 }
329 #else /* !CONFIG_HAS_DMA || !CONFIG_DMA_NEED_SYNC */
dma_dev_need_sync(const struct device * dev)330 static inline bool dma_dev_need_sync(const struct device *dev)
331 {
332 	return false;
333 }
dma_sync_single_for_cpu(struct device * dev,dma_addr_t addr,size_t size,enum dma_data_direction dir)334 static inline void dma_sync_single_for_cpu(struct device *dev, dma_addr_t addr,
335 		size_t size, enum dma_data_direction dir)
336 {
337 }
dma_sync_single_for_device(struct device * dev,dma_addr_t addr,size_t size,enum dma_data_direction dir)338 static inline void dma_sync_single_for_device(struct device *dev,
339 		dma_addr_t addr, size_t size, enum dma_data_direction dir)
340 {
341 }
dma_sync_sg_for_cpu(struct device * dev,struct scatterlist * sg,int nelems,enum dma_data_direction dir)342 static inline void dma_sync_sg_for_cpu(struct device *dev,
343 		struct scatterlist *sg, int nelems, enum dma_data_direction dir)
344 {
345 }
dma_sync_sg_for_device(struct device * dev,struct scatterlist * sg,int nelems,enum dma_data_direction dir)346 static inline void dma_sync_sg_for_device(struct device *dev,
347 		struct scatterlist *sg, int nelems, enum dma_data_direction dir)
348 {
349 }
dma_need_sync(struct device * dev,dma_addr_t dma_addr)350 static inline bool dma_need_sync(struct device *dev, dma_addr_t dma_addr)
351 {
352 	return false;
353 }
354 #endif /* !CONFIG_HAS_DMA || !CONFIG_DMA_NEED_SYNC */
355 
356 struct page *dma_alloc_pages(struct device *dev, size_t size,
357 		dma_addr_t *dma_handle, enum dma_data_direction dir, gfp_t gfp);
358 void dma_free_pages(struct device *dev, size_t size, struct page *page,
359 		dma_addr_t dma_handle, enum dma_data_direction dir);
360 int dma_mmap_pages(struct device *dev, struct vm_area_struct *vma,
361 		size_t size, struct page *page);
362 
dma_alloc_noncoherent(struct device * dev,size_t size,dma_addr_t * dma_handle,enum dma_data_direction dir,gfp_t gfp)363 static inline void *dma_alloc_noncoherent(struct device *dev, size_t size,
364 		dma_addr_t *dma_handle, enum dma_data_direction dir, gfp_t gfp)
365 {
366 	struct page *page = dma_alloc_pages(dev, size, dma_handle, dir, gfp);
367 	return page ? page_address(page) : NULL;
368 }
369 
dma_free_noncoherent(struct device * dev,size_t size,void * vaddr,dma_addr_t dma_handle,enum dma_data_direction dir)370 static inline void dma_free_noncoherent(struct device *dev, size_t size,
371 		void *vaddr, dma_addr_t dma_handle, enum dma_data_direction dir)
372 {
373 	dma_free_pages(dev, size, virt_to_page(vaddr), dma_handle, dir);
374 }
375 
dma_map_single_attrs(struct device * dev,void * ptr,size_t size,enum dma_data_direction dir,unsigned long attrs)376 static inline dma_addr_t dma_map_single_attrs(struct device *dev, void *ptr,
377 		size_t size, enum dma_data_direction dir, unsigned long attrs)
378 {
379 	/* DMA must never operate on areas that might be remapped. */
380 	if (dev_WARN_ONCE(dev, is_vmalloc_addr(ptr),
381 			  "rejecting DMA map of vmalloc memory\n"))
382 		return DMA_MAPPING_ERROR;
383 	debug_dma_map_single(dev, ptr, size);
384 	return dma_map_page_attrs(dev, virt_to_page(ptr), offset_in_page(ptr),
385 			size, dir, attrs);
386 }
387 
dma_unmap_single_attrs(struct device * dev,dma_addr_t addr,size_t size,enum dma_data_direction dir,unsigned long attrs)388 static inline void dma_unmap_single_attrs(struct device *dev, dma_addr_t addr,
389 		size_t size, enum dma_data_direction dir, unsigned long attrs)
390 {
391 	return dma_unmap_page_attrs(dev, addr, size, dir, attrs);
392 }
393 
dma_sync_single_range_for_cpu(struct device * dev,dma_addr_t addr,unsigned long offset,size_t size,enum dma_data_direction dir)394 static inline void dma_sync_single_range_for_cpu(struct device *dev,
395 		dma_addr_t addr, unsigned long offset, size_t size,
396 		enum dma_data_direction dir)
397 {
398 	return dma_sync_single_for_cpu(dev, addr + offset, size, dir);
399 }
400 
dma_sync_single_range_for_device(struct device * dev,dma_addr_t addr,unsigned long offset,size_t size,enum dma_data_direction dir)401 static inline void dma_sync_single_range_for_device(struct device *dev,
402 		dma_addr_t addr, unsigned long offset, size_t size,
403 		enum dma_data_direction dir)
404 {
405 	return dma_sync_single_for_device(dev, addr + offset, size, dir);
406 }
407 
408 /**
409  * dma_unmap_sgtable - Unmap the given buffer for DMA
410  * @dev:	The device for which to perform the DMA operation
411  * @sgt:	The sg_table object describing the buffer
412  * @dir:	DMA direction
413  * @attrs:	Optional DMA attributes for the unmap operation
414  *
415  * Unmaps a buffer described by a scatterlist stored in the given sg_table
416  * object for the @dir DMA operation by the @dev device. After this function
417  * the ownership of the buffer is transferred back to the CPU domain.
418  */
dma_unmap_sgtable(struct device * dev,struct sg_table * sgt,enum dma_data_direction dir,unsigned long attrs)419 static inline void dma_unmap_sgtable(struct device *dev, struct sg_table *sgt,
420 		enum dma_data_direction dir, unsigned long attrs)
421 {
422 	dma_unmap_sg_attrs(dev, sgt->sgl, sgt->orig_nents, dir, attrs);
423 }
424 
425 /**
426  * dma_sync_sgtable_for_cpu - Synchronize the given buffer for CPU access
427  * @dev:	The device for which to perform the DMA operation
428  * @sgt:	The sg_table object describing the buffer
429  * @dir:	DMA direction
430  *
431  * Performs the needed cache synchronization and moves the ownership of the
432  * buffer back to the CPU domain, so it is safe to perform any access to it
433  * by the CPU. Before doing any further DMA operations, one has to transfer
434  * the ownership of the buffer back to the DMA domain by calling the
435  * dma_sync_sgtable_for_device().
436  */
dma_sync_sgtable_for_cpu(struct device * dev,struct sg_table * sgt,enum dma_data_direction dir)437 static inline void dma_sync_sgtable_for_cpu(struct device *dev,
438 		struct sg_table *sgt, enum dma_data_direction dir)
439 {
440 	dma_sync_sg_for_cpu(dev, sgt->sgl, sgt->orig_nents, dir);
441 }
442 
443 /**
444  * dma_sync_sgtable_for_device - Synchronize the given buffer for DMA
445  * @dev:	The device for which to perform the DMA operation
446  * @sgt:	The sg_table object describing the buffer
447  * @dir:	DMA direction
448  *
449  * Performs the needed cache synchronization and moves the ownership of the
450  * buffer back to the DMA domain, so it is safe to perform the DMA operation.
451  * Once finished, one has to call dma_sync_sgtable_for_cpu() or
452  * dma_unmap_sgtable().
453  */
dma_sync_sgtable_for_device(struct device * dev,struct sg_table * sgt,enum dma_data_direction dir)454 static inline void dma_sync_sgtable_for_device(struct device *dev,
455 		struct sg_table *sgt, enum dma_data_direction dir)
456 {
457 	dma_sync_sg_for_device(dev, sgt->sgl, sgt->orig_nents, dir);
458 }
459 
460 #define dma_map_single(d, a, s, r) dma_map_single_attrs(d, a, s, r, 0)
461 #define dma_unmap_single(d, a, s, r) dma_unmap_single_attrs(d, a, s, r, 0)
462 #define dma_map_sg(d, s, n, r) dma_map_sg_attrs(d, s, n, r, 0)
463 #define dma_unmap_sg(d, s, n, r) dma_unmap_sg_attrs(d, s, n, r, 0)
464 #define dma_map_page(d, p, o, s, r) dma_map_page_attrs(d, p, o, s, r, 0)
465 #define dma_unmap_page(d, a, s, r) dma_unmap_page_attrs(d, a, s, r, 0)
466 #define dma_get_sgtable(d, t, v, h, s) dma_get_sgtable_attrs(d, t, v, h, s, 0)
467 #define dma_mmap_coherent(d, v, c, h, s) dma_mmap_attrs(d, v, c, h, s, 0)
468 
469 bool dma_coherent_ok(struct device *dev, phys_addr_t phys, size_t size);
470 
dma_alloc_coherent(struct device * dev,size_t size,dma_addr_t * dma_handle,gfp_t gfp)471 static inline void *dma_alloc_coherent(struct device *dev, size_t size,
472 		dma_addr_t *dma_handle, gfp_t gfp)
473 {
474 	return dma_alloc_attrs(dev, size, dma_handle, gfp,
475 			(gfp & __GFP_NOWARN) ? DMA_ATTR_NO_WARN : 0);
476 }
477 
dma_free_coherent(struct device * dev,size_t size,void * cpu_addr,dma_addr_t dma_handle)478 static inline void dma_free_coherent(struct device *dev, size_t size,
479 		void *cpu_addr, dma_addr_t dma_handle)
480 {
481 	return dma_free_attrs(dev, size, cpu_addr, dma_handle, 0);
482 }
483 
484 
dma_get_mask(struct device * dev)485 static inline u64 dma_get_mask(struct device *dev)
486 {
487 	if (dev->dma_mask && *dev->dma_mask)
488 		return *dev->dma_mask;
489 	return DMA_BIT_MASK(32);
490 }
491 
492 /*
493  * Set both the DMA mask and the coherent DMA mask to the same thing.
494  * Note that we don't check the return value from dma_set_coherent_mask()
495  * as the DMA API guarantees that the coherent DMA mask can be set to
496  * the same or smaller than the streaming DMA mask.
497  */
dma_set_mask_and_coherent(struct device * dev,u64 mask)498 static inline int dma_set_mask_and_coherent(struct device *dev, u64 mask)
499 {
500 	int rc = dma_set_mask(dev, mask);
501 	if (rc == 0)
502 		dma_set_coherent_mask(dev, mask);
503 	return rc;
504 }
505 
506 /*
507  * Similar to the above, except it deals with the case where the device
508  * does not have dev->dma_mask appropriately setup.
509  */
dma_coerce_mask_and_coherent(struct device * dev,u64 mask)510 static inline int dma_coerce_mask_and_coherent(struct device *dev, u64 mask)
511 {
512 	dev->dma_mask = &dev->coherent_dma_mask;
513 	return dma_set_mask_and_coherent(dev, mask);
514 }
515 
dma_get_max_seg_size(struct device * dev)516 static inline unsigned int dma_get_max_seg_size(struct device *dev)
517 {
518 	if (dev->dma_parms && dev->dma_parms->max_segment_size)
519 		return dev->dma_parms->max_segment_size;
520 	return SZ_64K;
521 }
522 
dma_set_max_seg_size(struct device * dev,unsigned int size)523 static inline void dma_set_max_seg_size(struct device *dev, unsigned int size)
524 {
525 	if (WARN_ON_ONCE(!dev->dma_parms))
526 		return;
527 	dev->dma_parms->max_segment_size = size;
528 }
529 
dma_get_seg_boundary(struct device * dev)530 static inline unsigned long dma_get_seg_boundary(struct device *dev)
531 {
532 	if (dev->dma_parms && dev->dma_parms->segment_boundary_mask)
533 		return dev->dma_parms->segment_boundary_mask;
534 	return ULONG_MAX;
535 }
536 
537 /**
538  * dma_get_seg_boundary_nr_pages - return the segment boundary in "page" units
539  * @dev: device to guery the boundary for
540  * @page_shift: ilog() of the IOMMU page size
541  *
542  * Return the segment boundary in IOMMU page units (which may be different from
543  * the CPU page size) for the passed in device.
544  *
545  * If @dev is NULL a boundary of U32_MAX is assumed, this case is just for
546  * non-DMA API callers.
547  */
dma_get_seg_boundary_nr_pages(struct device * dev,unsigned int page_shift)548 static inline unsigned long dma_get_seg_boundary_nr_pages(struct device *dev,
549 		unsigned int page_shift)
550 {
551 	if (!dev)
552 		return (U32_MAX >> page_shift) + 1;
553 	return (dma_get_seg_boundary(dev) >> page_shift) + 1;
554 }
555 
dma_set_seg_boundary(struct device * dev,unsigned long mask)556 static inline void dma_set_seg_boundary(struct device *dev, unsigned long mask)
557 {
558 	if (WARN_ON_ONCE(!dev->dma_parms))
559 		return;
560 	dev->dma_parms->segment_boundary_mask = mask;
561 }
562 
dma_get_min_align_mask(struct device * dev)563 static inline unsigned int dma_get_min_align_mask(struct device *dev)
564 {
565 	if (dev->dma_parms)
566 		return dev->dma_parms->min_align_mask;
567 	return 0;
568 }
569 
dma_set_min_align_mask(struct device * dev,unsigned int min_align_mask)570 static inline void dma_set_min_align_mask(struct device *dev,
571 		unsigned int min_align_mask)
572 {
573 	if (WARN_ON_ONCE(!dev->dma_parms))
574 		return;
575 	dev->dma_parms->min_align_mask = min_align_mask;
576 }
577 
578 #ifndef dma_get_cache_alignment
dma_get_cache_alignment(void)579 static inline int dma_get_cache_alignment(void)
580 {
581 #ifdef ARCH_HAS_DMA_MINALIGN
582 	return ARCH_DMA_MINALIGN;
583 #endif
584 	return 1;
585 }
586 #endif
587 
dmam_alloc_coherent(struct device * dev,size_t size,dma_addr_t * dma_handle,gfp_t gfp)588 static inline void *dmam_alloc_coherent(struct device *dev, size_t size,
589 		dma_addr_t *dma_handle, gfp_t gfp)
590 {
591 	return dmam_alloc_attrs(dev, size, dma_handle, gfp,
592 			(gfp & __GFP_NOWARN) ? DMA_ATTR_NO_WARN : 0);
593 }
594 
dma_alloc_wc(struct device * dev,size_t size,dma_addr_t * dma_addr,gfp_t gfp)595 static inline void *dma_alloc_wc(struct device *dev, size_t size,
596 				 dma_addr_t *dma_addr, gfp_t gfp)
597 {
598 	unsigned long attrs = DMA_ATTR_WRITE_COMBINE;
599 
600 	if (gfp & __GFP_NOWARN)
601 		attrs |= DMA_ATTR_NO_WARN;
602 
603 	return dma_alloc_attrs(dev, size, dma_addr, gfp, attrs);
604 }
605 
dma_free_wc(struct device * dev,size_t size,void * cpu_addr,dma_addr_t dma_addr)606 static inline void dma_free_wc(struct device *dev, size_t size,
607 			       void *cpu_addr, dma_addr_t dma_addr)
608 {
609 	return dma_free_attrs(dev, size, cpu_addr, dma_addr,
610 			      DMA_ATTR_WRITE_COMBINE);
611 }
612 
dma_mmap_wc(struct device * dev,struct vm_area_struct * vma,void * cpu_addr,dma_addr_t dma_addr,size_t size)613 static inline int dma_mmap_wc(struct device *dev,
614 			      struct vm_area_struct *vma,
615 			      void *cpu_addr, dma_addr_t dma_addr,
616 			      size_t size)
617 {
618 	return dma_mmap_attrs(dev, vma, cpu_addr, dma_addr, size,
619 			      DMA_ATTR_WRITE_COMBINE);
620 }
621 
622 #ifdef CONFIG_NEED_DMA_MAP_STATE
623 #define DEFINE_DMA_UNMAP_ADDR(ADDR_NAME)        dma_addr_t ADDR_NAME
624 #define DEFINE_DMA_UNMAP_LEN(LEN_NAME)          __u32 LEN_NAME
625 #define dma_unmap_addr(PTR, ADDR_NAME)           ((PTR)->ADDR_NAME)
626 #define dma_unmap_addr_set(PTR, ADDR_NAME, VAL)  (((PTR)->ADDR_NAME) = (VAL))
627 #define dma_unmap_len(PTR, LEN_NAME)             ((PTR)->LEN_NAME)
628 #define dma_unmap_len_set(PTR, LEN_NAME, VAL)    (((PTR)->LEN_NAME) = (VAL))
629 #else
630 #define DEFINE_DMA_UNMAP_ADDR(ADDR_NAME)
631 #define DEFINE_DMA_UNMAP_LEN(LEN_NAME)
632 #define dma_unmap_addr(PTR, ADDR_NAME)           (0)
633 #define dma_unmap_addr_set(PTR, ADDR_NAME, VAL)  do { } while (0)
634 #define dma_unmap_len(PTR, LEN_NAME)             (0)
635 #define dma_unmap_len_set(PTR, LEN_NAME, VAL)    do { } while (0)
636 #endif
637 
638 #endif /* _LINUX_DMA_MAPPING_H */
639