dma-noncoherent.c (9cdf083f981b8d37b3212400a359368661385099) dma-noncoherent.c (f8c55dc6e828324fc58c0bb32d72a5a4041d1c3b)
1// SPDX-License-Identifier: GPL-2.0
1/*
2/*
2 * This file is subject to the terms and conditions of the GNU General Public
3 * License. See the file "COPYING" in the main directory of this archive
4 * for more details.
5 *
6 * Copyright (C) 2000 Ani Joshi <ajoshi@unixbox.com>
3 * Copyright (C) 2000 Ani Joshi <ajoshi@unixbox.com>
7 * Copyright (C) 2000, 2001 Ralf Baechle <ralf@gnu.org>
4 * Copyright (C) 2000, 2001, 06 Ralf Baechle <ralf@linux-mips.org>
8 * swiped from i386, and cloned for MIPS by Geert, polished by Ralf.
9 */
5 * swiped from i386, and cloned for MIPS by Geert, polished by Ralf.
6 */
10#include <linux/types.h>
11#include <linux/mm.h>
12#include <linux/module.h>
13#include <linux/string.h>
14#include <linux/dma-mapping.h>
7#include <linux/dma-direct.h>
8#include <linux/dma-noncoherent.h>
9#include <linux/dma-contiguous.h>
10#include <linux/highmem.h>
15
16#include <asm/cache.h>
11
12#include <asm/cache.h>
13#include <asm/cpu-type.h>
14#include <asm/dma-coherence.h>
17#include <asm/io.h>
18
15#include <asm/io.h>
16
17#ifdef CONFIG_DMA_PERDEV_COHERENT
18static inline int dev_is_coherent(struct device *dev)
19{
20 return dev->archdata.dma_coherent;
21}
22#else
23static inline int dev_is_coherent(struct device *dev)
24{
25 switch (coherentio) {
26 default:
27 case IO_COHERENCE_DEFAULT:
28 return hw_coherentio;
29 case IO_COHERENCE_ENABLED:
30 return 1;
31 case IO_COHERENCE_DISABLED:
32 return 0;
33 }
34}
35#endif /* CONFIG_DMA_PERDEV_COHERENT */
36
19/*
37/*
20 * Warning on the terminology - Linux calls an uncached area coherent;
21 * MIPS terminology calls memory areas with hardware maintained coherency
22 * coherent.
38 * The affected CPUs below in 'cpu_needs_post_dma_flush()' can speculatively
39 * fill random cachelines with stale data at any time, requiring an extra
40 * flush post-DMA.
41 *
42 * Warning on the terminology - Linux calls an uncached area coherent; MIPS
43 * terminology calls memory areas with hardware maintained coherency coherent.
44 *
45 * Note that the R14000 and R16000 should also be checked for in this condition.
46 * However this function is only called on non-I/O-coherent systems and only the
47 * R10000 and R12000 are used in such systems, the SGI IP28 Indigo² rsp.
48 * SGI IP32 aka O2.
23 */
49 */
24
25void *dma_alloc_noncoherent(struct device *dev, size_t size,
26 dma_addr_t * dma_handle, gfp_t gfp)
50static inline bool cpu_needs_post_dma_flush(struct device *dev)
27{
51{
28 void *ret;
29 /* ignore region specifiers */
30 gfp &= ~(__GFP_DMA | __GFP_HIGHMEM);
52 if (dev_is_coherent(dev))
53 return false;
31
54
32 if (dev == NULL || (dev->coherent_dma_mask < 0xffffffff))
33 gfp |= GFP_DMA;
34 ret = (void *) __get_free_pages(gfp, get_order(size));
35
36 if (ret != NULL) {
37 memset(ret, 0, size);
38 *dma_handle = virt_to_phys(ret);
55 switch (boot_cpu_type()) {
56 case CPU_R10000:
57 case CPU_R12000:
58 case CPU_BMIPS5000:
59 return true;
60 default:
61 /*
62 * Presence of MAARs suggests that the CPU supports
63 * speculatively prefetching data, and therefore requires
64 * the post-DMA flush/invalidate.
65 */
66 return cpu_has_maar;
39 }
67 }
40
41 return ret;
42}
43
68}
69
44EXPORT_SYMBOL(dma_alloc_noncoherent);
45
46void *dma_alloc_coherent(struct device *dev, size_t size,
47 dma_addr_t * dma_handle, gfp_t gfp)
70void *arch_dma_alloc(struct device *dev, size_t size,
71 dma_addr_t *dma_handle, gfp_t gfp, unsigned long attrs)
48{
49 void *ret;
50
72{
73 void *ret;
74
51 ret = dma_alloc_noncoherent(dev, size, dma_handle, gfp);
52 if (ret) {
75 ret = dma_direct_alloc(dev, size, dma_handle, gfp, attrs);
76 if (!ret)
77 return NULL;
78
79 if (!dev_is_coherent(dev) && !(attrs & DMA_ATTR_NON_CONSISTENT)) {
53 dma_cache_wback_inv((unsigned long) ret, size);
54 ret = UNCAC_ADDR(ret);
55 }
56
57 return ret;
58}
59
80 dma_cache_wback_inv((unsigned long) ret, size);
81 ret = UNCAC_ADDR(ret);
82 }
83
84 return ret;
85}
86
60EXPORT_SYMBOL(dma_alloc_coherent);
61
62void dma_free_noncoherent(struct device *dev, size_t size, void *vaddr,
63 dma_addr_t dma_handle)
87void arch_dma_free(struct device *dev, size_t size, void *cpu_addr,
88 dma_addr_t dma_addr, unsigned long attrs)
64{
89{
65 free_pages((unsigned long) vaddr, get_order(size));
90 if (!(attrs & DMA_ATTR_NON_CONSISTENT) && !dev_is_coherent(dev))
91 cpu_addr = (void *)CAC_ADDR((unsigned long)cpu_addr);
92 dma_direct_free(dev, size, cpu_addr, dma_addr, attrs);
66}
67
93}
94
68EXPORT_SYMBOL(dma_free_noncoherent);
69
70void dma_free_coherent(struct device *dev, size_t size, void *vaddr,
71 dma_addr_t dma_handle)
95int arch_dma_mmap(struct device *dev, struct vm_area_struct *vma,
96 void *cpu_addr, dma_addr_t dma_addr, size_t size,
97 unsigned long attrs)
72{
98{
73 unsigned long addr = (unsigned long) vaddr;
99 unsigned long user_count = vma_pages(vma);
100 unsigned long count = PAGE_ALIGN(size) >> PAGE_SHIFT;
101 unsigned long addr = (unsigned long)cpu_addr;
102 unsigned long off = vma->vm_pgoff;
103 unsigned long pfn;
104 int ret = -ENXIO;
74
105
75 addr = CAC_ADDR(addr);
76 free_pages(addr, get_order(size));
77}
106 if (!dev_is_coherent(dev))
107 addr = CAC_ADDR(addr);
78
108
79EXPORT_SYMBOL(dma_free_coherent);
109 pfn = page_to_pfn(virt_to_page((void *)addr));
80
110
81static inline void __dma_sync(unsigned long addr, size_t size,
82 enum dma_data_direction direction)
111 if (attrs & DMA_ATTR_WRITE_COMBINE)
112 vma->vm_page_prot = pgprot_writecombine(vma->vm_page_prot);
113 else
114 vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot);
115
116 if (dma_mmap_from_dev_coherent(dev, vma, cpu_addr, size, &ret))
117 return ret;
118
119 if (off < count && user_count <= (count - off)) {
120 ret = remap_pfn_range(vma, vma->vm_start,
121 pfn + off,
122 user_count << PAGE_SHIFT,
123 vma->vm_page_prot);
124 }
125
126 return ret;
127}
128
129static inline void dma_sync_virt(void *addr, size_t size,
130 enum dma_data_direction dir)
83{
131{
84 switch (direction) {
132 switch (dir) {
85 case DMA_TO_DEVICE:
133 case DMA_TO_DEVICE:
86 dma_cache_wback(addr, size);
134 dma_cache_wback((unsigned long)addr, size);
87 break;
88
89 case DMA_FROM_DEVICE:
135 break;
136
137 case DMA_FROM_DEVICE:
90 dma_cache_inv(addr, size);
138 dma_cache_inv((unsigned long)addr, size);
91 break;
92
93 case DMA_BIDIRECTIONAL:
139 break;
140
141 case DMA_BIDIRECTIONAL:
94 dma_cache_wback_inv(addr, size);
142 dma_cache_wback_inv((unsigned long)addr, size);
95 break;
96
97 default:
98 BUG();
99 }
100}
101
143 break;
144
145 default:
146 BUG();
147 }
148}
149
102dma_addr_t dma_map_single(struct device *dev, void *ptr, size_t size,
103 enum dma_data_direction direction)
150/*
151 * A single sg entry may refer to multiple physically contiguous pages. But
152 * we still need to process highmem pages individually. If highmem is not
153 * configured then the bulk of this loop gets optimized out.
154 */
155static inline void dma_sync_phys(phys_addr_t paddr, size_t size,
156 enum dma_data_direction dir)
104{
157{
105 unsigned long addr = (unsigned long) ptr;
158 struct page *page = pfn_to_page(paddr >> PAGE_SHIFT);
159 unsigned long offset = paddr & ~PAGE_MASK;
160 size_t left = size;
106
161
107 __dma_sync(addr, size, direction);
162 do {
163 size_t len = left;
108
164
109 return virt_to_phys(ptr);
110}
165 if (PageHighMem(page)) {
166 void *addr;
111
167
112EXPORT_SYMBOL(dma_map_single);
168 if (offset + len > PAGE_SIZE) {
169 if (offset >= PAGE_SIZE) {
170 page += offset >> PAGE_SHIFT;
171 offset &= ~PAGE_MASK;
172 }
173 len = PAGE_SIZE - offset;
174 }
113
175
114void dma_unmap_single(struct device *dev, dma_addr_t dma_addr, size_t size,
115 enum dma_data_direction direction)
116{
117 unsigned long addr;
118 addr = dma_addr + PAGE_OFFSET;
119
120 //__dma_sync(addr, size, direction);
176 addr = kmap_atomic(page);
177 dma_sync_virt(addr + offset, len, dir);
178 kunmap_atomic(addr);
179 } else
180 dma_sync_virt(page_address(page) + offset, size, dir);
181 offset = 0;
182 page++;
183 left -= len;
184 } while (left);
121}
122
185}
186
123EXPORT_SYMBOL(dma_unmap_single);
124
125int dma_map_sg(struct device *dev, struct scatterlist *sg, int nents,
126 enum dma_data_direction direction)
187void arch_sync_dma_for_device(struct device *dev, phys_addr_t paddr,
188 size_t size, enum dma_data_direction dir)
127{
189{
128 int i;
129
130 BUG_ON(direction == DMA_NONE);
131
132 for (i = 0; i < nents; i++, sg++) {
133 unsigned long addr;
134
135 addr = (unsigned long) page_address(sg->page);
136 if (addr) {
137 __dma_sync(addr + sg->offset, sg->length, direction);
138 sg->dma_address = (dma_addr_t)page_to_phys(sg->page)
139 + sg->offset;
140 }
141 }
142
143 return nents;
190 if (!dev_is_coherent(dev))
191 dma_sync_phys(paddr, size, dir);
144}
145
192}
193
146EXPORT_SYMBOL(dma_map_sg);
147
148dma_addr_t dma_map_page(struct device *dev, struct page *page,
149 unsigned long offset, size_t size, enum dma_data_direction direction)
194void arch_sync_dma_for_cpu(struct device *dev, phys_addr_t paddr,
195 size_t size, enum dma_data_direction dir)
150{
196{
151 unsigned long addr;
152
153 BUG_ON(direction == DMA_NONE);
154
155 addr = (unsigned long) page_address(page) + offset;
156 dma_cache_wback_inv(addr, size);
157
158 return page_to_phys(page) + offset;
197 if (cpu_needs_post_dma_flush(dev))
198 dma_sync_phys(paddr, size, dir);
159}
160
199}
200
161EXPORT_SYMBOL(dma_map_page);
162
163void dma_unmap_page(struct device *dev, dma_addr_t dma_address, size_t size,
164 enum dma_data_direction direction)
201void arch_dma_cache_sync(struct device *dev, void *vaddr, size_t size,
202 enum dma_data_direction direction)
165{
166 BUG_ON(direction == DMA_NONE);
167
203{
204 BUG_ON(direction == DMA_NONE);
205
168 if (direction != DMA_TO_DEVICE) {
169 unsigned long addr;
170
171 addr = dma_address + PAGE_OFFSET;
172 dma_cache_wback_inv(addr, size);
173 }
206 if (!dev_is_coherent(dev))
207 dma_sync_virt(vaddr, size, direction);
174}
208}
175
176EXPORT_SYMBOL(dma_unmap_page);
177
178void dma_unmap_sg(struct device *dev, struct scatterlist *sg, int nhwentries,
179 enum dma_data_direction direction)
180{
181 unsigned long addr;
182 int i;
183
184 BUG_ON(direction == DMA_NONE);
185
186 if (direction == DMA_TO_DEVICE)
187 return;
188
189 for (i = 0; i < nhwentries; i++, sg++) {
190 addr = (unsigned long) page_address(sg->page);
191 if (addr)
192 __dma_sync(addr + sg->offset, sg->length, direction);
193 }
194}
195
196EXPORT_SYMBOL(dma_unmap_sg);
197
198void dma_sync_single_for_cpu(struct device *dev, dma_addr_t dma_handle,
199 size_t size, enum dma_data_direction direction)
200{
201 unsigned long addr;
202
203 BUG_ON(direction == DMA_NONE);
204
205 addr = dma_handle + PAGE_OFFSET;
206 __dma_sync(addr, size, direction);
207}
208
209EXPORT_SYMBOL(dma_sync_single_for_cpu);
210
211void dma_sync_single_for_device(struct device *dev, dma_addr_t dma_handle,
212 size_t size, enum dma_data_direction direction)
213{
214 unsigned long addr;
215
216 BUG_ON(direction == DMA_NONE);
217
218 addr = dma_handle + PAGE_OFFSET;
219 __dma_sync(addr, size, direction);
220}
221
222EXPORT_SYMBOL(dma_sync_single_for_device);
223
224void dma_sync_single_range_for_cpu(struct device *dev, dma_addr_t dma_handle,
225 unsigned long offset, size_t size, enum dma_data_direction direction)
226{
227 unsigned long addr;
228
229 BUG_ON(direction == DMA_NONE);
230
231 addr = dma_handle + offset + PAGE_OFFSET;
232 __dma_sync(addr, size, direction);
233}
234
235EXPORT_SYMBOL(dma_sync_single_range_for_cpu);
236
237void dma_sync_single_range_for_device(struct device *dev, dma_addr_t dma_handle,
238 unsigned long offset, size_t size, enum dma_data_direction direction)
239{
240 unsigned long addr;
241
242 BUG_ON(direction == DMA_NONE);
243
244 addr = dma_handle + offset + PAGE_OFFSET;
245 __dma_sync(addr, size, direction);
246}
247
248EXPORT_SYMBOL(dma_sync_single_range_for_device);
249
250void dma_sync_sg_for_cpu(struct device *dev, struct scatterlist *sg, int nelems,
251 enum dma_data_direction direction)
252{
253 int i;
254
255 BUG_ON(direction == DMA_NONE);
256
257 /* Make sure that gcc doesn't leave the empty loop body. */
258 for (i = 0; i < nelems; i++, sg++)
259 __dma_sync((unsigned long)page_address(sg->page),
260 sg->length, direction);
261}
262
263EXPORT_SYMBOL(dma_sync_sg_for_cpu);
264
265void dma_sync_sg_for_device(struct device *dev, struct scatterlist *sg, int nelems,
266 enum dma_data_direction direction)
267{
268 int i;
269
270 BUG_ON(direction == DMA_NONE);
271
272 /* Make sure that gcc doesn't leave the empty loop body. */
273 for (i = 0; i < nelems; i++, sg++)
274 __dma_sync((unsigned long)page_address(sg->page),
275 sg->length, direction);
276}
277
278EXPORT_SYMBOL(dma_sync_sg_for_device);
279
280int dma_mapping_error(dma_addr_t dma_addr)
281{
282 return 0;
283}
284
285EXPORT_SYMBOL(dma_mapping_error);
286
287int dma_supported(struct device *dev, u64 mask)
288{
289 /*
290 * we fall back to GFP_DMA when the mask isn't all 1s,
291 * so we can't guarantee allocations that must be
292 * within a tighter range than GFP_DMA..
293 */
294 if (mask < 0x00ffffff)
295 return 0;
296
297 return 1;
298}
299
300EXPORT_SYMBOL(dma_supported);
301
302int dma_is_consistent(struct device *dev, dma_addr_t dma_addr)
303{
304 return 1;
305}
306
307EXPORT_SYMBOL(dma_is_consistent);
308
309void dma_cache_sync(struct device *dev, void *vaddr, size_t size,
310 enum dma_data_direction direction)
311{
312 if (direction == DMA_NONE)
313 return;
314
315 dma_cache_wback_inv((unsigned long)vaddr, size);
316}
317
318EXPORT_SYMBOL(dma_cache_sync);
319
320/* The DAC routines are a PCIism.. */
321
322#ifdef CONFIG_PCI
323
324#include <linux/pci.h>
325
326dma64_addr_t pci_dac_page_to_dma(struct pci_dev *pdev,
327 struct page *page, unsigned long offset, int direction)
328{
329 return (dma64_addr_t)page_to_phys(page) + offset;
330}
331
332EXPORT_SYMBOL(pci_dac_page_to_dma);
333
334struct page *pci_dac_dma_to_page(struct pci_dev *pdev,
335 dma64_addr_t dma_addr)
336{
337 return mem_map + (dma_addr >> PAGE_SHIFT);
338}
339
340EXPORT_SYMBOL(pci_dac_dma_to_page);
341
342unsigned long pci_dac_dma_to_offset(struct pci_dev *pdev,
343 dma64_addr_t dma_addr)
344{
345 return dma_addr & ~PAGE_MASK;
346}
347
348EXPORT_SYMBOL(pci_dac_dma_to_offset);
349
350void pci_dac_dma_sync_single_for_cpu(struct pci_dev *pdev,
351 dma64_addr_t dma_addr, size_t len, int direction)
352{
353 BUG_ON(direction == PCI_DMA_NONE);
354
355 dma_cache_wback_inv(dma_addr + PAGE_OFFSET, len);
356}
357
358EXPORT_SYMBOL(pci_dac_dma_sync_single_for_cpu);
359
360void pci_dac_dma_sync_single_for_device(struct pci_dev *pdev,
361 dma64_addr_t dma_addr, size_t len, int direction)
362{
363 BUG_ON(direction == PCI_DMA_NONE);
364
365 dma_cache_wback_inv(dma_addr + PAGE_OFFSET, len);
366}
367
368EXPORT_SYMBOL(pci_dac_dma_sync_single_for_device);
369
370#endif /* CONFIG_PCI */