1 /* 2 * DMA coherent memory allocation. 3 * 4 * This program is free software; you can redistribute it and/or modify it 5 * under the terms of the GNU General Public License as published by the 6 * Free Software Foundation; either version 2 of the License, or (at your 7 * option) any later version. 8 * 9 * Copyright (C) 2002 - 2005 Tensilica Inc. 10 * Copyright (C) 2015 Cadence Design Systems Inc. 11 * 12 * Based on version for i386. 13 * 14 * Chris Zankel <chris@zankel.net> 15 * Joe Taylor <joe@tensilica.com, joetylr@yahoo.com> 16 */ 17 18 #include <linux/dma-contiguous.h> 19 #include <linux/gfp.h> 20 #include <linux/highmem.h> 21 #include <linux/mm.h> 22 #include <linux/module.h> 23 #include <linux/pci.h> 24 #include <linux/string.h> 25 #include <linux/types.h> 26 #include <asm/cacheflush.h> 27 #include <asm/io.h> 28 29 void dma_cache_sync(struct device *dev, void *vaddr, size_t size, 30 enum dma_data_direction dir) 31 { 32 switch (dir) { 33 case DMA_BIDIRECTIONAL: 34 __flush_invalidate_dcache_range((unsigned long)vaddr, size); 35 break; 36 37 case DMA_FROM_DEVICE: 38 __invalidate_dcache_range((unsigned long)vaddr, size); 39 break; 40 41 case DMA_TO_DEVICE: 42 __flush_dcache_range((unsigned long)vaddr, size); 43 break; 44 45 case DMA_NONE: 46 BUG(); 47 break; 48 } 49 } 50 EXPORT_SYMBOL(dma_cache_sync); 51 52 static void do_cache_op(dma_addr_t dma_handle, size_t size, 53 void (*fn)(unsigned long, unsigned long)) 54 { 55 unsigned long off = dma_handle & (PAGE_SIZE - 1); 56 unsigned long pfn = PFN_DOWN(dma_handle); 57 struct page *page = pfn_to_page(pfn); 58 59 if (!PageHighMem(page)) 60 fn((unsigned long)bus_to_virt(dma_handle), size); 61 else 62 while (size > 0) { 63 size_t sz = min_t(size_t, size, PAGE_SIZE - off); 64 void *vaddr = kmap_atomic(page); 65 66 fn((unsigned long)vaddr + off, sz); 67 kunmap_atomic(vaddr); 68 off = 0; 69 ++page; 70 size -= sz; 71 } 72 } 73 74 static void xtensa_sync_single_for_cpu(struct device *dev, 75 dma_addr_t dma_handle, size_t size, 76 enum dma_data_direction dir) 77 { 78 switch (dir) { 79 case DMA_BIDIRECTIONAL: 80 case DMA_FROM_DEVICE: 81 do_cache_op(dma_handle, size, __invalidate_dcache_range); 82 break; 83 84 case DMA_NONE: 85 BUG(); 86 break; 87 88 default: 89 break; 90 } 91 } 92 93 static void xtensa_sync_single_for_device(struct device *dev, 94 dma_addr_t dma_handle, size_t size, 95 enum dma_data_direction dir) 96 { 97 switch (dir) { 98 case DMA_BIDIRECTIONAL: 99 case DMA_TO_DEVICE: 100 if (XCHAL_DCACHE_IS_WRITEBACK) 101 do_cache_op(dma_handle, size, __flush_dcache_range); 102 break; 103 104 case DMA_NONE: 105 BUG(); 106 break; 107 108 default: 109 break; 110 } 111 } 112 113 static void xtensa_sync_sg_for_cpu(struct device *dev, 114 struct scatterlist *sg, int nents, 115 enum dma_data_direction dir) 116 { 117 struct scatterlist *s; 118 int i; 119 120 for_each_sg(sg, s, nents, i) { 121 xtensa_sync_single_for_cpu(dev, sg_dma_address(s), 122 sg_dma_len(s), dir); 123 } 124 } 125 126 static void xtensa_sync_sg_for_device(struct device *dev, 127 struct scatterlist *sg, int nents, 128 enum dma_data_direction dir) 129 { 130 struct scatterlist *s; 131 int i; 132 133 for_each_sg(sg, s, nents, i) { 134 xtensa_sync_single_for_device(dev, sg_dma_address(s), 135 sg_dma_len(s), dir); 136 } 137 } 138 139 /* 140 * Note: We assume that the full memory space is always mapped to 'kseg' 141 * Otherwise we have to use page attributes (not implemented). 142 */ 143 144 static void *xtensa_dma_alloc(struct device *dev, size_t size, 145 dma_addr_t *handle, gfp_t flag, 146 unsigned long attrs) 147 { 148 unsigned long ret; 149 unsigned long uncached = 0; 150 unsigned long count = PAGE_ALIGN(size) >> PAGE_SHIFT; 151 struct page *page = NULL; 152 153 /* ignore region speicifiers */ 154 155 flag &= ~(__GFP_DMA | __GFP_HIGHMEM); 156 157 if (dev == NULL || (dev->coherent_dma_mask < 0xffffffff)) 158 flag |= GFP_DMA; 159 160 if (gfpflags_allow_blocking(flag)) 161 page = dma_alloc_from_contiguous(dev, count, get_order(size), 162 flag); 163 164 if (!page) 165 page = alloc_pages(flag, get_order(size)); 166 167 if (!page) 168 return NULL; 169 170 ret = (unsigned long)page_address(page); 171 172 /* We currently don't support coherent memory outside KSEG */ 173 174 BUG_ON(ret < XCHAL_KSEG_CACHED_VADDR || 175 ret > XCHAL_KSEG_CACHED_VADDR + XCHAL_KSEG_SIZE - 1); 176 177 uncached = ret + XCHAL_KSEG_BYPASS_VADDR - XCHAL_KSEG_CACHED_VADDR; 178 *handle = virt_to_bus((void *)ret); 179 __invalidate_dcache_range(ret, size); 180 181 return (void *)uncached; 182 } 183 184 static void xtensa_dma_free(struct device *dev, size_t size, void *vaddr, 185 dma_addr_t dma_handle, unsigned long attrs) 186 { 187 unsigned long addr = (unsigned long)vaddr + 188 XCHAL_KSEG_CACHED_VADDR - XCHAL_KSEG_BYPASS_VADDR; 189 struct page *page = virt_to_page(addr); 190 unsigned long count = PAGE_ALIGN(size) >> PAGE_SHIFT; 191 192 BUG_ON(addr < XCHAL_KSEG_CACHED_VADDR || 193 addr > XCHAL_KSEG_CACHED_VADDR + XCHAL_KSEG_SIZE - 1); 194 195 if (!dma_release_from_contiguous(dev, page, count)) 196 __free_pages(page, get_order(size)); 197 } 198 199 static dma_addr_t xtensa_map_page(struct device *dev, struct page *page, 200 unsigned long offset, size_t size, 201 enum dma_data_direction dir, 202 unsigned long attrs) 203 { 204 dma_addr_t dma_handle = page_to_phys(page) + offset; 205 206 if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC)) 207 xtensa_sync_single_for_device(dev, dma_handle, size, dir); 208 209 return dma_handle; 210 } 211 212 static void xtensa_unmap_page(struct device *dev, dma_addr_t dma_handle, 213 size_t size, enum dma_data_direction dir, 214 unsigned long attrs) 215 { 216 if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC)) 217 xtensa_sync_single_for_cpu(dev, dma_handle, size, dir); 218 } 219 220 static int xtensa_map_sg(struct device *dev, struct scatterlist *sg, 221 int nents, enum dma_data_direction dir, 222 unsigned long attrs) 223 { 224 struct scatterlist *s; 225 int i; 226 227 for_each_sg(sg, s, nents, i) { 228 s->dma_address = xtensa_map_page(dev, sg_page(s), s->offset, 229 s->length, dir, attrs); 230 } 231 return nents; 232 } 233 234 static void xtensa_unmap_sg(struct device *dev, 235 struct scatterlist *sg, int nents, 236 enum dma_data_direction dir, 237 unsigned long attrs) 238 { 239 struct scatterlist *s; 240 int i; 241 242 for_each_sg(sg, s, nents, i) { 243 xtensa_unmap_page(dev, sg_dma_address(s), 244 sg_dma_len(s), dir, attrs); 245 } 246 } 247 248 int xtensa_dma_mapping_error(struct device *dev, dma_addr_t dma_addr) 249 { 250 return 0; 251 } 252 253 const struct dma_map_ops xtensa_dma_map_ops = { 254 .alloc = xtensa_dma_alloc, 255 .free = xtensa_dma_free, 256 .map_page = xtensa_map_page, 257 .unmap_page = xtensa_unmap_page, 258 .map_sg = xtensa_map_sg, 259 .unmap_sg = xtensa_unmap_sg, 260 .sync_single_for_cpu = xtensa_sync_single_for_cpu, 261 .sync_single_for_device = xtensa_sync_single_for_device, 262 .sync_sg_for_cpu = xtensa_sync_sg_for_cpu, 263 .sync_sg_for_device = xtensa_sync_sg_for_device, 264 .mapping_error = xtensa_dma_mapping_error, 265 }; 266 EXPORT_SYMBOL(xtensa_dma_map_ops); 267 268 #define PREALLOC_DMA_DEBUG_ENTRIES (1 << 16) 269 270 static int __init xtensa_dma_init(void) 271 { 272 dma_debug_init(PREALLOC_DMA_DEBUG_ENTRIES); 273 return 0; 274 } 275 fs_initcall(xtensa_dma_init); 276