1 #include <linux/dma-mapping.h> 2 #include <linux/dma-debug.h> 3 #include <linux/dmar.h> 4 #include <linux/bootmem.h> 5 #include <linux/pci.h> 6 7 #include <asm/proto.h> 8 #include <asm/dma.h> 9 #include <asm/iommu.h> 10 #include <asm/gart.h> 11 #include <asm/calgary.h> 12 #include <asm/amd_iommu.h> 13 14 static int forbid_dac __read_mostly; 15 16 struct dma_map_ops *dma_ops; 17 EXPORT_SYMBOL(dma_ops); 18 19 static int iommu_sac_force __read_mostly; 20 21 #ifdef CONFIG_IOMMU_DEBUG 22 int panic_on_overflow __read_mostly = 1; 23 int force_iommu __read_mostly = 1; 24 #else 25 int panic_on_overflow __read_mostly = 0; 26 int force_iommu __read_mostly = 0; 27 #endif 28 29 int iommu_merge __read_mostly = 0; 30 31 int no_iommu __read_mostly; 32 /* Set this to 1 if there is a HW IOMMU in the system */ 33 int iommu_detected __read_mostly = 0; 34 35 int iommu_pass_through; 36 37 dma_addr_t bad_dma_address __read_mostly = 0; 38 EXPORT_SYMBOL(bad_dma_address); 39 40 /* Dummy device used for NULL arguments (normally ISA). Better would 41 be probably a smaller DMA mask, but this is bug-to-bug compatible 42 to older i386. */ 43 struct device x86_dma_fallback_dev = { 44 .init_name = "fallback device", 45 .coherent_dma_mask = DMA_BIT_MASK(32), 46 .dma_mask = &x86_dma_fallback_dev.coherent_dma_mask, 47 }; 48 EXPORT_SYMBOL(x86_dma_fallback_dev); 49 50 /* Number of entries preallocated for DMA-API debugging */ 51 #define PREALLOC_DMA_DEBUG_ENTRIES 32768 52 53 int dma_set_mask(struct device *dev, u64 mask) 54 { 55 if (!dev->dma_mask || !dma_supported(dev, mask)) 56 return -EIO; 57 58 *dev->dma_mask = mask; 59 60 return 0; 61 } 62 EXPORT_SYMBOL(dma_set_mask); 63 64 #ifdef CONFIG_X86_64 65 static __initdata void *dma32_bootmem_ptr; 66 static unsigned long dma32_bootmem_size __initdata = (128ULL<<20); 67 68 static int __init parse_dma32_size_opt(char *p) 69 { 70 if (!p) 71 return -EINVAL; 72 dma32_bootmem_size = memparse(p, &p); 73 return 0; 74 } 75 early_param("dma32_size", parse_dma32_size_opt); 76 77 void __init dma32_reserve_bootmem(void) 78 { 79 unsigned long size, align; 80 if (max_pfn <= MAX_DMA32_PFN) 81 return; 82 83 /* 84 * check aperture_64.c allocate_aperture() for reason about 85 * using 512M as goal 86 */ 87 align = 64ULL<<20; 88 size = roundup(dma32_bootmem_size, align); 89 dma32_bootmem_ptr = __alloc_bootmem_nopanic(size, align, 90 512ULL<<20); 91 if (dma32_bootmem_ptr) 92 dma32_bootmem_size = size; 93 else 94 dma32_bootmem_size = 0; 95 } 96 static void __init dma32_free_bootmem(void) 97 { 98 99 if (max_pfn <= MAX_DMA32_PFN) 100 return; 101 102 if (!dma32_bootmem_ptr) 103 return; 104 105 free_bootmem(__pa(dma32_bootmem_ptr), dma32_bootmem_size); 106 107 dma32_bootmem_ptr = NULL; 108 dma32_bootmem_size = 0; 109 } 110 #endif 111 112 void __init pci_iommu_alloc(void) 113 { 114 #ifdef CONFIG_X86_64 115 /* free the range so iommu could get some range less than 4G */ 116 dma32_free_bootmem(); 117 #endif 118 119 /* 120 * The order of these functions is important for 121 * fall-back/fail-over reasons 122 */ 123 gart_iommu_hole_init(); 124 125 detect_calgary(); 126 127 detect_intel_iommu(); 128 129 amd_iommu_detect(); 130 131 pci_swiotlb_init(); 132 } 133 134 void *dma_generic_alloc_coherent(struct device *dev, size_t size, 135 dma_addr_t *dma_addr, gfp_t flag) 136 { 137 unsigned long dma_mask; 138 struct page *page; 139 dma_addr_t addr; 140 141 dma_mask = dma_alloc_coherent_mask(dev, flag); 142 143 flag |= __GFP_ZERO; 144 again: 145 page = alloc_pages_node(dev_to_node(dev), flag, get_order(size)); 146 if (!page) 147 return NULL; 148 149 addr = page_to_phys(page); 150 if (!is_buffer_dma_capable(dma_mask, addr, size)) { 151 __free_pages(page, get_order(size)); 152 153 if (dma_mask < DMA_BIT_MASK(32) && !(flag & GFP_DMA)) { 154 flag = (flag & ~GFP_DMA32) | GFP_DMA; 155 goto again; 156 } 157 158 return NULL; 159 } 160 161 *dma_addr = addr; 162 return page_address(page); 163 } 164 165 /* 166 * See <Documentation/x86_64/boot-options.txt> for the iommu kernel parameter 167 * documentation. 168 */ 169 static __init int iommu_setup(char *p) 170 { 171 iommu_merge = 1; 172 173 if (!p) 174 return -EINVAL; 175 176 while (*p) { 177 if (!strncmp(p, "off", 3)) 178 no_iommu = 1; 179 /* gart_parse_options has more force support */ 180 if (!strncmp(p, "force", 5)) 181 force_iommu = 1; 182 if (!strncmp(p, "noforce", 7)) { 183 iommu_merge = 0; 184 force_iommu = 0; 185 } 186 187 if (!strncmp(p, "biomerge", 8)) { 188 iommu_merge = 1; 189 force_iommu = 1; 190 } 191 if (!strncmp(p, "panic", 5)) 192 panic_on_overflow = 1; 193 if (!strncmp(p, "nopanic", 7)) 194 panic_on_overflow = 0; 195 if (!strncmp(p, "merge", 5)) { 196 iommu_merge = 1; 197 force_iommu = 1; 198 } 199 if (!strncmp(p, "nomerge", 7)) 200 iommu_merge = 0; 201 if (!strncmp(p, "forcesac", 8)) 202 iommu_sac_force = 1; 203 if (!strncmp(p, "allowdac", 8)) 204 forbid_dac = 0; 205 if (!strncmp(p, "nodac", 5)) 206 forbid_dac = -1; 207 if (!strncmp(p, "usedac", 6)) { 208 forbid_dac = -1; 209 return 1; 210 } 211 #ifdef CONFIG_SWIOTLB 212 if (!strncmp(p, "soft", 4)) 213 swiotlb = 1; 214 #endif 215 if (!strncmp(p, "pt", 2)) { 216 iommu_pass_through = 1; 217 return 1; 218 } 219 220 gart_parse_options(p); 221 222 #ifdef CONFIG_CALGARY_IOMMU 223 if (!strncmp(p, "calgary", 7)) 224 use_calgary = 1; 225 #endif /* CONFIG_CALGARY_IOMMU */ 226 227 p += strcspn(p, ","); 228 if (*p == ',') 229 ++p; 230 } 231 return 0; 232 } 233 early_param("iommu", iommu_setup); 234 235 int dma_supported(struct device *dev, u64 mask) 236 { 237 struct dma_map_ops *ops = get_dma_ops(dev); 238 239 #ifdef CONFIG_PCI 240 if (mask > 0xffffffff && forbid_dac > 0) { 241 dev_info(dev, "PCI: Disallowing DAC for device\n"); 242 return 0; 243 } 244 #endif 245 246 if (ops->dma_supported) 247 return ops->dma_supported(dev, mask); 248 249 /* Copied from i386. Doesn't make much sense, because it will 250 only work for pci_alloc_coherent. 251 The caller just has to use GFP_DMA in this case. */ 252 if (mask < DMA_BIT_MASK(24)) 253 return 0; 254 255 /* Tell the device to use SAC when IOMMU force is on. This 256 allows the driver to use cheaper accesses in some cases. 257 258 Problem with this is that if we overflow the IOMMU area and 259 return DAC as fallback address the device may not handle it 260 correctly. 261 262 As a special case some controllers have a 39bit address 263 mode that is as efficient as 32bit (aic79xx). Don't force 264 SAC for these. Assume all masks <= 40 bits are of this 265 type. Normally this doesn't make any difference, but gives 266 more gentle handling of IOMMU overflow. */ 267 if (iommu_sac_force && (mask >= DMA_BIT_MASK(40))) { 268 dev_info(dev, "Force SAC with mask %Lx\n", mask); 269 return 0; 270 } 271 272 return 1; 273 } 274 EXPORT_SYMBOL(dma_supported); 275 276 static int __init pci_iommu_init(void) 277 { 278 dma_debug_init(PREALLOC_DMA_DEBUG_ENTRIES); 279 280 #ifdef CONFIG_PCI 281 dma_debug_add_bus(&pci_bus_type); 282 #endif 283 284 calgary_iommu_init(); 285 286 intel_iommu_init(); 287 288 amd_iommu_init(); 289 290 gart_iommu_init(); 291 292 no_iommu_init(); 293 return 0; 294 } 295 296 void pci_iommu_shutdown(void) 297 { 298 gart_iommu_shutdown(); 299 300 amd_iommu_shutdown(); 301 } 302 /* Must execute after PCI subsystem */ 303 fs_initcall(pci_iommu_init); 304 305 #ifdef CONFIG_PCI 306 /* Many VIA bridges seem to corrupt data for DAC. Disable it here */ 307 308 static __devinit void via_no_dac(struct pci_dev *dev) 309 { 310 if ((dev->class >> 8) == PCI_CLASS_BRIDGE_PCI && forbid_dac == 0) { 311 dev_info(&dev->dev, "disabling DAC on VIA PCI bridge\n"); 312 forbid_dac = 1; 313 } 314 } 315 DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_VIA, PCI_ANY_ID, via_no_dac); 316 #endif 317