1 #include <linux/dma-mapping.h> 2 #include <linux/dma-debug.h> 3 #include <linux/dmar.h> 4 #include <linux/bootmem.h> 5 #include <linux/pci.h> 6 #include <linux/kmemleak.h> 7 8 #include <asm/proto.h> 9 #include <asm/dma.h> 10 #include <asm/iommu.h> 11 #include <asm/gart.h> 12 #include <asm/calgary.h> 13 #include <asm/amd_iommu.h> 14 15 static int forbid_dac __read_mostly; 16 17 struct dma_map_ops *dma_ops; 18 EXPORT_SYMBOL(dma_ops); 19 20 static int iommu_sac_force __read_mostly; 21 22 #ifdef CONFIG_IOMMU_DEBUG 23 int panic_on_overflow __read_mostly = 1; 24 int force_iommu __read_mostly = 1; 25 #else 26 int panic_on_overflow __read_mostly = 0; 27 int force_iommu __read_mostly = 0; 28 #endif 29 30 int iommu_merge __read_mostly = 0; 31 32 int no_iommu __read_mostly; 33 /* Set this to 1 if there is a HW IOMMU in the system */ 34 int iommu_detected __read_mostly = 0; 35 36 /* 37 * This variable becomes 1 if iommu=pt is passed on the kernel command line. 38 * If this variable is 1, IOMMU implementations do no DMA ranslation for 39 * devices and allow every device to access to whole physical memory. This is 40 * useful if a user want to use an IOMMU only for KVM device assignment to 41 * guests and not for driver dma translation. 42 */ 43 int iommu_pass_through __read_mostly; 44 45 dma_addr_t bad_dma_address __read_mostly = 0; 46 EXPORT_SYMBOL(bad_dma_address); 47 48 /* Dummy device used for NULL arguments (normally ISA). Better would 49 be probably a smaller DMA mask, but this is bug-to-bug compatible 50 to older i386. */ 51 struct device x86_dma_fallback_dev = { 52 .init_name = "fallback device", 53 .coherent_dma_mask = DMA_BIT_MASK(32), 54 .dma_mask = &x86_dma_fallback_dev.coherent_dma_mask, 55 }; 56 EXPORT_SYMBOL(x86_dma_fallback_dev); 57 58 /* Number of entries preallocated for DMA-API debugging */ 59 #define PREALLOC_DMA_DEBUG_ENTRIES 32768 60 61 int dma_set_mask(struct device *dev, u64 mask) 62 { 63 if (!dev->dma_mask || !dma_supported(dev, mask)) 64 return -EIO; 65 66 *dev->dma_mask = mask; 67 68 return 0; 69 } 70 EXPORT_SYMBOL(dma_set_mask); 71 72 #ifdef CONFIG_X86_64 73 static __initdata void *dma32_bootmem_ptr; 74 static unsigned long dma32_bootmem_size __initdata = (128ULL<<20); 75 76 static int __init parse_dma32_size_opt(char *p) 77 { 78 if (!p) 79 return -EINVAL; 80 dma32_bootmem_size = memparse(p, &p); 81 return 0; 82 } 83 early_param("dma32_size", parse_dma32_size_opt); 84 85 void __init dma32_reserve_bootmem(void) 86 { 87 unsigned long size, align; 88 if (max_pfn <= MAX_DMA32_PFN) 89 return; 90 91 /* 92 * check aperture_64.c allocate_aperture() for reason about 93 * using 512M as goal 94 */ 95 align = 64ULL<<20; 96 size = roundup(dma32_bootmem_size, align); 97 dma32_bootmem_ptr = __alloc_bootmem_nopanic(size, align, 98 512ULL<<20); 99 /* 100 * Kmemleak should not scan this block as it may not be mapped via the 101 * kernel direct mapping. 102 */ 103 kmemleak_ignore(dma32_bootmem_ptr); 104 if (dma32_bootmem_ptr) 105 dma32_bootmem_size = size; 106 else 107 dma32_bootmem_size = 0; 108 } 109 static void __init dma32_free_bootmem(void) 110 { 111 112 if (max_pfn <= MAX_DMA32_PFN) 113 return; 114 115 if (!dma32_bootmem_ptr) 116 return; 117 118 free_bootmem(__pa(dma32_bootmem_ptr), dma32_bootmem_size); 119 120 dma32_bootmem_ptr = NULL; 121 dma32_bootmem_size = 0; 122 } 123 #endif 124 125 void __init pci_iommu_alloc(void) 126 { 127 #ifdef CONFIG_X86_64 128 /* free the range so iommu could get some range less than 4G */ 129 dma32_free_bootmem(); 130 #endif 131 132 /* 133 * The order of these functions is important for 134 * fall-back/fail-over reasons 135 */ 136 gart_iommu_hole_init(); 137 138 detect_calgary(); 139 140 detect_intel_iommu(); 141 142 amd_iommu_detect(); 143 144 pci_swiotlb_init(); 145 } 146 147 void *dma_generic_alloc_coherent(struct device *dev, size_t size, 148 dma_addr_t *dma_addr, gfp_t flag) 149 { 150 unsigned long dma_mask; 151 struct page *page; 152 dma_addr_t addr; 153 154 dma_mask = dma_alloc_coherent_mask(dev, flag); 155 156 flag |= __GFP_ZERO; 157 again: 158 page = alloc_pages_node(dev_to_node(dev), flag, get_order(size)); 159 if (!page) 160 return NULL; 161 162 addr = page_to_phys(page); 163 if (addr + size > dma_mask) { 164 __free_pages(page, get_order(size)); 165 166 if (dma_mask < DMA_BIT_MASK(32) && !(flag & GFP_DMA)) { 167 flag = (flag & ~GFP_DMA32) | GFP_DMA; 168 goto again; 169 } 170 171 return NULL; 172 } 173 174 *dma_addr = addr; 175 return page_address(page); 176 } 177 178 /* 179 * See <Documentation/x86_64/boot-options.txt> for the iommu kernel parameter 180 * documentation. 181 */ 182 static __init int iommu_setup(char *p) 183 { 184 iommu_merge = 1; 185 186 if (!p) 187 return -EINVAL; 188 189 while (*p) { 190 if (!strncmp(p, "off", 3)) 191 no_iommu = 1; 192 /* gart_parse_options has more force support */ 193 if (!strncmp(p, "force", 5)) 194 force_iommu = 1; 195 if (!strncmp(p, "noforce", 7)) { 196 iommu_merge = 0; 197 force_iommu = 0; 198 } 199 200 if (!strncmp(p, "biomerge", 8)) { 201 iommu_merge = 1; 202 force_iommu = 1; 203 } 204 if (!strncmp(p, "panic", 5)) 205 panic_on_overflow = 1; 206 if (!strncmp(p, "nopanic", 7)) 207 panic_on_overflow = 0; 208 if (!strncmp(p, "merge", 5)) { 209 iommu_merge = 1; 210 force_iommu = 1; 211 } 212 if (!strncmp(p, "nomerge", 7)) 213 iommu_merge = 0; 214 if (!strncmp(p, "forcesac", 8)) 215 iommu_sac_force = 1; 216 if (!strncmp(p, "allowdac", 8)) 217 forbid_dac = 0; 218 if (!strncmp(p, "nodac", 5)) 219 forbid_dac = -1; 220 if (!strncmp(p, "usedac", 6)) { 221 forbid_dac = -1; 222 return 1; 223 } 224 #ifdef CONFIG_SWIOTLB 225 if (!strncmp(p, "soft", 4)) 226 swiotlb = 1; 227 #endif 228 if (!strncmp(p, "pt", 2)) 229 iommu_pass_through = 1; 230 231 gart_parse_options(p); 232 233 #ifdef CONFIG_CALGARY_IOMMU 234 if (!strncmp(p, "calgary", 7)) 235 use_calgary = 1; 236 #endif /* CONFIG_CALGARY_IOMMU */ 237 238 p += strcspn(p, ","); 239 if (*p == ',') 240 ++p; 241 } 242 return 0; 243 } 244 early_param("iommu", iommu_setup); 245 246 int dma_supported(struct device *dev, u64 mask) 247 { 248 struct dma_map_ops *ops = get_dma_ops(dev); 249 250 #ifdef CONFIG_PCI 251 if (mask > 0xffffffff && forbid_dac > 0) { 252 dev_info(dev, "PCI: Disallowing DAC for device\n"); 253 return 0; 254 } 255 #endif 256 257 if (ops->dma_supported) 258 return ops->dma_supported(dev, mask); 259 260 /* Copied from i386. Doesn't make much sense, because it will 261 only work for pci_alloc_coherent. 262 The caller just has to use GFP_DMA in this case. */ 263 if (mask < DMA_BIT_MASK(24)) 264 return 0; 265 266 /* Tell the device to use SAC when IOMMU force is on. This 267 allows the driver to use cheaper accesses in some cases. 268 269 Problem with this is that if we overflow the IOMMU area and 270 return DAC as fallback address the device may not handle it 271 correctly. 272 273 As a special case some controllers have a 39bit address 274 mode that is as efficient as 32bit (aic79xx). Don't force 275 SAC for these. Assume all masks <= 40 bits are of this 276 type. Normally this doesn't make any difference, but gives 277 more gentle handling of IOMMU overflow. */ 278 if (iommu_sac_force && (mask >= DMA_BIT_MASK(40))) { 279 dev_info(dev, "Force SAC with mask %Lx\n", mask); 280 return 0; 281 } 282 283 return 1; 284 } 285 EXPORT_SYMBOL(dma_supported); 286 287 static int __init pci_iommu_init(void) 288 { 289 dma_debug_init(PREALLOC_DMA_DEBUG_ENTRIES); 290 291 #ifdef CONFIG_PCI 292 dma_debug_add_bus(&pci_bus_type); 293 #endif 294 295 calgary_iommu_init(); 296 297 intel_iommu_init(); 298 299 amd_iommu_init(); 300 301 gart_iommu_init(); 302 303 no_iommu_init(); 304 return 0; 305 } 306 307 void pci_iommu_shutdown(void) 308 { 309 gart_iommu_shutdown(); 310 311 amd_iommu_shutdown(); 312 } 313 /* Must execute after PCI subsystem */ 314 fs_initcall(pci_iommu_init); 315 316 #ifdef CONFIG_PCI 317 /* Many VIA bridges seem to corrupt data for DAC. Disable it here */ 318 319 static __devinit void via_no_dac(struct pci_dev *dev) 320 { 321 if ((dev->class >> 8) == PCI_CLASS_BRIDGE_PCI && forbid_dac == 0) { 322 dev_info(&dev->dev, "disabling DAC on VIA PCI bridge\n"); 323 forbid_dac = 1; 324 } 325 } 326 DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_VIA, PCI_ANY_ID, via_no_dac); 327 #endif 328