1 #include <linux/dma-mapping.h> 2 #include <linux/dma-debug.h> 3 #include <linux/dmar.h> 4 #include <linux/bootmem.h> 5 #include <linux/gfp.h> 6 #include <linux/pci.h> 7 #include <linux/kmemleak.h> 8 9 #include <asm/proto.h> 10 #include <asm/dma.h> 11 #include <asm/iommu.h> 12 #include <asm/gart.h> 13 #include <asm/calgary.h> 14 #include <asm/amd_iommu.h> 15 #include <asm/x86_init.h> 16 17 static int forbid_dac __read_mostly; 18 19 struct dma_map_ops *dma_ops = &nommu_dma_ops; 20 EXPORT_SYMBOL(dma_ops); 21 22 static int iommu_sac_force __read_mostly; 23 24 #ifdef CONFIG_IOMMU_DEBUG 25 int panic_on_overflow __read_mostly = 1; 26 int force_iommu __read_mostly = 1; 27 #else 28 int panic_on_overflow __read_mostly = 0; 29 int force_iommu __read_mostly = 0; 30 #endif 31 32 int iommu_merge __read_mostly = 0; 33 34 int no_iommu __read_mostly; 35 /* Set this to 1 if there is a HW IOMMU in the system */ 36 int iommu_detected __read_mostly = 0; 37 38 /* 39 * This variable becomes 1 if iommu=pt is passed on the kernel command line. 40 * If this variable is 1, IOMMU implementations do no DMA translation for 41 * devices and allow every device to access to whole physical memory. This is 42 * useful if a user wants to use an IOMMU only for KVM device assignment to 43 * guests and not for driver dma translation. 44 */ 45 int iommu_pass_through __read_mostly; 46 47 /* Dummy device used for NULL arguments (normally ISA). */ 48 struct device x86_dma_fallback_dev = { 49 .init_name = "fallback device", 50 .coherent_dma_mask = ISA_DMA_BIT_MASK, 51 .dma_mask = &x86_dma_fallback_dev.coherent_dma_mask, 52 }; 53 EXPORT_SYMBOL(x86_dma_fallback_dev); 54 55 /* Number of entries preallocated for DMA-API debugging */ 56 #define PREALLOC_DMA_DEBUG_ENTRIES 32768 57 58 int dma_set_mask(struct device *dev, u64 mask) 59 { 60 if (!dev->dma_mask || !dma_supported(dev, mask)) 61 return -EIO; 62 63 *dev->dma_mask = mask; 64 65 return 0; 66 } 67 EXPORT_SYMBOL(dma_set_mask); 68 69 #if defined(CONFIG_X86_64) && !defined(CONFIG_NUMA) 70 static __initdata void *dma32_bootmem_ptr; 71 static unsigned long dma32_bootmem_size __initdata = (128ULL<<20); 72 73 static int __init parse_dma32_size_opt(char *p) 74 { 75 if (!p) 76 return -EINVAL; 77 dma32_bootmem_size = memparse(p, &p); 78 return 0; 79 } 80 early_param("dma32_size", parse_dma32_size_opt); 81 82 void __init dma32_reserve_bootmem(void) 83 { 84 unsigned long size, align; 85 if (max_pfn <= MAX_DMA32_PFN) 86 return; 87 88 /* 89 * check aperture_64.c allocate_aperture() for reason about 90 * using 512M as goal 91 */ 92 align = 64ULL<<20; 93 size = roundup(dma32_bootmem_size, align); 94 dma32_bootmem_ptr = __alloc_bootmem_nopanic(size, align, 95 512ULL<<20); 96 /* 97 * Kmemleak should not scan this block as it may not be mapped via the 98 * kernel direct mapping. 99 */ 100 kmemleak_ignore(dma32_bootmem_ptr); 101 if (dma32_bootmem_ptr) 102 dma32_bootmem_size = size; 103 else 104 dma32_bootmem_size = 0; 105 } 106 static void __init dma32_free_bootmem(void) 107 { 108 109 if (max_pfn <= MAX_DMA32_PFN) 110 return; 111 112 if (!dma32_bootmem_ptr) 113 return; 114 115 free_bootmem(__pa(dma32_bootmem_ptr), dma32_bootmem_size); 116 117 dma32_bootmem_ptr = NULL; 118 dma32_bootmem_size = 0; 119 } 120 #else 121 void __init dma32_reserve_bootmem(void) 122 { 123 } 124 static void __init dma32_free_bootmem(void) 125 { 126 } 127 128 #endif 129 130 void __init pci_iommu_alloc(void) 131 { 132 /* free the range so iommu could get some range less than 4G */ 133 dma32_free_bootmem(); 134 135 if (pci_swiotlb_detect()) 136 goto out; 137 138 gart_iommu_hole_init(); 139 140 detect_calgary(); 141 142 detect_intel_iommu(); 143 144 /* needs to be called after gart_iommu_hole_init */ 145 amd_iommu_detect(); 146 out: 147 pci_swiotlb_init(); 148 } 149 150 void *dma_generic_alloc_coherent(struct device *dev, size_t size, 151 dma_addr_t *dma_addr, gfp_t flag) 152 { 153 unsigned long dma_mask; 154 struct page *page; 155 dma_addr_t addr; 156 157 dma_mask = dma_alloc_coherent_mask(dev, flag); 158 159 flag |= __GFP_ZERO; 160 again: 161 page = alloc_pages_node(dev_to_node(dev), flag, get_order(size)); 162 if (!page) 163 return NULL; 164 165 addr = page_to_phys(page); 166 if (addr + size > dma_mask) { 167 __free_pages(page, get_order(size)); 168 169 if (dma_mask < DMA_BIT_MASK(32) && !(flag & GFP_DMA)) { 170 flag = (flag & ~GFP_DMA32) | GFP_DMA; 171 goto again; 172 } 173 174 return NULL; 175 } 176 177 *dma_addr = addr; 178 return page_address(page); 179 } 180 181 /* 182 * See <Documentation/x86_64/boot-options.txt> for the iommu kernel parameter 183 * documentation. 184 */ 185 static __init int iommu_setup(char *p) 186 { 187 iommu_merge = 1; 188 189 if (!p) 190 return -EINVAL; 191 192 while (*p) { 193 if (!strncmp(p, "off", 3)) 194 no_iommu = 1; 195 /* gart_parse_options has more force support */ 196 if (!strncmp(p, "force", 5)) 197 force_iommu = 1; 198 if (!strncmp(p, "noforce", 7)) { 199 iommu_merge = 0; 200 force_iommu = 0; 201 } 202 203 if (!strncmp(p, "biomerge", 8)) { 204 iommu_merge = 1; 205 force_iommu = 1; 206 } 207 if (!strncmp(p, "panic", 5)) 208 panic_on_overflow = 1; 209 if (!strncmp(p, "nopanic", 7)) 210 panic_on_overflow = 0; 211 if (!strncmp(p, "merge", 5)) { 212 iommu_merge = 1; 213 force_iommu = 1; 214 } 215 if (!strncmp(p, "nomerge", 7)) 216 iommu_merge = 0; 217 if (!strncmp(p, "forcesac", 8)) 218 iommu_sac_force = 1; 219 if (!strncmp(p, "allowdac", 8)) 220 forbid_dac = 0; 221 if (!strncmp(p, "nodac", 5)) 222 forbid_dac = 1; 223 if (!strncmp(p, "usedac", 6)) { 224 forbid_dac = -1; 225 return 1; 226 } 227 #ifdef CONFIG_SWIOTLB 228 if (!strncmp(p, "soft", 4)) 229 swiotlb = 1; 230 #endif 231 if (!strncmp(p, "pt", 2)) 232 iommu_pass_through = 1; 233 234 gart_parse_options(p); 235 236 #ifdef CONFIG_CALGARY_IOMMU 237 if (!strncmp(p, "calgary", 7)) 238 use_calgary = 1; 239 #endif /* CONFIG_CALGARY_IOMMU */ 240 241 p += strcspn(p, ","); 242 if (*p == ',') 243 ++p; 244 } 245 return 0; 246 } 247 early_param("iommu", iommu_setup); 248 249 int dma_supported(struct device *dev, u64 mask) 250 { 251 struct dma_map_ops *ops = get_dma_ops(dev); 252 253 #ifdef CONFIG_PCI 254 if (mask > 0xffffffff && forbid_dac > 0) { 255 dev_info(dev, "PCI: Disallowing DAC for device\n"); 256 return 0; 257 } 258 #endif 259 260 if (ops->dma_supported) 261 return ops->dma_supported(dev, mask); 262 263 /* Copied from i386. Doesn't make much sense, because it will 264 only work for pci_alloc_coherent. 265 The caller just has to use GFP_DMA in this case. */ 266 if (mask < DMA_BIT_MASK(24)) 267 return 0; 268 269 /* Tell the device to use SAC when IOMMU force is on. This 270 allows the driver to use cheaper accesses in some cases. 271 272 Problem with this is that if we overflow the IOMMU area and 273 return DAC as fallback address the device may not handle it 274 correctly. 275 276 As a special case some controllers have a 39bit address 277 mode that is as efficient as 32bit (aic79xx). Don't force 278 SAC for these. Assume all masks <= 40 bits are of this 279 type. Normally this doesn't make any difference, but gives 280 more gentle handling of IOMMU overflow. */ 281 if (iommu_sac_force && (mask >= DMA_BIT_MASK(40))) { 282 dev_info(dev, "Force SAC with mask %Lx\n", mask); 283 return 0; 284 } 285 286 return 1; 287 } 288 EXPORT_SYMBOL(dma_supported); 289 290 static int __init pci_iommu_init(void) 291 { 292 dma_debug_init(PREALLOC_DMA_DEBUG_ENTRIES); 293 294 #ifdef CONFIG_PCI 295 dma_debug_add_bus(&pci_bus_type); 296 #endif 297 x86_init.iommu.iommu_init(); 298 299 if (swiotlb) { 300 printk(KERN_INFO "PCI-DMA: " 301 "Using software bounce buffering for IO (SWIOTLB)\n"); 302 swiotlb_print_info(); 303 } else 304 swiotlb_free(); 305 306 return 0; 307 } 308 /* Must execute after PCI subsystem */ 309 rootfs_initcall(pci_iommu_init); 310 311 #ifdef CONFIG_PCI 312 /* Many VIA bridges seem to corrupt data for DAC. Disable it here */ 313 314 static __devinit void via_no_dac(struct pci_dev *dev) 315 { 316 if ((dev->class >> 8) == PCI_CLASS_BRIDGE_PCI && forbid_dac == 0) { 317 dev_info(&dev->dev, "disabling DAC on VIA PCI bridge\n"); 318 forbid_dac = 1; 319 } 320 } 321 DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_VIA, PCI_ANY_ID, via_no_dac); 322 #endif 323