1 // SPDX-License-Identifier: GPL-2.0 2 #include <linux/dma-map-ops.h> 3 #include <linux/dma-direct.h> 4 #include <linux/iommu.h> 5 #include <linux/dmar.h> 6 #include <linux/export.h> 7 #include <linux/memblock.h> 8 #include <linux/gfp.h> 9 #include <linux/pci.h> 10 #include <linux/amd-iommu.h> 11 12 #include <asm/proto.h> 13 #include <asm/dma.h> 14 #include <asm/iommu.h> 15 #include <asm/gart.h> 16 #include <asm/x86_init.h> 17 18 #include <xen/xen.h> 19 #include <xen/swiotlb-xen.h> 20 21 static bool disable_dac_quirk __read_mostly; 22 23 const struct dma_map_ops *dma_ops; 24 EXPORT_SYMBOL(dma_ops); 25 26 #ifdef CONFIG_IOMMU_DEBUG 27 int panic_on_overflow __read_mostly = 1; 28 int force_iommu __read_mostly = 1; 29 #else 30 int panic_on_overflow __read_mostly = 0; 31 int force_iommu __read_mostly = 0; 32 #endif 33 34 int iommu_merge __read_mostly = 0; 35 36 int no_iommu __read_mostly; 37 /* Set this to 1 if there is a HW IOMMU in the system */ 38 int iommu_detected __read_mostly = 0; 39 40 #ifdef CONFIG_SWIOTLB 41 bool x86_swiotlb_enable; 42 static unsigned int x86_swiotlb_flags; 43 44 static void __init pci_swiotlb_detect(void) 45 { 46 /* don't initialize swiotlb if iommu=off (no_iommu=1) */ 47 if (!no_iommu && max_possible_pfn > MAX_DMA32_PFN) 48 x86_swiotlb_enable = true; 49 50 /* 51 * Set swiotlb to 1 so that bounce buffers are allocated and used for 52 * devices that can't support DMA to encrypted memory. 53 */ 54 if (cc_platform_has(CC_ATTR_HOST_MEM_ENCRYPT)) 55 x86_swiotlb_enable = true; 56 57 /* 58 * Guest with guest memory encryption currently perform all DMA through 59 * bounce buffers as the hypervisor can't access arbitrary VM memory 60 * that is not explicitly shared with it. 61 */ 62 if (cc_platform_has(CC_ATTR_GUEST_MEM_ENCRYPT)) 63 x86_swiotlb_enable = true; 64 } 65 #else 66 static inline void __init pci_swiotlb_detect(void) 67 { 68 } 69 #define x86_swiotlb_flags 0 70 #endif /* CONFIG_SWIOTLB */ 71 72 #ifdef CONFIG_SWIOTLB_XEN 73 static bool xen_swiotlb_enabled(void) 74 { 75 return xen_initial_domain() || x86_swiotlb_enable || 76 (IS_ENABLED(CONFIG_XEN_PCIDEV_FRONTEND) && xen_pv_pci_possible); 77 } 78 79 static void __init pci_xen_swiotlb_init(void) 80 { 81 if (!xen_swiotlb_enabled()) 82 return; 83 x86_swiotlb_enable = true; 84 x86_swiotlb_flags |= SWIOTLB_ANY; 85 swiotlb_init_remap(true, x86_swiotlb_flags, xen_swiotlb_fixup); 86 dma_ops = &xen_swiotlb_dma_ops; 87 if (IS_ENABLED(CONFIG_PCI)) 88 pci_request_acs(); 89 } 90 #else 91 static inline void __init pci_xen_swiotlb_init(void) 92 { 93 } 94 #endif /* CONFIG_SWIOTLB_XEN */ 95 96 void __init pci_iommu_alloc(void) 97 { 98 if (xen_pv_domain()) { 99 pci_xen_swiotlb_init(); 100 return; 101 } 102 pci_swiotlb_detect(); 103 gart_iommu_hole_init(); 104 amd_iommu_detect(); 105 detect_intel_iommu(); 106 swiotlb_init(x86_swiotlb_enable, x86_swiotlb_flags); 107 } 108 109 static __init int iommu_setup(char *p) 110 { 111 iommu_merge = 1; 112 113 if (!p) 114 return -EINVAL; 115 116 while (*p) { 117 if (!strncmp(p, "off", 3)) 118 no_iommu = 1; 119 /* gart_parse_options has more force support */ 120 if (!strncmp(p, "force", 5)) 121 force_iommu = 1; 122 if (!strncmp(p, "noforce", 7)) { 123 iommu_merge = 0; 124 force_iommu = 0; 125 } 126 127 if (!strncmp(p, "biomerge", 8)) { 128 iommu_merge = 1; 129 force_iommu = 1; 130 } 131 if (!strncmp(p, "panic", 5)) 132 panic_on_overflow = 1; 133 if (!strncmp(p, "nopanic", 7)) 134 panic_on_overflow = 0; 135 if (!strncmp(p, "merge", 5)) { 136 iommu_merge = 1; 137 force_iommu = 1; 138 } 139 if (!strncmp(p, "nomerge", 7)) 140 iommu_merge = 0; 141 if (!strncmp(p, "forcesac", 8)) 142 pr_warn("forcesac option ignored.\n"); 143 if (!strncmp(p, "allowdac", 8)) 144 pr_warn("allowdac option ignored.\n"); 145 if (!strncmp(p, "nodac", 5)) 146 pr_warn("nodac option ignored.\n"); 147 if (!strncmp(p, "usedac", 6)) { 148 disable_dac_quirk = true; 149 return 1; 150 } 151 #ifdef CONFIG_SWIOTLB 152 if (!strncmp(p, "soft", 4)) 153 x86_swiotlb_enable = true; 154 #endif 155 if (!strncmp(p, "pt", 2)) 156 iommu_set_default_passthrough(true); 157 if (!strncmp(p, "nopt", 4)) 158 iommu_set_default_translated(true); 159 160 gart_parse_options(p); 161 162 p += strcspn(p, ","); 163 if (*p == ',') 164 ++p; 165 } 166 return 0; 167 } 168 early_param("iommu", iommu_setup); 169 170 static int __init pci_iommu_init(void) 171 { 172 x86_init.iommu.iommu_init(); 173 174 #ifdef CONFIG_SWIOTLB 175 /* An IOMMU turned us off. */ 176 if (x86_swiotlb_enable) { 177 pr_info("PCI-DMA: Using software bounce buffering for IO (SWIOTLB)\n"); 178 swiotlb_print_info(); 179 } else { 180 swiotlb_exit(); 181 } 182 #endif 183 184 return 0; 185 } 186 /* Must execute after PCI subsystem */ 187 rootfs_initcall(pci_iommu_init); 188 189 #ifdef CONFIG_PCI 190 /* Many VIA bridges seem to corrupt data for DAC. Disable it here */ 191 192 static int via_no_dac_cb(struct pci_dev *pdev, void *data) 193 { 194 pdev->dev.bus_dma_limit = DMA_BIT_MASK(32); 195 return 0; 196 } 197 198 static void via_no_dac(struct pci_dev *dev) 199 { 200 if (!disable_dac_quirk) { 201 dev_info(&dev->dev, "disabling DAC on VIA PCI bridge\n"); 202 pci_walk_bus(dev->subordinate, via_no_dac_cb, NULL); 203 } 204 } 205 DECLARE_PCI_FIXUP_CLASS_FINAL(PCI_VENDOR_ID_VIA, PCI_ANY_ID, 206 PCI_CLASS_BRIDGE_PCI, 8, via_no_dac); 207 #endif 208