1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Implementation of the IOMMU SVA API for the ARM SMMUv3 4 */ 5 6 #include <linux/mm.h> 7 #include <linux/mmu_context.h> 8 #include <linux/mmu_notifier.h> 9 #include <linux/sched/mm.h> 10 #include <linux/slab.h> 11 #include <kunit/visibility.h> 12 13 #include "arm-smmu-v3.h" 14 #include "../../io-pgtable-arm.h" 15 16 static void __maybe_unused 17 arm_smmu_update_s1_domain_cd_entry(struct arm_smmu_domain *smmu_domain) 18 { 19 struct arm_smmu_master_domain *master_domain; 20 struct arm_smmu_cd target_cd; 21 unsigned long flags; 22 23 spin_lock_irqsave(&smmu_domain->devices_lock, flags); 24 list_for_each_entry(master_domain, &smmu_domain->devices, devices_elm) { 25 struct arm_smmu_master *master = master_domain->master; 26 struct arm_smmu_cd *cdptr; 27 28 cdptr = arm_smmu_get_cd_ptr(master, master_domain->ssid); 29 if (WARN_ON(!cdptr)) 30 continue; 31 32 arm_smmu_make_s1_cd(&target_cd, master, smmu_domain); 33 arm_smmu_write_cd_entry(master, master_domain->ssid, cdptr, 34 &target_cd); 35 } 36 spin_unlock_irqrestore(&smmu_domain->devices_lock, flags); 37 } 38 39 static u64 page_size_to_cd(void) 40 { 41 static_assert(PAGE_SIZE == SZ_4K || PAGE_SIZE == SZ_16K || 42 PAGE_SIZE == SZ_64K); 43 if (PAGE_SIZE == SZ_64K) 44 return ARM_LPAE_TCR_TG0_64K; 45 if (PAGE_SIZE == SZ_16K) 46 return ARM_LPAE_TCR_TG0_16K; 47 return ARM_LPAE_TCR_TG0_4K; 48 } 49 50 VISIBLE_IF_KUNIT 51 void arm_smmu_make_sva_cd(struct arm_smmu_cd *target, 52 struct arm_smmu_master *master, struct mm_struct *mm, 53 u16 asid) 54 { 55 u64 par; 56 57 memset(target, 0, sizeof(*target)); 58 59 par = cpuid_feature_extract_unsigned_field( 60 read_sanitised_ftr_reg(SYS_ID_AA64MMFR0_EL1), 61 ID_AA64MMFR0_EL1_PARANGE_SHIFT); 62 63 target->data[0] = cpu_to_le64( 64 CTXDESC_CD_0_TCR_EPD1 | 65 #ifdef __BIG_ENDIAN 66 CTXDESC_CD_0_ENDI | 67 #endif 68 CTXDESC_CD_0_V | 69 FIELD_PREP(CTXDESC_CD_0_TCR_IPS, par) | 70 CTXDESC_CD_0_AA64 | 71 (master->stall_enabled ? CTXDESC_CD_0_S : 0) | 72 CTXDESC_CD_0_R | 73 CTXDESC_CD_0_A | 74 CTXDESC_CD_0_ASET | 75 FIELD_PREP(CTXDESC_CD_0_ASID, asid)); 76 77 /* 78 * If no MM is passed then this creates a SVA entry that faults 79 * everything. arm_smmu_write_cd_entry() can hitlessly go between these 80 * two entries types since TTB0 is ignored by HW when EPD0 is set. 81 */ 82 if (mm) { 83 target->data[0] |= cpu_to_le64( 84 FIELD_PREP(CTXDESC_CD_0_TCR_T0SZ, 85 64ULL - vabits_actual) | 86 FIELD_PREP(CTXDESC_CD_0_TCR_TG0, page_size_to_cd()) | 87 FIELD_PREP(CTXDESC_CD_0_TCR_IRGN0, 88 ARM_LPAE_TCR_RGN_WBWA) | 89 FIELD_PREP(CTXDESC_CD_0_TCR_ORGN0, 90 ARM_LPAE_TCR_RGN_WBWA) | 91 FIELD_PREP(CTXDESC_CD_0_TCR_SH0, ARM_LPAE_TCR_SH_IS)); 92 93 target->data[1] = cpu_to_le64(virt_to_phys(mm->pgd) & 94 CTXDESC_CD_1_TTB0_MASK); 95 } else { 96 target->data[0] |= cpu_to_le64(CTXDESC_CD_0_TCR_EPD0); 97 98 /* 99 * Disable stall and immediately generate an abort if stall 100 * disable is permitted. This speeds up cleanup for an unclean 101 * exit if the device is still doing a lot of DMA. 102 */ 103 if (!(master->smmu->features & ARM_SMMU_FEAT_STALL_FORCE)) 104 target->data[0] &= 105 cpu_to_le64(~(CTXDESC_CD_0_S | CTXDESC_CD_0_R)); 106 } 107 108 /* 109 * MAIR value is pretty much constant and global, so we can just get it 110 * from the current CPU register 111 */ 112 target->data[3] = cpu_to_le64(read_sysreg(mair_el1)); 113 114 /* 115 * Note that we don't bother with S1PIE on the SMMU, we just rely on 116 * our default encoding scheme matching direct permissions anyway. 117 * SMMU has no notion of S1POE nor GCS, so make sure that is clear if 118 * either is enabled for CPUs, just in case anyone imagines otherwise. 119 */ 120 if (system_supports_poe() || system_supports_gcs()) 121 dev_warn_once(master->smmu->dev, "SVA devices ignore permission overlays and GCS\n"); 122 } 123 EXPORT_SYMBOL_IF_KUNIT(arm_smmu_make_sva_cd); 124 125 /* 126 * Cloned from the MAX_TLBI_OPS in arch/arm64/include/asm/tlbflush.h, this 127 * is used as a threshold to replace per-page TLBI commands to issue in the 128 * command queue with an address-space TLBI command, when SMMU w/o a range 129 * invalidation feature handles too many per-page TLBI commands, which will 130 * otherwise result in a soft lockup. 131 */ 132 #define CMDQ_MAX_TLBI_OPS (1 << (PAGE_SHIFT - 3)) 133 134 static void arm_smmu_mm_arch_invalidate_secondary_tlbs(struct mmu_notifier *mn, 135 struct mm_struct *mm, 136 unsigned long start, 137 unsigned long end) 138 { 139 struct arm_smmu_domain *smmu_domain = 140 container_of(mn, struct arm_smmu_domain, mmu_notifier); 141 size_t size; 142 143 /* 144 * The mm_types defines vm_end as the first byte after the end address, 145 * different from IOMMU subsystem using the last address of an address 146 * range. So do a simple translation here by calculating size correctly. 147 */ 148 size = end - start; 149 if (!(smmu_domain->smmu->features & ARM_SMMU_FEAT_RANGE_INV)) { 150 if (size >= CMDQ_MAX_TLBI_OPS * PAGE_SIZE) 151 size = 0; 152 } else { 153 if (size == ULONG_MAX) 154 size = 0; 155 } 156 157 if (!size) 158 arm_smmu_tlb_inv_asid(smmu_domain->smmu, smmu_domain->cd.asid); 159 else 160 arm_smmu_tlb_inv_range_asid(start, size, smmu_domain->cd.asid, 161 PAGE_SIZE, false, smmu_domain); 162 163 arm_smmu_atc_inv_domain(smmu_domain, start, size); 164 } 165 166 static void arm_smmu_mm_release(struct mmu_notifier *mn, struct mm_struct *mm) 167 { 168 struct arm_smmu_domain *smmu_domain = 169 container_of(mn, struct arm_smmu_domain, mmu_notifier); 170 struct arm_smmu_master_domain *master_domain; 171 unsigned long flags; 172 173 /* 174 * DMA may still be running. Keep the cd valid to avoid C_BAD_CD events, 175 * but disable translation. 176 */ 177 spin_lock_irqsave(&smmu_domain->devices_lock, flags); 178 list_for_each_entry(master_domain, &smmu_domain->devices, 179 devices_elm) { 180 struct arm_smmu_master *master = master_domain->master; 181 struct arm_smmu_cd target; 182 struct arm_smmu_cd *cdptr; 183 184 cdptr = arm_smmu_get_cd_ptr(master, master_domain->ssid); 185 if (WARN_ON(!cdptr)) 186 continue; 187 arm_smmu_make_sva_cd(&target, master, NULL, 188 smmu_domain->cd.asid); 189 arm_smmu_write_cd_entry(master, master_domain->ssid, cdptr, 190 &target); 191 } 192 spin_unlock_irqrestore(&smmu_domain->devices_lock, flags); 193 194 arm_smmu_tlb_inv_asid(smmu_domain->smmu, smmu_domain->cd.asid); 195 arm_smmu_atc_inv_domain(smmu_domain, 0, 0); 196 } 197 198 static void arm_smmu_mmu_notifier_free(struct mmu_notifier *mn) 199 { 200 kfree(container_of(mn, struct arm_smmu_domain, mmu_notifier)); 201 } 202 203 static const struct mmu_notifier_ops arm_smmu_mmu_notifier_ops = { 204 .arch_invalidate_secondary_tlbs = arm_smmu_mm_arch_invalidate_secondary_tlbs, 205 .release = arm_smmu_mm_release, 206 .free_notifier = arm_smmu_mmu_notifier_free, 207 }; 208 209 bool arm_smmu_sva_supported(struct arm_smmu_device *smmu) 210 { 211 unsigned long reg, fld; 212 unsigned long oas; 213 unsigned long asid_bits; 214 u32 feat_mask = ARM_SMMU_FEAT_COHERENCY; 215 216 if (vabits_actual == 52) { 217 /* We don't support LPA2 */ 218 if (PAGE_SIZE != SZ_64K) 219 return false; 220 feat_mask |= ARM_SMMU_FEAT_VAX; 221 } 222 223 if ((smmu->features & feat_mask) != feat_mask) 224 return false; 225 226 if (!(smmu->pgsize_bitmap & PAGE_SIZE)) 227 return false; 228 229 /* 230 * Get the smallest PA size of all CPUs (sanitized by cpufeature). We're 231 * not even pretending to support AArch32 here. Abort if the MMU outputs 232 * addresses larger than what we support. 233 */ 234 reg = read_sanitised_ftr_reg(SYS_ID_AA64MMFR0_EL1); 235 fld = cpuid_feature_extract_unsigned_field(reg, ID_AA64MMFR0_EL1_PARANGE_SHIFT); 236 oas = id_aa64mmfr0_parange_to_phys_shift(fld); 237 if (smmu->oas < oas) 238 return false; 239 240 /* We can support bigger ASIDs than the CPU, but not smaller */ 241 fld = cpuid_feature_extract_unsigned_field(reg, ID_AA64MMFR0_EL1_ASIDBITS_SHIFT); 242 asid_bits = fld ? 16 : 8; 243 if (smmu->asid_bits < asid_bits) 244 return false; 245 246 /* 247 * See max_pinned_asids in arch/arm64/mm/context.c. The following is 248 * generally the maximum number of bindable processes. 249 */ 250 if (arm64_kernel_unmapped_at_el0()) 251 asid_bits--; 252 dev_dbg(smmu->dev, "%d shared contexts\n", (1 << asid_bits) - 253 num_possible_cpus() - 2); 254 255 return true; 256 } 257 258 void arm_smmu_sva_notifier_synchronize(void) 259 { 260 /* 261 * Some MMU notifiers may still be waiting to be freed, using 262 * arm_smmu_mmu_notifier_free(). Wait for them. 263 */ 264 mmu_notifier_synchronize(); 265 } 266 267 static int arm_smmu_sva_set_dev_pasid(struct iommu_domain *domain, 268 struct device *dev, ioasid_t id, 269 struct iommu_domain *old) 270 { 271 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); 272 struct arm_smmu_master *master = dev_iommu_priv_get(dev); 273 struct arm_smmu_cd target; 274 int ret; 275 276 if (!(master->smmu->features & ARM_SMMU_FEAT_SVA)) 277 return -EOPNOTSUPP; 278 279 /* Prevent arm_smmu_mm_release from being called while we are attaching */ 280 if (!mmget_not_zero(domain->mm)) 281 return -EINVAL; 282 283 /* 284 * This does not need the arm_smmu_asid_lock because SVA domains never 285 * get reassigned 286 */ 287 arm_smmu_make_sva_cd(&target, master, domain->mm, smmu_domain->cd.asid); 288 ret = arm_smmu_set_pasid(master, smmu_domain, id, &target, old); 289 290 mmput(domain->mm); 291 return ret; 292 } 293 294 static void arm_smmu_sva_domain_free(struct iommu_domain *domain) 295 { 296 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); 297 298 /* 299 * Ensure the ASID is empty in the iommu cache before allowing reuse. 300 */ 301 arm_smmu_tlb_inv_asid(smmu_domain->smmu, smmu_domain->cd.asid); 302 303 /* 304 * Notice that the arm_smmu_mm_arch_invalidate_secondary_tlbs op can 305 * still be called/running at this point. We allow the ASID to be 306 * reused, and if there is a race then it just suffers harmless 307 * unnecessary invalidation. 308 */ 309 xa_erase(&arm_smmu_asid_xa, smmu_domain->cd.asid); 310 311 /* 312 * Actual free is defered to the SRCU callback 313 * arm_smmu_mmu_notifier_free() 314 */ 315 mmu_notifier_put(&smmu_domain->mmu_notifier); 316 } 317 318 static const struct iommu_domain_ops arm_smmu_sva_domain_ops = { 319 .set_dev_pasid = arm_smmu_sva_set_dev_pasid, 320 .free = arm_smmu_sva_domain_free 321 }; 322 323 struct iommu_domain *arm_smmu_sva_domain_alloc(struct device *dev, 324 struct mm_struct *mm) 325 { 326 struct arm_smmu_master *master = dev_iommu_priv_get(dev); 327 struct arm_smmu_device *smmu = master->smmu; 328 struct arm_smmu_domain *smmu_domain; 329 u32 asid; 330 int ret; 331 332 if (!(master->smmu->features & ARM_SMMU_FEAT_SVA)) 333 return ERR_PTR(-EOPNOTSUPP); 334 335 smmu_domain = arm_smmu_domain_alloc(); 336 if (IS_ERR(smmu_domain)) 337 return ERR_CAST(smmu_domain); 338 smmu_domain->domain.type = IOMMU_DOMAIN_SVA; 339 smmu_domain->domain.ops = &arm_smmu_sva_domain_ops; 340 341 /* 342 * Choose page_size as the leaf page size for invalidation when 343 * ARM_SMMU_FEAT_RANGE_INV is present 344 */ 345 smmu_domain->domain.pgsize_bitmap = PAGE_SIZE; 346 smmu_domain->smmu = smmu; 347 348 ret = xa_alloc(&arm_smmu_asid_xa, &asid, smmu_domain, 349 XA_LIMIT(1, (1 << smmu->asid_bits) - 1), GFP_KERNEL); 350 if (ret) 351 goto err_free; 352 353 smmu_domain->cd.asid = asid; 354 smmu_domain->mmu_notifier.ops = &arm_smmu_mmu_notifier_ops; 355 ret = mmu_notifier_register(&smmu_domain->mmu_notifier, mm); 356 if (ret) 357 goto err_asid; 358 359 return &smmu_domain->domain; 360 361 err_asid: 362 xa_erase(&arm_smmu_asid_xa, smmu_domain->cd.asid); 363 err_free: 364 kfree(smmu_domain); 365 return ERR_PTR(ret); 366 } 367