1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Implementation of the IOMMU SVA API for the ARM SMMUv3 4 */ 5 6 #include <linux/mm.h> 7 #include <linux/mmu_context.h> 8 #include <linux/mmu_notifier.h> 9 #include <linux/sched/mm.h> 10 #include <linux/slab.h> 11 #include <kunit/visibility.h> 12 13 #include "arm-smmu-v3.h" 14 #include "../../io-pgtable-arm.h" 15 16 static void __maybe_unused 17 arm_smmu_update_s1_domain_cd_entry(struct arm_smmu_domain *smmu_domain) 18 { 19 struct arm_smmu_master_domain *master_domain; 20 struct arm_smmu_cd target_cd; 21 unsigned long flags; 22 23 spin_lock_irqsave(&smmu_domain->devices_lock, flags); 24 list_for_each_entry(master_domain, &smmu_domain->devices, devices_elm) { 25 struct arm_smmu_master *master = master_domain->master; 26 struct arm_smmu_cd *cdptr; 27 28 cdptr = arm_smmu_get_cd_ptr(master, master_domain->ssid); 29 if (WARN_ON(!cdptr)) 30 continue; 31 32 arm_smmu_make_s1_cd(&target_cd, master, smmu_domain); 33 arm_smmu_write_cd_entry(master, master_domain->ssid, cdptr, 34 &target_cd); 35 } 36 spin_unlock_irqrestore(&smmu_domain->devices_lock, flags); 37 } 38 39 static u64 page_size_to_cd(void) 40 { 41 static_assert(PAGE_SIZE == SZ_4K || PAGE_SIZE == SZ_16K || 42 PAGE_SIZE == SZ_64K); 43 if (PAGE_SIZE == SZ_64K) 44 return ARM_LPAE_TCR_TG0_64K; 45 if (PAGE_SIZE == SZ_16K) 46 return ARM_LPAE_TCR_TG0_16K; 47 return ARM_LPAE_TCR_TG0_4K; 48 } 49 50 VISIBLE_IF_KUNIT 51 void arm_smmu_make_sva_cd(struct arm_smmu_cd *target, 52 struct arm_smmu_master *master, struct mm_struct *mm, 53 u16 asid) 54 { 55 u64 par; 56 57 memset(target, 0, sizeof(*target)); 58 59 par = cpuid_feature_extract_unsigned_field( 60 read_sanitised_ftr_reg(SYS_ID_AA64MMFR0_EL1), 61 ID_AA64MMFR0_EL1_PARANGE_SHIFT); 62 63 target->data[0] = cpu_to_le64( 64 CTXDESC_CD_0_TCR_EPD1 | 65 #ifdef __BIG_ENDIAN 66 CTXDESC_CD_0_ENDI | 67 #endif 68 CTXDESC_CD_0_V | 69 FIELD_PREP(CTXDESC_CD_0_TCR_IPS, par) | 70 CTXDESC_CD_0_AA64 | 71 (master->stall_enabled ? CTXDESC_CD_0_S : 0) | 72 CTXDESC_CD_0_R | 73 CTXDESC_CD_0_A | 74 CTXDESC_CD_0_ASET | 75 FIELD_PREP(CTXDESC_CD_0_ASID, asid)); 76 77 /* 78 * If no MM is passed then this creates a SVA entry that faults 79 * everything. arm_smmu_write_cd_entry() can hitlessly go between these 80 * two entries types since TTB0 is ignored by HW when EPD0 is set. 81 */ 82 if (mm) { 83 target->data[0] |= cpu_to_le64( 84 FIELD_PREP(CTXDESC_CD_0_TCR_T0SZ, 85 64ULL - vabits_actual) | 86 FIELD_PREP(CTXDESC_CD_0_TCR_TG0, page_size_to_cd()) | 87 FIELD_PREP(CTXDESC_CD_0_TCR_IRGN0, 88 ARM_LPAE_TCR_RGN_WBWA) | 89 FIELD_PREP(CTXDESC_CD_0_TCR_ORGN0, 90 ARM_LPAE_TCR_RGN_WBWA) | 91 FIELD_PREP(CTXDESC_CD_0_TCR_SH0, ARM_LPAE_TCR_SH_IS)); 92 93 target->data[1] = cpu_to_le64(virt_to_phys(mm->pgd) & 94 CTXDESC_CD_1_TTB0_MASK); 95 } else { 96 target->data[0] |= cpu_to_le64(CTXDESC_CD_0_TCR_EPD0); 97 98 /* 99 * Disable stall and immediately generate an abort if stall 100 * disable is permitted. This speeds up cleanup for an unclean 101 * exit if the device is still doing a lot of DMA. 102 */ 103 if (!(master->smmu->features & ARM_SMMU_FEAT_STALL_FORCE)) 104 target->data[0] &= 105 cpu_to_le64(~(CTXDESC_CD_0_S | CTXDESC_CD_0_R)); 106 } 107 108 /* 109 * MAIR value is pretty much constant and global, so we can just get it 110 * from the current CPU register 111 */ 112 target->data[3] = cpu_to_le64(read_sysreg(mair_el1)); 113 114 /* 115 * Note that we don't bother with S1PIE on the SMMU, we just rely on 116 * our default encoding scheme matching direct permissions anyway. 117 * SMMU has no notion of S1POE nor GCS, so make sure that is clear if 118 * either is enabled for CPUs, just in case anyone imagines otherwise. 119 */ 120 if (system_supports_poe() || system_supports_gcs()) 121 dev_warn_once(master->smmu->dev, "SVA devices ignore permission overlays and GCS\n"); 122 } 123 EXPORT_SYMBOL_IF_KUNIT(arm_smmu_make_sva_cd); 124 125 /* 126 * Cloned from the MAX_TLBI_OPS in arch/arm64/include/asm/tlbflush.h, this 127 * is used as a threshold to replace per-page TLBI commands to issue in the 128 * command queue with an address-space TLBI command, when SMMU w/o a range 129 * invalidation feature handles too many per-page TLBI commands, which will 130 * otherwise result in a soft lockup. 131 */ 132 #define CMDQ_MAX_TLBI_OPS (1 << (PAGE_SHIFT - 3)) 133 134 static void arm_smmu_mm_arch_invalidate_secondary_tlbs(struct mmu_notifier *mn, 135 struct mm_struct *mm, 136 unsigned long start, 137 unsigned long end) 138 { 139 struct arm_smmu_domain *smmu_domain = 140 container_of(mn, struct arm_smmu_domain, mmu_notifier); 141 size_t size; 142 143 /* 144 * The mm_types defines vm_end as the first byte after the end address, 145 * different from IOMMU subsystem using the last address of an address 146 * range. So do a simple translation here by calculating size correctly. 147 */ 148 size = end - start; 149 if (!(smmu_domain->smmu->features & ARM_SMMU_FEAT_RANGE_INV)) { 150 if (size >= CMDQ_MAX_TLBI_OPS * PAGE_SIZE) 151 size = 0; 152 } else { 153 if (size == ULONG_MAX) 154 size = 0; 155 } 156 157 if (!size) 158 arm_smmu_tlb_inv_asid(smmu_domain->smmu, smmu_domain->cd.asid); 159 else 160 arm_smmu_tlb_inv_range_asid(start, size, smmu_domain->cd.asid, 161 PAGE_SIZE, false, smmu_domain); 162 163 arm_smmu_atc_inv_domain(smmu_domain, start, size); 164 } 165 166 static void arm_smmu_mm_release(struct mmu_notifier *mn, struct mm_struct *mm) 167 { 168 struct arm_smmu_domain *smmu_domain = 169 container_of(mn, struct arm_smmu_domain, mmu_notifier); 170 struct arm_smmu_master_domain *master_domain; 171 unsigned long flags; 172 173 /* 174 * DMA may still be running. Keep the cd valid to avoid C_BAD_CD events, 175 * but disable translation. 176 */ 177 spin_lock_irqsave(&smmu_domain->devices_lock, flags); 178 list_for_each_entry(master_domain, &smmu_domain->devices, 179 devices_elm) { 180 struct arm_smmu_master *master = master_domain->master; 181 struct arm_smmu_cd target; 182 struct arm_smmu_cd *cdptr; 183 184 cdptr = arm_smmu_get_cd_ptr(master, master_domain->ssid); 185 if (WARN_ON(!cdptr)) 186 continue; 187 arm_smmu_make_sva_cd(&target, master, NULL, 188 smmu_domain->cd.asid); 189 arm_smmu_write_cd_entry(master, master_domain->ssid, cdptr, 190 &target); 191 } 192 spin_unlock_irqrestore(&smmu_domain->devices_lock, flags); 193 194 arm_smmu_tlb_inv_asid(smmu_domain->smmu, smmu_domain->cd.asid); 195 arm_smmu_atc_inv_domain(smmu_domain, 0, 0); 196 } 197 198 static void arm_smmu_mmu_notifier_free(struct mmu_notifier *mn) 199 { 200 kfree(container_of(mn, struct arm_smmu_domain, mmu_notifier)); 201 } 202 203 static const struct mmu_notifier_ops arm_smmu_mmu_notifier_ops = { 204 .arch_invalidate_secondary_tlbs = arm_smmu_mm_arch_invalidate_secondary_tlbs, 205 .release = arm_smmu_mm_release, 206 .free_notifier = arm_smmu_mmu_notifier_free, 207 }; 208 209 bool arm_smmu_sva_supported(struct arm_smmu_device *smmu) 210 { 211 unsigned long reg, fld; 212 unsigned long oas; 213 unsigned long asid_bits; 214 u32 feat_mask = ARM_SMMU_FEAT_COHERENCY; 215 216 if (vabits_actual == 52) { 217 /* We don't support LPA2 */ 218 if (PAGE_SIZE != SZ_64K) 219 return false; 220 feat_mask |= ARM_SMMU_FEAT_VAX; 221 } 222 223 if (system_supports_bbml2_noabort()) 224 feat_mask |= ARM_SMMU_FEAT_BBML2; 225 226 if ((smmu->features & feat_mask) != feat_mask) 227 return false; 228 229 if (!(smmu->pgsize_bitmap & PAGE_SIZE)) 230 return false; 231 232 /* 233 * Get the smallest PA size of all CPUs (sanitized by cpufeature). We're 234 * not even pretending to support AArch32 here. Abort if the MMU outputs 235 * addresses larger than what we support. 236 */ 237 reg = read_sanitised_ftr_reg(SYS_ID_AA64MMFR0_EL1); 238 fld = cpuid_feature_extract_unsigned_field(reg, ID_AA64MMFR0_EL1_PARANGE_SHIFT); 239 oas = id_aa64mmfr0_parange_to_phys_shift(fld); 240 if (smmu->oas < oas) 241 return false; 242 243 /* We can support bigger ASIDs than the CPU, but not smaller */ 244 fld = cpuid_feature_extract_unsigned_field(reg, ID_AA64MMFR0_EL1_ASIDBITS_SHIFT); 245 asid_bits = fld ? 16 : 8; 246 if (smmu->asid_bits < asid_bits) 247 return false; 248 249 /* 250 * See max_pinned_asids in arch/arm64/mm/context.c. The following is 251 * generally the maximum number of bindable processes. 252 */ 253 if (arm64_kernel_unmapped_at_el0()) 254 asid_bits--; 255 dev_dbg(smmu->dev, "%d shared contexts\n", (1 << asid_bits) - 256 num_possible_cpus() - 2); 257 258 return true; 259 } 260 261 void arm_smmu_sva_notifier_synchronize(void) 262 { 263 /* 264 * Some MMU notifiers may still be waiting to be freed, using 265 * arm_smmu_mmu_notifier_free(). Wait for them. 266 */ 267 mmu_notifier_synchronize(); 268 } 269 270 static int arm_smmu_sva_set_dev_pasid(struct iommu_domain *domain, 271 struct device *dev, ioasid_t id, 272 struct iommu_domain *old) 273 { 274 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); 275 struct arm_smmu_master *master = dev_iommu_priv_get(dev); 276 struct arm_smmu_cd target; 277 int ret; 278 279 if (!(master->smmu->features & ARM_SMMU_FEAT_SVA)) 280 return -EOPNOTSUPP; 281 282 /* Prevent arm_smmu_mm_release from being called while we are attaching */ 283 if (!mmget_not_zero(domain->mm)) 284 return -EINVAL; 285 286 /* 287 * This does not need the arm_smmu_asid_lock because SVA domains never 288 * get reassigned 289 */ 290 arm_smmu_make_sva_cd(&target, master, domain->mm, smmu_domain->cd.asid); 291 ret = arm_smmu_set_pasid(master, smmu_domain, id, &target, old); 292 293 mmput(domain->mm); 294 return ret; 295 } 296 297 static void arm_smmu_sva_domain_free(struct iommu_domain *domain) 298 { 299 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain); 300 301 /* 302 * Ensure the ASID is empty in the iommu cache before allowing reuse. 303 */ 304 arm_smmu_tlb_inv_asid(smmu_domain->smmu, smmu_domain->cd.asid); 305 306 /* 307 * Notice that the arm_smmu_mm_arch_invalidate_secondary_tlbs op can 308 * still be called/running at this point. We allow the ASID to be 309 * reused, and if there is a race then it just suffers harmless 310 * unnecessary invalidation. 311 */ 312 xa_erase(&arm_smmu_asid_xa, smmu_domain->cd.asid); 313 314 /* 315 * Actual free is defered to the SRCU callback 316 * arm_smmu_mmu_notifier_free() 317 */ 318 mmu_notifier_put(&smmu_domain->mmu_notifier); 319 } 320 321 static const struct iommu_domain_ops arm_smmu_sva_domain_ops = { 322 .set_dev_pasid = arm_smmu_sva_set_dev_pasid, 323 .free = arm_smmu_sva_domain_free 324 }; 325 326 struct iommu_domain *arm_smmu_sva_domain_alloc(struct device *dev, 327 struct mm_struct *mm) 328 { 329 struct arm_smmu_master *master = dev_iommu_priv_get(dev); 330 struct arm_smmu_device *smmu = master->smmu; 331 struct arm_smmu_domain *smmu_domain; 332 u32 asid; 333 int ret; 334 335 if (!(master->smmu->features & ARM_SMMU_FEAT_SVA)) 336 return ERR_PTR(-EOPNOTSUPP); 337 338 smmu_domain = arm_smmu_domain_alloc(); 339 if (IS_ERR(smmu_domain)) 340 return ERR_CAST(smmu_domain); 341 smmu_domain->domain.type = IOMMU_DOMAIN_SVA; 342 smmu_domain->domain.ops = &arm_smmu_sva_domain_ops; 343 344 /* 345 * Choose page_size as the leaf page size for invalidation when 346 * ARM_SMMU_FEAT_RANGE_INV is present 347 */ 348 smmu_domain->domain.pgsize_bitmap = PAGE_SIZE; 349 smmu_domain->smmu = smmu; 350 351 ret = xa_alloc(&arm_smmu_asid_xa, &asid, smmu_domain, 352 XA_LIMIT(1, (1 << smmu->asid_bits) - 1), GFP_KERNEL); 353 if (ret) 354 goto err_free; 355 356 smmu_domain->cd.asid = asid; 357 smmu_domain->mmu_notifier.ops = &arm_smmu_mmu_notifier_ops; 358 ret = mmu_notifier_register(&smmu_domain->mmu_notifier, mm); 359 if (ret) 360 goto err_asid; 361 362 return &smmu_domain->domain; 363 364 err_asid: 365 xa_erase(&arm_smmu_asid_xa, smmu_domain->cd.asid); 366 err_free: 367 kfree(smmu_domain); 368 return ERR_PTR(ret); 369 } 370