1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Implementation of the IOMMU SVA API for the ARM SMMUv3 4 */ 5 6 #include <linux/mm.h> 7 #include <linux/mmu_context.h> 8 #include <linux/slab.h> 9 10 #include "arm-smmu-v3.h" 11 #include "../../io-pgtable-arm.h" 12 13 /* 14 * Check if the CPU ASID is available on the SMMU side. If a private context 15 * descriptor is using it, try to replace it. 16 */ 17 static struct arm_smmu_ctx_desc * 18 arm_smmu_share_asid(struct mm_struct *mm, u16 asid) 19 { 20 int ret; 21 u32 new_asid; 22 struct arm_smmu_ctx_desc *cd; 23 struct arm_smmu_device *smmu; 24 struct arm_smmu_domain *smmu_domain; 25 26 cd = xa_load(&arm_smmu_asid_xa, asid); 27 if (!cd) 28 return NULL; 29 30 if (cd->mm) { 31 if (WARN_ON(cd->mm != mm)) 32 return ERR_PTR(-EINVAL); 33 /* All devices bound to this mm use the same cd struct. */ 34 refcount_inc(&cd->refs); 35 return cd; 36 } 37 38 smmu_domain = container_of(cd, struct arm_smmu_domain, s1_cfg.cd); 39 smmu = smmu_domain->smmu; 40 41 ret = xa_alloc(&arm_smmu_asid_xa, &new_asid, cd, 42 XA_LIMIT(1, (1 << smmu->asid_bits) - 1), GFP_KERNEL); 43 if (ret) 44 return ERR_PTR(-ENOSPC); 45 /* 46 * Race with unmap: TLB invalidations will start targeting the new ASID, 47 * which isn't assigned yet. We'll do an invalidate-all on the old ASID 48 * later, so it doesn't matter. 49 */ 50 cd->asid = new_asid; 51 /* 52 * Update ASID and invalidate CD in all associated masters. There will 53 * be some overlap between use of both ASIDs, until we invalidate the 54 * TLB. 55 */ 56 arm_smmu_write_ctx_desc(smmu_domain, 0, cd); 57 58 /* Invalidate TLB entries previously associated with that context */ 59 arm_smmu_tlb_inv_asid(smmu, asid); 60 61 xa_erase(&arm_smmu_asid_xa, asid); 62 return NULL; 63 } 64 65 __maybe_unused 66 static struct arm_smmu_ctx_desc *arm_smmu_alloc_shared_cd(struct mm_struct *mm) 67 { 68 u16 asid; 69 int err = 0; 70 u64 tcr, par, reg; 71 struct arm_smmu_ctx_desc *cd; 72 struct arm_smmu_ctx_desc *ret = NULL; 73 74 asid = arm64_mm_context_get(mm); 75 if (!asid) 76 return ERR_PTR(-ESRCH); 77 78 cd = kzalloc(sizeof(*cd), GFP_KERNEL); 79 if (!cd) { 80 err = -ENOMEM; 81 goto out_put_context; 82 } 83 84 refcount_set(&cd->refs, 1); 85 86 mutex_lock(&arm_smmu_asid_lock); 87 ret = arm_smmu_share_asid(mm, asid); 88 if (ret) { 89 mutex_unlock(&arm_smmu_asid_lock); 90 goto out_free_cd; 91 } 92 93 err = xa_insert(&arm_smmu_asid_xa, asid, cd, GFP_KERNEL); 94 mutex_unlock(&arm_smmu_asid_lock); 95 96 if (err) 97 goto out_free_asid; 98 99 tcr = FIELD_PREP(CTXDESC_CD_0_TCR_T0SZ, 64ULL - vabits_actual) | 100 FIELD_PREP(CTXDESC_CD_0_TCR_IRGN0, ARM_LPAE_TCR_RGN_WBWA) | 101 FIELD_PREP(CTXDESC_CD_0_TCR_ORGN0, ARM_LPAE_TCR_RGN_WBWA) | 102 FIELD_PREP(CTXDESC_CD_0_TCR_SH0, ARM_LPAE_TCR_SH_IS) | 103 CTXDESC_CD_0_TCR_EPD1 | CTXDESC_CD_0_AA64; 104 105 switch (PAGE_SIZE) { 106 case SZ_4K: 107 tcr |= FIELD_PREP(CTXDESC_CD_0_TCR_TG0, ARM_LPAE_TCR_TG0_4K); 108 break; 109 case SZ_16K: 110 tcr |= FIELD_PREP(CTXDESC_CD_0_TCR_TG0, ARM_LPAE_TCR_TG0_16K); 111 break; 112 case SZ_64K: 113 tcr |= FIELD_PREP(CTXDESC_CD_0_TCR_TG0, ARM_LPAE_TCR_TG0_64K); 114 break; 115 default: 116 WARN_ON(1); 117 err = -EINVAL; 118 goto out_free_asid; 119 } 120 121 reg = read_sanitised_ftr_reg(SYS_ID_AA64MMFR0_EL1); 122 par = cpuid_feature_extract_unsigned_field(reg, ID_AA64MMFR0_PARANGE_SHIFT); 123 tcr |= FIELD_PREP(CTXDESC_CD_0_TCR_IPS, par); 124 125 cd->ttbr = virt_to_phys(mm->pgd); 126 cd->tcr = tcr; 127 /* 128 * MAIR value is pretty much constant and global, so we can just get it 129 * from the current CPU register 130 */ 131 cd->mair = read_sysreg(mair_el1); 132 cd->asid = asid; 133 cd->mm = mm; 134 135 return cd; 136 137 out_free_asid: 138 arm_smmu_free_asid(cd); 139 out_free_cd: 140 kfree(cd); 141 out_put_context: 142 arm64_mm_context_put(mm); 143 return err < 0 ? ERR_PTR(err) : ret; 144 } 145 146 __maybe_unused 147 static void arm_smmu_free_shared_cd(struct arm_smmu_ctx_desc *cd) 148 { 149 if (arm_smmu_free_asid(cd)) { 150 /* Unpin ASID */ 151 arm64_mm_context_put(cd->mm); 152 kfree(cd); 153 } 154 } 155