1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * Implementation of the IOMMU SVA API for the ARM SMMUv3
4 */
5
6 #include <linux/mm.h>
7 #include <linux/mmu_context.h>
8 #include <linux/mmu_notifier.h>
9 #include <linux/sched/mm.h>
10 #include <linux/slab.h>
11 #include <kunit/visibility.h>
12
13 #include "arm-smmu-v3.h"
14 #include "../../io-pgtable-arm.h"
15
16 static void __maybe_unused
arm_smmu_update_s1_domain_cd_entry(struct arm_smmu_domain * smmu_domain)17 arm_smmu_update_s1_domain_cd_entry(struct arm_smmu_domain *smmu_domain)
18 {
19 struct arm_smmu_master_domain *master_domain;
20 struct arm_smmu_cd target_cd;
21 unsigned long flags;
22
23 spin_lock_irqsave(&smmu_domain->devices_lock, flags);
24 list_for_each_entry(master_domain, &smmu_domain->devices, devices_elm) {
25 struct arm_smmu_master *master = master_domain->master;
26 struct arm_smmu_cd *cdptr;
27
28 cdptr = arm_smmu_get_cd_ptr(master, master_domain->ssid);
29 if (WARN_ON(!cdptr))
30 continue;
31
32 arm_smmu_make_s1_cd(&target_cd, master, smmu_domain);
33 arm_smmu_write_cd_entry(master, master_domain->ssid, cdptr,
34 &target_cd);
35 }
36 spin_unlock_irqrestore(&smmu_domain->devices_lock, flags);
37 }
38
page_size_to_cd(void)39 static u64 page_size_to_cd(void)
40 {
41 static_assert(PAGE_SIZE == SZ_4K || PAGE_SIZE == SZ_16K ||
42 PAGE_SIZE == SZ_64K);
43 if (PAGE_SIZE == SZ_64K)
44 return ARM_LPAE_TCR_TG0_64K;
45 if (PAGE_SIZE == SZ_16K)
46 return ARM_LPAE_TCR_TG0_16K;
47 return ARM_LPAE_TCR_TG0_4K;
48 }
49
50 VISIBLE_IF_KUNIT
arm_smmu_make_sva_cd(struct arm_smmu_cd * target,struct arm_smmu_master * master,struct mm_struct * mm,u16 asid)51 void arm_smmu_make_sva_cd(struct arm_smmu_cd *target,
52 struct arm_smmu_master *master, struct mm_struct *mm,
53 u16 asid)
54 {
55 u64 par;
56
57 memset(target, 0, sizeof(*target));
58
59 par = cpuid_feature_extract_unsigned_field(
60 read_sanitised_ftr_reg(SYS_ID_AA64MMFR0_EL1),
61 ID_AA64MMFR0_EL1_PARANGE_SHIFT);
62
63 target->data[0] = cpu_to_le64(
64 CTXDESC_CD_0_TCR_EPD1 |
65 #ifdef __BIG_ENDIAN
66 CTXDESC_CD_0_ENDI |
67 #endif
68 CTXDESC_CD_0_V |
69 FIELD_PREP(CTXDESC_CD_0_TCR_IPS, par) |
70 CTXDESC_CD_0_AA64 |
71 (master->stall_enabled ? CTXDESC_CD_0_S : 0) |
72 CTXDESC_CD_0_R |
73 CTXDESC_CD_0_A |
74 CTXDESC_CD_0_ASET |
75 FIELD_PREP(CTXDESC_CD_0_ASID, asid));
76
77 /*
78 * If no MM is passed then this creates a SVA entry that faults
79 * everything. arm_smmu_write_cd_entry() can hitlessly go between these
80 * two entries types since TTB0 is ignored by HW when EPD0 is set.
81 */
82 if (mm) {
83 target->data[0] |= cpu_to_le64(
84 FIELD_PREP(CTXDESC_CD_0_TCR_T0SZ,
85 64ULL - vabits_actual) |
86 FIELD_PREP(CTXDESC_CD_0_TCR_TG0, page_size_to_cd()) |
87 FIELD_PREP(CTXDESC_CD_0_TCR_IRGN0,
88 ARM_LPAE_TCR_RGN_WBWA) |
89 FIELD_PREP(CTXDESC_CD_0_TCR_ORGN0,
90 ARM_LPAE_TCR_RGN_WBWA) |
91 FIELD_PREP(CTXDESC_CD_0_TCR_SH0, ARM_LPAE_TCR_SH_IS));
92
93 target->data[1] = cpu_to_le64(virt_to_phys(mm->pgd) &
94 CTXDESC_CD_1_TTB0_MASK);
95 } else {
96 target->data[0] |= cpu_to_le64(CTXDESC_CD_0_TCR_EPD0);
97
98 /*
99 * Disable stall and immediately generate an abort if stall
100 * disable is permitted. This speeds up cleanup for an unclean
101 * exit if the device is still doing a lot of DMA.
102 */
103 if (!(master->smmu->features & ARM_SMMU_FEAT_STALL_FORCE))
104 target->data[0] &=
105 cpu_to_le64(~(CTXDESC_CD_0_S | CTXDESC_CD_0_R));
106 }
107
108 /*
109 * MAIR value is pretty much constant and global, so we can just get it
110 * from the current CPU register
111 */
112 target->data[3] = cpu_to_le64(read_sysreg(mair_el1));
113
114 /*
115 * Note that we don't bother with S1PIE on the SMMU, we just rely on
116 * our default encoding scheme matching direct permissions anyway.
117 * SMMU has no notion of S1POE nor GCS, so make sure that is clear if
118 * either is enabled for CPUs, just in case anyone imagines otherwise.
119 */
120 if (system_supports_poe() || system_supports_gcs())
121 dev_warn_once(master->smmu->dev, "SVA devices ignore permission overlays and GCS\n");
122 }
123 EXPORT_SYMBOL_IF_KUNIT(arm_smmu_make_sva_cd);
124
125 /*
126 * Cloned from the MAX_TLBI_OPS in arch/arm64/include/asm/tlbflush.h, this
127 * is used as a threshold to replace per-page TLBI commands to issue in the
128 * command queue with an address-space TLBI command, when SMMU w/o a range
129 * invalidation feature handles too many per-page TLBI commands, which will
130 * otherwise result in a soft lockup.
131 */
132 #define CMDQ_MAX_TLBI_OPS (1 << (PAGE_SHIFT - 3))
133
arm_smmu_mm_arch_invalidate_secondary_tlbs(struct mmu_notifier * mn,struct mm_struct * mm,unsigned long start,unsigned long end)134 static void arm_smmu_mm_arch_invalidate_secondary_tlbs(struct mmu_notifier *mn,
135 struct mm_struct *mm,
136 unsigned long start,
137 unsigned long end)
138 {
139 struct arm_smmu_domain *smmu_domain =
140 container_of(mn, struct arm_smmu_domain, mmu_notifier);
141 size_t size;
142
143 /*
144 * The mm_types defines vm_end as the first byte after the end address,
145 * different from IOMMU subsystem using the last address of an address
146 * range. So do a simple translation here by calculating size correctly.
147 */
148 size = end - start;
149 if (!(smmu_domain->smmu->features & ARM_SMMU_FEAT_RANGE_INV)) {
150 if (size >= CMDQ_MAX_TLBI_OPS * PAGE_SIZE)
151 size = 0;
152 } else {
153 if (size == ULONG_MAX)
154 size = 0;
155 }
156
157 if (!size)
158 arm_smmu_tlb_inv_asid(smmu_domain->smmu, smmu_domain->cd.asid);
159 else
160 arm_smmu_tlb_inv_range_asid(start, size, smmu_domain->cd.asid,
161 PAGE_SIZE, false, smmu_domain);
162
163 arm_smmu_atc_inv_domain(smmu_domain, start, size);
164 }
165
arm_smmu_mm_release(struct mmu_notifier * mn,struct mm_struct * mm)166 static void arm_smmu_mm_release(struct mmu_notifier *mn, struct mm_struct *mm)
167 {
168 struct arm_smmu_domain *smmu_domain =
169 container_of(mn, struct arm_smmu_domain, mmu_notifier);
170 struct arm_smmu_master_domain *master_domain;
171 unsigned long flags;
172
173 /*
174 * DMA may still be running. Keep the cd valid to avoid C_BAD_CD events,
175 * but disable translation.
176 */
177 spin_lock_irqsave(&smmu_domain->devices_lock, flags);
178 list_for_each_entry(master_domain, &smmu_domain->devices,
179 devices_elm) {
180 struct arm_smmu_master *master = master_domain->master;
181 struct arm_smmu_cd target;
182 struct arm_smmu_cd *cdptr;
183
184 cdptr = arm_smmu_get_cd_ptr(master, master_domain->ssid);
185 if (WARN_ON(!cdptr))
186 continue;
187 arm_smmu_make_sva_cd(&target, master, NULL,
188 smmu_domain->cd.asid);
189 arm_smmu_write_cd_entry(master, master_domain->ssid, cdptr,
190 &target);
191 }
192 spin_unlock_irqrestore(&smmu_domain->devices_lock, flags);
193
194 arm_smmu_tlb_inv_asid(smmu_domain->smmu, smmu_domain->cd.asid);
195 arm_smmu_atc_inv_domain(smmu_domain, 0, 0);
196 }
197
arm_smmu_mmu_notifier_free(struct mmu_notifier * mn)198 static void arm_smmu_mmu_notifier_free(struct mmu_notifier *mn)
199 {
200 kfree(container_of(mn, struct arm_smmu_domain, mmu_notifier));
201 }
202
203 static const struct mmu_notifier_ops arm_smmu_mmu_notifier_ops = {
204 .arch_invalidate_secondary_tlbs = arm_smmu_mm_arch_invalidate_secondary_tlbs,
205 .release = arm_smmu_mm_release,
206 .free_notifier = arm_smmu_mmu_notifier_free,
207 };
208
arm_smmu_sva_supported(struct arm_smmu_device * smmu)209 bool arm_smmu_sva_supported(struct arm_smmu_device *smmu)
210 {
211 unsigned long reg, fld;
212 unsigned long oas;
213 unsigned long asid_bits;
214 u32 feat_mask = ARM_SMMU_FEAT_COHERENCY;
215
216 if (vabits_actual == 52) {
217 /* We don't support LPA2 */
218 if (PAGE_SIZE != SZ_64K)
219 return false;
220 feat_mask |= ARM_SMMU_FEAT_VAX;
221 }
222
223 if (system_supports_bbml2_noabort())
224 feat_mask |= ARM_SMMU_FEAT_BBML2;
225
226 if ((smmu->features & feat_mask) != feat_mask)
227 return false;
228
229 if (!(smmu->pgsize_bitmap & PAGE_SIZE))
230 return false;
231
232 /*
233 * Get the smallest PA size of all CPUs (sanitized by cpufeature). We're
234 * not even pretending to support AArch32 here. Abort if the MMU outputs
235 * addresses larger than what we support.
236 */
237 reg = read_sanitised_ftr_reg(SYS_ID_AA64MMFR0_EL1);
238 fld = cpuid_feature_extract_unsigned_field(reg, ID_AA64MMFR0_EL1_PARANGE_SHIFT);
239 oas = id_aa64mmfr0_parange_to_phys_shift(fld);
240 if (smmu->oas < oas)
241 return false;
242
243 /* We can support bigger ASIDs than the CPU, but not smaller */
244 fld = cpuid_feature_extract_unsigned_field(reg, ID_AA64MMFR0_EL1_ASIDBITS_SHIFT);
245 asid_bits = fld ? 16 : 8;
246 if (smmu->asid_bits < asid_bits)
247 return false;
248
249 /*
250 * See max_pinned_asids in arch/arm64/mm/context.c. The following is
251 * generally the maximum number of bindable processes.
252 */
253 if (arm64_kernel_unmapped_at_el0())
254 asid_bits--;
255 dev_dbg(smmu->dev, "%d shared contexts\n", (1 << asid_bits) -
256 num_possible_cpus() - 2);
257
258 return true;
259 }
260
arm_smmu_sva_notifier_synchronize(void)261 void arm_smmu_sva_notifier_synchronize(void)
262 {
263 /*
264 * Some MMU notifiers may still be waiting to be freed, using
265 * arm_smmu_mmu_notifier_free(). Wait for them.
266 */
267 mmu_notifier_synchronize();
268 }
269
arm_smmu_sva_set_dev_pasid(struct iommu_domain * domain,struct device * dev,ioasid_t id,struct iommu_domain * old)270 static int arm_smmu_sva_set_dev_pasid(struct iommu_domain *domain,
271 struct device *dev, ioasid_t id,
272 struct iommu_domain *old)
273 {
274 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
275 struct arm_smmu_master *master = dev_iommu_priv_get(dev);
276 struct arm_smmu_cd target;
277 int ret;
278
279 if (!(master->smmu->features & ARM_SMMU_FEAT_SVA))
280 return -EOPNOTSUPP;
281
282 /* Prevent arm_smmu_mm_release from being called while we are attaching */
283 if (!mmget_not_zero(domain->mm))
284 return -EINVAL;
285
286 /*
287 * This does not need the arm_smmu_asid_lock because SVA domains never
288 * get reassigned
289 */
290 arm_smmu_make_sva_cd(&target, master, domain->mm, smmu_domain->cd.asid);
291 ret = arm_smmu_set_pasid(master, smmu_domain, id, &target, old);
292
293 mmput(domain->mm);
294 return ret;
295 }
296
arm_smmu_sva_domain_free(struct iommu_domain * domain)297 static void arm_smmu_sva_domain_free(struct iommu_domain *domain)
298 {
299 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
300
301 /*
302 * Ensure the ASID is empty in the iommu cache before allowing reuse.
303 */
304 arm_smmu_tlb_inv_asid(smmu_domain->smmu, smmu_domain->cd.asid);
305
306 /*
307 * Notice that the arm_smmu_mm_arch_invalidate_secondary_tlbs op can
308 * still be called/running at this point. We allow the ASID to be
309 * reused, and if there is a race then it just suffers harmless
310 * unnecessary invalidation.
311 */
312 xa_erase(&arm_smmu_asid_xa, smmu_domain->cd.asid);
313
314 /*
315 * Actual free is defered to the SRCU callback
316 * arm_smmu_mmu_notifier_free()
317 */
318 mmu_notifier_put(&smmu_domain->mmu_notifier);
319 }
320
321 static const struct iommu_domain_ops arm_smmu_sva_domain_ops = {
322 .set_dev_pasid = arm_smmu_sva_set_dev_pasid,
323 .free = arm_smmu_sva_domain_free
324 };
325
arm_smmu_sva_domain_alloc(struct device * dev,struct mm_struct * mm)326 struct iommu_domain *arm_smmu_sva_domain_alloc(struct device *dev,
327 struct mm_struct *mm)
328 {
329 struct arm_smmu_master *master = dev_iommu_priv_get(dev);
330 struct arm_smmu_device *smmu = master->smmu;
331 struct arm_smmu_domain *smmu_domain;
332 u32 asid;
333 int ret;
334
335 if (!(master->smmu->features & ARM_SMMU_FEAT_SVA))
336 return ERR_PTR(-EOPNOTSUPP);
337
338 smmu_domain = arm_smmu_domain_alloc();
339 if (IS_ERR(smmu_domain))
340 return ERR_CAST(smmu_domain);
341 smmu_domain->domain.type = IOMMU_DOMAIN_SVA;
342 smmu_domain->domain.ops = &arm_smmu_sva_domain_ops;
343
344 /*
345 * Choose page_size as the leaf page size for invalidation when
346 * ARM_SMMU_FEAT_RANGE_INV is present
347 */
348 smmu_domain->domain.pgsize_bitmap = PAGE_SIZE;
349 smmu_domain->smmu = smmu;
350
351 ret = xa_alloc(&arm_smmu_asid_xa, &asid, smmu_domain,
352 XA_LIMIT(1, (1 << smmu->asid_bits) - 1), GFP_KERNEL);
353 if (ret)
354 goto err_free;
355
356 smmu_domain->cd.asid = asid;
357 smmu_domain->mmu_notifier.ops = &arm_smmu_mmu_notifier_ops;
358 ret = mmu_notifier_register(&smmu_domain->mmu_notifier, mm);
359 if (ret)
360 goto err_asid;
361
362 return &smmu_domain->domain;
363
364 err_asid:
365 xa_erase(&arm_smmu_asid_xa, smmu_domain->cd.asid);
366 err_free:
367 kfree(smmu_domain);
368 return ERR_PTR(ret);
369 }
370