1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * IOMMU API for ARM architected SMMU implementations.
4 *
5 * Copyright (C) 2013 ARM Limited
6 *
7 * Author: Will Deacon <will.deacon@arm.com>
8 *
9 * This driver currently supports:
10 * - SMMUv1 and v2 implementations
11 * - Stream-matching and stream-indexing
12 * - v7/v8 long-descriptor format
13 * - Non-secure access to the SMMU
14 * - Context fault reporting
15 * - Extended Stream ID (16 bit)
16 */
17
18 #define pr_fmt(fmt) "arm-smmu: " fmt
19
20 #include <linux/acpi.h>
21 #include <linux/acpi_iort.h>
22 #include <linux/bitfield.h>
23 #include <linux/delay.h>
24 #include <linux/dma-mapping.h>
25 #include <linux/err.h>
26 #include <linux/interrupt.h>
27 #include <linux/io.h>
28 #include <linux/iopoll.h>
29 #include <linux/module.h>
30 #include <linux/of.h>
31 #include <linux/of_address.h>
32 #include <linux/pci.h>
33 #include <linux/platform_device.h>
34 #include <linux/pm_runtime.h>
35 #include <linux/ratelimit.h>
36 #include <linux/slab.h>
37 #include <linux/string_choices.h>
38
39 #include <linux/fsl/mc.h>
40
41 #include "arm-smmu.h"
42 #include "../../dma-iommu.h"
43
44 /*
45 * Apparently, some Qualcomm arm64 platforms which appear to expose their SMMU
46 * global register space are still, in fact, using a hypervisor to mediate it
47 * by trapping and emulating register accesses. Sadly, some deployed versions
48 * of said trapping code have bugs wherein they go horribly wrong for stores
49 * using r31 (i.e. XZR/WZR) as the source register.
50 */
51 #define QCOM_DUMMY_VAL -1
52
53 #define MSI_IOVA_BASE 0x8000000
54 #define MSI_IOVA_LENGTH 0x100000
55
56 static int force_stage;
57 module_param(force_stage, int, S_IRUGO);
58 MODULE_PARM_DESC(force_stage,
59 "Force SMMU mappings to be installed at a particular stage of translation. A value of '1' or '2' forces the corresponding stage. All other values are ignored (i.e. no stage is forced). Note that selecting a specific stage will disable support for nested translation.");
60 static bool disable_bypass =
61 IS_ENABLED(CONFIG_ARM_SMMU_DISABLE_BYPASS_BY_DEFAULT);
62 module_param(disable_bypass, bool, S_IRUGO);
63 MODULE_PARM_DESC(disable_bypass,
64 "Disable bypass streams such that incoming transactions from devices that are not attached to an iommu domain will report an abort back to the device and will not be allowed to pass through the SMMU.");
65
66 #define s2cr_init_val (struct arm_smmu_s2cr){ \
67 .type = disable_bypass ? S2CR_TYPE_FAULT : S2CR_TYPE_BYPASS, \
68 }
69
70 static bool using_legacy_binding, using_generic_binding;
71
arm_smmu_rpm_get(struct arm_smmu_device * smmu)72 static inline int arm_smmu_rpm_get(struct arm_smmu_device *smmu)
73 {
74 if (pm_runtime_enabled(smmu->dev))
75 return pm_runtime_resume_and_get(smmu->dev);
76
77 return 0;
78 }
79
arm_smmu_rpm_put(struct arm_smmu_device * smmu)80 static inline void arm_smmu_rpm_put(struct arm_smmu_device *smmu)
81 {
82 if (pm_runtime_enabled(smmu->dev)) {
83 pm_runtime_mark_last_busy(smmu->dev);
84 __pm_runtime_put_autosuspend(smmu->dev);
85
86 }
87 }
88
arm_smmu_rpm_use_autosuspend(struct arm_smmu_device * smmu)89 static void arm_smmu_rpm_use_autosuspend(struct arm_smmu_device *smmu)
90 {
91 /*
92 * Setup an autosuspend delay to avoid bouncing runpm state.
93 * Otherwise, if a driver for a suspended consumer device
94 * unmaps buffers, it will runpm resume/suspend for each one.
95 *
96 * For example, when used by a GPU device, when an application
97 * or game exits, it can trigger unmapping 100s or 1000s of
98 * buffers. With a runpm cycle for each buffer, that adds up
99 * to 5-10sec worth of reprogramming the context bank, while
100 * the system appears to be locked up to the user.
101 */
102 pm_runtime_set_autosuspend_delay(smmu->dev, 20);
103 pm_runtime_use_autosuspend(smmu->dev);
104 }
105
to_smmu_domain(struct iommu_domain * dom)106 static struct arm_smmu_domain *to_smmu_domain(struct iommu_domain *dom)
107 {
108 return container_of(dom, struct arm_smmu_domain, domain);
109 }
110
111 static struct platform_driver arm_smmu_driver;
112 static struct iommu_ops arm_smmu_ops;
113
114 #ifdef CONFIG_ARM_SMMU_LEGACY_DT_BINDINGS
dev_get_dev_node(struct device * dev)115 static struct device_node *dev_get_dev_node(struct device *dev)
116 {
117 if (dev_is_pci(dev)) {
118 struct pci_bus *bus = to_pci_dev(dev)->bus;
119
120 while (!pci_is_root_bus(bus))
121 bus = bus->parent;
122 return of_node_get(bus->bridge->parent->of_node);
123 }
124
125 return of_node_get(dev->of_node);
126 }
127
__arm_smmu_get_pci_sid(struct pci_dev * pdev,u16 alias,void * data)128 static int __arm_smmu_get_pci_sid(struct pci_dev *pdev, u16 alias, void *data)
129 {
130 *((__be32 *)data) = cpu_to_be32(alias);
131 return 0; /* Continue walking */
132 }
133
__find_legacy_master_phandle(struct device * dev,void * data)134 static int __find_legacy_master_phandle(struct device *dev, void *data)
135 {
136 struct of_phandle_iterator *it = *(void **)data;
137 struct device_node *np = it->node;
138 int err;
139
140 of_for_each_phandle(it, err, dev->of_node, "mmu-masters",
141 "#stream-id-cells", -1)
142 if (it->node == np) {
143 *(void **)data = dev;
144 return 1;
145 }
146 it->node = np;
147 return err == -ENOENT ? 0 : err;
148 }
149
arm_smmu_register_legacy_master(struct device * dev,struct arm_smmu_device ** smmu)150 static int arm_smmu_register_legacy_master(struct device *dev,
151 struct arm_smmu_device **smmu)
152 {
153 struct device *smmu_dev;
154 struct device_node *np;
155 struct of_phandle_iterator it;
156 void *data = ⁢
157 u32 *sids;
158 __be32 pci_sid;
159 int err;
160
161 np = dev_get_dev_node(dev);
162 if (!np || !of_property_present(np, "#stream-id-cells")) {
163 of_node_put(np);
164 return -ENODEV;
165 }
166
167 it.node = np;
168 err = driver_for_each_device(&arm_smmu_driver.driver, NULL, &data,
169 __find_legacy_master_phandle);
170 smmu_dev = data;
171 of_node_put(np);
172 if (err == 0)
173 return -ENODEV;
174 if (err < 0)
175 return err;
176
177 if (dev_is_pci(dev)) {
178 /* "mmu-masters" assumes Stream ID == Requester ID */
179 pci_for_each_dma_alias(to_pci_dev(dev), __arm_smmu_get_pci_sid,
180 &pci_sid);
181 it.cur = &pci_sid;
182 it.cur_count = 1;
183 }
184
185 err = iommu_fwspec_init(dev, NULL);
186 if (err)
187 return err;
188
189 sids = kcalloc(it.cur_count, sizeof(*sids), GFP_KERNEL);
190 if (!sids)
191 return -ENOMEM;
192
193 *smmu = dev_get_drvdata(smmu_dev);
194 of_phandle_iterator_args(&it, sids, it.cur_count);
195 err = iommu_fwspec_add_ids(dev, sids, it.cur_count);
196 kfree(sids);
197 return err;
198 }
199 #else
arm_smmu_register_legacy_master(struct device * dev,struct arm_smmu_device ** smmu)200 static int arm_smmu_register_legacy_master(struct device *dev,
201 struct arm_smmu_device **smmu)
202 {
203 return -ENODEV;
204 }
205 #endif /* CONFIG_ARM_SMMU_LEGACY_DT_BINDINGS */
206
__arm_smmu_free_bitmap(unsigned long * map,int idx)207 static void __arm_smmu_free_bitmap(unsigned long *map, int idx)
208 {
209 clear_bit(idx, map);
210 }
211
212 /* Wait for any pending TLB invalidations to complete */
__arm_smmu_tlb_sync(struct arm_smmu_device * smmu,int page,int sync,int status)213 static void __arm_smmu_tlb_sync(struct arm_smmu_device *smmu, int page,
214 int sync, int status)
215 {
216 unsigned int spin_cnt, delay;
217 u32 reg;
218
219 if (smmu->impl && unlikely(smmu->impl->tlb_sync))
220 return smmu->impl->tlb_sync(smmu, page, sync, status);
221
222 arm_smmu_writel(smmu, page, sync, QCOM_DUMMY_VAL);
223 for (delay = 1; delay < TLB_LOOP_TIMEOUT; delay *= 2) {
224 for (spin_cnt = TLB_SPIN_COUNT; spin_cnt > 0; spin_cnt--) {
225 reg = arm_smmu_readl(smmu, page, status);
226 if (!(reg & ARM_SMMU_sTLBGSTATUS_GSACTIVE))
227 return;
228 cpu_relax();
229 }
230 udelay(delay);
231 }
232 dev_err_ratelimited(smmu->dev,
233 "TLB sync timed out -- SMMU may be deadlocked\n");
234 }
235
arm_smmu_tlb_sync_global(struct arm_smmu_device * smmu)236 static void arm_smmu_tlb_sync_global(struct arm_smmu_device *smmu)
237 {
238 unsigned long flags;
239
240 spin_lock_irqsave(&smmu->global_sync_lock, flags);
241 __arm_smmu_tlb_sync(smmu, ARM_SMMU_GR0, ARM_SMMU_GR0_sTLBGSYNC,
242 ARM_SMMU_GR0_sTLBGSTATUS);
243 spin_unlock_irqrestore(&smmu->global_sync_lock, flags);
244 }
245
arm_smmu_tlb_sync_context(struct arm_smmu_domain * smmu_domain)246 static void arm_smmu_tlb_sync_context(struct arm_smmu_domain *smmu_domain)
247 {
248 struct arm_smmu_device *smmu = smmu_domain->smmu;
249 unsigned long flags;
250
251 spin_lock_irqsave(&smmu_domain->cb_lock, flags);
252 __arm_smmu_tlb_sync(smmu, ARM_SMMU_CB(smmu, smmu_domain->cfg.cbndx),
253 ARM_SMMU_CB_TLBSYNC, ARM_SMMU_CB_TLBSTATUS);
254 spin_unlock_irqrestore(&smmu_domain->cb_lock, flags);
255 }
256
arm_smmu_tlb_inv_context_s1(void * cookie)257 static void arm_smmu_tlb_inv_context_s1(void *cookie)
258 {
259 struct arm_smmu_domain *smmu_domain = cookie;
260 /*
261 * The TLBI write may be relaxed, so ensure that PTEs cleared by the
262 * current CPU are visible beforehand.
263 */
264 wmb();
265 arm_smmu_cb_write(smmu_domain->smmu, smmu_domain->cfg.cbndx,
266 ARM_SMMU_CB_S1_TLBIASID, smmu_domain->cfg.asid);
267 arm_smmu_tlb_sync_context(smmu_domain);
268 }
269
arm_smmu_tlb_inv_context_s2(void * cookie)270 static void arm_smmu_tlb_inv_context_s2(void *cookie)
271 {
272 struct arm_smmu_domain *smmu_domain = cookie;
273 struct arm_smmu_device *smmu = smmu_domain->smmu;
274
275 /* See above */
276 wmb();
277 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_TLBIVMID, smmu_domain->cfg.vmid);
278 arm_smmu_tlb_sync_global(smmu);
279 }
280
arm_smmu_tlb_inv_range_s1(unsigned long iova,size_t size,size_t granule,void * cookie,int reg)281 static void arm_smmu_tlb_inv_range_s1(unsigned long iova, size_t size,
282 size_t granule, void *cookie, int reg)
283 {
284 struct arm_smmu_domain *smmu_domain = cookie;
285 struct arm_smmu_device *smmu = smmu_domain->smmu;
286 struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
287 int idx = cfg->cbndx;
288
289 if (smmu->features & ARM_SMMU_FEAT_COHERENT_WALK)
290 wmb();
291
292 if (cfg->fmt != ARM_SMMU_CTX_FMT_AARCH64) {
293 iova = (iova >> 12) << 12;
294 iova |= cfg->asid;
295 do {
296 arm_smmu_cb_write(smmu, idx, reg, iova);
297 iova += granule;
298 } while (size -= granule);
299 } else {
300 iova >>= 12;
301 iova |= (u64)cfg->asid << 48;
302 do {
303 arm_smmu_cb_writeq(smmu, idx, reg, iova);
304 iova += granule >> 12;
305 } while (size -= granule);
306 }
307 }
308
arm_smmu_tlb_inv_range_s2(unsigned long iova,size_t size,size_t granule,void * cookie,int reg)309 static void arm_smmu_tlb_inv_range_s2(unsigned long iova, size_t size,
310 size_t granule, void *cookie, int reg)
311 {
312 struct arm_smmu_domain *smmu_domain = cookie;
313 struct arm_smmu_device *smmu = smmu_domain->smmu;
314 int idx = smmu_domain->cfg.cbndx;
315
316 if (smmu->features & ARM_SMMU_FEAT_COHERENT_WALK)
317 wmb();
318
319 iova >>= 12;
320 do {
321 if (smmu_domain->cfg.fmt == ARM_SMMU_CTX_FMT_AARCH64)
322 arm_smmu_cb_writeq(smmu, idx, reg, iova);
323 else
324 arm_smmu_cb_write(smmu, idx, reg, iova);
325 iova += granule >> 12;
326 } while (size -= granule);
327 }
328
arm_smmu_tlb_inv_walk_s1(unsigned long iova,size_t size,size_t granule,void * cookie)329 static void arm_smmu_tlb_inv_walk_s1(unsigned long iova, size_t size,
330 size_t granule, void *cookie)
331 {
332 struct arm_smmu_domain *smmu_domain = cookie;
333 struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
334
335 if (cfg->flush_walk_prefer_tlbiasid) {
336 arm_smmu_tlb_inv_context_s1(cookie);
337 } else {
338 arm_smmu_tlb_inv_range_s1(iova, size, granule, cookie,
339 ARM_SMMU_CB_S1_TLBIVA);
340 arm_smmu_tlb_sync_context(cookie);
341 }
342 }
343
arm_smmu_tlb_add_page_s1(struct iommu_iotlb_gather * gather,unsigned long iova,size_t granule,void * cookie)344 static void arm_smmu_tlb_add_page_s1(struct iommu_iotlb_gather *gather,
345 unsigned long iova, size_t granule,
346 void *cookie)
347 {
348 arm_smmu_tlb_inv_range_s1(iova, granule, granule, cookie,
349 ARM_SMMU_CB_S1_TLBIVAL);
350 }
351
arm_smmu_tlb_inv_walk_s2(unsigned long iova,size_t size,size_t granule,void * cookie)352 static void arm_smmu_tlb_inv_walk_s2(unsigned long iova, size_t size,
353 size_t granule, void *cookie)
354 {
355 arm_smmu_tlb_inv_range_s2(iova, size, granule, cookie,
356 ARM_SMMU_CB_S2_TLBIIPAS2);
357 arm_smmu_tlb_sync_context(cookie);
358 }
359
arm_smmu_tlb_add_page_s2(struct iommu_iotlb_gather * gather,unsigned long iova,size_t granule,void * cookie)360 static void arm_smmu_tlb_add_page_s2(struct iommu_iotlb_gather *gather,
361 unsigned long iova, size_t granule,
362 void *cookie)
363 {
364 arm_smmu_tlb_inv_range_s2(iova, granule, granule, cookie,
365 ARM_SMMU_CB_S2_TLBIIPAS2L);
366 }
367
arm_smmu_tlb_inv_walk_s2_v1(unsigned long iova,size_t size,size_t granule,void * cookie)368 static void arm_smmu_tlb_inv_walk_s2_v1(unsigned long iova, size_t size,
369 size_t granule, void *cookie)
370 {
371 arm_smmu_tlb_inv_context_s2(cookie);
372 }
373 /*
374 * On MMU-401 at least, the cost of firing off multiple TLBIVMIDs appears
375 * almost negligible, but the benefit of getting the first one in as far ahead
376 * of the sync as possible is significant, hence we don't just make this a
377 * no-op and call arm_smmu_tlb_inv_context_s2() from .iotlb_sync as you might
378 * think.
379 */
arm_smmu_tlb_add_page_s2_v1(struct iommu_iotlb_gather * gather,unsigned long iova,size_t granule,void * cookie)380 static void arm_smmu_tlb_add_page_s2_v1(struct iommu_iotlb_gather *gather,
381 unsigned long iova, size_t granule,
382 void *cookie)
383 {
384 struct arm_smmu_domain *smmu_domain = cookie;
385 struct arm_smmu_device *smmu = smmu_domain->smmu;
386
387 if (smmu->features & ARM_SMMU_FEAT_COHERENT_WALK)
388 wmb();
389
390 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_TLBIVMID, smmu_domain->cfg.vmid);
391 }
392
393 static const struct iommu_flush_ops arm_smmu_s1_tlb_ops = {
394 .tlb_flush_all = arm_smmu_tlb_inv_context_s1,
395 .tlb_flush_walk = arm_smmu_tlb_inv_walk_s1,
396 .tlb_add_page = arm_smmu_tlb_add_page_s1,
397 };
398
399 static const struct iommu_flush_ops arm_smmu_s2_tlb_ops_v2 = {
400 .tlb_flush_all = arm_smmu_tlb_inv_context_s2,
401 .tlb_flush_walk = arm_smmu_tlb_inv_walk_s2,
402 .tlb_add_page = arm_smmu_tlb_add_page_s2,
403 };
404
405 static const struct iommu_flush_ops arm_smmu_s2_tlb_ops_v1 = {
406 .tlb_flush_all = arm_smmu_tlb_inv_context_s2,
407 .tlb_flush_walk = arm_smmu_tlb_inv_walk_s2_v1,
408 .tlb_add_page = arm_smmu_tlb_add_page_s2_v1,
409 };
410
411
arm_smmu_read_context_fault_info(struct arm_smmu_device * smmu,int idx,struct arm_smmu_context_fault_info * cfi)412 void arm_smmu_read_context_fault_info(struct arm_smmu_device *smmu, int idx,
413 struct arm_smmu_context_fault_info *cfi)
414 {
415 cfi->iova = arm_smmu_cb_readq(smmu, idx, ARM_SMMU_CB_FAR);
416 cfi->fsr = arm_smmu_cb_read(smmu, idx, ARM_SMMU_CB_FSR);
417 cfi->fsynr = arm_smmu_cb_read(smmu, idx, ARM_SMMU_CB_FSYNR0);
418 cfi->cbfrsynra = arm_smmu_gr1_read(smmu, ARM_SMMU_GR1_CBFRSYNRA(idx));
419 }
420
arm_smmu_print_context_fault_info(struct arm_smmu_device * smmu,int idx,const struct arm_smmu_context_fault_info * cfi)421 void arm_smmu_print_context_fault_info(struct arm_smmu_device *smmu, int idx,
422 const struct arm_smmu_context_fault_info *cfi)
423 {
424 dev_err(smmu->dev,
425 "Unhandled context fault: fsr=0x%x, iova=0x%08lx, fsynr=0x%x, cbfrsynra=0x%x, cb=%d\n",
426 cfi->fsr, cfi->iova, cfi->fsynr, cfi->cbfrsynra, idx);
427
428 dev_err(smmu->dev, "FSR = %08x [%s%sFormat=%u%s%s%s%s%s%s%s%s], SID=0x%x\n",
429 cfi->fsr,
430 (cfi->fsr & ARM_SMMU_CB_FSR_MULTI) ? "MULTI " : "",
431 (cfi->fsr & ARM_SMMU_CB_FSR_SS) ? "SS " : "",
432 (u32)FIELD_GET(ARM_SMMU_CB_FSR_FORMAT, cfi->fsr),
433 (cfi->fsr & ARM_SMMU_CB_FSR_UUT) ? " UUT" : "",
434 (cfi->fsr & ARM_SMMU_CB_FSR_ASF) ? " ASF" : "",
435 (cfi->fsr & ARM_SMMU_CB_FSR_TLBLKF) ? " TLBLKF" : "",
436 (cfi->fsr & ARM_SMMU_CB_FSR_TLBMCF) ? " TLBMCF" : "",
437 (cfi->fsr & ARM_SMMU_CB_FSR_EF) ? " EF" : "",
438 (cfi->fsr & ARM_SMMU_CB_FSR_PF) ? " PF" : "",
439 (cfi->fsr & ARM_SMMU_CB_FSR_AFF) ? " AFF" : "",
440 (cfi->fsr & ARM_SMMU_CB_FSR_TF) ? " TF" : "",
441 cfi->cbfrsynra);
442
443 dev_err(smmu->dev, "FSYNR0 = %08x [S1CBNDX=%u%s%s%s%s%s%s PLVL=%u]\n",
444 cfi->fsynr,
445 (u32)FIELD_GET(ARM_SMMU_CB_FSYNR0_S1CBNDX, cfi->fsynr),
446 (cfi->fsynr & ARM_SMMU_CB_FSYNR0_AFR) ? " AFR" : "",
447 (cfi->fsynr & ARM_SMMU_CB_FSYNR0_PTWF) ? " PTWF" : "",
448 (cfi->fsynr & ARM_SMMU_CB_FSYNR0_NSATTR) ? " NSATTR" : "",
449 (cfi->fsynr & ARM_SMMU_CB_FSYNR0_IND) ? " IND" : "",
450 (cfi->fsynr & ARM_SMMU_CB_FSYNR0_PNU) ? " PNU" : "",
451 (cfi->fsynr & ARM_SMMU_CB_FSYNR0_WNR) ? " WNR" : "",
452 (u32)FIELD_GET(ARM_SMMU_CB_FSYNR0_PLVL, cfi->fsynr));
453 }
454
arm_smmu_context_fault(int irq,void * dev)455 static irqreturn_t arm_smmu_context_fault(int irq, void *dev)
456 {
457 struct arm_smmu_context_fault_info cfi;
458 struct arm_smmu_domain *smmu_domain = dev;
459 struct arm_smmu_device *smmu = smmu_domain->smmu;
460 static DEFINE_RATELIMIT_STATE(rs, DEFAULT_RATELIMIT_INTERVAL,
461 DEFAULT_RATELIMIT_BURST);
462 int idx = smmu_domain->cfg.cbndx;
463 int ret;
464
465 arm_smmu_read_context_fault_info(smmu, idx, &cfi);
466
467 if (!(cfi.fsr & ARM_SMMU_CB_FSR_FAULT))
468 return IRQ_NONE;
469
470 ret = report_iommu_fault(&smmu_domain->domain, NULL, cfi.iova,
471 cfi.fsynr & ARM_SMMU_CB_FSYNR0_WNR ? IOMMU_FAULT_WRITE : IOMMU_FAULT_READ);
472
473 if (ret == -ENOSYS && __ratelimit(&rs))
474 arm_smmu_print_context_fault_info(smmu, idx, &cfi);
475
476 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_FSR, cfi.fsr);
477 return IRQ_HANDLED;
478 }
479
arm_smmu_global_fault(int irq,void * dev)480 static irqreturn_t arm_smmu_global_fault(int irq, void *dev)
481 {
482 u32 gfsr, gfsynr0, gfsynr1, gfsynr2;
483 struct arm_smmu_device *smmu = dev;
484 static DEFINE_RATELIMIT_STATE(rs, DEFAULT_RATELIMIT_INTERVAL,
485 DEFAULT_RATELIMIT_BURST);
486
487 gfsr = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sGFSR);
488 gfsynr0 = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sGFSYNR0);
489 gfsynr1 = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sGFSYNR1);
490 gfsynr2 = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sGFSYNR2);
491
492 if (!gfsr)
493 return IRQ_NONE;
494
495 if (__ratelimit(&rs)) {
496 if (IS_ENABLED(CONFIG_ARM_SMMU_DISABLE_BYPASS_BY_DEFAULT) &&
497 (gfsr & ARM_SMMU_sGFSR_USF))
498 dev_err(smmu->dev,
499 "Blocked unknown Stream ID 0x%hx; boot with \"arm-smmu.disable_bypass=0\" to allow, but this may have security implications\n",
500 (u16)gfsynr1);
501 else
502 dev_err(smmu->dev,
503 "Unexpected global fault, this could be serious\n");
504 dev_err(smmu->dev,
505 "\tGFSR 0x%08x, GFSYNR0 0x%08x, GFSYNR1 0x%08x, GFSYNR2 0x%08x\n",
506 gfsr, gfsynr0, gfsynr1, gfsynr2);
507 }
508
509 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_sGFSR, gfsr);
510 return IRQ_HANDLED;
511 }
512
arm_smmu_init_context_bank(struct arm_smmu_domain * smmu_domain,struct io_pgtable_cfg * pgtbl_cfg)513 static void arm_smmu_init_context_bank(struct arm_smmu_domain *smmu_domain,
514 struct io_pgtable_cfg *pgtbl_cfg)
515 {
516 struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
517 struct arm_smmu_cb *cb = &smmu_domain->smmu->cbs[cfg->cbndx];
518 bool stage1 = cfg->cbar != CBAR_TYPE_S2_TRANS;
519
520 cb->cfg = cfg;
521
522 /* TCR */
523 if (stage1) {
524 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) {
525 cb->tcr[0] = pgtbl_cfg->arm_v7s_cfg.tcr;
526 } else {
527 cb->tcr[0] = arm_smmu_lpae_tcr(pgtbl_cfg);
528 cb->tcr[1] = arm_smmu_lpae_tcr2(pgtbl_cfg);
529 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64)
530 cb->tcr[1] |= ARM_SMMU_TCR2_AS;
531 else
532 cb->tcr[0] |= ARM_SMMU_TCR_EAE;
533 }
534 } else {
535 cb->tcr[0] = arm_smmu_lpae_vtcr(pgtbl_cfg);
536 }
537
538 /* TTBRs */
539 if (stage1) {
540 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) {
541 cb->ttbr[0] = pgtbl_cfg->arm_v7s_cfg.ttbr;
542 cb->ttbr[1] = 0;
543 } else {
544 cb->ttbr[0] = FIELD_PREP(ARM_SMMU_TTBRn_ASID,
545 cfg->asid);
546 cb->ttbr[1] = FIELD_PREP(ARM_SMMU_TTBRn_ASID,
547 cfg->asid);
548
549 if (pgtbl_cfg->quirks & IO_PGTABLE_QUIRK_ARM_TTBR1)
550 cb->ttbr[1] |= pgtbl_cfg->arm_lpae_s1_cfg.ttbr;
551 else
552 cb->ttbr[0] |= pgtbl_cfg->arm_lpae_s1_cfg.ttbr;
553 }
554 } else {
555 cb->ttbr[0] = pgtbl_cfg->arm_lpae_s2_cfg.vttbr;
556 }
557
558 /* MAIRs (stage-1 only) */
559 if (stage1) {
560 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) {
561 cb->mair[0] = pgtbl_cfg->arm_v7s_cfg.prrr;
562 cb->mair[1] = pgtbl_cfg->arm_v7s_cfg.nmrr;
563 } else {
564 cb->mair[0] = pgtbl_cfg->arm_lpae_s1_cfg.mair;
565 cb->mair[1] = pgtbl_cfg->arm_lpae_s1_cfg.mair >> 32;
566 }
567 }
568 }
569
arm_smmu_write_context_bank(struct arm_smmu_device * smmu,int idx)570 void arm_smmu_write_context_bank(struct arm_smmu_device *smmu, int idx)
571 {
572 u32 reg;
573 bool stage1;
574 struct arm_smmu_cb *cb = &smmu->cbs[idx];
575 struct arm_smmu_cfg *cfg = cb->cfg;
576
577 /* Unassigned context banks only need disabling */
578 if (!cfg) {
579 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_SCTLR, 0);
580 return;
581 }
582
583 stage1 = cfg->cbar != CBAR_TYPE_S2_TRANS;
584
585 /* CBA2R */
586 if (smmu->version > ARM_SMMU_V1) {
587 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64)
588 reg = ARM_SMMU_CBA2R_VA64;
589 else
590 reg = 0;
591 /* 16-bit VMIDs live in CBA2R */
592 if (smmu->features & ARM_SMMU_FEAT_VMID16)
593 reg |= FIELD_PREP(ARM_SMMU_CBA2R_VMID16, cfg->vmid);
594
595 arm_smmu_gr1_write(smmu, ARM_SMMU_GR1_CBA2R(idx), reg);
596 }
597
598 /* CBAR */
599 reg = FIELD_PREP(ARM_SMMU_CBAR_TYPE, cfg->cbar);
600 if (smmu->version < ARM_SMMU_V2)
601 reg |= FIELD_PREP(ARM_SMMU_CBAR_IRPTNDX, cfg->irptndx);
602
603 /*
604 * Use the weakest shareability/memory types, so they are
605 * overridden by the ttbcr/pte.
606 */
607 if (stage1) {
608 reg |= FIELD_PREP(ARM_SMMU_CBAR_S1_BPSHCFG,
609 ARM_SMMU_CBAR_S1_BPSHCFG_NSH) |
610 FIELD_PREP(ARM_SMMU_CBAR_S1_MEMATTR,
611 ARM_SMMU_CBAR_S1_MEMATTR_WB);
612 } else if (!(smmu->features & ARM_SMMU_FEAT_VMID16)) {
613 /* 8-bit VMIDs live in CBAR */
614 reg |= FIELD_PREP(ARM_SMMU_CBAR_VMID, cfg->vmid);
615 }
616 arm_smmu_gr1_write(smmu, ARM_SMMU_GR1_CBAR(idx), reg);
617
618 /*
619 * TCR
620 * We must write this before the TTBRs, since it determines the
621 * access behaviour of some fields (in particular, ASID[15:8]).
622 */
623 if (stage1 && smmu->version > ARM_SMMU_V1)
624 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_TCR2, cb->tcr[1]);
625 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_TCR, cb->tcr[0]);
626
627 /* TTBRs */
628 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) {
629 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_CONTEXTIDR, cfg->asid);
630 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_TTBR0, cb->ttbr[0]);
631 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_TTBR1, cb->ttbr[1]);
632 } else {
633 arm_smmu_cb_writeq(smmu, idx, ARM_SMMU_CB_TTBR0, cb->ttbr[0]);
634 if (stage1)
635 arm_smmu_cb_writeq(smmu, idx, ARM_SMMU_CB_TTBR1,
636 cb->ttbr[1]);
637 }
638
639 /* MAIRs (stage-1 only) */
640 if (stage1) {
641 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_S1_MAIR0, cb->mair[0]);
642 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_S1_MAIR1, cb->mair[1]);
643 }
644
645 /* SCTLR */
646 reg = ARM_SMMU_SCTLR_CFIE | ARM_SMMU_SCTLR_CFRE | ARM_SMMU_SCTLR_AFE |
647 ARM_SMMU_SCTLR_TRE | ARM_SMMU_SCTLR_M;
648 if (stage1)
649 reg |= ARM_SMMU_SCTLR_S1_ASIDPNE;
650 if (IS_ENABLED(CONFIG_CPU_BIG_ENDIAN))
651 reg |= ARM_SMMU_SCTLR_E;
652
653 if (smmu->impl && smmu->impl->write_sctlr)
654 smmu->impl->write_sctlr(smmu, idx, reg);
655 else
656 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_SCTLR, reg);
657 }
658
arm_smmu_alloc_context_bank(struct arm_smmu_domain * smmu_domain,struct arm_smmu_device * smmu,struct device * dev,unsigned int start)659 static int arm_smmu_alloc_context_bank(struct arm_smmu_domain *smmu_domain,
660 struct arm_smmu_device *smmu,
661 struct device *dev, unsigned int start)
662 {
663 if (smmu->impl && smmu->impl->alloc_context_bank)
664 return smmu->impl->alloc_context_bank(smmu_domain, smmu, dev, start);
665
666 return __arm_smmu_alloc_bitmap(smmu->context_map, start, smmu->num_context_banks);
667 }
668
arm_smmu_init_domain_context(struct arm_smmu_domain * smmu_domain,struct arm_smmu_device * smmu,struct device * dev)669 static int arm_smmu_init_domain_context(struct arm_smmu_domain *smmu_domain,
670 struct arm_smmu_device *smmu,
671 struct device *dev)
672 {
673 int irq, start, ret = 0;
674 unsigned long ias, oas;
675 struct io_pgtable_ops *pgtbl_ops;
676 struct io_pgtable_cfg pgtbl_cfg;
677 enum io_pgtable_fmt fmt;
678 struct iommu_domain *domain = &smmu_domain->domain;
679 struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
680 irqreturn_t (*context_fault)(int irq, void *dev);
681
682 mutex_lock(&smmu_domain->init_mutex);
683 if (smmu_domain->smmu)
684 goto out_unlock;
685
686 /*
687 * Mapping the requested stage onto what we support is surprisingly
688 * complicated, mainly because the spec allows S1+S2 SMMUs without
689 * support for nested translation. That means we end up with the
690 * following table:
691 *
692 * Requested Supported Actual
693 * S1 N S1
694 * S1 S1+S2 S1
695 * S1 S2 S2
696 * S1 S1 S1
697 * N N N
698 * N S1+S2 S2
699 * N S2 S2
700 * N S1 S1
701 *
702 * Note that you can't actually request stage-2 mappings.
703 */
704 if (!(smmu->features & ARM_SMMU_FEAT_TRANS_S1))
705 smmu_domain->stage = ARM_SMMU_DOMAIN_S2;
706 if (!(smmu->features & ARM_SMMU_FEAT_TRANS_S2))
707 smmu_domain->stage = ARM_SMMU_DOMAIN_S1;
708
709 /*
710 * Choosing a suitable context format is even more fiddly. Until we
711 * grow some way for the caller to express a preference, and/or move
712 * the decision into the io-pgtable code where it arguably belongs,
713 * just aim for the closest thing to the rest of the system, and hope
714 * that the hardware isn't esoteric enough that we can't assume AArch64
715 * support to be a superset of AArch32 support...
716 */
717 if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH32_L)
718 cfg->fmt = ARM_SMMU_CTX_FMT_AARCH32_L;
719 if (IS_ENABLED(CONFIG_IOMMU_IO_PGTABLE_ARMV7S) &&
720 !IS_ENABLED(CONFIG_64BIT) && !IS_ENABLED(CONFIG_ARM_LPAE) &&
721 (smmu->features & ARM_SMMU_FEAT_FMT_AARCH32_S) &&
722 (smmu_domain->stage == ARM_SMMU_DOMAIN_S1))
723 cfg->fmt = ARM_SMMU_CTX_FMT_AARCH32_S;
724 if ((IS_ENABLED(CONFIG_64BIT) || cfg->fmt == ARM_SMMU_CTX_FMT_NONE) &&
725 (smmu->features & (ARM_SMMU_FEAT_FMT_AARCH64_64K |
726 ARM_SMMU_FEAT_FMT_AARCH64_16K |
727 ARM_SMMU_FEAT_FMT_AARCH64_4K)))
728 cfg->fmt = ARM_SMMU_CTX_FMT_AARCH64;
729
730 if (cfg->fmt == ARM_SMMU_CTX_FMT_NONE) {
731 ret = -EINVAL;
732 goto out_unlock;
733 }
734
735 switch (smmu_domain->stage) {
736 case ARM_SMMU_DOMAIN_S1:
737 cfg->cbar = CBAR_TYPE_S1_TRANS_S2_BYPASS;
738 start = smmu->num_s2_context_banks;
739 ias = smmu->va_size;
740 oas = smmu->ipa_size;
741 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64) {
742 fmt = ARM_64_LPAE_S1;
743 } else if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_L) {
744 fmt = ARM_32_LPAE_S1;
745 ias = min(ias, 32UL);
746 oas = min(oas, 40UL);
747 } else {
748 fmt = ARM_V7S;
749 ias = min(ias, 32UL);
750 oas = min(oas, 32UL);
751 }
752 smmu_domain->flush_ops = &arm_smmu_s1_tlb_ops;
753 break;
754 case ARM_SMMU_DOMAIN_NESTED:
755 /*
756 * We will likely want to change this if/when KVM gets
757 * involved.
758 */
759 case ARM_SMMU_DOMAIN_S2:
760 cfg->cbar = CBAR_TYPE_S2_TRANS;
761 start = 0;
762 ias = smmu->ipa_size;
763 oas = smmu->pa_size;
764 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64) {
765 fmt = ARM_64_LPAE_S2;
766 } else {
767 fmt = ARM_32_LPAE_S2;
768 ias = min(ias, 40UL);
769 oas = min(oas, 40UL);
770 }
771 if (smmu->version == ARM_SMMU_V2)
772 smmu_domain->flush_ops = &arm_smmu_s2_tlb_ops_v2;
773 else
774 smmu_domain->flush_ops = &arm_smmu_s2_tlb_ops_v1;
775 break;
776 default:
777 ret = -EINVAL;
778 goto out_unlock;
779 }
780
781 ret = arm_smmu_alloc_context_bank(smmu_domain, smmu, dev, start);
782 if (ret < 0) {
783 goto out_unlock;
784 }
785
786 smmu_domain->smmu = smmu;
787
788 cfg->cbndx = ret;
789 if (smmu->version < ARM_SMMU_V2) {
790 cfg->irptndx = atomic_inc_return(&smmu->irptndx);
791 cfg->irptndx %= smmu->num_context_irqs;
792 } else {
793 cfg->irptndx = cfg->cbndx;
794 }
795
796 if (smmu_domain->stage == ARM_SMMU_DOMAIN_S2)
797 cfg->vmid = cfg->cbndx + 1;
798 else
799 cfg->asid = cfg->cbndx;
800
801 pgtbl_cfg = (struct io_pgtable_cfg) {
802 .pgsize_bitmap = smmu->pgsize_bitmap,
803 .ias = ias,
804 .oas = oas,
805 .coherent_walk = smmu->features & ARM_SMMU_FEAT_COHERENT_WALK,
806 .tlb = smmu_domain->flush_ops,
807 .iommu_dev = smmu->dev,
808 };
809
810 if (smmu->impl && smmu->impl->init_context) {
811 ret = smmu->impl->init_context(smmu_domain, &pgtbl_cfg, dev);
812 if (ret)
813 goto out_clear_smmu;
814 }
815
816 if (smmu_domain->pgtbl_quirks)
817 pgtbl_cfg.quirks |= smmu_domain->pgtbl_quirks;
818
819 pgtbl_ops = alloc_io_pgtable_ops(fmt, &pgtbl_cfg, smmu_domain);
820 if (!pgtbl_ops) {
821 ret = -ENOMEM;
822 goto out_clear_smmu;
823 }
824
825 /* Update the domain's page sizes to reflect the page table format */
826 domain->pgsize_bitmap = pgtbl_cfg.pgsize_bitmap;
827
828 if (pgtbl_cfg.quirks & IO_PGTABLE_QUIRK_ARM_TTBR1) {
829 domain->geometry.aperture_start = ~0UL << ias;
830 domain->geometry.aperture_end = ~0UL;
831 } else {
832 domain->geometry.aperture_end = (1UL << ias) - 1;
833 }
834
835 domain->geometry.force_aperture = true;
836
837 /* Initialise the context bank with our page table cfg */
838 arm_smmu_init_context_bank(smmu_domain, &pgtbl_cfg);
839 arm_smmu_write_context_bank(smmu, cfg->cbndx);
840
841 /*
842 * Request context fault interrupt. Do this last to avoid the
843 * handler seeing a half-initialised domain state.
844 */
845 irq = smmu->irqs[cfg->irptndx];
846
847 if (smmu->impl && smmu->impl->context_fault)
848 context_fault = smmu->impl->context_fault;
849 else
850 context_fault = arm_smmu_context_fault;
851
852 if (smmu->impl && smmu->impl->context_fault_needs_threaded_irq)
853 ret = devm_request_threaded_irq(smmu->dev, irq, NULL,
854 context_fault,
855 IRQF_ONESHOT | IRQF_SHARED,
856 "arm-smmu-context-fault",
857 smmu_domain);
858 else
859 ret = devm_request_irq(smmu->dev, irq, context_fault, IRQF_SHARED,
860 "arm-smmu-context-fault", smmu_domain);
861
862 if (ret < 0) {
863 dev_err(smmu->dev, "failed to request context IRQ %d (%u)\n",
864 cfg->irptndx, irq);
865 cfg->irptndx = ARM_SMMU_INVALID_IRPTNDX;
866 }
867
868 mutex_unlock(&smmu_domain->init_mutex);
869
870 /* Publish page table ops for map/unmap */
871 smmu_domain->pgtbl_ops = pgtbl_ops;
872 return 0;
873
874 out_clear_smmu:
875 __arm_smmu_free_bitmap(smmu->context_map, cfg->cbndx);
876 smmu_domain->smmu = NULL;
877 out_unlock:
878 mutex_unlock(&smmu_domain->init_mutex);
879 return ret;
880 }
881
arm_smmu_destroy_domain_context(struct arm_smmu_domain * smmu_domain)882 static void arm_smmu_destroy_domain_context(struct arm_smmu_domain *smmu_domain)
883 {
884 struct arm_smmu_device *smmu = smmu_domain->smmu;
885 struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
886 int ret, irq;
887
888 if (!smmu)
889 return;
890
891 ret = arm_smmu_rpm_get(smmu);
892 if (ret < 0)
893 return;
894
895 /*
896 * Disable the context bank and free the page tables before freeing
897 * it.
898 */
899 smmu->cbs[cfg->cbndx].cfg = NULL;
900 arm_smmu_write_context_bank(smmu, cfg->cbndx);
901
902 if (cfg->irptndx != ARM_SMMU_INVALID_IRPTNDX) {
903 irq = smmu->irqs[cfg->irptndx];
904 devm_free_irq(smmu->dev, irq, smmu_domain);
905 }
906
907 free_io_pgtable_ops(smmu_domain->pgtbl_ops);
908 __arm_smmu_free_bitmap(smmu->context_map, cfg->cbndx);
909
910 arm_smmu_rpm_put(smmu);
911 }
912
arm_smmu_domain_alloc_paging(struct device * dev)913 static struct iommu_domain *arm_smmu_domain_alloc_paging(struct device *dev)
914 {
915 struct arm_smmu_domain *smmu_domain;
916
917 /*
918 * Allocate the domain and initialise some of its data structures.
919 * We can't really do anything meaningful until we've added a
920 * master.
921 */
922 smmu_domain = kzalloc(sizeof(*smmu_domain), GFP_KERNEL);
923 if (!smmu_domain)
924 return NULL;
925
926 mutex_init(&smmu_domain->init_mutex);
927 spin_lock_init(&smmu_domain->cb_lock);
928
929 return &smmu_domain->domain;
930 }
931
arm_smmu_domain_free(struct iommu_domain * domain)932 static void arm_smmu_domain_free(struct iommu_domain *domain)
933 {
934 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
935
936 /*
937 * Free the domain resources. We assume that all devices have
938 * already been detached.
939 */
940 arm_smmu_destroy_domain_context(smmu_domain);
941 kfree(smmu_domain);
942 }
943
arm_smmu_write_smr(struct arm_smmu_device * smmu,int idx)944 static void arm_smmu_write_smr(struct arm_smmu_device *smmu, int idx)
945 {
946 struct arm_smmu_smr *smr = smmu->smrs + idx;
947 u32 reg = FIELD_PREP(ARM_SMMU_SMR_ID, smr->id) |
948 FIELD_PREP(ARM_SMMU_SMR_MASK, smr->mask);
949
950 if (!(smmu->features & ARM_SMMU_FEAT_EXIDS) && smr->valid)
951 reg |= ARM_SMMU_SMR_VALID;
952 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_SMR(idx), reg);
953 }
954
arm_smmu_write_s2cr(struct arm_smmu_device * smmu,int idx)955 static void arm_smmu_write_s2cr(struct arm_smmu_device *smmu, int idx)
956 {
957 struct arm_smmu_s2cr *s2cr = smmu->s2crs + idx;
958 u32 reg;
959
960 if (smmu->impl && smmu->impl->write_s2cr) {
961 smmu->impl->write_s2cr(smmu, idx);
962 return;
963 }
964
965 reg = FIELD_PREP(ARM_SMMU_S2CR_TYPE, s2cr->type) |
966 FIELD_PREP(ARM_SMMU_S2CR_CBNDX, s2cr->cbndx) |
967 FIELD_PREP(ARM_SMMU_S2CR_PRIVCFG, s2cr->privcfg);
968
969 if (smmu->features & ARM_SMMU_FEAT_EXIDS && smmu->smrs &&
970 smmu->smrs[idx].valid)
971 reg |= ARM_SMMU_S2CR_EXIDVALID;
972 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_S2CR(idx), reg);
973 }
974
arm_smmu_write_sme(struct arm_smmu_device * smmu,int idx)975 static void arm_smmu_write_sme(struct arm_smmu_device *smmu, int idx)
976 {
977 arm_smmu_write_s2cr(smmu, idx);
978 if (smmu->smrs)
979 arm_smmu_write_smr(smmu, idx);
980 }
981
982 /*
983 * The width of SMR's mask field depends on sCR0_EXIDENABLE, so this function
984 * should be called after sCR0 is written.
985 */
arm_smmu_test_smr_masks(struct arm_smmu_device * smmu)986 static void arm_smmu_test_smr_masks(struct arm_smmu_device *smmu)
987 {
988 u32 smr;
989 int i;
990
991 if (!smmu->smrs)
992 return;
993 /*
994 * If we've had to accommodate firmware memory regions, we may
995 * have live SMRs by now; tread carefully...
996 *
997 * Somewhat perversely, not having a free SMR for this test implies we
998 * can get away without it anyway, as we'll only be able to 'allocate'
999 * these SMRs for the ID/mask values we're already trusting to be OK.
1000 */
1001 for (i = 0; i < smmu->num_mapping_groups; i++)
1002 if (!smmu->smrs[i].valid)
1003 goto smr_ok;
1004 return;
1005 smr_ok:
1006 /*
1007 * SMR.ID bits may not be preserved if the corresponding MASK
1008 * bits are set, so check each one separately. We can reject
1009 * masters later if they try to claim IDs outside these masks.
1010 */
1011 smr = FIELD_PREP(ARM_SMMU_SMR_ID, smmu->streamid_mask);
1012 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_SMR(i), smr);
1013 smr = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_SMR(i));
1014 smmu->streamid_mask = FIELD_GET(ARM_SMMU_SMR_ID, smr);
1015
1016 smr = FIELD_PREP(ARM_SMMU_SMR_MASK, smmu->streamid_mask);
1017 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_SMR(i), smr);
1018 smr = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_SMR(i));
1019 smmu->smr_mask_mask = FIELD_GET(ARM_SMMU_SMR_MASK, smr);
1020 }
1021
arm_smmu_find_sme(struct arm_smmu_device * smmu,u16 id,u16 mask)1022 static int arm_smmu_find_sme(struct arm_smmu_device *smmu, u16 id, u16 mask)
1023 {
1024 struct arm_smmu_smr *smrs = smmu->smrs;
1025 int i, free_idx = -ENOSPC;
1026
1027 /* Stream indexing is blissfully easy */
1028 if (!smrs)
1029 return id;
1030
1031 /* Validating SMRs is... less so */
1032 for (i = 0; i < smmu->num_mapping_groups; ++i) {
1033 if (!smrs[i].valid) {
1034 /*
1035 * Note the first free entry we come across, which
1036 * we'll claim in the end if nothing else matches.
1037 */
1038 if (free_idx < 0)
1039 free_idx = i;
1040 continue;
1041 }
1042 /*
1043 * If the new entry is _entirely_ matched by an existing entry,
1044 * then reuse that, with the guarantee that there also cannot
1045 * be any subsequent conflicting entries. In normal use we'd
1046 * expect simply identical entries for this case, but there's
1047 * no harm in accommodating the generalisation.
1048 */
1049 if ((mask & smrs[i].mask) == mask &&
1050 !((id ^ smrs[i].id) & ~smrs[i].mask))
1051 return i;
1052 /*
1053 * If the new entry has any other overlap with an existing one,
1054 * though, then there always exists at least one stream ID
1055 * which would cause a conflict, and we can't allow that risk.
1056 */
1057 if (!((id ^ smrs[i].id) & ~(smrs[i].mask | mask)))
1058 return -EINVAL;
1059 }
1060
1061 return free_idx;
1062 }
1063
arm_smmu_free_sme(struct arm_smmu_device * smmu,int idx)1064 static bool arm_smmu_free_sme(struct arm_smmu_device *smmu, int idx)
1065 {
1066 if (--smmu->s2crs[idx].count)
1067 return false;
1068
1069 smmu->s2crs[idx] = s2cr_init_val;
1070 if (smmu->smrs)
1071 smmu->smrs[idx].valid = false;
1072
1073 return true;
1074 }
1075
arm_smmu_master_alloc_smes(struct device * dev)1076 static int arm_smmu_master_alloc_smes(struct device *dev)
1077 {
1078 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
1079 struct arm_smmu_master_cfg *cfg = dev_iommu_priv_get(dev);
1080 struct arm_smmu_device *smmu = cfg->smmu;
1081 struct arm_smmu_smr *smrs = smmu->smrs;
1082 int i, idx, ret;
1083
1084 mutex_lock(&smmu->stream_map_mutex);
1085 /* Figure out a viable stream map entry allocation */
1086 for_each_cfg_sme(cfg, fwspec, i, idx) {
1087 u16 sid = FIELD_GET(ARM_SMMU_SMR_ID, fwspec->ids[i]);
1088 u16 mask = FIELD_GET(ARM_SMMU_SMR_MASK, fwspec->ids[i]);
1089
1090 if (idx != INVALID_SMENDX) {
1091 ret = -EEXIST;
1092 goto out_err;
1093 }
1094
1095 ret = arm_smmu_find_sme(smmu, sid, mask);
1096 if (ret < 0)
1097 goto out_err;
1098
1099 idx = ret;
1100 if (smrs && smmu->s2crs[idx].count == 0) {
1101 smrs[idx].id = sid;
1102 smrs[idx].mask = mask;
1103 smrs[idx].valid = true;
1104 }
1105 smmu->s2crs[idx].count++;
1106 cfg->smendx[i] = (s16)idx;
1107 }
1108
1109 /* It worked! Now, poke the actual hardware */
1110 for_each_cfg_sme(cfg, fwspec, i, idx)
1111 arm_smmu_write_sme(smmu, idx);
1112
1113 mutex_unlock(&smmu->stream_map_mutex);
1114 return 0;
1115
1116 out_err:
1117 while (i--) {
1118 arm_smmu_free_sme(smmu, cfg->smendx[i]);
1119 cfg->smendx[i] = INVALID_SMENDX;
1120 }
1121 mutex_unlock(&smmu->stream_map_mutex);
1122 return ret;
1123 }
1124
arm_smmu_master_free_smes(struct arm_smmu_master_cfg * cfg,struct iommu_fwspec * fwspec)1125 static void arm_smmu_master_free_smes(struct arm_smmu_master_cfg *cfg,
1126 struct iommu_fwspec *fwspec)
1127 {
1128 struct arm_smmu_device *smmu = cfg->smmu;
1129 int i, idx;
1130
1131 mutex_lock(&smmu->stream_map_mutex);
1132 for_each_cfg_sme(cfg, fwspec, i, idx) {
1133 if (arm_smmu_free_sme(smmu, idx))
1134 arm_smmu_write_sme(smmu, idx);
1135 cfg->smendx[i] = INVALID_SMENDX;
1136 }
1137 mutex_unlock(&smmu->stream_map_mutex);
1138 }
1139
arm_smmu_master_install_s2crs(struct arm_smmu_master_cfg * cfg,enum arm_smmu_s2cr_type type,u8 cbndx,struct iommu_fwspec * fwspec)1140 static void arm_smmu_master_install_s2crs(struct arm_smmu_master_cfg *cfg,
1141 enum arm_smmu_s2cr_type type,
1142 u8 cbndx, struct iommu_fwspec *fwspec)
1143 {
1144 struct arm_smmu_device *smmu = cfg->smmu;
1145 struct arm_smmu_s2cr *s2cr = smmu->s2crs;
1146 int i, idx;
1147
1148 for_each_cfg_sme(cfg, fwspec, i, idx) {
1149 if (type == s2cr[idx].type && cbndx == s2cr[idx].cbndx)
1150 continue;
1151
1152 s2cr[idx].type = type;
1153 s2cr[idx].privcfg = S2CR_PRIVCFG_DEFAULT;
1154 s2cr[idx].cbndx = cbndx;
1155 arm_smmu_write_s2cr(smmu, idx);
1156 }
1157 }
1158
arm_smmu_attach_dev(struct iommu_domain * domain,struct device * dev)1159 static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev)
1160 {
1161 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1162 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
1163 struct arm_smmu_master_cfg *cfg;
1164 struct arm_smmu_device *smmu;
1165 int ret;
1166
1167 /*
1168 * FIXME: The arch/arm DMA API code tries to attach devices to its own
1169 * domains between of_xlate() and probe_device() - we have no way to cope
1170 * with that, so until ARM gets converted to rely on groups and default
1171 * domains, just say no (but more politely than by dereferencing NULL).
1172 * This should be at least a WARN_ON once that's sorted.
1173 */
1174 cfg = dev_iommu_priv_get(dev);
1175 if (!cfg)
1176 return -ENODEV;
1177
1178 smmu = cfg->smmu;
1179
1180 ret = arm_smmu_rpm_get(smmu);
1181 if (ret < 0)
1182 return ret;
1183
1184 /* Ensure that the domain is finalised */
1185 ret = arm_smmu_init_domain_context(smmu_domain, smmu, dev);
1186 if (ret < 0)
1187 goto rpm_put;
1188
1189 /*
1190 * Sanity check the domain. We don't support domains across
1191 * different SMMUs.
1192 */
1193 if (smmu_domain->smmu != smmu) {
1194 ret = -EINVAL;
1195 goto rpm_put;
1196 }
1197
1198 /* Looks ok, so add the device to the domain */
1199 arm_smmu_master_install_s2crs(cfg, S2CR_TYPE_TRANS,
1200 smmu_domain->cfg.cbndx, fwspec);
1201 rpm_put:
1202 arm_smmu_rpm_put(smmu);
1203 return ret;
1204 }
1205
arm_smmu_attach_dev_type(struct device * dev,enum arm_smmu_s2cr_type type)1206 static int arm_smmu_attach_dev_type(struct device *dev,
1207 enum arm_smmu_s2cr_type type)
1208 {
1209 struct arm_smmu_master_cfg *cfg = dev_iommu_priv_get(dev);
1210 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
1211 struct arm_smmu_device *smmu;
1212 int ret;
1213
1214 if (!cfg)
1215 return -ENODEV;
1216 smmu = cfg->smmu;
1217
1218 ret = arm_smmu_rpm_get(smmu);
1219 if (ret < 0)
1220 return ret;
1221
1222 arm_smmu_master_install_s2crs(cfg, type, 0, fwspec);
1223 arm_smmu_rpm_put(smmu);
1224 return 0;
1225 }
1226
arm_smmu_attach_dev_identity(struct iommu_domain * domain,struct device * dev)1227 static int arm_smmu_attach_dev_identity(struct iommu_domain *domain,
1228 struct device *dev)
1229 {
1230 return arm_smmu_attach_dev_type(dev, S2CR_TYPE_BYPASS);
1231 }
1232
1233 static const struct iommu_domain_ops arm_smmu_identity_ops = {
1234 .attach_dev = arm_smmu_attach_dev_identity,
1235 };
1236
1237 static struct iommu_domain arm_smmu_identity_domain = {
1238 .type = IOMMU_DOMAIN_IDENTITY,
1239 .ops = &arm_smmu_identity_ops,
1240 };
1241
arm_smmu_attach_dev_blocked(struct iommu_domain * domain,struct device * dev)1242 static int arm_smmu_attach_dev_blocked(struct iommu_domain *domain,
1243 struct device *dev)
1244 {
1245 return arm_smmu_attach_dev_type(dev, S2CR_TYPE_FAULT);
1246 }
1247
1248 static const struct iommu_domain_ops arm_smmu_blocked_ops = {
1249 .attach_dev = arm_smmu_attach_dev_blocked,
1250 };
1251
1252 static struct iommu_domain arm_smmu_blocked_domain = {
1253 .type = IOMMU_DOMAIN_BLOCKED,
1254 .ops = &arm_smmu_blocked_ops,
1255 };
1256
arm_smmu_map_pages(struct iommu_domain * domain,unsigned long iova,phys_addr_t paddr,size_t pgsize,size_t pgcount,int prot,gfp_t gfp,size_t * mapped)1257 static int arm_smmu_map_pages(struct iommu_domain *domain, unsigned long iova,
1258 phys_addr_t paddr, size_t pgsize, size_t pgcount,
1259 int prot, gfp_t gfp, size_t *mapped)
1260 {
1261 struct io_pgtable_ops *ops = to_smmu_domain(domain)->pgtbl_ops;
1262 struct arm_smmu_device *smmu = to_smmu_domain(domain)->smmu;
1263 int ret;
1264
1265 if (!ops)
1266 return -ENODEV;
1267
1268 arm_smmu_rpm_get(smmu);
1269 ret = ops->map_pages(ops, iova, paddr, pgsize, pgcount, prot, gfp, mapped);
1270 arm_smmu_rpm_put(smmu);
1271
1272 return ret;
1273 }
1274
arm_smmu_unmap_pages(struct iommu_domain * domain,unsigned long iova,size_t pgsize,size_t pgcount,struct iommu_iotlb_gather * iotlb_gather)1275 static size_t arm_smmu_unmap_pages(struct iommu_domain *domain, unsigned long iova,
1276 size_t pgsize, size_t pgcount,
1277 struct iommu_iotlb_gather *iotlb_gather)
1278 {
1279 struct io_pgtable_ops *ops = to_smmu_domain(domain)->pgtbl_ops;
1280 struct arm_smmu_device *smmu = to_smmu_domain(domain)->smmu;
1281 size_t ret;
1282
1283 if (!ops)
1284 return 0;
1285
1286 arm_smmu_rpm_get(smmu);
1287 ret = ops->unmap_pages(ops, iova, pgsize, pgcount, iotlb_gather);
1288 arm_smmu_rpm_put(smmu);
1289
1290 return ret;
1291 }
1292
arm_smmu_flush_iotlb_all(struct iommu_domain * domain)1293 static void arm_smmu_flush_iotlb_all(struct iommu_domain *domain)
1294 {
1295 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1296 struct arm_smmu_device *smmu = smmu_domain->smmu;
1297
1298 if (smmu_domain->flush_ops) {
1299 arm_smmu_rpm_get(smmu);
1300 smmu_domain->flush_ops->tlb_flush_all(smmu_domain);
1301 arm_smmu_rpm_put(smmu);
1302 }
1303 }
1304
arm_smmu_iotlb_sync(struct iommu_domain * domain,struct iommu_iotlb_gather * gather)1305 static void arm_smmu_iotlb_sync(struct iommu_domain *domain,
1306 struct iommu_iotlb_gather *gather)
1307 {
1308 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1309 struct arm_smmu_device *smmu = smmu_domain->smmu;
1310
1311 if (!smmu)
1312 return;
1313
1314 arm_smmu_rpm_get(smmu);
1315 if (smmu->version == ARM_SMMU_V2 ||
1316 smmu_domain->stage == ARM_SMMU_DOMAIN_S1)
1317 arm_smmu_tlb_sync_context(smmu_domain);
1318 else
1319 arm_smmu_tlb_sync_global(smmu);
1320 arm_smmu_rpm_put(smmu);
1321 }
1322
arm_smmu_iova_to_phys_hard(struct iommu_domain * domain,dma_addr_t iova)1323 static phys_addr_t arm_smmu_iova_to_phys_hard(struct iommu_domain *domain,
1324 dma_addr_t iova)
1325 {
1326 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1327 struct arm_smmu_device *smmu = smmu_domain->smmu;
1328 struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
1329 struct io_pgtable_ops *ops= smmu_domain->pgtbl_ops;
1330 struct device *dev = smmu->dev;
1331 void __iomem *reg;
1332 u32 tmp;
1333 u64 phys;
1334 unsigned long va, flags;
1335 int ret, idx = cfg->cbndx;
1336 phys_addr_t addr = 0;
1337
1338 ret = arm_smmu_rpm_get(smmu);
1339 if (ret < 0)
1340 return 0;
1341
1342 spin_lock_irqsave(&smmu_domain->cb_lock, flags);
1343 va = iova & ~0xfffUL;
1344 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64)
1345 arm_smmu_cb_writeq(smmu, idx, ARM_SMMU_CB_ATS1PR, va);
1346 else
1347 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_ATS1PR, va);
1348
1349 reg = arm_smmu_page(smmu, ARM_SMMU_CB(smmu, idx)) + ARM_SMMU_CB_ATSR;
1350 if (readl_poll_timeout_atomic(reg, tmp, !(tmp & ARM_SMMU_CB_ATSR_ACTIVE),
1351 5, 50)) {
1352 spin_unlock_irqrestore(&smmu_domain->cb_lock, flags);
1353 dev_err(dev,
1354 "iova to phys timed out on %pad. Falling back to software table walk.\n",
1355 &iova);
1356 arm_smmu_rpm_put(smmu);
1357 return ops->iova_to_phys(ops, iova);
1358 }
1359
1360 phys = arm_smmu_cb_readq(smmu, idx, ARM_SMMU_CB_PAR);
1361 spin_unlock_irqrestore(&smmu_domain->cb_lock, flags);
1362 if (phys & ARM_SMMU_CB_PAR_F) {
1363 dev_err(dev, "translation fault!\n");
1364 dev_err(dev, "PAR = 0x%llx\n", phys);
1365 goto out;
1366 }
1367
1368 addr = (phys & GENMASK_ULL(39, 12)) | (iova & 0xfff);
1369 out:
1370 arm_smmu_rpm_put(smmu);
1371
1372 return addr;
1373 }
1374
arm_smmu_iova_to_phys(struct iommu_domain * domain,dma_addr_t iova)1375 static phys_addr_t arm_smmu_iova_to_phys(struct iommu_domain *domain,
1376 dma_addr_t iova)
1377 {
1378 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1379 struct io_pgtable_ops *ops = smmu_domain->pgtbl_ops;
1380
1381 if (!ops)
1382 return 0;
1383
1384 if (smmu_domain->smmu->features & ARM_SMMU_FEAT_TRANS_OPS &&
1385 smmu_domain->stage == ARM_SMMU_DOMAIN_S1)
1386 return arm_smmu_iova_to_phys_hard(domain, iova);
1387
1388 return ops->iova_to_phys(ops, iova);
1389 }
1390
arm_smmu_capable(struct device * dev,enum iommu_cap cap)1391 static bool arm_smmu_capable(struct device *dev, enum iommu_cap cap)
1392 {
1393 struct arm_smmu_master_cfg *cfg = dev_iommu_priv_get(dev);
1394
1395 switch (cap) {
1396 case IOMMU_CAP_CACHE_COHERENCY:
1397 /*
1398 * It's overwhelmingly the case in practice that when the pagetable
1399 * walk interface is connected to a coherent interconnect, all the
1400 * translation interfaces are too. Furthermore if the device is
1401 * natively coherent, then its translation interface must also be.
1402 */
1403 return cfg->smmu->features & ARM_SMMU_FEAT_COHERENT_WALK ||
1404 device_get_dma_attr(dev) == DEV_DMA_COHERENT;
1405 case IOMMU_CAP_NOEXEC:
1406 case IOMMU_CAP_DEFERRED_FLUSH:
1407 return true;
1408 default:
1409 return false;
1410 }
1411 }
1412
1413 static
arm_smmu_get_by_fwnode(struct fwnode_handle * fwnode)1414 struct arm_smmu_device *arm_smmu_get_by_fwnode(struct fwnode_handle *fwnode)
1415 {
1416 struct device *dev = bus_find_device_by_fwnode(&platform_bus_type, fwnode);
1417
1418 put_device(dev);
1419 return dev ? dev_get_drvdata(dev) : NULL;
1420 }
1421
arm_smmu_probe_device(struct device * dev)1422 static struct iommu_device *arm_smmu_probe_device(struct device *dev)
1423 {
1424 struct arm_smmu_device *smmu = NULL;
1425 struct arm_smmu_master_cfg *cfg;
1426 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
1427 int i, ret;
1428
1429 if (using_legacy_binding) {
1430 ret = arm_smmu_register_legacy_master(dev, &smmu);
1431
1432 /*
1433 * If dev->iommu_fwspec is initally NULL, arm_smmu_register_legacy_master()
1434 * will allocate/initialise a new one. Thus we need to update fwspec for
1435 * later use.
1436 */
1437 fwspec = dev_iommu_fwspec_get(dev);
1438 if (ret)
1439 goto out_free;
1440 } else {
1441 smmu = arm_smmu_get_by_fwnode(fwspec->iommu_fwnode);
1442 }
1443
1444 ret = -EINVAL;
1445 for (i = 0; i < fwspec->num_ids; i++) {
1446 u16 sid = FIELD_GET(ARM_SMMU_SMR_ID, fwspec->ids[i]);
1447 u16 mask = FIELD_GET(ARM_SMMU_SMR_MASK, fwspec->ids[i]);
1448
1449 if (sid & ~smmu->streamid_mask) {
1450 dev_err(dev, "stream ID 0x%x out of range for SMMU (0x%x)\n",
1451 sid, smmu->streamid_mask);
1452 goto out_free;
1453 }
1454 if (mask & ~smmu->smr_mask_mask) {
1455 dev_err(dev, "SMR mask 0x%x out of range for SMMU (0x%x)\n",
1456 mask, smmu->smr_mask_mask);
1457 goto out_free;
1458 }
1459 }
1460
1461 ret = -ENOMEM;
1462 cfg = kzalloc(offsetof(struct arm_smmu_master_cfg, smendx[i]),
1463 GFP_KERNEL);
1464 if (!cfg)
1465 goto out_free;
1466
1467 cfg->smmu = smmu;
1468 dev_iommu_priv_set(dev, cfg);
1469 while (i--)
1470 cfg->smendx[i] = INVALID_SMENDX;
1471
1472 ret = arm_smmu_rpm_get(smmu);
1473 if (ret < 0)
1474 goto out_cfg_free;
1475
1476 ret = arm_smmu_master_alloc_smes(dev);
1477 arm_smmu_rpm_put(smmu);
1478
1479 if (ret)
1480 goto out_cfg_free;
1481
1482 device_link_add(dev, smmu->dev,
1483 DL_FLAG_PM_RUNTIME | DL_FLAG_AUTOREMOVE_SUPPLIER);
1484
1485 return &smmu->iommu;
1486
1487 out_cfg_free:
1488 kfree(cfg);
1489 out_free:
1490 return ERR_PTR(ret);
1491 }
1492
arm_smmu_release_device(struct device * dev)1493 static void arm_smmu_release_device(struct device *dev)
1494 {
1495 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
1496 struct arm_smmu_master_cfg *cfg = dev_iommu_priv_get(dev);
1497 int ret;
1498
1499 ret = arm_smmu_rpm_get(cfg->smmu);
1500 if (ret < 0)
1501 return;
1502
1503 arm_smmu_master_free_smes(cfg, fwspec);
1504
1505 arm_smmu_rpm_put(cfg->smmu);
1506
1507 kfree(cfg);
1508 }
1509
arm_smmu_probe_finalize(struct device * dev)1510 static void arm_smmu_probe_finalize(struct device *dev)
1511 {
1512 struct arm_smmu_master_cfg *cfg;
1513 struct arm_smmu_device *smmu;
1514
1515 cfg = dev_iommu_priv_get(dev);
1516 smmu = cfg->smmu;
1517
1518 if (smmu->impl && smmu->impl->probe_finalize)
1519 smmu->impl->probe_finalize(smmu, dev);
1520 }
1521
arm_smmu_device_group(struct device * dev)1522 static struct iommu_group *arm_smmu_device_group(struct device *dev)
1523 {
1524 struct arm_smmu_master_cfg *cfg = dev_iommu_priv_get(dev);
1525 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
1526 struct arm_smmu_device *smmu = cfg->smmu;
1527 struct iommu_group *group = NULL;
1528 int i, idx;
1529
1530 mutex_lock(&smmu->stream_map_mutex);
1531 for_each_cfg_sme(cfg, fwspec, i, idx) {
1532 if (group && smmu->s2crs[idx].group &&
1533 group != smmu->s2crs[idx].group) {
1534 mutex_unlock(&smmu->stream_map_mutex);
1535 return ERR_PTR(-EINVAL);
1536 }
1537
1538 group = smmu->s2crs[idx].group;
1539 }
1540
1541 if (group) {
1542 mutex_unlock(&smmu->stream_map_mutex);
1543 return iommu_group_ref_get(group);
1544 }
1545
1546 if (dev_is_pci(dev))
1547 group = pci_device_group(dev);
1548 else if (dev_is_fsl_mc(dev))
1549 group = fsl_mc_device_group(dev);
1550 else
1551 group = generic_device_group(dev);
1552
1553 /* Remember group for faster lookups */
1554 if (!IS_ERR(group))
1555 for_each_cfg_sme(cfg, fwspec, i, idx)
1556 smmu->s2crs[idx].group = group;
1557
1558 mutex_unlock(&smmu->stream_map_mutex);
1559 return group;
1560 }
1561
arm_smmu_set_pgtable_quirks(struct iommu_domain * domain,unsigned long quirks)1562 static int arm_smmu_set_pgtable_quirks(struct iommu_domain *domain,
1563 unsigned long quirks)
1564 {
1565 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1566 int ret = 0;
1567
1568 mutex_lock(&smmu_domain->init_mutex);
1569 if (smmu_domain->smmu)
1570 ret = -EPERM;
1571 else
1572 smmu_domain->pgtbl_quirks = quirks;
1573 mutex_unlock(&smmu_domain->init_mutex);
1574
1575 return ret;
1576 }
1577
arm_smmu_of_xlate(struct device * dev,const struct of_phandle_args * args)1578 static int arm_smmu_of_xlate(struct device *dev,
1579 const struct of_phandle_args *args)
1580 {
1581 u32 mask, fwid = 0;
1582
1583 if (args->args_count > 0)
1584 fwid |= FIELD_PREP(ARM_SMMU_SMR_ID, args->args[0]);
1585
1586 if (args->args_count > 1)
1587 fwid |= FIELD_PREP(ARM_SMMU_SMR_MASK, args->args[1]);
1588 else if (!of_property_read_u32(args->np, "stream-match-mask", &mask))
1589 fwid |= FIELD_PREP(ARM_SMMU_SMR_MASK, mask);
1590
1591 return iommu_fwspec_add_ids(dev, &fwid, 1);
1592 }
1593
arm_smmu_get_resv_regions(struct device * dev,struct list_head * head)1594 static void arm_smmu_get_resv_regions(struct device *dev,
1595 struct list_head *head)
1596 {
1597 struct iommu_resv_region *region;
1598 int prot = IOMMU_WRITE | IOMMU_NOEXEC | IOMMU_MMIO;
1599
1600 region = iommu_alloc_resv_region(MSI_IOVA_BASE, MSI_IOVA_LENGTH,
1601 prot, IOMMU_RESV_SW_MSI, GFP_KERNEL);
1602 if (!region)
1603 return;
1604
1605 list_add_tail(®ion->list, head);
1606
1607 iommu_dma_get_resv_regions(dev, head);
1608 }
1609
arm_smmu_def_domain_type(struct device * dev)1610 static int arm_smmu_def_domain_type(struct device *dev)
1611 {
1612 struct arm_smmu_master_cfg *cfg = dev_iommu_priv_get(dev);
1613 const struct arm_smmu_impl *impl = cfg->smmu->impl;
1614
1615 if (using_legacy_binding)
1616 return IOMMU_DOMAIN_IDENTITY;
1617
1618 if (impl && impl->def_domain_type)
1619 return impl->def_domain_type(dev);
1620
1621 return 0;
1622 }
1623
1624 static struct iommu_ops arm_smmu_ops = {
1625 .identity_domain = &arm_smmu_identity_domain,
1626 .blocked_domain = &arm_smmu_blocked_domain,
1627 .capable = arm_smmu_capable,
1628 .domain_alloc_paging = arm_smmu_domain_alloc_paging,
1629 .probe_device = arm_smmu_probe_device,
1630 .release_device = arm_smmu_release_device,
1631 .probe_finalize = arm_smmu_probe_finalize,
1632 .device_group = arm_smmu_device_group,
1633 .of_xlate = arm_smmu_of_xlate,
1634 .get_resv_regions = arm_smmu_get_resv_regions,
1635 .def_domain_type = arm_smmu_def_domain_type,
1636 .pgsize_bitmap = -1UL, /* Restricted during device attach */
1637 .owner = THIS_MODULE,
1638 .default_domain_ops = &(const struct iommu_domain_ops) {
1639 .attach_dev = arm_smmu_attach_dev,
1640 .map_pages = arm_smmu_map_pages,
1641 .unmap_pages = arm_smmu_unmap_pages,
1642 .flush_iotlb_all = arm_smmu_flush_iotlb_all,
1643 .iotlb_sync = arm_smmu_iotlb_sync,
1644 .iova_to_phys = arm_smmu_iova_to_phys,
1645 .set_pgtable_quirks = arm_smmu_set_pgtable_quirks,
1646 .free = arm_smmu_domain_free,
1647 }
1648 };
1649
arm_smmu_device_reset(struct arm_smmu_device * smmu)1650 static void arm_smmu_device_reset(struct arm_smmu_device *smmu)
1651 {
1652 int i;
1653 u32 reg;
1654
1655 /* clear global FSR */
1656 reg = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sGFSR);
1657 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_sGFSR, reg);
1658
1659 /*
1660 * Reset stream mapping groups: Initial values mark all SMRn as
1661 * invalid and all S2CRn as bypass unless overridden.
1662 */
1663 for (i = 0; i < smmu->num_mapping_groups; ++i)
1664 arm_smmu_write_sme(smmu, i);
1665
1666 /* Make sure all context banks are disabled and clear CB_FSR */
1667 for (i = 0; i < smmu->num_context_banks; ++i) {
1668 arm_smmu_write_context_bank(smmu, i);
1669 arm_smmu_cb_write(smmu, i, ARM_SMMU_CB_FSR, ARM_SMMU_CB_FSR_FAULT);
1670 }
1671
1672 /* Invalidate the TLB, just in case */
1673 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_TLBIALLH, QCOM_DUMMY_VAL);
1674 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_TLBIALLNSNH, QCOM_DUMMY_VAL);
1675
1676 reg = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sCR0);
1677
1678 /* Enable fault reporting */
1679 reg |= (ARM_SMMU_sCR0_GFRE | ARM_SMMU_sCR0_GFIE |
1680 ARM_SMMU_sCR0_GCFGFRE | ARM_SMMU_sCR0_GCFGFIE);
1681
1682 /* Disable TLB broadcasting. */
1683 reg |= (ARM_SMMU_sCR0_VMIDPNE | ARM_SMMU_sCR0_PTM);
1684
1685 /* Enable client access, handling unmatched streams as appropriate */
1686 reg &= ~ARM_SMMU_sCR0_CLIENTPD;
1687 if (disable_bypass)
1688 reg |= ARM_SMMU_sCR0_USFCFG;
1689 else
1690 reg &= ~ARM_SMMU_sCR0_USFCFG;
1691
1692 /* Disable forced broadcasting */
1693 reg &= ~ARM_SMMU_sCR0_FB;
1694
1695 /* Don't upgrade barriers */
1696 reg &= ~(ARM_SMMU_sCR0_BSU);
1697
1698 if (smmu->features & ARM_SMMU_FEAT_VMID16)
1699 reg |= ARM_SMMU_sCR0_VMID16EN;
1700
1701 if (smmu->features & ARM_SMMU_FEAT_EXIDS)
1702 reg |= ARM_SMMU_sCR0_EXIDENABLE;
1703
1704 if (smmu->impl && smmu->impl->reset)
1705 smmu->impl->reset(smmu);
1706
1707 /* Push the button */
1708 arm_smmu_tlb_sync_global(smmu);
1709 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_sCR0, reg);
1710 }
1711
arm_smmu_id_size_to_bits(int size)1712 static int arm_smmu_id_size_to_bits(int size)
1713 {
1714 switch (size) {
1715 case 0:
1716 return 32;
1717 case 1:
1718 return 36;
1719 case 2:
1720 return 40;
1721 case 3:
1722 return 42;
1723 case 4:
1724 return 44;
1725 case 5:
1726 default:
1727 return 48;
1728 }
1729 }
1730
arm_smmu_device_cfg_probe(struct arm_smmu_device * smmu)1731 static int arm_smmu_device_cfg_probe(struct arm_smmu_device *smmu)
1732 {
1733 unsigned int size;
1734 u32 id;
1735 bool cttw_reg, cttw_fw = smmu->features & ARM_SMMU_FEAT_COHERENT_WALK;
1736 int i, ret;
1737
1738 dev_notice(smmu->dev, "probing hardware configuration...\n");
1739 dev_notice(smmu->dev, "SMMUv%d with:\n",
1740 smmu->version == ARM_SMMU_V2 ? 2 : 1);
1741
1742 /* ID0 */
1743 id = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_ID0);
1744
1745 /* Restrict available stages based on module parameter */
1746 if (force_stage == 1)
1747 id &= ~(ARM_SMMU_ID0_S2TS | ARM_SMMU_ID0_NTS);
1748 else if (force_stage == 2)
1749 id &= ~(ARM_SMMU_ID0_S1TS | ARM_SMMU_ID0_NTS);
1750
1751 if (id & ARM_SMMU_ID0_S1TS) {
1752 smmu->features |= ARM_SMMU_FEAT_TRANS_S1;
1753 dev_notice(smmu->dev, "\tstage 1 translation\n");
1754 }
1755
1756 if (id & ARM_SMMU_ID0_S2TS) {
1757 smmu->features |= ARM_SMMU_FEAT_TRANS_S2;
1758 dev_notice(smmu->dev, "\tstage 2 translation\n");
1759 }
1760
1761 if (id & ARM_SMMU_ID0_NTS) {
1762 smmu->features |= ARM_SMMU_FEAT_TRANS_NESTED;
1763 dev_notice(smmu->dev, "\tnested translation\n");
1764 }
1765
1766 if (!(smmu->features &
1767 (ARM_SMMU_FEAT_TRANS_S1 | ARM_SMMU_FEAT_TRANS_S2))) {
1768 dev_err(smmu->dev, "\tno translation support!\n");
1769 return -ENODEV;
1770 }
1771
1772 if ((id & ARM_SMMU_ID0_S1TS) &&
1773 ((smmu->version < ARM_SMMU_V2) || !(id & ARM_SMMU_ID0_ATOSNS))) {
1774 smmu->features |= ARM_SMMU_FEAT_TRANS_OPS;
1775 dev_notice(smmu->dev, "\taddress translation ops\n");
1776 }
1777
1778 /*
1779 * In order for DMA API calls to work properly, we must defer to what
1780 * the FW says about coherency, regardless of what the hardware claims.
1781 * Fortunately, this also opens up a workaround for systems where the
1782 * ID register value has ended up configured incorrectly.
1783 */
1784 cttw_reg = !!(id & ARM_SMMU_ID0_CTTW);
1785 if (cttw_fw || cttw_reg)
1786 dev_notice(smmu->dev, "\t%scoherent table walk\n",
1787 cttw_fw ? "" : "non-");
1788 if (cttw_fw != cttw_reg)
1789 dev_notice(smmu->dev,
1790 "\t(IDR0.CTTW overridden by FW configuration)\n");
1791
1792 /* Max. number of entries we have for stream matching/indexing */
1793 if (smmu->version == ARM_SMMU_V2 && id & ARM_SMMU_ID0_EXIDS) {
1794 smmu->features |= ARM_SMMU_FEAT_EXIDS;
1795 size = 1 << 16;
1796 } else {
1797 size = 1 << FIELD_GET(ARM_SMMU_ID0_NUMSIDB, id);
1798 }
1799 smmu->streamid_mask = size - 1;
1800 if (id & ARM_SMMU_ID0_SMS) {
1801 smmu->features |= ARM_SMMU_FEAT_STREAM_MATCH;
1802 size = FIELD_GET(ARM_SMMU_ID0_NUMSMRG, id);
1803 if (size == 0) {
1804 dev_err(smmu->dev,
1805 "stream-matching supported, but no SMRs present!\n");
1806 return -ENODEV;
1807 }
1808
1809 /* Zero-initialised to mark as invalid */
1810 smmu->smrs = devm_kcalloc(smmu->dev, size, sizeof(*smmu->smrs),
1811 GFP_KERNEL);
1812 if (!smmu->smrs)
1813 return -ENOMEM;
1814
1815 dev_notice(smmu->dev,
1816 "\tstream matching with %u register groups", size);
1817 }
1818 /* s2cr->type == 0 means translation, so initialise explicitly */
1819 smmu->s2crs = devm_kmalloc_array(smmu->dev, size, sizeof(*smmu->s2crs),
1820 GFP_KERNEL);
1821 if (!smmu->s2crs)
1822 return -ENOMEM;
1823 for (i = 0; i < size; i++)
1824 smmu->s2crs[i] = s2cr_init_val;
1825
1826 smmu->num_mapping_groups = size;
1827 mutex_init(&smmu->stream_map_mutex);
1828 spin_lock_init(&smmu->global_sync_lock);
1829
1830 if (smmu->version < ARM_SMMU_V2 ||
1831 !(id & ARM_SMMU_ID0_PTFS_NO_AARCH32)) {
1832 smmu->features |= ARM_SMMU_FEAT_FMT_AARCH32_L;
1833 if (!(id & ARM_SMMU_ID0_PTFS_NO_AARCH32S))
1834 smmu->features |= ARM_SMMU_FEAT_FMT_AARCH32_S;
1835 }
1836
1837 /* ID1 */
1838 id = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_ID1);
1839 smmu->pgshift = (id & ARM_SMMU_ID1_PAGESIZE) ? 16 : 12;
1840
1841 /* Check for size mismatch of SMMU address space from mapped region */
1842 size = 1 << (FIELD_GET(ARM_SMMU_ID1_NUMPAGENDXB, id) + 1);
1843 if (smmu->numpage != 2 * size << smmu->pgshift)
1844 dev_warn(smmu->dev,
1845 "SMMU address space size (0x%x) differs from mapped region size (0x%x)!\n",
1846 2 * size << smmu->pgshift, smmu->numpage);
1847 /* Now properly encode NUMPAGE to subsequently derive SMMU_CB_BASE */
1848 smmu->numpage = size;
1849
1850 smmu->num_s2_context_banks = FIELD_GET(ARM_SMMU_ID1_NUMS2CB, id);
1851 smmu->num_context_banks = FIELD_GET(ARM_SMMU_ID1_NUMCB, id);
1852 if (smmu->num_s2_context_banks > smmu->num_context_banks) {
1853 dev_err(smmu->dev, "impossible number of S2 context banks!\n");
1854 return -ENODEV;
1855 }
1856 dev_notice(smmu->dev, "\t%u context banks (%u stage-2 only)\n",
1857 smmu->num_context_banks, smmu->num_s2_context_banks);
1858 smmu->cbs = devm_kcalloc(smmu->dev, smmu->num_context_banks,
1859 sizeof(*smmu->cbs), GFP_KERNEL);
1860 if (!smmu->cbs)
1861 return -ENOMEM;
1862
1863 /* ID2 */
1864 id = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_ID2);
1865 size = arm_smmu_id_size_to_bits(FIELD_GET(ARM_SMMU_ID2_IAS, id));
1866 smmu->ipa_size = size;
1867
1868 /* The output mask is also applied for bypass */
1869 size = arm_smmu_id_size_to_bits(FIELD_GET(ARM_SMMU_ID2_OAS, id));
1870 smmu->pa_size = size;
1871
1872 if (id & ARM_SMMU_ID2_VMID16)
1873 smmu->features |= ARM_SMMU_FEAT_VMID16;
1874
1875 /*
1876 * What the page table walker can address actually depends on which
1877 * descriptor format is in use, but since a) we don't know that yet,
1878 * and b) it can vary per context bank, this will have to do...
1879 */
1880 if (dma_set_mask_and_coherent(smmu->dev, DMA_BIT_MASK(size)))
1881 dev_warn(smmu->dev,
1882 "failed to set DMA mask for table walker\n");
1883
1884 if (smmu->version < ARM_SMMU_V2) {
1885 smmu->va_size = smmu->ipa_size;
1886 if (smmu->version == ARM_SMMU_V1_64K)
1887 smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_64K;
1888 } else {
1889 size = FIELD_GET(ARM_SMMU_ID2_UBS, id);
1890 smmu->va_size = arm_smmu_id_size_to_bits(size);
1891 if (id & ARM_SMMU_ID2_PTFS_4K)
1892 smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_4K;
1893 if (id & ARM_SMMU_ID2_PTFS_16K)
1894 smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_16K;
1895 if (id & ARM_SMMU_ID2_PTFS_64K)
1896 smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_64K;
1897 }
1898
1899 if (smmu->impl && smmu->impl->cfg_probe) {
1900 ret = smmu->impl->cfg_probe(smmu);
1901 if (ret)
1902 return ret;
1903 }
1904
1905 /* Now we've corralled the various formats, what'll it do? */
1906 if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH32_S)
1907 smmu->pgsize_bitmap |= SZ_4K | SZ_64K | SZ_1M | SZ_16M;
1908 if (smmu->features &
1909 (ARM_SMMU_FEAT_FMT_AARCH32_L | ARM_SMMU_FEAT_FMT_AARCH64_4K))
1910 smmu->pgsize_bitmap |= SZ_4K | SZ_2M | SZ_1G;
1911 if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH64_16K)
1912 smmu->pgsize_bitmap |= SZ_16K | SZ_32M;
1913 if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH64_64K)
1914 smmu->pgsize_bitmap |= SZ_64K | SZ_512M;
1915
1916 if (arm_smmu_ops.pgsize_bitmap == -1UL)
1917 arm_smmu_ops.pgsize_bitmap = smmu->pgsize_bitmap;
1918 else
1919 arm_smmu_ops.pgsize_bitmap |= smmu->pgsize_bitmap;
1920 dev_notice(smmu->dev, "\tSupported page sizes: 0x%08lx\n",
1921 smmu->pgsize_bitmap);
1922
1923
1924 if (smmu->features & ARM_SMMU_FEAT_TRANS_S1)
1925 dev_notice(smmu->dev, "\tStage-1: %lu-bit VA -> %lu-bit IPA\n",
1926 smmu->va_size, smmu->ipa_size);
1927
1928 if (smmu->features & ARM_SMMU_FEAT_TRANS_S2)
1929 dev_notice(smmu->dev, "\tStage-2: %lu-bit IPA -> %lu-bit PA\n",
1930 smmu->ipa_size, smmu->pa_size);
1931
1932 return 0;
1933 }
1934
1935 struct arm_smmu_match_data {
1936 enum arm_smmu_arch_version version;
1937 enum arm_smmu_implementation model;
1938 };
1939
1940 #define ARM_SMMU_MATCH_DATA(name, ver, imp) \
1941 static const struct arm_smmu_match_data name = { .version = ver, .model = imp }
1942
1943 ARM_SMMU_MATCH_DATA(smmu_generic_v1, ARM_SMMU_V1, GENERIC_SMMU);
1944 ARM_SMMU_MATCH_DATA(smmu_generic_v2, ARM_SMMU_V2, GENERIC_SMMU);
1945 ARM_SMMU_MATCH_DATA(arm_mmu401, ARM_SMMU_V1_64K, GENERIC_SMMU);
1946 ARM_SMMU_MATCH_DATA(arm_mmu500, ARM_SMMU_V2, ARM_MMU500);
1947 ARM_SMMU_MATCH_DATA(cavium_smmuv2, ARM_SMMU_V2, CAVIUM_SMMUV2);
1948 ARM_SMMU_MATCH_DATA(qcom_smmuv2, ARM_SMMU_V2, QCOM_SMMUV2);
1949
1950 static const struct of_device_id arm_smmu_of_match[] = {
1951 { .compatible = "arm,smmu-v1", .data = &smmu_generic_v1 },
1952 { .compatible = "arm,smmu-v2", .data = &smmu_generic_v2 },
1953 { .compatible = "arm,mmu-400", .data = &smmu_generic_v1 },
1954 { .compatible = "arm,mmu-401", .data = &arm_mmu401 },
1955 { .compatible = "arm,mmu-500", .data = &arm_mmu500 },
1956 { .compatible = "cavium,smmu-v2", .data = &cavium_smmuv2 },
1957 { .compatible = "nvidia,smmu-500", .data = &arm_mmu500 },
1958 { .compatible = "qcom,smmu-v2", .data = &qcom_smmuv2 },
1959 { },
1960 };
1961 MODULE_DEVICE_TABLE(of, arm_smmu_of_match);
1962
1963 #ifdef CONFIG_ACPI
acpi_smmu_get_data(u32 model,struct arm_smmu_device * smmu)1964 static int acpi_smmu_get_data(u32 model, struct arm_smmu_device *smmu)
1965 {
1966 int ret = 0;
1967
1968 switch (model) {
1969 case ACPI_IORT_SMMU_V1:
1970 case ACPI_IORT_SMMU_CORELINK_MMU400:
1971 smmu->version = ARM_SMMU_V1;
1972 smmu->model = GENERIC_SMMU;
1973 break;
1974 case ACPI_IORT_SMMU_CORELINK_MMU401:
1975 smmu->version = ARM_SMMU_V1_64K;
1976 smmu->model = GENERIC_SMMU;
1977 break;
1978 case ACPI_IORT_SMMU_V2:
1979 smmu->version = ARM_SMMU_V2;
1980 smmu->model = GENERIC_SMMU;
1981 break;
1982 case ACPI_IORT_SMMU_CORELINK_MMU500:
1983 smmu->version = ARM_SMMU_V2;
1984 smmu->model = ARM_MMU500;
1985 break;
1986 case ACPI_IORT_SMMU_CAVIUM_THUNDERX:
1987 smmu->version = ARM_SMMU_V2;
1988 smmu->model = CAVIUM_SMMUV2;
1989 break;
1990 default:
1991 ret = -ENODEV;
1992 }
1993
1994 return ret;
1995 }
1996
arm_smmu_device_acpi_probe(struct arm_smmu_device * smmu,u32 * global_irqs,u32 * pmu_irqs)1997 static int arm_smmu_device_acpi_probe(struct arm_smmu_device *smmu,
1998 u32 *global_irqs, u32 *pmu_irqs)
1999 {
2000 struct device *dev = smmu->dev;
2001 struct acpi_iort_node *node =
2002 *(struct acpi_iort_node **)dev_get_platdata(dev);
2003 struct acpi_iort_smmu *iort_smmu;
2004 int ret;
2005
2006 /* Retrieve SMMU1/2 specific data */
2007 iort_smmu = (struct acpi_iort_smmu *)node->node_data;
2008
2009 ret = acpi_smmu_get_data(iort_smmu->model, smmu);
2010 if (ret < 0)
2011 return ret;
2012
2013 /* Ignore the configuration access interrupt */
2014 *global_irqs = 1;
2015 *pmu_irqs = 0;
2016
2017 if (iort_smmu->flags & ACPI_IORT_SMMU_COHERENT_WALK)
2018 smmu->features |= ARM_SMMU_FEAT_COHERENT_WALK;
2019
2020 return 0;
2021 }
2022 #else
arm_smmu_device_acpi_probe(struct arm_smmu_device * smmu,u32 * global_irqs,u32 * pmu_irqs)2023 static inline int arm_smmu_device_acpi_probe(struct arm_smmu_device *smmu,
2024 u32 *global_irqs, u32 *pmu_irqs)
2025 {
2026 return -ENODEV;
2027 }
2028 #endif
2029
arm_smmu_device_dt_probe(struct arm_smmu_device * smmu,u32 * global_irqs,u32 * pmu_irqs)2030 static int arm_smmu_device_dt_probe(struct arm_smmu_device *smmu,
2031 u32 *global_irqs, u32 *pmu_irqs)
2032 {
2033 const struct arm_smmu_match_data *data;
2034 struct device *dev = smmu->dev;
2035 bool legacy_binding;
2036
2037 if (of_property_read_u32(dev->of_node, "#global-interrupts", global_irqs))
2038 return dev_err_probe(dev, -ENODEV,
2039 "missing #global-interrupts property\n");
2040 *pmu_irqs = 0;
2041
2042 data = of_device_get_match_data(dev);
2043 smmu->version = data->version;
2044 smmu->model = data->model;
2045
2046 legacy_binding = of_find_property(dev->of_node, "mmu-masters", NULL);
2047 if (legacy_binding && !using_generic_binding) {
2048 if (!using_legacy_binding) {
2049 pr_notice("deprecated \"mmu-masters\" DT property in use; %s support unavailable\n",
2050 IS_ENABLED(CONFIG_ARM_SMMU_LEGACY_DT_BINDINGS) ? "DMA API" : "SMMU");
2051 }
2052 using_legacy_binding = true;
2053 } else if (!legacy_binding && !using_legacy_binding) {
2054 using_generic_binding = true;
2055 } else {
2056 dev_err(dev, "not probing due to mismatched DT properties\n");
2057 return -ENODEV;
2058 }
2059
2060 if (of_dma_is_coherent(dev->of_node))
2061 smmu->features |= ARM_SMMU_FEAT_COHERENT_WALK;
2062
2063 return 0;
2064 }
2065
arm_smmu_rmr_install_bypass_smr(struct arm_smmu_device * smmu)2066 static void arm_smmu_rmr_install_bypass_smr(struct arm_smmu_device *smmu)
2067 {
2068 struct list_head rmr_list;
2069 struct iommu_resv_region *e;
2070 int idx, cnt = 0;
2071 u32 reg;
2072
2073 INIT_LIST_HEAD(&rmr_list);
2074 iort_get_rmr_sids(dev_fwnode(smmu->dev), &rmr_list);
2075
2076 /*
2077 * Rather than trying to look at existing mappings that
2078 * are setup by the firmware and then invalidate the ones
2079 * that do no have matching RMR entries, just disable the
2080 * SMMU until it gets enabled again in the reset routine.
2081 */
2082 reg = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sCR0);
2083 reg |= ARM_SMMU_sCR0_CLIENTPD;
2084 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_sCR0, reg);
2085
2086 list_for_each_entry(e, &rmr_list, list) {
2087 struct iommu_iort_rmr_data *rmr;
2088 int i;
2089
2090 rmr = container_of(e, struct iommu_iort_rmr_data, rr);
2091 for (i = 0; i < rmr->num_sids; i++) {
2092 idx = arm_smmu_find_sme(smmu, rmr->sids[i], ~0);
2093 if (idx < 0)
2094 continue;
2095
2096 if (smmu->s2crs[idx].count == 0) {
2097 smmu->smrs[idx].id = rmr->sids[i];
2098 smmu->smrs[idx].mask = 0;
2099 smmu->smrs[idx].valid = true;
2100 }
2101 smmu->s2crs[idx].count++;
2102 smmu->s2crs[idx].type = S2CR_TYPE_BYPASS;
2103 smmu->s2crs[idx].privcfg = S2CR_PRIVCFG_DEFAULT;
2104
2105 cnt++;
2106 }
2107 }
2108
2109 dev_notice(smmu->dev, "\tpreserved %d boot mapping%s\n", cnt,
2110 str_plural(cnt));
2111 iort_put_rmr_sids(dev_fwnode(smmu->dev), &rmr_list);
2112 }
2113
arm_smmu_device_probe(struct platform_device * pdev)2114 static int arm_smmu_device_probe(struct platform_device *pdev)
2115 {
2116 struct resource *res;
2117 struct arm_smmu_device *smmu;
2118 struct device *dev = &pdev->dev;
2119 int num_irqs, i, err;
2120 u32 global_irqs, pmu_irqs;
2121 irqreturn_t (*global_fault)(int irq, void *dev);
2122
2123 smmu = devm_kzalloc(dev, sizeof(*smmu), GFP_KERNEL);
2124 if (!smmu) {
2125 dev_err(dev, "failed to allocate arm_smmu_device\n");
2126 return -ENOMEM;
2127 }
2128 smmu->dev = dev;
2129
2130 if (dev->of_node)
2131 err = arm_smmu_device_dt_probe(smmu, &global_irqs, &pmu_irqs);
2132 else
2133 err = arm_smmu_device_acpi_probe(smmu, &global_irqs, &pmu_irqs);
2134 if (err)
2135 return err;
2136
2137 smmu->base = devm_platform_get_and_ioremap_resource(pdev, 0, &res);
2138 if (IS_ERR(smmu->base))
2139 return PTR_ERR(smmu->base);
2140 smmu->ioaddr = res->start;
2141
2142 /*
2143 * The resource size should effectively match the value of SMMU_TOP;
2144 * stash that temporarily until we know PAGESIZE to validate it with.
2145 */
2146 smmu->numpage = resource_size(res);
2147
2148 smmu = arm_smmu_impl_init(smmu);
2149 if (IS_ERR(smmu))
2150 return PTR_ERR(smmu);
2151
2152 num_irqs = platform_irq_count(pdev);
2153
2154 smmu->num_context_irqs = num_irqs - global_irqs - pmu_irqs;
2155 if (smmu->num_context_irqs <= 0)
2156 return dev_err_probe(dev, -ENODEV,
2157 "found %d interrupts but expected at least %d\n",
2158 num_irqs, global_irqs + pmu_irqs + 1);
2159
2160 smmu->irqs = devm_kcalloc(dev, smmu->num_context_irqs,
2161 sizeof(*smmu->irqs), GFP_KERNEL);
2162 if (!smmu->irqs)
2163 return dev_err_probe(dev, -ENOMEM, "failed to allocate %d irqs\n",
2164 smmu->num_context_irqs);
2165
2166 for (i = 0; i < smmu->num_context_irqs; i++) {
2167 int irq = platform_get_irq(pdev, global_irqs + pmu_irqs + i);
2168
2169 if (irq < 0)
2170 return irq;
2171 smmu->irqs[i] = irq;
2172 }
2173
2174 err = devm_clk_bulk_get_all(dev, &smmu->clks);
2175 if (err < 0) {
2176 dev_err(dev, "failed to get clocks %d\n", err);
2177 return err;
2178 }
2179 smmu->num_clks = err;
2180
2181 err = clk_bulk_prepare_enable(smmu->num_clks, smmu->clks);
2182 if (err)
2183 return err;
2184
2185 err = arm_smmu_device_cfg_probe(smmu);
2186 if (err)
2187 return err;
2188
2189 if (smmu->version == ARM_SMMU_V2) {
2190 if (smmu->num_context_banks > smmu->num_context_irqs) {
2191 dev_err(dev,
2192 "found only %d context irq(s) but %d required\n",
2193 smmu->num_context_irqs, smmu->num_context_banks);
2194 return -ENODEV;
2195 }
2196
2197 /* Ignore superfluous interrupts */
2198 smmu->num_context_irqs = smmu->num_context_banks;
2199 }
2200
2201 if (smmu->impl && smmu->impl->global_fault)
2202 global_fault = smmu->impl->global_fault;
2203 else
2204 global_fault = arm_smmu_global_fault;
2205
2206 for (i = 0; i < global_irqs; i++) {
2207 int irq = platform_get_irq(pdev, i);
2208
2209 if (irq < 0)
2210 return irq;
2211
2212 err = devm_request_irq(dev, irq, global_fault, IRQF_SHARED,
2213 "arm-smmu global fault", smmu);
2214 if (err)
2215 return dev_err_probe(dev, err,
2216 "failed to request global IRQ %d (%u)\n",
2217 i, irq);
2218 }
2219
2220 platform_set_drvdata(pdev, smmu);
2221
2222 /* Check for RMRs and install bypass SMRs if any */
2223 arm_smmu_rmr_install_bypass_smr(smmu);
2224
2225 arm_smmu_device_reset(smmu);
2226 arm_smmu_test_smr_masks(smmu);
2227
2228 err = iommu_device_sysfs_add(&smmu->iommu, smmu->dev, NULL,
2229 "smmu.%pa", &smmu->ioaddr);
2230 if (err)
2231 return dev_err_probe(dev, err, "Failed to register iommu in sysfs\n");
2232
2233 err = iommu_device_register(&smmu->iommu, &arm_smmu_ops,
2234 using_legacy_binding ? NULL : dev);
2235 if (err) {
2236 iommu_device_sysfs_remove(&smmu->iommu);
2237 return dev_err_probe(dev, err, "Failed to register iommu\n");
2238 }
2239
2240 /*
2241 * We want to avoid touching dev->power.lock in fastpaths unless
2242 * it's really going to do something useful - pm_runtime_enabled()
2243 * can serve as an ideal proxy for that decision. So, conditionally
2244 * enable pm_runtime.
2245 */
2246 if (dev->pm_domain) {
2247 pm_runtime_set_active(dev);
2248 pm_runtime_enable(dev);
2249 arm_smmu_rpm_use_autosuspend(smmu);
2250 }
2251
2252 return 0;
2253 }
2254
arm_smmu_device_shutdown(struct platform_device * pdev)2255 static void arm_smmu_device_shutdown(struct platform_device *pdev)
2256 {
2257 struct arm_smmu_device *smmu = platform_get_drvdata(pdev);
2258
2259 if (!bitmap_empty(smmu->context_map, ARM_SMMU_MAX_CBS))
2260 dev_notice(&pdev->dev, "disabling translation\n");
2261
2262 arm_smmu_rpm_get(smmu);
2263 /* Turn the thing off */
2264 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_sCR0, ARM_SMMU_sCR0_CLIENTPD);
2265 arm_smmu_rpm_put(smmu);
2266
2267 if (pm_runtime_enabled(smmu->dev))
2268 pm_runtime_force_suspend(smmu->dev);
2269 else
2270 clk_bulk_disable(smmu->num_clks, smmu->clks);
2271
2272 clk_bulk_unprepare(smmu->num_clks, smmu->clks);
2273 }
2274
arm_smmu_device_remove(struct platform_device * pdev)2275 static void arm_smmu_device_remove(struct platform_device *pdev)
2276 {
2277 struct arm_smmu_device *smmu = platform_get_drvdata(pdev);
2278
2279 iommu_device_unregister(&smmu->iommu);
2280 iommu_device_sysfs_remove(&smmu->iommu);
2281
2282 arm_smmu_device_shutdown(pdev);
2283 }
2284
arm_smmu_runtime_resume(struct device * dev)2285 static int __maybe_unused arm_smmu_runtime_resume(struct device *dev)
2286 {
2287 struct arm_smmu_device *smmu = dev_get_drvdata(dev);
2288 int ret;
2289
2290 ret = clk_bulk_enable(smmu->num_clks, smmu->clks);
2291 if (ret)
2292 return ret;
2293
2294 arm_smmu_device_reset(smmu);
2295
2296 return 0;
2297 }
2298
arm_smmu_runtime_suspend(struct device * dev)2299 static int __maybe_unused arm_smmu_runtime_suspend(struct device *dev)
2300 {
2301 struct arm_smmu_device *smmu = dev_get_drvdata(dev);
2302
2303 clk_bulk_disable(smmu->num_clks, smmu->clks);
2304
2305 return 0;
2306 }
2307
arm_smmu_pm_resume(struct device * dev)2308 static int __maybe_unused arm_smmu_pm_resume(struct device *dev)
2309 {
2310 int ret;
2311 struct arm_smmu_device *smmu = dev_get_drvdata(dev);
2312
2313 ret = clk_bulk_prepare(smmu->num_clks, smmu->clks);
2314 if (ret)
2315 return ret;
2316
2317 if (pm_runtime_suspended(dev))
2318 return 0;
2319
2320 ret = arm_smmu_runtime_resume(dev);
2321 if (ret)
2322 clk_bulk_unprepare(smmu->num_clks, smmu->clks);
2323
2324 return ret;
2325 }
2326
arm_smmu_pm_suspend(struct device * dev)2327 static int __maybe_unused arm_smmu_pm_suspend(struct device *dev)
2328 {
2329 int ret = 0;
2330 struct arm_smmu_device *smmu = dev_get_drvdata(dev);
2331
2332 if (pm_runtime_suspended(dev))
2333 goto clk_unprepare;
2334
2335 ret = arm_smmu_runtime_suspend(dev);
2336 if (ret)
2337 return ret;
2338
2339 clk_unprepare:
2340 clk_bulk_unprepare(smmu->num_clks, smmu->clks);
2341 return ret;
2342 }
2343
2344 static const struct dev_pm_ops arm_smmu_pm_ops = {
2345 SET_SYSTEM_SLEEP_PM_OPS(arm_smmu_pm_suspend, arm_smmu_pm_resume)
2346 SET_RUNTIME_PM_OPS(arm_smmu_runtime_suspend,
2347 arm_smmu_runtime_resume, NULL)
2348 };
2349
2350 static struct platform_driver arm_smmu_driver = {
2351 .driver = {
2352 .name = "arm-smmu",
2353 .of_match_table = arm_smmu_of_match,
2354 .pm = &arm_smmu_pm_ops,
2355 .suppress_bind_attrs = true,
2356 },
2357 .probe = arm_smmu_device_probe,
2358 .remove = arm_smmu_device_remove,
2359 .shutdown = arm_smmu_device_shutdown,
2360 };
2361 module_platform_driver(arm_smmu_driver);
2362
2363 MODULE_DESCRIPTION("IOMMU API for ARM architected SMMU implementations");
2364 MODULE_AUTHOR("Will Deacon <will@kernel.org>");
2365 MODULE_ALIAS("platform:arm-smmu");
2366 MODULE_LICENSE("GPL v2");
2367