1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * IOMMU API for ARM architected SMMU implementations.
4 *
5 * Copyright (C) 2013 ARM Limited
6 *
7 * Author: Will Deacon <will.deacon@arm.com>
8 *
9 * This driver currently supports:
10 * - SMMUv1 and v2 implementations
11 * - Stream-matching and stream-indexing
12 * - v7/v8 long-descriptor format
13 * - Non-secure access to the SMMU
14 * - Context fault reporting
15 * - Extended Stream ID (16 bit)
16 */
17
18 #define pr_fmt(fmt) "arm-smmu: " fmt
19
20 #include <linux/acpi.h>
21 #include <linux/acpi_iort.h>
22 #include <linux/bitfield.h>
23 #include <linux/delay.h>
24 #include <linux/dma-mapping.h>
25 #include <linux/err.h>
26 #include <linux/interrupt.h>
27 #include <linux/io.h>
28 #include <linux/iopoll.h>
29 #include <linux/module.h>
30 #include <linux/of.h>
31 #include <linux/of_address.h>
32 #include <linux/pci.h>
33 #include <linux/platform_device.h>
34 #include <linux/pm_runtime.h>
35 #include <linux/ratelimit.h>
36 #include <linux/slab.h>
37 #include <linux/string_choices.h>
38
39 #include <linux/fsl/mc.h>
40
41 #include "arm-smmu.h"
42 #include "../../dma-iommu.h"
43
44 /*
45 * Apparently, some Qualcomm arm64 platforms which appear to expose their SMMU
46 * global register space are still, in fact, using a hypervisor to mediate it
47 * by trapping and emulating register accesses. Sadly, some deployed versions
48 * of said trapping code have bugs wherein they go horribly wrong for stores
49 * using r31 (i.e. XZR/WZR) as the source register.
50 */
51 #define QCOM_DUMMY_VAL -1
52
53 #define MSI_IOVA_BASE 0x8000000
54 #define MSI_IOVA_LENGTH 0x100000
55
56 static int force_stage;
57 module_param(force_stage, int, S_IRUGO);
58 MODULE_PARM_DESC(force_stage,
59 "Force SMMU mappings to be installed at a particular stage of translation. A value of '1' or '2' forces the corresponding stage. All other values are ignored (i.e. no stage is forced). Note that selecting a specific stage will disable support for nested translation.");
60 static bool disable_bypass =
61 IS_ENABLED(CONFIG_ARM_SMMU_DISABLE_BYPASS_BY_DEFAULT);
62 module_param(disable_bypass, bool, S_IRUGO);
63 MODULE_PARM_DESC(disable_bypass,
64 "Disable bypass streams such that incoming transactions from devices that are not attached to an iommu domain will report an abort back to the device and will not be allowed to pass through the SMMU.");
65
66 #define s2cr_init_val (struct arm_smmu_s2cr){ \
67 .type = disable_bypass ? S2CR_TYPE_FAULT : S2CR_TYPE_BYPASS, \
68 }
69
70 static bool using_legacy_binding, using_generic_binding;
71
arm_smmu_rpm_get(struct arm_smmu_device * smmu)72 static inline int arm_smmu_rpm_get(struct arm_smmu_device *smmu)
73 {
74 if (pm_runtime_enabled(smmu->dev))
75 return pm_runtime_resume_and_get(smmu->dev);
76
77 return 0;
78 }
79
arm_smmu_rpm_put(struct arm_smmu_device * smmu)80 static inline void arm_smmu_rpm_put(struct arm_smmu_device *smmu)
81 {
82 if (pm_runtime_enabled(smmu->dev)) {
83 pm_runtime_mark_last_busy(smmu->dev);
84 __pm_runtime_put_autosuspend(smmu->dev);
85
86 }
87 }
88
arm_smmu_rpm_use_autosuspend(struct arm_smmu_device * smmu)89 static void arm_smmu_rpm_use_autosuspend(struct arm_smmu_device *smmu)
90 {
91 /*
92 * Setup an autosuspend delay to avoid bouncing runpm state.
93 * Otherwise, if a driver for a suspended consumer device
94 * unmaps buffers, it will runpm resume/suspend for each one.
95 *
96 * For example, when used by a GPU device, when an application
97 * or game exits, it can trigger unmapping 100s or 1000s of
98 * buffers. With a runpm cycle for each buffer, that adds up
99 * to 5-10sec worth of reprogramming the context bank, while
100 * the system appears to be locked up to the user.
101 */
102 pm_runtime_set_autosuspend_delay(smmu->dev, 20);
103 pm_runtime_use_autosuspend(smmu->dev);
104 }
105
to_smmu_domain(struct iommu_domain * dom)106 static struct arm_smmu_domain *to_smmu_domain(struct iommu_domain *dom)
107 {
108 return container_of(dom, struct arm_smmu_domain, domain);
109 }
110
111 static struct platform_driver arm_smmu_driver;
112 static const struct iommu_ops arm_smmu_ops;
113
114 #ifdef CONFIG_ARM_SMMU_LEGACY_DT_BINDINGS
dev_get_dev_node(struct device * dev)115 static struct device_node *dev_get_dev_node(struct device *dev)
116 {
117 if (dev_is_pci(dev)) {
118 struct pci_bus *bus = to_pci_dev(dev)->bus;
119
120 while (!pci_is_root_bus(bus))
121 bus = bus->parent;
122 return of_node_get(bus->bridge->parent->of_node);
123 }
124
125 return of_node_get(dev->of_node);
126 }
127
__arm_smmu_get_pci_sid(struct pci_dev * pdev,u16 alias,void * data)128 static int __arm_smmu_get_pci_sid(struct pci_dev *pdev, u16 alias, void *data)
129 {
130 *((__be32 *)data) = cpu_to_be32(alias);
131 return 0; /* Continue walking */
132 }
133
__find_legacy_master_phandle(struct device * dev,void * data)134 static int __find_legacy_master_phandle(struct device *dev, void *data)
135 {
136 struct of_phandle_iterator *it = *(void **)data;
137 struct device_node *np = it->node;
138 int err;
139
140 of_for_each_phandle(it, err, dev->of_node, "mmu-masters",
141 "#stream-id-cells", -1)
142 if (it->node == np) {
143 *(void **)data = dev;
144 return 1;
145 }
146 it->node = np;
147 return err == -ENOENT ? 0 : err;
148 }
149
arm_smmu_register_legacy_master(struct device * dev,struct arm_smmu_device ** smmu)150 static int arm_smmu_register_legacy_master(struct device *dev,
151 struct arm_smmu_device **smmu)
152 {
153 struct device *smmu_dev;
154 struct device_node *np;
155 struct of_phandle_iterator it;
156 void *data = ⁢
157 u32 *sids;
158 __be32 pci_sid;
159 int err;
160
161 np = dev_get_dev_node(dev);
162 if (!np || !of_property_present(np, "#stream-id-cells")) {
163 of_node_put(np);
164 return -ENODEV;
165 }
166
167 it.node = np;
168 err = driver_for_each_device(&arm_smmu_driver.driver, NULL, &data,
169 __find_legacy_master_phandle);
170 smmu_dev = data;
171 of_node_put(np);
172 if (err == 0)
173 return -ENODEV;
174 if (err < 0)
175 return err;
176
177 if (dev_is_pci(dev)) {
178 /* "mmu-masters" assumes Stream ID == Requester ID */
179 pci_for_each_dma_alias(to_pci_dev(dev), __arm_smmu_get_pci_sid,
180 &pci_sid);
181 it.cur = &pci_sid;
182 it.cur_count = 1;
183 }
184
185 err = iommu_fwspec_init(dev, NULL);
186 if (err)
187 return err;
188
189 sids = kcalloc(it.cur_count, sizeof(*sids), GFP_KERNEL);
190 if (!sids)
191 return -ENOMEM;
192
193 *smmu = dev_get_drvdata(smmu_dev);
194 of_phandle_iterator_args(&it, sids, it.cur_count);
195 err = iommu_fwspec_add_ids(dev, sids, it.cur_count);
196 kfree(sids);
197 return err;
198 }
199 #else
arm_smmu_register_legacy_master(struct device * dev,struct arm_smmu_device ** smmu)200 static int arm_smmu_register_legacy_master(struct device *dev,
201 struct arm_smmu_device **smmu)
202 {
203 return -ENODEV;
204 }
205 #endif /* CONFIG_ARM_SMMU_LEGACY_DT_BINDINGS */
206
__arm_smmu_free_bitmap(unsigned long * map,int idx)207 static void __arm_smmu_free_bitmap(unsigned long *map, int idx)
208 {
209 clear_bit(idx, map);
210 }
211
212 /* Wait for any pending TLB invalidations to complete */
__arm_smmu_tlb_sync(struct arm_smmu_device * smmu,int page,int sync,int status)213 static void __arm_smmu_tlb_sync(struct arm_smmu_device *smmu, int page,
214 int sync, int status)
215 {
216 unsigned int spin_cnt, delay;
217 u32 reg;
218
219 if (smmu->impl && unlikely(smmu->impl->tlb_sync))
220 return smmu->impl->tlb_sync(smmu, page, sync, status);
221
222 arm_smmu_writel(smmu, page, sync, QCOM_DUMMY_VAL);
223 for (delay = 1; delay < TLB_LOOP_TIMEOUT; delay *= 2) {
224 for (spin_cnt = TLB_SPIN_COUNT; spin_cnt > 0; spin_cnt--) {
225 reg = arm_smmu_readl(smmu, page, status);
226 if (!(reg & ARM_SMMU_sTLBGSTATUS_GSACTIVE))
227 return;
228 cpu_relax();
229 }
230 udelay(delay);
231 }
232 dev_err_ratelimited(smmu->dev,
233 "TLB sync timed out -- SMMU may be deadlocked\n");
234 }
235
arm_smmu_tlb_sync_global(struct arm_smmu_device * smmu)236 static void arm_smmu_tlb_sync_global(struct arm_smmu_device *smmu)
237 {
238 unsigned long flags;
239
240 spin_lock_irqsave(&smmu->global_sync_lock, flags);
241 __arm_smmu_tlb_sync(smmu, ARM_SMMU_GR0, ARM_SMMU_GR0_sTLBGSYNC,
242 ARM_SMMU_GR0_sTLBGSTATUS);
243 spin_unlock_irqrestore(&smmu->global_sync_lock, flags);
244 }
245
arm_smmu_tlb_sync_context(struct arm_smmu_domain * smmu_domain)246 static void arm_smmu_tlb_sync_context(struct arm_smmu_domain *smmu_domain)
247 {
248 struct arm_smmu_device *smmu = smmu_domain->smmu;
249 unsigned long flags;
250
251 spin_lock_irqsave(&smmu_domain->cb_lock, flags);
252 __arm_smmu_tlb_sync(smmu, ARM_SMMU_CB(smmu, smmu_domain->cfg.cbndx),
253 ARM_SMMU_CB_TLBSYNC, ARM_SMMU_CB_TLBSTATUS);
254 spin_unlock_irqrestore(&smmu_domain->cb_lock, flags);
255 }
256
arm_smmu_tlb_inv_context_s1(void * cookie)257 static void arm_smmu_tlb_inv_context_s1(void *cookie)
258 {
259 struct arm_smmu_domain *smmu_domain = cookie;
260 /*
261 * The TLBI write may be relaxed, so ensure that PTEs cleared by the
262 * current CPU are visible beforehand.
263 */
264 wmb();
265 arm_smmu_cb_write(smmu_domain->smmu, smmu_domain->cfg.cbndx,
266 ARM_SMMU_CB_S1_TLBIASID, smmu_domain->cfg.asid);
267 arm_smmu_tlb_sync_context(smmu_domain);
268 }
269
arm_smmu_tlb_inv_context_s2(void * cookie)270 static void arm_smmu_tlb_inv_context_s2(void *cookie)
271 {
272 struct arm_smmu_domain *smmu_domain = cookie;
273 struct arm_smmu_device *smmu = smmu_domain->smmu;
274
275 /* See above */
276 wmb();
277 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_TLBIVMID, smmu_domain->cfg.vmid);
278 arm_smmu_tlb_sync_global(smmu);
279 }
280
arm_smmu_tlb_inv_range_s1(unsigned long iova,size_t size,size_t granule,void * cookie,int reg)281 static void arm_smmu_tlb_inv_range_s1(unsigned long iova, size_t size,
282 size_t granule, void *cookie, int reg)
283 {
284 struct arm_smmu_domain *smmu_domain = cookie;
285 struct arm_smmu_device *smmu = smmu_domain->smmu;
286 struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
287 int idx = cfg->cbndx;
288
289 if (smmu->features & ARM_SMMU_FEAT_COHERENT_WALK)
290 wmb();
291
292 if (cfg->fmt != ARM_SMMU_CTX_FMT_AARCH64) {
293 iova = (iova >> 12) << 12;
294 iova |= cfg->asid;
295 do {
296 arm_smmu_cb_write(smmu, idx, reg, iova);
297 iova += granule;
298 } while (size -= granule);
299 } else {
300 iova >>= 12;
301 iova |= (u64)cfg->asid << 48;
302 do {
303 arm_smmu_cb_writeq(smmu, idx, reg, iova);
304 iova += granule >> 12;
305 } while (size -= granule);
306 }
307 }
308
arm_smmu_tlb_inv_range_s2(unsigned long iova,size_t size,size_t granule,void * cookie,int reg)309 static void arm_smmu_tlb_inv_range_s2(unsigned long iova, size_t size,
310 size_t granule, void *cookie, int reg)
311 {
312 struct arm_smmu_domain *smmu_domain = cookie;
313 struct arm_smmu_device *smmu = smmu_domain->smmu;
314 int idx = smmu_domain->cfg.cbndx;
315
316 if (smmu->features & ARM_SMMU_FEAT_COHERENT_WALK)
317 wmb();
318
319 iova >>= 12;
320 do {
321 if (smmu_domain->cfg.fmt == ARM_SMMU_CTX_FMT_AARCH64)
322 arm_smmu_cb_writeq(smmu, idx, reg, iova);
323 else
324 arm_smmu_cb_write(smmu, idx, reg, iova);
325 iova += granule >> 12;
326 } while (size -= granule);
327 }
328
arm_smmu_tlb_inv_walk_s1(unsigned long iova,size_t size,size_t granule,void * cookie)329 static void arm_smmu_tlb_inv_walk_s1(unsigned long iova, size_t size,
330 size_t granule, void *cookie)
331 {
332 struct arm_smmu_domain *smmu_domain = cookie;
333 struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
334
335 if (cfg->flush_walk_prefer_tlbiasid) {
336 arm_smmu_tlb_inv_context_s1(cookie);
337 } else {
338 arm_smmu_tlb_inv_range_s1(iova, size, granule, cookie,
339 ARM_SMMU_CB_S1_TLBIVA);
340 arm_smmu_tlb_sync_context(cookie);
341 }
342 }
343
arm_smmu_tlb_add_page_s1(struct iommu_iotlb_gather * gather,unsigned long iova,size_t granule,void * cookie)344 static void arm_smmu_tlb_add_page_s1(struct iommu_iotlb_gather *gather,
345 unsigned long iova, size_t granule,
346 void *cookie)
347 {
348 arm_smmu_tlb_inv_range_s1(iova, granule, granule, cookie,
349 ARM_SMMU_CB_S1_TLBIVAL);
350 }
351
arm_smmu_tlb_inv_walk_s2(unsigned long iova,size_t size,size_t granule,void * cookie)352 static void arm_smmu_tlb_inv_walk_s2(unsigned long iova, size_t size,
353 size_t granule, void *cookie)
354 {
355 arm_smmu_tlb_inv_range_s2(iova, size, granule, cookie,
356 ARM_SMMU_CB_S2_TLBIIPAS2);
357 arm_smmu_tlb_sync_context(cookie);
358 }
359
arm_smmu_tlb_add_page_s2(struct iommu_iotlb_gather * gather,unsigned long iova,size_t granule,void * cookie)360 static void arm_smmu_tlb_add_page_s2(struct iommu_iotlb_gather *gather,
361 unsigned long iova, size_t granule,
362 void *cookie)
363 {
364 arm_smmu_tlb_inv_range_s2(iova, granule, granule, cookie,
365 ARM_SMMU_CB_S2_TLBIIPAS2L);
366 }
367
arm_smmu_tlb_inv_walk_s2_v1(unsigned long iova,size_t size,size_t granule,void * cookie)368 static void arm_smmu_tlb_inv_walk_s2_v1(unsigned long iova, size_t size,
369 size_t granule, void *cookie)
370 {
371 arm_smmu_tlb_inv_context_s2(cookie);
372 }
373 /*
374 * On MMU-401 at least, the cost of firing off multiple TLBIVMIDs appears
375 * almost negligible, but the benefit of getting the first one in as far ahead
376 * of the sync as possible is significant, hence we don't just make this a
377 * no-op and call arm_smmu_tlb_inv_context_s2() from .iotlb_sync as you might
378 * think.
379 */
arm_smmu_tlb_add_page_s2_v1(struct iommu_iotlb_gather * gather,unsigned long iova,size_t granule,void * cookie)380 static void arm_smmu_tlb_add_page_s2_v1(struct iommu_iotlb_gather *gather,
381 unsigned long iova, size_t granule,
382 void *cookie)
383 {
384 struct arm_smmu_domain *smmu_domain = cookie;
385 struct arm_smmu_device *smmu = smmu_domain->smmu;
386
387 if (smmu->features & ARM_SMMU_FEAT_COHERENT_WALK)
388 wmb();
389
390 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_TLBIVMID, smmu_domain->cfg.vmid);
391 }
392
393 static const struct iommu_flush_ops arm_smmu_s1_tlb_ops = {
394 .tlb_flush_all = arm_smmu_tlb_inv_context_s1,
395 .tlb_flush_walk = arm_smmu_tlb_inv_walk_s1,
396 .tlb_add_page = arm_smmu_tlb_add_page_s1,
397 };
398
399 static const struct iommu_flush_ops arm_smmu_s2_tlb_ops_v2 = {
400 .tlb_flush_all = arm_smmu_tlb_inv_context_s2,
401 .tlb_flush_walk = arm_smmu_tlb_inv_walk_s2,
402 .tlb_add_page = arm_smmu_tlb_add_page_s2,
403 };
404
405 static const struct iommu_flush_ops arm_smmu_s2_tlb_ops_v1 = {
406 .tlb_flush_all = arm_smmu_tlb_inv_context_s2,
407 .tlb_flush_walk = arm_smmu_tlb_inv_walk_s2_v1,
408 .tlb_add_page = arm_smmu_tlb_add_page_s2_v1,
409 };
410
411
arm_smmu_read_context_fault_info(struct arm_smmu_device * smmu,int idx,struct arm_smmu_context_fault_info * cfi)412 void arm_smmu_read_context_fault_info(struct arm_smmu_device *smmu, int idx,
413 struct arm_smmu_context_fault_info *cfi)
414 {
415 cfi->iova = arm_smmu_cb_readq(smmu, idx, ARM_SMMU_CB_FAR);
416 cfi->fsr = arm_smmu_cb_read(smmu, idx, ARM_SMMU_CB_FSR);
417 cfi->fsynr = arm_smmu_cb_read(smmu, idx, ARM_SMMU_CB_FSYNR0);
418 cfi->cbfrsynra = arm_smmu_gr1_read(smmu, ARM_SMMU_GR1_CBFRSYNRA(idx));
419 }
420
arm_smmu_print_context_fault_info(struct arm_smmu_device * smmu,int idx,const struct arm_smmu_context_fault_info * cfi)421 void arm_smmu_print_context_fault_info(struct arm_smmu_device *smmu, int idx,
422 const struct arm_smmu_context_fault_info *cfi)
423 {
424 dev_err(smmu->dev,
425 "Unhandled context fault: fsr=0x%x, iova=0x%08lx, fsynr=0x%x, cbfrsynra=0x%x, cb=%d\n",
426 cfi->fsr, cfi->iova, cfi->fsynr, cfi->cbfrsynra, idx);
427
428 dev_err(smmu->dev, "FSR = %08x [%s%sFormat=%u%s%s%s%s%s%s%s%s], SID=0x%x\n",
429 cfi->fsr,
430 (cfi->fsr & ARM_SMMU_CB_FSR_MULTI) ? "MULTI " : "",
431 (cfi->fsr & ARM_SMMU_CB_FSR_SS) ? "SS " : "",
432 (u32)FIELD_GET(ARM_SMMU_CB_FSR_FORMAT, cfi->fsr),
433 (cfi->fsr & ARM_SMMU_CB_FSR_UUT) ? " UUT" : "",
434 (cfi->fsr & ARM_SMMU_CB_FSR_ASF) ? " ASF" : "",
435 (cfi->fsr & ARM_SMMU_CB_FSR_TLBLKF) ? " TLBLKF" : "",
436 (cfi->fsr & ARM_SMMU_CB_FSR_TLBMCF) ? " TLBMCF" : "",
437 (cfi->fsr & ARM_SMMU_CB_FSR_EF) ? " EF" : "",
438 (cfi->fsr & ARM_SMMU_CB_FSR_PF) ? " PF" : "",
439 (cfi->fsr & ARM_SMMU_CB_FSR_AFF) ? " AFF" : "",
440 (cfi->fsr & ARM_SMMU_CB_FSR_TF) ? " TF" : "",
441 cfi->cbfrsynra);
442
443 dev_err(smmu->dev, "FSYNR0 = %08x [S1CBNDX=%u%s%s%s%s%s%s PLVL=%u]\n",
444 cfi->fsynr,
445 (u32)FIELD_GET(ARM_SMMU_CB_FSYNR0_S1CBNDX, cfi->fsynr),
446 (cfi->fsynr & ARM_SMMU_CB_FSYNR0_AFR) ? " AFR" : "",
447 (cfi->fsynr & ARM_SMMU_CB_FSYNR0_PTWF) ? " PTWF" : "",
448 (cfi->fsynr & ARM_SMMU_CB_FSYNR0_NSATTR) ? " NSATTR" : "",
449 (cfi->fsynr & ARM_SMMU_CB_FSYNR0_IND) ? " IND" : "",
450 (cfi->fsynr & ARM_SMMU_CB_FSYNR0_PNU) ? " PNU" : "",
451 (cfi->fsynr & ARM_SMMU_CB_FSYNR0_WNR) ? " WNR" : "",
452 (u32)FIELD_GET(ARM_SMMU_CB_FSYNR0_PLVL, cfi->fsynr));
453 }
454
arm_smmu_context_fault(int irq,void * dev)455 static irqreturn_t arm_smmu_context_fault(int irq, void *dev)
456 {
457 struct arm_smmu_context_fault_info cfi;
458 struct arm_smmu_domain *smmu_domain = dev;
459 struct arm_smmu_device *smmu = smmu_domain->smmu;
460 static DEFINE_RATELIMIT_STATE(rs, DEFAULT_RATELIMIT_INTERVAL,
461 DEFAULT_RATELIMIT_BURST);
462 int idx = smmu_domain->cfg.cbndx;
463 int ret;
464
465 arm_smmu_read_context_fault_info(smmu, idx, &cfi);
466
467 if (!(cfi.fsr & ARM_SMMU_CB_FSR_FAULT))
468 return IRQ_NONE;
469
470 ret = report_iommu_fault(&smmu_domain->domain, NULL, cfi.iova,
471 cfi.fsynr & ARM_SMMU_CB_FSYNR0_WNR ? IOMMU_FAULT_WRITE : IOMMU_FAULT_READ);
472
473 if (ret == -ENOSYS && __ratelimit(&rs))
474 arm_smmu_print_context_fault_info(smmu, idx, &cfi);
475
476 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_FSR, cfi.fsr);
477
478 if (cfi.fsr & ARM_SMMU_CB_FSR_SS) {
479 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_RESUME,
480 ret == -EAGAIN ? 0 : ARM_SMMU_RESUME_TERMINATE);
481 }
482
483 return IRQ_HANDLED;
484 }
485
arm_smmu_global_fault(int irq,void * dev)486 static irqreturn_t arm_smmu_global_fault(int irq, void *dev)
487 {
488 u32 gfsr, gfsynr0, gfsynr1, gfsynr2;
489 struct arm_smmu_device *smmu = dev;
490 static DEFINE_RATELIMIT_STATE(rs, DEFAULT_RATELIMIT_INTERVAL,
491 DEFAULT_RATELIMIT_BURST);
492
493 gfsr = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sGFSR);
494 gfsynr0 = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sGFSYNR0);
495 gfsynr1 = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sGFSYNR1);
496 gfsynr2 = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sGFSYNR2);
497
498 if (!gfsr)
499 return IRQ_NONE;
500
501 if (__ratelimit(&rs)) {
502 if (IS_ENABLED(CONFIG_ARM_SMMU_DISABLE_BYPASS_BY_DEFAULT) &&
503 (gfsr & ARM_SMMU_sGFSR_USF))
504 dev_err(smmu->dev,
505 "Blocked unknown Stream ID 0x%hx; boot with \"arm-smmu.disable_bypass=0\" to allow, but this may have security implications\n",
506 (u16)gfsynr1);
507 else
508 dev_err(smmu->dev,
509 "Unexpected global fault, this could be serious\n");
510 dev_err(smmu->dev,
511 "\tGFSR 0x%08x, GFSYNR0 0x%08x, GFSYNR1 0x%08x, GFSYNR2 0x%08x\n",
512 gfsr, gfsynr0, gfsynr1, gfsynr2);
513 }
514
515 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_sGFSR, gfsr);
516 return IRQ_HANDLED;
517 }
518
arm_smmu_init_context_bank(struct arm_smmu_domain * smmu_domain,struct io_pgtable_cfg * pgtbl_cfg)519 static void arm_smmu_init_context_bank(struct arm_smmu_domain *smmu_domain,
520 struct io_pgtable_cfg *pgtbl_cfg)
521 {
522 struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
523 struct arm_smmu_cb *cb = &smmu_domain->smmu->cbs[cfg->cbndx];
524 bool stage1 = cfg->cbar != CBAR_TYPE_S2_TRANS;
525
526 cb->cfg = cfg;
527
528 /* TCR */
529 if (stage1) {
530 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) {
531 cb->tcr[0] = pgtbl_cfg->arm_v7s_cfg.tcr;
532 } else {
533 cb->tcr[0] = arm_smmu_lpae_tcr(pgtbl_cfg);
534 cb->tcr[1] = arm_smmu_lpae_tcr2(pgtbl_cfg);
535 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64)
536 cb->tcr[1] |= ARM_SMMU_TCR2_AS;
537 else
538 cb->tcr[0] |= ARM_SMMU_TCR_EAE;
539 }
540 } else {
541 cb->tcr[0] = arm_smmu_lpae_vtcr(pgtbl_cfg);
542 }
543
544 /* TTBRs */
545 if (stage1) {
546 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) {
547 cb->ttbr[0] = pgtbl_cfg->arm_v7s_cfg.ttbr;
548 cb->ttbr[1] = 0;
549 } else {
550 cb->ttbr[0] = FIELD_PREP(ARM_SMMU_TTBRn_ASID,
551 cfg->asid);
552 cb->ttbr[1] = FIELD_PREP(ARM_SMMU_TTBRn_ASID,
553 cfg->asid);
554
555 if (pgtbl_cfg->quirks & IO_PGTABLE_QUIRK_ARM_TTBR1)
556 cb->ttbr[1] |= pgtbl_cfg->arm_lpae_s1_cfg.ttbr;
557 else
558 cb->ttbr[0] |= pgtbl_cfg->arm_lpae_s1_cfg.ttbr;
559 }
560 } else {
561 cb->ttbr[0] = pgtbl_cfg->arm_lpae_s2_cfg.vttbr;
562 }
563
564 /* MAIRs (stage-1 only) */
565 if (stage1) {
566 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) {
567 cb->mair[0] = pgtbl_cfg->arm_v7s_cfg.prrr;
568 cb->mair[1] = pgtbl_cfg->arm_v7s_cfg.nmrr;
569 } else {
570 cb->mair[0] = pgtbl_cfg->arm_lpae_s1_cfg.mair;
571 cb->mair[1] = pgtbl_cfg->arm_lpae_s1_cfg.mair >> 32;
572 }
573 }
574 }
575
arm_smmu_write_context_bank(struct arm_smmu_device * smmu,int idx)576 void arm_smmu_write_context_bank(struct arm_smmu_device *smmu, int idx)
577 {
578 u32 reg;
579 bool stage1;
580 struct arm_smmu_cb *cb = &smmu->cbs[idx];
581 struct arm_smmu_cfg *cfg = cb->cfg;
582
583 /* Unassigned context banks only need disabling */
584 if (!cfg) {
585 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_SCTLR, 0);
586 return;
587 }
588
589 stage1 = cfg->cbar != CBAR_TYPE_S2_TRANS;
590
591 /* CBA2R */
592 if (smmu->version > ARM_SMMU_V1) {
593 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64)
594 reg = ARM_SMMU_CBA2R_VA64;
595 else
596 reg = 0;
597 /* 16-bit VMIDs live in CBA2R */
598 if (smmu->features & ARM_SMMU_FEAT_VMID16)
599 reg |= FIELD_PREP(ARM_SMMU_CBA2R_VMID16, cfg->vmid);
600
601 arm_smmu_gr1_write(smmu, ARM_SMMU_GR1_CBA2R(idx), reg);
602 }
603
604 /* CBAR */
605 reg = FIELD_PREP(ARM_SMMU_CBAR_TYPE, cfg->cbar);
606 if (smmu->version < ARM_SMMU_V2)
607 reg |= FIELD_PREP(ARM_SMMU_CBAR_IRPTNDX, cfg->irptndx);
608
609 /*
610 * Use the weakest shareability/memory types, so they are
611 * overridden by the ttbcr/pte.
612 */
613 if (stage1) {
614 reg |= FIELD_PREP(ARM_SMMU_CBAR_S1_BPSHCFG,
615 ARM_SMMU_CBAR_S1_BPSHCFG_NSH) |
616 FIELD_PREP(ARM_SMMU_CBAR_S1_MEMATTR,
617 ARM_SMMU_CBAR_S1_MEMATTR_WB);
618 } else if (!(smmu->features & ARM_SMMU_FEAT_VMID16)) {
619 /* 8-bit VMIDs live in CBAR */
620 reg |= FIELD_PREP(ARM_SMMU_CBAR_VMID, cfg->vmid);
621 }
622 arm_smmu_gr1_write(smmu, ARM_SMMU_GR1_CBAR(idx), reg);
623
624 /*
625 * TCR
626 * We must write this before the TTBRs, since it determines the
627 * access behaviour of some fields (in particular, ASID[15:8]).
628 */
629 if (stage1 && smmu->version > ARM_SMMU_V1)
630 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_TCR2, cb->tcr[1]);
631 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_TCR, cb->tcr[0]);
632
633 /* TTBRs */
634 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) {
635 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_CONTEXTIDR, cfg->asid);
636 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_TTBR0, cb->ttbr[0]);
637 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_TTBR1, cb->ttbr[1]);
638 } else {
639 arm_smmu_cb_writeq(smmu, idx, ARM_SMMU_CB_TTBR0, cb->ttbr[0]);
640 if (stage1)
641 arm_smmu_cb_writeq(smmu, idx, ARM_SMMU_CB_TTBR1,
642 cb->ttbr[1]);
643 }
644
645 /* MAIRs (stage-1 only) */
646 if (stage1) {
647 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_S1_MAIR0, cb->mair[0]);
648 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_S1_MAIR1, cb->mair[1]);
649 }
650
651 /* SCTLR */
652 reg = ARM_SMMU_SCTLR_CFIE | ARM_SMMU_SCTLR_CFRE | ARM_SMMU_SCTLR_AFE |
653 ARM_SMMU_SCTLR_TRE | ARM_SMMU_SCTLR_M;
654 if (stage1)
655 reg |= ARM_SMMU_SCTLR_S1_ASIDPNE;
656 if (IS_ENABLED(CONFIG_CPU_BIG_ENDIAN))
657 reg |= ARM_SMMU_SCTLR_E;
658
659 if (smmu->impl && smmu->impl->write_sctlr)
660 smmu->impl->write_sctlr(smmu, idx, reg);
661 else
662 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_SCTLR, reg);
663 }
664
arm_smmu_alloc_context_bank(struct arm_smmu_domain * smmu_domain,struct arm_smmu_device * smmu,struct device * dev,unsigned int start)665 static int arm_smmu_alloc_context_bank(struct arm_smmu_domain *smmu_domain,
666 struct arm_smmu_device *smmu,
667 struct device *dev, unsigned int start)
668 {
669 if (smmu->impl && smmu->impl->alloc_context_bank)
670 return smmu->impl->alloc_context_bank(smmu_domain, smmu, dev, start);
671
672 return __arm_smmu_alloc_bitmap(smmu->context_map, start, smmu->num_context_banks);
673 }
674
arm_smmu_init_domain_context(struct arm_smmu_domain * smmu_domain,struct arm_smmu_device * smmu,struct device * dev)675 static int arm_smmu_init_domain_context(struct arm_smmu_domain *smmu_domain,
676 struct arm_smmu_device *smmu,
677 struct device *dev)
678 {
679 int irq, start, ret = 0;
680 unsigned long ias, oas;
681 struct io_pgtable_ops *pgtbl_ops;
682 struct io_pgtable_cfg pgtbl_cfg;
683 enum io_pgtable_fmt fmt;
684 struct iommu_domain *domain = &smmu_domain->domain;
685 struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
686 irqreturn_t (*context_fault)(int irq, void *dev);
687
688 mutex_lock(&smmu_domain->init_mutex);
689 if (smmu_domain->smmu)
690 goto out_unlock;
691
692 /*
693 * Mapping the requested stage onto what we support is surprisingly
694 * complicated, mainly because the spec allows S1+S2 SMMUs without
695 * support for nested translation. That means we end up with the
696 * following table:
697 *
698 * Requested Supported Actual
699 * S1 N S1
700 * S1 S1+S2 S1
701 * S1 S2 S2
702 * S1 S1 S1
703 * N N N
704 * N S1+S2 S2
705 * N S2 S2
706 * N S1 S1
707 *
708 * Note that you can't actually request stage-2 mappings.
709 */
710 if (!(smmu->features & ARM_SMMU_FEAT_TRANS_S1))
711 smmu_domain->stage = ARM_SMMU_DOMAIN_S2;
712 if (!(smmu->features & ARM_SMMU_FEAT_TRANS_S2))
713 smmu_domain->stage = ARM_SMMU_DOMAIN_S1;
714
715 /*
716 * Choosing a suitable context format is even more fiddly. Until we
717 * grow some way for the caller to express a preference, and/or move
718 * the decision into the io-pgtable code where it arguably belongs,
719 * just aim for the closest thing to the rest of the system, and hope
720 * that the hardware isn't esoteric enough that we can't assume AArch64
721 * support to be a superset of AArch32 support...
722 */
723 if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH32_L)
724 cfg->fmt = ARM_SMMU_CTX_FMT_AARCH32_L;
725 if (IS_ENABLED(CONFIG_IOMMU_IO_PGTABLE_ARMV7S) &&
726 !IS_ENABLED(CONFIG_64BIT) && !IS_ENABLED(CONFIG_ARM_LPAE) &&
727 (smmu->features & ARM_SMMU_FEAT_FMT_AARCH32_S) &&
728 (smmu_domain->stage == ARM_SMMU_DOMAIN_S1))
729 cfg->fmt = ARM_SMMU_CTX_FMT_AARCH32_S;
730 if ((IS_ENABLED(CONFIG_64BIT) || cfg->fmt == ARM_SMMU_CTX_FMT_NONE) &&
731 (smmu->features & (ARM_SMMU_FEAT_FMT_AARCH64_64K |
732 ARM_SMMU_FEAT_FMT_AARCH64_16K |
733 ARM_SMMU_FEAT_FMT_AARCH64_4K)))
734 cfg->fmt = ARM_SMMU_CTX_FMT_AARCH64;
735
736 if (cfg->fmt == ARM_SMMU_CTX_FMT_NONE) {
737 ret = -EINVAL;
738 goto out_unlock;
739 }
740
741 switch (smmu_domain->stage) {
742 case ARM_SMMU_DOMAIN_S1:
743 cfg->cbar = CBAR_TYPE_S1_TRANS_S2_BYPASS;
744 start = smmu->num_s2_context_banks;
745 ias = smmu->va_size;
746 oas = smmu->ipa_size;
747 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64) {
748 fmt = ARM_64_LPAE_S1;
749 } else if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_L) {
750 fmt = ARM_32_LPAE_S1;
751 ias = min(ias, 32UL);
752 oas = min(oas, 40UL);
753 } else {
754 fmt = ARM_V7S;
755 ias = min(ias, 32UL);
756 oas = min(oas, 32UL);
757 }
758 smmu_domain->flush_ops = &arm_smmu_s1_tlb_ops;
759 break;
760 case ARM_SMMU_DOMAIN_NESTED:
761 /*
762 * We will likely want to change this if/when KVM gets
763 * involved.
764 */
765 case ARM_SMMU_DOMAIN_S2:
766 cfg->cbar = CBAR_TYPE_S2_TRANS;
767 start = 0;
768 ias = smmu->ipa_size;
769 oas = smmu->pa_size;
770 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64) {
771 fmt = ARM_64_LPAE_S2;
772 } else {
773 fmt = ARM_32_LPAE_S2;
774 ias = min(ias, 40UL);
775 oas = min(oas, 40UL);
776 }
777 if (smmu->version == ARM_SMMU_V2)
778 smmu_domain->flush_ops = &arm_smmu_s2_tlb_ops_v2;
779 else
780 smmu_domain->flush_ops = &arm_smmu_s2_tlb_ops_v1;
781 break;
782 default:
783 ret = -EINVAL;
784 goto out_unlock;
785 }
786
787 ret = arm_smmu_alloc_context_bank(smmu_domain, smmu, dev, start);
788 if (ret < 0) {
789 goto out_unlock;
790 }
791
792 smmu_domain->smmu = smmu;
793
794 cfg->cbndx = ret;
795 if (smmu->version < ARM_SMMU_V2) {
796 cfg->irptndx = atomic_inc_return(&smmu->irptndx);
797 cfg->irptndx %= smmu->num_context_irqs;
798 } else {
799 cfg->irptndx = cfg->cbndx;
800 }
801
802 if (smmu_domain->stage == ARM_SMMU_DOMAIN_S2)
803 cfg->vmid = cfg->cbndx + 1;
804 else
805 cfg->asid = cfg->cbndx;
806
807 pgtbl_cfg = (struct io_pgtable_cfg) {
808 .pgsize_bitmap = smmu->pgsize_bitmap,
809 .ias = ias,
810 .oas = oas,
811 .coherent_walk = smmu->features & ARM_SMMU_FEAT_COHERENT_WALK,
812 .tlb = smmu_domain->flush_ops,
813 .iommu_dev = smmu->dev,
814 };
815
816 if (smmu->impl && smmu->impl->init_context) {
817 ret = smmu->impl->init_context(smmu_domain, &pgtbl_cfg, dev);
818 if (ret)
819 goto out_clear_smmu;
820 }
821
822 if (smmu_domain->pgtbl_quirks)
823 pgtbl_cfg.quirks |= smmu_domain->pgtbl_quirks;
824
825 pgtbl_ops = alloc_io_pgtable_ops(fmt, &pgtbl_cfg, smmu_domain);
826 if (!pgtbl_ops) {
827 ret = -ENOMEM;
828 goto out_clear_smmu;
829 }
830
831 /* Update the domain's page sizes to reflect the page table format */
832 domain->pgsize_bitmap = pgtbl_cfg.pgsize_bitmap;
833
834 if (pgtbl_cfg.quirks & IO_PGTABLE_QUIRK_ARM_TTBR1) {
835 domain->geometry.aperture_start = ~0UL << ias;
836 domain->geometry.aperture_end = ~0UL;
837 } else {
838 domain->geometry.aperture_end = (1UL << ias) - 1;
839 }
840
841 domain->geometry.force_aperture = true;
842
843 /* Initialise the context bank with our page table cfg */
844 arm_smmu_init_context_bank(smmu_domain, &pgtbl_cfg);
845 arm_smmu_write_context_bank(smmu, cfg->cbndx);
846
847 /*
848 * Request context fault interrupt. Do this last to avoid the
849 * handler seeing a half-initialised domain state.
850 */
851 irq = smmu->irqs[cfg->irptndx];
852
853 if (smmu->impl && smmu->impl->context_fault)
854 context_fault = smmu->impl->context_fault;
855 else
856 context_fault = arm_smmu_context_fault;
857
858 if (smmu->impl && smmu->impl->context_fault_needs_threaded_irq)
859 ret = devm_request_threaded_irq(smmu->dev, irq, NULL,
860 context_fault,
861 IRQF_ONESHOT | IRQF_SHARED,
862 "arm-smmu-context-fault",
863 smmu_domain);
864 else
865 ret = devm_request_irq(smmu->dev, irq, context_fault, IRQF_SHARED,
866 "arm-smmu-context-fault", smmu_domain);
867
868 if (ret < 0) {
869 dev_err(smmu->dev, "failed to request context IRQ %d (%u)\n",
870 cfg->irptndx, irq);
871 cfg->irptndx = ARM_SMMU_INVALID_IRPTNDX;
872 }
873
874 mutex_unlock(&smmu_domain->init_mutex);
875
876 /* Publish page table ops for map/unmap */
877 smmu_domain->pgtbl_ops = pgtbl_ops;
878 return 0;
879
880 out_clear_smmu:
881 __arm_smmu_free_bitmap(smmu->context_map, cfg->cbndx);
882 smmu_domain->smmu = NULL;
883 out_unlock:
884 mutex_unlock(&smmu_domain->init_mutex);
885 return ret;
886 }
887
arm_smmu_destroy_domain_context(struct arm_smmu_domain * smmu_domain)888 static void arm_smmu_destroy_domain_context(struct arm_smmu_domain *smmu_domain)
889 {
890 struct arm_smmu_device *smmu = smmu_domain->smmu;
891 struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
892 int ret, irq;
893
894 if (!smmu)
895 return;
896
897 ret = arm_smmu_rpm_get(smmu);
898 if (ret < 0)
899 return;
900
901 /*
902 * Disable the context bank and free the page tables before freeing
903 * it.
904 */
905 smmu->cbs[cfg->cbndx].cfg = NULL;
906 arm_smmu_write_context_bank(smmu, cfg->cbndx);
907
908 if (cfg->irptndx != ARM_SMMU_INVALID_IRPTNDX) {
909 irq = smmu->irqs[cfg->irptndx];
910 devm_free_irq(smmu->dev, irq, smmu_domain);
911 }
912
913 free_io_pgtable_ops(smmu_domain->pgtbl_ops);
914 __arm_smmu_free_bitmap(smmu->context_map, cfg->cbndx);
915
916 arm_smmu_rpm_put(smmu);
917 }
918
arm_smmu_domain_alloc_paging(struct device * dev)919 static struct iommu_domain *arm_smmu_domain_alloc_paging(struct device *dev)
920 {
921 struct arm_smmu_domain *smmu_domain;
922 struct arm_smmu_master_cfg *cfg = dev_iommu_priv_get(dev);
923 struct arm_smmu_device *smmu = cfg->smmu;
924
925 /*
926 * Allocate the domain and initialise some of its data structures.
927 * We can't really do anything meaningful until we've added a
928 * master.
929 */
930 smmu_domain = kzalloc(sizeof(*smmu_domain), GFP_KERNEL);
931 if (!smmu_domain)
932 return NULL;
933
934 mutex_init(&smmu_domain->init_mutex);
935 spin_lock_init(&smmu_domain->cb_lock);
936 smmu_domain->domain.pgsize_bitmap = smmu->pgsize_bitmap;
937
938 return &smmu_domain->domain;
939 }
940
arm_smmu_domain_free(struct iommu_domain * domain)941 static void arm_smmu_domain_free(struct iommu_domain *domain)
942 {
943 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
944
945 /*
946 * Free the domain resources. We assume that all devices have
947 * already been detached.
948 */
949 arm_smmu_destroy_domain_context(smmu_domain);
950 kfree(smmu_domain);
951 }
952
arm_smmu_write_smr(struct arm_smmu_device * smmu,int idx)953 static void arm_smmu_write_smr(struct arm_smmu_device *smmu, int idx)
954 {
955 struct arm_smmu_smr *smr = smmu->smrs + idx;
956 u32 reg = FIELD_PREP(ARM_SMMU_SMR_ID, smr->id) |
957 FIELD_PREP(ARM_SMMU_SMR_MASK, smr->mask);
958
959 if (!(smmu->features & ARM_SMMU_FEAT_EXIDS) && smr->valid)
960 reg |= ARM_SMMU_SMR_VALID;
961 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_SMR(idx), reg);
962 }
963
arm_smmu_write_s2cr(struct arm_smmu_device * smmu,int idx)964 static void arm_smmu_write_s2cr(struct arm_smmu_device *smmu, int idx)
965 {
966 struct arm_smmu_s2cr *s2cr = smmu->s2crs + idx;
967 u32 reg;
968
969 if (smmu->impl && smmu->impl->write_s2cr) {
970 smmu->impl->write_s2cr(smmu, idx);
971 return;
972 }
973
974 reg = FIELD_PREP(ARM_SMMU_S2CR_TYPE, s2cr->type) |
975 FIELD_PREP(ARM_SMMU_S2CR_CBNDX, s2cr->cbndx) |
976 FIELD_PREP(ARM_SMMU_S2CR_PRIVCFG, s2cr->privcfg);
977
978 if (smmu->features & ARM_SMMU_FEAT_EXIDS && smmu->smrs &&
979 smmu->smrs[idx].valid)
980 reg |= ARM_SMMU_S2CR_EXIDVALID;
981 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_S2CR(idx), reg);
982 }
983
arm_smmu_write_sme(struct arm_smmu_device * smmu,int idx)984 static void arm_smmu_write_sme(struct arm_smmu_device *smmu, int idx)
985 {
986 arm_smmu_write_s2cr(smmu, idx);
987 if (smmu->smrs)
988 arm_smmu_write_smr(smmu, idx);
989 }
990
991 /*
992 * The width of SMR's mask field depends on sCR0_EXIDENABLE, so this function
993 * should be called after sCR0 is written.
994 */
arm_smmu_test_smr_masks(struct arm_smmu_device * smmu)995 static void arm_smmu_test_smr_masks(struct arm_smmu_device *smmu)
996 {
997 u32 smr;
998 int i;
999
1000 if (!smmu->smrs)
1001 return;
1002 /*
1003 * If we've had to accommodate firmware memory regions, we may
1004 * have live SMRs by now; tread carefully...
1005 *
1006 * Somewhat perversely, not having a free SMR for this test implies we
1007 * can get away without it anyway, as we'll only be able to 'allocate'
1008 * these SMRs for the ID/mask values we're already trusting to be OK.
1009 */
1010 for (i = 0; i < smmu->num_mapping_groups; i++)
1011 if (!smmu->smrs[i].valid)
1012 goto smr_ok;
1013 return;
1014 smr_ok:
1015 /*
1016 * SMR.ID bits may not be preserved if the corresponding MASK
1017 * bits are set, so check each one separately. We can reject
1018 * masters later if they try to claim IDs outside these masks.
1019 */
1020 smr = FIELD_PREP(ARM_SMMU_SMR_ID, smmu->streamid_mask);
1021 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_SMR(i), smr);
1022 smr = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_SMR(i));
1023 smmu->streamid_mask = FIELD_GET(ARM_SMMU_SMR_ID, smr);
1024
1025 smr = FIELD_PREP(ARM_SMMU_SMR_MASK, smmu->streamid_mask);
1026 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_SMR(i), smr);
1027 smr = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_SMR(i));
1028 smmu->smr_mask_mask = FIELD_GET(ARM_SMMU_SMR_MASK, smr);
1029 }
1030
arm_smmu_find_sme(struct arm_smmu_device * smmu,u16 id,u16 mask)1031 static int arm_smmu_find_sme(struct arm_smmu_device *smmu, u16 id, u16 mask)
1032 {
1033 struct arm_smmu_smr *smrs = smmu->smrs;
1034 int i, free_idx = -ENOSPC;
1035
1036 /* Stream indexing is blissfully easy */
1037 if (!smrs)
1038 return id;
1039
1040 /* Validating SMRs is... less so */
1041 for (i = 0; i < smmu->num_mapping_groups; ++i) {
1042 if (!smrs[i].valid) {
1043 /*
1044 * Note the first free entry we come across, which
1045 * we'll claim in the end if nothing else matches.
1046 */
1047 if (free_idx < 0)
1048 free_idx = i;
1049 continue;
1050 }
1051 /*
1052 * If the new entry is _entirely_ matched by an existing entry,
1053 * then reuse that, with the guarantee that there also cannot
1054 * be any subsequent conflicting entries. In normal use we'd
1055 * expect simply identical entries for this case, but there's
1056 * no harm in accommodating the generalisation.
1057 */
1058 if ((mask & smrs[i].mask) == mask &&
1059 !((id ^ smrs[i].id) & ~smrs[i].mask))
1060 return i;
1061 /*
1062 * If the new entry has any other overlap with an existing one,
1063 * though, then there always exists at least one stream ID
1064 * which would cause a conflict, and we can't allow that risk.
1065 */
1066 if (!((id ^ smrs[i].id) & ~(smrs[i].mask | mask)))
1067 return -EINVAL;
1068 }
1069
1070 return free_idx;
1071 }
1072
arm_smmu_free_sme(struct arm_smmu_device * smmu,int idx)1073 static bool arm_smmu_free_sme(struct arm_smmu_device *smmu, int idx)
1074 {
1075 if (--smmu->s2crs[idx].count)
1076 return false;
1077
1078 smmu->s2crs[idx] = s2cr_init_val;
1079 if (smmu->smrs)
1080 smmu->smrs[idx].valid = false;
1081
1082 return true;
1083 }
1084
arm_smmu_master_alloc_smes(struct device * dev)1085 static int arm_smmu_master_alloc_smes(struct device *dev)
1086 {
1087 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
1088 struct arm_smmu_master_cfg *cfg = dev_iommu_priv_get(dev);
1089 struct arm_smmu_device *smmu = cfg->smmu;
1090 struct arm_smmu_smr *smrs = smmu->smrs;
1091 int i, idx, ret;
1092
1093 mutex_lock(&smmu->stream_map_mutex);
1094 /* Figure out a viable stream map entry allocation */
1095 for_each_cfg_sme(cfg, fwspec, i, idx) {
1096 u16 sid = FIELD_GET(ARM_SMMU_SMR_ID, fwspec->ids[i]);
1097 u16 mask = FIELD_GET(ARM_SMMU_SMR_MASK, fwspec->ids[i]);
1098
1099 if (idx != INVALID_SMENDX) {
1100 ret = -EEXIST;
1101 goto out_err;
1102 }
1103
1104 ret = arm_smmu_find_sme(smmu, sid, mask);
1105 if (ret < 0)
1106 goto out_err;
1107
1108 idx = ret;
1109 if (smrs && smmu->s2crs[idx].count == 0) {
1110 smrs[idx].id = sid;
1111 smrs[idx].mask = mask;
1112 smrs[idx].valid = true;
1113 }
1114 smmu->s2crs[idx].count++;
1115 cfg->smendx[i] = (s16)idx;
1116 }
1117
1118 /* It worked! Now, poke the actual hardware */
1119 for_each_cfg_sme(cfg, fwspec, i, idx)
1120 arm_smmu_write_sme(smmu, idx);
1121
1122 mutex_unlock(&smmu->stream_map_mutex);
1123 return 0;
1124
1125 out_err:
1126 while (i--) {
1127 arm_smmu_free_sme(smmu, cfg->smendx[i]);
1128 cfg->smendx[i] = INVALID_SMENDX;
1129 }
1130 mutex_unlock(&smmu->stream_map_mutex);
1131 return ret;
1132 }
1133
arm_smmu_master_free_smes(struct arm_smmu_master_cfg * cfg,struct iommu_fwspec * fwspec)1134 static void arm_smmu_master_free_smes(struct arm_smmu_master_cfg *cfg,
1135 struct iommu_fwspec *fwspec)
1136 {
1137 struct arm_smmu_device *smmu = cfg->smmu;
1138 int i, idx;
1139
1140 mutex_lock(&smmu->stream_map_mutex);
1141 for_each_cfg_sme(cfg, fwspec, i, idx) {
1142 if (arm_smmu_free_sme(smmu, idx))
1143 arm_smmu_write_sme(smmu, idx);
1144 cfg->smendx[i] = INVALID_SMENDX;
1145 }
1146 mutex_unlock(&smmu->stream_map_mutex);
1147 }
1148
arm_smmu_master_install_s2crs(struct arm_smmu_master_cfg * cfg,enum arm_smmu_s2cr_type type,u8 cbndx,struct iommu_fwspec * fwspec)1149 static void arm_smmu_master_install_s2crs(struct arm_smmu_master_cfg *cfg,
1150 enum arm_smmu_s2cr_type type,
1151 u8 cbndx, struct iommu_fwspec *fwspec)
1152 {
1153 struct arm_smmu_device *smmu = cfg->smmu;
1154 struct arm_smmu_s2cr *s2cr = smmu->s2crs;
1155 int i, idx;
1156
1157 for_each_cfg_sme(cfg, fwspec, i, idx) {
1158 if (type == s2cr[idx].type && cbndx == s2cr[idx].cbndx)
1159 continue;
1160
1161 s2cr[idx].type = type;
1162 s2cr[idx].privcfg = S2CR_PRIVCFG_DEFAULT;
1163 s2cr[idx].cbndx = cbndx;
1164 arm_smmu_write_s2cr(smmu, idx);
1165 }
1166 }
1167
arm_smmu_attach_dev(struct iommu_domain * domain,struct device * dev,struct iommu_domain * old)1168 static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev,
1169 struct iommu_domain *old)
1170 {
1171 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1172 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
1173 struct arm_smmu_master_cfg *cfg;
1174 struct arm_smmu_device *smmu;
1175 int ret;
1176
1177 /*
1178 * FIXME: The arch/arm DMA API code tries to attach devices to its own
1179 * domains between of_xlate() and probe_device() - we have no way to cope
1180 * with that, so until ARM gets converted to rely on groups and default
1181 * domains, just say no (but more politely than by dereferencing NULL).
1182 * This should be at least a WARN_ON once that's sorted.
1183 */
1184 cfg = dev_iommu_priv_get(dev);
1185 if (!cfg)
1186 return -ENODEV;
1187
1188 smmu = cfg->smmu;
1189
1190 ret = arm_smmu_rpm_get(smmu);
1191 if (ret < 0)
1192 return ret;
1193
1194 /* Ensure that the domain is finalised */
1195 ret = arm_smmu_init_domain_context(smmu_domain, smmu, dev);
1196 if (ret < 0)
1197 goto rpm_put;
1198
1199 /*
1200 * Sanity check the domain. We don't support domains across
1201 * different SMMUs.
1202 */
1203 if (smmu_domain->smmu != smmu) {
1204 ret = -EINVAL;
1205 goto rpm_put;
1206 }
1207
1208 /* Looks ok, so add the device to the domain */
1209 arm_smmu_master_install_s2crs(cfg, S2CR_TYPE_TRANS,
1210 smmu_domain->cfg.cbndx, fwspec);
1211 rpm_put:
1212 arm_smmu_rpm_put(smmu);
1213 return ret;
1214 }
1215
arm_smmu_attach_dev_type(struct device * dev,enum arm_smmu_s2cr_type type)1216 static int arm_smmu_attach_dev_type(struct device *dev,
1217 enum arm_smmu_s2cr_type type)
1218 {
1219 struct arm_smmu_master_cfg *cfg = dev_iommu_priv_get(dev);
1220 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
1221 struct arm_smmu_device *smmu;
1222 int ret;
1223
1224 if (!cfg)
1225 return -ENODEV;
1226 smmu = cfg->smmu;
1227
1228 ret = arm_smmu_rpm_get(smmu);
1229 if (ret < 0)
1230 return ret;
1231
1232 arm_smmu_master_install_s2crs(cfg, type, 0, fwspec);
1233 arm_smmu_rpm_put(smmu);
1234 return 0;
1235 }
1236
arm_smmu_attach_dev_identity(struct iommu_domain * domain,struct device * dev,struct iommu_domain * old)1237 static int arm_smmu_attach_dev_identity(struct iommu_domain *domain,
1238 struct device *dev,
1239 struct iommu_domain *old)
1240 {
1241 return arm_smmu_attach_dev_type(dev, S2CR_TYPE_BYPASS);
1242 }
1243
1244 static const struct iommu_domain_ops arm_smmu_identity_ops = {
1245 .attach_dev = arm_smmu_attach_dev_identity,
1246 };
1247
1248 static struct iommu_domain arm_smmu_identity_domain = {
1249 .type = IOMMU_DOMAIN_IDENTITY,
1250 .ops = &arm_smmu_identity_ops,
1251 };
1252
arm_smmu_attach_dev_blocked(struct iommu_domain * domain,struct device * dev,struct iommu_domain * old)1253 static int arm_smmu_attach_dev_blocked(struct iommu_domain *domain,
1254 struct device *dev,
1255 struct iommu_domain *old)
1256 {
1257 return arm_smmu_attach_dev_type(dev, S2CR_TYPE_FAULT);
1258 }
1259
1260 static const struct iommu_domain_ops arm_smmu_blocked_ops = {
1261 .attach_dev = arm_smmu_attach_dev_blocked,
1262 };
1263
1264 static struct iommu_domain arm_smmu_blocked_domain = {
1265 .type = IOMMU_DOMAIN_BLOCKED,
1266 .ops = &arm_smmu_blocked_ops,
1267 };
1268
arm_smmu_map_pages(struct iommu_domain * domain,unsigned long iova,phys_addr_t paddr,size_t pgsize,size_t pgcount,int prot,gfp_t gfp,size_t * mapped)1269 static int arm_smmu_map_pages(struct iommu_domain *domain, unsigned long iova,
1270 phys_addr_t paddr, size_t pgsize, size_t pgcount,
1271 int prot, gfp_t gfp, size_t *mapped)
1272 {
1273 struct io_pgtable_ops *ops = to_smmu_domain(domain)->pgtbl_ops;
1274 struct arm_smmu_device *smmu = to_smmu_domain(domain)->smmu;
1275 int ret;
1276
1277 if (!ops)
1278 return -ENODEV;
1279
1280 arm_smmu_rpm_get(smmu);
1281 ret = ops->map_pages(ops, iova, paddr, pgsize, pgcount, prot, gfp, mapped);
1282 arm_smmu_rpm_put(smmu);
1283
1284 return ret;
1285 }
1286
arm_smmu_unmap_pages(struct iommu_domain * domain,unsigned long iova,size_t pgsize,size_t pgcount,struct iommu_iotlb_gather * iotlb_gather)1287 static size_t arm_smmu_unmap_pages(struct iommu_domain *domain, unsigned long iova,
1288 size_t pgsize, size_t pgcount,
1289 struct iommu_iotlb_gather *iotlb_gather)
1290 {
1291 struct io_pgtable_ops *ops = to_smmu_domain(domain)->pgtbl_ops;
1292 struct arm_smmu_device *smmu = to_smmu_domain(domain)->smmu;
1293 size_t ret;
1294
1295 if (!ops)
1296 return 0;
1297
1298 arm_smmu_rpm_get(smmu);
1299 ret = ops->unmap_pages(ops, iova, pgsize, pgcount, iotlb_gather);
1300 arm_smmu_rpm_put(smmu);
1301
1302 return ret;
1303 }
1304
arm_smmu_flush_iotlb_all(struct iommu_domain * domain)1305 static void arm_smmu_flush_iotlb_all(struct iommu_domain *domain)
1306 {
1307 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1308 struct arm_smmu_device *smmu = smmu_domain->smmu;
1309
1310 if (smmu_domain->flush_ops) {
1311 arm_smmu_rpm_get(smmu);
1312 smmu_domain->flush_ops->tlb_flush_all(smmu_domain);
1313 arm_smmu_rpm_put(smmu);
1314 }
1315 }
1316
arm_smmu_iotlb_sync(struct iommu_domain * domain,struct iommu_iotlb_gather * gather)1317 static void arm_smmu_iotlb_sync(struct iommu_domain *domain,
1318 struct iommu_iotlb_gather *gather)
1319 {
1320 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1321 struct arm_smmu_device *smmu = smmu_domain->smmu;
1322
1323 if (!smmu)
1324 return;
1325
1326 arm_smmu_rpm_get(smmu);
1327 if (smmu->version == ARM_SMMU_V2 ||
1328 smmu_domain->stage == ARM_SMMU_DOMAIN_S1)
1329 arm_smmu_tlb_sync_context(smmu_domain);
1330 else
1331 arm_smmu_tlb_sync_global(smmu);
1332 arm_smmu_rpm_put(smmu);
1333 }
1334
arm_smmu_iova_to_phys_hard(struct iommu_domain * domain,dma_addr_t iova)1335 static phys_addr_t arm_smmu_iova_to_phys_hard(struct iommu_domain *domain,
1336 dma_addr_t iova)
1337 {
1338 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1339 struct arm_smmu_device *smmu = smmu_domain->smmu;
1340 struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
1341 struct io_pgtable_ops *ops= smmu_domain->pgtbl_ops;
1342 struct device *dev = smmu->dev;
1343 void __iomem *reg;
1344 u32 tmp;
1345 u64 phys;
1346 unsigned long va, flags;
1347 int ret, idx = cfg->cbndx;
1348 phys_addr_t addr = 0;
1349
1350 ret = arm_smmu_rpm_get(smmu);
1351 if (ret < 0)
1352 return 0;
1353
1354 spin_lock_irqsave(&smmu_domain->cb_lock, flags);
1355 va = iova & ~0xfffUL;
1356 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64)
1357 arm_smmu_cb_writeq(smmu, idx, ARM_SMMU_CB_ATS1PR, va);
1358 else
1359 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_ATS1PR, va);
1360
1361 reg = arm_smmu_page(smmu, ARM_SMMU_CB(smmu, idx)) + ARM_SMMU_CB_ATSR;
1362 if (readl_poll_timeout_atomic(reg, tmp, !(tmp & ARM_SMMU_CB_ATSR_ACTIVE),
1363 5, 50)) {
1364 spin_unlock_irqrestore(&smmu_domain->cb_lock, flags);
1365 dev_err(dev,
1366 "iova to phys timed out on %pad. Falling back to software table walk.\n",
1367 &iova);
1368 arm_smmu_rpm_put(smmu);
1369 return ops->iova_to_phys(ops, iova);
1370 }
1371
1372 phys = arm_smmu_cb_readq(smmu, idx, ARM_SMMU_CB_PAR);
1373 spin_unlock_irqrestore(&smmu_domain->cb_lock, flags);
1374 if (phys & ARM_SMMU_CB_PAR_F) {
1375 dev_err(dev, "translation fault!\n");
1376 dev_err(dev, "PAR = 0x%llx\n", phys);
1377 goto out;
1378 }
1379
1380 addr = (phys & GENMASK_ULL(39, 12)) | (iova & 0xfff);
1381 out:
1382 arm_smmu_rpm_put(smmu);
1383
1384 return addr;
1385 }
1386
arm_smmu_iova_to_phys(struct iommu_domain * domain,dma_addr_t iova)1387 static phys_addr_t arm_smmu_iova_to_phys(struct iommu_domain *domain,
1388 dma_addr_t iova)
1389 {
1390 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1391 struct io_pgtable_ops *ops = smmu_domain->pgtbl_ops;
1392
1393 if (!ops)
1394 return 0;
1395
1396 if (smmu_domain->smmu->features & ARM_SMMU_FEAT_TRANS_OPS &&
1397 smmu_domain->stage == ARM_SMMU_DOMAIN_S1)
1398 return arm_smmu_iova_to_phys_hard(domain, iova);
1399
1400 return ops->iova_to_phys(ops, iova);
1401 }
1402
arm_smmu_capable(struct device * dev,enum iommu_cap cap)1403 static bool arm_smmu_capable(struct device *dev, enum iommu_cap cap)
1404 {
1405 struct arm_smmu_master_cfg *cfg = dev_iommu_priv_get(dev);
1406
1407 switch (cap) {
1408 case IOMMU_CAP_CACHE_COHERENCY:
1409 /*
1410 * It's overwhelmingly the case in practice that when the pagetable
1411 * walk interface is connected to a coherent interconnect, all the
1412 * translation interfaces are too. Furthermore if the device is
1413 * natively coherent, then its translation interface must also be.
1414 */
1415 return cfg->smmu->features & ARM_SMMU_FEAT_COHERENT_WALK ||
1416 device_get_dma_attr(dev) == DEV_DMA_COHERENT;
1417 case IOMMU_CAP_NOEXEC:
1418 case IOMMU_CAP_DEFERRED_FLUSH:
1419 return true;
1420 default:
1421 return false;
1422 }
1423 }
1424
1425 static
arm_smmu_get_by_fwnode(struct fwnode_handle * fwnode)1426 struct arm_smmu_device *arm_smmu_get_by_fwnode(struct fwnode_handle *fwnode)
1427 {
1428 struct device *dev = bus_find_device_by_fwnode(&platform_bus_type, fwnode);
1429
1430 put_device(dev);
1431 return dev ? dev_get_drvdata(dev) : NULL;
1432 }
1433
arm_smmu_probe_device(struct device * dev)1434 static struct iommu_device *arm_smmu_probe_device(struct device *dev)
1435 {
1436 struct arm_smmu_device *smmu = NULL;
1437 struct arm_smmu_master_cfg *cfg;
1438 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
1439 int i, ret;
1440
1441 if (using_legacy_binding) {
1442 ret = arm_smmu_register_legacy_master(dev, &smmu);
1443
1444 /*
1445 * If dev->iommu_fwspec is initally NULL, arm_smmu_register_legacy_master()
1446 * will allocate/initialise a new one. Thus we need to update fwspec for
1447 * later use.
1448 */
1449 fwspec = dev_iommu_fwspec_get(dev);
1450 if (ret)
1451 goto out_free;
1452 } else {
1453 smmu = arm_smmu_get_by_fwnode(fwspec->iommu_fwnode);
1454 }
1455
1456 ret = -EINVAL;
1457 for (i = 0; i < fwspec->num_ids; i++) {
1458 u16 sid = FIELD_GET(ARM_SMMU_SMR_ID, fwspec->ids[i]);
1459 u16 mask = FIELD_GET(ARM_SMMU_SMR_MASK, fwspec->ids[i]);
1460
1461 if (sid & ~smmu->streamid_mask) {
1462 dev_err(dev, "stream ID 0x%x out of range for SMMU (0x%x)\n",
1463 sid, smmu->streamid_mask);
1464 goto out_free;
1465 }
1466 if (mask & ~smmu->smr_mask_mask) {
1467 dev_err(dev, "SMR mask 0x%x out of range for SMMU (0x%x)\n",
1468 mask, smmu->smr_mask_mask);
1469 goto out_free;
1470 }
1471 }
1472
1473 ret = -ENOMEM;
1474 cfg = kzalloc(offsetof(struct arm_smmu_master_cfg, smendx[i]),
1475 GFP_KERNEL);
1476 if (!cfg)
1477 goto out_free;
1478
1479 cfg->smmu = smmu;
1480 dev_iommu_priv_set(dev, cfg);
1481 while (i--)
1482 cfg->smendx[i] = INVALID_SMENDX;
1483
1484 ret = arm_smmu_rpm_get(smmu);
1485 if (ret < 0)
1486 goto out_cfg_free;
1487
1488 ret = arm_smmu_master_alloc_smes(dev);
1489 arm_smmu_rpm_put(smmu);
1490
1491 if (ret)
1492 goto out_cfg_free;
1493
1494 device_link_add(dev, smmu->dev,
1495 DL_FLAG_PM_RUNTIME | DL_FLAG_AUTOREMOVE_SUPPLIER);
1496
1497 return &smmu->iommu;
1498
1499 out_cfg_free:
1500 kfree(cfg);
1501 out_free:
1502 return ERR_PTR(ret);
1503 }
1504
arm_smmu_release_device(struct device * dev)1505 static void arm_smmu_release_device(struct device *dev)
1506 {
1507 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
1508 struct arm_smmu_master_cfg *cfg = dev_iommu_priv_get(dev);
1509 int ret;
1510
1511 ret = arm_smmu_rpm_get(cfg->smmu);
1512 if (ret < 0)
1513 return;
1514
1515 arm_smmu_master_free_smes(cfg, fwspec);
1516
1517 arm_smmu_rpm_put(cfg->smmu);
1518
1519 kfree(cfg);
1520 }
1521
arm_smmu_probe_finalize(struct device * dev)1522 static void arm_smmu_probe_finalize(struct device *dev)
1523 {
1524 struct arm_smmu_master_cfg *cfg;
1525 struct arm_smmu_device *smmu;
1526
1527 cfg = dev_iommu_priv_get(dev);
1528 smmu = cfg->smmu;
1529
1530 if (smmu->impl && smmu->impl->probe_finalize)
1531 smmu->impl->probe_finalize(smmu, dev);
1532 }
1533
arm_smmu_device_group(struct device * dev)1534 static struct iommu_group *arm_smmu_device_group(struct device *dev)
1535 {
1536 struct arm_smmu_master_cfg *cfg = dev_iommu_priv_get(dev);
1537 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
1538 struct arm_smmu_device *smmu = cfg->smmu;
1539 struct iommu_group *group = NULL;
1540 int i, idx;
1541
1542 mutex_lock(&smmu->stream_map_mutex);
1543 for_each_cfg_sme(cfg, fwspec, i, idx) {
1544 if (group && smmu->s2crs[idx].group &&
1545 group != smmu->s2crs[idx].group) {
1546 mutex_unlock(&smmu->stream_map_mutex);
1547 return ERR_PTR(-EINVAL);
1548 }
1549
1550 group = smmu->s2crs[idx].group;
1551 }
1552
1553 if (group) {
1554 mutex_unlock(&smmu->stream_map_mutex);
1555 return iommu_group_ref_get(group);
1556 }
1557
1558 if (dev_is_pci(dev))
1559 group = pci_device_group(dev);
1560 else if (dev_is_fsl_mc(dev))
1561 group = fsl_mc_device_group(dev);
1562 else
1563 group = generic_device_group(dev);
1564
1565 /* Remember group for faster lookups */
1566 if (!IS_ERR(group))
1567 for_each_cfg_sme(cfg, fwspec, i, idx)
1568 smmu->s2crs[idx].group = group;
1569
1570 mutex_unlock(&smmu->stream_map_mutex);
1571 return group;
1572 }
1573
arm_smmu_set_pgtable_quirks(struct iommu_domain * domain,unsigned long quirks)1574 static int arm_smmu_set_pgtable_quirks(struct iommu_domain *domain,
1575 unsigned long quirks)
1576 {
1577 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1578 int ret = 0;
1579
1580 mutex_lock(&smmu_domain->init_mutex);
1581 if (smmu_domain->smmu)
1582 ret = -EPERM;
1583 else
1584 smmu_domain->pgtbl_quirks = quirks;
1585 mutex_unlock(&smmu_domain->init_mutex);
1586
1587 return ret;
1588 }
1589
arm_smmu_of_xlate(struct device * dev,const struct of_phandle_args * args)1590 static int arm_smmu_of_xlate(struct device *dev,
1591 const struct of_phandle_args *args)
1592 {
1593 u32 mask, fwid = 0;
1594
1595 if (args->args_count > 0)
1596 fwid |= FIELD_PREP(ARM_SMMU_SMR_ID, args->args[0]);
1597
1598 if (args->args_count > 1)
1599 fwid |= FIELD_PREP(ARM_SMMU_SMR_MASK, args->args[1]);
1600 else if (!of_property_read_u32(args->np, "stream-match-mask", &mask))
1601 fwid |= FIELD_PREP(ARM_SMMU_SMR_MASK, mask);
1602
1603 return iommu_fwspec_add_ids(dev, &fwid, 1);
1604 }
1605
arm_smmu_get_resv_regions(struct device * dev,struct list_head * head)1606 static void arm_smmu_get_resv_regions(struct device *dev,
1607 struct list_head *head)
1608 {
1609 struct iommu_resv_region *region;
1610 int prot = IOMMU_WRITE | IOMMU_NOEXEC | IOMMU_MMIO;
1611
1612 region = iommu_alloc_resv_region(MSI_IOVA_BASE, MSI_IOVA_LENGTH,
1613 prot, IOMMU_RESV_SW_MSI, GFP_KERNEL);
1614 if (!region)
1615 return;
1616
1617 list_add_tail(®ion->list, head);
1618
1619 iommu_dma_get_resv_regions(dev, head);
1620 }
1621
arm_smmu_def_domain_type(struct device * dev)1622 static int arm_smmu_def_domain_type(struct device *dev)
1623 {
1624 struct arm_smmu_master_cfg *cfg = dev_iommu_priv_get(dev);
1625 const struct arm_smmu_impl *impl = cfg->smmu->impl;
1626
1627 if (using_legacy_binding)
1628 return IOMMU_DOMAIN_IDENTITY;
1629
1630 if (impl && impl->def_domain_type)
1631 return impl->def_domain_type(dev);
1632
1633 return 0;
1634 }
1635
1636 static const struct iommu_ops arm_smmu_ops = {
1637 .identity_domain = &arm_smmu_identity_domain,
1638 .blocked_domain = &arm_smmu_blocked_domain,
1639 .capable = arm_smmu_capable,
1640 .domain_alloc_paging = arm_smmu_domain_alloc_paging,
1641 .probe_device = arm_smmu_probe_device,
1642 .release_device = arm_smmu_release_device,
1643 .probe_finalize = arm_smmu_probe_finalize,
1644 .device_group = arm_smmu_device_group,
1645 .of_xlate = arm_smmu_of_xlate,
1646 .get_resv_regions = arm_smmu_get_resv_regions,
1647 .def_domain_type = arm_smmu_def_domain_type,
1648 .owner = THIS_MODULE,
1649 .default_domain_ops = &(const struct iommu_domain_ops) {
1650 .attach_dev = arm_smmu_attach_dev,
1651 .map_pages = arm_smmu_map_pages,
1652 .unmap_pages = arm_smmu_unmap_pages,
1653 .flush_iotlb_all = arm_smmu_flush_iotlb_all,
1654 .iotlb_sync = arm_smmu_iotlb_sync,
1655 .iova_to_phys = arm_smmu_iova_to_phys,
1656 .set_pgtable_quirks = arm_smmu_set_pgtable_quirks,
1657 .free = arm_smmu_domain_free,
1658 }
1659 };
1660
arm_smmu_device_reset(struct arm_smmu_device * smmu)1661 static void arm_smmu_device_reset(struct arm_smmu_device *smmu)
1662 {
1663 int i;
1664 u32 reg;
1665
1666 /* clear global FSR */
1667 reg = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sGFSR);
1668 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_sGFSR, reg);
1669
1670 /*
1671 * Reset stream mapping groups: Initial values mark all SMRn as
1672 * invalid and all S2CRn as bypass unless overridden.
1673 */
1674 for (i = 0; i < smmu->num_mapping_groups; ++i)
1675 arm_smmu_write_sme(smmu, i);
1676
1677 /* Make sure all context banks are disabled and clear CB_FSR */
1678 for (i = 0; i < smmu->num_context_banks; ++i) {
1679 arm_smmu_write_context_bank(smmu, i);
1680 arm_smmu_cb_write(smmu, i, ARM_SMMU_CB_FSR, ARM_SMMU_CB_FSR_FAULT);
1681 }
1682
1683 /* Invalidate the TLB, just in case */
1684 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_TLBIALLH, QCOM_DUMMY_VAL);
1685 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_TLBIALLNSNH, QCOM_DUMMY_VAL);
1686
1687 reg = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sCR0);
1688
1689 /* Enable fault reporting */
1690 reg |= (ARM_SMMU_sCR0_GFRE | ARM_SMMU_sCR0_GFIE |
1691 ARM_SMMU_sCR0_GCFGFRE | ARM_SMMU_sCR0_GCFGFIE);
1692
1693 /* Disable TLB broadcasting. */
1694 reg |= (ARM_SMMU_sCR0_VMIDPNE | ARM_SMMU_sCR0_PTM);
1695
1696 /* Enable client access, handling unmatched streams as appropriate */
1697 reg &= ~ARM_SMMU_sCR0_CLIENTPD;
1698 if (disable_bypass)
1699 reg |= ARM_SMMU_sCR0_USFCFG;
1700 else
1701 reg &= ~ARM_SMMU_sCR0_USFCFG;
1702
1703 /* Disable forced broadcasting */
1704 reg &= ~ARM_SMMU_sCR0_FB;
1705
1706 /* Don't upgrade barriers */
1707 reg &= ~(ARM_SMMU_sCR0_BSU);
1708
1709 if (smmu->features & ARM_SMMU_FEAT_VMID16)
1710 reg |= ARM_SMMU_sCR0_VMID16EN;
1711
1712 if (smmu->features & ARM_SMMU_FEAT_EXIDS)
1713 reg |= ARM_SMMU_sCR0_EXIDENABLE;
1714
1715 if (smmu->impl && smmu->impl->reset)
1716 smmu->impl->reset(smmu);
1717
1718 /* Push the button */
1719 arm_smmu_tlb_sync_global(smmu);
1720 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_sCR0, reg);
1721 }
1722
arm_smmu_id_size_to_bits(int size)1723 static int arm_smmu_id_size_to_bits(int size)
1724 {
1725 switch (size) {
1726 case 0:
1727 return 32;
1728 case 1:
1729 return 36;
1730 case 2:
1731 return 40;
1732 case 3:
1733 return 42;
1734 case 4:
1735 return 44;
1736 case 5:
1737 default:
1738 return 48;
1739 }
1740 }
1741
arm_smmu_device_cfg_probe(struct arm_smmu_device * smmu)1742 static int arm_smmu_device_cfg_probe(struct arm_smmu_device *smmu)
1743 {
1744 unsigned int size;
1745 u32 id;
1746 bool cttw_reg, cttw_fw = smmu->features & ARM_SMMU_FEAT_COHERENT_WALK;
1747 int i, ret;
1748
1749 dev_notice(smmu->dev, "probing hardware configuration...\n");
1750 dev_notice(smmu->dev, "SMMUv%d with:\n",
1751 smmu->version == ARM_SMMU_V2 ? 2 : 1);
1752
1753 /* ID0 */
1754 id = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_ID0);
1755
1756 /* Restrict available stages based on module parameter */
1757 if (force_stage == 1)
1758 id &= ~(ARM_SMMU_ID0_S2TS | ARM_SMMU_ID0_NTS);
1759 else if (force_stage == 2)
1760 id &= ~(ARM_SMMU_ID0_S1TS | ARM_SMMU_ID0_NTS);
1761
1762 if (id & ARM_SMMU_ID0_S1TS) {
1763 smmu->features |= ARM_SMMU_FEAT_TRANS_S1;
1764 dev_notice(smmu->dev, "\tstage 1 translation\n");
1765 }
1766
1767 if (id & ARM_SMMU_ID0_S2TS) {
1768 smmu->features |= ARM_SMMU_FEAT_TRANS_S2;
1769 dev_notice(smmu->dev, "\tstage 2 translation\n");
1770 }
1771
1772 if (id & ARM_SMMU_ID0_NTS) {
1773 smmu->features |= ARM_SMMU_FEAT_TRANS_NESTED;
1774 dev_notice(smmu->dev, "\tnested translation\n");
1775 }
1776
1777 if (!(smmu->features &
1778 (ARM_SMMU_FEAT_TRANS_S1 | ARM_SMMU_FEAT_TRANS_S2))) {
1779 dev_err(smmu->dev, "\tno translation support!\n");
1780 return -ENODEV;
1781 }
1782
1783 if ((id & ARM_SMMU_ID0_S1TS) &&
1784 ((smmu->version < ARM_SMMU_V2) || !(id & ARM_SMMU_ID0_ATOSNS))) {
1785 smmu->features |= ARM_SMMU_FEAT_TRANS_OPS;
1786 dev_notice(smmu->dev, "\taddress translation ops\n");
1787 }
1788
1789 /*
1790 * In order for DMA API calls to work properly, we must defer to what
1791 * the FW says about coherency, regardless of what the hardware claims.
1792 * Fortunately, this also opens up a workaround for systems where the
1793 * ID register value has ended up configured incorrectly.
1794 */
1795 cttw_reg = !!(id & ARM_SMMU_ID0_CTTW);
1796 if (cttw_fw || cttw_reg)
1797 dev_notice(smmu->dev, "\t%scoherent table walk\n",
1798 cttw_fw ? "" : "non-");
1799 if (cttw_fw != cttw_reg)
1800 dev_notice(smmu->dev,
1801 "\t(IDR0.CTTW overridden by FW configuration)\n");
1802
1803 /* Max. number of entries we have for stream matching/indexing */
1804 if (smmu->version == ARM_SMMU_V2 && id & ARM_SMMU_ID0_EXIDS) {
1805 smmu->features |= ARM_SMMU_FEAT_EXIDS;
1806 size = 1 << 16;
1807 } else {
1808 size = 1 << FIELD_GET(ARM_SMMU_ID0_NUMSIDB, id);
1809 }
1810 smmu->streamid_mask = size - 1;
1811 if (id & ARM_SMMU_ID0_SMS) {
1812 smmu->features |= ARM_SMMU_FEAT_STREAM_MATCH;
1813 size = FIELD_GET(ARM_SMMU_ID0_NUMSMRG, id);
1814 if (size == 0) {
1815 dev_err(smmu->dev,
1816 "stream-matching supported, but no SMRs present!\n");
1817 return -ENODEV;
1818 }
1819
1820 /* Zero-initialised to mark as invalid */
1821 smmu->smrs = devm_kcalloc(smmu->dev, size, sizeof(*smmu->smrs),
1822 GFP_KERNEL);
1823 if (!smmu->smrs)
1824 return -ENOMEM;
1825
1826 dev_notice(smmu->dev,
1827 "\tstream matching with %u register groups", size);
1828 }
1829 /* s2cr->type == 0 means translation, so initialise explicitly */
1830 smmu->s2crs = devm_kmalloc_array(smmu->dev, size, sizeof(*smmu->s2crs),
1831 GFP_KERNEL);
1832 if (!smmu->s2crs)
1833 return -ENOMEM;
1834 for (i = 0; i < size; i++)
1835 smmu->s2crs[i] = s2cr_init_val;
1836
1837 smmu->num_mapping_groups = size;
1838 mutex_init(&smmu->stream_map_mutex);
1839 spin_lock_init(&smmu->global_sync_lock);
1840
1841 if (smmu->version < ARM_SMMU_V2 ||
1842 !(id & ARM_SMMU_ID0_PTFS_NO_AARCH32)) {
1843 smmu->features |= ARM_SMMU_FEAT_FMT_AARCH32_L;
1844 if (!(id & ARM_SMMU_ID0_PTFS_NO_AARCH32S))
1845 smmu->features |= ARM_SMMU_FEAT_FMT_AARCH32_S;
1846 }
1847
1848 /* ID1 */
1849 id = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_ID1);
1850 smmu->pgshift = (id & ARM_SMMU_ID1_PAGESIZE) ? 16 : 12;
1851
1852 /* Check for size mismatch of SMMU address space from mapped region */
1853 size = 1 << (FIELD_GET(ARM_SMMU_ID1_NUMPAGENDXB, id) + 1);
1854 if (smmu->numpage != 2 * size << smmu->pgshift)
1855 dev_warn(smmu->dev,
1856 "SMMU address space size (0x%x) differs from mapped region size (0x%x)!\n",
1857 2 * size << smmu->pgshift, smmu->numpage);
1858 /* Now properly encode NUMPAGE to subsequently derive SMMU_CB_BASE */
1859 smmu->numpage = size;
1860
1861 smmu->num_s2_context_banks = FIELD_GET(ARM_SMMU_ID1_NUMS2CB, id);
1862 smmu->num_context_banks = FIELD_GET(ARM_SMMU_ID1_NUMCB, id);
1863 if (smmu->num_s2_context_banks > smmu->num_context_banks) {
1864 dev_err(smmu->dev, "impossible number of S2 context banks!\n");
1865 return -ENODEV;
1866 }
1867 dev_notice(smmu->dev, "\t%u context banks (%u stage-2 only)\n",
1868 smmu->num_context_banks, smmu->num_s2_context_banks);
1869 smmu->cbs = devm_kcalloc(smmu->dev, smmu->num_context_banks,
1870 sizeof(*smmu->cbs), GFP_KERNEL);
1871 if (!smmu->cbs)
1872 return -ENOMEM;
1873
1874 /* ID2 */
1875 id = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_ID2);
1876 size = arm_smmu_id_size_to_bits(FIELD_GET(ARM_SMMU_ID2_IAS, id));
1877 smmu->ipa_size = size;
1878
1879 /* The output mask is also applied for bypass */
1880 size = arm_smmu_id_size_to_bits(FIELD_GET(ARM_SMMU_ID2_OAS, id));
1881 smmu->pa_size = size;
1882
1883 if (id & ARM_SMMU_ID2_VMID16)
1884 smmu->features |= ARM_SMMU_FEAT_VMID16;
1885
1886 /*
1887 * What the page table walker can address actually depends on which
1888 * descriptor format is in use, but since a) we don't know that yet,
1889 * and b) it can vary per context bank, this will have to do...
1890 */
1891 if (dma_set_mask_and_coherent(smmu->dev, DMA_BIT_MASK(size)))
1892 dev_warn(smmu->dev,
1893 "failed to set DMA mask for table walker\n");
1894
1895 if (smmu->version < ARM_SMMU_V2) {
1896 smmu->va_size = smmu->ipa_size;
1897 if (smmu->version == ARM_SMMU_V1_64K)
1898 smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_64K;
1899 } else {
1900 size = FIELD_GET(ARM_SMMU_ID2_UBS, id);
1901 smmu->va_size = arm_smmu_id_size_to_bits(size);
1902 if (id & ARM_SMMU_ID2_PTFS_4K)
1903 smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_4K;
1904 if (id & ARM_SMMU_ID2_PTFS_16K)
1905 smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_16K;
1906 if (id & ARM_SMMU_ID2_PTFS_64K)
1907 smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_64K;
1908 }
1909
1910 if (smmu->impl && smmu->impl->cfg_probe) {
1911 ret = smmu->impl->cfg_probe(smmu);
1912 if (ret)
1913 return ret;
1914 }
1915
1916 /* Now we've corralled the various formats, what'll it do? */
1917 if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH32_S)
1918 smmu->pgsize_bitmap |= SZ_4K | SZ_64K | SZ_1M | SZ_16M;
1919 if (smmu->features &
1920 (ARM_SMMU_FEAT_FMT_AARCH32_L | ARM_SMMU_FEAT_FMT_AARCH64_4K))
1921 smmu->pgsize_bitmap |= SZ_4K | SZ_2M | SZ_1G;
1922 if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH64_16K)
1923 smmu->pgsize_bitmap |= SZ_16K | SZ_32M;
1924 if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH64_64K)
1925 smmu->pgsize_bitmap |= SZ_64K | SZ_512M;
1926
1927 dev_notice(smmu->dev, "\tSupported page sizes: 0x%08lx\n",
1928 smmu->pgsize_bitmap);
1929
1930
1931 if (smmu->features & ARM_SMMU_FEAT_TRANS_S1)
1932 dev_notice(smmu->dev, "\tStage-1: %lu-bit VA -> %lu-bit IPA\n",
1933 smmu->va_size, smmu->ipa_size);
1934
1935 if (smmu->features & ARM_SMMU_FEAT_TRANS_S2)
1936 dev_notice(smmu->dev, "\tStage-2: %lu-bit IPA -> %lu-bit PA\n",
1937 smmu->ipa_size, smmu->pa_size);
1938
1939 return 0;
1940 }
1941
1942 struct arm_smmu_match_data {
1943 enum arm_smmu_arch_version version;
1944 enum arm_smmu_implementation model;
1945 };
1946
1947 #define ARM_SMMU_MATCH_DATA(name, ver, imp) \
1948 static const struct arm_smmu_match_data name = { .version = ver, .model = imp }
1949
1950 ARM_SMMU_MATCH_DATA(smmu_generic_v1, ARM_SMMU_V1, GENERIC_SMMU);
1951 ARM_SMMU_MATCH_DATA(smmu_generic_v2, ARM_SMMU_V2, GENERIC_SMMU);
1952 ARM_SMMU_MATCH_DATA(arm_mmu401, ARM_SMMU_V1_64K, GENERIC_SMMU);
1953 ARM_SMMU_MATCH_DATA(arm_mmu500, ARM_SMMU_V2, ARM_MMU500);
1954 ARM_SMMU_MATCH_DATA(cavium_smmuv2, ARM_SMMU_V2, CAVIUM_SMMUV2);
1955 ARM_SMMU_MATCH_DATA(qcom_smmuv2, ARM_SMMU_V2, QCOM_SMMUV2);
1956
1957 static const struct of_device_id arm_smmu_of_match[] = {
1958 { .compatible = "arm,smmu-v1", .data = &smmu_generic_v1 },
1959 { .compatible = "arm,smmu-v2", .data = &smmu_generic_v2 },
1960 { .compatible = "arm,mmu-400", .data = &smmu_generic_v1 },
1961 { .compatible = "arm,mmu-401", .data = &arm_mmu401 },
1962 { .compatible = "arm,mmu-500", .data = &arm_mmu500 },
1963 { .compatible = "cavium,smmu-v2", .data = &cavium_smmuv2 },
1964 { .compatible = "nvidia,smmu-500", .data = &arm_mmu500 },
1965 { .compatible = "qcom,smmu-v2", .data = &qcom_smmuv2 },
1966 { },
1967 };
1968 MODULE_DEVICE_TABLE(of, arm_smmu_of_match);
1969
1970 #ifdef CONFIG_ACPI
acpi_smmu_get_data(u32 model,struct arm_smmu_device * smmu)1971 static int acpi_smmu_get_data(u32 model, struct arm_smmu_device *smmu)
1972 {
1973 int ret = 0;
1974
1975 switch (model) {
1976 case ACPI_IORT_SMMU_V1:
1977 case ACPI_IORT_SMMU_CORELINK_MMU400:
1978 smmu->version = ARM_SMMU_V1;
1979 smmu->model = GENERIC_SMMU;
1980 break;
1981 case ACPI_IORT_SMMU_CORELINK_MMU401:
1982 smmu->version = ARM_SMMU_V1_64K;
1983 smmu->model = GENERIC_SMMU;
1984 break;
1985 case ACPI_IORT_SMMU_V2:
1986 smmu->version = ARM_SMMU_V2;
1987 smmu->model = GENERIC_SMMU;
1988 break;
1989 case ACPI_IORT_SMMU_CORELINK_MMU500:
1990 smmu->version = ARM_SMMU_V2;
1991 smmu->model = ARM_MMU500;
1992 break;
1993 case ACPI_IORT_SMMU_CAVIUM_THUNDERX:
1994 smmu->version = ARM_SMMU_V2;
1995 smmu->model = CAVIUM_SMMUV2;
1996 break;
1997 default:
1998 ret = -ENODEV;
1999 }
2000
2001 return ret;
2002 }
2003
arm_smmu_device_acpi_probe(struct arm_smmu_device * smmu,u32 * global_irqs,u32 * pmu_irqs)2004 static int arm_smmu_device_acpi_probe(struct arm_smmu_device *smmu,
2005 u32 *global_irqs, u32 *pmu_irqs)
2006 {
2007 struct device *dev = smmu->dev;
2008 struct acpi_iort_node *node =
2009 *(struct acpi_iort_node **)dev_get_platdata(dev);
2010 struct acpi_iort_smmu *iort_smmu;
2011 int ret;
2012
2013 /* Retrieve SMMU1/2 specific data */
2014 iort_smmu = (struct acpi_iort_smmu *)node->node_data;
2015
2016 ret = acpi_smmu_get_data(iort_smmu->model, smmu);
2017 if (ret < 0)
2018 return ret;
2019
2020 /* Ignore the configuration access interrupt */
2021 *global_irqs = 1;
2022 *pmu_irqs = 0;
2023
2024 if (iort_smmu->flags & ACPI_IORT_SMMU_COHERENT_WALK)
2025 smmu->features |= ARM_SMMU_FEAT_COHERENT_WALK;
2026
2027 return 0;
2028 }
2029 #else
arm_smmu_device_acpi_probe(struct arm_smmu_device * smmu,u32 * global_irqs,u32 * pmu_irqs)2030 static inline int arm_smmu_device_acpi_probe(struct arm_smmu_device *smmu,
2031 u32 *global_irqs, u32 *pmu_irqs)
2032 {
2033 return -ENODEV;
2034 }
2035 #endif
2036
arm_smmu_device_dt_probe(struct arm_smmu_device * smmu,u32 * global_irqs,u32 * pmu_irqs)2037 static int arm_smmu_device_dt_probe(struct arm_smmu_device *smmu,
2038 u32 *global_irqs, u32 *pmu_irqs)
2039 {
2040 const struct arm_smmu_match_data *data;
2041 struct device *dev = smmu->dev;
2042 bool legacy_binding;
2043
2044 if (of_property_read_u32(dev->of_node, "#global-interrupts", global_irqs))
2045 return dev_err_probe(dev, -ENODEV,
2046 "missing #global-interrupts property\n");
2047 *pmu_irqs = 0;
2048
2049 data = of_device_get_match_data(dev);
2050 smmu->version = data->version;
2051 smmu->model = data->model;
2052
2053 legacy_binding = of_find_property(dev->of_node, "mmu-masters", NULL);
2054 if (legacy_binding && !using_generic_binding) {
2055 if (!using_legacy_binding) {
2056 pr_notice("deprecated \"mmu-masters\" DT property in use; %s support unavailable\n",
2057 IS_ENABLED(CONFIG_ARM_SMMU_LEGACY_DT_BINDINGS) ? "DMA API" : "SMMU");
2058 }
2059 using_legacy_binding = true;
2060 } else if (!legacy_binding && !using_legacy_binding) {
2061 using_generic_binding = true;
2062 } else {
2063 dev_err(dev, "not probing due to mismatched DT properties\n");
2064 return -ENODEV;
2065 }
2066
2067 if (of_dma_is_coherent(dev->of_node))
2068 smmu->features |= ARM_SMMU_FEAT_COHERENT_WALK;
2069
2070 return 0;
2071 }
2072
arm_smmu_rmr_install_bypass_smr(struct arm_smmu_device * smmu)2073 static void arm_smmu_rmr_install_bypass_smr(struct arm_smmu_device *smmu)
2074 {
2075 struct list_head rmr_list;
2076 struct iommu_resv_region *e;
2077 int idx, cnt = 0;
2078 u32 reg;
2079
2080 INIT_LIST_HEAD(&rmr_list);
2081 iort_get_rmr_sids(dev_fwnode(smmu->dev), &rmr_list);
2082
2083 /*
2084 * Rather than trying to look at existing mappings that
2085 * are setup by the firmware and then invalidate the ones
2086 * that do no have matching RMR entries, just disable the
2087 * SMMU until it gets enabled again in the reset routine.
2088 */
2089 reg = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sCR0);
2090 reg |= ARM_SMMU_sCR0_CLIENTPD;
2091 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_sCR0, reg);
2092
2093 list_for_each_entry(e, &rmr_list, list) {
2094 struct iommu_iort_rmr_data *rmr;
2095 int i;
2096
2097 rmr = container_of(e, struct iommu_iort_rmr_data, rr);
2098 for (i = 0; i < rmr->num_sids; i++) {
2099 idx = arm_smmu_find_sme(smmu, rmr->sids[i], ~0);
2100 if (idx < 0)
2101 continue;
2102
2103 if (smmu->s2crs[idx].count == 0) {
2104 smmu->smrs[idx].id = rmr->sids[i];
2105 smmu->smrs[idx].mask = 0;
2106 smmu->smrs[idx].valid = true;
2107 }
2108 smmu->s2crs[idx].count++;
2109 smmu->s2crs[idx].type = S2CR_TYPE_BYPASS;
2110 smmu->s2crs[idx].privcfg = S2CR_PRIVCFG_DEFAULT;
2111
2112 cnt++;
2113 }
2114 }
2115
2116 dev_notice(smmu->dev, "\tpreserved %d boot mapping%s\n", cnt,
2117 str_plural(cnt));
2118 iort_put_rmr_sids(dev_fwnode(smmu->dev), &rmr_list);
2119 }
2120
arm_smmu_device_probe(struct platform_device * pdev)2121 static int arm_smmu_device_probe(struct platform_device *pdev)
2122 {
2123 struct resource *res;
2124 struct arm_smmu_device *smmu;
2125 struct device *dev = &pdev->dev;
2126 int num_irqs, i, err;
2127 u32 global_irqs, pmu_irqs;
2128 irqreturn_t (*global_fault)(int irq, void *dev);
2129
2130 smmu = devm_kzalloc(dev, sizeof(*smmu), GFP_KERNEL);
2131 if (!smmu) {
2132 dev_err(dev, "failed to allocate arm_smmu_device\n");
2133 return -ENOMEM;
2134 }
2135 smmu->dev = dev;
2136
2137 if (dev->of_node)
2138 err = arm_smmu_device_dt_probe(smmu, &global_irqs, &pmu_irqs);
2139 else
2140 err = arm_smmu_device_acpi_probe(smmu, &global_irqs, &pmu_irqs);
2141 if (err)
2142 return err;
2143
2144 smmu->base = devm_platform_get_and_ioremap_resource(pdev, 0, &res);
2145 if (IS_ERR(smmu->base))
2146 return PTR_ERR(smmu->base);
2147 smmu->ioaddr = res->start;
2148
2149 /*
2150 * The resource size should effectively match the value of SMMU_TOP;
2151 * stash that temporarily until we know PAGESIZE to validate it with.
2152 */
2153 smmu->numpage = resource_size(res);
2154
2155 smmu = arm_smmu_impl_init(smmu);
2156 if (IS_ERR(smmu))
2157 return PTR_ERR(smmu);
2158
2159 num_irqs = platform_irq_count(pdev);
2160
2161 smmu->num_context_irqs = num_irqs - global_irqs - pmu_irqs;
2162 if (smmu->num_context_irqs <= 0)
2163 return dev_err_probe(dev, -ENODEV,
2164 "found %d interrupts but expected at least %d\n",
2165 num_irqs, global_irqs + pmu_irqs + 1);
2166
2167 smmu->irqs = devm_kcalloc(dev, smmu->num_context_irqs,
2168 sizeof(*smmu->irqs), GFP_KERNEL);
2169 if (!smmu->irqs)
2170 return dev_err_probe(dev, -ENOMEM, "failed to allocate %d irqs\n",
2171 smmu->num_context_irqs);
2172
2173 for (i = 0; i < smmu->num_context_irqs; i++) {
2174 int irq = platform_get_irq(pdev, global_irqs + pmu_irqs + i);
2175
2176 if (irq < 0)
2177 return irq;
2178 smmu->irqs[i] = irq;
2179 }
2180
2181 err = devm_clk_bulk_get_all(dev, &smmu->clks);
2182 if (err < 0) {
2183 dev_err(dev, "failed to get clocks %d\n", err);
2184 return err;
2185 }
2186 smmu->num_clks = err;
2187
2188 err = clk_bulk_prepare_enable(smmu->num_clks, smmu->clks);
2189 if (err)
2190 return err;
2191
2192 err = arm_smmu_device_cfg_probe(smmu);
2193 if (err)
2194 return err;
2195
2196 if (smmu->version == ARM_SMMU_V2) {
2197 if (smmu->num_context_banks > smmu->num_context_irqs) {
2198 dev_err(dev,
2199 "found only %d context irq(s) but %d required\n",
2200 smmu->num_context_irqs, smmu->num_context_banks);
2201 return -ENODEV;
2202 }
2203
2204 /* Ignore superfluous interrupts */
2205 smmu->num_context_irqs = smmu->num_context_banks;
2206 }
2207
2208 if (smmu->impl && smmu->impl->global_fault)
2209 global_fault = smmu->impl->global_fault;
2210 else
2211 global_fault = arm_smmu_global_fault;
2212
2213 for (i = 0; i < global_irqs; i++) {
2214 int irq = platform_get_irq(pdev, i);
2215
2216 if (irq < 0)
2217 return irq;
2218
2219 err = devm_request_irq(dev, irq, global_fault, IRQF_SHARED,
2220 "arm-smmu global fault", smmu);
2221 if (err)
2222 return dev_err_probe(dev, err,
2223 "failed to request global IRQ %d (%u)\n",
2224 i, irq);
2225 }
2226
2227 platform_set_drvdata(pdev, smmu);
2228
2229 /* Check for RMRs and install bypass SMRs if any */
2230 arm_smmu_rmr_install_bypass_smr(smmu);
2231
2232 arm_smmu_device_reset(smmu);
2233 arm_smmu_test_smr_masks(smmu);
2234
2235 err = iommu_device_sysfs_add(&smmu->iommu, smmu->dev, NULL,
2236 "smmu.%pa", &smmu->ioaddr);
2237 if (err)
2238 return dev_err_probe(dev, err, "Failed to register iommu in sysfs\n");
2239
2240 err = iommu_device_register(&smmu->iommu, &arm_smmu_ops,
2241 using_legacy_binding ? NULL : dev);
2242 if (err) {
2243 iommu_device_sysfs_remove(&smmu->iommu);
2244 return dev_err_probe(dev, err, "Failed to register iommu\n");
2245 }
2246
2247 /*
2248 * We want to avoid touching dev->power.lock in fastpaths unless
2249 * it's really going to do something useful - pm_runtime_enabled()
2250 * can serve as an ideal proxy for that decision. So, conditionally
2251 * enable pm_runtime.
2252 */
2253 if (dev->pm_domain) {
2254 pm_runtime_set_active(dev);
2255 pm_runtime_enable(dev);
2256 arm_smmu_rpm_use_autosuspend(smmu);
2257 }
2258
2259 return 0;
2260 }
2261
arm_smmu_device_shutdown(struct platform_device * pdev)2262 static void arm_smmu_device_shutdown(struct platform_device *pdev)
2263 {
2264 struct arm_smmu_device *smmu = platform_get_drvdata(pdev);
2265
2266 if (!bitmap_empty(smmu->context_map, ARM_SMMU_MAX_CBS))
2267 dev_notice(&pdev->dev, "disabling translation\n");
2268
2269 arm_smmu_rpm_get(smmu);
2270 /* Turn the thing off */
2271 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_sCR0, ARM_SMMU_sCR0_CLIENTPD);
2272 arm_smmu_rpm_put(smmu);
2273
2274 if (pm_runtime_enabled(smmu->dev))
2275 pm_runtime_force_suspend(smmu->dev);
2276 else
2277 clk_bulk_disable(smmu->num_clks, smmu->clks);
2278
2279 clk_bulk_unprepare(smmu->num_clks, smmu->clks);
2280 }
2281
arm_smmu_device_remove(struct platform_device * pdev)2282 static void arm_smmu_device_remove(struct platform_device *pdev)
2283 {
2284 struct arm_smmu_device *smmu = platform_get_drvdata(pdev);
2285
2286 iommu_device_unregister(&smmu->iommu);
2287 iommu_device_sysfs_remove(&smmu->iommu);
2288
2289 arm_smmu_device_shutdown(pdev);
2290 }
2291
arm_smmu_runtime_resume(struct device * dev)2292 static int __maybe_unused arm_smmu_runtime_resume(struct device *dev)
2293 {
2294 struct arm_smmu_device *smmu = dev_get_drvdata(dev);
2295 int ret;
2296
2297 ret = clk_bulk_enable(smmu->num_clks, smmu->clks);
2298 if (ret)
2299 return ret;
2300
2301 arm_smmu_device_reset(smmu);
2302
2303 return 0;
2304 }
2305
arm_smmu_runtime_suspend(struct device * dev)2306 static int __maybe_unused arm_smmu_runtime_suspend(struct device *dev)
2307 {
2308 struct arm_smmu_device *smmu = dev_get_drvdata(dev);
2309
2310 clk_bulk_disable(smmu->num_clks, smmu->clks);
2311
2312 return 0;
2313 }
2314
arm_smmu_pm_resume(struct device * dev)2315 static int __maybe_unused arm_smmu_pm_resume(struct device *dev)
2316 {
2317 int ret;
2318 struct arm_smmu_device *smmu = dev_get_drvdata(dev);
2319
2320 ret = clk_bulk_prepare(smmu->num_clks, smmu->clks);
2321 if (ret)
2322 return ret;
2323
2324 if (pm_runtime_suspended(dev))
2325 return 0;
2326
2327 ret = arm_smmu_runtime_resume(dev);
2328 if (ret)
2329 clk_bulk_unprepare(smmu->num_clks, smmu->clks);
2330
2331 return ret;
2332 }
2333
arm_smmu_pm_suspend(struct device * dev)2334 static int __maybe_unused arm_smmu_pm_suspend(struct device *dev)
2335 {
2336 int ret = 0;
2337 struct arm_smmu_device *smmu = dev_get_drvdata(dev);
2338
2339 if (pm_runtime_suspended(dev))
2340 goto clk_unprepare;
2341
2342 ret = arm_smmu_runtime_suspend(dev);
2343 if (ret)
2344 return ret;
2345
2346 clk_unprepare:
2347 clk_bulk_unprepare(smmu->num_clks, smmu->clks);
2348 return ret;
2349 }
2350
2351 static const struct dev_pm_ops arm_smmu_pm_ops = {
2352 SET_SYSTEM_SLEEP_PM_OPS(arm_smmu_pm_suspend, arm_smmu_pm_resume)
2353 SET_RUNTIME_PM_OPS(arm_smmu_runtime_suspend,
2354 arm_smmu_runtime_resume, NULL)
2355 };
2356
2357 static struct platform_driver arm_smmu_driver = {
2358 .driver = {
2359 .name = "arm-smmu",
2360 .of_match_table = arm_smmu_of_match,
2361 .pm = &arm_smmu_pm_ops,
2362 .suppress_bind_attrs = true,
2363 },
2364 .probe = arm_smmu_device_probe,
2365 .remove = arm_smmu_device_remove,
2366 .shutdown = arm_smmu_device_shutdown,
2367 };
2368 module_platform_driver(arm_smmu_driver);
2369
2370 MODULE_DESCRIPTION("IOMMU API for ARM architected SMMU implementations");
2371 MODULE_AUTHOR("Will Deacon <will@kernel.org>");
2372 MODULE_ALIAS("platform:arm-smmu");
2373 MODULE_LICENSE("GPL v2");
2374