1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * IOMMU API for ARM architected SMMU implementations.
4 *
5 * Copyright (C) 2013 ARM Limited
6 *
7 * Author: Will Deacon <will.deacon@arm.com>
8 *
9 * This driver currently supports:
10 * - SMMUv1 and v2 implementations
11 * - Stream-matching and stream-indexing
12 * - v7/v8 long-descriptor format
13 * - Non-secure access to the SMMU
14 * - Context fault reporting
15 * - Extended Stream ID (16 bit)
16 */
17
18 #define pr_fmt(fmt) "arm-smmu: " fmt
19
20 #include <linux/acpi.h>
21 #include <linux/acpi_iort.h>
22 #include <linux/bitfield.h>
23 #include <linux/delay.h>
24 #include <linux/dma-mapping.h>
25 #include <linux/err.h>
26 #include <linux/interrupt.h>
27 #include <linux/io.h>
28 #include <linux/iopoll.h>
29 #include <linux/module.h>
30 #include <linux/of.h>
31 #include <linux/of_address.h>
32 #include <linux/pci.h>
33 #include <linux/platform_device.h>
34 #include <linux/pm_runtime.h>
35 #include <linux/ratelimit.h>
36 #include <linux/slab.h>
37 #include <linux/string_choices.h>
38
39 #include <linux/fsl/mc.h>
40
41 #include "arm-smmu.h"
42 #include "../../dma-iommu.h"
43
44 /*
45 * Apparently, some Qualcomm arm64 platforms which appear to expose their SMMU
46 * global register space are still, in fact, using a hypervisor to mediate it
47 * by trapping and emulating register accesses. Sadly, some deployed versions
48 * of said trapping code have bugs wherein they go horribly wrong for stores
49 * using r31 (i.e. XZR/WZR) as the source register.
50 */
51 #define QCOM_DUMMY_VAL -1
52
53 #define MSI_IOVA_BASE 0x8000000
54 #define MSI_IOVA_LENGTH 0x100000
55
56 static int force_stage;
57 module_param(force_stage, int, S_IRUGO);
58 MODULE_PARM_DESC(force_stage,
59 "Force SMMU mappings to be installed at a particular stage of translation. A value of '1' or '2' forces the corresponding stage. All other values are ignored (i.e. no stage is forced). Note that selecting a specific stage will disable support for nested translation.");
60 static bool disable_bypass =
61 IS_ENABLED(CONFIG_ARM_SMMU_DISABLE_BYPASS_BY_DEFAULT);
62 module_param(disable_bypass, bool, S_IRUGO);
63 MODULE_PARM_DESC(disable_bypass,
64 "Disable bypass streams such that incoming transactions from devices that are not attached to an iommu domain will report an abort back to the device and will not be allowed to pass through the SMMU.");
65
66 #define s2cr_init_val (struct arm_smmu_s2cr){ \
67 .type = disable_bypass ? S2CR_TYPE_FAULT : S2CR_TYPE_BYPASS, \
68 }
69
70 static bool using_legacy_binding, using_generic_binding;
71
arm_smmu_rpm_get(struct arm_smmu_device * smmu)72 static inline int arm_smmu_rpm_get(struct arm_smmu_device *smmu)
73 {
74 if (pm_runtime_enabled(smmu->dev))
75 return pm_runtime_resume_and_get(smmu->dev);
76
77 return 0;
78 }
79
arm_smmu_rpm_put(struct arm_smmu_device * smmu)80 static inline void arm_smmu_rpm_put(struct arm_smmu_device *smmu)
81 {
82 if (pm_runtime_enabled(smmu->dev)) {
83 pm_runtime_mark_last_busy(smmu->dev);
84 __pm_runtime_put_autosuspend(smmu->dev);
85
86 }
87 }
88
arm_smmu_rpm_use_autosuspend(struct arm_smmu_device * smmu)89 static void arm_smmu_rpm_use_autosuspend(struct arm_smmu_device *smmu)
90 {
91 /*
92 * Setup an autosuspend delay to avoid bouncing runpm state.
93 * Otherwise, if a driver for a suspended consumer device
94 * unmaps buffers, it will runpm resume/suspend for each one.
95 *
96 * For example, when used by a GPU device, when an application
97 * or game exits, it can trigger unmapping 100s or 1000s of
98 * buffers. With a runpm cycle for each buffer, that adds up
99 * to 5-10sec worth of reprogramming the context bank, while
100 * the system appears to be locked up to the user.
101 */
102 pm_runtime_set_autosuspend_delay(smmu->dev, 20);
103 pm_runtime_use_autosuspend(smmu->dev);
104 }
105
to_smmu_domain(struct iommu_domain * dom)106 static struct arm_smmu_domain *to_smmu_domain(struct iommu_domain *dom)
107 {
108 return container_of(dom, struct arm_smmu_domain, domain);
109 }
110
111 static struct platform_driver arm_smmu_driver;
112 static const struct iommu_ops arm_smmu_ops;
113
114 #ifdef CONFIG_ARM_SMMU_LEGACY_DT_BINDINGS
dev_get_dev_node(struct device * dev)115 static struct device_node *dev_get_dev_node(struct device *dev)
116 {
117 if (dev_is_pci(dev)) {
118 struct pci_bus *bus = to_pci_dev(dev)->bus;
119
120 while (!pci_is_root_bus(bus))
121 bus = bus->parent;
122 return of_node_get(bus->bridge->parent->of_node);
123 }
124
125 return of_node_get(dev->of_node);
126 }
127
__arm_smmu_get_pci_sid(struct pci_dev * pdev,u16 alias,void * data)128 static int __arm_smmu_get_pci_sid(struct pci_dev *pdev, u16 alias, void *data)
129 {
130 *((__be32 *)data) = cpu_to_be32(alias);
131 return 0; /* Continue walking */
132 }
133
__find_legacy_master_phandle(struct device * dev,void * data)134 static int __find_legacy_master_phandle(struct device *dev, void *data)
135 {
136 struct of_phandle_iterator *it = *(void **)data;
137 struct device_node *np = it->node;
138 int err;
139
140 of_for_each_phandle(it, err, dev->of_node, "mmu-masters",
141 "#stream-id-cells", -1)
142 if (it->node == np) {
143 *(void **)data = dev;
144 return 1;
145 }
146 it->node = np;
147 return err == -ENOENT ? 0 : err;
148 }
149
arm_smmu_register_legacy_master(struct device * dev,struct arm_smmu_device ** smmu)150 static int arm_smmu_register_legacy_master(struct device *dev,
151 struct arm_smmu_device **smmu)
152 {
153 struct device *smmu_dev;
154 struct device_node *np;
155 struct of_phandle_iterator it;
156 void *data = ⁢
157 u32 *sids;
158 __be32 pci_sid;
159 int err;
160
161 np = dev_get_dev_node(dev);
162 if (!np || !of_property_present(np, "#stream-id-cells")) {
163 of_node_put(np);
164 return -ENODEV;
165 }
166
167 it.node = np;
168 err = driver_for_each_device(&arm_smmu_driver.driver, NULL, &data,
169 __find_legacy_master_phandle);
170 smmu_dev = data;
171 of_node_put(np);
172 if (err == 0)
173 return -ENODEV;
174 if (err < 0)
175 return err;
176
177 if (dev_is_pci(dev)) {
178 /* "mmu-masters" assumes Stream ID == Requester ID */
179 pci_for_each_dma_alias(to_pci_dev(dev), __arm_smmu_get_pci_sid,
180 &pci_sid);
181 it.cur = &pci_sid;
182 it.cur_count = 1;
183 }
184
185 err = iommu_fwspec_init(dev, NULL);
186 if (err)
187 return err;
188
189 sids = kcalloc(it.cur_count, sizeof(*sids), GFP_KERNEL);
190 if (!sids)
191 return -ENOMEM;
192
193 *smmu = dev_get_drvdata(smmu_dev);
194 of_phandle_iterator_args(&it, sids, it.cur_count);
195 err = iommu_fwspec_add_ids(dev, sids, it.cur_count);
196 kfree(sids);
197 return err;
198 }
199 #else
arm_smmu_register_legacy_master(struct device * dev,struct arm_smmu_device ** smmu)200 static int arm_smmu_register_legacy_master(struct device *dev,
201 struct arm_smmu_device **smmu)
202 {
203 return -ENODEV;
204 }
205 #endif /* CONFIG_ARM_SMMU_LEGACY_DT_BINDINGS */
206
__arm_smmu_free_bitmap(unsigned long * map,int idx)207 static void __arm_smmu_free_bitmap(unsigned long *map, int idx)
208 {
209 clear_bit(idx, map);
210 }
211
212 /* Wait for any pending TLB invalidations to complete */
__arm_smmu_tlb_sync(struct arm_smmu_device * smmu,int page,int sync,int status)213 static void __arm_smmu_tlb_sync(struct arm_smmu_device *smmu, int page,
214 int sync, int status)
215 {
216 unsigned int spin_cnt, delay;
217 u32 reg;
218
219 if (smmu->impl && unlikely(smmu->impl->tlb_sync))
220 return smmu->impl->tlb_sync(smmu, page, sync, status);
221
222 arm_smmu_writel(smmu, page, sync, QCOM_DUMMY_VAL);
223 for (delay = 1; delay < TLB_LOOP_TIMEOUT; delay *= 2) {
224 for (spin_cnt = TLB_SPIN_COUNT; spin_cnt > 0; spin_cnt--) {
225 reg = arm_smmu_readl(smmu, page, status);
226 if (!(reg & ARM_SMMU_sTLBGSTATUS_GSACTIVE))
227 return;
228 cpu_relax();
229 }
230 udelay(delay);
231 }
232 dev_err_ratelimited(smmu->dev,
233 "TLB sync timed out -- SMMU may be deadlocked\n");
234 }
235
arm_smmu_tlb_sync_global(struct arm_smmu_device * smmu)236 static void arm_smmu_tlb_sync_global(struct arm_smmu_device *smmu)
237 {
238 unsigned long flags;
239
240 spin_lock_irqsave(&smmu->global_sync_lock, flags);
241 __arm_smmu_tlb_sync(smmu, ARM_SMMU_GR0, ARM_SMMU_GR0_sTLBGSYNC,
242 ARM_SMMU_GR0_sTLBGSTATUS);
243 spin_unlock_irqrestore(&smmu->global_sync_lock, flags);
244 }
245
arm_smmu_tlb_sync_context(struct arm_smmu_domain * smmu_domain)246 static void arm_smmu_tlb_sync_context(struct arm_smmu_domain *smmu_domain)
247 {
248 struct arm_smmu_device *smmu = smmu_domain->smmu;
249 unsigned long flags;
250
251 spin_lock_irqsave(&smmu_domain->cb_lock, flags);
252 __arm_smmu_tlb_sync(smmu, ARM_SMMU_CB(smmu, smmu_domain->cfg.cbndx),
253 ARM_SMMU_CB_TLBSYNC, ARM_SMMU_CB_TLBSTATUS);
254 spin_unlock_irqrestore(&smmu_domain->cb_lock, flags);
255 }
256
arm_smmu_tlb_inv_context_s1(void * cookie)257 static void arm_smmu_tlb_inv_context_s1(void *cookie)
258 {
259 struct arm_smmu_domain *smmu_domain = cookie;
260 /*
261 * The TLBI write may be relaxed, so ensure that PTEs cleared by the
262 * current CPU are visible beforehand.
263 */
264 wmb();
265 arm_smmu_cb_write(smmu_domain->smmu, smmu_domain->cfg.cbndx,
266 ARM_SMMU_CB_S1_TLBIASID, smmu_domain->cfg.asid);
267 arm_smmu_tlb_sync_context(smmu_domain);
268 }
269
arm_smmu_tlb_inv_context_s2(void * cookie)270 static void arm_smmu_tlb_inv_context_s2(void *cookie)
271 {
272 struct arm_smmu_domain *smmu_domain = cookie;
273 struct arm_smmu_device *smmu = smmu_domain->smmu;
274
275 /* See above */
276 wmb();
277 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_TLBIVMID, smmu_domain->cfg.vmid);
278 arm_smmu_tlb_sync_global(smmu);
279 }
280
arm_smmu_tlb_inv_range_s1(unsigned long iova,size_t size,size_t granule,void * cookie,int reg)281 static void arm_smmu_tlb_inv_range_s1(unsigned long iova, size_t size,
282 size_t granule, void *cookie, int reg)
283 {
284 struct arm_smmu_domain *smmu_domain = cookie;
285 struct arm_smmu_device *smmu = smmu_domain->smmu;
286 struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
287 int idx = cfg->cbndx;
288
289 if (smmu->features & ARM_SMMU_FEAT_COHERENT_WALK)
290 wmb();
291
292 if (cfg->fmt != ARM_SMMU_CTX_FMT_AARCH64) {
293 iova = (iova >> 12) << 12;
294 iova |= cfg->asid;
295 do {
296 arm_smmu_cb_write(smmu, idx, reg, iova);
297 iova += granule;
298 } while (size -= granule);
299 } else {
300 iova >>= 12;
301 iova |= (u64)cfg->asid << 48;
302 do {
303 arm_smmu_cb_writeq(smmu, idx, reg, iova);
304 iova += granule >> 12;
305 } while (size -= granule);
306 }
307 }
308
arm_smmu_tlb_inv_range_s2(unsigned long iova,size_t size,size_t granule,void * cookie,int reg)309 static void arm_smmu_tlb_inv_range_s2(unsigned long iova, size_t size,
310 size_t granule, void *cookie, int reg)
311 {
312 struct arm_smmu_domain *smmu_domain = cookie;
313 struct arm_smmu_device *smmu = smmu_domain->smmu;
314 int idx = smmu_domain->cfg.cbndx;
315
316 if (smmu->features & ARM_SMMU_FEAT_COHERENT_WALK)
317 wmb();
318
319 iova >>= 12;
320 do {
321 if (smmu_domain->cfg.fmt == ARM_SMMU_CTX_FMT_AARCH64)
322 arm_smmu_cb_writeq(smmu, idx, reg, iova);
323 else
324 arm_smmu_cb_write(smmu, idx, reg, iova);
325 iova += granule >> 12;
326 } while (size -= granule);
327 }
328
arm_smmu_tlb_inv_walk_s1(unsigned long iova,size_t size,size_t granule,void * cookie)329 static void arm_smmu_tlb_inv_walk_s1(unsigned long iova, size_t size,
330 size_t granule, void *cookie)
331 {
332 struct arm_smmu_domain *smmu_domain = cookie;
333 struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
334
335 if (cfg->flush_walk_prefer_tlbiasid) {
336 arm_smmu_tlb_inv_context_s1(cookie);
337 } else {
338 arm_smmu_tlb_inv_range_s1(iova, size, granule, cookie,
339 ARM_SMMU_CB_S1_TLBIVA);
340 arm_smmu_tlb_sync_context(cookie);
341 }
342 }
343
arm_smmu_tlb_add_page_s1(struct iommu_iotlb_gather * gather,unsigned long iova,size_t granule,void * cookie)344 static void arm_smmu_tlb_add_page_s1(struct iommu_iotlb_gather *gather,
345 unsigned long iova, size_t granule,
346 void *cookie)
347 {
348 arm_smmu_tlb_inv_range_s1(iova, granule, granule, cookie,
349 ARM_SMMU_CB_S1_TLBIVAL);
350 }
351
arm_smmu_tlb_inv_walk_s2(unsigned long iova,size_t size,size_t granule,void * cookie)352 static void arm_smmu_tlb_inv_walk_s2(unsigned long iova, size_t size,
353 size_t granule, void *cookie)
354 {
355 arm_smmu_tlb_inv_range_s2(iova, size, granule, cookie,
356 ARM_SMMU_CB_S2_TLBIIPAS2);
357 arm_smmu_tlb_sync_context(cookie);
358 }
359
arm_smmu_tlb_add_page_s2(struct iommu_iotlb_gather * gather,unsigned long iova,size_t granule,void * cookie)360 static void arm_smmu_tlb_add_page_s2(struct iommu_iotlb_gather *gather,
361 unsigned long iova, size_t granule,
362 void *cookie)
363 {
364 arm_smmu_tlb_inv_range_s2(iova, granule, granule, cookie,
365 ARM_SMMU_CB_S2_TLBIIPAS2L);
366 }
367
arm_smmu_tlb_inv_walk_s2_v1(unsigned long iova,size_t size,size_t granule,void * cookie)368 static void arm_smmu_tlb_inv_walk_s2_v1(unsigned long iova, size_t size,
369 size_t granule, void *cookie)
370 {
371 arm_smmu_tlb_inv_context_s2(cookie);
372 }
373 /*
374 * On MMU-401 at least, the cost of firing off multiple TLBIVMIDs appears
375 * almost negligible, but the benefit of getting the first one in as far ahead
376 * of the sync as possible is significant, hence we don't just make this a
377 * no-op and call arm_smmu_tlb_inv_context_s2() from .iotlb_sync as you might
378 * think.
379 */
arm_smmu_tlb_add_page_s2_v1(struct iommu_iotlb_gather * gather,unsigned long iova,size_t granule,void * cookie)380 static void arm_smmu_tlb_add_page_s2_v1(struct iommu_iotlb_gather *gather,
381 unsigned long iova, size_t granule,
382 void *cookie)
383 {
384 struct arm_smmu_domain *smmu_domain = cookie;
385 struct arm_smmu_device *smmu = smmu_domain->smmu;
386
387 if (smmu->features & ARM_SMMU_FEAT_COHERENT_WALK)
388 wmb();
389
390 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_TLBIVMID, smmu_domain->cfg.vmid);
391 }
392
393 static const struct iommu_flush_ops arm_smmu_s1_tlb_ops = {
394 .tlb_flush_all = arm_smmu_tlb_inv_context_s1,
395 .tlb_flush_walk = arm_smmu_tlb_inv_walk_s1,
396 .tlb_add_page = arm_smmu_tlb_add_page_s1,
397 };
398
399 static const struct iommu_flush_ops arm_smmu_s2_tlb_ops_v2 = {
400 .tlb_flush_all = arm_smmu_tlb_inv_context_s2,
401 .tlb_flush_walk = arm_smmu_tlb_inv_walk_s2,
402 .tlb_add_page = arm_smmu_tlb_add_page_s2,
403 };
404
405 static const struct iommu_flush_ops arm_smmu_s2_tlb_ops_v1 = {
406 .tlb_flush_all = arm_smmu_tlb_inv_context_s2,
407 .tlb_flush_walk = arm_smmu_tlb_inv_walk_s2_v1,
408 .tlb_add_page = arm_smmu_tlb_add_page_s2_v1,
409 };
410
411
arm_smmu_read_context_fault_info(struct arm_smmu_device * smmu,int idx,struct arm_smmu_context_fault_info * cfi)412 void arm_smmu_read_context_fault_info(struct arm_smmu_device *smmu, int idx,
413 struct arm_smmu_context_fault_info *cfi)
414 {
415 cfi->iova = arm_smmu_cb_readq(smmu, idx, ARM_SMMU_CB_FAR);
416 cfi->fsr = arm_smmu_cb_read(smmu, idx, ARM_SMMU_CB_FSR);
417 cfi->fsynr = arm_smmu_cb_read(smmu, idx, ARM_SMMU_CB_FSYNR0);
418 cfi->cbfrsynra = arm_smmu_gr1_read(smmu, ARM_SMMU_GR1_CBFRSYNRA(idx));
419 }
420
arm_smmu_print_context_fault_info(struct arm_smmu_device * smmu,int idx,const struct arm_smmu_context_fault_info * cfi)421 void arm_smmu_print_context_fault_info(struct arm_smmu_device *smmu, int idx,
422 const struct arm_smmu_context_fault_info *cfi)
423 {
424 dev_err(smmu->dev,
425 "Unhandled context fault: fsr=0x%x, iova=0x%08lx, fsynr=0x%x, cbfrsynra=0x%x, cb=%d\n",
426 cfi->fsr, cfi->iova, cfi->fsynr, cfi->cbfrsynra, idx);
427
428 dev_err(smmu->dev, "FSR = %08x [%s%sFormat=%u%s%s%s%s%s%s%s%s], SID=0x%x\n",
429 cfi->fsr,
430 (cfi->fsr & ARM_SMMU_CB_FSR_MULTI) ? "MULTI " : "",
431 (cfi->fsr & ARM_SMMU_CB_FSR_SS) ? "SS " : "",
432 (u32)FIELD_GET(ARM_SMMU_CB_FSR_FORMAT, cfi->fsr),
433 (cfi->fsr & ARM_SMMU_CB_FSR_UUT) ? " UUT" : "",
434 (cfi->fsr & ARM_SMMU_CB_FSR_ASF) ? " ASF" : "",
435 (cfi->fsr & ARM_SMMU_CB_FSR_TLBLKF) ? " TLBLKF" : "",
436 (cfi->fsr & ARM_SMMU_CB_FSR_TLBMCF) ? " TLBMCF" : "",
437 (cfi->fsr & ARM_SMMU_CB_FSR_EF) ? " EF" : "",
438 (cfi->fsr & ARM_SMMU_CB_FSR_PF) ? " PF" : "",
439 (cfi->fsr & ARM_SMMU_CB_FSR_AFF) ? " AFF" : "",
440 (cfi->fsr & ARM_SMMU_CB_FSR_TF) ? " TF" : "",
441 cfi->cbfrsynra);
442
443 dev_err(smmu->dev, "FSYNR0 = %08x [S1CBNDX=%u%s%s%s%s%s%s PLVL=%u]\n",
444 cfi->fsynr,
445 (u32)FIELD_GET(ARM_SMMU_CB_FSYNR0_S1CBNDX, cfi->fsynr),
446 (cfi->fsynr & ARM_SMMU_CB_FSYNR0_AFR) ? " AFR" : "",
447 (cfi->fsynr & ARM_SMMU_CB_FSYNR0_PTWF) ? " PTWF" : "",
448 (cfi->fsynr & ARM_SMMU_CB_FSYNR0_NSATTR) ? " NSATTR" : "",
449 (cfi->fsynr & ARM_SMMU_CB_FSYNR0_IND) ? " IND" : "",
450 (cfi->fsynr & ARM_SMMU_CB_FSYNR0_PNU) ? " PNU" : "",
451 (cfi->fsynr & ARM_SMMU_CB_FSYNR0_WNR) ? " WNR" : "",
452 (u32)FIELD_GET(ARM_SMMU_CB_FSYNR0_PLVL, cfi->fsynr));
453 }
454
arm_smmu_context_fault(int irq,void * dev)455 static irqreturn_t arm_smmu_context_fault(int irq, void *dev)
456 {
457 struct arm_smmu_context_fault_info cfi;
458 struct arm_smmu_domain *smmu_domain = dev;
459 struct arm_smmu_device *smmu = smmu_domain->smmu;
460 static DEFINE_RATELIMIT_STATE(rs, DEFAULT_RATELIMIT_INTERVAL,
461 DEFAULT_RATELIMIT_BURST);
462 int idx = smmu_domain->cfg.cbndx;
463 int ret;
464
465 arm_smmu_read_context_fault_info(smmu, idx, &cfi);
466
467 if (!(cfi.fsr & ARM_SMMU_CB_FSR_FAULT))
468 return IRQ_NONE;
469
470 ret = report_iommu_fault(&smmu_domain->domain, NULL, cfi.iova,
471 cfi.fsynr & ARM_SMMU_CB_FSYNR0_WNR ? IOMMU_FAULT_WRITE : IOMMU_FAULT_READ);
472
473 if (ret == -ENOSYS && __ratelimit(&rs))
474 arm_smmu_print_context_fault_info(smmu, idx, &cfi);
475
476 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_FSR, cfi.fsr);
477
478 if (cfi.fsr & ARM_SMMU_CB_FSR_SS) {
479 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_RESUME,
480 ret == -EAGAIN ? 0 : ARM_SMMU_RESUME_TERMINATE);
481 }
482
483 return IRQ_HANDLED;
484 }
485
arm_smmu_global_fault(int irq,void * dev)486 static irqreturn_t arm_smmu_global_fault(int irq, void *dev)
487 {
488 u32 gfsr, gfsynr0, gfsynr1, gfsynr2;
489 struct arm_smmu_device *smmu = dev;
490 static DEFINE_RATELIMIT_STATE(rs, DEFAULT_RATELIMIT_INTERVAL,
491 DEFAULT_RATELIMIT_BURST);
492
493 gfsr = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sGFSR);
494 gfsynr0 = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sGFSYNR0);
495 gfsynr1 = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sGFSYNR1);
496 gfsynr2 = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sGFSYNR2);
497
498 if (!gfsr)
499 return IRQ_NONE;
500
501 if (__ratelimit(&rs)) {
502 if (IS_ENABLED(CONFIG_ARM_SMMU_DISABLE_BYPASS_BY_DEFAULT) &&
503 (gfsr & ARM_SMMU_sGFSR_USF))
504 dev_err(smmu->dev,
505 "Blocked unknown Stream ID 0x%hx; boot with \"arm-smmu.disable_bypass=0\" to allow, but this may have security implications\n",
506 (u16)gfsynr1);
507 else
508 dev_err(smmu->dev,
509 "Unexpected global fault, this could be serious\n");
510 dev_err(smmu->dev,
511 "\tGFSR 0x%08x, GFSYNR0 0x%08x, GFSYNR1 0x%08x, GFSYNR2 0x%08x\n",
512 gfsr, gfsynr0, gfsynr1, gfsynr2);
513 }
514
515 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_sGFSR, gfsr);
516 return IRQ_HANDLED;
517 }
518
arm_smmu_init_context_bank(struct arm_smmu_domain * smmu_domain,struct io_pgtable_cfg * pgtbl_cfg)519 static void arm_smmu_init_context_bank(struct arm_smmu_domain *smmu_domain,
520 struct io_pgtable_cfg *pgtbl_cfg)
521 {
522 struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
523 struct arm_smmu_cb *cb = &smmu_domain->smmu->cbs[cfg->cbndx];
524 bool stage1 = cfg->cbar != CBAR_TYPE_S2_TRANS;
525
526 cb->cfg = cfg;
527
528 /* TCR */
529 if (stage1) {
530 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) {
531 cb->tcr[0] = pgtbl_cfg->arm_v7s_cfg.tcr;
532 } else {
533 cb->tcr[0] = arm_smmu_lpae_tcr(pgtbl_cfg);
534 cb->tcr[1] = arm_smmu_lpae_tcr2(pgtbl_cfg);
535 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64)
536 cb->tcr[1] |= ARM_SMMU_TCR2_AS;
537 else
538 cb->tcr[0] |= ARM_SMMU_TCR_EAE;
539 }
540 } else {
541 cb->tcr[0] = arm_smmu_lpae_vtcr(pgtbl_cfg);
542 }
543
544 /* TTBRs */
545 if (stage1) {
546 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) {
547 cb->ttbr[0] = pgtbl_cfg->arm_v7s_cfg.ttbr;
548 cb->ttbr[1] = 0;
549 } else {
550 cb->ttbr[0] = FIELD_PREP(ARM_SMMU_TTBRn_ASID,
551 cfg->asid);
552 cb->ttbr[1] = FIELD_PREP(ARM_SMMU_TTBRn_ASID,
553 cfg->asid);
554
555 if (pgtbl_cfg->quirks & IO_PGTABLE_QUIRK_ARM_TTBR1)
556 cb->ttbr[1] |= pgtbl_cfg->arm_lpae_s1_cfg.ttbr;
557 else
558 cb->ttbr[0] |= pgtbl_cfg->arm_lpae_s1_cfg.ttbr;
559 }
560 } else {
561 cb->ttbr[0] = pgtbl_cfg->arm_lpae_s2_cfg.vttbr;
562 }
563
564 /* MAIRs (stage-1 only) */
565 if (stage1) {
566 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) {
567 cb->mair[0] = pgtbl_cfg->arm_v7s_cfg.prrr;
568 cb->mair[1] = pgtbl_cfg->arm_v7s_cfg.nmrr;
569 } else {
570 cb->mair[0] = pgtbl_cfg->arm_lpae_s1_cfg.mair;
571 cb->mair[1] = pgtbl_cfg->arm_lpae_s1_cfg.mair >> 32;
572 }
573 }
574 }
575
arm_smmu_write_context_bank(struct arm_smmu_device * smmu,int idx)576 void arm_smmu_write_context_bank(struct arm_smmu_device *smmu, int idx)
577 {
578 u32 reg;
579 bool stage1;
580 struct arm_smmu_cb *cb = &smmu->cbs[idx];
581 struct arm_smmu_cfg *cfg = cb->cfg;
582
583 /* Unassigned context banks only need disabling */
584 if (!cfg) {
585 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_SCTLR, 0);
586 return;
587 }
588
589 stage1 = cfg->cbar != CBAR_TYPE_S2_TRANS;
590
591 /* CBA2R */
592 if (smmu->version > ARM_SMMU_V1) {
593 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64)
594 reg = ARM_SMMU_CBA2R_VA64;
595 else
596 reg = 0;
597 /* 16-bit VMIDs live in CBA2R */
598 if (smmu->features & ARM_SMMU_FEAT_VMID16)
599 reg |= FIELD_PREP(ARM_SMMU_CBA2R_VMID16, cfg->vmid);
600
601 arm_smmu_gr1_write(smmu, ARM_SMMU_GR1_CBA2R(idx), reg);
602 }
603
604 /* CBAR */
605 reg = FIELD_PREP(ARM_SMMU_CBAR_TYPE, cfg->cbar);
606 if (smmu->version < ARM_SMMU_V2)
607 reg |= FIELD_PREP(ARM_SMMU_CBAR_IRPTNDX, cfg->irptndx);
608
609 /*
610 * Use the weakest shareability/memory types, so they are
611 * overridden by the ttbcr/pte.
612 */
613 if (stage1) {
614 reg |= FIELD_PREP(ARM_SMMU_CBAR_S1_BPSHCFG,
615 ARM_SMMU_CBAR_S1_BPSHCFG_NSH) |
616 FIELD_PREP(ARM_SMMU_CBAR_S1_MEMATTR,
617 ARM_SMMU_CBAR_S1_MEMATTR_WB);
618 } else if (!(smmu->features & ARM_SMMU_FEAT_VMID16)) {
619 /* 8-bit VMIDs live in CBAR */
620 reg |= FIELD_PREP(ARM_SMMU_CBAR_VMID, cfg->vmid);
621 }
622 arm_smmu_gr1_write(smmu, ARM_SMMU_GR1_CBAR(idx), reg);
623
624 /*
625 * TCR
626 * We must write this before the TTBRs, since it determines the
627 * access behaviour of some fields (in particular, ASID[15:8]).
628 */
629 if (stage1 && smmu->version > ARM_SMMU_V1)
630 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_TCR2, cb->tcr[1]);
631 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_TCR, cb->tcr[0]);
632
633 /* TTBRs */
634 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) {
635 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_CONTEXTIDR, cfg->asid);
636 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_TTBR0, cb->ttbr[0]);
637 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_TTBR1, cb->ttbr[1]);
638 } else {
639 arm_smmu_cb_writeq(smmu, idx, ARM_SMMU_CB_TTBR0, cb->ttbr[0]);
640 if (stage1)
641 arm_smmu_cb_writeq(smmu, idx, ARM_SMMU_CB_TTBR1,
642 cb->ttbr[1]);
643 }
644
645 /* MAIRs (stage-1 only) */
646 if (stage1) {
647 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_S1_MAIR0, cb->mair[0]);
648 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_S1_MAIR1, cb->mair[1]);
649 }
650
651 /* SCTLR */
652 reg = ARM_SMMU_SCTLR_CFIE | ARM_SMMU_SCTLR_CFRE | ARM_SMMU_SCTLR_AFE |
653 ARM_SMMU_SCTLR_TRE | ARM_SMMU_SCTLR_M;
654 if (stage1)
655 reg |= ARM_SMMU_SCTLR_S1_ASIDPNE;
656 if (IS_ENABLED(CONFIG_CPU_BIG_ENDIAN))
657 reg |= ARM_SMMU_SCTLR_E;
658
659 if (smmu->impl && smmu->impl->write_sctlr)
660 smmu->impl->write_sctlr(smmu, idx, reg);
661 else
662 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_SCTLR, reg);
663 }
664
arm_smmu_alloc_context_bank(struct arm_smmu_domain * smmu_domain,struct arm_smmu_device * smmu,struct device * dev,unsigned int start)665 static int arm_smmu_alloc_context_bank(struct arm_smmu_domain *smmu_domain,
666 struct arm_smmu_device *smmu,
667 struct device *dev, unsigned int start)
668 {
669 if (smmu->impl && smmu->impl->alloc_context_bank)
670 return smmu->impl->alloc_context_bank(smmu_domain, smmu, dev, start);
671
672 return __arm_smmu_alloc_bitmap(smmu->context_map, start, smmu->num_context_banks);
673 }
674
arm_smmu_init_domain_context(struct arm_smmu_domain * smmu_domain,struct arm_smmu_device * smmu,struct device * dev)675 static int arm_smmu_init_domain_context(struct arm_smmu_domain *smmu_domain,
676 struct arm_smmu_device *smmu,
677 struct device *dev)
678 {
679 int irq, start, ret = 0;
680 unsigned long ias, oas;
681 struct io_pgtable_ops *pgtbl_ops;
682 struct io_pgtable_cfg pgtbl_cfg;
683 enum io_pgtable_fmt fmt;
684 struct iommu_domain *domain = &smmu_domain->domain;
685 struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
686 irqreturn_t (*context_fault)(int irq, void *dev);
687
688 mutex_lock(&smmu_domain->init_mutex);
689 if (smmu_domain->smmu)
690 goto out_unlock;
691
692 /*
693 * Mapping the requested stage onto what we support is surprisingly
694 * complicated, mainly because the spec allows S1+S2 SMMUs without
695 * support for nested translation. That means we end up with the
696 * following table:
697 *
698 * Requested Supported Actual
699 * S1 N S1
700 * S1 S1+S2 S1
701 * S1 S2 S2
702 * S1 S1 S1
703 * N N N
704 * N S1+S2 S2
705 * N S2 S2
706 * N S1 S1
707 *
708 * Note that you can't actually request stage-2 mappings.
709 */
710 if (!(smmu->features & ARM_SMMU_FEAT_TRANS_S1))
711 smmu_domain->stage = ARM_SMMU_DOMAIN_S2;
712 if (!(smmu->features & ARM_SMMU_FEAT_TRANS_S2))
713 smmu_domain->stage = ARM_SMMU_DOMAIN_S1;
714
715 /*
716 * Choosing a suitable context format is even more fiddly. Until we
717 * grow some way for the caller to express a preference, and/or move
718 * the decision into the io-pgtable code where it arguably belongs,
719 * just aim for the closest thing to the rest of the system, and hope
720 * that the hardware isn't esoteric enough that we can't assume AArch64
721 * support to be a superset of AArch32 support...
722 */
723 if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH32_L)
724 cfg->fmt = ARM_SMMU_CTX_FMT_AARCH32_L;
725 if (IS_ENABLED(CONFIG_IOMMU_IO_PGTABLE_ARMV7S) &&
726 !IS_ENABLED(CONFIG_64BIT) && !IS_ENABLED(CONFIG_ARM_LPAE) &&
727 (smmu->features & ARM_SMMU_FEAT_FMT_AARCH32_S) &&
728 (smmu_domain->stage == ARM_SMMU_DOMAIN_S1))
729 cfg->fmt = ARM_SMMU_CTX_FMT_AARCH32_S;
730 if ((IS_ENABLED(CONFIG_64BIT) || cfg->fmt == ARM_SMMU_CTX_FMT_NONE) &&
731 (smmu->features & (ARM_SMMU_FEAT_FMT_AARCH64_64K |
732 ARM_SMMU_FEAT_FMT_AARCH64_16K |
733 ARM_SMMU_FEAT_FMT_AARCH64_4K)))
734 cfg->fmt = ARM_SMMU_CTX_FMT_AARCH64;
735
736 if (cfg->fmt == ARM_SMMU_CTX_FMT_NONE) {
737 ret = -EINVAL;
738 goto out_unlock;
739 }
740
741 switch (smmu_domain->stage) {
742 case ARM_SMMU_DOMAIN_S1:
743 cfg->cbar = CBAR_TYPE_S1_TRANS_S2_BYPASS;
744 start = smmu->num_s2_context_banks;
745 ias = smmu->va_size;
746 oas = smmu->ipa_size;
747 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64) {
748 fmt = ARM_64_LPAE_S1;
749 } else if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_L) {
750 fmt = ARM_32_LPAE_S1;
751 ias = min(ias, 32UL);
752 oas = min(oas, 40UL);
753 } else {
754 fmt = ARM_V7S;
755 ias = min(ias, 32UL);
756 oas = min(oas, 32UL);
757 }
758 smmu_domain->flush_ops = &arm_smmu_s1_tlb_ops;
759 break;
760 case ARM_SMMU_DOMAIN_NESTED:
761 /*
762 * We will likely want to change this if/when KVM gets
763 * involved.
764 */
765 case ARM_SMMU_DOMAIN_S2:
766 cfg->cbar = CBAR_TYPE_S2_TRANS;
767 start = 0;
768 ias = smmu->ipa_size;
769 oas = smmu->pa_size;
770 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64) {
771 fmt = ARM_64_LPAE_S2;
772 } else {
773 fmt = ARM_32_LPAE_S2;
774 ias = min(ias, 40UL);
775 oas = min(oas, 40UL);
776 }
777 if (smmu->version == ARM_SMMU_V2)
778 smmu_domain->flush_ops = &arm_smmu_s2_tlb_ops_v2;
779 else
780 smmu_domain->flush_ops = &arm_smmu_s2_tlb_ops_v1;
781 break;
782 default:
783 ret = -EINVAL;
784 goto out_unlock;
785 }
786
787 ret = arm_smmu_alloc_context_bank(smmu_domain, smmu, dev, start);
788 if (ret < 0) {
789 goto out_unlock;
790 }
791
792 smmu_domain->smmu = smmu;
793
794 cfg->cbndx = ret;
795 if (smmu->version < ARM_SMMU_V2) {
796 cfg->irptndx = atomic_inc_return(&smmu->irptndx);
797 cfg->irptndx %= smmu->num_context_irqs;
798 } else {
799 cfg->irptndx = cfg->cbndx;
800 }
801
802 if (smmu_domain->stage == ARM_SMMU_DOMAIN_S2)
803 cfg->vmid = cfg->cbndx + 1;
804 else
805 cfg->asid = cfg->cbndx;
806
807 pgtbl_cfg = (struct io_pgtable_cfg) {
808 .pgsize_bitmap = smmu->pgsize_bitmap,
809 .ias = ias,
810 .oas = oas,
811 .coherent_walk = smmu->features & ARM_SMMU_FEAT_COHERENT_WALK,
812 .tlb = smmu_domain->flush_ops,
813 .iommu_dev = smmu->dev,
814 };
815
816 if (smmu->impl && smmu->impl->init_context) {
817 ret = smmu->impl->init_context(smmu_domain, &pgtbl_cfg, dev);
818 if (ret)
819 goto out_clear_smmu;
820 }
821
822 if (smmu_domain->pgtbl_quirks)
823 pgtbl_cfg.quirks |= smmu_domain->pgtbl_quirks;
824
825 pgtbl_ops = alloc_io_pgtable_ops(fmt, &pgtbl_cfg, smmu_domain);
826 if (!pgtbl_ops) {
827 ret = -ENOMEM;
828 goto out_clear_smmu;
829 }
830
831 /* Update the domain's page sizes to reflect the page table format */
832 domain->pgsize_bitmap = pgtbl_cfg.pgsize_bitmap;
833
834 if (pgtbl_cfg.quirks & IO_PGTABLE_QUIRK_ARM_TTBR1) {
835 domain->geometry.aperture_start = ~0UL << ias;
836 domain->geometry.aperture_end = ~0UL;
837 } else {
838 domain->geometry.aperture_end = (1UL << ias) - 1;
839 }
840
841 domain->geometry.force_aperture = true;
842
843 /* Initialise the context bank with our page table cfg */
844 arm_smmu_init_context_bank(smmu_domain, &pgtbl_cfg);
845 arm_smmu_write_context_bank(smmu, cfg->cbndx);
846
847 /*
848 * Request context fault interrupt. Do this last to avoid the
849 * handler seeing a half-initialised domain state.
850 */
851 irq = smmu->irqs[cfg->irptndx];
852
853 if (smmu->impl && smmu->impl->context_fault)
854 context_fault = smmu->impl->context_fault;
855 else
856 context_fault = arm_smmu_context_fault;
857
858 if (smmu->impl && smmu->impl->context_fault_needs_threaded_irq)
859 ret = devm_request_threaded_irq(smmu->dev, irq, NULL,
860 context_fault,
861 IRQF_ONESHOT | IRQF_SHARED,
862 "arm-smmu-context-fault",
863 smmu_domain);
864 else
865 ret = devm_request_irq(smmu->dev, irq, context_fault, IRQF_SHARED,
866 "arm-smmu-context-fault", smmu_domain);
867
868 if (ret < 0) {
869 dev_err(smmu->dev, "failed to request context IRQ %d (%u)\n",
870 cfg->irptndx, irq);
871 cfg->irptndx = ARM_SMMU_INVALID_IRPTNDX;
872 }
873
874 mutex_unlock(&smmu_domain->init_mutex);
875
876 /* Publish page table ops for map/unmap */
877 smmu_domain->pgtbl_ops = pgtbl_ops;
878 return 0;
879
880 out_clear_smmu:
881 __arm_smmu_free_bitmap(smmu->context_map, cfg->cbndx);
882 smmu_domain->smmu = NULL;
883 out_unlock:
884 mutex_unlock(&smmu_domain->init_mutex);
885 return ret;
886 }
887
arm_smmu_destroy_domain_context(struct arm_smmu_domain * smmu_domain)888 static void arm_smmu_destroy_domain_context(struct arm_smmu_domain *smmu_domain)
889 {
890 struct arm_smmu_device *smmu = smmu_domain->smmu;
891 struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
892 int ret, irq;
893
894 if (!smmu)
895 return;
896
897 ret = arm_smmu_rpm_get(smmu);
898 if (ret < 0)
899 return;
900
901 /*
902 * Disable the context bank and free the page tables before freeing
903 * it.
904 */
905 smmu->cbs[cfg->cbndx].cfg = NULL;
906 arm_smmu_write_context_bank(smmu, cfg->cbndx);
907
908 if (cfg->irptndx != ARM_SMMU_INVALID_IRPTNDX) {
909 irq = smmu->irqs[cfg->irptndx];
910 devm_free_irq(smmu->dev, irq, smmu_domain);
911 }
912
913 free_io_pgtable_ops(smmu_domain->pgtbl_ops);
914 __arm_smmu_free_bitmap(smmu->context_map, cfg->cbndx);
915
916 arm_smmu_rpm_put(smmu);
917 }
918
arm_smmu_domain_alloc_paging(struct device * dev)919 static struct iommu_domain *arm_smmu_domain_alloc_paging(struct device *dev)
920 {
921 struct arm_smmu_domain *smmu_domain;
922 struct arm_smmu_master_cfg *cfg = dev_iommu_priv_get(dev);
923 struct arm_smmu_device *smmu = cfg->smmu;
924
925 /*
926 * Allocate the domain and initialise some of its data structures.
927 * We can't really do anything meaningful until we've added a
928 * master.
929 */
930 smmu_domain = kzalloc(sizeof(*smmu_domain), GFP_KERNEL);
931 if (!smmu_domain)
932 return NULL;
933
934 mutex_init(&smmu_domain->init_mutex);
935 spin_lock_init(&smmu_domain->cb_lock);
936 smmu_domain->domain.pgsize_bitmap = smmu->pgsize_bitmap;
937
938 return &smmu_domain->domain;
939 }
940
arm_smmu_domain_free(struct iommu_domain * domain)941 static void arm_smmu_domain_free(struct iommu_domain *domain)
942 {
943 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
944
945 /*
946 * Free the domain resources. We assume that all devices have
947 * already been detached.
948 */
949 arm_smmu_destroy_domain_context(smmu_domain);
950 kfree(smmu_domain);
951 }
952
arm_smmu_write_smr(struct arm_smmu_device * smmu,int idx)953 static void arm_smmu_write_smr(struct arm_smmu_device *smmu, int idx)
954 {
955 struct arm_smmu_smr *smr = smmu->smrs + idx;
956 u32 reg = FIELD_PREP(ARM_SMMU_SMR_ID, smr->id) |
957 FIELD_PREP(ARM_SMMU_SMR_MASK, smr->mask);
958
959 if (!(smmu->features & ARM_SMMU_FEAT_EXIDS) && smr->valid)
960 reg |= ARM_SMMU_SMR_VALID;
961 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_SMR(idx), reg);
962 }
963
arm_smmu_write_s2cr(struct arm_smmu_device * smmu,int idx)964 static void arm_smmu_write_s2cr(struct arm_smmu_device *smmu, int idx)
965 {
966 struct arm_smmu_s2cr *s2cr = smmu->s2crs + idx;
967 u32 reg;
968
969 if (smmu->impl && smmu->impl->write_s2cr) {
970 smmu->impl->write_s2cr(smmu, idx);
971 return;
972 }
973
974 reg = FIELD_PREP(ARM_SMMU_S2CR_TYPE, s2cr->type) |
975 FIELD_PREP(ARM_SMMU_S2CR_CBNDX, s2cr->cbndx) |
976 FIELD_PREP(ARM_SMMU_S2CR_PRIVCFG, s2cr->privcfg);
977
978 if (smmu->features & ARM_SMMU_FEAT_EXIDS && smmu->smrs &&
979 smmu->smrs[idx].valid)
980 reg |= ARM_SMMU_S2CR_EXIDVALID;
981 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_S2CR(idx), reg);
982 }
983
arm_smmu_write_sme(struct arm_smmu_device * smmu,int idx)984 static void arm_smmu_write_sme(struct arm_smmu_device *smmu, int idx)
985 {
986 arm_smmu_write_s2cr(smmu, idx);
987 if (smmu->smrs)
988 arm_smmu_write_smr(smmu, idx);
989 }
990
991 /*
992 * The width of SMR's mask field depends on sCR0_EXIDENABLE, so this function
993 * should be called after sCR0 is written.
994 */
arm_smmu_test_smr_masks(struct arm_smmu_device * smmu)995 static void arm_smmu_test_smr_masks(struct arm_smmu_device *smmu)
996 {
997 u32 smr;
998 int i;
999
1000 if (!smmu->smrs)
1001 return;
1002 /*
1003 * If we've had to accommodate firmware memory regions, we may
1004 * have live SMRs by now; tread carefully...
1005 *
1006 * Somewhat perversely, not having a free SMR for this test implies we
1007 * can get away without it anyway, as we'll only be able to 'allocate'
1008 * these SMRs for the ID/mask values we're already trusting to be OK.
1009 */
1010 for (i = 0; i < smmu->num_mapping_groups; i++)
1011 if (!smmu->smrs[i].valid)
1012 goto smr_ok;
1013 return;
1014 smr_ok:
1015 /*
1016 * SMR.ID bits may not be preserved if the corresponding MASK
1017 * bits are set, so check each one separately. We can reject
1018 * masters later if they try to claim IDs outside these masks.
1019 */
1020 smr = FIELD_PREP(ARM_SMMU_SMR_ID, smmu->streamid_mask);
1021 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_SMR(i), smr);
1022 smr = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_SMR(i));
1023 smmu->streamid_mask = FIELD_GET(ARM_SMMU_SMR_ID, smr);
1024
1025 smr = FIELD_PREP(ARM_SMMU_SMR_MASK, smmu->streamid_mask);
1026 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_SMR(i), smr);
1027 smr = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_SMR(i));
1028 smmu->smr_mask_mask = FIELD_GET(ARM_SMMU_SMR_MASK, smr);
1029 }
1030
arm_smmu_find_sme(struct arm_smmu_device * smmu,u16 id,u16 mask)1031 static int arm_smmu_find_sme(struct arm_smmu_device *smmu, u16 id, u16 mask)
1032 {
1033 struct arm_smmu_smr *smrs = smmu->smrs;
1034 int i, free_idx = -ENOSPC;
1035
1036 /* Stream indexing is blissfully easy */
1037 if (!smrs)
1038 return id;
1039
1040 /* Validating SMRs is... less so */
1041 for (i = 0; i < smmu->num_mapping_groups; ++i) {
1042 if (!smrs[i].valid) {
1043 /*
1044 * Note the first free entry we come across, which
1045 * we'll claim in the end if nothing else matches.
1046 */
1047 if (free_idx < 0)
1048 free_idx = i;
1049 continue;
1050 }
1051 /*
1052 * If the new entry is _entirely_ matched by an existing entry,
1053 * then reuse that, with the guarantee that there also cannot
1054 * be any subsequent conflicting entries. In normal use we'd
1055 * expect simply identical entries for this case, but there's
1056 * no harm in accommodating the generalisation.
1057 */
1058 if ((mask & smrs[i].mask) == mask &&
1059 !((id ^ smrs[i].id) & ~smrs[i].mask))
1060 return i;
1061 /*
1062 * If the new entry has any other overlap with an existing one,
1063 * though, then there always exists at least one stream ID
1064 * which would cause a conflict, and we can't allow that risk.
1065 */
1066 if (!((id ^ smrs[i].id) & ~(smrs[i].mask | mask)))
1067 return -EINVAL;
1068 }
1069
1070 return free_idx;
1071 }
1072
arm_smmu_free_sme(struct arm_smmu_device * smmu,int idx)1073 static bool arm_smmu_free_sme(struct arm_smmu_device *smmu, int idx)
1074 {
1075 if (--smmu->s2crs[idx].count)
1076 return false;
1077
1078 smmu->s2crs[idx] = s2cr_init_val;
1079 if (smmu->smrs)
1080 smmu->smrs[idx].valid = false;
1081
1082 return true;
1083 }
1084
arm_smmu_master_alloc_smes(struct device * dev)1085 static int arm_smmu_master_alloc_smes(struct device *dev)
1086 {
1087 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
1088 struct arm_smmu_master_cfg *cfg = dev_iommu_priv_get(dev);
1089 struct arm_smmu_device *smmu = cfg->smmu;
1090 struct arm_smmu_smr *smrs = smmu->smrs;
1091 int i, idx, ret;
1092
1093 mutex_lock(&smmu->stream_map_mutex);
1094 /* Figure out a viable stream map entry allocation */
1095 for_each_cfg_sme(cfg, fwspec, i, idx) {
1096 u16 sid = FIELD_GET(ARM_SMMU_SMR_ID, fwspec->ids[i]);
1097 u16 mask = FIELD_GET(ARM_SMMU_SMR_MASK, fwspec->ids[i]);
1098
1099 if (idx != INVALID_SMENDX) {
1100 ret = -EEXIST;
1101 goto out_err;
1102 }
1103
1104 ret = arm_smmu_find_sme(smmu, sid, mask);
1105 if (ret < 0)
1106 goto out_err;
1107
1108 idx = ret;
1109 if (smrs && smmu->s2crs[idx].count == 0) {
1110 smrs[idx].id = sid;
1111 smrs[idx].mask = mask;
1112 smrs[idx].valid = true;
1113 }
1114 smmu->s2crs[idx].count++;
1115 cfg->smendx[i] = (s16)idx;
1116 }
1117
1118 /* It worked! Now, poke the actual hardware */
1119 for_each_cfg_sme(cfg, fwspec, i, idx)
1120 arm_smmu_write_sme(smmu, idx);
1121
1122 mutex_unlock(&smmu->stream_map_mutex);
1123 return 0;
1124
1125 out_err:
1126 while (i--) {
1127 arm_smmu_free_sme(smmu, cfg->smendx[i]);
1128 cfg->smendx[i] = INVALID_SMENDX;
1129 }
1130 mutex_unlock(&smmu->stream_map_mutex);
1131 return ret;
1132 }
1133
arm_smmu_master_free_smes(struct arm_smmu_master_cfg * cfg,struct iommu_fwspec * fwspec)1134 static void arm_smmu_master_free_smes(struct arm_smmu_master_cfg *cfg,
1135 struct iommu_fwspec *fwspec)
1136 {
1137 struct arm_smmu_device *smmu = cfg->smmu;
1138 int i, idx;
1139
1140 mutex_lock(&smmu->stream_map_mutex);
1141 for_each_cfg_sme(cfg, fwspec, i, idx) {
1142 if (arm_smmu_free_sme(smmu, idx))
1143 arm_smmu_write_sme(smmu, idx);
1144 cfg->smendx[i] = INVALID_SMENDX;
1145 }
1146 mutex_unlock(&smmu->stream_map_mutex);
1147 }
1148
arm_smmu_master_install_s2crs(struct arm_smmu_master_cfg * cfg,enum arm_smmu_s2cr_type type,u8 cbndx,struct iommu_fwspec * fwspec)1149 static void arm_smmu_master_install_s2crs(struct arm_smmu_master_cfg *cfg,
1150 enum arm_smmu_s2cr_type type,
1151 u8 cbndx, struct iommu_fwspec *fwspec)
1152 {
1153 struct arm_smmu_device *smmu = cfg->smmu;
1154 struct arm_smmu_s2cr *s2cr = smmu->s2crs;
1155 int i, idx;
1156
1157 for_each_cfg_sme(cfg, fwspec, i, idx) {
1158 if (type == s2cr[idx].type && cbndx == s2cr[idx].cbndx)
1159 continue;
1160
1161 s2cr[idx].type = type;
1162 s2cr[idx].privcfg = S2CR_PRIVCFG_DEFAULT;
1163 s2cr[idx].cbndx = cbndx;
1164 arm_smmu_write_s2cr(smmu, idx);
1165 }
1166 }
1167
arm_smmu_attach_dev(struct iommu_domain * domain,struct device * dev)1168 static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev)
1169 {
1170 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1171 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
1172 struct arm_smmu_master_cfg *cfg;
1173 struct arm_smmu_device *smmu;
1174 int ret;
1175
1176 /*
1177 * FIXME: The arch/arm DMA API code tries to attach devices to its own
1178 * domains between of_xlate() and probe_device() - we have no way to cope
1179 * with that, so until ARM gets converted to rely on groups and default
1180 * domains, just say no (but more politely than by dereferencing NULL).
1181 * This should be at least a WARN_ON once that's sorted.
1182 */
1183 cfg = dev_iommu_priv_get(dev);
1184 if (!cfg)
1185 return -ENODEV;
1186
1187 smmu = cfg->smmu;
1188
1189 ret = arm_smmu_rpm_get(smmu);
1190 if (ret < 0)
1191 return ret;
1192
1193 /* Ensure that the domain is finalised */
1194 ret = arm_smmu_init_domain_context(smmu_domain, smmu, dev);
1195 if (ret < 0)
1196 goto rpm_put;
1197
1198 /*
1199 * Sanity check the domain. We don't support domains across
1200 * different SMMUs.
1201 */
1202 if (smmu_domain->smmu != smmu) {
1203 ret = -EINVAL;
1204 goto rpm_put;
1205 }
1206
1207 /* Looks ok, so add the device to the domain */
1208 arm_smmu_master_install_s2crs(cfg, S2CR_TYPE_TRANS,
1209 smmu_domain->cfg.cbndx, fwspec);
1210 rpm_put:
1211 arm_smmu_rpm_put(smmu);
1212 return ret;
1213 }
1214
arm_smmu_attach_dev_type(struct device * dev,enum arm_smmu_s2cr_type type)1215 static int arm_smmu_attach_dev_type(struct device *dev,
1216 enum arm_smmu_s2cr_type type)
1217 {
1218 struct arm_smmu_master_cfg *cfg = dev_iommu_priv_get(dev);
1219 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
1220 struct arm_smmu_device *smmu;
1221 int ret;
1222
1223 if (!cfg)
1224 return -ENODEV;
1225 smmu = cfg->smmu;
1226
1227 ret = arm_smmu_rpm_get(smmu);
1228 if (ret < 0)
1229 return ret;
1230
1231 arm_smmu_master_install_s2crs(cfg, type, 0, fwspec);
1232 arm_smmu_rpm_put(smmu);
1233 return 0;
1234 }
1235
arm_smmu_attach_dev_identity(struct iommu_domain * domain,struct device * dev)1236 static int arm_smmu_attach_dev_identity(struct iommu_domain *domain,
1237 struct device *dev)
1238 {
1239 return arm_smmu_attach_dev_type(dev, S2CR_TYPE_BYPASS);
1240 }
1241
1242 static const struct iommu_domain_ops arm_smmu_identity_ops = {
1243 .attach_dev = arm_smmu_attach_dev_identity,
1244 };
1245
1246 static struct iommu_domain arm_smmu_identity_domain = {
1247 .type = IOMMU_DOMAIN_IDENTITY,
1248 .ops = &arm_smmu_identity_ops,
1249 };
1250
arm_smmu_attach_dev_blocked(struct iommu_domain * domain,struct device * dev)1251 static int arm_smmu_attach_dev_blocked(struct iommu_domain *domain,
1252 struct device *dev)
1253 {
1254 return arm_smmu_attach_dev_type(dev, S2CR_TYPE_FAULT);
1255 }
1256
1257 static const struct iommu_domain_ops arm_smmu_blocked_ops = {
1258 .attach_dev = arm_smmu_attach_dev_blocked,
1259 };
1260
1261 static struct iommu_domain arm_smmu_blocked_domain = {
1262 .type = IOMMU_DOMAIN_BLOCKED,
1263 .ops = &arm_smmu_blocked_ops,
1264 };
1265
arm_smmu_map_pages(struct iommu_domain * domain,unsigned long iova,phys_addr_t paddr,size_t pgsize,size_t pgcount,int prot,gfp_t gfp,size_t * mapped)1266 static int arm_smmu_map_pages(struct iommu_domain *domain, unsigned long iova,
1267 phys_addr_t paddr, size_t pgsize, size_t pgcount,
1268 int prot, gfp_t gfp, size_t *mapped)
1269 {
1270 struct io_pgtable_ops *ops = to_smmu_domain(domain)->pgtbl_ops;
1271 struct arm_smmu_device *smmu = to_smmu_domain(domain)->smmu;
1272 int ret;
1273
1274 if (!ops)
1275 return -ENODEV;
1276
1277 arm_smmu_rpm_get(smmu);
1278 ret = ops->map_pages(ops, iova, paddr, pgsize, pgcount, prot, gfp, mapped);
1279 arm_smmu_rpm_put(smmu);
1280
1281 return ret;
1282 }
1283
arm_smmu_unmap_pages(struct iommu_domain * domain,unsigned long iova,size_t pgsize,size_t pgcount,struct iommu_iotlb_gather * iotlb_gather)1284 static size_t arm_smmu_unmap_pages(struct iommu_domain *domain, unsigned long iova,
1285 size_t pgsize, size_t pgcount,
1286 struct iommu_iotlb_gather *iotlb_gather)
1287 {
1288 struct io_pgtable_ops *ops = to_smmu_domain(domain)->pgtbl_ops;
1289 struct arm_smmu_device *smmu = to_smmu_domain(domain)->smmu;
1290 size_t ret;
1291
1292 if (!ops)
1293 return 0;
1294
1295 arm_smmu_rpm_get(smmu);
1296 ret = ops->unmap_pages(ops, iova, pgsize, pgcount, iotlb_gather);
1297 arm_smmu_rpm_put(smmu);
1298
1299 return ret;
1300 }
1301
arm_smmu_flush_iotlb_all(struct iommu_domain * domain)1302 static void arm_smmu_flush_iotlb_all(struct iommu_domain *domain)
1303 {
1304 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1305 struct arm_smmu_device *smmu = smmu_domain->smmu;
1306
1307 if (smmu_domain->flush_ops) {
1308 arm_smmu_rpm_get(smmu);
1309 smmu_domain->flush_ops->tlb_flush_all(smmu_domain);
1310 arm_smmu_rpm_put(smmu);
1311 }
1312 }
1313
arm_smmu_iotlb_sync(struct iommu_domain * domain,struct iommu_iotlb_gather * gather)1314 static void arm_smmu_iotlb_sync(struct iommu_domain *domain,
1315 struct iommu_iotlb_gather *gather)
1316 {
1317 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1318 struct arm_smmu_device *smmu = smmu_domain->smmu;
1319
1320 if (!smmu)
1321 return;
1322
1323 arm_smmu_rpm_get(smmu);
1324 if (smmu->version == ARM_SMMU_V2 ||
1325 smmu_domain->stage == ARM_SMMU_DOMAIN_S1)
1326 arm_smmu_tlb_sync_context(smmu_domain);
1327 else
1328 arm_smmu_tlb_sync_global(smmu);
1329 arm_smmu_rpm_put(smmu);
1330 }
1331
arm_smmu_iova_to_phys_hard(struct iommu_domain * domain,dma_addr_t iova)1332 static phys_addr_t arm_smmu_iova_to_phys_hard(struct iommu_domain *domain,
1333 dma_addr_t iova)
1334 {
1335 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1336 struct arm_smmu_device *smmu = smmu_domain->smmu;
1337 struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
1338 struct io_pgtable_ops *ops= smmu_domain->pgtbl_ops;
1339 struct device *dev = smmu->dev;
1340 void __iomem *reg;
1341 u32 tmp;
1342 u64 phys;
1343 unsigned long va, flags;
1344 int ret, idx = cfg->cbndx;
1345 phys_addr_t addr = 0;
1346
1347 ret = arm_smmu_rpm_get(smmu);
1348 if (ret < 0)
1349 return 0;
1350
1351 spin_lock_irqsave(&smmu_domain->cb_lock, flags);
1352 va = iova & ~0xfffUL;
1353 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64)
1354 arm_smmu_cb_writeq(smmu, idx, ARM_SMMU_CB_ATS1PR, va);
1355 else
1356 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_ATS1PR, va);
1357
1358 reg = arm_smmu_page(smmu, ARM_SMMU_CB(smmu, idx)) + ARM_SMMU_CB_ATSR;
1359 if (readl_poll_timeout_atomic(reg, tmp, !(tmp & ARM_SMMU_CB_ATSR_ACTIVE),
1360 5, 50)) {
1361 spin_unlock_irqrestore(&smmu_domain->cb_lock, flags);
1362 dev_err(dev,
1363 "iova to phys timed out on %pad. Falling back to software table walk.\n",
1364 &iova);
1365 arm_smmu_rpm_put(smmu);
1366 return ops->iova_to_phys(ops, iova);
1367 }
1368
1369 phys = arm_smmu_cb_readq(smmu, idx, ARM_SMMU_CB_PAR);
1370 spin_unlock_irqrestore(&smmu_domain->cb_lock, flags);
1371 if (phys & ARM_SMMU_CB_PAR_F) {
1372 dev_err(dev, "translation fault!\n");
1373 dev_err(dev, "PAR = 0x%llx\n", phys);
1374 goto out;
1375 }
1376
1377 addr = (phys & GENMASK_ULL(39, 12)) | (iova & 0xfff);
1378 out:
1379 arm_smmu_rpm_put(smmu);
1380
1381 return addr;
1382 }
1383
arm_smmu_iova_to_phys(struct iommu_domain * domain,dma_addr_t iova)1384 static phys_addr_t arm_smmu_iova_to_phys(struct iommu_domain *domain,
1385 dma_addr_t iova)
1386 {
1387 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1388 struct io_pgtable_ops *ops = smmu_domain->pgtbl_ops;
1389
1390 if (!ops)
1391 return 0;
1392
1393 if (smmu_domain->smmu->features & ARM_SMMU_FEAT_TRANS_OPS &&
1394 smmu_domain->stage == ARM_SMMU_DOMAIN_S1)
1395 return arm_smmu_iova_to_phys_hard(domain, iova);
1396
1397 return ops->iova_to_phys(ops, iova);
1398 }
1399
arm_smmu_capable(struct device * dev,enum iommu_cap cap)1400 static bool arm_smmu_capable(struct device *dev, enum iommu_cap cap)
1401 {
1402 struct arm_smmu_master_cfg *cfg = dev_iommu_priv_get(dev);
1403
1404 switch (cap) {
1405 case IOMMU_CAP_CACHE_COHERENCY:
1406 /*
1407 * It's overwhelmingly the case in practice that when the pagetable
1408 * walk interface is connected to a coherent interconnect, all the
1409 * translation interfaces are too. Furthermore if the device is
1410 * natively coherent, then its translation interface must also be.
1411 */
1412 return cfg->smmu->features & ARM_SMMU_FEAT_COHERENT_WALK ||
1413 device_get_dma_attr(dev) == DEV_DMA_COHERENT;
1414 case IOMMU_CAP_NOEXEC:
1415 case IOMMU_CAP_DEFERRED_FLUSH:
1416 return true;
1417 default:
1418 return false;
1419 }
1420 }
1421
1422 static
arm_smmu_get_by_fwnode(struct fwnode_handle * fwnode)1423 struct arm_smmu_device *arm_smmu_get_by_fwnode(struct fwnode_handle *fwnode)
1424 {
1425 struct device *dev = bus_find_device_by_fwnode(&platform_bus_type, fwnode);
1426
1427 put_device(dev);
1428 return dev ? dev_get_drvdata(dev) : NULL;
1429 }
1430
arm_smmu_probe_device(struct device * dev)1431 static struct iommu_device *arm_smmu_probe_device(struct device *dev)
1432 {
1433 struct arm_smmu_device *smmu = NULL;
1434 struct arm_smmu_master_cfg *cfg;
1435 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
1436 int i, ret;
1437
1438 if (using_legacy_binding) {
1439 ret = arm_smmu_register_legacy_master(dev, &smmu);
1440
1441 /*
1442 * If dev->iommu_fwspec is initally NULL, arm_smmu_register_legacy_master()
1443 * will allocate/initialise a new one. Thus we need to update fwspec for
1444 * later use.
1445 */
1446 fwspec = dev_iommu_fwspec_get(dev);
1447 if (ret)
1448 goto out_free;
1449 } else {
1450 smmu = arm_smmu_get_by_fwnode(fwspec->iommu_fwnode);
1451 }
1452
1453 ret = -EINVAL;
1454 for (i = 0; i < fwspec->num_ids; i++) {
1455 u16 sid = FIELD_GET(ARM_SMMU_SMR_ID, fwspec->ids[i]);
1456 u16 mask = FIELD_GET(ARM_SMMU_SMR_MASK, fwspec->ids[i]);
1457
1458 if (sid & ~smmu->streamid_mask) {
1459 dev_err(dev, "stream ID 0x%x out of range for SMMU (0x%x)\n",
1460 sid, smmu->streamid_mask);
1461 goto out_free;
1462 }
1463 if (mask & ~smmu->smr_mask_mask) {
1464 dev_err(dev, "SMR mask 0x%x out of range for SMMU (0x%x)\n",
1465 mask, smmu->smr_mask_mask);
1466 goto out_free;
1467 }
1468 }
1469
1470 ret = -ENOMEM;
1471 cfg = kzalloc(offsetof(struct arm_smmu_master_cfg, smendx[i]),
1472 GFP_KERNEL);
1473 if (!cfg)
1474 goto out_free;
1475
1476 cfg->smmu = smmu;
1477 dev_iommu_priv_set(dev, cfg);
1478 while (i--)
1479 cfg->smendx[i] = INVALID_SMENDX;
1480
1481 ret = arm_smmu_rpm_get(smmu);
1482 if (ret < 0)
1483 goto out_cfg_free;
1484
1485 ret = arm_smmu_master_alloc_smes(dev);
1486 arm_smmu_rpm_put(smmu);
1487
1488 if (ret)
1489 goto out_cfg_free;
1490
1491 device_link_add(dev, smmu->dev,
1492 DL_FLAG_PM_RUNTIME | DL_FLAG_AUTOREMOVE_SUPPLIER);
1493
1494 return &smmu->iommu;
1495
1496 out_cfg_free:
1497 kfree(cfg);
1498 out_free:
1499 return ERR_PTR(ret);
1500 }
1501
arm_smmu_release_device(struct device * dev)1502 static void arm_smmu_release_device(struct device *dev)
1503 {
1504 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
1505 struct arm_smmu_master_cfg *cfg = dev_iommu_priv_get(dev);
1506 int ret;
1507
1508 ret = arm_smmu_rpm_get(cfg->smmu);
1509 if (ret < 0)
1510 return;
1511
1512 arm_smmu_master_free_smes(cfg, fwspec);
1513
1514 arm_smmu_rpm_put(cfg->smmu);
1515
1516 kfree(cfg);
1517 }
1518
arm_smmu_probe_finalize(struct device * dev)1519 static void arm_smmu_probe_finalize(struct device *dev)
1520 {
1521 struct arm_smmu_master_cfg *cfg;
1522 struct arm_smmu_device *smmu;
1523
1524 cfg = dev_iommu_priv_get(dev);
1525 smmu = cfg->smmu;
1526
1527 if (smmu->impl && smmu->impl->probe_finalize)
1528 smmu->impl->probe_finalize(smmu, dev);
1529 }
1530
arm_smmu_device_group(struct device * dev)1531 static struct iommu_group *arm_smmu_device_group(struct device *dev)
1532 {
1533 struct arm_smmu_master_cfg *cfg = dev_iommu_priv_get(dev);
1534 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
1535 struct arm_smmu_device *smmu = cfg->smmu;
1536 struct iommu_group *group = NULL;
1537 int i, idx;
1538
1539 mutex_lock(&smmu->stream_map_mutex);
1540 for_each_cfg_sme(cfg, fwspec, i, idx) {
1541 if (group && smmu->s2crs[idx].group &&
1542 group != smmu->s2crs[idx].group) {
1543 mutex_unlock(&smmu->stream_map_mutex);
1544 return ERR_PTR(-EINVAL);
1545 }
1546
1547 group = smmu->s2crs[idx].group;
1548 }
1549
1550 if (group) {
1551 mutex_unlock(&smmu->stream_map_mutex);
1552 return iommu_group_ref_get(group);
1553 }
1554
1555 if (dev_is_pci(dev))
1556 group = pci_device_group(dev);
1557 else if (dev_is_fsl_mc(dev))
1558 group = fsl_mc_device_group(dev);
1559 else
1560 group = generic_device_group(dev);
1561
1562 /* Remember group for faster lookups */
1563 if (!IS_ERR(group))
1564 for_each_cfg_sme(cfg, fwspec, i, idx)
1565 smmu->s2crs[idx].group = group;
1566
1567 mutex_unlock(&smmu->stream_map_mutex);
1568 return group;
1569 }
1570
arm_smmu_set_pgtable_quirks(struct iommu_domain * domain,unsigned long quirks)1571 static int arm_smmu_set_pgtable_quirks(struct iommu_domain *domain,
1572 unsigned long quirks)
1573 {
1574 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1575 int ret = 0;
1576
1577 mutex_lock(&smmu_domain->init_mutex);
1578 if (smmu_domain->smmu)
1579 ret = -EPERM;
1580 else
1581 smmu_domain->pgtbl_quirks = quirks;
1582 mutex_unlock(&smmu_domain->init_mutex);
1583
1584 return ret;
1585 }
1586
arm_smmu_of_xlate(struct device * dev,const struct of_phandle_args * args)1587 static int arm_smmu_of_xlate(struct device *dev,
1588 const struct of_phandle_args *args)
1589 {
1590 u32 mask, fwid = 0;
1591
1592 if (args->args_count > 0)
1593 fwid |= FIELD_PREP(ARM_SMMU_SMR_ID, args->args[0]);
1594
1595 if (args->args_count > 1)
1596 fwid |= FIELD_PREP(ARM_SMMU_SMR_MASK, args->args[1]);
1597 else if (!of_property_read_u32(args->np, "stream-match-mask", &mask))
1598 fwid |= FIELD_PREP(ARM_SMMU_SMR_MASK, mask);
1599
1600 return iommu_fwspec_add_ids(dev, &fwid, 1);
1601 }
1602
arm_smmu_get_resv_regions(struct device * dev,struct list_head * head)1603 static void arm_smmu_get_resv_regions(struct device *dev,
1604 struct list_head *head)
1605 {
1606 struct iommu_resv_region *region;
1607 int prot = IOMMU_WRITE | IOMMU_NOEXEC | IOMMU_MMIO;
1608
1609 region = iommu_alloc_resv_region(MSI_IOVA_BASE, MSI_IOVA_LENGTH,
1610 prot, IOMMU_RESV_SW_MSI, GFP_KERNEL);
1611 if (!region)
1612 return;
1613
1614 list_add_tail(®ion->list, head);
1615
1616 iommu_dma_get_resv_regions(dev, head);
1617 }
1618
arm_smmu_def_domain_type(struct device * dev)1619 static int arm_smmu_def_domain_type(struct device *dev)
1620 {
1621 struct arm_smmu_master_cfg *cfg = dev_iommu_priv_get(dev);
1622 const struct arm_smmu_impl *impl = cfg->smmu->impl;
1623
1624 if (using_legacy_binding)
1625 return IOMMU_DOMAIN_IDENTITY;
1626
1627 if (impl && impl->def_domain_type)
1628 return impl->def_domain_type(dev);
1629
1630 return 0;
1631 }
1632
1633 static const struct iommu_ops arm_smmu_ops = {
1634 .identity_domain = &arm_smmu_identity_domain,
1635 .blocked_domain = &arm_smmu_blocked_domain,
1636 .capable = arm_smmu_capable,
1637 .domain_alloc_paging = arm_smmu_domain_alloc_paging,
1638 .probe_device = arm_smmu_probe_device,
1639 .release_device = arm_smmu_release_device,
1640 .probe_finalize = arm_smmu_probe_finalize,
1641 .device_group = arm_smmu_device_group,
1642 .of_xlate = arm_smmu_of_xlate,
1643 .get_resv_regions = arm_smmu_get_resv_regions,
1644 .def_domain_type = arm_smmu_def_domain_type,
1645 .owner = THIS_MODULE,
1646 .default_domain_ops = &(const struct iommu_domain_ops) {
1647 .attach_dev = arm_smmu_attach_dev,
1648 .map_pages = arm_smmu_map_pages,
1649 .unmap_pages = arm_smmu_unmap_pages,
1650 .flush_iotlb_all = arm_smmu_flush_iotlb_all,
1651 .iotlb_sync = arm_smmu_iotlb_sync,
1652 .iova_to_phys = arm_smmu_iova_to_phys,
1653 .set_pgtable_quirks = arm_smmu_set_pgtable_quirks,
1654 .free = arm_smmu_domain_free,
1655 }
1656 };
1657
arm_smmu_device_reset(struct arm_smmu_device * smmu)1658 static void arm_smmu_device_reset(struct arm_smmu_device *smmu)
1659 {
1660 int i;
1661 u32 reg;
1662
1663 /* clear global FSR */
1664 reg = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sGFSR);
1665 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_sGFSR, reg);
1666
1667 /*
1668 * Reset stream mapping groups: Initial values mark all SMRn as
1669 * invalid and all S2CRn as bypass unless overridden.
1670 */
1671 for (i = 0; i < smmu->num_mapping_groups; ++i)
1672 arm_smmu_write_sme(smmu, i);
1673
1674 /* Make sure all context banks are disabled and clear CB_FSR */
1675 for (i = 0; i < smmu->num_context_banks; ++i) {
1676 arm_smmu_write_context_bank(smmu, i);
1677 arm_smmu_cb_write(smmu, i, ARM_SMMU_CB_FSR, ARM_SMMU_CB_FSR_FAULT);
1678 }
1679
1680 /* Invalidate the TLB, just in case */
1681 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_TLBIALLH, QCOM_DUMMY_VAL);
1682 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_TLBIALLNSNH, QCOM_DUMMY_VAL);
1683
1684 reg = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sCR0);
1685
1686 /* Enable fault reporting */
1687 reg |= (ARM_SMMU_sCR0_GFRE | ARM_SMMU_sCR0_GFIE |
1688 ARM_SMMU_sCR0_GCFGFRE | ARM_SMMU_sCR0_GCFGFIE);
1689
1690 /* Disable TLB broadcasting. */
1691 reg |= (ARM_SMMU_sCR0_VMIDPNE | ARM_SMMU_sCR0_PTM);
1692
1693 /* Enable client access, handling unmatched streams as appropriate */
1694 reg &= ~ARM_SMMU_sCR0_CLIENTPD;
1695 if (disable_bypass)
1696 reg |= ARM_SMMU_sCR0_USFCFG;
1697 else
1698 reg &= ~ARM_SMMU_sCR0_USFCFG;
1699
1700 /* Disable forced broadcasting */
1701 reg &= ~ARM_SMMU_sCR0_FB;
1702
1703 /* Don't upgrade barriers */
1704 reg &= ~(ARM_SMMU_sCR0_BSU);
1705
1706 if (smmu->features & ARM_SMMU_FEAT_VMID16)
1707 reg |= ARM_SMMU_sCR0_VMID16EN;
1708
1709 if (smmu->features & ARM_SMMU_FEAT_EXIDS)
1710 reg |= ARM_SMMU_sCR0_EXIDENABLE;
1711
1712 if (smmu->impl && smmu->impl->reset)
1713 smmu->impl->reset(smmu);
1714
1715 /* Push the button */
1716 arm_smmu_tlb_sync_global(smmu);
1717 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_sCR0, reg);
1718 }
1719
arm_smmu_id_size_to_bits(int size)1720 static int arm_smmu_id_size_to_bits(int size)
1721 {
1722 switch (size) {
1723 case 0:
1724 return 32;
1725 case 1:
1726 return 36;
1727 case 2:
1728 return 40;
1729 case 3:
1730 return 42;
1731 case 4:
1732 return 44;
1733 case 5:
1734 default:
1735 return 48;
1736 }
1737 }
1738
arm_smmu_device_cfg_probe(struct arm_smmu_device * smmu)1739 static int arm_smmu_device_cfg_probe(struct arm_smmu_device *smmu)
1740 {
1741 unsigned int size;
1742 u32 id;
1743 bool cttw_reg, cttw_fw = smmu->features & ARM_SMMU_FEAT_COHERENT_WALK;
1744 int i, ret;
1745
1746 dev_notice(smmu->dev, "probing hardware configuration...\n");
1747 dev_notice(smmu->dev, "SMMUv%d with:\n",
1748 smmu->version == ARM_SMMU_V2 ? 2 : 1);
1749
1750 /* ID0 */
1751 id = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_ID0);
1752
1753 /* Restrict available stages based on module parameter */
1754 if (force_stage == 1)
1755 id &= ~(ARM_SMMU_ID0_S2TS | ARM_SMMU_ID0_NTS);
1756 else if (force_stage == 2)
1757 id &= ~(ARM_SMMU_ID0_S1TS | ARM_SMMU_ID0_NTS);
1758
1759 if (id & ARM_SMMU_ID0_S1TS) {
1760 smmu->features |= ARM_SMMU_FEAT_TRANS_S1;
1761 dev_notice(smmu->dev, "\tstage 1 translation\n");
1762 }
1763
1764 if (id & ARM_SMMU_ID0_S2TS) {
1765 smmu->features |= ARM_SMMU_FEAT_TRANS_S2;
1766 dev_notice(smmu->dev, "\tstage 2 translation\n");
1767 }
1768
1769 if (id & ARM_SMMU_ID0_NTS) {
1770 smmu->features |= ARM_SMMU_FEAT_TRANS_NESTED;
1771 dev_notice(smmu->dev, "\tnested translation\n");
1772 }
1773
1774 if (!(smmu->features &
1775 (ARM_SMMU_FEAT_TRANS_S1 | ARM_SMMU_FEAT_TRANS_S2))) {
1776 dev_err(smmu->dev, "\tno translation support!\n");
1777 return -ENODEV;
1778 }
1779
1780 if ((id & ARM_SMMU_ID0_S1TS) &&
1781 ((smmu->version < ARM_SMMU_V2) || !(id & ARM_SMMU_ID0_ATOSNS))) {
1782 smmu->features |= ARM_SMMU_FEAT_TRANS_OPS;
1783 dev_notice(smmu->dev, "\taddress translation ops\n");
1784 }
1785
1786 /*
1787 * In order for DMA API calls to work properly, we must defer to what
1788 * the FW says about coherency, regardless of what the hardware claims.
1789 * Fortunately, this also opens up a workaround for systems where the
1790 * ID register value has ended up configured incorrectly.
1791 */
1792 cttw_reg = !!(id & ARM_SMMU_ID0_CTTW);
1793 if (cttw_fw || cttw_reg)
1794 dev_notice(smmu->dev, "\t%scoherent table walk\n",
1795 cttw_fw ? "" : "non-");
1796 if (cttw_fw != cttw_reg)
1797 dev_notice(smmu->dev,
1798 "\t(IDR0.CTTW overridden by FW configuration)\n");
1799
1800 /* Max. number of entries we have for stream matching/indexing */
1801 if (smmu->version == ARM_SMMU_V2 && id & ARM_SMMU_ID0_EXIDS) {
1802 smmu->features |= ARM_SMMU_FEAT_EXIDS;
1803 size = 1 << 16;
1804 } else {
1805 size = 1 << FIELD_GET(ARM_SMMU_ID0_NUMSIDB, id);
1806 }
1807 smmu->streamid_mask = size - 1;
1808 if (id & ARM_SMMU_ID0_SMS) {
1809 smmu->features |= ARM_SMMU_FEAT_STREAM_MATCH;
1810 size = FIELD_GET(ARM_SMMU_ID0_NUMSMRG, id);
1811 if (size == 0) {
1812 dev_err(smmu->dev,
1813 "stream-matching supported, but no SMRs present!\n");
1814 return -ENODEV;
1815 }
1816
1817 /* Zero-initialised to mark as invalid */
1818 smmu->smrs = devm_kcalloc(smmu->dev, size, sizeof(*smmu->smrs),
1819 GFP_KERNEL);
1820 if (!smmu->smrs)
1821 return -ENOMEM;
1822
1823 dev_notice(smmu->dev,
1824 "\tstream matching with %u register groups", size);
1825 }
1826 /* s2cr->type == 0 means translation, so initialise explicitly */
1827 smmu->s2crs = devm_kmalloc_array(smmu->dev, size, sizeof(*smmu->s2crs),
1828 GFP_KERNEL);
1829 if (!smmu->s2crs)
1830 return -ENOMEM;
1831 for (i = 0; i < size; i++)
1832 smmu->s2crs[i] = s2cr_init_val;
1833
1834 smmu->num_mapping_groups = size;
1835 mutex_init(&smmu->stream_map_mutex);
1836 spin_lock_init(&smmu->global_sync_lock);
1837
1838 if (smmu->version < ARM_SMMU_V2 ||
1839 !(id & ARM_SMMU_ID0_PTFS_NO_AARCH32)) {
1840 smmu->features |= ARM_SMMU_FEAT_FMT_AARCH32_L;
1841 if (!(id & ARM_SMMU_ID0_PTFS_NO_AARCH32S))
1842 smmu->features |= ARM_SMMU_FEAT_FMT_AARCH32_S;
1843 }
1844
1845 /* ID1 */
1846 id = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_ID1);
1847 smmu->pgshift = (id & ARM_SMMU_ID1_PAGESIZE) ? 16 : 12;
1848
1849 /* Check for size mismatch of SMMU address space from mapped region */
1850 size = 1 << (FIELD_GET(ARM_SMMU_ID1_NUMPAGENDXB, id) + 1);
1851 if (smmu->numpage != 2 * size << smmu->pgshift)
1852 dev_warn(smmu->dev,
1853 "SMMU address space size (0x%x) differs from mapped region size (0x%x)!\n",
1854 2 * size << smmu->pgshift, smmu->numpage);
1855 /* Now properly encode NUMPAGE to subsequently derive SMMU_CB_BASE */
1856 smmu->numpage = size;
1857
1858 smmu->num_s2_context_banks = FIELD_GET(ARM_SMMU_ID1_NUMS2CB, id);
1859 smmu->num_context_banks = FIELD_GET(ARM_SMMU_ID1_NUMCB, id);
1860 if (smmu->num_s2_context_banks > smmu->num_context_banks) {
1861 dev_err(smmu->dev, "impossible number of S2 context banks!\n");
1862 return -ENODEV;
1863 }
1864 dev_notice(smmu->dev, "\t%u context banks (%u stage-2 only)\n",
1865 smmu->num_context_banks, smmu->num_s2_context_banks);
1866 smmu->cbs = devm_kcalloc(smmu->dev, smmu->num_context_banks,
1867 sizeof(*smmu->cbs), GFP_KERNEL);
1868 if (!smmu->cbs)
1869 return -ENOMEM;
1870
1871 /* ID2 */
1872 id = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_ID2);
1873 size = arm_smmu_id_size_to_bits(FIELD_GET(ARM_SMMU_ID2_IAS, id));
1874 smmu->ipa_size = size;
1875
1876 /* The output mask is also applied for bypass */
1877 size = arm_smmu_id_size_to_bits(FIELD_GET(ARM_SMMU_ID2_OAS, id));
1878 smmu->pa_size = size;
1879
1880 if (id & ARM_SMMU_ID2_VMID16)
1881 smmu->features |= ARM_SMMU_FEAT_VMID16;
1882
1883 /*
1884 * What the page table walker can address actually depends on which
1885 * descriptor format is in use, but since a) we don't know that yet,
1886 * and b) it can vary per context bank, this will have to do...
1887 */
1888 if (dma_set_mask_and_coherent(smmu->dev, DMA_BIT_MASK(size)))
1889 dev_warn(smmu->dev,
1890 "failed to set DMA mask for table walker\n");
1891
1892 if (smmu->version < ARM_SMMU_V2) {
1893 smmu->va_size = smmu->ipa_size;
1894 if (smmu->version == ARM_SMMU_V1_64K)
1895 smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_64K;
1896 } else {
1897 size = FIELD_GET(ARM_SMMU_ID2_UBS, id);
1898 smmu->va_size = arm_smmu_id_size_to_bits(size);
1899 if (id & ARM_SMMU_ID2_PTFS_4K)
1900 smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_4K;
1901 if (id & ARM_SMMU_ID2_PTFS_16K)
1902 smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_16K;
1903 if (id & ARM_SMMU_ID2_PTFS_64K)
1904 smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_64K;
1905 }
1906
1907 if (smmu->impl && smmu->impl->cfg_probe) {
1908 ret = smmu->impl->cfg_probe(smmu);
1909 if (ret)
1910 return ret;
1911 }
1912
1913 /* Now we've corralled the various formats, what'll it do? */
1914 if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH32_S)
1915 smmu->pgsize_bitmap |= SZ_4K | SZ_64K | SZ_1M | SZ_16M;
1916 if (smmu->features &
1917 (ARM_SMMU_FEAT_FMT_AARCH32_L | ARM_SMMU_FEAT_FMT_AARCH64_4K))
1918 smmu->pgsize_bitmap |= SZ_4K | SZ_2M | SZ_1G;
1919 if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH64_16K)
1920 smmu->pgsize_bitmap |= SZ_16K | SZ_32M;
1921 if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH64_64K)
1922 smmu->pgsize_bitmap |= SZ_64K | SZ_512M;
1923
1924 dev_notice(smmu->dev, "\tSupported page sizes: 0x%08lx\n",
1925 smmu->pgsize_bitmap);
1926
1927
1928 if (smmu->features & ARM_SMMU_FEAT_TRANS_S1)
1929 dev_notice(smmu->dev, "\tStage-1: %lu-bit VA -> %lu-bit IPA\n",
1930 smmu->va_size, smmu->ipa_size);
1931
1932 if (smmu->features & ARM_SMMU_FEAT_TRANS_S2)
1933 dev_notice(smmu->dev, "\tStage-2: %lu-bit IPA -> %lu-bit PA\n",
1934 smmu->ipa_size, smmu->pa_size);
1935
1936 return 0;
1937 }
1938
1939 struct arm_smmu_match_data {
1940 enum arm_smmu_arch_version version;
1941 enum arm_smmu_implementation model;
1942 };
1943
1944 #define ARM_SMMU_MATCH_DATA(name, ver, imp) \
1945 static const struct arm_smmu_match_data name = { .version = ver, .model = imp }
1946
1947 ARM_SMMU_MATCH_DATA(smmu_generic_v1, ARM_SMMU_V1, GENERIC_SMMU);
1948 ARM_SMMU_MATCH_DATA(smmu_generic_v2, ARM_SMMU_V2, GENERIC_SMMU);
1949 ARM_SMMU_MATCH_DATA(arm_mmu401, ARM_SMMU_V1_64K, GENERIC_SMMU);
1950 ARM_SMMU_MATCH_DATA(arm_mmu500, ARM_SMMU_V2, ARM_MMU500);
1951 ARM_SMMU_MATCH_DATA(cavium_smmuv2, ARM_SMMU_V2, CAVIUM_SMMUV2);
1952 ARM_SMMU_MATCH_DATA(qcom_smmuv2, ARM_SMMU_V2, QCOM_SMMUV2);
1953
1954 static const struct of_device_id arm_smmu_of_match[] = {
1955 { .compatible = "arm,smmu-v1", .data = &smmu_generic_v1 },
1956 { .compatible = "arm,smmu-v2", .data = &smmu_generic_v2 },
1957 { .compatible = "arm,mmu-400", .data = &smmu_generic_v1 },
1958 { .compatible = "arm,mmu-401", .data = &arm_mmu401 },
1959 { .compatible = "arm,mmu-500", .data = &arm_mmu500 },
1960 { .compatible = "cavium,smmu-v2", .data = &cavium_smmuv2 },
1961 { .compatible = "nvidia,smmu-500", .data = &arm_mmu500 },
1962 { .compatible = "qcom,smmu-v2", .data = &qcom_smmuv2 },
1963 { },
1964 };
1965 MODULE_DEVICE_TABLE(of, arm_smmu_of_match);
1966
1967 #ifdef CONFIG_ACPI
acpi_smmu_get_data(u32 model,struct arm_smmu_device * smmu)1968 static int acpi_smmu_get_data(u32 model, struct arm_smmu_device *smmu)
1969 {
1970 int ret = 0;
1971
1972 switch (model) {
1973 case ACPI_IORT_SMMU_V1:
1974 case ACPI_IORT_SMMU_CORELINK_MMU400:
1975 smmu->version = ARM_SMMU_V1;
1976 smmu->model = GENERIC_SMMU;
1977 break;
1978 case ACPI_IORT_SMMU_CORELINK_MMU401:
1979 smmu->version = ARM_SMMU_V1_64K;
1980 smmu->model = GENERIC_SMMU;
1981 break;
1982 case ACPI_IORT_SMMU_V2:
1983 smmu->version = ARM_SMMU_V2;
1984 smmu->model = GENERIC_SMMU;
1985 break;
1986 case ACPI_IORT_SMMU_CORELINK_MMU500:
1987 smmu->version = ARM_SMMU_V2;
1988 smmu->model = ARM_MMU500;
1989 break;
1990 case ACPI_IORT_SMMU_CAVIUM_THUNDERX:
1991 smmu->version = ARM_SMMU_V2;
1992 smmu->model = CAVIUM_SMMUV2;
1993 break;
1994 default:
1995 ret = -ENODEV;
1996 }
1997
1998 return ret;
1999 }
2000
arm_smmu_device_acpi_probe(struct arm_smmu_device * smmu,u32 * global_irqs,u32 * pmu_irqs)2001 static int arm_smmu_device_acpi_probe(struct arm_smmu_device *smmu,
2002 u32 *global_irqs, u32 *pmu_irqs)
2003 {
2004 struct device *dev = smmu->dev;
2005 struct acpi_iort_node *node =
2006 *(struct acpi_iort_node **)dev_get_platdata(dev);
2007 struct acpi_iort_smmu *iort_smmu;
2008 int ret;
2009
2010 /* Retrieve SMMU1/2 specific data */
2011 iort_smmu = (struct acpi_iort_smmu *)node->node_data;
2012
2013 ret = acpi_smmu_get_data(iort_smmu->model, smmu);
2014 if (ret < 0)
2015 return ret;
2016
2017 /* Ignore the configuration access interrupt */
2018 *global_irqs = 1;
2019 *pmu_irqs = 0;
2020
2021 if (iort_smmu->flags & ACPI_IORT_SMMU_COHERENT_WALK)
2022 smmu->features |= ARM_SMMU_FEAT_COHERENT_WALK;
2023
2024 return 0;
2025 }
2026 #else
arm_smmu_device_acpi_probe(struct arm_smmu_device * smmu,u32 * global_irqs,u32 * pmu_irqs)2027 static inline int arm_smmu_device_acpi_probe(struct arm_smmu_device *smmu,
2028 u32 *global_irqs, u32 *pmu_irqs)
2029 {
2030 return -ENODEV;
2031 }
2032 #endif
2033
arm_smmu_device_dt_probe(struct arm_smmu_device * smmu,u32 * global_irqs,u32 * pmu_irqs)2034 static int arm_smmu_device_dt_probe(struct arm_smmu_device *smmu,
2035 u32 *global_irqs, u32 *pmu_irqs)
2036 {
2037 const struct arm_smmu_match_data *data;
2038 struct device *dev = smmu->dev;
2039 bool legacy_binding;
2040
2041 if (of_property_read_u32(dev->of_node, "#global-interrupts", global_irqs))
2042 return dev_err_probe(dev, -ENODEV,
2043 "missing #global-interrupts property\n");
2044 *pmu_irqs = 0;
2045
2046 data = of_device_get_match_data(dev);
2047 smmu->version = data->version;
2048 smmu->model = data->model;
2049
2050 legacy_binding = of_find_property(dev->of_node, "mmu-masters", NULL);
2051 if (legacy_binding && !using_generic_binding) {
2052 if (!using_legacy_binding) {
2053 pr_notice("deprecated \"mmu-masters\" DT property in use; %s support unavailable\n",
2054 IS_ENABLED(CONFIG_ARM_SMMU_LEGACY_DT_BINDINGS) ? "DMA API" : "SMMU");
2055 }
2056 using_legacy_binding = true;
2057 } else if (!legacy_binding && !using_legacy_binding) {
2058 using_generic_binding = true;
2059 } else {
2060 dev_err(dev, "not probing due to mismatched DT properties\n");
2061 return -ENODEV;
2062 }
2063
2064 if (of_dma_is_coherent(dev->of_node))
2065 smmu->features |= ARM_SMMU_FEAT_COHERENT_WALK;
2066
2067 return 0;
2068 }
2069
arm_smmu_rmr_install_bypass_smr(struct arm_smmu_device * smmu)2070 static void arm_smmu_rmr_install_bypass_smr(struct arm_smmu_device *smmu)
2071 {
2072 struct list_head rmr_list;
2073 struct iommu_resv_region *e;
2074 int idx, cnt = 0;
2075 u32 reg;
2076
2077 INIT_LIST_HEAD(&rmr_list);
2078 iort_get_rmr_sids(dev_fwnode(smmu->dev), &rmr_list);
2079
2080 /*
2081 * Rather than trying to look at existing mappings that
2082 * are setup by the firmware and then invalidate the ones
2083 * that do no have matching RMR entries, just disable the
2084 * SMMU until it gets enabled again in the reset routine.
2085 */
2086 reg = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sCR0);
2087 reg |= ARM_SMMU_sCR0_CLIENTPD;
2088 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_sCR0, reg);
2089
2090 list_for_each_entry(e, &rmr_list, list) {
2091 struct iommu_iort_rmr_data *rmr;
2092 int i;
2093
2094 rmr = container_of(e, struct iommu_iort_rmr_data, rr);
2095 for (i = 0; i < rmr->num_sids; i++) {
2096 idx = arm_smmu_find_sme(smmu, rmr->sids[i], ~0);
2097 if (idx < 0)
2098 continue;
2099
2100 if (smmu->s2crs[idx].count == 0) {
2101 smmu->smrs[idx].id = rmr->sids[i];
2102 smmu->smrs[idx].mask = 0;
2103 smmu->smrs[idx].valid = true;
2104 }
2105 smmu->s2crs[idx].count++;
2106 smmu->s2crs[idx].type = S2CR_TYPE_BYPASS;
2107 smmu->s2crs[idx].privcfg = S2CR_PRIVCFG_DEFAULT;
2108
2109 cnt++;
2110 }
2111 }
2112
2113 dev_notice(smmu->dev, "\tpreserved %d boot mapping%s\n", cnt,
2114 str_plural(cnt));
2115 iort_put_rmr_sids(dev_fwnode(smmu->dev), &rmr_list);
2116 }
2117
arm_smmu_device_probe(struct platform_device * pdev)2118 static int arm_smmu_device_probe(struct platform_device *pdev)
2119 {
2120 struct resource *res;
2121 struct arm_smmu_device *smmu;
2122 struct device *dev = &pdev->dev;
2123 int num_irqs, i, err;
2124 u32 global_irqs, pmu_irqs;
2125 irqreturn_t (*global_fault)(int irq, void *dev);
2126
2127 smmu = devm_kzalloc(dev, sizeof(*smmu), GFP_KERNEL);
2128 if (!smmu) {
2129 dev_err(dev, "failed to allocate arm_smmu_device\n");
2130 return -ENOMEM;
2131 }
2132 smmu->dev = dev;
2133
2134 if (dev->of_node)
2135 err = arm_smmu_device_dt_probe(smmu, &global_irqs, &pmu_irqs);
2136 else
2137 err = arm_smmu_device_acpi_probe(smmu, &global_irqs, &pmu_irqs);
2138 if (err)
2139 return err;
2140
2141 smmu->base = devm_platform_get_and_ioremap_resource(pdev, 0, &res);
2142 if (IS_ERR(smmu->base))
2143 return PTR_ERR(smmu->base);
2144 smmu->ioaddr = res->start;
2145
2146 /*
2147 * The resource size should effectively match the value of SMMU_TOP;
2148 * stash that temporarily until we know PAGESIZE to validate it with.
2149 */
2150 smmu->numpage = resource_size(res);
2151
2152 smmu = arm_smmu_impl_init(smmu);
2153 if (IS_ERR(smmu))
2154 return PTR_ERR(smmu);
2155
2156 num_irqs = platform_irq_count(pdev);
2157
2158 smmu->num_context_irqs = num_irqs - global_irqs - pmu_irqs;
2159 if (smmu->num_context_irqs <= 0)
2160 return dev_err_probe(dev, -ENODEV,
2161 "found %d interrupts but expected at least %d\n",
2162 num_irqs, global_irqs + pmu_irqs + 1);
2163
2164 smmu->irqs = devm_kcalloc(dev, smmu->num_context_irqs,
2165 sizeof(*smmu->irqs), GFP_KERNEL);
2166 if (!smmu->irqs)
2167 return dev_err_probe(dev, -ENOMEM, "failed to allocate %d irqs\n",
2168 smmu->num_context_irqs);
2169
2170 for (i = 0; i < smmu->num_context_irqs; i++) {
2171 int irq = platform_get_irq(pdev, global_irqs + pmu_irqs + i);
2172
2173 if (irq < 0)
2174 return irq;
2175 smmu->irqs[i] = irq;
2176 }
2177
2178 err = devm_clk_bulk_get_all(dev, &smmu->clks);
2179 if (err < 0) {
2180 dev_err(dev, "failed to get clocks %d\n", err);
2181 return err;
2182 }
2183 smmu->num_clks = err;
2184
2185 err = clk_bulk_prepare_enable(smmu->num_clks, smmu->clks);
2186 if (err)
2187 return err;
2188
2189 err = arm_smmu_device_cfg_probe(smmu);
2190 if (err)
2191 return err;
2192
2193 if (smmu->version == ARM_SMMU_V2) {
2194 if (smmu->num_context_banks > smmu->num_context_irqs) {
2195 dev_err(dev,
2196 "found only %d context irq(s) but %d required\n",
2197 smmu->num_context_irqs, smmu->num_context_banks);
2198 return -ENODEV;
2199 }
2200
2201 /* Ignore superfluous interrupts */
2202 smmu->num_context_irqs = smmu->num_context_banks;
2203 }
2204
2205 if (smmu->impl && smmu->impl->global_fault)
2206 global_fault = smmu->impl->global_fault;
2207 else
2208 global_fault = arm_smmu_global_fault;
2209
2210 for (i = 0; i < global_irqs; i++) {
2211 int irq = platform_get_irq(pdev, i);
2212
2213 if (irq < 0)
2214 return irq;
2215
2216 err = devm_request_irq(dev, irq, global_fault, IRQF_SHARED,
2217 "arm-smmu global fault", smmu);
2218 if (err)
2219 return dev_err_probe(dev, err,
2220 "failed to request global IRQ %d (%u)\n",
2221 i, irq);
2222 }
2223
2224 platform_set_drvdata(pdev, smmu);
2225
2226 /* Check for RMRs and install bypass SMRs if any */
2227 arm_smmu_rmr_install_bypass_smr(smmu);
2228
2229 arm_smmu_device_reset(smmu);
2230 arm_smmu_test_smr_masks(smmu);
2231
2232 err = iommu_device_sysfs_add(&smmu->iommu, smmu->dev, NULL,
2233 "smmu.%pa", &smmu->ioaddr);
2234 if (err)
2235 return dev_err_probe(dev, err, "Failed to register iommu in sysfs\n");
2236
2237 err = iommu_device_register(&smmu->iommu, &arm_smmu_ops,
2238 using_legacy_binding ? NULL : dev);
2239 if (err) {
2240 iommu_device_sysfs_remove(&smmu->iommu);
2241 return dev_err_probe(dev, err, "Failed to register iommu\n");
2242 }
2243
2244 /*
2245 * We want to avoid touching dev->power.lock in fastpaths unless
2246 * it's really going to do something useful - pm_runtime_enabled()
2247 * can serve as an ideal proxy for that decision. So, conditionally
2248 * enable pm_runtime.
2249 */
2250 if (dev->pm_domain) {
2251 pm_runtime_set_active(dev);
2252 pm_runtime_enable(dev);
2253 arm_smmu_rpm_use_autosuspend(smmu);
2254 }
2255
2256 return 0;
2257 }
2258
arm_smmu_device_shutdown(struct platform_device * pdev)2259 static void arm_smmu_device_shutdown(struct platform_device *pdev)
2260 {
2261 struct arm_smmu_device *smmu = platform_get_drvdata(pdev);
2262
2263 if (!bitmap_empty(smmu->context_map, ARM_SMMU_MAX_CBS))
2264 dev_notice(&pdev->dev, "disabling translation\n");
2265
2266 arm_smmu_rpm_get(smmu);
2267 /* Turn the thing off */
2268 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_sCR0, ARM_SMMU_sCR0_CLIENTPD);
2269 arm_smmu_rpm_put(smmu);
2270
2271 if (pm_runtime_enabled(smmu->dev))
2272 pm_runtime_force_suspend(smmu->dev);
2273 else
2274 clk_bulk_disable(smmu->num_clks, smmu->clks);
2275
2276 clk_bulk_unprepare(smmu->num_clks, smmu->clks);
2277 }
2278
arm_smmu_device_remove(struct platform_device * pdev)2279 static void arm_smmu_device_remove(struct platform_device *pdev)
2280 {
2281 struct arm_smmu_device *smmu = platform_get_drvdata(pdev);
2282
2283 iommu_device_unregister(&smmu->iommu);
2284 iommu_device_sysfs_remove(&smmu->iommu);
2285
2286 arm_smmu_device_shutdown(pdev);
2287 }
2288
arm_smmu_runtime_resume(struct device * dev)2289 static int __maybe_unused arm_smmu_runtime_resume(struct device *dev)
2290 {
2291 struct arm_smmu_device *smmu = dev_get_drvdata(dev);
2292 int ret;
2293
2294 ret = clk_bulk_enable(smmu->num_clks, smmu->clks);
2295 if (ret)
2296 return ret;
2297
2298 arm_smmu_device_reset(smmu);
2299
2300 return 0;
2301 }
2302
arm_smmu_runtime_suspend(struct device * dev)2303 static int __maybe_unused arm_smmu_runtime_suspend(struct device *dev)
2304 {
2305 struct arm_smmu_device *smmu = dev_get_drvdata(dev);
2306
2307 clk_bulk_disable(smmu->num_clks, smmu->clks);
2308
2309 return 0;
2310 }
2311
arm_smmu_pm_resume(struct device * dev)2312 static int __maybe_unused arm_smmu_pm_resume(struct device *dev)
2313 {
2314 int ret;
2315 struct arm_smmu_device *smmu = dev_get_drvdata(dev);
2316
2317 ret = clk_bulk_prepare(smmu->num_clks, smmu->clks);
2318 if (ret)
2319 return ret;
2320
2321 if (pm_runtime_suspended(dev))
2322 return 0;
2323
2324 ret = arm_smmu_runtime_resume(dev);
2325 if (ret)
2326 clk_bulk_unprepare(smmu->num_clks, smmu->clks);
2327
2328 return ret;
2329 }
2330
arm_smmu_pm_suspend(struct device * dev)2331 static int __maybe_unused arm_smmu_pm_suspend(struct device *dev)
2332 {
2333 int ret = 0;
2334 struct arm_smmu_device *smmu = dev_get_drvdata(dev);
2335
2336 if (pm_runtime_suspended(dev))
2337 goto clk_unprepare;
2338
2339 ret = arm_smmu_runtime_suspend(dev);
2340 if (ret)
2341 return ret;
2342
2343 clk_unprepare:
2344 clk_bulk_unprepare(smmu->num_clks, smmu->clks);
2345 return ret;
2346 }
2347
2348 static const struct dev_pm_ops arm_smmu_pm_ops = {
2349 SET_SYSTEM_SLEEP_PM_OPS(arm_smmu_pm_suspend, arm_smmu_pm_resume)
2350 SET_RUNTIME_PM_OPS(arm_smmu_runtime_suspend,
2351 arm_smmu_runtime_resume, NULL)
2352 };
2353
2354 static struct platform_driver arm_smmu_driver = {
2355 .driver = {
2356 .name = "arm-smmu",
2357 .of_match_table = arm_smmu_of_match,
2358 .pm = &arm_smmu_pm_ops,
2359 .suppress_bind_attrs = true,
2360 },
2361 .probe = arm_smmu_device_probe,
2362 .remove = arm_smmu_device_remove,
2363 .shutdown = arm_smmu_device_shutdown,
2364 };
2365 module_platform_driver(arm_smmu_driver);
2366
2367 MODULE_DESCRIPTION("IOMMU API for ARM architected SMMU implementations");
2368 MODULE_AUTHOR("Will Deacon <will@kernel.org>");
2369 MODULE_ALIAS("platform:arm-smmu");
2370 MODULE_LICENSE("GPL v2");
2371