1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * IOMMU API for ARM architected SMMU implementations.
4 *
5 * Copyright (C) 2013 ARM Limited
6 *
7 * Author: Will Deacon <will.deacon@arm.com>
8 *
9 * This driver currently supports:
10 * - SMMUv1 and v2 implementations
11 * - Stream-matching and stream-indexing
12 * - v7/v8 long-descriptor format
13 * - Non-secure access to the SMMU
14 * - Context fault reporting
15 * - Extended Stream ID (16 bit)
16 */
17
18 #define pr_fmt(fmt) "arm-smmu: " fmt
19
20 #include <linux/acpi.h>
21 #include <linux/acpi_iort.h>
22 #include <linux/bitfield.h>
23 #include <linux/delay.h>
24 #include <linux/dma-mapping.h>
25 #include <linux/err.h>
26 #include <linux/interrupt.h>
27 #include <linux/io.h>
28 #include <linux/iopoll.h>
29 #include <linux/module.h>
30 #include <linux/of.h>
31 #include <linux/of_address.h>
32 #include <linux/pci.h>
33 #include <linux/platform_device.h>
34 #include <linux/pm_runtime.h>
35 #include <linux/ratelimit.h>
36 #include <linux/slab.h>
37 #include <linux/string_choices.h>
38
39 #include <linux/fsl/mc.h>
40
41 #include "arm-smmu.h"
42 #include "../../dma-iommu.h"
43
44 /*
45 * Apparently, some Qualcomm arm64 platforms which appear to expose their SMMU
46 * global register space are still, in fact, using a hypervisor to mediate it
47 * by trapping and emulating register accesses. Sadly, some deployed versions
48 * of said trapping code have bugs wherein they go horribly wrong for stores
49 * using r31 (i.e. XZR/WZR) as the source register.
50 */
51 #define QCOM_DUMMY_VAL -1
52
53 #define MSI_IOVA_BASE 0x8000000
54 #define MSI_IOVA_LENGTH 0x100000
55
56 static int force_stage;
57 module_param(force_stage, int, S_IRUGO);
58 MODULE_PARM_DESC(force_stage,
59 "Force SMMU mappings to be installed at a particular stage of translation. A value of '1' or '2' forces the corresponding stage. All other values are ignored (i.e. no stage is forced). Note that selecting a specific stage will disable support for nested translation.");
60 static bool disable_bypass =
61 IS_ENABLED(CONFIG_ARM_SMMU_DISABLE_BYPASS_BY_DEFAULT);
62 module_param(disable_bypass, bool, S_IRUGO);
63 MODULE_PARM_DESC(disable_bypass,
64 "Disable bypass streams such that incoming transactions from devices that are not attached to an iommu domain will report an abort back to the device and will not be allowed to pass through the SMMU.");
65
66 #define s2cr_init_val (struct arm_smmu_s2cr){ \
67 .type = disable_bypass ? S2CR_TYPE_FAULT : S2CR_TYPE_BYPASS, \
68 }
69
70 static bool using_legacy_binding, using_generic_binding;
71
arm_smmu_rpm_get(struct arm_smmu_device * smmu)72 static inline int arm_smmu_rpm_get(struct arm_smmu_device *smmu)
73 {
74 if (pm_runtime_enabled(smmu->dev))
75 return pm_runtime_resume_and_get(smmu->dev);
76
77 return 0;
78 }
79
arm_smmu_rpm_put(struct arm_smmu_device * smmu)80 static inline void arm_smmu_rpm_put(struct arm_smmu_device *smmu)
81 {
82 if (pm_runtime_enabled(smmu->dev)) {
83 pm_runtime_mark_last_busy(smmu->dev);
84 __pm_runtime_put_autosuspend(smmu->dev);
85
86 }
87 }
88
arm_smmu_rpm_use_autosuspend(struct arm_smmu_device * smmu)89 static void arm_smmu_rpm_use_autosuspend(struct arm_smmu_device *smmu)
90 {
91 /*
92 * Setup an autosuspend delay to avoid bouncing runpm state.
93 * Otherwise, if a driver for a suspended consumer device
94 * unmaps buffers, it will runpm resume/suspend for each one.
95 *
96 * For example, when used by a GPU device, when an application
97 * or game exits, it can trigger unmapping 100s or 1000s of
98 * buffers. With a runpm cycle for each buffer, that adds up
99 * to 5-10sec worth of reprogramming the context bank, while
100 * the system appears to be locked up to the user.
101 */
102 pm_runtime_set_autosuspend_delay(smmu->dev, 20);
103 pm_runtime_use_autosuspend(smmu->dev);
104 }
105
to_smmu_domain(struct iommu_domain * dom)106 static struct arm_smmu_domain *to_smmu_domain(struct iommu_domain *dom)
107 {
108 return container_of(dom, struct arm_smmu_domain, domain);
109 }
110
111 static struct platform_driver arm_smmu_driver;
112 static struct iommu_ops arm_smmu_ops;
113
114 #ifdef CONFIG_ARM_SMMU_LEGACY_DT_BINDINGS
dev_get_dev_node(struct device * dev)115 static struct device_node *dev_get_dev_node(struct device *dev)
116 {
117 if (dev_is_pci(dev)) {
118 struct pci_bus *bus = to_pci_dev(dev)->bus;
119
120 while (!pci_is_root_bus(bus))
121 bus = bus->parent;
122 return of_node_get(bus->bridge->parent->of_node);
123 }
124
125 return of_node_get(dev->of_node);
126 }
127
__arm_smmu_get_pci_sid(struct pci_dev * pdev,u16 alias,void * data)128 static int __arm_smmu_get_pci_sid(struct pci_dev *pdev, u16 alias, void *data)
129 {
130 *((__be32 *)data) = cpu_to_be32(alias);
131 return 0; /* Continue walking */
132 }
133
__find_legacy_master_phandle(struct device * dev,void * data)134 static int __find_legacy_master_phandle(struct device *dev, void *data)
135 {
136 struct of_phandle_iterator *it = *(void **)data;
137 struct device_node *np = it->node;
138 int err;
139
140 of_for_each_phandle(it, err, dev->of_node, "mmu-masters",
141 "#stream-id-cells", -1)
142 if (it->node == np) {
143 *(void **)data = dev;
144 return 1;
145 }
146 it->node = np;
147 return err == -ENOENT ? 0 : err;
148 }
149
arm_smmu_register_legacy_master(struct device * dev,struct arm_smmu_device ** smmu)150 static int arm_smmu_register_legacy_master(struct device *dev,
151 struct arm_smmu_device **smmu)
152 {
153 struct device *smmu_dev;
154 struct device_node *np;
155 struct of_phandle_iterator it;
156 void *data = ⁢
157 u32 *sids;
158 __be32 pci_sid;
159 int err;
160
161 np = dev_get_dev_node(dev);
162 if (!np || !of_property_present(np, "#stream-id-cells")) {
163 of_node_put(np);
164 return -ENODEV;
165 }
166
167 it.node = np;
168 err = driver_for_each_device(&arm_smmu_driver.driver, NULL, &data,
169 __find_legacy_master_phandle);
170 smmu_dev = data;
171 of_node_put(np);
172 if (err == 0)
173 return -ENODEV;
174 if (err < 0)
175 return err;
176
177 if (dev_is_pci(dev)) {
178 /* "mmu-masters" assumes Stream ID == Requester ID */
179 pci_for_each_dma_alias(to_pci_dev(dev), __arm_smmu_get_pci_sid,
180 &pci_sid);
181 it.cur = &pci_sid;
182 it.cur_count = 1;
183 }
184
185 err = iommu_fwspec_init(dev, NULL);
186 if (err)
187 return err;
188
189 sids = kcalloc(it.cur_count, sizeof(*sids), GFP_KERNEL);
190 if (!sids)
191 return -ENOMEM;
192
193 *smmu = dev_get_drvdata(smmu_dev);
194 of_phandle_iterator_args(&it, sids, it.cur_count);
195 err = iommu_fwspec_add_ids(dev, sids, it.cur_count);
196 kfree(sids);
197 return err;
198 }
199 #else
arm_smmu_register_legacy_master(struct device * dev,struct arm_smmu_device ** smmu)200 static int arm_smmu_register_legacy_master(struct device *dev,
201 struct arm_smmu_device **smmu)
202 {
203 return -ENODEV;
204 }
205 #endif /* CONFIG_ARM_SMMU_LEGACY_DT_BINDINGS */
206
__arm_smmu_free_bitmap(unsigned long * map,int idx)207 static void __arm_smmu_free_bitmap(unsigned long *map, int idx)
208 {
209 clear_bit(idx, map);
210 }
211
212 /* Wait for any pending TLB invalidations to complete */
__arm_smmu_tlb_sync(struct arm_smmu_device * smmu,int page,int sync,int status)213 static void __arm_smmu_tlb_sync(struct arm_smmu_device *smmu, int page,
214 int sync, int status)
215 {
216 unsigned int spin_cnt, delay;
217 u32 reg;
218
219 if (smmu->impl && unlikely(smmu->impl->tlb_sync))
220 return smmu->impl->tlb_sync(smmu, page, sync, status);
221
222 arm_smmu_writel(smmu, page, sync, QCOM_DUMMY_VAL);
223 for (delay = 1; delay < TLB_LOOP_TIMEOUT; delay *= 2) {
224 for (spin_cnt = TLB_SPIN_COUNT; spin_cnt > 0; spin_cnt--) {
225 reg = arm_smmu_readl(smmu, page, status);
226 if (!(reg & ARM_SMMU_sTLBGSTATUS_GSACTIVE))
227 return;
228 cpu_relax();
229 }
230 udelay(delay);
231 }
232 dev_err_ratelimited(smmu->dev,
233 "TLB sync timed out -- SMMU may be deadlocked\n");
234 }
235
arm_smmu_tlb_sync_global(struct arm_smmu_device * smmu)236 static void arm_smmu_tlb_sync_global(struct arm_smmu_device *smmu)
237 {
238 unsigned long flags;
239
240 spin_lock_irqsave(&smmu->global_sync_lock, flags);
241 __arm_smmu_tlb_sync(smmu, ARM_SMMU_GR0, ARM_SMMU_GR0_sTLBGSYNC,
242 ARM_SMMU_GR0_sTLBGSTATUS);
243 spin_unlock_irqrestore(&smmu->global_sync_lock, flags);
244 }
245
arm_smmu_tlb_sync_context(struct arm_smmu_domain * smmu_domain)246 static void arm_smmu_tlb_sync_context(struct arm_smmu_domain *smmu_domain)
247 {
248 struct arm_smmu_device *smmu = smmu_domain->smmu;
249 unsigned long flags;
250
251 spin_lock_irqsave(&smmu_domain->cb_lock, flags);
252 __arm_smmu_tlb_sync(smmu, ARM_SMMU_CB(smmu, smmu_domain->cfg.cbndx),
253 ARM_SMMU_CB_TLBSYNC, ARM_SMMU_CB_TLBSTATUS);
254 spin_unlock_irqrestore(&smmu_domain->cb_lock, flags);
255 }
256
arm_smmu_tlb_inv_context_s1(void * cookie)257 static void arm_smmu_tlb_inv_context_s1(void *cookie)
258 {
259 struct arm_smmu_domain *smmu_domain = cookie;
260 /*
261 * The TLBI write may be relaxed, so ensure that PTEs cleared by the
262 * current CPU are visible beforehand.
263 */
264 wmb();
265 arm_smmu_cb_write(smmu_domain->smmu, smmu_domain->cfg.cbndx,
266 ARM_SMMU_CB_S1_TLBIASID, smmu_domain->cfg.asid);
267 arm_smmu_tlb_sync_context(smmu_domain);
268 }
269
arm_smmu_tlb_inv_context_s2(void * cookie)270 static void arm_smmu_tlb_inv_context_s2(void *cookie)
271 {
272 struct arm_smmu_domain *smmu_domain = cookie;
273 struct arm_smmu_device *smmu = smmu_domain->smmu;
274
275 /* See above */
276 wmb();
277 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_TLBIVMID, smmu_domain->cfg.vmid);
278 arm_smmu_tlb_sync_global(smmu);
279 }
280
arm_smmu_tlb_inv_range_s1(unsigned long iova,size_t size,size_t granule,void * cookie,int reg)281 static void arm_smmu_tlb_inv_range_s1(unsigned long iova, size_t size,
282 size_t granule, void *cookie, int reg)
283 {
284 struct arm_smmu_domain *smmu_domain = cookie;
285 struct arm_smmu_device *smmu = smmu_domain->smmu;
286 struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
287 int idx = cfg->cbndx;
288
289 if (smmu->features & ARM_SMMU_FEAT_COHERENT_WALK)
290 wmb();
291
292 if (cfg->fmt != ARM_SMMU_CTX_FMT_AARCH64) {
293 iova = (iova >> 12) << 12;
294 iova |= cfg->asid;
295 do {
296 arm_smmu_cb_write(smmu, idx, reg, iova);
297 iova += granule;
298 } while (size -= granule);
299 } else {
300 iova >>= 12;
301 iova |= (u64)cfg->asid << 48;
302 do {
303 arm_smmu_cb_writeq(smmu, idx, reg, iova);
304 iova += granule >> 12;
305 } while (size -= granule);
306 }
307 }
308
arm_smmu_tlb_inv_range_s2(unsigned long iova,size_t size,size_t granule,void * cookie,int reg)309 static void arm_smmu_tlb_inv_range_s2(unsigned long iova, size_t size,
310 size_t granule, void *cookie, int reg)
311 {
312 struct arm_smmu_domain *smmu_domain = cookie;
313 struct arm_smmu_device *smmu = smmu_domain->smmu;
314 int idx = smmu_domain->cfg.cbndx;
315
316 if (smmu->features & ARM_SMMU_FEAT_COHERENT_WALK)
317 wmb();
318
319 iova >>= 12;
320 do {
321 if (smmu_domain->cfg.fmt == ARM_SMMU_CTX_FMT_AARCH64)
322 arm_smmu_cb_writeq(smmu, idx, reg, iova);
323 else
324 arm_smmu_cb_write(smmu, idx, reg, iova);
325 iova += granule >> 12;
326 } while (size -= granule);
327 }
328
arm_smmu_tlb_inv_walk_s1(unsigned long iova,size_t size,size_t granule,void * cookie)329 static void arm_smmu_tlb_inv_walk_s1(unsigned long iova, size_t size,
330 size_t granule, void *cookie)
331 {
332 struct arm_smmu_domain *smmu_domain = cookie;
333 struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
334
335 if (cfg->flush_walk_prefer_tlbiasid) {
336 arm_smmu_tlb_inv_context_s1(cookie);
337 } else {
338 arm_smmu_tlb_inv_range_s1(iova, size, granule, cookie,
339 ARM_SMMU_CB_S1_TLBIVA);
340 arm_smmu_tlb_sync_context(cookie);
341 }
342 }
343
arm_smmu_tlb_add_page_s1(struct iommu_iotlb_gather * gather,unsigned long iova,size_t granule,void * cookie)344 static void arm_smmu_tlb_add_page_s1(struct iommu_iotlb_gather *gather,
345 unsigned long iova, size_t granule,
346 void *cookie)
347 {
348 arm_smmu_tlb_inv_range_s1(iova, granule, granule, cookie,
349 ARM_SMMU_CB_S1_TLBIVAL);
350 }
351
arm_smmu_tlb_inv_walk_s2(unsigned long iova,size_t size,size_t granule,void * cookie)352 static void arm_smmu_tlb_inv_walk_s2(unsigned long iova, size_t size,
353 size_t granule, void *cookie)
354 {
355 arm_smmu_tlb_inv_range_s2(iova, size, granule, cookie,
356 ARM_SMMU_CB_S2_TLBIIPAS2);
357 arm_smmu_tlb_sync_context(cookie);
358 }
359
arm_smmu_tlb_add_page_s2(struct iommu_iotlb_gather * gather,unsigned long iova,size_t granule,void * cookie)360 static void arm_smmu_tlb_add_page_s2(struct iommu_iotlb_gather *gather,
361 unsigned long iova, size_t granule,
362 void *cookie)
363 {
364 arm_smmu_tlb_inv_range_s2(iova, granule, granule, cookie,
365 ARM_SMMU_CB_S2_TLBIIPAS2L);
366 }
367
arm_smmu_tlb_inv_walk_s2_v1(unsigned long iova,size_t size,size_t granule,void * cookie)368 static void arm_smmu_tlb_inv_walk_s2_v1(unsigned long iova, size_t size,
369 size_t granule, void *cookie)
370 {
371 arm_smmu_tlb_inv_context_s2(cookie);
372 }
373 /*
374 * On MMU-401 at least, the cost of firing off multiple TLBIVMIDs appears
375 * almost negligible, but the benefit of getting the first one in as far ahead
376 * of the sync as possible is significant, hence we don't just make this a
377 * no-op and call arm_smmu_tlb_inv_context_s2() from .iotlb_sync as you might
378 * think.
379 */
arm_smmu_tlb_add_page_s2_v1(struct iommu_iotlb_gather * gather,unsigned long iova,size_t granule,void * cookie)380 static void arm_smmu_tlb_add_page_s2_v1(struct iommu_iotlb_gather *gather,
381 unsigned long iova, size_t granule,
382 void *cookie)
383 {
384 struct arm_smmu_domain *smmu_domain = cookie;
385 struct arm_smmu_device *smmu = smmu_domain->smmu;
386
387 if (smmu->features & ARM_SMMU_FEAT_COHERENT_WALK)
388 wmb();
389
390 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_TLBIVMID, smmu_domain->cfg.vmid);
391 }
392
393 static const struct iommu_flush_ops arm_smmu_s1_tlb_ops = {
394 .tlb_flush_all = arm_smmu_tlb_inv_context_s1,
395 .tlb_flush_walk = arm_smmu_tlb_inv_walk_s1,
396 .tlb_add_page = arm_smmu_tlb_add_page_s1,
397 };
398
399 static const struct iommu_flush_ops arm_smmu_s2_tlb_ops_v2 = {
400 .tlb_flush_all = arm_smmu_tlb_inv_context_s2,
401 .tlb_flush_walk = arm_smmu_tlb_inv_walk_s2,
402 .tlb_add_page = arm_smmu_tlb_add_page_s2,
403 };
404
405 static const struct iommu_flush_ops arm_smmu_s2_tlb_ops_v1 = {
406 .tlb_flush_all = arm_smmu_tlb_inv_context_s2,
407 .tlb_flush_walk = arm_smmu_tlb_inv_walk_s2_v1,
408 .tlb_add_page = arm_smmu_tlb_add_page_s2_v1,
409 };
410
411
arm_smmu_read_context_fault_info(struct arm_smmu_device * smmu,int idx,struct arm_smmu_context_fault_info * cfi)412 void arm_smmu_read_context_fault_info(struct arm_smmu_device *smmu, int idx,
413 struct arm_smmu_context_fault_info *cfi)
414 {
415 cfi->iova = arm_smmu_cb_readq(smmu, idx, ARM_SMMU_CB_FAR);
416 cfi->fsr = arm_smmu_cb_read(smmu, idx, ARM_SMMU_CB_FSR);
417 cfi->fsynr = arm_smmu_cb_read(smmu, idx, ARM_SMMU_CB_FSYNR0);
418 cfi->cbfrsynra = arm_smmu_gr1_read(smmu, ARM_SMMU_GR1_CBFRSYNRA(idx));
419 }
420
arm_smmu_print_context_fault_info(struct arm_smmu_device * smmu,int idx,const struct arm_smmu_context_fault_info * cfi)421 void arm_smmu_print_context_fault_info(struct arm_smmu_device *smmu, int idx,
422 const struct arm_smmu_context_fault_info *cfi)
423 {
424 dev_err(smmu->dev,
425 "Unhandled context fault: fsr=0x%x, iova=0x%08lx, fsynr=0x%x, cbfrsynra=0x%x, cb=%d\n",
426 cfi->fsr, cfi->iova, cfi->fsynr, cfi->cbfrsynra, idx);
427
428 dev_err(smmu->dev, "FSR = %08x [%s%sFormat=%u%s%s%s%s%s%s%s%s], SID=0x%x\n",
429 cfi->fsr,
430 (cfi->fsr & ARM_SMMU_CB_FSR_MULTI) ? "MULTI " : "",
431 (cfi->fsr & ARM_SMMU_CB_FSR_SS) ? "SS " : "",
432 (u32)FIELD_GET(ARM_SMMU_CB_FSR_FORMAT, cfi->fsr),
433 (cfi->fsr & ARM_SMMU_CB_FSR_UUT) ? " UUT" : "",
434 (cfi->fsr & ARM_SMMU_CB_FSR_ASF) ? " ASF" : "",
435 (cfi->fsr & ARM_SMMU_CB_FSR_TLBLKF) ? " TLBLKF" : "",
436 (cfi->fsr & ARM_SMMU_CB_FSR_TLBMCF) ? " TLBMCF" : "",
437 (cfi->fsr & ARM_SMMU_CB_FSR_EF) ? " EF" : "",
438 (cfi->fsr & ARM_SMMU_CB_FSR_PF) ? " PF" : "",
439 (cfi->fsr & ARM_SMMU_CB_FSR_AFF) ? " AFF" : "",
440 (cfi->fsr & ARM_SMMU_CB_FSR_TF) ? " TF" : "",
441 cfi->cbfrsynra);
442
443 dev_err(smmu->dev, "FSYNR0 = %08x [S1CBNDX=%u%s%s%s%s%s%s PLVL=%u]\n",
444 cfi->fsynr,
445 (u32)FIELD_GET(ARM_SMMU_CB_FSYNR0_S1CBNDX, cfi->fsynr),
446 (cfi->fsynr & ARM_SMMU_CB_FSYNR0_AFR) ? " AFR" : "",
447 (cfi->fsynr & ARM_SMMU_CB_FSYNR0_PTWF) ? " PTWF" : "",
448 (cfi->fsynr & ARM_SMMU_CB_FSYNR0_NSATTR) ? " NSATTR" : "",
449 (cfi->fsynr & ARM_SMMU_CB_FSYNR0_IND) ? " IND" : "",
450 (cfi->fsynr & ARM_SMMU_CB_FSYNR0_PNU) ? " PNU" : "",
451 (cfi->fsynr & ARM_SMMU_CB_FSYNR0_WNR) ? " WNR" : "",
452 (u32)FIELD_GET(ARM_SMMU_CB_FSYNR0_PLVL, cfi->fsynr));
453 }
454
arm_smmu_context_fault(int irq,void * dev)455 static irqreturn_t arm_smmu_context_fault(int irq, void *dev)
456 {
457 struct arm_smmu_context_fault_info cfi;
458 struct arm_smmu_domain *smmu_domain = dev;
459 struct arm_smmu_device *smmu = smmu_domain->smmu;
460 static DEFINE_RATELIMIT_STATE(rs, DEFAULT_RATELIMIT_INTERVAL,
461 DEFAULT_RATELIMIT_BURST);
462 int idx = smmu_domain->cfg.cbndx;
463 int ret;
464
465 arm_smmu_read_context_fault_info(smmu, idx, &cfi);
466
467 if (!(cfi.fsr & ARM_SMMU_CB_FSR_FAULT))
468 return IRQ_NONE;
469
470 ret = report_iommu_fault(&smmu_domain->domain, NULL, cfi.iova,
471 cfi.fsynr & ARM_SMMU_CB_FSYNR0_WNR ? IOMMU_FAULT_WRITE : IOMMU_FAULT_READ);
472
473 if (ret == -ENOSYS && __ratelimit(&rs))
474 arm_smmu_print_context_fault_info(smmu, idx, &cfi);
475
476 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_FSR, cfi.fsr);
477
478 if (cfi.fsr & ARM_SMMU_CB_FSR_SS) {
479 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_RESUME,
480 ret == -EAGAIN ? 0 : ARM_SMMU_RESUME_TERMINATE);
481 }
482
483 return IRQ_HANDLED;
484 }
485
arm_smmu_global_fault(int irq,void * dev)486 static irqreturn_t arm_smmu_global_fault(int irq, void *dev)
487 {
488 u32 gfsr, gfsynr0, gfsynr1, gfsynr2;
489 struct arm_smmu_device *smmu = dev;
490 static DEFINE_RATELIMIT_STATE(rs, DEFAULT_RATELIMIT_INTERVAL,
491 DEFAULT_RATELIMIT_BURST);
492
493 gfsr = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sGFSR);
494 gfsynr0 = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sGFSYNR0);
495 gfsynr1 = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sGFSYNR1);
496 gfsynr2 = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sGFSYNR2);
497
498 if (!gfsr)
499 return IRQ_NONE;
500
501 if (__ratelimit(&rs)) {
502 if (IS_ENABLED(CONFIG_ARM_SMMU_DISABLE_BYPASS_BY_DEFAULT) &&
503 (gfsr & ARM_SMMU_sGFSR_USF))
504 dev_err(smmu->dev,
505 "Blocked unknown Stream ID 0x%hx; boot with \"arm-smmu.disable_bypass=0\" to allow, but this may have security implications\n",
506 (u16)gfsynr1);
507 else
508 dev_err(smmu->dev,
509 "Unexpected global fault, this could be serious\n");
510 dev_err(smmu->dev,
511 "\tGFSR 0x%08x, GFSYNR0 0x%08x, GFSYNR1 0x%08x, GFSYNR2 0x%08x\n",
512 gfsr, gfsynr0, gfsynr1, gfsynr2);
513 }
514
515 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_sGFSR, gfsr);
516 return IRQ_HANDLED;
517 }
518
arm_smmu_init_context_bank(struct arm_smmu_domain * smmu_domain,struct io_pgtable_cfg * pgtbl_cfg)519 static void arm_smmu_init_context_bank(struct arm_smmu_domain *smmu_domain,
520 struct io_pgtable_cfg *pgtbl_cfg)
521 {
522 struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
523 struct arm_smmu_cb *cb = &smmu_domain->smmu->cbs[cfg->cbndx];
524 bool stage1 = cfg->cbar != CBAR_TYPE_S2_TRANS;
525
526 cb->cfg = cfg;
527
528 /* TCR */
529 if (stage1) {
530 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) {
531 cb->tcr[0] = pgtbl_cfg->arm_v7s_cfg.tcr;
532 } else {
533 cb->tcr[0] = arm_smmu_lpae_tcr(pgtbl_cfg);
534 cb->tcr[1] = arm_smmu_lpae_tcr2(pgtbl_cfg);
535 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64)
536 cb->tcr[1] |= ARM_SMMU_TCR2_AS;
537 else
538 cb->tcr[0] |= ARM_SMMU_TCR_EAE;
539 }
540 } else {
541 cb->tcr[0] = arm_smmu_lpae_vtcr(pgtbl_cfg);
542 }
543
544 /* TTBRs */
545 if (stage1) {
546 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) {
547 cb->ttbr[0] = pgtbl_cfg->arm_v7s_cfg.ttbr;
548 cb->ttbr[1] = 0;
549 } else {
550 cb->ttbr[0] = FIELD_PREP(ARM_SMMU_TTBRn_ASID,
551 cfg->asid);
552 cb->ttbr[1] = FIELD_PREP(ARM_SMMU_TTBRn_ASID,
553 cfg->asid);
554
555 if (pgtbl_cfg->quirks & IO_PGTABLE_QUIRK_ARM_TTBR1)
556 cb->ttbr[1] |= pgtbl_cfg->arm_lpae_s1_cfg.ttbr;
557 else
558 cb->ttbr[0] |= pgtbl_cfg->arm_lpae_s1_cfg.ttbr;
559 }
560 } else {
561 cb->ttbr[0] = pgtbl_cfg->arm_lpae_s2_cfg.vttbr;
562 }
563
564 /* MAIRs (stage-1 only) */
565 if (stage1) {
566 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) {
567 cb->mair[0] = pgtbl_cfg->arm_v7s_cfg.prrr;
568 cb->mair[1] = pgtbl_cfg->arm_v7s_cfg.nmrr;
569 } else {
570 cb->mair[0] = pgtbl_cfg->arm_lpae_s1_cfg.mair;
571 cb->mair[1] = pgtbl_cfg->arm_lpae_s1_cfg.mair >> 32;
572 }
573 }
574 }
575
arm_smmu_write_context_bank(struct arm_smmu_device * smmu,int idx)576 void arm_smmu_write_context_bank(struct arm_smmu_device *smmu, int idx)
577 {
578 u32 reg;
579 bool stage1;
580 struct arm_smmu_cb *cb = &smmu->cbs[idx];
581 struct arm_smmu_cfg *cfg = cb->cfg;
582
583 /* Unassigned context banks only need disabling */
584 if (!cfg) {
585 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_SCTLR, 0);
586 return;
587 }
588
589 stage1 = cfg->cbar != CBAR_TYPE_S2_TRANS;
590
591 /* CBA2R */
592 if (smmu->version > ARM_SMMU_V1) {
593 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64)
594 reg = ARM_SMMU_CBA2R_VA64;
595 else
596 reg = 0;
597 /* 16-bit VMIDs live in CBA2R */
598 if (smmu->features & ARM_SMMU_FEAT_VMID16)
599 reg |= FIELD_PREP(ARM_SMMU_CBA2R_VMID16, cfg->vmid);
600
601 arm_smmu_gr1_write(smmu, ARM_SMMU_GR1_CBA2R(idx), reg);
602 }
603
604 /* CBAR */
605 reg = FIELD_PREP(ARM_SMMU_CBAR_TYPE, cfg->cbar);
606 if (smmu->version < ARM_SMMU_V2)
607 reg |= FIELD_PREP(ARM_SMMU_CBAR_IRPTNDX, cfg->irptndx);
608
609 /*
610 * Use the weakest shareability/memory types, so they are
611 * overridden by the ttbcr/pte.
612 */
613 if (stage1) {
614 reg |= FIELD_PREP(ARM_SMMU_CBAR_S1_BPSHCFG,
615 ARM_SMMU_CBAR_S1_BPSHCFG_NSH) |
616 FIELD_PREP(ARM_SMMU_CBAR_S1_MEMATTR,
617 ARM_SMMU_CBAR_S1_MEMATTR_WB);
618 } else if (!(smmu->features & ARM_SMMU_FEAT_VMID16)) {
619 /* 8-bit VMIDs live in CBAR */
620 reg |= FIELD_PREP(ARM_SMMU_CBAR_VMID, cfg->vmid);
621 }
622 arm_smmu_gr1_write(smmu, ARM_SMMU_GR1_CBAR(idx), reg);
623
624 /*
625 * TCR
626 * We must write this before the TTBRs, since it determines the
627 * access behaviour of some fields (in particular, ASID[15:8]).
628 */
629 if (stage1 && smmu->version > ARM_SMMU_V1)
630 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_TCR2, cb->tcr[1]);
631 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_TCR, cb->tcr[0]);
632
633 /* TTBRs */
634 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) {
635 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_CONTEXTIDR, cfg->asid);
636 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_TTBR0, cb->ttbr[0]);
637 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_TTBR1, cb->ttbr[1]);
638 } else {
639 arm_smmu_cb_writeq(smmu, idx, ARM_SMMU_CB_TTBR0, cb->ttbr[0]);
640 if (stage1)
641 arm_smmu_cb_writeq(smmu, idx, ARM_SMMU_CB_TTBR1,
642 cb->ttbr[1]);
643 }
644
645 /* MAIRs (stage-1 only) */
646 if (stage1) {
647 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_S1_MAIR0, cb->mair[0]);
648 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_S1_MAIR1, cb->mair[1]);
649 }
650
651 /* SCTLR */
652 reg = ARM_SMMU_SCTLR_CFIE | ARM_SMMU_SCTLR_CFRE | ARM_SMMU_SCTLR_AFE |
653 ARM_SMMU_SCTLR_TRE | ARM_SMMU_SCTLR_M;
654 if (stage1)
655 reg |= ARM_SMMU_SCTLR_S1_ASIDPNE;
656 if (IS_ENABLED(CONFIG_CPU_BIG_ENDIAN))
657 reg |= ARM_SMMU_SCTLR_E;
658
659 if (smmu->impl && smmu->impl->write_sctlr)
660 smmu->impl->write_sctlr(smmu, idx, reg);
661 else
662 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_SCTLR, reg);
663 }
664
arm_smmu_alloc_context_bank(struct arm_smmu_domain * smmu_domain,struct arm_smmu_device * smmu,struct device * dev,unsigned int start)665 static int arm_smmu_alloc_context_bank(struct arm_smmu_domain *smmu_domain,
666 struct arm_smmu_device *smmu,
667 struct device *dev, unsigned int start)
668 {
669 if (smmu->impl && smmu->impl->alloc_context_bank)
670 return smmu->impl->alloc_context_bank(smmu_domain, smmu, dev, start);
671
672 return __arm_smmu_alloc_bitmap(smmu->context_map, start, smmu->num_context_banks);
673 }
674
arm_smmu_init_domain_context(struct arm_smmu_domain * smmu_domain,struct arm_smmu_device * smmu,struct device * dev)675 static int arm_smmu_init_domain_context(struct arm_smmu_domain *smmu_domain,
676 struct arm_smmu_device *smmu,
677 struct device *dev)
678 {
679 int irq, start, ret = 0;
680 unsigned long ias, oas;
681 struct io_pgtable_ops *pgtbl_ops;
682 struct io_pgtable_cfg pgtbl_cfg;
683 enum io_pgtable_fmt fmt;
684 struct iommu_domain *domain = &smmu_domain->domain;
685 struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
686 irqreturn_t (*context_fault)(int irq, void *dev);
687
688 mutex_lock(&smmu_domain->init_mutex);
689 if (smmu_domain->smmu)
690 goto out_unlock;
691
692 /*
693 * Mapping the requested stage onto what we support is surprisingly
694 * complicated, mainly because the spec allows S1+S2 SMMUs without
695 * support for nested translation. That means we end up with the
696 * following table:
697 *
698 * Requested Supported Actual
699 * S1 N S1
700 * S1 S1+S2 S1
701 * S1 S2 S2
702 * S1 S1 S1
703 * N N N
704 * N S1+S2 S2
705 * N S2 S2
706 * N S1 S1
707 *
708 * Note that you can't actually request stage-2 mappings.
709 */
710 if (!(smmu->features & ARM_SMMU_FEAT_TRANS_S1))
711 smmu_domain->stage = ARM_SMMU_DOMAIN_S2;
712 if (!(smmu->features & ARM_SMMU_FEAT_TRANS_S2))
713 smmu_domain->stage = ARM_SMMU_DOMAIN_S1;
714
715 /*
716 * Choosing a suitable context format is even more fiddly. Until we
717 * grow some way for the caller to express a preference, and/or move
718 * the decision into the io-pgtable code where it arguably belongs,
719 * just aim for the closest thing to the rest of the system, and hope
720 * that the hardware isn't esoteric enough that we can't assume AArch64
721 * support to be a superset of AArch32 support...
722 */
723 if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH32_L)
724 cfg->fmt = ARM_SMMU_CTX_FMT_AARCH32_L;
725 if (IS_ENABLED(CONFIG_IOMMU_IO_PGTABLE_ARMV7S) &&
726 !IS_ENABLED(CONFIG_64BIT) && !IS_ENABLED(CONFIG_ARM_LPAE) &&
727 (smmu->features & ARM_SMMU_FEAT_FMT_AARCH32_S) &&
728 (smmu_domain->stage == ARM_SMMU_DOMAIN_S1))
729 cfg->fmt = ARM_SMMU_CTX_FMT_AARCH32_S;
730 if ((IS_ENABLED(CONFIG_64BIT) || cfg->fmt == ARM_SMMU_CTX_FMT_NONE) &&
731 (smmu->features & (ARM_SMMU_FEAT_FMT_AARCH64_64K |
732 ARM_SMMU_FEAT_FMT_AARCH64_16K |
733 ARM_SMMU_FEAT_FMT_AARCH64_4K)))
734 cfg->fmt = ARM_SMMU_CTX_FMT_AARCH64;
735
736 if (cfg->fmt == ARM_SMMU_CTX_FMT_NONE) {
737 ret = -EINVAL;
738 goto out_unlock;
739 }
740
741 switch (smmu_domain->stage) {
742 case ARM_SMMU_DOMAIN_S1:
743 cfg->cbar = CBAR_TYPE_S1_TRANS_S2_BYPASS;
744 start = smmu->num_s2_context_banks;
745 ias = smmu->va_size;
746 oas = smmu->ipa_size;
747 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64) {
748 fmt = ARM_64_LPAE_S1;
749 } else if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_L) {
750 fmt = ARM_32_LPAE_S1;
751 ias = min(ias, 32UL);
752 oas = min(oas, 40UL);
753 } else {
754 fmt = ARM_V7S;
755 ias = min(ias, 32UL);
756 oas = min(oas, 32UL);
757 }
758 smmu_domain->flush_ops = &arm_smmu_s1_tlb_ops;
759 break;
760 case ARM_SMMU_DOMAIN_NESTED:
761 /*
762 * We will likely want to change this if/when KVM gets
763 * involved.
764 */
765 case ARM_SMMU_DOMAIN_S2:
766 cfg->cbar = CBAR_TYPE_S2_TRANS;
767 start = 0;
768 ias = smmu->ipa_size;
769 oas = smmu->pa_size;
770 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64) {
771 fmt = ARM_64_LPAE_S2;
772 } else {
773 fmt = ARM_32_LPAE_S2;
774 ias = min(ias, 40UL);
775 oas = min(oas, 40UL);
776 }
777 if (smmu->version == ARM_SMMU_V2)
778 smmu_domain->flush_ops = &arm_smmu_s2_tlb_ops_v2;
779 else
780 smmu_domain->flush_ops = &arm_smmu_s2_tlb_ops_v1;
781 break;
782 default:
783 ret = -EINVAL;
784 goto out_unlock;
785 }
786
787 ret = arm_smmu_alloc_context_bank(smmu_domain, smmu, dev, start);
788 if (ret < 0) {
789 goto out_unlock;
790 }
791
792 smmu_domain->smmu = smmu;
793
794 cfg->cbndx = ret;
795 if (smmu->version < ARM_SMMU_V2) {
796 cfg->irptndx = atomic_inc_return(&smmu->irptndx);
797 cfg->irptndx %= smmu->num_context_irqs;
798 } else {
799 cfg->irptndx = cfg->cbndx;
800 }
801
802 if (smmu_domain->stage == ARM_SMMU_DOMAIN_S2)
803 cfg->vmid = cfg->cbndx + 1;
804 else
805 cfg->asid = cfg->cbndx;
806
807 pgtbl_cfg = (struct io_pgtable_cfg) {
808 .pgsize_bitmap = smmu->pgsize_bitmap,
809 .ias = ias,
810 .oas = oas,
811 .coherent_walk = smmu->features & ARM_SMMU_FEAT_COHERENT_WALK,
812 .tlb = smmu_domain->flush_ops,
813 .iommu_dev = smmu->dev,
814 };
815
816 if (smmu->impl && smmu->impl->init_context) {
817 ret = smmu->impl->init_context(smmu_domain, &pgtbl_cfg, dev);
818 if (ret)
819 goto out_clear_smmu;
820 }
821
822 if (smmu_domain->pgtbl_quirks)
823 pgtbl_cfg.quirks |= smmu_domain->pgtbl_quirks;
824
825 pgtbl_ops = alloc_io_pgtable_ops(fmt, &pgtbl_cfg, smmu_domain);
826 if (!pgtbl_ops) {
827 ret = -ENOMEM;
828 goto out_clear_smmu;
829 }
830
831 /* Update the domain's page sizes to reflect the page table format */
832 domain->pgsize_bitmap = pgtbl_cfg.pgsize_bitmap;
833
834 if (pgtbl_cfg.quirks & IO_PGTABLE_QUIRK_ARM_TTBR1) {
835 domain->geometry.aperture_start = ~0UL << ias;
836 domain->geometry.aperture_end = ~0UL;
837 } else {
838 domain->geometry.aperture_end = (1UL << ias) - 1;
839 }
840
841 domain->geometry.force_aperture = true;
842
843 /* Initialise the context bank with our page table cfg */
844 arm_smmu_init_context_bank(smmu_domain, &pgtbl_cfg);
845 arm_smmu_write_context_bank(smmu, cfg->cbndx);
846
847 /*
848 * Request context fault interrupt. Do this last to avoid the
849 * handler seeing a half-initialised domain state.
850 */
851 irq = smmu->irqs[cfg->irptndx];
852
853 if (smmu->impl && smmu->impl->context_fault)
854 context_fault = smmu->impl->context_fault;
855 else
856 context_fault = arm_smmu_context_fault;
857
858 if (smmu->impl && smmu->impl->context_fault_needs_threaded_irq)
859 ret = devm_request_threaded_irq(smmu->dev, irq, NULL,
860 context_fault,
861 IRQF_ONESHOT | IRQF_SHARED,
862 "arm-smmu-context-fault",
863 smmu_domain);
864 else
865 ret = devm_request_irq(smmu->dev, irq, context_fault, IRQF_SHARED,
866 "arm-smmu-context-fault", smmu_domain);
867
868 if (ret < 0) {
869 dev_err(smmu->dev, "failed to request context IRQ %d (%u)\n",
870 cfg->irptndx, irq);
871 cfg->irptndx = ARM_SMMU_INVALID_IRPTNDX;
872 }
873
874 mutex_unlock(&smmu_domain->init_mutex);
875
876 /* Publish page table ops for map/unmap */
877 smmu_domain->pgtbl_ops = pgtbl_ops;
878 return 0;
879
880 out_clear_smmu:
881 __arm_smmu_free_bitmap(smmu->context_map, cfg->cbndx);
882 smmu_domain->smmu = NULL;
883 out_unlock:
884 mutex_unlock(&smmu_domain->init_mutex);
885 return ret;
886 }
887
arm_smmu_destroy_domain_context(struct arm_smmu_domain * smmu_domain)888 static void arm_smmu_destroy_domain_context(struct arm_smmu_domain *smmu_domain)
889 {
890 struct arm_smmu_device *smmu = smmu_domain->smmu;
891 struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
892 int ret, irq;
893
894 if (!smmu)
895 return;
896
897 ret = arm_smmu_rpm_get(smmu);
898 if (ret < 0)
899 return;
900
901 /*
902 * Disable the context bank and free the page tables before freeing
903 * it.
904 */
905 smmu->cbs[cfg->cbndx].cfg = NULL;
906 arm_smmu_write_context_bank(smmu, cfg->cbndx);
907
908 if (cfg->irptndx != ARM_SMMU_INVALID_IRPTNDX) {
909 irq = smmu->irqs[cfg->irptndx];
910 devm_free_irq(smmu->dev, irq, smmu_domain);
911 }
912
913 free_io_pgtable_ops(smmu_domain->pgtbl_ops);
914 __arm_smmu_free_bitmap(smmu->context_map, cfg->cbndx);
915
916 arm_smmu_rpm_put(smmu);
917 }
918
arm_smmu_domain_alloc_paging(struct device * dev)919 static struct iommu_domain *arm_smmu_domain_alloc_paging(struct device *dev)
920 {
921 struct arm_smmu_domain *smmu_domain;
922
923 /*
924 * Allocate the domain and initialise some of its data structures.
925 * We can't really do anything meaningful until we've added a
926 * master.
927 */
928 smmu_domain = kzalloc(sizeof(*smmu_domain), GFP_KERNEL);
929 if (!smmu_domain)
930 return NULL;
931
932 mutex_init(&smmu_domain->init_mutex);
933 spin_lock_init(&smmu_domain->cb_lock);
934
935 return &smmu_domain->domain;
936 }
937
arm_smmu_domain_free(struct iommu_domain * domain)938 static void arm_smmu_domain_free(struct iommu_domain *domain)
939 {
940 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
941
942 /*
943 * Free the domain resources. We assume that all devices have
944 * already been detached.
945 */
946 arm_smmu_destroy_domain_context(smmu_domain);
947 kfree(smmu_domain);
948 }
949
arm_smmu_write_smr(struct arm_smmu_device * smmu,int idx)950 static void arm_smmu_write_smr(struct arm_smmu_device *smmu, int idx)
951 {
952 struct arm_smmu_smr *smr = smmu->smrs + idx;
953 u32 reg = FIELD_PREP(ARM_SMMU_SMR_ID, smr->id) |
954 FIELD_PREP(ARM_SMMU_SMR_MASK, smr->mask);
955
956 if (!(smmu->features & ARM_SMMU_FEAT_EXIDS) && smr->valid)
957 reg |= ARM_SMMU_SMR_VALID;
958 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_SMR(idx), reg);
959 }
960
arm_smmu_write_s2cr(struct arm_smmu_device * smmu,int idx)961 static void arm_smmu_write_s2cr(struct arm_smmu_device *smmu, int idx)
962 {
963 struct arm_smmu_s2cr *s2cr = smmu->s2crs + idx;
964 u32 reg;
965
966 if (smmu->impl && smmu->impl->write_s2cr) {
967 smmu->impl->write_s2cr(smmu, idx);
968 return;
969 }
970
971 reg = FIELD_PREP(ARM_SMMU_S2CR_TYPE, s2cr->type) |
972 FIELD_PREP(ARM_SMMU_S2CR_CBNDX, s2cr->cbndx) |
973 FIELD_PREP(ARM_SMMU_S2CR_PRIVCFG, s2cr->privcfg);
974
975 if (smmu->features & ARM_SMMU_FEAT_EXIDS && smmu->smrs &&
976 smmu->smrs[idx].valid)
977 reg |= ARM_SMMU_S2CR_EXIDVALID;
978 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_S2CR(idx), reg);
979 }
980
arm_smmu_write_sme(struct arm_smmu_device * smmu,int idx)981 static void arm_smmu_write_sme(struct arm_smmu_device *smmu, int idx)
982 {
983 arm_smmu_write_s2cr(smmu, idx);
984 if (smmu->smrs)
985 arm_smmu_write_smr(smmu, idx);
986 }
987
988 /*
989 * The width of SMR's mask field depends on sCR0_EXIDENABLE, so this function
990 * should be called after sCR0 is written.
991 */
arm_smmu_test_smr_masks(struct arm_smmu_device * smmu)992 static void arm_smmu_test_smr_masks(struct arm_smmu_device *smmu)
993 {
994 u32 smr;
995 int i;
996
997 if (!smmu->smrs)
998 return;
999 /*
1000 * If we've had to accommodate firmware memory regions, we may
1001 * have live SMRs by now; tread carefully...
1002 *
1003 * Somewhat perversely, not having a free SMR for this test implies we
1004 * can get away without it anyway, as we'll only be able to 'allocate'
1005 * these SMRs for the ID/mask values we're already trusting to be OK.
1006 */
1007 for (i = 0; i < smmu->num_mapping_groups; i++)
1008 if (!smmu->smrs[i].valid)
1009 goto smr_ok;
1010 return;
1011 smr_ok:
1012 /*
1013 * SMR.ID bits may not be preserved if the corresponding MASK
1014 * bits are set, so check each one separately. We can reject
1015 * masters later if they try to claim IDs outside these masks.
1016 */
1017 smr = FIELD_PREP(ARM_SMMU_SMR_ID, smmu->streamid_mask);
1018 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_SMR(i), smr);
1019 smr = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_SMR(i));
1020 smmu->streamid_mask = FIELD_GET(ARM_SMMU_SMR_ID, smr);
1021
1022 smr = FIELD_PREP(ARM_SMMU_SMR_MASK, smmu->streamid_mask);
1023 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_SMR(i), smr);
1024 smr = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_SMR(i));
1025 smmu->smr_mask_mask = FIELD_GET(ARM_SMMU_SMR_MASK, smr);
1026 }
1027
arm_smmu_find_sme(struct arm_smmu_device * smmu,u16 id,u16 mask)1028 static int arm_smmu_find_sme(struct arm_smmu_device *smmu, u16 id, u16 mask)
1029 {
1030 struct arm_smmu_smr *smrs = smmu->smrs;
1031 int i, free_idx = -ENOSPC;
1032
1033 /* Stream indexing is blissfully easy */
1034 if (!smrs)
1035 return id;
1036
1037 /* Validating SMRs is... less so */
1038 for (i = 0; i < smmu->num_mapping_groups; ++i) {
1039 if (!smrs[i].valid) {
1040 /*
1041 * Note the first free entry we come across, which
1042 * we'll claim in the end if nothing else matches.
1043 */
1044 if (free_idx < 0)
1045 free_idx = i;
1046 continue;
1047 }
1048 /*
1049 * If the new entry is _entirely_ matched by an existing entry,
1050 * then reuse that, with the guarantee that there also cannot
1051 * be any subsequent conflicting entries. In normal use we'd
1052 * expect simply identical entries for this case, but there's
1053 * no harm in accommodating the generalisation.
1054 */
1055 if ((mask & smrs[i].mask) == mask &&
1056 !((id ^ smrs[i].id) & ~smrs[i].mask))
1057 return i;
1058 /*
1059 * If the new entry has any other overlap with an existing one,
1060 * though, then there always exists at least one stream ID
1061 * which would cause a conflict, and we can't allow that risk.
1062 */
1063 if (!((id ^ smrs[i].id) & ~(smrs[i].mask | mask)))
1064 return -EINVAL;
1065 }
1066
1067 return free_idx;
1068 }
1069
arm_smmu_free_sme(struct arm_smmu_device * smmu,int idx)1070 static bool arm_smmu_free_sme(struct arm_smmu_device *smmu, int idx)
1071 {
1072 if (--smmu->s2crs[idx].count)
1073 return false;
1074
1075 smmu->s2crs[idx] = s2cr_init_val;
1076 if (smmu->smrs)
1077 smmu->smrs[idx].valid = false;
1078
1079 return true;
1080 }
1081
arm_smmu_master_alloc_smes(struct device * dev)1082 static int arm_smmu_master_alloc_smes(struct device *dev)
1083 {
1084 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
1085 struct arm_smmu_master_cfg *cfg = dev_iommu_priv_get(dev);
1086 struct arm_smmu_device *smmu = cfg->smmu;
1087 struct arm_smmu_smr *smrs = smmu->smrs;
1088 int i, idx, ret;
1089
1090 mutex_lock(&smmu->stream_map_mutex);
1091 /* Figure out a viable stream map entry allocation */
1092 for_each_cfg_sme(cfg, fwspec, i, idx) {
1093 u16 sid = FIELD_GET(ARM_SMMU_SMR_ID, fwspec->ids[i]);
1094 u16 mask = FIELD_GET(ARM_SMMU_SMR_MASK, fwspec->ids[i]);
1095
1096 if (idx != INVALID_SMENDX) {
1097 ret = -EEXIST;
1098 goto out_err;
1099 }
1100
1101 ret = arm_smmu_find_sme(smmu, sid, mask);
1102 if (ret < 0)
1103 goto out_err;
1104
1105 idx = ret;
1106 if (smrs && smmu->s2crs[idx].count == 0) {
1107 smrs[idx].id = sid;
1108 smrs[idx].mask = mask;
1109 smrs[idx].valid = true;
1110 }
1111 smmu->s2crs[idx].count++;
1112 cfg->smendx[i] = (s16)idx;
1113 }
1114
1115 /* It worked! Now, poke the actual hardware */
1116 for_each_cfg_sme(cfg, fwspec, i, idx)
1117 arm_smmu_write_sme(smmu, idx);
1118
1119 mutex_unlock(&smmu->stream_map_mutex);
1120 return 0;
1121
1122 out_err:
1123 while (i--) {
1124 arm_smmu_free_sme(smmu, cfg->smendx[i]);
1125 cfg->smendx[i] = INVALID_SMENDX;
1126 }
1127 mutex_unlock(&smmu->stream_map_mutex);
1128 return ret;
1129 }
1130
arm_smmu_master_free_smes(struct arm_smmu_master_cfg * cfg,struct iommu_fwspec * fwspec)1131 static void arm_smmu_master_free_smes(struct arm_smmu_master_cfg *cfg,
1132 struct iommu_fwspec *fwspec)
1133 {
1134 struct arm_smmu_device *smmu = cfg->smmu;
1135 int i, idx;
1136
1137 mutex_lock(&smmu->stream_map_mutex);
1138 for_each_cfg_sme(cfg, fwspec, i, idx) {
1139 if (arm_smmu_free_sme(smmu, idx))
1140 arm_smmu_write_sme(smmu, idx);
1141 cfg->smendx[i] = INVALID_SMENDX;
1142 }
1143 mutex_unlock(&smmu->stream_map_mutex);
1144 }
1145
arm_smmu_master_install_s2crs(struct arm_smmu_master_cfg * cfg,enum arm_smmu_s2cr_type type,u8 cbndx,struct iommu_fwspec * fwspec)1146 static void arm_smmu_master_install_s2crs(struct arm_smmu_master_cfg *cfg,
1147 enum arm_smmu_s2cr_type type,
1148 u8 cbndx, struct iommu_fwspec *fwspec)
1149 {
1150 struct arm_smmu_device *smmu = cfg->smmu;
1151 struct arm_smmu_s2cr *s2cr = smmu->s2crs;
1152 int i, idx;
1153
1154 for_each_cfg_sme(cfg, fwspec, i, idx) {
1155 if (type == s2cr[idx].type && cbndx == s2cr[idx].cbndx)
1156 continue;
1157
1158 s2cr[idx].type = type;
1159 s2cr[idx].privcfg = S2CR_PRIVCFG_DEFAULT;
1160 s2cr[idx].cbndx = cbndx;
1161 arm_smmu_write_s2cr(smmu, idx);
1162 }
1163 }
1164
arm_smmu_attach_dev(struct iommu_domain * domain,struct device * dev)1165 static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev)
1166 {
1167 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1168 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
1169 struct arm_smmu_master_cfg *cfg;
1170 struct arm_smmu_device *smmu;
1171 int ret;
1172
1173 /*
1174 * FIXME: The arch/arm DMA API code tries to attach devices to its own
1175 * domains between of_xlate() and probe_device() - we have no way to cope
1176 * with that, so until ARM gets converted to rely on groups and default
1177 * domains, just say no (but more politely than by dereferencing NULL).
1178 * This should be at least a WARN_ON once that's sorted.
1179 */
1180 cfg = dev_iommu_priv_get(dev);
1181 if (!cfg)
1182 return -ENODEV;
1183
1184 smmu = cfg->smmu;
1185
1186 ret = arm_smmu_rpm_get(smmu);
1187 if (ret < 0)
1188 return ret;
1189
1190 /* Ensure that the domain is finalised */
1191 ret = arm_smmu_init_domain_context(smmu_domain, smmu, dev);
1192 if (ret < 0)
1193 goto rpm_put;
1194
1195 /*
1196 * Sanity check the domain. We don't support domains across
1197 * different SMMUs.
1198 */
1199 if (smmu_domain->smmu != smmu) {
1200 ret = -EINVAL;
1201 goto rpm_put;
1202 }
1203
1204 /* Looks ok, so add the device to the domain */
1205 arm_smmu_master_install_s2crs(cfg, S2CR_TYPE_TRANS,
1206 smmu_domain->cfg.cbndx, fwspec);
1207 rpm_put:
1208 arm_smmu_rpm_put(smmu);
1209 return ret;
1210 }
1211
arm_smmu_attach_dev_type(struct device * dev,enum arm_smmu_s2cr_type type)1212 static int arm_smmu_attach_dev_type(struct device *dev,
1213 enum arm_smmu_s2cr_type type)
1214 {
1215 struct arm_smmu_master_cfg *cfg = dev_iommu_priv_get(dev);
1216 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
1217 struct arm_smmu_device *smmu;
1218 int ret;
1219
1220 if (!cfg)
1221 return -ENODEV;
1222 smmu = cfg->smmu;
1223
1224 ret = arm_smmu_rpm_get(smmu);
1225 if (ret < 0)
1226 return ret;
1227
1228 arm_smmu_master_install_s2crs(cfg, type, 0, fwspec);
1229 arm_smmu_rpm_put(smmu);
1230 return 0;
1231 }
1232
arm_smmu_attach_dev_identity(struct iommu_domain * domain,struct device * dev)1233 static int arm_smmu_attach_dev_identity(struct iommu_domain *domain,
1234 struct device *dev)
1235 {
1236 return arm_smmu_attach_dev_type(dev, S2CR_TYPE_BYPASS);
1237 }
1238
1239 static const struct iommu_domain_ops arm_smmu_identity_ops = {
1240 .attach_dev = arm_smmu_attach_dev_identity,
1241 };
1242
1243 static struct iommu_domain arm_smmu_identity_domain = {
1244 .type = IOMMU_DOMAIN_IDENTITY,
1245 .ops = &arm_smmu_identity_ops,
1246 };
1247
arm_smmu_attach_dev_blocked(struct iommu_domain * domain,struct device * dev)1248 static int arm_smmu_attach_dev_blocked(struct iommu_domain *domain,
1249 struct device *dev)
1250 {
1251 return arm_smmu_attach_dev_type(dev, S2CR_TYPE_FAULT);
1252 }
1253
1254 static const struct iommu_domain_ops arm_smmu_blocked_ops = {
1255 .attach_dev = arm_smmu_attach_dev_blocked,
1256 };
1257
1258 static struct iommu_domain arm_smmu_blocked_domain = {
1259 .type = IOMMU_DOMAIN_BLOCKED,
1260 .ops = &arm_smmu_blocked_ops,
1261 };
1262
arm_smmu_map_pages(struct iommu_domain * domain,unsigned long iova,phys_addr_t paddr,size_t pgsize,size_t pgcount,int prot,gfp_t gfp,size_t * mapped)1263 static int arm_smmu_map_pages(struct iommu_domain *domain, unsigned long iova,
1264 phys_addr_t paddr, size_t pgsize, size_t pgcount,
1265 int prot, gfp_t gfp, size_t *mapped)
1266 {
1267 struct io_pgtable_ops *ops = to_smmu_domain(domain)->pgtbl_ops;
1268 struct arm_smmu_device *smmu = to_smmu_domain(domain)->smmu;
1269 int ret;
1270
1271 if (!ops)
1272 return -ENODEV;
1273
1274 arm_smmu_rpm_get(smmu);
1275 ret = ops->map_pages(ops, iova, paddr, pgsize, pgcount, prot, gfp, mapped);
1276 arm_smmu_rpm_put(smmu);
1277
1278 return ret;
1279 }
1280
arm_smmu_unmap_pages(struct iommu_domain * domain,unsigned long iova,size_t pgsize,size_t pgcount,struct iommu_iotlb_gather * iotlb_gather)1281 static size_t arm_smmu_unmap_pages(struct iommu_domain *domain, unsigned long iova,
1282 size_t pgsize, size_t pgcount,
1283 struct iommu_iotlb_gather *iotlb_gather)
1284 {
1285 struct io_pgtable_ops *ops = to_smmu_domain(domain)->pgtbl_ops;
1286 struct arm_smmu_device *smmu = to_smmu_domain(domain)->smmu;
1287 size_t ret;
1288
1289 if (!ops)
1290 return 0;
1291
1292 arm_smmu_rpm_get(smmu);
1293 ret = ops->unmap_pages(ops, iova, pgsize, pgcount, iotlb_gather);
1294 arm_smmu_rpm_put(smmu);
1295
1296 return ret;
1297 }
1298
arm_smmu_flush_iotlb_all(struct iommu_domain * domain)1299 static void arm_smmu_flush_iotlb_all(struct iommu_domain *domain)
1300 {
1301 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1302 struct arm_smmu_device *smmu = smmu_domain->smmu;
1303
1304 if (smmu_domain->flush_ops) {
1305 arm_smmu_rpm_get(smmu);
1306 smmu_domain->flush_ops->tlb_flush_all(smmu_domain);
1307 arm_smmu_rpm_put(smmu);
1308 }
1309 }
1310
arm_smmu_iotlb_sync(struct iommu_domain * domain,struct iommu_iotlb_gather * gather)1311 static void arm_smmu_iotlb_sync(struct iommu_domain *domain,
1312 struct iommu_iotlb_gather *gather)
1313 {
1314 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1315 struct arm_smmu_device *smmu = smmu_domain->smmu;
1316
1317 if (!smmu)
1318 return;
1319
1320 arm_smmu_rpm_get(smmu);
1321 if (smmu->version == ARM_SMMU_V2 ||
1322 smmu_domain->stage == ARM_SMMU_DOMAIN_S1)
1323 arm_smmu_tlb_sync_context(smmu_domain);
1324 else
1325 arm_smmu_tlb_sync_global(smmu);
1326 arm_smmu_rpm_put(smmu);
1327 }
1328
arm_smmu_iova_to_phys_hard(struct iommu_domain * domain,dma_addr_t iova)1329 static phys_addr_t arm_smmu_iova_to_phys_hard(struct iommu_domain *domain,
1330 dma_addr_t iova)
1331 {
1332 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1333 struct arm_smmu_device *smmu = smmu_domain->smmu;
1334 struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
1335 struct io_pgtable_ops *ops= smmu_domain->pgtbl_ops;
1336 struct device *dev = smmu->dev;
1337 void __iomem *reg;
1338 u32 tmp;
1339 u64 phys;
1340 unsigned long va, flags;
1341 int ret, idx = cfg->cbndx;
1342 phys_addr_t addr = 0;
1343
1344 ret = arm_smmu_rpm_get(smmu);
1345 if (ret < 0)
1346 return 0;
1347
1348 spin_lock_irqsave(&smmu_domain->cb_lock, flags);
1349 va = iova & ~0xfffUL;
1350 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64)
1351 arm_smmu_cb_writeq(smmu, idx, ARM_SMMU_CB_ATS1PR, va);
1352 else
1353 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_ATS1PR, va);
1354
1355 reg = arm_smmu_page(smmu, ARM_SMMU_CB(smmu, idx)) + ARM_SMMU_CB_ATSR;
1356 if (readl_poll_timeout_atomic(reg, tmp, !(tmp & ARM_SMMU_CB_ATSR_ACTIVE),
1357 5, 50)) {
1358 spin_unlock_irqrestore(&smmu_domain->cb_lock, flags);
1359 dev_err(dev,
1360 "iova to phys timed out on %pad. Falling back to software table walk.\n",
1361 &iova);
1362 arm_smmu_rpm_put(smmu);
1363 return ops->iova_to_phys(ops, iova);
1364 }
1365
1366 phys = arm_smmu_cb_readq(smmu, idx, ARM_SMMU_CB_PAR);
1367 spin_unlock_irqrestore(&smmu_domain->cb_lock, flags);
1368 if (phys & ARM_SMMU_CB_PAR_F) {
1369 dev_err(dev, "translation fault!\n");
1370 dev_err(dev, "PAR = 0x%llx\n", phys);
1371 goto out;
1372 }
1373
1374 addr = (phys & GENMASK_ULL(39, 12)) | (iova & 0xfff);
1375 out:
1376 arm_smmu_rpm_put(smmu);
1377
1378 return addr;
1379 }
1380
arm_smmu_iova_to_phys(struct iommu_domain * domain,dma_addr_t iova)1381 static phys_addr_t arm_smmu_iova_to_phys(struct iommu_domain *domain,
1382 dma_addr_t iova)
1383 {
1384 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1385 struct io_pgtable_ops *ops = smmu_domain->pgtbl_ops;
1386
1387 if (!ops)
1388 return 0;
1389
1390 if (smmu_domain->smmu->features & ARM_SMMU_FEAT_TRANS_OPS &&
1391 smmu_domain->stage == ARM_SMMU_DOMAIN_S1)
1392 return arm_smmu_iova_to_phys_hard(domain, iova);
1393
1394 return ops->iova_to_phys(ops, iova);
1395 }
1396
arm_smmu_capable(struct device * dev,enum iommu_cap cap)1397 static bool arm_smmu_capable(struct device *dev, enum iommu_cap cap)
1398 {
1399 struct arm_smmu_master_cfg *cfg = dev_iommu_priv_get(dev);
1400
1401 switch (cap) {
1402 case IOMMU_CAP_CACHE_COHERENCY:
1403 /*
1404 * It's overwhelmingly the case in practice that when the pagetable
1405 * walk interface is connected to a coherent interconnect, all the
1406 * translation interfaces are too. Furthermore if the device is
1407 * natively coherent, then its translation interface must also be.
1408 */
1409 return cfg->smmu->features & ARM_SMMU_FEAT_COHERENT_WALK ||
1410 device_get_dma_attr(dev) == DEV_DMA_COHERENT;
1411 case IOMMU_CAP_NOEXEC:
1412 case IOMMU_CAP_DEFERRED_FLUSH:
1413 return true;
1414 default:
1415 return false;
1416 }
1417 }
1418
1419 static
arm_smmu_get_by_fwnode(struct fwnode_handle * fwnode)1420 struct arm_smmu_device *arm_smmu_get_by_fwnode(struct fwnode_handle *fwnode)
1421 {
1422 struct device *dev = bus_find_device_by_fwnode(&platform_bus_type, fwnode);
1423
1424 put_device(dev);
1425 return dev ? dev_get_drvdata(dev) : NULL;
1426 }
1427
arm_smmu_probe_device(struct device * dev)1428 static struct iommu_device *arm_smmu_probe_device(struct device *dev)
1429 {
1430 struct arm_smmu_device *smmu = NULL;
1431 struct arm_smmu_master_cfg *cfg;
1432 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
1433 int i, ret;
1434
1435 if (using_legacy_binding) {
1436 ret = arm_smmu_register_legacy_master(dev, &smmu);
1437
1438 /*
1439 * If dev->iommu_fwspec is initally NULL, arm_smmu_register_legacy_master()
1440 * will allocate/initialise a new one. Thus we need to update fwspec for
1441 * later use.
1442 */
1443 fwspec = dev_iommu_fwspec_get(dev);
1444 if (ret)
1445 goto out_free;
1446 } else {
1447 smmu = arm_smmu_get_by_fwnode(fwspec->iommu_fwnode);
1448 }
1449
1450 ret = -EINVAL;
1451 for (i = 0; i < fwspec->num_ids; i++) {
1452 u16 sid = FIELD_GET(ARM_SMMU_SMR_ID, fwspec->ids[i]);
1453 u16 mask = FIELD_GET(ARM_SMMU_SMR_MASK, fwspec->ids[i]);
1454
1455 if (sid & ~smmu->streamid_mask) {
1456 dev_err(dev, "stream ID 0x%x out of range for SMMU (0x%x)\n",
1457 sid, smmu->streamid_mask);
1458 goto out_free;
1459 }
1460 if (mask & ~smmu->smr_mask_mask) {
1461 dev_err(dev, "SMR mask 0x%x out of range for SMMU (0x%x)\n",
1462 mask, smmu->smr_mask_mask);
1463 goto out_free;
1464 }
1465 }
1466
1467 ret = -ENOMEM;
1468 cfg = kzalloc(offsetof(struct arm_smmu_master_cfg, smendx[i]),
1469 GFP_KERNEL);
1470 if (!cfg)
1471 goto out_free;
1472
1473 cfg->smmu = smmu;
1474 dev_iommu_priv_set(dev, cfg);
1475 while (i--)
1476 cfg->smendx[i] = INVALID_SMENDX;
1477
1478 ret = arm_smmu_rpm_get(smmu);
1479 if (ret < 0)
1480 goto out_cfg_free;
1481
1482 ret = arm_smmu_master_alloc_smes(dev);
1483 arm_smmu_rpm_put(smmu);
1484
1485 if (ret)
1486 goto out_cfg_free;
1487
1488 device_link_add(dev, smmu->dev,
1489 DL_FLAG_PM_RUNTIME | DL_FLAG_AUTOREMOVE_SUPPLIER);
1490
1491 return &smmu->iommu;
1492
1493 out_cfg_free:
1494 kfree(cfg);
1495 out_free:
1496 return ERR_PTR(ret);
1497 }
1498
arm_smmu_release_device(struct device * dev)1499 static void arm_smmu_release_device(struct device *dev)
1500 {
1501 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
1502 struct arm_smmu_master_cfg *cfg = dev_iommu_priv_get(dev);
1503 int ret;
1504
1505 ret = arm_smmu_rpm_get(cfg->smmu);
1506 if (ret < 0)
1507 return;
1508
1509 arm_smmu_master_free_smes(cfg, fwspec);
1510
1511 arm_smmu_rpm_put(cfg->smmu);
1512
1513 kfree(cfg);
1514 }
1515
arm_smmu_probe_finalize(struct device * dev)1516 static void arm_smmu_probe_finalize(struct device *dev)
1517 {
1518 struct arm_smmu_master_cfg *cfg;
1519 struct arm_smmu_device *smmu;
1520
1521 cfg = dev_iommu_priv_get(dev);
1522 smmu = cfg->smmu;
1523
1524 if (smmu->impl && smmu->impl->probe_finalize)
1525 smmu->impl->probe_finalize(smmu, dev);
1526 }
1527
arm_smmu_device_group(struct device * dev)1528 static struct iommu_group *arm_smmu_device_group(struct device *dev)
1529 {
1530 struct arm_smmu_master_cfg *cfg = dev_iommu_priv_get(dev);
1531 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
1532 struct arm_smmu_device *smmu = cfg->smmu;
1533 struct iommu_group *group = NULL;
1534 int i, idx;
1535
1536 mutex_lock(&smmu->stream_map_mutex);
1537 for_each_cfg_sme(cfg, fwspec, i, idx) {
1538 if (group && smmu->s2crs[idx].group &&
1539 group != smmu->s2crs[idx].group) {
1540 mutex_unlock(&smmu->stream_map_mutex);
1541 return ERR_PTR(-EINVAL);
1542 }
1543
1544 group = smmu->s2crs[idx].group;
1545 }
1546
1547 if (group) {
1548 mutex_unlock(&smmu->stream_map_mutex);
1549 return iommu_group_ref_get(group);
1550 }
1551
1552 if (dev_is_pci(dev))
1553 group = pci_device_group(dev);
1554 else if (dev_is_fsl_mc(dev))
1555 group = fsl_mc_device_group(dev);
1556 else
1557 group = generic_device_group(dev);
1558
1559 /* Remember group for faster lookups */
1560 if (!IS_ERR(group))
1561 for_each_cfg_sme(cfg, fwspec, i, idx)
1562 smmu->s2crs[idx].group = group;
1563
1564 mutex_unlock(&smmu->stream_map_mutex);
1565 return group;
1566 }
1567
arm_smmu_set_pgtable_quirks(struct iommu_domain * domain,unsigned long quirks)1568 static int arm_smmu_set_pgtable_quirks(struct iommu_domain *domain,
1569 unsigned long quirks)
1570 {
1571 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1572 int ret = 0;
1573
1574 mutex_lock(&smmu_domain->init_mutex);
1575 if (smmu_domain->smmu)
1576 ret = -EPERM;
1577 else
1578 smmu_domain->pgtbl_quirks = quirks;
1579 mutex_unlock(&smmu_domain->init_mutex);
1580
1581 return ret;
1582 }
1583
arm_smmu_of_xlate(struct device * dev,const struct of_phandle_args * args)1584 static int arm_smmu_of_xlate(struct device *dev,
1585 const struct of_phandle_args *args)
1586 {
1587 u32 mask, fwid = 0;
1588
1589 if (args->args_count > 0)
1590 fwid |= FIELD_PREP(ARM_SMMU_SMR_ID, args->args[0]);
1591
1592 if (args->args_count > 1)
1593 fwid |= FIELD_PREP(ARM_SMMU_SMR_MASK, args->args[1]);
1594 else if (!of_property_read_u32(args->np, "stream-match-mask", &mask))
1595 fwid |= FIELD_PREP(ARM_SMMU_SMR_MASK, mask);
1596
1597 return iommu_fwspec_add_ids(dev, &fwid, 1);
1598 }
1599
arm_smmu_get_resv_regions(struct device * dev,struct list_head * head)1600 static void arm_smmu_get_resv_regions(struct device *dev,
1601 struct list_head *head)
1602 {
1603 struct iommu_resv_region *region;
1604 int prot = IOMMU_WRITE | IOMMU_NOEXEC | IOMMU_MMIO;
1605
1606 region = iommu_alloc_resv_region(MSI_IOVA_BASE, MSI_IOVA_LENGTH,
1607 prot, IOMMU_RESV_SW_MSI, GFP_KERNEL);
1608 if (!region)
1609 return;
1610
1611 list_add_tail(®ion->list, head);
1612
1613 iommu_dma_get_resv_regions(dev, head);
1614 }
1615
arm_smmu_def_domain_type(struct device * dev)1616 static int arm_smmu_def_domain_type(struct device *dev)
1617 {
1618 struct arm_smmu_master_cfg *cfg = dev_iommu_priv_get(dev);
1619 const struct arm_smmu_impl *impl = cfg->smmu->impl;
1620
1621 if (using_legacy_binding)
1622 return IOMMU_DOMAIN_IDENTITY;
1623
1624 if (impl && impl->def_domain_type)
1625 return impl->def_domain_type(dev);
1626
1627 return 0;
1628 }
1629
1630 static struct iommu_ops arm_smmu_ops = {
1631 .identity_domain = &arm_smmu_identity_domain,
1632 .blocked_domain = &arm_smmu_blocked_domain,
1633 .capable = arm_smmu_capable,
1634 .domain_alloc_paging = arm_smmu_domain_alloc_paging,
1635 .probe_device = arm_smmu_probe_device,
1636 .release_device = arm_smmu_release_device,
1637 .probe_finalize = arm_smmu_probe_finalize,
1638 .device_group = arm_smmu_device_group,
1639 .of_xlate = arm_smmu_of_xlate,
1640 .get_resv_regions = arm_smmu_get_resv_regions,
1641 .def_domain_type = arm_smmu_def_domain_type,
1642 .pgsize_bitmap = -1UL, /* Restricted during device attach */
1643 .owner = THIS_MODULE,
1644 .default_domain_ops = &(const struct iommu_domain_ops) {
1645 .attach_dev = arm_smmu_attach_dev,
1646 .map_pages = arm_smmu_map_pages,
1647 .unmap_pages = arm_smmu_unmap_pages,
1648 .flush_iotlb_all = arm_smmu_flush_iotlb_all,
1649 .iotlb_sync = arm_smmu_iotlb_sync,
1650 .iova_to_phys = arm_smmu_iova_to_phys,
1651 .set_pgtable_quirks = arm_smmu_set_pgtable_quirks,
1652 .free = arm_smmu_domain_free,
1653 }
1654 };
1655
arm_smmu_device_reset(struct arm_smmu_device * smmu)1656 static void arm_smmu_device_reset(struct arm_smmu_device *smmu)
1657 {
1658 int i;
1659 u32 reg;
1660
1661 /* clear global FSR */
1662 reg = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sGFSR);
1663 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_sGFSR, reg);
1664
1665 /*
1666 * Reset stream mapping groups: Initial values mark all SMRn as
1667 * invalid and all S2CRn as bypass unless overridden.
1668 */
1669 for (i = 0; i < smmu->num_mapping_groups; ++i)
1670 arm_smmu_write_sme(smmu, i);
1671
1672 /* Make sure all context banks are disabled and clear CB_FSR */
1673 for (i = 0; i < smmu->num_context_banks; ++i) {
1674 arm_smmu_write_context_bank(smmu, i);
1675 arm_smmu_cb_write(smmu, i, ARM_SMMU_CB_FSR, ARM_SMMU_CB_FSR_FAULT);
1676 }
1677
1678 /* Invalidate the TLB, just in case */
1679 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_TLBIALLH, QCOM_DUMMY_VAL);
1680 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_TLBIALLNSNH, QCOM_DUMMY_VAL);
1681
1682 reg = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sCR0);
1683
1684 /* Enable fault reporting */
1685 reg |= (ARM_SMMU_sCR0_GFRE | ARM_SMMU_sCR0_GFIE |
1686 ARM_SMMU_sCR0_GCFGFRE | ARM_SMMU_sCR0_GCFGFIE);
1687
1688 /* Disable TLB broadcasting. */
1689 reg |= (ARM_SMMU_sCR0_VMIDPNE | ARM_SMMU_sCR0_PTM);
1690
1691 /* Enable client access, handling unmatched streams as appropriate */
1692 reg &= ~ARM_SMMU_sCR0_CLIENTPD;
1693 if (disable_bypass)
1694 reg |= ARM_SMMU_sCR0_USFCFG;
1695 else
1696 reg &= ~ARM_SMMU_sCR0_USFCFG;
1697
1698 /* Disable forced broadcasting */
1699 reg &= ~ARM_SMMU_sCR0_FB;
1700
1701 /* Don't upgrade barriers */
1702 reg &= ~(ARM_SMMU_sCR0_BSU);
1703
1704 if (smmu->features & ARM_SMMU_FEAT_VMID16)
1705 reg |= ARM_SMMU_sCR0_VMID16EN;
1706
1707 if (smmu->features & ARM_SMMU_FEAT_EXIDS)
1708 reg |= ARM_SMMU_sCR0_EXIDENABLE;
1709
1710 if (smmu->impl && smmu->impl->reset)
1711 smmu->impl->reset(smmu);
1712
1713 /* Push the button */
1714 arm_smmu_tlb_sync_global(smmu);
1715 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_sCR0, reg);
1716 }
1717
arm_smmu_id_size_to_bits(int size)1718 static int arm_smmu_id_size_to_bits(int size)
1719 {
1720 switch (size) {
1721 case 0:
1722 return 32;
1723 case 1:
1724 return 36;
1725 case 2:
1726 return 40;
1727 case 3:
1728 return 42;
1729 case 4:
1730 return 44;
1731 case 5:
1732 default:
1733 return 48;
1734 }
1735 }
1736
arm_smmu_device_cfg_probe(struct arm_smmu_device * smmu)1737 static int arm_smmu_device_cfg_probe(struct arm_smmu_device *smmu)
1738 {
1739 unsigned int size;
1740 u32 id;
1741 bool cttw_reg, cttw_fw = smmu->features & ARM_SMMU_FEAT_COHERENT_WALK;
1742 int i, ret;
1743
1744 dev_notice(smmu->dev, "probing hardware configuration...\n");
1745 dev_notice(smmu->dev, "SMMUv%d with:\n",
1746 smmu->version == ARM_SMMU_V2 ? 2 : 1);
1747
1748 /* ID0 */
1749 id = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_ID0);
1750
1751 /* Restrict available stages based on module parameter */
1752 if (force_stage == 1)
1753 id &= ~(ARM_SMMU_ID0_S2TS | ARM_SMMU_ID0_NTS);
1754 else if (force_stage == 2)
1755 id &= ~(ARM_SMMU_ID0_S1TS | ARM_SMMU_ID0_NTS);
1756
1757 if (id & ARM_SMMU_ID0_S1TS) {
1758 smmu->features |= ARM_SMMU_FEAT_TRANS_S1;
1759 dev_notice(smmu->dev, "\tstage 1 translation\n");
1760 }
1761
1762 if (id & ARM_SMMU_ID0_S2TS) {
1763 smmu->features |= ARM_SMMU_FEAT_TRANS_S2;
1764 dev_notice(smmu->dev, "\tstage 2 translation\n");
1765 }
1766
1767 if (id & ARM_SMMU_ID0_NTS) {
1768 smmu->features |= ARM_SMMU_FEAT_TRANS_NESTED;
1769 dev_notice(smmu->dev, "\tnested translation\n");
1770 }
1771
1772 if (!(smmu->features &
1773 (ARM_SMMU_FEAT_TRANS_S1 | ARM_SMMU_FEAT_TRANS_S2))) {
1774 dev_err(smmu->dev, "\tno translation support!\n");
1775 return -ENODEV;
1776 }
1777
1778 if ((id & ARM_SMMU_ID0_S1TS) &&
1779 ((smmu->version < ARM_SMMU_V2) || !(id & ARM_SMMU_ID0_ATOSNS))) {
1780 smmu->features |= ARM_SMMU_FEAT_TRANS_OPS;
1781 dev_notice(smmu->dev, "\taddress translation ops\n");
1782 }
1783
1784 /*
1785 * In order for DMA API calls to work properly, we must defer to what
1786 * the FW says about coherency, regardless of what the hardware claims.
1787 * Fortunately, this also opens up a workaround for systems where the
1788 * ID register value has ended up configured incorrectly.
1789 */
1790 cttw_reg = !!(id & ARM_SMMU_ID0_CTTW);
1791 if (cttw_fw || cttw_reg)
1792 dev_notice(smmu->dev, "\t%scoherent table walk\n",
1793 cttw_fw ? "" : "non-");
1794 if (cttw_fw != cttw_reg)
1795 dev_notice(smmu->dev,
1796 "\t(IDR0.CTTW overridden by FW configuration)\n");
1797
1798 /* Max. number of entries we have for stream matching/indexing */
1799 if (smmu->version == ARM_SMMU_V2 && id & ARM_SMMU_ID0_EXIDS) {
1800 smmu->features |= ARM_SMMU_FEAT_EXIDS;
1801 size = 1 << 16;
1802 } else {
1803 size = 1 << FIELD_GET(ARM_SMMU_ID0_NUMSIDB, id);
1804 }
1805 smmu->streamid_mask = size - 1;
1806 if (id & ARM_SMMU_ID0_SMS) {
1807 smmu->features |= ARM_SMMU_FEAT_STREAM_MATCH;
1808 size = FIELD_GET(ARM_SMMU_ID0_NUMSMRG, id);
1809 if (size == 0) {
1810 dev_err(smmu->dev,
1811 "stream-matching supported, but no SMRs present!\n");
1812 return -ENODEV;
1813 }
1814
1815 /* Zero-initialised to mark as invalid */
1816 smmu->smrs = devm_kcalloc(smmu->dev, size, sizeof(*smmu->smrs),
1817 GFP_KERNEL);
1818 if (!smmu->smrs)
1819 return -ENOMEM;
1820
1821 dev_notice(smmu->dev,
1822 "\tstream matching with %u register groups", size);
1823 }
1824 /* s2cr->type == 0 means translation, so initialise explicitly */
1825 smmu->s2crs = devm_kmalloc_array(smmu->dev, size, sizeof(*smmu->s2crs),
1826 GFP_KERNEL);
1827 if (!smmu->s2crs)
1828 return -ENOMEM;
1829 for (i = 0; i < size; i++)
1830 smmu->s2crs[i] = s2cr_init_val;
1831
1832 smmu->num_mapping_groups = size;
1833 mutex_init(&smmu->stream_map_mutex);
1834 spin_lock_init(&smmu->global_sync_lock);
1835
1836 if (smmu->version < ARM_SMMU_V2 ||
1837 !(id & ARM_SMMU_ID0_PTFS_NO_AARCH32)) {
1838 smmu->features |= ARM_SMMU_FEAT_FMT_AARCH32_L;
1839 if (!(id & ARM_SMMU_ID0_PTFS_NO_AARCH32S))
1840 smmu->features |= ARM_SMMU_FEAT_FMT_AARCH32_S;
1841 }
1842
1843 /* ID1 */
1844 id = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_ID1);
1845 smmu->pgshift = (id & ARM_SMMU_ID1_PAGESIZE) ? 16 : 12;
1846
1847 /* Check for size mismatch of SMMU address space from mapped region */
1848 size = 1 << (FIELD_GET(ARM_SMMU_ID1_NUMPAGENDXB, id) + 1);
1849 if (smmu->numpage != 2 * size << smmu->pgshift)
1850 dev_warn(smmu->dev,
1851 "SMMU address space size (0x%x) differs from mapped region size (0x%x)!\n",
1852 2 * size << smmu->pgshift, smmu->numpage);
1853 /* Now properly encode NUMPAGE to subsequently derive SMMU_CB_BASE */
1854 smmu->numpage = size;
1855
1856 smmu->num_s2_context_banks = FIELD_GET(ARM_SMMU_ID1_NUMS2CB, id);
1857 smmu->num_context_banks = FIELD_GET(ARM_SMMU_ID1_NUMCB, id);
1858 if (smmu->num_s2_context_banks > smmu->num_context_banks) {
1859 dev_err(smmu->dev, "impossible number of S2 context banks!\n");
1860 return -ENODEV;
1861 }
1862 dev_notice(smmu->dev, "\t%u context banks (%u stage-2 only)\n",
1863 smmu->num_context_banks, smmu->num_s2_context_banks);
1864 smmu->cbs = devm_kcalloc(smmu->dev, smmu->num_context_banks,
1865 sizeof(*smmu->cbs), GFP_KERNEL);
1866 if (!smmu->cbs)
1867 return -ENOMEM;
1868
1869 /* ID2 */
1870 id = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_ID2);
1871 size = arm_smmu_id_size_to_bits(FIELD_GET(ARM_SMMU_ID2_IAS, id));
1872 smmu->ipa_size = size;
1873
1874 /* The output mask is also applied for bypass */
1875 size = arm_smmu_id_size_to_bits(FIELD_GET(ARM_SMMU_ID2_OAS, id));
1876 smmu->pa_size = size;
1877
1878 if (id & ARM_SMMU_ID2_VMID16)
1879 smmu->features |= ARM_SMMU_FEAT_VMID16;
1880
1881 /*
1882 * What the page table walker can address actually depends on which
1883 * descriptor format is in use, but since a) we don't know that yet,
1884 * and b) it can vary per context bank, this will have to do...
1885 */
1886 if (dma_set_mask_and_coherent(smmu->dev, DMA_BIT_MASK(size)))
1887 dev_warn(smmu->dev,
1888 "failed to set DMA mask for table walker\n");
1889
1890 if (smmu->version < ARM_SMMU_V2) {
1891 smmu->va_size = smmu->ipa_size;
1892 if (smmu->version == ARM_SMMU_V1_64K)
1893 smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_64K;
1894 } else {
1895 size = FIELD_GET(ARM_SMMU_ID2_UBS, id);
1896 smmu->va_size = arm_smmu_id_size_to_bits(size);
1897 if (id & ARM_SMMU_ID2_PTFS_4K)
1898 smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_4K;
1899 if (id & ARM_SMMU_ID2_PTFS_16K)
1900 smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_16K;
1901 if (id & ARM_SMMU_ID2_PTFS_64K)
1902 smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_64K;
1903 }
1904
1905 if (smmu->impl && smmu->impl->cfg_probe) {
1906 ret = smmu->impl->cfg_probe(smmu);
1907 if (ret)
1908 return ret;
1909 }
1910
1911 /* Now we've corralled the various formats, what'll it do? */
1912 if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH32_S)
1913 smmu->pgsize_bitmap |= SZ_4K | SZ_64K | SZ_1M | SZ_16M;
1914 if (smmu->features &
1915 (ARM_SMMU_FEAT_FMT_AARCH32_L | ARM_SMMU_FEAT_FMT_AARCH64_4K))
1916 smmu->pgsize_bitmap |= SZ_4K | SZ_2M | SZ_1G;
1917 if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH64_16K)
1918 smmu->pgsize_bitmap |= SZ_16K | SZ_32M;
1919 if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH64_64K)
1920 smmu->pgsize_bitmap |= SZ_64K | SZ_512M;
1921
1922 if (arm_smmu_ops.pgsize_bitmap == -1UL)
1923 arm_smmu_ops.pgsize_bitmap = smmu->pgsize_bitmap;
1924 else
1925 arm_smmu_ops.pgsize_bitmap |= smmu->pgsize_bitmap;
1926 dev_notice(smmu->dev, "\tSupported page sizes: 0x%08lx\n",
1927 smmu->pgsize_bitmap);
1928
1929
1930 if (smmu->features & ARM_SMMU_FEAT_TRANS_S1)
1931 dev_notice(smmu->dev, "\tStage-1: %lu-bit VA -> %lu-bit IPA\n",
1932 smmu->va_size, smmu->ipa_size);
1933
1934 if (smmu->features & ARM_SMMU_FEAT_TRANS_S2)
1935 dev_notice(smmu->dev, "\tStage-2: %lu-bit IPA -> %lu-bit PA\n",
1936 smmu->ipa_size, smmu->pa_size);
1937
1938 return 0;
1939 }
1940
1941 struct arm_smmu_match_data {
1942 enum arm_smmu_arch_version version;
1943 enum arm_smmu_implementation model;
1944 };
1945
1946 #define ARM_SMMU_MATCH_DATA(name, ver, imp) \
1947 static const struct arm_smmu_match_data name = { .version = ver, .model = imp }
1948
1949 ARM_SMMU_MATCH_DATA(smmu_generic_v1, ARM_SMMU_V1, GENERIC_SMMU);
1950 ARM_SMMU_MATCH_DATA(smmu_generic_v2, ARM_SMMU_V2, GENERIC_SMMU);
1951 ARM_SMMU_MATCH_DATA(arm_mmu401, ARM_SMMU_V1_64K, GENERIC_SMMU);
1952 ARM_SMMU_MATCH_DATA(arm_mmu500, ARM_SMMU_V2, ARM_MMU500);
1953 ARM_SMMU_MATCH_DATA(cavium_smmuv2, ARM_SMMU_V2, CAVIUM_SMMUV2);
1954 ARM_SMMU_MATCH_DATA(qcom_smmuv2, ARM_SMMU_V2, QCOM_SMMUV2);
1955
1956 static const struct of_device_id arm_smmu_of_match[] = {
1957 { .compatible = "arm,smmu-v1", .data = &smmu_generic_v1 },
1958 { .compatible = "arm,smmu-v2", .data = &smmu_generic_v2 },
1959 { .compatible = "arm,mmu-400", .data = &smmu_generic_v1 },
1960 { .compatible = "arm,mmu-401", .data = &arm_mmu401 },
1961 { .compatible = "arm,mmu-500", .data = &arm_mmu500 },
1962 { .compatible = "cavium,smmu-v2", .data = &cavium_smmuv2 },
1963 { .compatible = "nvidia,smmu-500", .data = &arm_mmu500 },
1964 { .compatible = "qcom,smmu-v2", .data = &qcom_smmuv2 },
1965 { },
1966 };
1967 MODULE_DEVICE_TABLE(of, arm_smmu_of_match);
1968
1969 #ifdef CONFIG_ACPI
acpi_smmu_get_data(u32 model,struct arm_smmu_device * smmu)1970 static int acpi_smmu_get_data(u32 model, struct arm_smmu_device *smmu)
1971 {
1972 int ret = 0;
1973
1974 switch (model) {
1975 case ACPI_IORT_SMMU_V1:
1976 case ACPI_IORT_SMMU_CORELINK_MMU400:
1977 smmu->version = ARM_SMMU_V1;
1978 smmu->model = GENERIC_SMMU;
1979 break;
1980 case ACPI_IORT_SMMU_CORELINK_MMU401:
1981 smmu->version = ARM_SMMU_V1_64K;
1982 smmu->model = GENERIC_SMMU;
1983 break;
1984 case ACPI_IORT_SMMU_V2:
1985 smmu->version = ARM_SMMU_V2;
1986 smmu->model = GENERIC_SMMU;
1987 break;
1988 case ACPI_IORT_SMMU_CORELINK_MMU500:
1989 smmu->version = ARM_SMMU_V2;
1990 smmu->model = ARM_MMU500;
1991 break;
1992 case ACPI_IORT_SMMU_CAVIUM_THUNDERX:
1993 smmu->version = ARM_SMMU_V2;
1994 smmu->model = CAVIUM_SMMUV2;
1995 break;
1996 default:
1997 ret = -ENODEV;
1998 }
1999
2000 return ret;
2001 }
2002
arm_smmu_device_acpi_probe(struct arm_smmu_device * smmu,u32 * global_irqs,u32 * pmu_irqs)2003 static int arm_smmu_device_acpi_probe(struct arm_smmu_device *smmu,
2004 u32 *global_irqs, u32 *pmu_irqs)
2005 {
2006 struct device *dev = smmu->dev;
2007 struct acpi_iort_node *node =
2008 *(struct acpi_iort_node **)dev_get_platdata(dev);
2009 struct acpi_iort_smmu *iort_smmu;
2010 int ret;
2011
2012 /* Retrieve SMMU1/2 specific data */
2013 iort_smmu = (struct acpi_iort_smmu *)node->node_data;
2014
2015 ret = acpi_smmu_get_data(iort_smmu->model, smmu);
2016 if (ret < 0)
2017 return ret;
2018
2019 /* Ignore the configuration access interrupt */
2020 *global_irqs = 1;
2021 *pmu_irqs = 0;
2022
2023 if (iort_smmu->flags & ACPI_IORT_SMMU_COHERENT_WALK)
2024 smmu->features |= ARM_SMMU_FEAT_COHERENT_WALK;
2025
2026 return 0;
2027 }
2028 #else
arm_smmu_device_acpi_probe(struct arm_smmu_device * smmu,u32 * global_irqs,u32 * pmu_irqs)2029 static inline int arm_smmu_device_acpi_probe(struct arm_smmu_device *smmu,
2030 u32 *global_irqs, u32 *pmu_irqs)
2031 {
2032 return -ENODEV;
2033 }
2034 #endif
2035
arm_smmu_device_dt_probe(struct arm_smmu_device * smmu,u32 * global_irqs,u32 * pmu_irqs)2036 static int arm_smmu_device_dt_probe(struct arm_smmu_device *smmu,
2037 u32 *global_irqs, u32 *pmu_irqs)
2038 {
2039 const struct arm_smmu_match_data *data;
2040 struct device *dev = smmu->dev;
2041 bool legacy_binding;
2042
2043 if (of_property_read_u32(dev->of_node, "#global-interrupts", global_irqs))
2044 return dev_err_probe(dev, -ENODEV,
2045 "missing #global-interrupts property\n");
2046 *pmu_irqs = 0;
2047
2048 data = of_device_get_match_data(dev);
2049 smmu->version = data->version;
2050 smmu->model = data->model;
2051
2052 legacy_binding = of_find_property(dev->of_node, "mmu-masters", NULL);
2053 if (legacy_binding && !using_generic_binding) {
2054 if (!using_legacy_binding) {
2055 pr_notice("deprecated \"mmu-masters\" DT property in use; %s support unavailable\n",
2056 IS_ENABLED(CONFIG_ARM_SMMU_LEGACY_DT_BINDINGS) ? "DMA API" : "SMMU");
2057 }
2058 using_legacy_binding = true;
2059 } else if (!legacy_binding && !using_legacy_binding) {
2060 using_generic_binding = true;
2061 } else {
2062 dev_err(dev, "not probing due to mismatched DT properties\n");
2063 return -ENODEV;
2064 }
2065
2066 if (of_dma_is_coherent(dev->of_node))
2067 smmu->features |= ARM_SMMU_FEAT_COHERENT_WALK;
2068
2069 return 0;
2070 }
2071
arm_smmu_rmr_install_bypass_smr(struct arm_smmu_device * smmu)2072 static void arm_smmu_rmr_install_bypass_smr(struct arm_smmu_device *smmu)
2073 {
2074 struct list_head rmr_list;
2075 struct iommu_resv_region *e;
2076 int idx, cnt = 0;
2077 u32 reg;
2078
2079 INIT_LIST_HEAD(&rmr_list);
2080 iort_get_rmr_sids(dev_fwnode(smmu->dev), &rmr_list);
2081
2082 /*
2083 * Rather than trying to look at existing mappings that
2084 * are setup by the firmware and then invalidate the ones
2085 * that do no have matching RMR entries, just disable the
2086 * SMMU until it gets enabled again in the reset routine.
2087 */
2088 reg = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sCR0);
2089 reg |= ARM_SMMU_sCR0_CLIENTPD;
2090 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_sCR0, reg);
2091
2092 list_for_each_entry(e, &rmr_list, list) {
2093 struct iommu_iort_rmr_data *rmr;
2094 int i;
2095
2096 rmr = container_of(e, struct iommu_iort_rmr_data, rr);
2097 for (i = 0; i < rmr->num_sids; i++) {
2098 idx = arm_smmu_find_sme(smmu, rmr->sids[i], ~0);
2099 if (idx < 0)
2100 continue;
2101
2102 if (smmu->s2crs[idx].count == 0) {
2103 smmu->smrs[idx].id = rmr->sids[i];
2104 smmu->smrs[idx].mask = 0;
2105 smmu->smrs[idx].valid = true;
2106 }
2107 smmu->s2crs[idx].count++;
2108 smmu->s2crs[idx].type = S2CR_TYPE_BYPASS;
2109 smmu->s2crs[idx].privcfg = S2CR_PRIVCFG_DEFAULT;
2110
2111 cnt++;
2112 }
2113 }
2114
2115 dev_notice(smmu->dev, "\tpreserved %d boot mapping%s\n", cnt,
2116 str_plural(cnt));
2117 iort_put_rmr_sids(dev_fwnode(smmu->dev), &rmr_list);
2118 }
2119
arm_smmu_device_probe(struct platform_device * pdev)2120 static int arm_smmu_device_probe(struct platform_device *pdev)
2121 {
2122 struct resource *res;
2123 struct arm_smmu_device *smmu;
2124 struct device *dev = &pdev->dev;
2125 int num_irqs, i, err;
2126 u32 global_irqs, pmu_irqs;
2127 irqreturn_t (*global_fault)(int irq, void *dev);
2128
2129 smmu = devm_kzalloc(dev, sizeof(*smmu), GFP_KERNEL);
2130 if (!smmu) {
2131 dev_err(dev, "failed to allocate arm_smmu_device\n");
2132 return -ENOMEM;
2133 }
2134 smmu->dev = dev;
2135
2136 if (dev->of_node)
2137 err = arm_smmu_device_dt_probe(smmu, &global_irqs, &pmu_irqs);
2138 else
2139 err = arm_smmu_device_acpi_probe(smmu, &global_irqs, &pmu_irqs);
2140 if (err)
2141 return err;
2142
2143 smmu->base = devm_platform_get_and_ioremap_resource(pdev, 0, &res);
2144 if (IS_ERR(smmu->base))
2145 return PTR_ERR(smmu->base);
2146 smmu->ioaddr = res->start;
2147
2148 /*
2149 * The resource size should effectively match the value of SMMU_TOP;
2150 * stash that temporarily until we know PAGESIZE to validate it with.
2151 */
2152 smmu->numpage = resource_size(res);
2153
2154 smmu = arm_smmu_impl_init(smmu);
2155 if (IS_ERR(smmu))
2156 return PTR_ERR(smmu);
2157
2158 num_irqs = platform_irq_count(pdev);
2159
2160 smmu->num_context_irqs = num_irqs - global_irqs - pmu_irqs;
2161 if (smmu->num_context_irqs <= 0)
2162 return dev_err_probe(dev, -ENODEV,
2163 "found %d interrupts but expected at least %d\n",
2164 num_irqs, global_irqs + pmu_irqs + 1);
2165
2166 smmu->irqs = devm_kcalloc(dev, smmu->num_context_irqs,
2167 sizeof(*smmu->irqs), GFP_KERNEL);
2168 if (!smmu->irqs)
2169 return dev_err_probe(dev, -ENOMEM, "failed to allocate %d irqs\n",
2170 smmu->num_context_irqs);
2171
2172 for (i = 0; i < smmu->num_context_irqs; i++) {
2173 int irq = platform_get_irq(pdev, global_irqs + pmu_irqs + i);
2174
2175 if (irq < 0)
2176 return irq;
2177 smmu->irqs[i] = irq;
2178 }
2179
2180 err = devm_clk_bulk_get_all(dev, &smmu->clks);
2181 if (err < 0) {
2182 dev_err(dev, "failed to get clocks %d\n", err);
2183 return err;
2184 }
2185 smmu->num_clks = err;
2186
2187 err = clk_bulk_prepare_enable(smmu->num_clks, smmu->clks);
2188 if (err)
2189 return err;
2190
2191 err = arm_smmu_device_cfg_probe(smmu);
2192 if (err)
2193 return err;
2194
2195 if (smmu->version == ARM_SMMU_V2) {
2196 if (smmu->num_context_banks > smmu->num_context_irqs) {
2197 dev_err(dev,
2198 "found only %d context irq(s) but %d required\n",
2199 smmu->num_context_irqs, smmu->num_context_banks);
2200 return -ENODEV;
2201 }
2202
2203 /* Ignore superfluous interrupts */
2204 smmu->num_context_irqs = smmu->num_context_banks;
2205 }
2206
2207 if (smmu->impl && smmu->impl->global_fault)
2208 global_fault = smmu->impl->global_fault;
2209 else
2210 global_fault = arm_smmu_global_fault;
2211
2212 for (i = 0; i < global_irqs; i++) {
2213 int irq = platform_get_irq(pdev, i);
2214
2215 if (irq < 0)
2216 return irq;
2217
2218 err = devm_request_irq(dev, irq, global_fault, IRQF_SHARED,
2219 "arm-smmu global fault", smmu);
2220 if (err)
2221 return dev_err_probe(dev, err,
2222 "failed to request global IRQ %d (%u)\n",
2223 i, irq);
2224 }
2225
2226 platform_set_drvdata(pdev, smmu);
2227
2228 /* Check for RMRs and install bypass SMRs if any */
2229 arm_smmu_rmr_install_bypass_smr(smmu);
2230
2231 arm_smmu_device_reset(smmu);
2232 arm_smmu_test_smr_masks(smmu);
2233
2234 err = iommu_device_sysfs_add(&smmu->iommu, smmu->dev, NULL,
2235 "smmu.%pa", &smmu->ioaddr);
2236 if (err)
2237 return dev_err_probe(dev, err, "Failed to register iommu in sysfs\n");
2238
2239 err = iommu_device_register(&smmu->iommu, &arm_smmu_ops,
2240 using_legacy_binding ? NULL : dev);
2241 if (err) {
2242 iommu_device_sysfs_remove(&smmu->iommu);
2243 return dev_err_probe(dev, err, "Failed to register iommu\n");
2244 }
2245
2246 /*
2247 * We want to avoid touching dev->power.lock in fastpaths unless
2248 * it's really going to do something useful - pm_runtime_enabled()
2249 * can serve as an ideal proxy for that decision. So, conditionally
2250 * enable pm_runtime.
2251 */
2252 if (dev->pm_domain) {
2253 pm_runtime_set_active(dev);
2254 pm_runtime_enable(dev);
2255 arm_smmu_rpm_use_autosuspend(smmu);
2256 }
2257
2258 return 0;
2259 }
2260
arm_smmu_device_shutdown(struct platform_device * pdev)2261 static void arm_smmu_device_shutdown(struct platform_device *pdev)
2262 {
2263 struct arm_smmu_device *smmu = platform_get_drvdata(pdev);
2264
2265 if (!bitmap_empty(smmu->context_map, ARM_SMMU_MAX_CBS))
2266 dev_notice(&pdev->dev, "disabling translation\n");
2267
2268 arm_smmu_rpm_get(smmu);
2269 /* Turn the thing off */
2270 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_sCR0, ARM_SMMU_sCR0_CLIENTPD);
2271 arm_smmu_rpm_put(smmu);
2272
2273 if (pm_runtime_enabled(smmu->dev))
2274 pm_runtime_force_suspend(smmu->dev);
2275 else
2276 clk_bulk_disable(smmu->num_clks, smmu->clks);
2277
2278 clk_bulk_unprepare(smmu->num_clks, smmu->clks);
2279 }
2280
arm_smmu_device_remove(struct platform_device * pdev)2281 static void arm_smmu_device_remove(struct platform_device *pdev)
2282 {
2283 struct arm_smmu_device *smmu = platform_get_drvdata(pdev);
2284
2285 iommu_device_unregister(&smmu->iommu);
2286 iommu_device_sysfs_remove(&smmu->iommu);
2287
2288 arm_smmu_device_shutdown(pdev);
2289 }
2290
arm_smmu_runtime_resume(struct device * dev)2291 static int __maybe_unused arm_smmu_runtime_resume(struct device *dev)
2292 {
2293 struct arm_smmu_device *smmu = dev_get_drvdata(dev);
2294 int ret;
2295
2296 ret = clk_bulk_enable(smmu->num_clks, smmu->clks);
2297 if (ret)
2298 return ret;
2299
2300 arm_smmu_device_reset(smmu);
2301
2302 return 0;
2303 }
2304
arm_smmu_runtime_suspend(struct device * dev)2305 static int __maybe_unused arm_smmu_runtime_suspend(struct device *dev)
2306 {
2307 struct arm_smmu_device *smmu = dev_get_drvdata(dev);
2308
2309 clk_bulk_disable(smmu->num_clks, smmu->clks);
2310
2311 return 0;
2312 }
2313
arm_smmu_pm_resume(struct device * dev)2314 static int __maybe_unused arm_smmu_pm_resume(struct device *dev)
2315 {
2316 int ret;
2317 struct arm_smmu_device *smmu = dev_get_drvdata(dev);
2318
2319 ret = clk_bulk_prepare(smmu->num_clks, smmu->clks);
2320 if (ret)
2321 return ret;
2322
2323 if (pm_runtime_suspended(dev))
2324 return 0;
2325
2326 ret = arm_smmu_runtime_resume(dev);
2327 if (ret)
2328 clk_bulk_unprepare(smmu->num_clks, smmu->clks);
2329
2330 return ret;
2331 }
2332
arm_smmu_pm_suspend(struct device * dev)2333 static int __maybe_unused arm_smmu_pm_suspend(struct device *dev)
2334 {
2335 int ret = 0;
2336 struct arm_smmu_device *smmu = dev_get_drvdata(dev);
2337
2338 if (pm_runtime_suspended(dev))
2339 goto clk_unprepare;
2340
2341 ret = arm_smmu_runtime_suspend(dev);
2342 if (ret)
2343 return ret;
2344
2345 clk_unprepare:
2346 clk_bulk_unprepare(smmu->num_clks, smmu->clks);
2347 return ret;
2348 }
2349
2350 static const struct dev_pm_ops arm_smmu_pm_ops = {
2351 SET_SYSTEM_SLEEP_PM_OPS(arm_smmu_pm_suspend, arm_smmu_pm_resume)
2352 SET_RUNTIME_PM_OPS(arm_smmu_runtime_suspend,
2353 arm_smmu_runtime_resume, NULL)
2354 };
2355
2356 static struct platform_driver arm_smmu_driver = {
2357 .driver = {
2358 .name = "arm-smmu",
2359 .of_match_table = arm_smmu_of_match,
2360 .pm = &arm_smmu_pm_ops,
2361 .suppress_bind_attrs = true,
2362 },
2363 .probe = arm_smmu_device_probe,
2364 .remove = arm_smmu_device_remove,
2365 .shutdown = arm_smmu_device_shutdown,
2366 };
2367 module_platform_driver(arm_smmu_driver);
2368
2369 MODULE_DESCRIPTION("IOMMU API for ARM architected SMMU implementations");
2370 MODULE_AUTHOR("Will Deacon <will@kernel.org>");
2371 MODULE_ALIAS("platform:arm-smmu");
2372 MODULE_LICENSE("GPL v2");
2373