1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * IOMMU API for ARM architected SMMU implementations.
4 *
5 * Copyright (C) 2013 ARM Limited
6 *
7 * Author: Will Deacon <will.deacon@arm.com>
8 *
9 * This driver currently supports:
10 * - SMMUv1 and v2 implementations
11 * - Stream-matching and stream-indexing
12 * - v7/v8 long-descriptor format
13 * - Non-secure access to the SMMU
14 * - Context fault reporting
15 * - Extended Stream ID (16 bit)
16 */
17
18 #define pr_fmt(fmt) "arm-smmu: " fmt
19
20 #include <linux/acpi.h>
21 #include <linux/acpi_iort.h>
22 #include <linux/bitfield.h>
23 #include <linux/delay.h>
24 #include <linux/dma-mapping.h>
25 #include <linux/err.h>
26 #include <linux/interrupt.h>
27 #include <linux/io.h>
28 #include <linux/iopoll.h>
29 #include <linux/module.h>
30 #include <linux/of.h>
31 #include <linux/of_address.h>
32 #include <linux/pci.h>
33 #include <linux/platform_device.h>
34 #include <linux/pm_runtime.h>
35 #include <linux/ratelimit.h>
36 #include <linux/slab.h>
37 #include <linux/string_choices.h>
38
39 #include <linux/fsl/mc.h>
40
41 #include "arm-smmu.h"
42 #include "../../dma-iommu.h"
43
44 /*
45 * Apparently, some Qualcomm arm64 platforms which appear to expose their SMMU
46 * global register space are still, in fact, using a hypervisor to mediate it
47 * by trapping and emulating register accesses. Sadly, some deployed versions
48 * of said trapping code have bugs wherein they go horribly wrong for stores
49 * using r31 (i.e. XZR/WZR) as the source register.
50 */
51 #define QCOM_DUMMY_VAL -1
52
53 #define MSI_IOVA_BASE 0x8000000
54 #define MSI_IOVA_LENGTH 0x100000
55
56 static int force_stage;
57 module_param(force_stage, int, S_IRUGO);
58 MODULE_PARM_DESC(force_stage,
59 "Force SMMU mappings to be installed at a particular stage of translation. A value of '1' or '2' forces the corresponding stage. All other values are ignored (i.e. no stage is forced). Note that selecting a specific stage will disable support for nested translation.");
60 static bool disable_bypass =
61 IS_ENABLED(CONFIG_ARM_SMMU_DISABLE_BYPASS_BY_DEFAULT);
62 module_param(disable_bypass, bool, S_IRUGO);
63 MODULE_PARM_DESC(disable_bypass,
64 "Disable bypass streams such that incoming transactions from devices that are not attached to an iommu domain will report an abort back to the device and will not be allowed to pass through the SMMU.");
65
66 #define s2cr_init_val (struct arm_smmu_s2cr){ \
67 .type = disable_bypass ? S2CR_TYPE_FAULT : S2CR_TYPE_BYPASS, \
68 }
69
70 static bool using_legacy_binding, using_generic_binding;
71
arm_smmu_rpm_get(struct arm_smmu_device * smmu)72 static inline int arm_smmu_rpm_get(struct arm_smmu_device *smmu)
73 {
74 if (pm_runtime_enabled(smmu->dev))
75 return pm_runtime_resume_and_get(smmu->dev);
76
77 return 0;
78 }
79
arm_smmu_rpm_put(struct arm_smmu_device * smmu)80 static inline void arm_smmu_rpm_put(struct arm_smmu_device *smmu)
81 {
82 if (pm_runtime_enabled(smmu->dev))
83 pm_runtime_put_autosuspend(smmu->dev);
84 }
85
arm_smmu_rpm_use_autosuspend(struct arm_smmu_device * smmu)86 static void arm_smmu_rpm_use_autosuspend(struct arm_smmu_device *smmu)
87 {
88 /*
89 * Setup an autosuspend delay to avoid bouncing runpm state.
90 * Otherwise, if a driver for a suspended consumer device
91 * unmaps buffers, it will runpm resume/suspend for each one.
92 *
93 * For example, when used by a GPU device, when an application
94 * or game exits, it can trigger unmapping 100s or 1000s of
95 * buffers. With a runpm cycle for each buffer, that adds up
96 * to 5-10sec worth of reprogramming the context bank, while
97 * the system appears to be locked up to the user.
98 */
99 pm_runtime_set_autosuspend_delay(smmu->dev, 20);
100 pm_runtime_use_autosuspend(smmu->dev);
101 }
102
to_smmu_domain(struct iommu_domain * dom)103 static struct arm_smmu_domain *to_smmu_domain(struct iommu_domain *dom)
104 {
105 return container_of(dom, struct arm_smmu_domain, domain);
106 }
107
108 static struct platform_driver arm_smmu_driver;
109 static struct iommu_ops arm_smmu_ops;
110
111 #ifdef CONFIG_ARM_SMMU_LEGACY_DT_BINDINGS
dev_get_dev_node(struct device * dev)112 static struct device_node *dev_get_dev_node(struct device *dev)
113 {
114 if (dev_is_pci(dev)) {
115 struct pci_bus *bus = to_pci_dev(dev)->bus;
116
117 while (!pci_is_root_bus(bus))
118 bus = bus->parent;
119 return of_node_get(bus->bridge->parent->of_node);
120 }
121
122 return of_node_get(dev->of_node);
123 }
124
__arm_smmu_get_pci_sid(struct pci_dev * pdev,u16 alias,void * data)125 static int __arm_smmu_get_pci_sid(struct pci_dev *pdev, u16 alias, void *data)
126 {
127 *((__be32 *)data) = cpu_to_be32(alias);
128 return 0; /* Continue walking */
129 }
130
__find_legacy_master_phandle(struct device * dev,void * data)131 static int __find_legacy_master_phandle(struct device *dev, void *data)
132 {
133 struct of_phandle_iterator *it = *(void **)data;
134 struct device_node *np = it->node;
135 int err;
136
137 of_for_each_phandle(it, err, dev->of_node, "mmu-masters",
138 "#stream-id-cells", -1)
139 if (it->node == np) {
140 *(void **)data = dev;
141 return 1;
142 }
143 it->node = np;
144 return err == -ENOENT ? 0 : err;
145 }
146
arm_smmu_register_legacy_master(struct device * dev,struct arm_smmu_device ** smmu)147 static int arm_smmu_register_legacy_master(struct device *dev,
148 struct arm_smmu_device **smmu)
149 {
150 struct device *smmu_dev;
151 struct device_node *np;
152 struct of_phandle_iterator it;
153 void *data = ⁢
154 u32 *sids;
155 __be32 pci_sid;
156 int err;
157
158 np = dev_get_dev_node(dev);
159 if (!np || !of_property_present(np, "#stream-id-cells")) {
160 of_node_put(np);
161 return -ENODEV;
162 }
163
164 it.node = np;
165 err = driver_for_each_device(&arm_smmu_driver.driver, NULL, &data,
166 __find_legacy_master_phandle);
167 smmu_dev = data;
168 of_node_put(np);
169 if (err == 0)
170 return -ENODEV;
171 if (err < 0)
172 return err;
173
174 if (dev_is_pci(dev)) {
175 /* "mmu-masters" assumes Stream ID == Requester ID */
176 pci_for_each_dma_alias(to_pci_dev(dev), __arm_smmu_get_pci_sid,
177 &pci_sid);
178 it.cur = &pci_sid;
179 it.cur_count = 1;
180 }
181
182 err = iommu_fwspec_init(dev, NULL);
183 if (err)
184 return err;
185
186 sids = kcalloc(it.cur_count, sizeof(*sids), GFP_KERNEL);
187 if (!sids)
188 return -ENOMEM;
189
190 *smmu = dev_get_drvdata(smmu_dev);
191 of_phandle_iterator_args(&it, sids, it.cur_count);
192 err = iommu_fwspec_add_ids(dev, sids, it.cur_count);
193 kfree(sids);
194 return err;
195 }
196 #else
arm_smmu_register_legacy_master(struct device * dev,struct arm_smmu_device ** smmu)197 static int arm_smmu_register_legacy_master(struct device *dev,
198 struct arm_smmu_device **smmu)
199 {
200 return -ENODEV;
201 }
202 #endif /* CONFIG_ARM_SMMU_LEGACY_DT_BINDINGS */
203
__arm_smmu_free_bitmap(unsigned long * map,int idx)204 static void __arm_smmu_free_bitmap(unsigned long *map, int idx)
205 {
206 clear_bit(idx, map);
207 }
208
209 /* Wait for any pending TLB invalidations to complete */
__arm_smmu_tlb_sync(struct arm_smmu_device * smmu,int page,int sync,int status)210 static void __arm_smmu_tlb_sync(struct arm_smmu_device *smmu, int page,
211 int sync, int status)
212 {
213 unsigned int spin_cnt, delay;
214 u32 reg;
215
216 if (smmu->impl && unlikely(smmu->impl->tlb_sync))
217 return smmu->impl->tlb_sync(smmu, page, sync, status);
218
219 arm_smmu_writel(smmu, page, sync, QCOM_DUMMY_VAL);
220 for (delay = 1; delay < TLB_LOOP_TIMEOUT; delay *= 2) {
221 for (spin_cnt = TLB_SPIN_COUNT; spin_cnt > 0; spin_cnt--) {
222 reg = arm_smmu_readl(smmu, page, status);
223 if (!(reg & ARM_SMMU_sTLBGSTATUS_GSACTIVE))
224 return;
225 cpu_relax();
226 }
227 udelay(delay);
228 }
229 dev_err_ratelimited(smmu->dev,
230 "TLB sync timed out -- SMMU may be deadlocked\n");
231 }
232
arm_smmu_tlb_sync_global(struct arm_smmu_device * smmu)233 static void arm_smmu_tlb_sync_global(struct arm_smmu_device *smmu)
234 {
235 unsigned long flags;
236
237 spin_lock_irqsave(&smmu->global_sync_lock, flags);
238 __arm_smmu_tlb_sync(smmu, ARM_SMMU_GR0, ARM_SMMU_GR0_sTLBGSYNC,
239 ARM_SMMU_GR0_sTLBGSTATUS);
240 spin_unlock_irqrestore(&smmu->global_sync_lock, flags);
241 }
242
arm_smmu_tlb_sync_context(struct arm_smmu_domain * smmu_domain)243 static void arm_smmu_tlb_sync_context(struct arm_smmu_domain *smmu_domain)
244 {
245 struct arm_smmu_device *smmu = smmu_domain->smmu;
246 unsigned long flags;
247
248 spin_lock_irqsave(&smmu_domain->cb_lock, flags);
249 __arm_smmu_tlb_sync(smmu, ARM_SMMU_CB(smmu, smmu_domain->cfg.cbndx),
250 ARM_SMMU_CB_TLBSYNC, ARM_SMMU_CB_TLBSTATUS);
251 spin_unlock_irqrestore(&smmu_domain->cb_lock, flags);
252 }
253
arm_smmu_tlb_inv_context_s1(void * cookie)254 static void arm_smmu_tlb_inv_context_s1(void *cookie)
255 {
256 struct arm_smmu_domain *smmu_domain = cookie;
257 /*
258 * The TLBI write may be relaxed, so ensure that PTEs cleared by the
259 * current CPU are visible beforehand.
260 */
261 wmb();
262 arm_smmu_cb_write(smmu_domain->smmu, smmu_domain->cfg.cbndx,
263 ARM_SMMU_CB_S1_TLBIASID, smmu_domain->cfg.asid);
264 arm_smmu_tlb_sync_context(smmu_domain);
265 }
266
arm_smmu_tlb_inv_context_s2(void * cookie)267 static void arm_smmu_tlb_inv_context_s2(void *cookie)
268 {
269 struct arm_smmu_domain *smmu_domain = cookie;
270 struct arm_smmu_device *smmu = smmu_domain->smmu;
271
272 /* See above */
273 wmb();
274 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_TLBIVMID, smmu_domain->cfg.vmid);
275 arm_smmu_tlb_sync_global(smmu);
276 }
277
arm_smmu_tlb_inv_range_s1(unsigned long iova,size_t size,size_t granule,void * cookie,int reg)278 static void arm_smmu_tlb_inv_range_s1(unsigned long iova, size_t size,
279 size_t granule, void *cookie, int reg)
280 {
281 struct arm_smmu_domain *smmu_domain = cookie;
282 struct arm_smmu_device *smmu = smmu_domain->smmu;
283 struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
284 int idx = cfg->cbndx;
285
286 if (smmu->features & ARM_SMMU_FEAT_COHERENT_WALK)
287 wmb();
288
289 if (cfg->fmt != ARM_SMMU_CTX_FMT_AARCH64) {
290 iova = (iova >> 12) << 12;
291 iova |= cfg->asid;
292 do {
293 arm_smmu_cb_write(smmu, idx, reg, iova);
294 iova += granule;
295 } while (size -= granule);
296 } else {
297 iova >>= 12;
298 iova |= (u64)cfg->asid << 48;
299 do {
300 arm_smmu_cb_writeq(smmu, idx, reg, iova);
301 iova += granule >> 12;
302 } while (size -= granule);
303 }
304 }
305
arm_smmu_tlb_inv_range_s2(unsigned long iova,size_t size,size_t granule,void * cookie,int reg)306 static void arm_smmu_tlb_inv_range_s2(unsigned long iova, size_t size,
307 size_t granule, void *cookie, int reg)
308 {
309 struct arm_smmu_domain *smmu_domain = cookie;
310 struct arm_smmu_device *smmu = smmu_domain->smmu;
311 int idx = smmu_domain->cfg.cbndx;
312
313 if (smmu->features & ARM_SMMU_FEAT_COHERENT_WALK)
314 wmb();
315
316 iova >>= 12;
317 do {
318 if (smmu_domain->cfg.fmt == ARM_SMMU_CTX_FMT_AARCH64)
319 arm_smmu_cb_writeq(smmu, idx, reg, iova);
320 else
321 arm_smmu_cb_write(smmu, idx, reg, iova);
322 iova += granule >> 12;
323 } while (size -= granule);
324 }
325
arm_smmu_tlb_inv_walk_s1(unsigned long iova,size_t size,size_t granule,void * cookie)326 static void arm_smmu_tlb_inv_walk_s1(unsigned long iova, size_t size,
327 size_t granule, void *cookie)
328 {
329 struct arm_smmu_domain *smmu_domain = cookie;
330 struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
331
332 if (cfg->flush_walk_prefer_tlbiasid) {
333 arm_smmu_tlb_inv_context_s1(cookie);
334 } else {
335 arm_smmu_tlb_inv_range_s1(iova, size, granule, cookie,
336 ARM_SMMU_CB_S1_TLBIVA);
337 arm_smmu_tlb_sync_context(cookie);
338 }
339 }
340
arm_smmu_tlb_add_page_s1(struct iommu_iotlb_gather * gather,unsigned long iova,size_t granule,void * cookie)341 static void arm_smmu_tlb_add_page_s1(struct iommu_iotlb_gather *gather,
342 unsigned long iova, size_t granule,
343 void *cookie)
344 {
345 arm_smmu_tlb_inv_range_s1(iova, granule, granule, cookie,
346 ARM_SMMU_CB_S1_TLBIVAL);
347 }
348
arm_smmu_tlb_inv_walk_s2(unsigned long iova,size_t size,size_t granule,void * cookie)349 static void arm_smmu_tlb_inv_walk_s2(unsigned long iova, size_t size,
350 size_t granule, void *cookie)
351 {
352 arm_smmu_tlb_inv_range_s2(iova, size, granule, cookie,
353 ARM_SMMU_CB_S2_TLBIIPAS2);
354 arm_smmu_tlb_sync_context(cookie);
355 }
356
arm_smmu_tlb_add_page_s2(struct iommu_iotlb_gather * gather,unsigned long iova,size_t granule,void * cookie)357 static void arm_smmu_tlb_add_page_s2(struct iommu_iotlb_gather *gather,
358 unsigned long iova, size_t granule,
359 void *cookie)
360 {
361 arm_smmu_tlb_inv_range_s2(iova, granule, granule, cookie,
362 ARM_SMMU_CB_S2_TLBIIPAS2L);
363 }
364
arm_smmu_tlb_inv_walk_s2_v1(unsigned long iova,size_t size,size_t granule,void * cookie)365 static void arm_smmu_tlb_inv_walk_s2_v1(unsigned long iova, size_t size,
366 size_t granule, void *cookie)
367 {
368 arm_smmu_tlb_inv_context_s2(cookie);
369 }
370 /*
371 * On MMU-401 at least, the cost of firing off multiple TLBIVMIDs appears
372 * almost negligible, but the benefit of getting the first one in as far ahead
373 * of the sync as possible is significant, hence we don't just make this a
374 * no-op and call arm_smmu_tlb_inv_context_s2() from .iotlb_sync as you might
375 * think.
376 */
arm_smmu_tlb_add_page_s2_v1(struct iommu_iotlb_gather * gather,unsigned long iova,size_t granule,void * cookie)377 static void arm_smmu_tlb_add_page_s2_v1(struct iommu_iotlb_gather *gather,
378 unsigned long iova, size_t granule,
379 void *cookie)
380 {
381 struct arm_smmu_domain *smmu_domain = cookie;
382 struct arm_smmu_device *smmu = smmu_domain->smmu;
383
384 if (smmu->features & ARM_SMMU_FEAT_COHERENT_WALK)
385 wmb();
386
387 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_TLBIVMID, smmu_domain->cfg.vmid);
388 }
389
390 static const struct iommu_flush_ops arm_smmu_s1_tlb_ops = {
391 .tlb_flush_all = arm_smmu_tlb_inv_context_s1,
392 .tlb_flush_walk = arm_smmu_tlb_inv_walk_s1,
393 .tlb_add_page = arm_smmu_tlb_add_page_s1,
394 };
395
396 static const struct iommu_flush_ops arm_smmu_s2_tlb_ops_v2 = {
397 .tlb_flush_all = arm_smmu_tlb_inv_context_s2,
398 .tlb_flush_walk = arm_smmu_tlb_inv_walk_s2,
399 .tlb_add_page = arm_smmu_tlb_add_page_s2,
400 };
401
402 static const struct iommu_flush_ops arm_smmu_s2_tlb_ops_v1 = {
403 .tlb_flush_all = arm_smmu_tlb_inv_context_s2,
404 .tlb_flush_walk = arm_smmu_tlb_inv_walk_s2_v1,
405 .tlb_add_page = arm_smmu_tlb_add_page_s2_v1,
406 };
407
408
arm_smmu_read_context_fault_info(struct arm_smmu_device * smmu,int idx,struct arm_smmu_context_fault_info * cfi)409 void arm_smmu_read_context_fault_info(struct arm_smmu_device *smmu, int idx,
410 struct arm_smmu_context_fault_info *cfi)
411 {
412 cfi->iova = arm_smmu_cb_readq(smmu, idx, ARM_SMMU_CB_FAR);
413 cfi->fsr = arm_smmu_cb_read(smmu, idx, ARM_SMMU_CB_FSR);
414 cfi->fsynr = arm_smmu_cb_read(smmu, idx, ARM_SMMU_CB_FSYNR0);
415 cfi->cbfrsynra = arm_smmu_gr1_read(smmu, ARM_SMMU_GR1_CBFRSYNRA(idx));
416 }
417
arm_smmu_print_context_fault_info(struct arm_smmu_device * smmu,int idx,const struct arm_smmu_context_fault_info * cfi)418 void arm_smmu_print_context_fault_info(struct arm_smmu_device *smmu, int idx,
419 const struct arm_smmu_context_fault_info *cfi)
420 {
421 dev_err(smmu->dev,
422 "Unhandled context fault: fsr=0x%x, iova=0x%08lx, fsynr=0x%x, cbfrsynra=0x%x, cb=%d\n",
423 cfi->fsr, cfi->iova, cfi->fsynr, cfi->cbfrsynra, idx);
424
425 dev_err(smmu->dev, "FSR = %08x [%s%sFormat=%u%s%s%s%s%s%s%s%s], SID=0x%x\n",
426 cfi->fsr,
427 (cfi->fsr & ARM_SMMU_CB_FSR_MULTI) ? "MULTI " : "",
428 (cfi->fsr & ARM_SMMU_CB_FSR_SS) ? "SS " : "",
429 (u32)FIELD_GET(ARM_SMMU_CB_FSR_FORMAT, cfi->fsr),
430 (cfi->fsr & ARM_SMMU_CB_FSR_UUT) ? " UUT" : "",
431 (cfi->fsr & ARM_SMMU_CB_FSR_ASF) ? " ASF" : "",
432 (cfi->fsr & ARM_SMMU_CB_FSR_TLBLKF) ? " TLBLKF" : "",
433 (cfi->fsr & ARM_SMMU_CB_FSR_TLBMCF) ? " TLBMCF" : "",
434 (cfi->fsr & ARM_SMMU_CB_FSR_EF) ? " EF" : "",
435 (cfi->fsr & ARM_SMMU_CB_FSR_PF) ? " PF" : "",
436 (cfi->fsr & ARM_SMMU_CB_FSR_AFF) ? " AFF" : "",
437 (cfi->fsr & ARM_SMMU_CB_FSR_TF) ? " TF" : "",
438 cfi->cbfrsynra);
439
440 dev_err(smmu->dev, "FSYNR0 = %08x [S1CBNDX=%u%s%s%s%s%s%s PLVL=%u]\n",
441 cfi->fsynr,
442 (u32)FIELD_GET(ARM_SMMU_CB_FSYNR0_S1CBNDX, cfi->fsynr),
443 (cfi->fsynr & ARM_SMMU_CB_FSYNR0_AFR) ? " AFR" : "",
444 (cfi->fsynr & ARM_SMMU_CB_FSYNR0_PTWF) ? " PTWF" : "",
445 (cfi->fsynr & ARM_SMMU_CB_FSYNR0_NSATTR) ? " NSATTR" : "",
446 (cfi->fsynr & ARM_SMMU_CB_FSYNR0_IND) ? " IND" : "",
447 (cfi->fsynr & ARM_SMMU_CB_FSYNR0_PNU) ? " PNU" : "",
448 (cfi->fsynr & ARM_SMMU_CB_FSYNR0_WNR) ? " WNR" : "",
449 (u32)FIELD_GET(ARM_SMMU_CB_FSYNR0_PLVL, cfi->fsynr));
450 }
451
arm_smmu_context_fault(int irq,void * dev)452 static irqreturn_t arm_smmu_context_fault(int irq, void *dev)
453 {
454 struct arm_smmu_context_fault_info cfi;
455 struct arm_smmu_domain *smmu_domain = dev;
456 struct arm_smmu_device *smmu = smmu_domain->smmu;
457 static DEFINE_RATELIMIT_STATE(rs, DEFAULT_RATELIMIT_INTERVAL,
458 DEFAULT_RATELIMIT_BURST);
459 int idx = smmu_domain->cfg.cbndx;
460 int ret;
461
462 arm_smmu_read_context_fault_info(smmu, idx, &cfi);
463
464 if (!(cfi.fsr & ARM_SMMU_CB_FSR_FAULT))
465 return IRQ_NONE;
466
467 ret = report_iommu_fault(&smmu_domain->domain, NULL, cfi.iova,
468 cfi.fsynr & ARM_SMMU_CB_FSYNR0_WNR ? IOMMU_FAULT_WRITE : IOMMU_FAULT_READ);
469
470 if (ret == -ENOSYS && __ratelimit(&rs))
471 arm_smmu_print_context_fault_info(smmu, idx, &cfi);
472
473 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_FSR, cfi.fsr);
474 return IRQ_HANDLED;
475 }
476
arm_smmu_global_fault(int irq,void * dev)477 static irqreturn_t arm_smmu_global_fault(int irq, void *dev)
478 {
479 u32 gfsr, gfsynr0, gfsynr1, gfsynr2;
480 struct arm_smmu_device *smmu = dev;
481 static DEFINE_RATELIMIT_STATE(rs, DEFAULT_RATELIMIT_INTERVAL,
482 DEFAULT_RATELIMIT_BURST);
483
484 gfsr = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sGFSR);
485 gfsynr0 = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sGFSYNR0);
486 gfsynr1 = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sGFSYNR1);
487 gfsynr2 = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sGFSYNR2);
488
489 if (!gfsr)
490 return IRQ_NONE;
491
492 if (__ratelimit(&rs)) {
493 if (IS_ENABLED(CONFIG_ARM_SMMU_DISABLE_BYPASS_BY_DEFAULT) &&
494 (gfsr & ARM_SMMU_sGFSR_USF))
495 dev_err(smmu->dev,
496 "Blocked unknown Stream ID 0x%hx; boot with \"arm-smmu.disable_bypass=0\" to allow, but this may have security implications\n",
497 (u16)gfsynr1);
498 else
499 dev_err(smmu->dev,
500 "Unexpected global fault, this could be serious\n");
501 dev_err(smmu->dev,
502 "\tGFSR 0x%08x, GFSYNR0 0x%08x, GFSYNR1 0x%08x, GFSYNR2 0x%08x\n",
503 gfsr, gfsynr0, gfsynr1, gfsynr2);
504 }
505
506 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_sGFSR, gfsr);
507 return IRQ_HANDLED;
508 }
509
arm_smmu_init_context_bank(struct arm_smmu_domain * smmu_domain,struct io_pgtable_cfg * pgtbl_cfg)510 static void arm_smmu_init_context_bank(struct arm_smmu_domain *smmu_domain,
511 struct io_pgtable_cfg *pgtbl_cfg)
512 {
513 struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
514 struct arm_smmu_cb *cb = &smmu_domain->smmu->cbs[cfg->cbndx];
515 bool stage1 = cfg->cbar != CBAR_TYPE_S2_TRANS;
516
517 cb->cfg = cfg;
518
519 /* TCR */
520 if (stage1) {
521 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) {
522 cb->tcr[0] = pgtbl_cfg->arm_v7s_cfg.tcr;
523 } else {
524 cb->tcr[0] = arm_smmu_lpae_tcr(pgtbl_cfg);
525 cb->tcr[1] = arm_smmu_lpae_tcr2(pgtbl_cfg);
526 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64)
527 cb->tcr[1] |= ARM_SMMU_TCR2_AS;
528 else
529 cb->tcr[0] |= ARM_SMMU_TCR_EAE;
530 }
531 } else {
532 cb->tcr[0] = arm_smmu_lpae_vtcr(pgtbl_cfg);
533 }
534
535 /* TTBRs */
536 if (stage1) {
537 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) {
538 cb->ttbr[0] = pgtbl_cfg->arm_v7s_cfg.ttbr;
539 cb->ttbr[1] = 0;
540 } else {
541 cb->ttbr[0] = FIELD_PREP(ARM_SMMU_TTBRn_ASID,
542 cfg->asid);
543 cb->ttbr[1] = FIELD_PREP(ARM_SMMU_TTBRn_ASID,
544 cfg->asid);
545
546 if (pgtbl_cfg->quirks & IO_PGTABLE_QUIRK_ARM_TTBR1)
547 cb->ttbr[1] |= pgtbl_cfg->arm_lpae_s1_cfg.ttbr;
548 else
549 cb->ttbr[0] |= pgtbl_cfg->arm_lpae_s1_cfg.ttbr;
550 }
551 } else {
552 cb->ttbr[0] = pgtbl_cfg->arm_lpae_s2_cfg.vttbr;
553 }
554
555 /* MAIRs (stage-1 only) */
556 if (stage1) {
557 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) {
558 cb->mair[0] = pgtbl_cfg->arm_v7s_cfg.prrr;
559 cb->mair[1] = pgtbl_cfg->arm_v7s_cfg.nmrr;
560 } else {
561 cb->mair[0] = pgtbl_cfg->arm_lpae_s1_cfg.mair;
562 cb->mair[1] = pgtbl_cfg->arm_lpae_s1_cfg.mair >> 32;
563 }
564 }
565 }
566
arm_smmu_write_context_bank(struct arm_smmu_device * smmu,int idx)567 void arm_smmu_write_context_bank(struct arm_smmu_device *smmu, int idx)
568 {
569 u32 reg;
570 bool stage1;
571 struct arm_smmu_cb *cb = &smmu->cbs[idx];
572 struct arm_smmu_cfg *cfg = cb->cfg;
573
574 /* Unassigned context banks only need disabling */
575 if (!cfg) {
576 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_SCTLR, 0);
577 return;
578 }
579
580 stage1 = cfg->cbar != CBAR_TYPE_S2_TRANS;
581
582 /* CBA2R */
583 if (smmu->version > ARM_SMMU_V1) {
584 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64)
585 reg = ARM_SMMU_CBA2R_VA64;
586 else
587 reg = 0;
588 /* 16-bit VMIDs live in CBA2R */
589 if (smmu->features & ARM_SMMU_FEAT_VMID16)
590 reg |= FIELD_PREP(ARM_SMMU_CBA2R_VMID16, cfg->vmid);
591
592 arm_smmu_gr1_write(smmu, ARM_SMMU_GR1_CBA2R(idx), reg);
593 }
594
595 /* CBAR */
596 reg = FIELD_PREP(ARM_SMMU_CBAR_TYPE, cfg->cbar);
597 if (smmu->version < ARM_SMMU_V2)
598 reg |= FIELD_PREP(ARM_SMMU_CBAR_IRPTNDX, cfg->irptndx);
599
600 /*
601 * Use the weakest shareability/memory types, so they are
602 * overridden by the ttbcr/pte.
603 */
604 if (stage1) {
605 reg |= FIELD_PREP(ARM_SMMU_CBAR_S1_BPSHCFG,
606 ARM_SMMU_CBAR_S1_BPSHCFG_NSH) |
607 FIELD_PREP(ARM_SMMU_CBAR_S1_MEMATTR,
608 ARM_SMMU_CBAR_S1_MEMATTR_WB);
609 } else if (!(smmu->features & ARM_SMMU_FEAT_VMID16)) {
610 /* 8-bit VMIDs live in CBAR */
611 reg |= FIELD_PREP(ARM_SMMU_CBAR_VMID, cfg->vmid);
612 }
613 arm_smmu_gr1_write(smmu, ARM_SMMU_GR1_CBAR(idx), reg);
614
615 /*
616 * TCR
617 * We must write this before the TTBRs, since it determines the
618 * access behaviour of some fields (in particular, ASID[15:8]).
619 */
620 if (stage1 && smmu->version > ARM_SMMU_V1)
621 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_TCR2, cb->tcr[1]);
622 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_TCR, cb->tcr[0]);
623
624 /* TTBRs */
625 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_S) {
626 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_CONTEXTIDR, cfg->asid);
627 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_TTBR0, cb->ttbr[0]);
628 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_TTBR1, cb->ttbr[1]);
629 } else {
630 arm_smmu_cb_writeq(smmu, idx, ARM_SMMU_CB_TTBR0, cb->ttbr[0]);
631 if (stage1)
632 arm_smmu_cb_writeq(smmu, idx, ARM_SMMU_CB_TTBR1,
633 cb->ttbr[1]);
634 }
635
636 /* MAIRs (stage-1 only) */
637 if (stage1) {
638 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_S1_MAIR0, cb->mair[0]);
639 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_S1_MAIR1, cb->mair[1]);
640 }
641
642 /* SCTLR */
643 reg = ARM_SMMU_SCTLR_CFIE | ARM_SMMU_SCTLR_CFRE | ARM_SMMU_SCTLR_AFE |
644 ARM_SMMU_SCTLR_TRE | ARM_SMMU_SCTLR_M;
645 if (stage1)
646 reg |= ARM_SMMU_SCTLR_S1_ASIDPNE;
647 if (IS_ENABLED(CONFIG_CPU_BIG_ENDIAN))
648 reg |= ARM_SMMU_SCTLR_E;
649
650 if (smmu->impl && smmu->impl->write_sctlr)
651 smmu->impl->write_sctlr(smmu, idx, reg);
652 else
653 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_SCTLR, reg);
654 }
655
arm_smmu_alloc_context_bank(struct arm_smmu_domain * smmu_domain,struct arm_smmu_device * smmu,struct device * dev,unsigned int start)656 static int arm_smmu_alloc_context_bank(struct arm_smmu_domain *smmu_domain,
657 struct arm_smmu_device *smmu,
658 struct device *dev, unsigned int start)
659 {
660 if (smmu->impl && smmu->impl->alloc_context_bank)
661 return smmu->impl->alloc_context_bank(smmu_domain, smmu, dev, start);
662
663 return __arm_smmu_alloc_bitmap(smmu->context_map, start, smmu->num_context_banks);
664 }
665
arm_smmu_init_domain_context(struct arm_smmu_domain * smmu_domain,struct arm_smmu_device * smmu,struct device * dev)666 static int arm_smmu_init_domain_context(struct arm_smmu_domain *smmu_domain,
667 struct arm_smmu_device *smmu,
668 struct device *dev)
669 {
670 int irq, start, ret = 0;
671 unsigned long ias, oas;
672 struct io_pgtable_ops *pgtbl_ops;
673 struct io_pgtable_cfg pgtbl_cfg;
674 enum io_pgtable_fmt fmt;
675 struct iommu_domain *domain = &smmu_domain->domain;
676 struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
677 irqreturn_t (*context_fault)(int irq, void *dev);
678
679 mutex_lock(&smmu_domain->init_mutex);
680 if (smmu_domain->smmu)
681 goto out_unlock;
682
683 /*
684 * Mapping the requested stage onto what we support is surprisingly
685 * complicated, mainly because the spec allows S1+S2 SMMUs without
686 * support for nested translation. That means we end up with the
687 * following table:
688 *
689 * Requested Supported Actual
690 * S1 N S1
691 * S1 S1+S2 S1
692 * S1 S2 S2
693 * S1 S1 S1
694 * N N N
695 * N S1+S2 S2
696 * N S2 S2
697 * N S1 S1
698 *
699 * Note that you can't actually request stage-2 mappings.
700 */
701 if (!(smmu->features & ARM_SMMU_FEAT_TRANS_S1))
702 smmu_domain->stage = ARM_SMMU_DOMAIN_S2;
703 if (!(smmu->features & ARM_SMMU_FEAT_TRANS_S2))
704 smmu_domain->stage = ARM_SMMU_DOMAIN_S1;
705
706 /*
707 * Choosing a suitable context format is even more fiddly. Until we
708 * grow some way for the caller to express a preference, and/or move
709 * the decision into the io-pgtable code where it arguably belongs,
710 * just aim for the closest thing to the rest of the system, and hope
711 * that the hardware isn't esoteric enough that we can't assume AArch64
712 * support to be a superset of AArch32 support...
713 */
714 if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH32_L)
715 cfg->fmt = ARM_SMMU_CTX_FMT_AARCH32_L;
716 if (IS_ENABLED(CONFIG_IOMMU_IO_PGTABLE_ARMV7S) &&
717 !IS_ENABLED(CONFIG_64BIT) && !IS_ENABLED(CONFIG_ARM_LPAE) &&
718 (smmu->features & ARM_SMMU_FEAT_FMT_AARCH32_S) &&
719 (smmu_domain->stage == ARM_SMMU_DOMAIN_S1))
720 cfg->fmt = ARM_SMMU_CTX_FMT_AARCH32_S;
721 if ((IS_ENABLED(CONFIG_64BIT) || cfg->fmt == ARM_SMMU_CTX_FMT_NONE) &&
722 (smmu->features & (ARM_SMMU_FEAT_FMT_AARCH64_64K |
723 ARM_SMMU_FEAT_FMT_AARCH64_16K |
724 ARM_SMMU_FEAT_FMT_AARCH64_4K)))
725 cfg->fmt = ARM_SMMU_CTX_FMT_AARCH64;
726
727 if (cfg->fmt == ARM_SMMU_CTX_FMT_NONE) {
728 ret = -EINVAL;
729 goto out_unlock;
730 }
731
732 switch (smmu_domain->stage) {
733 case ARM_SMMU_DOMAIN_S1:
734 cfg->cbar = CBAR_TYPE_S1_TRANS_S2_BYPASS;
735 start = smmu->num_s2_context_banks;
736 ias = smmu->va_size;
737 oas = smmu->ipa_size;
738 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64) {
739 fmt = ARM_64_LPAE_S1;
740 } else if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH32_L) {
741 fmt = ARM_32_LPAE_S1;
742 ias = min(ias, 32UL);
743 oas = min(oas, 40UL);
744 } else {
745 fmt = ARM_V7S;
746 ias = min(ias, 32UL);
747 oas = min(oas, 32UL);
748 }
749 smmu_domain->flush_ops = &arm_smmu_s1_tlb_ops;
750 break;
751 case ARM_SMMU_DOMAIN_NESTED:
752 /*
753 * We will likely want to change this if/when KVM gets
754 * involved.
755 */
756 case ARM_SMMU_DOMAIN_S2:
757 cfg->cbar = CBAR_TYPE_S2_TRANS;
758 start = 0;
759 ias = smmu->ipa_size;
760 oas = smmu->pa_size;
761 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64) {
762 fmt = ARM_64_LPAE_S2;
763 } else {
764 fmt = ARM_32_LPAE_S2;
765 ias = min(ias, 40UL);
766 oas = min(oas, 40UL);
767 }
768 if (smmu->version == ARM_SMMU_V2)
769 smmu_domain->flush_ops = &arm_smmu_s2_tlb_ops_v2;
770 else
771 smmu_domain->flush_ops = &arm_smmu_s2_tlb_ops_v1;
772 break;
773 default:
774 ret = -EINVAL;
775 goto out_unlock;
776 }
777
778 ret = arm_smmu_alloc_context_bank(smmu_domain, smmu, dev, start);
779 if (ret < 0) {
780 goto out_unlock;
781 }
782
783 smmu_domain->smmu = smmu;
784
785 cfg->cbndx = ret;
786 if (smmu->version < ARM_SMMU_V2) {
787 cfg->irptndx = atomic_inc_return(&smmu->irptndx);
788 cfg->irptndx %= smmu->num_context_irqs;
789 } else {
790 cfg->irptndx = cfg->cbndx;
791 }
792
793 if (smmu_domain->stage == ARM_SMMU_DOMAIN_S2)
794 cfg->vmid = cfg->cbndx + 1;
795 else
796 cfg->asid = cfg->cbndx;
797
798 pgtbl_cfg = (struct io_pgtable_cfg) {
799 .pgsize_bitmap = smmu->pgsize_bitmap,
800 .ias = ias,
801 .oas = oas,
802 .coherent_walk = smmu->features & ARM_SMMU_FEAT_COHERENT_WALK,
803 .tlb = smmu_domain->flush_ops,
804 .iommu_dev = smmu->dev,
805 };
806
807 if (smmu->impl && smmu->impl->init_context) {
808 ret = smmu->impl->init_context(smmu_domain, &pgtbl_cfg, dev);
809 if (ret)
810 goto out_clear_smmu;
811 }
812
813 if (smmu_domain->pgtbl_quirks)
814 pgtbl_cfg.quirks |= smmu_domain->pgtbl_quirks;
815
816 pgtbl_ops = alloc_io_pgtable_ops(fmt, &pgtbl_cfg, smmu_domain);
817 if (!pgtbl_ops) {
818 ret = -ENOMEM;
819 goto out_clear_smmu;
820 }
821
822 /* Update the domain's page sizes to reflect the page table format */
823 domain->pgsize_bitmap = pgtbl_cfg.pgsize_bitmap;
824
825 if (pgtbl_cfg.quirks & IO_PGTABLE_QUIRK_ARM_TTBR1) {
826 domain->geometry.aperture_start = ~0UL << ias;
827 domain->geometry.aperture_end = ~0UL;
828 } else {
829 domain->geometry.aperture_end = (1UL << ias) - 1;
830 }
831
832 domain->geometry.force_aperture = true;
833
834 /* Initialise the context bank with our page table cfg */
835 arm_smmu_init_context_bank(smmu_domain, &pgtbl_cfg);
836 arm_smmu_write_context_bank(smmu, cfg->cbndx);
837
838 /*
839 * Request context fault interrupt. Do this last to avoid the
840 * handler seeing a half-initialised domain state.
841 */
842 irq = smmu->irqs[cfg->irptndx];
843
844 if (smmu->impl && smmu->impl->context_fault)
845 context_fault = smmu->impl->context_fault;
846 else
847 context_fault = arm_smmu_context_fault;
848
849 if (smmu->impl && smmu->impl->context_fault_needs_threaded_irq)
850 ret = devm_request_threaded_irq(smmu->dev, irq, NULL,
851 context_fault,
852 IRQF_ONESHOT | IRQF_SHARED,
853 "arm-smmu-context-fault",
854 smmu_domain);
855 else
856 ret = devm_request_irq(smmu->dev, irq, context_fault, IRQF_SHARED,
857 "arm-smmu-context-fault", smmu_domain);
858
859 if (ret < 0) {
860 dev_err(smmu->dev, "failed to request context IRQ %d (%u)\n",
861 cfg->irptndx, irq);
862 cfg->irptndx = ARM_SMMU_INVALID_IRPTNDX;
863 }
864
865 mutex_unlock(&smmu_domain->init_mutex);
866
867 /* Publish page table ops for map/unmap */
868 smmu_domain->pgtbl_ops = pgtbl_ops;
869 return 0;
870
871 out_clear_smmu:
872 __arm_smmu_free_bitmap(smmu->context_map, cfg->cbndx);
873 smmu_domain->smmu = NULL;
874 out_unlock:
875 mutex_unlock(&smmu_domain->init_mutex);
876 return ret;
877 }
878
arm_smmu_destroy_domain_context(struct arm_smmu_domain * smmu_domain)879 static void arm_smmu_destroy_domain_context(struct arm_smmu_domain *smmu_domain)
880 {
881 struct arm_smmu_device *smmu = smmu_domain->smmu;
882 struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
883 int ret, irq;
884
885 if (!smmu)
886 return;
887
888 ret = arm_smmu_rpm_get(smmu);
889 if (ret < 0)
890 return;
891
892 /*
893 * Disable the context bank and free the page tables before freeing
894 * it.
895 */
896 smmu->cbs[cfg->cbndx].cfg = NULL;
897 arm_smmu_write_context_bank(smmu, cfg->cbndx);
898
899 if (cfg->irptndx != ARM_SMMU_INVALID_IRPTNDX) {
900 irq = smmu->irqs[cfg->irptndx];
901 devm_free_irq(smmu->dev, irq, smmu_domain);
902 }
903
904 free_io_pgtable_ops(smmu_domain->pgtbl_ops);
905 __arm_smmu_free_bitmap(smmu->context_map, cfg->cbndx);
906
907 arm_smmu_rpm_put(smmu);
908 }
909
arm_smmu_domain_alloc_paging(struct device * dev)910 static struct iommu_domain *arm_smmu_domain_alloc_paging(struct device *dev)
911 {
912 struct arm_smmu_domain *smmu_domain;
913
914 /*
915 * Allocate the domain and initialise some of its data structures.
916 * We can't really do anything meaningful until we've added a
917 * master.
918 */
919 smmu_domain = kzalloc(sizeof(*smmu_domain), GFP_KERNEL);
920 if (!smmu_domain)
921 return NULL;
922
923 mutex_init(&smmu_domain->init_mutex);
924 spin_lock_init(&smmu_domain->cb_lock);
925
926 return &smmu_domain->domain;
927 }
928
arm_smmu_domain_free(struct iommu_domain * domain)929 static void arm_smmu_domain_free(struct iommu_domain *domain)
930 {
931 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
932
933 /*
934 * Free the domain resources. We assume that all devices have
935 * already been detached.
936 */
937 arm_smmu_destroy_domain_context(smmu_domain);
938 kfree(smmu_domain);
939 }
940
arm_smmu_write_smr(struct arm_smmu_device * smmu,int idx)941 static void arm_smmu_write_smr(struct arm_smmu_device *smmu, int idx)
942 {
943 struct arm_smmu_smr *smr = smmu->smrs + idx;
944 u32 reg = FIELD_PREP(ARM_SMMU_SMR_ID, smr->id) |
945 FIELD_PREP(ARM_SMMU_SMR_MASK, smr->mask);
946
947 if (!(smmu->features & ARM_SMMU_FEAT_EXIDS) && smr->valid)
948 reg |= ARM_SMMU_SMR_VALID;
949 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_SMR(idx), reg);
950 }
951
arm_smmu_write_s2cr(struct arm_smmu_device * smmu,int idx)952 static void arm_smmu_write_s2cr(struct arm_smmu_device *smmu, int idx)
953 {
954 struct arm_smmu_s2cr *s2cr = smmu->s2crs + idx;
955 u32 reg;
956
957 if (smmu->impl && smmu->impl->write_s2cr) {
958 smmu->impl->write_s2cr(smmu, idx);
959 return;
960 }
961
962 reg = FIELD_PREP(ARM_SMMU_S2CR_TYPE, s2cr->type) |
963 FIELD_PREP(ARM_SMMU_S2CR_CBNDX, s2cr->cbndx) |
964 FIELD_PREP(ARM_SMMU_S2CR_PRIVCFG, s2cr->privcfg);
965
966 if (smmu->features & ARM_SMMU_FEAT_EXIDS && smmu->smrs &&
967 smmu->smrs[idx].valid)
968 reg |= ARM_SMMU_S2CR_EXIDVALID;
969 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_S2CR(idx), reg);
970 }
971
arm_smmu_write_sme(struct arm_smmu_device * smmu,int idx)972 static void arm_smmu_write_sme(struct arm_smmu_device *smmu, int idx)
973 {
974 arm_smmu_write_s2cr(smmu, idx);
975 if (smmu->smrs)
976 arm_smmu_write_smr(smmu, idx);
977 }
978
979 /*
980 * The width of SMR's mask field depends on sCR0_EXIDENABLE, so this function
981 * should be called after sCR0 is written.
982 */
arm_smmu_test_smr_masks(struct arm_smmu_device * smmu)983 static void arm_smmu_test_smr_masks(struct arm_smmu_device *smmu)
984 {
985 u32 smr;
986 int i;
987
988 if (!smmu->smrs)
989 return;
990 /*
991 * If we've had to accommodate firmware memory regions, we may
992 * have live SMRs by now; tread carefully...
993 *
994 * Somewhat perversely, not having a free SMR for this test implies we
995 * can get away without it anyway, as we'll only be able to 'allocate'
996 * these SMRs for the ID/mask values we're already trusting to be OK.
997 */
998 for (i = 0; i < smmu->num_mapping_groups; i++)
999 if (!smmu->smrs[i].valid)
1000 goto smr_ok;
1001 return;
1002 smr_ok:
1003 /*
1004 * SMR.ID bits may not be preserved if the corresponding MASK
1005 * bits are set, so check each one separately. We can reject
1006 * masters later if they try to claim IDs outside these masks.
1007 */
1008 smr = FIELD_PREP(ARM_SMMU_SMR_ID, smmu->streamid_mask);
1009 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_SMR(i), smr);
1010 smr = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_SMR(i));
1011 smmu->streamid_mask = FIELD_GET(ARM_SMMU_SMR_ID, smr);
1012
1013 smr = FIELD_PREP(ARM_SMMU_SMR_MASK, smmu->streamid_mask);
1014 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_SMR(i), smr);
1015 smr = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_SMR(i));
1016 smmu->smr_mask_mask = FIELD_GET(ARM_SMMU_SMR_MASK, smr);
1017 }
1018
arm_smmu_find_sme(struct arm_smmu_device * smmu,u16 id,u16 mask)1019 static int arm_smmu_find_sme(struct arm_smmu_device *smmu, u16 id, u16 mask)
1020 {
1021 struct arm_smmu_smr *smrs = smmu->smrs;
1022 int i, free_idx = -ENOSPC;
1023
1024 /* Stream indexing is blissfully easy */
1025 if (!smrs)
1026 return id;
1027
1028 /* Validating SMRs is... less so */
1029 for (i = 0; i < smmu->num_mapping_groups; ++i) {
1030 if (!smrs[i].valid) {
1031 /*
1032 * Note the first free entry we come across, which
1033 * we'll claim in the end if nothing else matches.
1034 */
1035 if (free_idx < 0)
1036 free_idx = i;
1037 continue;
1038 }
1039 /*
1040 * If the new entry is _entirely_ matched by an existing entry,
1041 * then reuse that, with the guarantee that there also cannot
1042 * be any subsequent conflicting entries. In normal use we'd
1043 * expect simply identical entries for this case, but there's
1044 * no harm in accommodating the generalisation.
1045 */
1046 if ((mask & smrs[i].mask) == mask &&
1047 !((id ^ smrs[i].id) & ~smrs[i].mask))
1048 return i;
1049 /*
1050 * If the new entry has any other overlap with an existing one,
1051 * though, then there always exists at least one stream ID
1052 * which would cause a conflict, and we can't allow that risk.
1053 */
1054 if (!((id ^ smrs[i].id) & ~(smrs[i].mask | mask)))
1055 return -EINVAL;
1056 }
1057
1058 return free_idx;
1059 }
1060
arm_smmu_free_sme(struct arm_smmu_device * smmu,int idx)1061 static bool arm_smmu_free_sme(struct arm_smmu_device *smmu, int idx)
1062 {
1063 if (--smmu->s2crs[idx].count)
1064 return false;
1065
1066 smmu->s2crs[idx] = s2cr_init_val;
1067 if (smmu->smrs)
1068 smmu->smrs[idx].valid = false;
1069
1070 return true;
1071 }
1072
arm_smmu_master_alloc_smes(struct device * dev)1073 static int arm_smmu_master_alloc_smes(struct device *dev)
1074 {
1075 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
1076 struct arm_smmu_master_cfg *cfg = dev_iommu_priv_get(dev);
1077 struct arm_smmu_device *smmu = cfg->smmu;
1078 struct arm_smmu_smr *smrs = smmu->smrs;
1079 int i, idx, ret;
1080
1081 mutex_lock(&smmu->stream_map_mutex);
1082 /* Figure out a viable stream map entry allocation */
1083 for_each_cfg_sme(cfg, fwspec, i, idx) {
1084 u16 sid = FIELD_GET(ARM_SMMU_SMR_ID, fwspec->ids[i]);
1085 u16 mask = FIELD_GET(ARM_SMMU_SMR_MASK, fwspec->ids[i]);
1086
1087 if (idx != INVALID_SMENDX) {
1088 ret = -EEXIST;
1089 goto out_err;
1090 }
1091
1092 ret = arm_smmu_find_sme(smmu, sid, mask);
1093 if (ret < 0)
1094 goto out_err;
1095
1096 idx = ret;
1097 if (smrs && smmu->s2crs[idx].count == 0) {
1098 smrs[idx].id = sid;
1099 smrs[idx].mask = mask;
1100 smrs[idx].valid = true;
1101 }
1102 smmu->s2crs[idx].count++;
1103 cfg->smendx[i] = (s16)idx;
1104 }
1105
1106 /* It worked! Now, poke the actual hardware */
1107 for_each_cfg_sme(cfg, fwspec, i, idx)
1108 arm_smmu_write_sme(smmu, idx);
1109
1110 mutex_unlock(&smmu->stream_map_mutex);
1111 return 0;
1112
1113 out_err:
1114 while (i--) {
1115 arm_smmu_free_sme(smmu, cfg->smendx[i]);
1116 cfg->smendx[i] = INVALID_SMENDX;
1117 }
1118 mutex_unlock(&smmu->stream_map_mutex);
1119 return ret;
1120 }
1121
arm_smmu_master_free_smes(struct arm_smmu_master_cfg * cfg,struct iommu_fwspec * fwspec)1122 static void arm_smmu_master_free_smes(struct arm_smmu_master_cfg *cfg,
1123 struct iommu_fwspec *fwspec)
1124 {
1125 struct arm_smmu_device *smmu = cfg->smmu;
1126 int i, idx;
1127
1128 mutex_lock(&smmu->stream_map_mutex);
1129 for_each_cfg_sme(cfg, fwspec, i, idx) {
1130 if (arm_smmu_free_sme(smmu, idx))
1131 arm_smmu_write_sme(smmu, idx);
1132 cfg->smendx[i] = INVALID_SMENDX;
1133 }
1134 mutex_unlock(&smmu->stream_map_mutex);
1135 }
1136
arm_smmu_master_install_s2crs(struct arm_smmu_master_cfg * cfg,enum arm_smmu_s2cr_type type,u8 cbndx,struct iommu_fwspec * fwspec)1137 static void arm_smmu_master_install_s2crs(struct arm_smmu_master_cfg *cfg,
1138 enum arm_smmu_s2cr_type type,
1139 u8 cbndx, struct iommu_fwspec *fwspec)
1140 {
1141 struct arm_smmu_device *smmu = cfg->smmu;
1142 struct arm_smmu_s2cr *s2cr = smmu->s2crs;
1143 int i, idx;
1144
1145 for_each_cfg_sme(cfg, fwspec, i, idx) {
1146 if (type == s2cr[idx].type && cbndx == s2cr[idx].cbndx)
1147 continue;
1148
1149 s2cr[idx].type = type;
1150 s2cr[idx].privcfg = S2CR_PRIVCFG_DEFAULT;
1151 s2cr[idx].cbndx = cbndx;
1152 arm_smmu_write_s2cr(smmu, idx);
1153 }
1154 }
1155
arm_smmu_attach_dev(struct iommu_domain * domain,struct device * dev)1156 static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev)
1157 {
1158 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1159 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
1160 struct arm_smmu_master_cfg *cfg;
1161 struct arm_smmu_device *smmu;
1162 int ret;
1163
1164 /*
1165 * FIXME: The arch/arm DMA API code tries to attach devices to its own
1166 * domains between of_xlate() and probe_device() - we have no way to cope
1167 * with that, so until ARM gets converted to rely on groups and default
1168 * domains, just say no (but more politely than by dereferencing NULL).
1169 * This should be at least a WARN_ON once that's sorted.
1170 */
1171 cfg = dev_iommu_priv_get(dev);
1172 if (!cfg)
1173 return -ENODEV;
1174
1175 smmu = cfg->smmu;
1176
1177 ret = arm_smmu_rpm_get(smmu);
1178 if (ret < 0)
1179 return ret;
1180
1181 /* Ensure that the domain is finalised */
1182 ret = arm_smmu_init_domain_context(smmu_domain, smmu, dev);
1183 if (ret < 0)
1184 goto rpm_put;
1185
1186 /*
1187 * Sanity check the domain. We don't support domains across
1188 * different SMMUs.
1189 */
1190 if (smmu_domain->smmu != smmu) {
1191 ret = -EINVAL;
1192 goto rpm_put;
1193 }
1194
1195 /* Looks ok, so add the device to the domain */
1196 arm_smmu_master_install_s2crs(cfg, S2CR_TYPE_TRANS,
1197 smmu_domain->cfg.cbndx, fwspec);
1198 arm_smmu_rpm_use_autosuspend(smmu);
1199 rpm_put:
1200 arm_smmu_rpm_put(smmu);
1201 return ret;
1202 }
1203
arm_smmu_attach_dev_type(struct device * dev,enum arm_smmu_s2cr_type type)1204 static int arm_smmu_attach_dev_type(struct device *dev,
1205 enum arm_smmu_s2cr_type type)
1206 {
1207 struct arm_smmu_master_cfg *cfg = dev_iommu_priv_get(dev);
1208 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
1209 struct arm_smmu_device *smmu;
1210 int ret;
1211
1212 if (!cfg)
1213 return -ENODEV;
1214 smmu = cfg->smmu;
1215
1216 ret = arm_smmu_rpm_get(smmu);
1217 if (ret < 0)
1218 return ret;
1219
1220 arm_smmu_master_install_s2crs(cfg, type, 0, fwspec);
1221 arm_smmu_rpm_use_autosuspend(smmu);
1222 arm_smmu_rpm_put(smmu);
1223 return 0;
1224 }
1225
arm_smmu_attach_dev_identity(struct iommu_domain * domain,struct device * dev)1226 static int arm_smmu_attach_dev_identity(struct iommu_domain *domain,
1227 struct device *dev)
1228 {
1229 return arm_smmu_attach_dev_type(dev, S2CR_TYPE_BYPASS);
1230 }
1231
1232 static const struct iommu_domain_ops arm_smmu_identity_ops = {
1233 .attach_dev = arm_smmu_attach_dev_identity,
1234 };
1235
1236 static struct iommu_domain arm_smmu_identity_domain = {
1237 .type = IOMMU_DOMAIN_IDENTITY,
1238 .ops = &arm_smmu_identity_ops,
1239 };
1240
arm_smmu_attach_dev_blocked(struct iommu_domain * domain,struct device * dev)1241 static int arm_smmu_attach_dev_blocked(struct iommu_domain *domain,
1242 struct device *dev)
1243 {
1244 return arm_smmu_attach_dev_type(dev, S2CR_TYPE_FAULT);
1245 }
1246
1247 static const struct iommu_domain_ops arm_smmu_blocked_ops = {
1248 .attach_dev = arm_smmu_attach_dev_blocked,
1249 };
1250
1251 static struct iommu_domain arm_smmu_blocked_domain = {
1252 .type = IOMMU_DOMAIN_BLOCKED,
1253 .ops = &arm_smmu_blocked_ops,
1254 };
1255
arm_smmu_map_pages(struct iommu_domain * domain,unsigned long iova,phys_addr_t paddr,size_t pgsize,size_t pgcount,int prot,gfp_t gfp,size_t * mapped)1256 static int arm_smmu_map_pages(struct iommu_domain *domain, unsigned long iova,
1257 phys_addr_t paddr, size_t pgsize, size_t pgcount,
1258 int prot, gfp_t gfp, size_t *mapped)
1259 {
1260 struct io_pgtable_ops *ops = to_smmu_domain(domain)->pgtbl_ops;
1261 struct arm_smmu_device *smmu = to_smmu_domain(domain)->smmu;
1262 int ret;
1263
1264 if (!ops)
1265 return -ENODEV;
1266
1267 arm_smmu_rpm_get(smmu);
1268 ret = ops->map_pages(ops, iova, paddr, pgsize, pgcount, prot, gfp, mapped);
1269 arm_smmu_rpm_put(smmu);
1270
1271 return ret;
1272 }
1273
arm_smmu_unmap_pages(struct iommu_domain * domain,unsigned long iova,size_t pgsize,size_t pgcount,struct iommu_iotlb_gather * iotlb_gather)1274 static size_t arm_smmu_unmap_pages(struct iommu_domain *domain, unsigned long iova,
1275 size_t pgsize, size_t pgcount,
1276 struct iommu_iotlb_gather *iotlb_gather)
1277 {
1278 struct io_pgtable_ops *ops = to_smmu_domain(domain)->pgtbl_ops;
1279 struct arm_smmu_device *smmu = to_smmu_domain(domain)->smmu;
1280 size_t ret;
1281
1282 if (!ops)
1283 return 0;
1284
1285 arm_smmu_rpm_get(smmu);
1286 ret = ops->unmap_pages(ops, iova, pgsize, pgcount, iotlb_gather);
1287 arm_smmu_rpm_put(smmu);
1288
1289 return ret;
1290 }
1291
arm_smmu_flush_iotlb_all(struct iommu_domain * domain)1292 static void arm_smmu_flush_iotlb_all(struct iommu_domain *domain)
1293 {
1294 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1295 struct arm_smmu_device *smmu = smmu_domain->smmu;
1296
1297 if (smmu_domain->flush_ops) {
1298 arm_smmu_rpm_get(smmu);
1299 smmu_domain->flush_ops->tlb_flush_all(smmu_domain);
1300 arm_smmu_rpm_put(smmu);
1301 }
1302 }
1303
arm_smmu_iotlb_sync(struct iommu_domain * domain,struct iommu_iotlb_gather * gather)1304 static void arm_smmu_iotlb_sync(struct iommu_domain *domain,
1305 struct iommu_iotlb_gather *gather)
1306 {
1307 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1308 struct arm_smmu_device *smmu = smmu_domain->smmu;
1309
1310 if (!smmu)
1311 return;
1312
1313 arm_smmu_rpm_get(smmu);
1314 if (smmu->version == ARM_SMMU_V2 ||
1315 smmu_domain->stage == ARM_SMMU_DOMAIN_S1)
1316 arm_smmu_tlb_sync_context(smmu_domain);
1317 else
1318 arm_smmu_tlb_sync_global(smmu);
1319 arm_smmu_rpm_put(smmu);
1320 }
1321
arm_smmu_iova_to_phys_hard(struct iommu_domain * domain,dma_addr_t iova)1322 static phys_addr_t arm_smmu_iova_to_phys_hard(struct iommu_domain *domain,
1323 dma_addr_t iova)
1324 {
1325 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1326 struct arm_smmu_device *smmu = smmu_domain->smmu;
1327 struct arm_smmu_cfg *cfg = &smmu_domain->cfg;
1328 struct io_pgtable_ops *ops= smmu_domain->pgtbl_ops;
1329 struct device *dev = smmu->dev;
1330 void __iomem *reg;
1331 u32 tmp;
1332 u64 phys;
1333 unsigned long va, flags;
1334 int ret, idx = cfg->cbndx;
1335 phys_addr_t addr = 0;
1336
1337 ret = arm_smmu_rpm_get(smmu);
1338 if (ret < 0)
1339 return 0;
1340
1341 spin_lock_irqsave(&smmu_domain->cb_lock, flags);
1342 va = iova & ~0xfffUL;
1343 if (cfg->fmt == ARM_SMMU_CTX_FMT_AARCH64)
1344 arm_smmu_cb_writeq(smmu, idx, ARM_SMMU_CB_ATS1PR, va);
1345 else
1346 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_ATS1PR, va);
1347
1348 reg = arm_smmu_page(smmu, ARM_SMMU_CB(smmu, idx)) + ARM_SMMU_CB_ATSR;
1349 if (readl_poll_timeout_atomic(reg, tmp, !(tmp & ARM_SMMU_CB_ATSR_ACTIVE),
1350 5, 50)) {
1351 spin_unlock_irqrestore(&smmu_domain->cb_lock, flags);
1352 dev_err(dev,
1353 "iova to phys timed out on %pad. Falling back to software table walk.\n",
1354 &iova);
1355 arm_smmu_rpm_put(smmu);
1356 return ops->iova_to_phys(ops, iova);
1357 }
1358
1359 phys = arm_smmu_cb_readq(smmu, idx, ARM_SMMU_CB_PAR);
1360 spin_unlock_irqrestore(&smmu_domain->cb_lock, flags);
1361 if (phys & ARM_SMMU_CB_PAR_F) {
1362 dev_err(dev, "translation fault!\n");
1363 dev_err(dev, "PAR = 0x%llx\n", phys);
1364 goto out;
1365 }
1366
1367 addr = (phys & GENMASK_ULL(39, 12)) | (iova & 0xfff);
1368 out:
1369 arm_smmu_rpm_put(smmu);
1370
1371 return addr;
1372 }
1373
arm_smmu_iova_to_phys(struct iommu_domain * domain,dma_addr_t iova)1374 static phys_addr_t arm_smmu_iova_to_phys(struct iommu_domain *domain,
1375 dma_addr_t iova)
1376 {
1377 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1378 struct io_pgtable_ops *ops = smmu_domain->pgtbl_ops;
1379
1380 if (!ops)
1381 return 0;
1382
1383 if (smmu_domain->smmu->features & ARM_SMMU_FEAT_TRANS_OPS &&
1384 smmu_domain->stage == ARM_SMMU_DOMAIN_S1)
1385 return arm_smmu_iova_to_phys_hard(domain, iova);
1386
1387 return ops->iova_to_phys(ops, iova);
1388 }
1389
arm_smmu_capable(struct device * dev,enum iommu_cap cap)1390 static bool arm_smmu_capable(struct device *dev, enum iommu_cap cap)
1391 {
1392 struct arm_smmu_master_cfg *cfg = dev_iommu_priv_get(dev);
1393
1394 switch (cap) {
1395 case IOMMU_CAP_CACHE_COHERENCY:
1396 /*
1397 * It's overwhelmingly the case in practice that when the pagetable
1398 * walk interface is connected to a coherent interconnect, all the
1399 * translation interfaces are too. Furthermore if the device is
1400 * natively coherent, then its translation interface must also be.
1401 */
1402 return cfg->smmu->features & ARM_SMMU_FEAT_COHERENT_WALK ||
1403 device_get_dma_attr(dev) == DEV_DMA_COHERENT;
1404 case IOMMU_CAP_NOEXEC:
1405 case IOMMU_CAP_DEFERRED_FLUSH:
1406 return true;
1407 default:
1408 return false;
1409 }
1410 }
1411
1412 static
arm_smmu_get_by_fwnode(struct fwnode_handle * fwnode)1413 struct arm_smmu_device *arm_smmu_get_by_fwnode(struct fwnode_handle *fwnode)
1414 {
1415 struct device *dev = bus_find_device_by_fwnode(&platform_bus_type, fwnode);
1416
1417 put_device(dev);
1418 return dev ? dev_get_drvdata(dev) : NULL;
1419 }
1420
arm_smmu_probe_device(struct device * dev)1421 static struct iommu_device *arm_smmu_probe_device(struct device *dev)
1422 {
1423 struct arm_smmu_device *smmu = NULL;
1424 struct arm_smmu_master_cfg *cfg;
1425 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
1426 int i, ret;
1427
1428 if (using_legacy_binding) {
1429 ret = arm_smmu_register_legacy_master(dev, &smmu);
1430
1431 /*
1432 * If dev->iommu_fwspec is initally NULL, arm_smmu_register_legacy_master()
1433 * will allocate/initialise a new one. Thus we need to update fwspec for
1434 * later use.
1435 */
1436 fwspec = dev_iommu_fwspec_get(dev);
1437 if (ret)
1438 goto out_free;
1439 } else {
1440 smmu = arm_smmu_get_by_fwnode(fwspec->iommu_fwnode);
1441 }
1442
1443 ret = -EINVAL;
1444 for (i = 0; i < fwspec->num_ids; i++) {
1445 u16 sid = FIELD_GET(ARM_SMMU_SMR_ID, fwspec->ids[i]);
1446 u16 mask = FIELD_GET(ARM_SMMU_SMR_MASK, fwspec->ids[i]);
1447
1448 if (sid & ~smmu->streamid_mask) {
1449 dev_err(dev, "stream ID 0x%x out of range for SMMU (0x%x)\n",
1450 sid, smmu->streamid_mask);
1451 goto out_free;
1452 }
1453 if (mask & ~smmu->smr_mask_mask) {
1454 dev_err(dev, "SMR mask 0x%x out of range for SMMU (0x%x)\n",
1455 mask, smmu->smr_mask_mask);
1456 goto out_free;
1457 }
1458 }
1459
1460 ret = -ENOMEM;
1461 cfg = kzalloc(offsetof(struct arm_smmu_master_cfg, smendx[i]),
1462 GFP_KERNEL);
1463 if (!cfg)
1464 goto out_free;
1465
1466 cfg->smmu = smmu;
1467 dev_iommu_priv_set(dev, cfg);
1468 while (i--)
1469 cfg->smendx[i] = INVALID_SMENDX;
1470
1471 ret = arm_smmu_rpm_get(smmu);
1472 if (ret < 0)
1473 goto out_cfg_free;
1474
1475 ret = arm_smmu_master_alloc_smes(dev);
1476 arm_smmu_rpm_put(smmu);
1477
1478 if (ret)
1479 goto out_cfg_free;
1480
1481 device_link_add(dev, smmu->dev,
1482 DL_FLAG_PM_RUNTIME | DL_FLAG_AUTOREMOVE_SUPPLIER);
1483
1484 return &smmu->iommu;
1485
1486 out_cfg_free:
1487 kfree(cfg);
1488 out_free:
1489 iommu_fwspec_free(dev);
1490 return ERR_PTR(ret);
1491 }
1492
arm_smmu_release_device(struct device * dev)1493 static void arm_smmu_release_device(struct device *dev)
1494 {
1495 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
1496 struct arm_smmu_master_cfg *cfg = dev_iommu_priv_get(dev);
1497 int ret;
1498
1499 ret = arm_smmu_rpm_get(cfg->smmu);
1500 if (ret < 0)
1501 return;
1502
1503 arm_smmu_master_free_smes(cfg, fwspec);
1504
1505 arm_smmu_rpm_put(cfg->smmu);
1506
1507 kfree(cfg);
1508 }
1509
arm_smmu_probe_finalize(struct device * dev)1510 static void arm_smmu_probe_finalize(struct device *dev)
1511 {
1512 struct arm_smmu_master_cfg *cfg;
1513 struct arm_smmu_device *smmu;
1514
1515 cfg = dev_iommu_priv_get(dev);
1516 smmu = cfg->smmu;
1517
1518 if (smmu->impl && smmu->impl->probe_finalize)
1519 smmu->impl->probe_finalize(smmu, dev);
1520 }
1521
arm_smmu_device_group(struct device * dev)1522 static struct iommu_group *arm_smmu_device_group(struct device *dev)
1523 {
1524 struct arm_smmu_master_cfg *cfg = dev_iommu_priv_get(dev);
1525 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
1526 struct arm_smmu_device *smmu = cfg->smmu;
1527 struct iommu_group *group = NULL;
1528 int i, idx;
1529
1530 mutex_lock(&smmu->stream_map_mutex);
1531 for_each_cfg_sme(cfg, fwspec, i, idx) {
1532 if (group && smmu->s2crs[idx].group &&
1533 group != smmu->s2crs[idx].group) {
1534 mutex_unlock(&smmu->stream_map_mutex);
1535 return ERR_PTR(-EINVAL);
1536 }
1537
1538 group = smmu->s2crs[idx].group;
1539 }
1540
1541 if (group) {
1542 mutex_unlock(&smmu->stream_map_mutex);
1543 return iommu_group_ref_get(group);
1544 }
1545
1546 if (dev_is_pci(dev))
1547 group = pci_device_group(dev);
1548 else if (dev_is_fsl_mc(dev))
1549 group = fsl_mc_device_group(dev);
1550 else
1551 group = generic_device_group(dev);
1552
1553 /* Remember group for faster lookups */
1554 if (!IS_ERR(group))
1555 for_each_cfg_sme(cfg, fwspec, i, idx)
1556 smmu->s2crs[idx].group = group;
1557
1558 mutex_unlock(&smmu->stream_map_mutex);
1559 return group;
1560 }
1561
arm_smmu_set_pgtable_quirks(struct iommu_domain * domain,unsigned long quirks)1562 static int arm_smmu_set_pgtable_quirks(struct iommu_domain *domain,
1563 unsigned long quirks)
1564 {
1565 struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
1566 int ret = 0;
1567
1568 mutex_lock(&smmu_domain->init_mutex);
1569 if (smmu_domain->smmu)
1570 ret = -EPERM;
1571 else
1572 smmu_domain->pgtbl_quirks = quirks;
1573 mutex_unlock(&smmu_domain->init_mutex);
1574
1575 return ret;
1576 }
1577
arm_smmu_of_xlate(struct device * dev,const struct of_phandle_args * args)1578 static int arm_smmu_of_xlate(struct device *dev,
1579 const struct of_phandle_args *args)
1580 {
1581 u32 mask, fwid = 0;
1582
1583 if (args->args_count > 0)
1584 fwid |= FIELD_PREP(ARM_SMMU_SMR_ID, args->args[0]);
1585
1586 if (args->args_count > 1)
1587 fwid |= FIELD_PREP(ARM_SMMU_SMR_MASK, args->args[1]);
1588 else if (!of_property_read_u32(args->np, "stream-match-mask", &mask))
1589 fwid |= FIELD_PREP(ARM_SMMU_SMR_MASK, mask);
1590
1591 return iommu_fwspec_add_ids(dev, &fwid, 1);
1592 }
1593
arm_smmu_get_resv_regions(struct device * dev,struct list_head * head)1594 static void arm_smmu_get_resv_regions(struct device *dev,
1595 struct list_head *head)
1596 {
1597 struct iommu_resv_region *region;
1598 int prot = IOMMU_WRITE | IOMMU_NOEXEC | IOMMU_MMIO;
1599
1600 region = iommu_alloc_resv_region(MSI_IOVA_BASE, MSI_IOVA_LENGTH,
1601 prot, IOMMU_RESV_SW_MSI, GFP_KERNEL);
1602 if (!region)
1603 return;
1604
1605 list_add_tail(®ion->list, head);
1606
1607 iommu_dma_get_resv_regions(dev, head);
1608 }
1609
arm_smmu_def_domain_type(struct device * dev)1610 static int arm_smmu_def_domain_type(struct device *dev)
1611 {
1612 struct arm_smmu_master_cfg *cfg = dev_iommu_priv_get(dev);
1613 const struct arm_smmu_impl *impl = cfg->smmu->impl;
1614
1615 if (using_legacy_binding)
1616 return IOMMU_DOMAIN_IDENTITY;
1617
1618 if (impl && impl->def_domain_type)
1619 return impl->def_domain_type(dev);
1620
1621 return 0;
1622 }
1623
1624 static struct iommu_ops arm_smmu_ops = {
1625 .identity_domain = &arm_smmu_identity_domain,
1626 .blocked_domain = &arm_smmu_blocked_domain,
1627 .capable = arm_smmu_capable,
1628 .domain_alloc_paging = arm_smmu_domain_alloc_paging,
1629 .probe_device = arm_smmu_probe_device,
1630 .release_device = arm_smmu_release_device,
1631 .probe_finalize = arm_smmu_probe_finalize,
1632 .device_group = arm_smmu_device_group,
1633 .of_xlate = arm_smmu_of_xlate,
1634 .get_resv_regions = arm_smmu_get_resv_regions,
1635 .def_domain_type = arm_smmu_def_domain_type,
1636 .pgsize_bitmap = -1UL, /* Restricted during device attach */
1637 .owner = THIS_MODULE,
1638 .default_domain_ops = &(const struct iommu_domain_ops) {
1639 .attach_dev = arm_smmu_attach_dev,
1640 .map_pages = arm_smmu_map_pages,
1641 .unmap_pages = arm_smmu_unmap_pages,
1642 .flush_iotlb_all = arm_smmu_flush_iotlb_all,
1643 .iotlb_sync = arm_smmu_iotlb_sync,
1644 .iova_to_phys = arm_smmu_iova_to_phys,
1645 .set_pgtable_quirks = arm_smmu_set_pgtable_quirks,
1646 .free = arm_smmu_domain_free,
1647 }
1648 };
1649
arm_smmu_device_reset(struct arm_smmu_device * smmu)1650 static void arm_smmu_device_reset(struct arm_smmu_device *smmu)
1651 {
1652 int i;
1653 u32 reg;
1654
1655 /* clear global FSR */
1656 reg = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sGFSR);
1657 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_sGFSR, reg);
1658
1659 /*
1660 * Reset stream mapping groups: Initial values mark all SMRn as
1661 * invalid and all S2CRn as bypass unless overridden.
1662 */
1663 for (i = 0; i < smmu->num_mapping_groups; ++i)
1664 arm_smmu_write_sme(smmu, i);
1665
1666 /* Make sure all context banks are disabled and clear CB_FSR */
1667 for (i = 0; i < smmu->num_context_banks; ++i) {
1668 arm_smmu_write_context_bank(smmu, i);
1669 arm_smmu_cb_write(smmu, i, ARM_SMMU_CB_FSR, ARM_SMMU_CB_FSR_FAULT);
1670 }
1671
1672 /* Invalidate the TLB, just in case */
1673 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_TLBIALLH, QCOM_DUMMY_VAL);
1674 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_TLBIALLNSNH, QCOM_DUMMY_VAL);
1675
1676 reg = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sCR0);
1677
1678 /* Enable fault reporting */
1679 reg |= (ARM_SMMU_sCR0_GFRE | ARM_SMMU_sCR0_GFIE |
1680 ARM_SMMU_sCR0_GCFGFRE | ARM_SMMU_sCR0_GCFGFIE);
1681
1682 /* Disable TLB broadcasting. */
1683 reg |= (ARM_SMMU_sCR0_VMIDPNE | ARM_SMMU_sCR0_PTM);
1684
1685 /* Enable client access, handling unmatched streams as appropriate */
1686 reg &= ~ARM_SMMU_sCR0_CLIENTPD;
1687 if (disable_bypass)
1688 reg |= ARM_SMMU_sCR0_USFCFG;
1689 else
1690 reg &= ~ARM_SMMU_sCR0_USFCFG;
1691
1692 /* Disable forced broadcasting */
1693 reg &= ~ARM_SMMU_sCR0_FB;
1694
1695 /* Don't upgrade barriers */
1696 reg &= ~(ARM_SMMU_sCR0_BSU);
1697
1698 if (smmu->features & ARM_SMMU_FEAT_VMID16)
1699 reg |= ARM_SMMU_sCR0_VMID16EN;
1700
1701 if (smmu->features & ARM_SMMU_FEAT_EXIDS)
1702 reg |= ARM_SMMU_sCR0_EXIDENABLE;
1703
1704 if (smmu->impl && smmu->impl->reset)
1705 smmu->impl->reset(smmu);
1706
1707 /* Push the button */
1708 arm_smmu_tlb_sync_global(smmu);
1709 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_sCR0, reg);
1710 }
1711
arm_smmu_id_size_to_bits(int size)1712 static int arm_smmu_id_size_to_bits(int size)
1713 {
1714 switch (size) {
1715 case 0:
1716 return 32;
1717 case 1:
1718 return 36;
1719 case 2:
1720 return 40;
1721 case 3:
1722 return 42;
1723 case 4:
1724 return 44;
1725 case 5:
1726 default:
1727 return 48;
1728 }
1729 }
1730
arm_smmu_device_cfg_probe(struct arm_smmu_device * smmu)1731 static int arm_smmu_device_cfg_probe(struct arm_smmu_device *smmu)
1732 {
1733 unsigned int size;
1734 u32 id;
1735 bool cttw_reg, cttw_fw = smmu->features & ARM_SMMU_FEAT_COHERENT_WALK;
1736 int i, ret;
1737
1738 dev_notice(smmu->dev, "probing hardware configuration...\n");
1739 dev_notice(smmu->dev, "SMMUv%d with:\n",
1740 smmu->version == ARM_SMMU_V2 ? 2 : 1);
1741
1742 /* ID0 */
1743 id = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_ID0);
1744
1745 /* Restrict available stages based on module parameter */
1746 if (force_stage == 1)
1747 id &= ~(ARM_SMMU_ID0_S2TS | ARM_SMMU_ID0_NTS);
1748 else if (force_stage == 2)
1749 id &= ~(ARM_SMMU_ID0_S1TS | ARM_SMMU_ID0_NTS);
1750
1751 if (id & ARM_SMMU_ID0_S1TS) {
1752 smmu->features |= ARM_SMMU_FEAT_TRANS_S1;
1753 dev_notice(smmu->dev, "\tstage 1 translation\n");
1754 }
1755
1756 if (id & ARM_SMMU_ID0_S2TS) {
1757 smmu->features |= ARM_SMMU_FEAT_TRANS_S2;
1758 dev_notice(smmu->dev, "\tstage 2 translation\n");
1759 }
1760
1761 if (id & ARM_SMMU_ID0_NTS) {
1762 smmu->features |= ARM_SMMU_FEAT_TRANS_NESTED;
1763 dev_notice(smmu->dev, "\tnested translation\n");
1764 }
1765
1766 if (!(smmu->features &
1767 (ARM_SMMU_FEAT_TRANS_S1 | ARM_SMMU_FEAT_TRANS_S2))) {
1768 dev_err(smmu->dev, "\tno translation support!\n");
1769 return -ENODEV;
1770 }
1771
1772 if ((id & ARM_SMMU_ID0_S1TS) &&
1773 ((smmu->version < ARM_SMMU_V2) || !(id & ARM_SMMU_ID0_ATOSNS))) {
1774 smmu->features |= ARM_SMMU_FEAT_TRANS_OPS;
1775 dev_notice(smmu->dev, "\taddress translation ops\n");
1776 }
1777
1778 /*
1779 * In order for DMA API calls to work properly, we must defer to what
1780 * the FW says about coherency, regardless of what the hardware claims.
1781 * Fortunately, this also opens up a workaround for systems where the
1782 * ID register value has ended up configured incorrectly.
1783 */
1784 cttw_reg = !!(id & ARM_SMMU_ID0_CTTW);
1785 if (cttw_fw || cttw_reg)
1786 dev_notice(smmu->dev, "\t%scoherent table walk\n",
1787 cttw_fw ? "" : "non-");
1788 if (cttw_fw != cttw_reg)
1789 dev_notice(smmu->dev,
1790 "\t(IDR0.CTTW overridden by FW configuration)\n");
1791
1792 /* Max. number of entries we have for stream matching/indexing */
1793 if (smmu->version == ARM_SMMU_V2 && id & ARM_SMMU_ID0_EXIDS) {
1794 smmu->features |= ARM_SMMU_FEAT_EXIDS;
1795 size = 1 << 16;
1796 } else {
1797 size = 1 << FIELD_GET(ARM_SMMU_ID0_NUMSIDB, id);
1798 }
1799 smmu->streamid_mask = size - 1;
1800 if (id & ARM_SMMU_ID0_SMS) {
1801 smmu->features |= ARM_SMMU_FEAT_STREAM_MATCH;
1802 size = FIELD_GET(ARM_SMMU_ID0_NUMSMRG, id);
1803 if (size == 0) {
1804 dev_err(smmu->dev,
1805 "stream-matching supported, but no SMRs present!\n");
1806 return -ENODEV;
1807 }
1808
1809 /* Zero-initialised to mark as invalid */
1810 smmu->smrs = devm_kcalloc(smmu->dev, size, sizeof(*smmu->smrs),
1811 GFP_KERNEL);
1812 if (!smmu->smrs)
1813 return -ENOMEM;
1814
1815 dev_notice(smmu->dev,
1816 "\tstream matching with %u register groups", size);
1817 }
1818 /* s2cr->type == 0 means translation, so initialise explicitly */
1819 smmu->s2crs = devm_kmalloc_array(smmu->dev, size, sizeof(*smmu->s2crs),
1820 GFP_KERNEL);
1821 if (!smmu->s2crs)
1822 return -ENOMEM;
1823 for (i = 0; i < size; i++)
1824 smmu->s2crs[i] = s2cr_init_val;
1825
1826 smmu->num_mapping_groups = size;
1827 mutex_init(&smmu->stream_map_mutex);
1828 spin_lock_init(&smmu->global_sync_lock);
1829
1830 if (smmu->version < ARM_SMMU_V2 ||
1831 !(id & ARM_SMMU_ID0_PTFS_NO_AARCH32)) {
1832 smmu->features |= ARM_SMMU_FEAT_FMT_AARCH32_L;
1833 if (!(id & ARM_SMMU_ID0_PTFS_NO_AARCH32S))
1834 smmu->features |= ARM_SMMU_FEAT_FMT_AARCH32_S;
1835 }
1836
1837 /* ID1 */
1838 id = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_ID1);
1839 smmu->pgshift = (id & ARM_SMMU_ID1_PAGESIZE) ? 16 : 12;
1840
1841 /* Check for size mismatch of SMMU address space from mapped region */
1842 size = 1 << (FIELD_GET(ARM_SMMU_ID1_NUMPAGENDXB, id) + 1);
1843 if (smmu->numpage != 2 * size << smmu->pgshift)
1844 dev_warn(smmu->dev,
1845 "SMMU address space size (0x%x) differs from mapped region size (0x%x)!\n",
1846 2 * size << smmu->pgshift, smmu->numpage);
1847 /* Now properly encode NUMPAGE to subsequently derive SMMU_CB_BASE */
1848 smmu->numpage = size;
1849
1850 smmu->num_s2_context_banks = FIELD_GET(ARM_SMMU_ID1_NUMS2CB, id);
1851 smmu->num_context_banks = FIELD_GET(ARM_SMMU_ID1_NUMCB, id);
1852 if (smmu->num_s2_context_banks > smmu->num_context_banks) {
1853 dev_err(smmu->dev, "impossible number of S2 context banks!\n");
1854 return -ENODEV;
1855 }
1856 dev_notice(smmu->dev, "\t%u context banks (%u stage-2 only)\n",
1857 smmu->num_context_banks, smmu->num_s2_context_banks);
1858 smmu->cbs = devm_kcalloc(smmu->dev, smmu->num_context_banks,
1859 sizeof(*smmu->cbs), GFP_KERNEL);
1860 if (!smmu->cbs)
1861 return -ENOMEM;
1862
1863 /* ID2 */
1864 id = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_ID2);
1865 size = arm_smmu_id_size_to_bits(FIELD_GET(ARM_SMMU_ID2_IAS, id));
1866 smmu->ipa_size = size;
1867
1868 /* The output mask is also applied for bypass */
1869 size = arm_smmu_id_size_to_bits(FIELD_GET(ARM_SMMU_ID2_OAS, id));
1870 smmu->pa_size = size;
1871
1872 if (id & ARM_SMMU_ID2_VMID16)
1873 smmu->features |= ARM_SMMU_FEAT_VMID16;
1874
1875 /*
1876 * What the page table walker can address actually depends on which
1877 * descriptor format is in use, but since a) we don't know that yet,
1878 * and b) it can vary per context bank, this will have to do...
1879 */
1880 if (dma_set_mask_and_coherent(smmu->dev, DMA_BIT_MASK(size)))
1881 dev_warn(smmu->dev,
1882 "failed to set DMA mask for table walker\n");
1883
1884 if (smmu->version < ARM_SMMU_V2) {
1885 smmu->va_size = smmu->ipa_size;
1886 if (smmu->version == ARM_SMMU_V1_64K)
1887 smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_64K;
1888 } else {
1889 size = FIELD_GET(ARM_SMMU_ID2_UBS, id);
1890 smmu->va_size = arm_smmu_id_size_to_bits(size);
1891 if (id & ARM_SMMU_ID2_PTFS_4K)
1892 smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_4K;
1893 if (id & ARM_SMMU_ID2_PTFS_16K)
1894 smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_16K;
1895 if (id & ARM_SMMU_ID2_PTFS_64K)
1896 smmu->features |= ARM_SMMU_FEAT_FMT_AARCH64_64K;
1897 }
1898
1899 if (smmu->impl && smmu->impl->cfg_probe) {
1900 ret = smmu->impl->cfg_probe(smmu);
1901 if (ret)
1902 return ret;
1903 }
1904
1905 /* Now we've corralled the various formats, what'll it do? */
1906 if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH32_S)
1907 smmu->pgsize_bitmap |= SZ_4K | SZ_64K | SZ_1M | SZ_16M;
1908 if (smmu->features &
1909 (ARM_SMMU_FEAT_FMT_AARCH32_L | ARM_SMMU_FEAT_FMT_AARCH64_4K))
1910 smmu->pgsize_bitmap |= SZ_4K | SZ_2M | SZ_1G;
1911 if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH64_16K)
1912 smmu->pgsize_bitmap |= SZ_16K | SZ_32M;
1913 if (smmu->features & ARM_SMMU_FEAT_FMT_AARCH64_64K)
1914 smmu->pgsize_bitmap |= SZ_64K | SZ_512M;
1915
1916 if (arm_smmu_ops.pgsize_bitmap == -1UL)
1917 arm_smmu_ops.pgsize_bitmap = smmu->pgsize_bitmap;
1918 else
1919 arm_smmu_ops.pgsize_bitmap |= smmu->pgsize_bitmap;
1920 dev_notice(smmu->dev, "\tSupported page sizes: 0x%08lx\n",
1921 smmu->pgsize_bitmap);
1922
1923
1924 if (smmu->features & ARM_SMMU_FEAT_TRANS_S1)
1925 dev_notice(smmu->dev, "\tStage-1: %lu-bit VA -> %lu-bit IPA\n",
1926 smmu->va_size, smmu->ipa_size);
1927
1928 if (smmu->features & ARM_SMMU_FEAT_TRANS_S2)
1929 dev_notice(smmu->dev, "\tStage-2: %lu-bit IPA -> %lu-bit PA\n",
1930 smmu->ipa_size, smmu->pa_size);
1931
1932 return 0;
1933 }
1934
1935 struct arm_smmu_match_data {
1936 enum arm_smmu_arch_version version;
1937 enum arm_smmu_implementation model;
1938 };
1939
1940 #define ARM_SMMU_MATCH_DATA(name, ver, imp) \
1941 static const struct arm_smmu_match_data name = { .version = ver, .model = imp }
1942
1943 ARM_SMMU_MATCH_DATA(smmu_generic_v1, ARM_SMMU_V1, GENERIC_SMMU);
1944 ARM_SMMU_MATCH_DATA(smmu_generic_v2, ARM_SMMU_V2, GENERIC_SMMU);
1945 ARM_SMMU_MATCH_DATA(arm_mmu401, ARM_SMMU_V1_64K, GENERIC_SMMU);
1946 ARM_SMMU_MATCH_DATA(arm_mmu500, ARM_SMMU_V2, ARM_MMU500);
1947 ARM_SMMU_MATCH_DATA(cavium_smmuv2, ARM_SMMU_V2, CAVIUM_SMMUV2);
1948 ARM_SMMU_MATCH_DATA(qcom_smmuv2, ARM_SMMU_V2, QCOM_SMMUV2);
1949
1950 static const struct of_device_id arm_smmu_of_match[] = {
1951 { .compatible = "arm,smmu-v1", .data = &smmu_generic_v1 },
1952 { .compatible = "arm,smmu-v2", .data = &smmu_generic_v2 },
1953 { .compatible = "arm,mmu-400", .data = &smmu_generic_v1 },
1954 { .compatible = "arm,mmu-401", .data = &arm_mmu401 },
1955 { .compatible = "arm,mmu-500", .data = &arm_mmu500 },
1956 { .compatible = "cavium,smmu-v2", .data = &cavium_smmuv2 },
1957 { .compatible = "nvidia,smmu-500", .data = &arm_mmu500 },
1958 { .compatible = "qcom,smmu-v2", .data = &qcom_smmuv2 },
1959 { },
1960 };
1961 MODULE_DEVICE_TABLE(of, arm_smmu_of_match);
1962
1963 #ifdef CONFIG_ACPI
acpi_smmu_get_data(u32 model,struct arm_smmu_device * smmu)1964 static int acpi_smmu_get_data(u32 model, struct arm_smmu_device *smmu)
1965 {
1966 int ret = 0;
1967
1968 switch (model) {
1969 case ACPI_IORT_SMMU_V1:
1970 case ACPI_IORT_SMMU_CORELINK_MMU400:
1971 smmu->version = ARM_SMMU_V1;
1972 smmu->model = GENERIC_SMMU;
1973 break;
1974 case ACPI_IORT_SMMU_CORELINK_MMU401:
1975 smmu->version = ARM_SMMU_V1_64K;
1976 smmu->model = GENERIC_SMMU;
1977 break;
1978 case ACPI_IORT_SMMU_V2:
1979 smmu->version = ARM_SMMU_V2;
1980 smmu->model = GENERIC_SMMU;
1981 break;
1982 case ACPI_IORT_SMMU_CORELINK_MMU500:
1983 smmu->version = ARM_SMMU_V2;
1984 smmu->model = ARM_MMU500;
1985 break;
1986 case ACPI_IORT_SMMU_CAVIUM_THUNDERX:
1987 smmu->version = ARM_SMMU_V2;
1988 smmu->model = CAVIUM_SMMUV2;
1989 break;
1990 default:
1991 ret = -ENODEV;
1992 }
1993
1994 return ret;
1995 }
1996
arm_smmu_device_acpi_probe(struct arm_smmu_device * smmu,u32 * global_irqs,u32 * pmu_irqs)1997 static int arm_smmu_device_acpi_probe(struct arm_smmu_device *smmu,
1998 u32 *global_irqs, u32 *pmu_irqs)
1999 {
2000 struct device *dev = smmu->dev;
2001 struct acpi_iort_node *node =
2002 *(struct acpi_iort_node **)dev_get_platdata(dev);
2003 struct acpi_iort_smmu *iort_smmu;
2004 int ret;
2005
2006 /* Retrieve SMMU1/2 specific data */
2007 iort_smmu = (struct acpi_iort_smmu *)node->node_data;
2008
2009 ret = acpi_smmu_get_data(iort_smmu->model, smmu);
2010 if (ret < 0)
2011 return ret;
2012
2013 /* Ignore the configuration access interrupt */
2014 *global_irqs = 1;
2015 *pmu_irqs = 0;
2016
2017 if (iort_smmu->flags & ACPI_IORT_SMMU_COHERENT_WALK)
2018 smmu->features |= ARM_SMMU_FEAT_COHERENT_WALK;
2019
2020 return 0;
2021 }
2022 #else
arm_smmu_device_acpi_probe(struct arm_smmu_device * smmu,u32 * global_irqs,u32 * pmu_irqs)2023 static inline int arm_smmu_device_acpi_probe(struct arm_smmu_device *smmu,
2024 u32 *global_irqs, u32 *pmu_irqs)
2025 {
2026 return -ENODEV;
2027 }
2028 #endif
2029
arm_smmu_device_dt_probe(struct arm_smmu_device * smmu,u32 * global_irqs,u32 * pmu_irqs)2030 static int arm_smmu_device_dt_probe(struct arm_smmu_device *smmu,
2031 u32 *global_irqs, u32 *pmu_irqs)
2032 {
2033 const struct arm_smmu_match_data *data;
2034 struct device *dev = smmu->dev;
2035 bool legacy_binding;
2036
2037 if (of_property_read_u32(dev->of_node, "#global-interrupts", global_irqs))
2038 return dev_err_probe(dev, -ENODEV,
2039 "missing #global-interrupts property\n");
2040 *pmu_irqs = 0;
2041
2042 data = of_device_get_match_data(dev);
2043 smmu->version = data->version;
2044 smmu->model = data->model;
2045
2046 legacy_binding = of_find_property(dev->of_node, "mmu-masters", NULL);
2047 if (legacy_binding && !using_generic_binding) {
2048 if (!using_legacy_binding) {
2049 pr_notice("deprecated \"mmu-masters\" DT property in use; %s support unavailable\n",
2050 IS_ENABLED(CONFIG_ARM_SMMU_LEGACY_DT_BINDINGS) ? "DMA API" : "SMMU");
2051 }
2052 using_legacy_binding = true;
2053 } else if (!legacy_binding && !using_legacy_binding) {
2054 using_generic_binding = true;
2055 } else {
2056 dev_err(dev, "not probing due to mismatched DT properties\n");
2057 return -ENODEV;
2058 }
2059
2060 if (of_dma_is_coherent(dev->of_node))
2061 smmu->features |= ARM_SMMU_FEAT_COHERENT_WALK;
2062
2063 return 0;
2064 }
2065
arm_smmu_rmr_install_bypass_smr(struct arm_smmu_device * smmu)2066 static void arm_smmu_rmr_install_bypass_smr(struct arm_smmu_device *smmu)
2067 {
2068 struct list_head rmr_list;
2069 struct iommu_resv_region *e;
2070 int idx, cnt = 0;
2071 u32 reg;
2072
2073 INIT_LIST_HEAD(&rmr_list);
2074 iort_get_rmr_sids(dev_fwnode(smmu->dev), &rmr_list);
2075
2076 /*
2077 * Rather than trying to look at existing mappings that
2078 * are setup by the firmware and then invalidate the ones
2079 * that do no have matching RMR entries, just disable the
2080 * SMMU until it gets enabled again in the reset routine.
2081 */
2082 reg = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sCR0);
2083 reg |= ARM_SMMU_sCR0_CLIENTPD;
2084 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_sCR0, reg);
2085
2086 list_for_each_entry(e, &rmr_list, list) {
2087 struct iommu_iort_rmr_data *rmr;
2088 int i;
2089
2090 rmr = container_of(e, struct iommu_iort_rmr_data, rr);
2091 for (i = 0; i < rmr->num_sids; i++) {
2092 idx = arm_smmu_find_sme(smmu, rmr->sids[i], ~0);
2093 if (idx < 0)
2094 continue;
2095
2096 if (smmu->s2crs[idx].count == 0) {
2097 smmu->smrs[idx].id = rmr->sids[i];
2098 smmu->smrs[idx].mask = 0;
2099 smmu->smrs[idx].valid = true;
2100 }
2101 smmu->s2crs[idx].count++;
2102 smmu->s2crs[idx].type = S2CR_TYPE_BYPASS;
2103 smmu->s2crs[idx].privcfg = S2CR_PRIVCFG_DEFAULT;
2104
2105 cnt++;
2106 }
2107 }
2108
2109 dev_notice(smmu->dev, "\tpreserved %d boot mapping%s\n", cnt,
2110 str_plural(cnt));
2111 iort_put_rmr_sids(dev_fwnode(smmu->dev), &rmr_list);
2112 }
2113
arm_smmu_device_probe(struct platform_device * pdev)2114 static int arm_smmu_device_probe(struct platform_device *pdev)
2115 {
2116 struct resource *res;
2117 struct arm_smmu_device *smmu;
2118 struct device *dev = &pdev->dev;
2119 int num_irqs, i, err;
2120 u32 global_irqs, pmu_irqs;
2121 irqreturn_t (*global_fault)(int irq, void *dev);
2122
2123 smmu = devm_kzalloc(dev, sizeof(*smmu), GFP_KERNEL);
2124 if (!smmu) {
2125 dev_err(dev, "failed to allocate arm_smmu_device\n");
2126 return -ENOMEM;
2127 }
2128 smmu->dev = dev;
2129
2130 if (dev->of_node)
2131 err = arm_smmu_device_dt_probe(smmu, &global_irqs, &pmu_irqs);
2132 else
2133 err = arm_smmu_device_acpi_probe(smmu, &global_irqs, &pmu_irqs);
2134 if (err)
2135 return err;
2136
2137 smmu->base = devm_platform_get_and_ioremap_resource(pdev, 0, &res);
2138 if (IS_ERR(smmu->base))
2139 return PTR_ERR(smmu->base);
2140 smmu->ioaddr = res->start;
2141
2142 /*
2143 * The resource size should effectively match the value of SMMU_TOP;
2144 * stash that temporarily until we know PAGESIZE to validate it with.
2145 */
2146 smmu->numpage = resource_size(res);
2147
2148 smmu = arm_smmu_impl_init(smmu);
2149 if (IS_ERR(smmu))
2150 return PTR_ERR(smmu);
2151
2152 num_irqs = platform_irq_count(pdev);
2153
2154 smmu->num_context_irqs = num_irqs - global_irqs - pmu_irqs;
2155 if (smmu->num_context_irqs <= 0)
2156 return dev_err_probe(dev, -ENODEV,
2157 "found %d interrupts but expected at least %d\n",
2158 num_irqs, global_irqs + pmu_irqs + 1);
2159
2160 smmu->irqs = devm_kcalloc(dev, smmu->num_context_irqs,
2161 sizeof(*smmu->irqs), GFP_KERNEL);
2162 if (!smmu->irqs)
2163 return dev_err_probe(dev, -ENOMEM, "failed to allocate %d irqs\n",
2164 smmu->num_context_irqs);
2165
2166 for (i = 0; i < smmu->num_context_irqs; i++) {
2167 int irq = platform_get_irq(pdev, global_irqs + pmu_irqs + i);
2168
2169 if (irq < 0)
2170 return irq;
2171 smmu->irqs[i] = irq;
2172 }
2173
2174 err = devm_clk_bulk_get_all(dev, &smmu->clks);
2175 if (err < 0) {
2176 dev_err(dev, "failed to get clocks %d\n", err);
2177 return err;
2178 }
2179 smmu->num_clks = err;
2180
2181 err = clk_bulk_prepare_enable(smmu->num_clks, smmu->clks);
2182 if (err)
2183 return err;
2184
2185 err = arm_smmu_device_cfg_probe(smmu);
2186 if (err)
2187 return err;
2188
2189 if (smmu->version == ARM_SMMU_V2) {
2190 if (smmu->num_context_banks > smmu->num_context_irqs) {
2191 dev_err(dev,
2192 "found only %d context irq(s) but %d required\n",
2193 smmu->num_context_irqs, smmu->num_context_banks);
2194 return -ENODEV;
2195 }
2196
2197 /* Ignore superfluous interrupts */
2198 smmu->num_context_irqs = smmu->num_context_banks;
2199 }
2200
2201 if (smmu->impl && smmu->impl->global_fault)
2202 global_fault = smmu->impl->global_fault;
2203 else
2204 global_fault = arm_smmu_global_fault;
2205
2206 for (i = 0; i < global_irqs; i++) {
2207 int irq = platform_get_irq(pdev, i);
2208
2209 if (irq < 0)
2210 return irq;
2211
2212 err = devm_request_irq(dev, irq, global_fault, IRQF_SHARED,
2213 "arm-smmu global fault", smmu);
2214 if (err)
2215 return dev_err_probe(dev, err,
2216 "failed to request global IRQ %d (%u)\n",
2217 i, irq);
2218 }
2219
2220 platform_set_drvdata(pdev, smmu);
2221
2222 /* Check for RMRs and install bypass SMRs if any */
2223 arm_smmu_rmr_install_bypass_smr(smmu);
2224
2225 arm_smmu_device_reset(smmu);
2226 arm_smmu_test_smr_masks(smmu);
2227
2228 err = iommu_device_sysfs_add(&smmu->iommu, smmu->dev, NULL,
2229 "smmu.%pa", &smmu->ioaddr);
2230 if (err)
2231 return dev_err_probe(dev, err, "Failed to register iommu in sysfs\n");
2232
2233 err = iommu_device_register(&smmu->iommu, &arm_smmu_ops,
2234 using_legacy_binding ? NULL : dev);
2235 if (err) {
2236 iommu_device_sysfs_remove(&smmu->iommu);
2237 return dev_err_probe(dev, err, "Failed to register iommu\n");
2238 }
2239
2240 /*
2241 * We want to avoid touching dev->power.lock in fastpaths unless
2242 * it's really going to do something useful - pm_runtime_enabled()
2243 * can serve as an ideal proxy for that decision. So, conditionally
2244 * enable pm_runtime.
2245 */
2246 if (dev->pm_domain) {
2247 pm_runtime_set_active(dev);
2248 pm_runtime_enable(dev);
2249 }
2250
2251 return 0;
2252 }
2253
arm_smmu_device_shutdown(struct platform_device * pdev)2254 static void arm_smmu_device_shutdown(struct platform_device *pdev)
2255 {
2256 struct arm_smmu_device *smmu = platform_get_drvdata(pdev);
2257
2258 if (!bitmap_empty(smmu->context_map, ARM_SMMU_MAX_CBS))
2259 dev_notice(&pdev->dev, "disabling translation\n");
2260
2261 arm_smmu_rpm_get(smmu);
2262 /* Turn the thing off */
2263 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_sCR0, ARM_SMMU_sCR0_CLIENTPD);
2264 arm_smmu_rpm_put(smmu);
2265
2266 if (pm_runtime_enabled(smmu->dev))
2267 pm_runtime_force_suspend(smmu->dev);
2268 else
2269 clk_bulk_disable(smmu->num_clks, smmu->clks);
2270
2271 clk_bulk_unprepare(smmu->num_clks, smmu->clks);
2272 }
2273
arm_smmu_device_remove(struct platform_device * pdev)2274 static void arm_smmu_device_remove(struct platform_device *pdev)
2275 {
2276 struct arm_smmu_device *smmu = platform_get_drvdata(pdev);
2277
2278 iommu_device_unregister(&smmu->iommu);
2279 iommu_device_sysfs_remove(&smmu->iommu);
2280
2281 arm_smmu_device_shutdown(pdev);
2282 }
2283
arm_smmu_runtime_resume(struct device * dev)2284 static int __maybe_unused arm_smmu_runtime_resume(struct device *dev)
2285 {
2286 struct arm_smmu_device *smmu = dev_get_drvdata(dev);
2287 int ret;
2288
2289 ret = clk_bulk_enable(smmu->num_clks, smmu->clks);
2290 if (ret)
2291 return ret;
2292
2293 arm_smmu_device_reset(smmu);
2294
2295 return 0;
2296 }
2297
arm_smmu_runtime_suspend(struct device * dev)2298 static int __maybe_unused arm_smmu_runtime_suspend(struct device *dev)
2299 {
2300 struct arm_smmu_device *smmu = dev_get_drvdata(dev);
2301
2302 clk_bulk_disable(smmu->num_clks, smmu->clks);
2303
2304 return 0;
2305 }
2306
arm_smmu_pm_resume(struct device * dev)2307 static int __maybe_unused arm_smmu_pm_resume(struct device *dev)
2308 {
2309 int ret;
2310 struct arm_smmu_device *smmu = dev_get_drvdata(dev);
2311
2312 ret = clk_bulk_prepare(smmu->num_clks, smmu->clks);
2313 if (ret)
2314 return ret;
2315
2316 if (pm_runtime_suspended(dev))
2317 return 0;
2318
2319 ret = arm_smmu_runtime_resume(dev);
2320 if (ret)
2321 clk_bulk_unprepare(smmu->num_clks, smmu->clks);
2322
2323 return ret;
2324 }
2325
arm_smmu_pm_suspend(struct device * dev)2326 static int __maybe_unused arm_smmu_pm_suspend(struct device *dev)
2327 {
2328 int ret = 0;
2329 struct arm_smmu_device *smmu = dev_get_drvdata(dev);
2330
2331 if (pm_runtime_suspended(dev))
2332 goto clk_unprepare;
2333
2334 ret = arm_smmu_runtime_suspend(dev);
2335 if (ret)
2336 return ret;
2337
2338 clk_unprepare:
2339 clk_bulk_unprepare(smmu->num_clks, smmu->clks);
2340 return ret;
2341 }
2342
2343 static const struct dev_pm_ops arm_smmu_pm_ops = {
2344 SET_SYSTEM_SLEEP_PM_OPS(arm_smmu_pm_suspend, arm_smmu_pm_resume)
2345 SET_RUNTIME_PM_OPS(arm_smmu_runtime_suspend,
2346 arm_smmu_runtime_resume, NULL)
2347 };
2348
2349 static struct platform_driver arm_smmu_driver = {
2350 .driver = {
2351 .name = "arm-smmu",
2352 .of_match_table = arm_smmu_of_match,
2353 .pm = &arm_smmu_pm_ops,
2354 .suppress_bind_attrs = true,
2355 },
2356 .probe = arm_smmu_device_probe,
2357 .remove = arm_smmu_device_remove,
2358 .shutdown = arm_smmu_device_shutdown,
2359 };
2360 module_platform_driver(arm_smmu_driver);
2361
2362 MODULE_DESCRIPTION("IOMMU API for ARM architected SMMU implementations");
2363 MODULE_AUTHOR("Will Deacon <will@kernel.org>");
2364 MODULE_ALIAS("platform:arm-smmu");
2365 MODULE_LICENSE("GPL v2");
2366