1 // SPDX-License-Identifier: GPL-2.0-only
2 // Miscellaneous Arm SMMU implementation and integration quirks
3 // Copyright (C) 2019 Arm Limited
4
5 #define pr_fmt(fmt) "arm-smmu: " fmt
6
7 #include <linux/bitfield.h>
8 #include <linux/of.h>
9
10 #include "arm-smmu.h"
11
12
arm_smmu_gr0_ns(int offset)13 static int arm_smmu_gr0_ns(int offset)
14 {
15 switch (offset) {
16 case ARM_SMMU_GR0_sCR0:
17 case ARM_SMMU_GR0_sACR:
18 case ARM_SMMU_GR0_sGFSR:
19 case ARM_SMMU_GR0_sGFSYNR0:
20 case ARM_SMMU_GR0_sGFSYNR1:
21 case ARM_SMMU_GR0_sGFSYNR2:
22 return offset + 0x400;
23 default:
24 return offset;
25 }
26 }
27
arm_smmu_read_ns(struct arm_smmu_device * smmu,int page,int offset)28 static u32 arm_smmu_read_ns(struct arm_smmu_device *smmu, int page,
29 int offset)
30 {
31 if (page == ARM_SMMU_GR0)
32 offset = arm_smmu_gr0_ns(offset);
33 return readl_relaxed(arm_smmu_page(smmu, page) + offset);
34 }
35
arm_smmu_write_ns(struct arm_smmu_device * smmu,int page,int offset,u32 val)36 static void arm_smmu_write_ns(struct arm_smmu_device *smmu, int page,
37 int offset, u32 val)
38 {
39 if (page == ARM_SMMU_GR0)
40 offset = arm_smmu_gr0_ns(offset);
41 writel_relaxed(val, arm_smmu_page(smmu, page) + offset);
42 }
43
44 /* Since we don't care for sGFAR, we can do without 64-bit accessors */
45 static const struct arm_smmu_impl calxeda_impl = {
46 .read_reg = arm_smmu_read_ns,
47 .write_reg = arm_smmu_write_ns,
48 };
49
50
51 struct cavium_smmu {
52 struct arm_smmu_device smmu;
53 u32 id_base;
54 };
55
cavium_cfg_probe(struct arm_smmu_device * smmu)56 static int cavium_cfg_probe(struct arm_smmu_device *smmu)
57 {
58 static atomic_t context_count = ATOMIC_INIT(0);
59 struct cavium_smmu *cs = container_of(smmu, struct cavium_smmu, smmu);
60 /*
61 * Cavium CN88xx erratum #27704.
62 * Ensure ASID and VMID allocation is unique across all SMMUs in
63 * the system.
64 */
65 cs->id_base = atomic_fetch_add(smmu->num_context_banks, &context_count);
66 dev_notice(smmu->dev, "\tenabling workaround for Cavium erratum 27704\n");
67
68 return 0;
69 }
70
cavium_init_context(struct arm_smmu_domain * smmu_domain,struct io_pgtable_cfg * pgtbl_cfg,struct device * dev)71 static int cavium_init_context(struct arm_smmu_domain *smmu_domain,
72 struct io_pgtable_cfg *pgtbl_cfg, struct device *dev)
73 {
74 struct cavium_smmu *cs = container_of(smmu_domain->smmu,
75 struct cavium_smmu, smmu);
76
77 if (smmu_domain->stage == ARM_SMMU_DOMAIN_S2)
78 smmu_domain->cfg.vmid += cs->id_base;
79 else
80 smmu_domain->cfg.asid += cs->id_base;
81
82 return 0;
83 }
84
85 static const struct arm_smmu_impl cavium_impl = {
86 .cfg_probe = cavium_cfg_probe,
87 .init_context = cavium_init_context,
88 };
89
cavium_smmu_impl_init(struct arm_smmu_device * smmu)90 static struct arm_smmu_device *cavium_smmu_impl_init(struct arm_smmu_device *smmu)
91 {
92 struct cavium_smmu *cs;
93
94 cs = devm_krealloc(smmu->dev, smmu, sizeof(*cs), GFP_KERNEL);
95 if (!cs)
96 return ERR_PTR(-ENOMEM);
97
98 cs->smmu.impl = &cavium_impl;
99
100 return &cs->smmu;
101 }
102
103
104 #define ARM_MMU500_ACTLR_CPRE (1 << 1)
105
106 #define ARM_MMU500_ACR_CACHE_LOCK (1 << 26)
107 #define ARM_MMU500_ACR_S2CRB_TLBEN (1 << 10)
108 #define ARM_MMU500_ACR_SMTNMB_TLBEN (1 << 8)
109
arm_mmu500_reset(struct arm_smmu_device * smmu)110 int arm_mmu500_reset(struct arm_smmu_device *smmu)
111 {
112 u32 reg, major;
113 int i;
114 /*
115 * On MMU-500 r2p0 onwards we need to clear ACR.CACHE_LOCK before
116 * writes to the context bank ACTLRs will stick. And we just hope that
117 * Secure has also cleared SACR.CACHE_LOCK for this to take effect...
118 */
119 reg = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_ID7);
120 major = FIELD_GET(ARM_SMMU_ID7_MAJOR, reg);
121 reg = arm_smmu_gr0_read(smmu, ARM_SMMU_GR0_sACR);
122 if (major >= 2)
123 reg &= ~ARM_MMU500_ACR_CACHE_LOCK;
124 /*
125 * Allow unmatched Stream IDs to allocate bypass
126 * TLB entries for reduced latency.
127 */
128 reg |= ARM_MMU500_ACR_SMTNMB_TLBEN | ARM_MMU500_ACR_S2CRB_TLBEN;
129 arm_smmu_gr0_write(smmu, ARM_SMMU_GR0_sACR, reg);
130
131 /*
132 * Disable MMU-500's not-particularly-beneficial next-page
133 * prefetcher for the sake of at least 5 known errata.
134 */
135 for (i = 0; i < smmu->num_context_banks; ++i) {
136 reg = arm_smmu_cb_read(smmu, i, ARM_SMMU_CB_ACTLR);
137 reg &= ~ARM_MMU500_ACTLR_CPRE;
138 arm_smmu_cb_write(smmu, i, ARM_SMMU_CB_ACTLR, reg);
139 reg = arm_smmu_cb_read(smmu, i, ARM_SMMU_CB_ACTLR);
140 if (reg & ARM_MMU500_ACTLR_CPRE)
141 dev_warn_once(smmu->dev, "Failed to disable prefetcher for errata workarounds, check SACR.CACHE_LOCK\n");
142 }
143
144 return 0;
145 }
146
147 static const struct arm_smmu_impl arm_mmu500_impl = {
148 .reset = arm_mmu500_reset,
149 };
150
mrvl_mmu500_readq(struct arm_smmu_device * smmu,int page,int off)151 static u64 mrvl_mmu500_readq(struct arm_smmu_device *smmu, int page, int off)
152 {
153 /*
154 * Marvell Armada-AP806 erratum #582743.
155 * Split all the readq to double readl
156 */
157 return hi_lo_readq_relaxed(arm_smmu_page(smmu, page) + off);
158 }
159
mrvl_mmu500_writeq(struct arm_smmu_device * smmu,int page,int off,u64 val)160 static void mrvl_mmu500_writeq(struct arm_smmu_device *smmu, int page, int off,
161 u64 val)
162 {
163 /*
164 * Marvell Armada-AP806 erratum #582743.
165 * Split all the writeq to double writel
166 */
167 hi_lo_writeq_relaxed(val, arm_smmu_page(smmu, page) + off);
168 }
169
mrvl_mmu500_cfg_probe(struct arm_smmu_device * smmu)170 static int mrvl_mmu500_cfg_probe(struct arm_smmu_device *smmu)
171 {
172
173 /*
174 * Armada-AP806 erratum #582743.
175 * Hide the SMMU_IDR2.PTFSv8 fields to sidestep the AArch64
176 * formats altogether and allow using 32 bits access on the
177 * interconnect.
178 */
179 smmu->features &= ~(ARM_SMMU_FEAT_FMT_AARCH64_4K |
180 ARM_SMMU_FEAT_FMT_AARCH64_16K |
181 ARM_SMMU_FEAT_FMT_AARCH64_64K);
182
183 return 0;
184 }
185
186 static const struct arm_smmu_impl mrvl_mmu500_impl = {
187 .read_reg64 = mrvl_mmu500_readq,
188 .write_reg64 = mrvl_mmu500_writeq,
189 .cfg_probe = mrvl_mmu500_cfg_probe,
190 .reset = arm_mmu500_reset,
191 };
192
193
arm_smmu_impl_init(struct arm_smmu_device * smmu)194 struct arm_smmu_device *arm_smmu_impl_init(struct arm_smmu_device *smmu)
195 {
196 const struct device_node *np = smmu->dev->of_node;
197
198 /*
199 * Set the impl for model-specific implementation quirks first,
200 * such that platform integration quirks can pick it up and
201 * inherit from it if necessary.
202 */
203 switch (smmu->model) {
204 case ARM_MMU500:
205 smmu->impl = &arm_mmu500_impl;
206 break;
207 case CAVIUM_SMMUV2:
208 return cavium_smmu_impl_init(smmu);
209 default:
210 break;
211 }
212
213 /* This is implicitly MMU-400 */
214 if (of_property_read_bool(np, "calxeda,smmu-secure-config-access"))
215 smmu->impl = &calxeda_impl;
216
217 if (of_device_is_compatible(np, "nvidia,tegra234-smmu") ||
218 of_device_is_compatible(np, "nvidia,tegra194-smmu") ||
219 of_device_is_compatible(np, "nvidia,tegra186-smmu"))
220 return nvidia_smmu_impl_init(smmu);
221
222 if (IS_ENABLED(CONFIG_ARM_SMMU_QCOM))
223 smmu = qcom_smmu_impl_init(smmu);
224
225 if (of_device_is_compatible(np, "marvell,ap806-smmu-500"))
226 smmu->impl = &mrvl_mmu500_impl;
227
228 return smmu;
229 }
230