1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Copyright (c) 2022 Qualcomm Innovation Center, Inc. All rights reserved.
4 * Copyright (c) 2024 Qualcomm Innovation Center, Inc. All rights reserved.
5 */
6
7 #include <linux/cleanup.h>
8 #include <linux/device.h>
9 #include <linux/interconnect.h>
10 #include <linux/firmware/qcom/qcom_scm.h>
11 #include <linux/iopoll.h>
12 #include <linux/list.h>
13 #include <linux/mutex.h>
14 #include <linux/platform_device.h>
15 #include <linux/ratelimit.h>
16 #include <linux/spinlock.h>
17
18 #include "arm-smmu.h"
19 #include "arm-smmu-qcom.h"
20
21 #define TBU_DBG_TIMEOUT_US 100
22 #define DEBUG_AXUSER_REG 0x30
23 #define DEBUG_AXUSER_CDMID GENMASK_ULL(43, 36)
24 #define DEBUG_AXUSER_CDMID_VAL 0xff
25 #define DEBUG_PAR_REG 0x28
26 #define DEBUG_PAR_FAULT_VAL BIT(0)
27 #define DEBUG_PAR_PA GENMASK_ULL(47, 12)
28 #define DEBUG_SID_HALT_REG 0x0
29 #define DEBUG_SID_HALT_VAL BIT(16)
30 #define DEBUG_SID_HALT_SID GENMASK(9, 0)
31 #define DEBUG_SR_HALT_ACK_REG 0x20
32 #define DEBUG_SR_HALT_ACK_VAL BIT(1)
33 #define DEBUG_SR_ECATS_RUNNING_VAL BIT(0)
34 #define DEBUG_TXN_AXCACHE GENMASK(5, 2)
35 #define DEBUG_TXN_AXPROT GENMASK(8, 6)
36 #define DEBUG_TXN_AXPROT_PRIV 0x1
37 #define DEBUG_TXN_AXPROT_NSEC 0x2
38 #define DEBUG_TXN_TRIGG_REG 0x18
39 #define DEBUG_TXN_TRIGGER BIT(0)
40 #define DEBUG_VA_ADDR_REG 0x8
41
42 static LIST_HEAD(tbu_list);
43 static DEFINE_MUTEX(tbu_list_lock);
44 static DEFINE_SPINLOCK(atos_lock);
45
46 struct qcom_tbu {
47 struct device *dev;
48 struct device_node *smmu_np;
49 u32 sid_range[2];
50 struct list_head list;
51 struct clk *clk;
52 struct icc_path *path;
53 void __iomem *base;
54 spinlock_t halt_lock; /* multiple halt or resume can't execute concurrently */
55 int halt_count;
56 };
57
to_qcom_smmu(struct arm_smmu_device * smmu)58 static struct qcom_smmu *to_qcom_smmu(struct arm_smmu_device *smmu)
59 {
60 return container_of(smmu, struct qcom_smmu, smmu);
61 }
62
qcom_smmu_tlb_sync_debug(struct arm_smmu_device * smmu)63 void qcom_smmu_tlb_sync_debug(struct arm_smmu_device *smmu)
64 {
65 int ret;
66 u32 tbu_pwr_status, sync_inv_ack, sync_inv_progress;
67 struct qcom_smmu *qsmmu = container_of(smmu, struct qcom_smmu, smmu);
68 const struct qcom_smmu_config *cfg;
69 static DEFINE_RATELIMIT_STATE(rs, DEFAULT_RATELIMIT_INTERVAL,
70 DEFAULT_RATELIMIT_BURST);
71
72 if (__ratelimit(&rs)) {
73 dev_err(smmu->dev, "TLB sync timed out -- SMMU may be deadlocked\n");
74
75 cfg = qsmmu->data->cfg;
76 if (!cfg)
77 return;
78
79 ret = qcom_scm_io_readl(smmu->ioaddr + cfg->reg_offset[QCOM_SMMU_TBU_PWR_STATUS],
80 &tbu_pwr_status);
81 if (ret)
82 dev_err(smmu->dev,
83 "Failed to read TBU power status: %d\n", ret);
84
85 ret = qcom_scm_io_readl(smmu->ioaddr + cfg->reg_offset[QCOM_SMMU_STATS_SYNC_INV_TBU_ACK],
86 &sync_inv_ack);
87 if (ret)
88 dev_err(smmu->dev,
89 "Failed to read TBU sync/inv ack status: %d\n", ret);
90
91 ret = qcom_scm_io_readl(smmu->ioaddr + cfg->reg_offset[QCOM_SMMU_MMU2QSS_AND_SAFE_WAIT_CNTR],
92 &sync_inv_progress);
93 if (ret)
94 dev_err(smmu->dev,
95 "Failed to read TCU syn/inv progress: %d\n", ret);
96
97 dev_err(smmu->dev,
98 "TBU: power_status %#x sync_inv_ack %#x sync_inv_progress %#x\n",
99 tbu_pwr_status, sync_inv_ack, sync_inv_progress);
100 }
101 }
102
qcom_find_tbu(struct qcom_smmu * qsmmu,u32 sid)103 static struct qcom_tbu *qcom_find_tbu(struct qcom_smmu *qsmmu, u32 sid)
104 {
105 struct qcom_tbu *tbu;
106 u32 start, end;
107
108 guard(mutex)(&tbu_list_lock);
109
110 if (list_empty(&tbu_list))
111 return NULL;
112
113 list_for_each_entry(tbu, &tbu_list, list) {
114 start = tbu->sid_range[0];
115 end = start + tbu->sid_range[1];
116
117 if (qsmmu->smmu.dev->of_node == tbu->smmu_np &&
118 start <= sid && sid < end)
119 return tbu;
120 }
121 dev_err(qsmmu->smmu.dev, "Unable to find TBU for sid 0x%x\n", sid);
122
123 return NULL;
124 }
125
qcom_tbu_halt(struct qcom_tbu * tbu,struct arm_smmu_domain * smmu_domain)126 static int qcom_tbu_halt(struct qcom_tbu *tbu, struct arm_smmu_domain *smmu_domain)
127 {
128 struct arm_smmu_device *smmu = smmu_domain->smmu;
129 int ret = 0, idx = smmu_domain->cfg.cbndx;
130 u32 val, fsr, status;
131
132 guard(spinlock_irqsave)(&tbu->halt_lock);
133 if (tbu->halt_count) {
134 tbu->halt_count++;
135 return ret;
136 }
137
138 val = readl_relaxed(tbu->base + DEBUG_SID_HALT_REG);
139 val |= DEBUG_SID_HALT_VAL;
140 writel_relaxed(val, tbu->base + DEBUG_SID_HALT_REG);
141
142 fsr = arm_smmu_cb_read(smmu, idx, ARM_SMMU_CB_FSR);
143 if ((fsr & ARM_SMMU_CB_FSR_FAULT) && (fsr & ARM_SMMU_CB_FSR_SS)) {
144 u32 sctlr_orig, sctlr;
145
146 /*
147 * We are in a fault. Our request to halt the bus will not
148 * complete until transactions in front of us (such as the fault
149 * itself) have completed. Disable iommu faults and terminate
150 * any existing transactions.
151 */
152 sctlr_orig = arm_smmu_cb_read(smmu, idx, ARM_SMMU_CB_SCTLR);
153 sctlr = sctlr_orig & ~(ARM_SMMU_SCTLR_CFCFG | ARM_SMMU_SCTLR_CFIE);
154 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_SCTLR, sctlr);
155 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_FSR, fsr);
156 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_RESUME, ARM_SMMU_RESUME_TERMINATE);
157 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_SCTLR, sctlr_orig);
158 }
159
160 if (readl_poll_timeout_atomic(tbu->base + DEBUG_SR_HALT_ACK_REG, status,
161 (status & DEBUG_SR_HALT_ACK_VAL),
162 0, TBU_DBG_TIMEOUT_US)) {
163 dev_err(tbu->dev, "Timeout while trying to halt TBU!\n");
164 ret = -ETIMEDOUT;
165
166 val = readl_relaxed(tbu->base + DEBUG_SID_HALT_REG);
167 val &= ~DEBUG_SID_HALT_VAL;
168 writel_relaxed(val, tbu->base + DEBUG_SID_HALT_REG);
169
170 return ret;
171 }
172
173 tbu->halt_count = 1;
174
175 return ret;
176 }
177
qcom_tbu_resume(struct qcom_tbu * tbu)178 static void qcom_tbu_resume(struct qcom_tbu *tbu)
179 {
180 u32 val;
181
182 guard(spinlock_irqsave)(&tbu->halt_lock);
183 if (!tbu->halt_count) {
184 WARN(1, "%s: halt_count is 0", dev_name(tbu->dev));
185 return;
186 }
187
188 if (tbu->halt_count > 1) {
189 tbu->halt_count--;
190 return;
191 }
192
193 val = readl_relaxed(tbu->base + DEBUG_SID_HALT_REG);
194 val &= ~DEBUG_SID_HALT_VAL;
195 writel_relaxed(val, tbu->base + DEBUG_SID_HALT_REG);
196
197 tbu->halt_count = 0;
198 }
199
qcom_tbu_trigger_atos(struct arm_smmu_domain * smmu_domain,struct qcom_tbu * tbu,dma_addr_t iova,u32 sid)200 static phys_addr_t qcom_tbu_trigger_atos(struct arm_smmu_domain *smmu_domain,
201 struct qcom_tbu *tbu, dma_addr_t iova, u32 sid)
202 {
203 bool atos_timedout = false;
204 phys_addr_t phys = 0;
205 ktime_t timeout;
206 u64 val;
207
208 /* Set address and stream-id */
209 val = readq_relaxed(tbu->base + DEBUG_SID_HALT_REG);
210 val &= ~DEBUG_SID_HALT_SID;
211 val |= FIELD_PREP(DEBUG_SID_HALT_SID, sid);
212 writeq_relaxed(val, tbu->base + DEBUG_SID_HALT_REG);
213 writeq_relaxed(iova, tbu->base + DEBUG_VA_ADDR_REG);
214 val = FIELD_PREP(DEBUG_AXUSER_CDMID, DEBUG_AXUSER_CDMID_VAL);
215 writeq_relaxed(val, tbu->base + DEBUG_AXUSER_REG);
216
217 /* Write-back read and write-allocate */
218 val = FIELD_PREP(DEBUG_TXN_AXCACHE, 0xf);
219
220 /* Non-secure access */
221 val |= FIELD_PREP(DEBUG_TXN_AXPROT, DEBUG_TXN_AXPROT_NSEC);
222
223 /* Privileged access */
224 val |= FIELD_PREP(DEBUG_TXN_AXPROT, DEBUG_TXN_AXPROT_PRIV);
225
226 val |= DEBUG_TXN_TRIGGER;
227 writeq_relaxed(val, tbu->base + DEBUG_TXN_TRIGG_REG);
228
229 timeout = ktime_add_us(ktime_get(), TBU_DBG_TIMEOUT_US);
230 for (;;) {
231 val = readl_relaxed(tbu->base + DEBUG_SR_HALT_ACK_REG);
232 if (!(val & DEBUG_SR_ECATS_RUNNING_VAL))
233 break;
234 val = readl_relaxed(tbu->base + DEBUG_PAR_REG);
235 if (val & DEBUG_PAR_FAULT_VAL)
236 break;
237 if (ktime_compare(ktime_get(), timeout) > 0) {
238 atos_timedout = true;
239 break;
240 }
241 }
242
243 val = readq_relaxed(tbu->base + DEBUG_PAR_REG);
244 if (val & DEBUG_PAR_FAULT_VAL)
245 dev_err(tbu->dev, "ATOS generated a fault interrupt! PAR = %llx, SID=0x%x\n",
246 val, sid);
247 else if (atos_timedout)
248 dev_err_ratelimited(tbu->dev, "ATOS translation timed out!\n");
249 else
250 phys = FIELD_GET(DEBUG_PAR_PA, val);
251
252 /* Reset hardware */
253 writeq_relaxed(0, tbu->base + DEBUG_TXN_TRIGG_REG);
254 writeq_relaxed(0, tbu->base + DEBUG_VA_ADDR_REG);
255 val = readl_relaxed(tbu->base + DEBUG_SID_HALT_REG);
256 val &= ~DEBUG_SID_HALT_SID;
257 writel_relaxed(val, tbu->base + DEBUG_SID_HALT_REG);
258
259 return phys;
260 }
261
qcom_iova_to_phys(struct arm_smmu_domain * smmu_domain,dma_addr_t iova,u32 sid)262 static phys_addr_t qcom_iova_to_phys(struct arm_smmu_domain *smmu_domain,
263 dma_addr_t iova, u32 sid)
264 {
265 struct arm_smmu_device *smmu = smmu_domain->smmu;
266 struct qcom_smmu *qsmmu = to_qcom_smmu(smmu);
267 int idx = smmu_domain->cfg.cbndx;
268 struct qcom_tbu *tbu;
269 u32 sctlr_orig, sctlr;
270 phys_addr_t phys = 0;
271 int attempt = 0;
272 int ret;
273 u64 fsr;
274
275 tbu = qcom_find_tbu(qsmmu, sid);
276 if (!tbu)
277 return 0;
278
279 ret = icc_set_bw(tbu->path, 0, UINT_MAX);
280 if (ret)
281 return ret;
282
283 ret = clk_prepare_enable(tbu->clk);
284 if (ret)
285 goto disable_icc;
286
287 ret = qcom_tbu_halt(tbu, smmu_domain);
288 if (ret)
289 goto disable_clk;
290
291 /*
292 * ATOS/ECATS can trigger the fault interrupt, so disable it temporarily
293 * and check for an interrupt manually.
294 */
295 sctlr_orig = arm_smmu_cb_read(smmu, idx, ARM_SMMU_CB_SCTLR);
296 sctlr = sctlr_orig & ~(ARM_SMMU_SCTLR_CFCFG | ARM_SMMU_SCTLR_CFIE);
297 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_SCTLR, sctlr);
298
299 fsr = arm_smmu_cb_read(smmu, idx, ARM_SMMU_CB_FSR);
300 if (fsr & ARM_SMMU_CB_FSR_FAULT) {
301 /* Clear pending interrupts */
302 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_FSR, fsr);
303
304 /*
305 * TBU halt takes care of resuming any stalled transcation.
306 * Kept it here for completeness sake.
307 */
308 if (fsr & ARM_SMMU_CB_FSR_SS)
309 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_RESUME,
310 ARM_SMMU_RESUME_TERMINATE);
311 }
312
313 /* Only one concurrent atos operation */
314 scoped_guard(spinlock_irqsave, &atos_lock) {
315 /*
316 * If the translation fails, attempt the lookup more time."
317 */
318 do {
319 phys = qcom_tbu_trigger_atos(smmu_domain, tbu, iova, sid);
320
321 fsr = arm_smmu_cb_read(smmu, idx, ARM_SMMU_CB_FSR);
322 if (fsr & ARM_SMMU_CB_FSR_FAULT) {
323 /* Clear pending interrupts */
324 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_FSR, fsr);
325
326 if (fsr & ARM_SMMU_CB_FSR_SS)
327 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_RESUME,
328 ARM_SMMU_RESUME_TERMINATE);
329 }
330 } while (!phys && attempt++ < 2);
331
332 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_SCTLR, sctlr_orig);
333 }
334 qcom_tbu_resume(tbu);
335
336 /* Read to complete prior write transcations */
337 readl_relaxed(tbu->base + DEBUG_SR_HALT_ACK_REG);
338
339 disable_clk:
340 clk_disable_unprepare(tbu->clk);
341 disable_icc:
342 icc_set_bw(tbu->path, 0, 0);
343
344 return phys;
345 }
346
qcom_smmu_iova_to_phys_hard(struct arm_smmu_domain * smmu_domain,dma_addr_t iova)347 static phys_addr_t qcom_smmu_iova_to_phys_hard(struct arm_smmu_domain *smmu_domain, dma_addr_t iova)
348 {
349 struct arm_smmu_device *smmu = smmu_domain->smmu;
350 int idx = smmu_domain->cfg.cbndx;
351 u32 frsynra;
352 u16 sid;
353
354 frsynra = arm_smmu_gr1_read(smmu, ARM_SMMU_GR1_CBFRSYNRA(idx));
355 sid = FIELD_GET(ARM_SMMU_CBFRSYNRA_SID, frsynra);
356
357 return qcom_iova_to_phys(smmu_domain, iova, sid);
358 }
359
qcom_smmu_verify_fault(struct arm_smmu_domain * smmu_domain,dma_addr_t iova,u32 fsr)360 static phys_addr_t qcom_smmu_verify_fault(struct arm_smmu_domain *smmu_domain, dma_addr_t iova, u32 fsr)
361 {
362 struct io_pgtable *iop = io_pgtable_ops_to_pgtable(smmu_domain->pgtbl_ops);
363 struct arm_smmu_device *smmu = smmu_domain->smmu;
364 phys_addr_t phys_post_tlbiall;
365 phys_addr_t phys;
366
367 phys = qcom_smmu_iova_to_phys_hard(smmu_domain, iova);
368 io_pgtable_tlb_flush_all(iop);
369 phys_post_tlbiall = qcom_smmu_iova_to_phys_hard(smmu_domain, iova);
370
371 if (phys != phys_post_tlbiall) {
372 dev_err(smmu->dev,
373 "ATOS results differed across TLBIALL... (before: %pa after: %pa)\n",
374 &phys, &phys_post_tlbiall);
375 }
376
377 return (phys == 0 ? phys_post_tlbiall : phys);
378 }
379
qcom_smmu_context_fault(int irq,void * dev)380 irqreturn_t qcom_smmu_context_fault(int irq, void *dev)
381 {
382 struct arm_smmu_domain *smmu_domain = dev;
383 struct io_pgtable_ops *ops = smmu_domain->pgtbl_ops;
384 struct arm_smmu_device *smmu = smmu_domain->smmu;
385 struct arm_smmu_context_fault_info cfi;
386 u32 resume = 0;
387 int idx = smmu_domain->cfg.cbndx;
388 phys_addr_t phys_soft;
389 int ret, tmp;
390
391 static DEFINE_RATELIMIT_STATE(_rs,
392 DEFAULT_RATELIMIT_INTERVAL,
393 DEFAULT_RATELIMIT_BURST);
394
395 arm_smmu_read_context_fault_info(smmu, idx, &cfi);
396
397 if (!(cfi.fsr & ARM_SMMU_CB_FSR_FAULT))
398 return IRQ_NONE;
399
400 if (list_empty(&tbu_list)) {
401 ret = report_iommu_fault(&smmu_domain->domain, NULL, cfi.iova,
402 cfi.fsynr & ARM_SMMU_CB_FSYNR0_WNR ? IOMMU_FAULT_WRITE : IOMMU_FAULT_READ);
403
404 if (ret == -ENOSYS)
405 arm_smmu_print_context_fault_info(smmu, idx, &cfi);
406
407 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_FSR, cfi.fsr);
408
409 if (cfi.fsr & ARM_SMMU_CB_FSR_SS) {
410 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_RESUME,
411 ret == -EAGAIN ? 0 : ARM_SMMU_RESUME_TERMINATE);
412 }
413
414 return IRQ_HANDLED;
415 }
416
417 phys_soft = ops->iova_to_phys(ops, cfi.iova);
418
419 tmp = report_iommu_fault(&smmu_domain->domain, NULL, cfi.iova,
420 cfi.fsynr & ARM_SMMU_CB_FSYNR0_WNR ? IOMMU_FAULT_WRITE : IOMMU_FAULT_READ);
421 if (!tmp || tmp == -EBUSY) {
422 ret = IRQ_HANDLED;
423 resume = ARM_SMMU_RESUME_TERMINATE;
424 } else if (tmp == -EAGAIN) {
425 ret = IRQ_HANDLED;
426 resume = 0;
427 } else {
428 phys_addr_t phys_atos = qcom_smmu_verify_fault(smmu_domain, cfi.iova, cfi.fsr);
429
430 if (__ratelimit(&_rs)) {
431 arm_smmu_print_context_fault_info(smmu, idx, &cfi);
432
433 dev_err(smmu->dev,
434 "soft iova-to-phys=%pa\n", &phys_soft);
435 if (!phys_soft)
436 dev_err(smmu->dev,
437 "SOFTWARE TABLE WALK FAILED! Looks like %s accessed an unmapped address!\n",
438 dev_name(smmu->dev));
439 if (phys_atos)
440 dev_err(smmu->dev, "hard iova-to-phys (ATOS)=%pa\n",
441 &phys_atos);
442 else
443 dev_err(smmu->dev, "hard iova-to-phys (ATOS) failed\n");
444 }
445 ret = IRQ_NONE;
446 resume = ARM_SMMU_RESUME_TERMINATE;
447 }
448
449 /*
450 * If the client returns -EBUSY, do not clear FSR and do not RESUME
451 * if stalled. This is required to keep the IOMMU client stalled on
452 * the outstanding fault. This gives the client a chance to take any
453 * debug action and then terminate the stalled transaction.
454 * So, the sequence in case of stall on fault should be:
455 * 1) Do not clear FSR or write to RESUME here
456 * 2) Client takes any debug action
457 * 3) Client terminates the stalled transaction and resumes the IOMMU
458 * 4) Client clears FSR. The FSR should only be cleared after 3) and
459 * not before so that the fault remains outstanding. This ensures
460 * SCTLR.HUPCF has the desired effect if subsequent transactions also
461 * need to be terminated.
462 */
463 if (tmp != -EBUSY) {
464 /* Clear the faulting FSR */
465 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_FSR, cfi.fsr);
466
467 /* Retry or terminate any stalled transactions */
468 if (cfi.fsr & ARM_SMMU_CB_FSR_SS)
469 arm_smmu_cb_write(smmu, idx, ARM_SMMU_CB_RESUME, resume);
470 }
471
472 return ret;
473 }
474
qcom_tbu_probe(struct platform_device * pdev)475 int qcom_tbu_probe(struct platform_device *pdev)
476 {
477 struct of_phandle_args args = { .args_count = 2 };
478 struct device_node *np = pdev->dev.of_node;
479 struct device *dev = &pdev->dev;
480 struct qcom_tbu *tbu;
481
482 tbu = devm_kzalloc(dev, sizeof(*tbu), GFP_KERNEL);
483 if (!tbu)
484 return -ENOMEM;
485
486 tbu->dev = dev;
487 INIT_LIST_HEAD(&tbu->list);
488 spin_lock_init(&tbu->halt_lock);
489
490 if (of_parse_phandle_with_args(np, "qcom,stream-id-range", "#iommu-cells", 0, &args)) {
491 dev_err(dev, "Cannot parse the 'qcom,stream-id-range' DT property\n");
492 return -EINVAL;
493 }
494
495 tbu->smmu_np = args.np;
496 tbu->sid_range[0] = args.args[0];
497 tbu->sid_range[1] = args.args[1];
498 of_node_put(args.np);
499
500 tbu->base = devm_of_iomap(dev, np, 0, NULL);
501 if (IS_ERR(tbu->base))
502 return PTR_ERR(tbu->base);
503
504 tbu->clk = devm_clk_get_optional(dev, NULL);
505 if (IS_ERR(tbu->clk))
506 return PTR_ERR(tbu->clk);
507
508 tbu->path = devm_of_icc_get(dev, NULL);
509 if (IS_ERR(tbu->path))
510 return PTR_ERR(tbu->path);
511
512 guard(mutex)(&tbu_list_lock);
513 list_add_tail(&tbu->list, &tbu_list);
514
515 return 0;
516 }
517