1e86d1aa8SWill Deacon // SPDX-License-Identifier: GPL-2.0
2e86d1aa8SWill Deacon /*
3e86d1aa8SWill Deacon * IOMMU API for ARM architected SMMUv3 implementations.
4e86d1aa8SWill Deacon *
5e86d1aa8SWill Deacon * Copyright (C) 2015 ARM Limited
6e86d1aa8SWill Deacon *
7e86d1aa8SWill Deacon * Author: Will Deacon <will.deacon@arm.com>
8e86d1aa8SWill Deacon *
9e86d1aa8SWill Deacon * This driver is powered by bad coffee and bombay mix.
10e86d1aa8SWill Deacon */
11e86d1aa8SWill Deacon
12e86d1aa8SWill Deacon #include <linux/acpi.h>
13e86d1aa8SWill Deacon #include <linux/acpi_iort.h>
14e86d1aa8SWill Deacon #include <linux/bitops.h>
15e86d1aa8SWill Deacon #include <linux/crash_dump.h>
16e86d1aa8SWill Deacon #include <linux/delay.h>
17e86d1aa8SWill Deacon #include <linux/err.h>
18e86d1aa8SWill Deacon #include <linux/interrupt.h>
19e86d1aa8SWill Deacon #include <linux/io-pgtable.h>
20e86d1aa8SWill Deacon #include <linux/iopoll.h>
21e86d1aa8SWill Deacon #include <linux/module.h>
22e86d1aa8SWill Deacon #include <linux/msi.h>
23e86d1aa8SWill Deacon #include <linux/of.h>
24e86d1aa8SWill Deacon #include <linux/of_address.h>
25e86d1aa8SWill Deacon #include <linux/of_platform.h>
26e86d1aa8SWill Deacon #include <linux/pci.h>
27e86d1aa8SWill Deacon #include <linux/pci-ats.h>
28e86d1aa8SWill Deacon #include <linux/platform_device.h>
2956e1a4ccSJason Gunthorpe #include <kunit/visibility.h>
30eb054d67SJoao Martins #include <uapi/linux/iommufd.h>
31e86d1aa8SWill Deacon
32e881e783SJean-Philippe Brucker #include "arm-smmu-v3.h"
33f2042ed2SRobin Murphy #include "../../dma-iommu.h"
34e86d1aa8SWill Deacon
35bd07a20aSBarry Song static bool disable_msipolling;
36bd07a20aSBarry Song module_param(disable_msipolling, bool, 0444);
37bd07a20aSBarry Song MODULE_PARM_DESC(disable_msipolling,
38bd07a20aSBarry Song "Disable MSI-based polling for CMD_SYNC completion.");
39bd07a20aSBarry Song
4052acd7d8SShameer Kolothum static struct iommu_ops arm_smmu_ops;
41eb054d67SJoao Martins static struct iommu_dirty_ops arm_smmu_dirty_ops;
4252acd7d8SShameer Kolothum
43e86d1aa8SWill Deacon enum arm_smmu_msi_index {
44e86d1aa8SWill Deacon EVTQ_MSI_INDEX,
45e86d1aa8SWill Deacon GERROR_MSI_INDEX,
46e86d1aa8SWill Deacon PRIQ_MSI_INDEX,
47e86d1aa8SWill Deacon ARM_SMMU_MAX_MSIS,
48e86d1aa8SWill Deacon };
49e86d1aa8SWill Deacon
50de31c355SJason Gunthorpe #define NUM_ENTRY_QWORDS 8
51de31c355SJason Gunthorpe static_assert(sizeof(struct arm_smmu_ste) == NUM_ENTRY_QWORDS * sizeof(u64));
5278a5fbe8SJason Gunthorpe static_assert(sizeof(struct arm_smmu_cd) == NUM_ENTRY_QWORDS * sizeof(u64));
537da51af9SJason Gunthorpe
54e86d1aa8SWill Deacon static phys_addr_t arm_smmu_msi_cfg[ARM_SMMU_MAX_MSIS][3] = {
55e86d1aa8SWill Deacon [EVTQ_MSI_INDEX] = {
56e86d1aa8SWill Deacon ARM_SMMU_EVTQ_IRQ_CFG0,
57e86d1aa8SWill Deacon ARM_SMMU_EVTQ_IRQ_CFG1,
58e86d1aa8SWill Deacon ARM_SMMU_EVTQ_IRQ_CFG2,
59e86d1aa8SWill Deacon },
60e86d1aa8SWill Deacon [GERROR_MSI_INDEX] = {
61e86d1aa8SWill Deacon ARM_SMMU_GERROR_IRQ_CFG0,
62e86d1aa8SWill Deacon ARM_SMMU_GERROR_IRQ_CFG1,
63e86d1aa8SWill Deacon ARM_SMMU_GERROR_IRQ_CFG2,
64e86d1aa8SWill Deacon },
65e86d1aa8SWill Deacon [PRIQ_MSI_INDEX] = {
66e86d1aa8SWill Deacon ARM_SMMU_PRIQ_IRQ_CFG0,
67e86d1aa8SWill Deacon ARM_SMMU_PRIQ_IRQ_CFG1,
68e86d1aa8SWill Deacon ARM_SMMU_PRIQ_IRQ_CFG2,
69e86d1aa8SWill Deacon },
70e86d1aa8SWill Deacon };
71e86d1aa8SWill Deacon
72e86d1aa8SWill Deacon struct arm_smmu_option_prop {
73e86d1aa8SWill Deacon u32 opt;
74e86d1aa8SWill Deacon const char *prop;
75e86d1aa8SWill Deacon };
76e86d1aa8SWill Deacon
773f1ce8e8SJean-Philippe Brucker DEFINE_XARRAY_ALLOC1(arm_smmu_asid_xa);
783f1ce8e8SJean-Philippe Brucker DEFINE_MUTEX(arm_smmu_asid_lock);
79e86d1aa8SWill Deacon
80e86d1aa8SWill Deacon static struct arm_smmu_option_prop arm_smmu_options[] = {
81e86d1aa8SWill Deacon { ARM_SMMU_OPT_SKIP_PREFETCH, "hisilicon,broken-prefetch-cmd" },
82e86d1aa8SWill Deacon { ARM_SMMU_OPT_PAGE0_REGS_ONLY, "cavium,cn9900-broken-page1-regspace"},
83e86d1aa8SWill Deacon { 0, NULL},
84e86d1aa8SWill Deacon };
85e86d1aa8SWill Deacon
86d8cd2006SJason Gunthorpe static int arm_smmu_domain_finalise(struct arm_smmu_domain *smmu_domain,
87eb054d67SJoao Martins struct arm_smmu_device *smmu, u32 flags);
88b2f4c0fcSJason Gunthorpe static int arm_smmu_alloc_cd_tables(struct arm_smmu_master *master);
89d8cd2006SJason Gunthorpe
parse_driver_options(struct arm_smmu_device * smmu)90e86d1aa8SWill Deacon static void parse_driver_options(struct arm_smmu_device *smmu)
91e86d1aa8SWill Deacon {
92e86d1aa8SWill Deacon int i = 0;
93e86d1aa8SWill Deacon
94e86d1aa8SWill Deacon do {
95e86d1aa8SWill Deacon if (of_property_read_bool(smmu->dev->of_node,
96e86d1aa8SWill Deacon arm_smmu_options[i].prop)) {
97e86d1aa8SWill Deacon smmu->options |= arm_smmu_options[i].opt;
98e86d1aa8SWill Deacon dev_notice(smmu->dev, "option %s\n",
99e86d1aa8SWill Deacon arm_smmu_options[i].prop);
100e86d1aa8SWill Deacon }
101e86d1aa8SWill Deacon } while (arm_smmu_options[++i].opt);
102e86d1aa8SWill Deacon }
103e86d1aa8SWill Deacon
104e86d1aa8SWill Deacon /* Low-level queue manipulation functions */
queue_has_space(struct arm_smmu_ll_queue * q,u32 n)105e86d1aa8SWill Deacon static bool queue_has_space(struct arm_smmu_ll_queue *q, u32 n)
106e86d1aa8SWill Deacon {
107e86d1aa8SWill Deacon u32 space, prod, cons;
108e86d1aa8SWill Deacon
109e86d1aa8SWill Deacon prod = Q_IDX(q, q->prod);
110e86d1aa8SWill Deacon cons = Q_IDX(q, q->cons);
111e86d1aa8SWill Deacon
112e86d1aa8SWill Deacon if (Q_WRP(q, q->prod) == Q_WRP(q, q->cons))
113e86d1aa8SWill Deacon space = (1 << q->max_n_shift) - (prod - cons);
114e86d1aa8SWill Deacon else
115e86d1aa8SWill Deacon space = cons - prod;
116e86d1aa8SWill Deacon
117e86d1aa8SWill Deacon return space >= n;
118e86d1aa8SWill Deacon }
119e86d1aa8SWill Deacon
queue_full(struct arm_smmu_ll_queue * q)120e86d1aa8SWill Deacon static bool queue_full(struct arm_smmu_ll_queue *q)
121e86d1aa8SWill Deacon {
122e86d1aa8SWill Deacon return Q_IDX(q, q->prod) == Q_IDX(q, q->cons) &&
123e86d1aa8SWill Deacon Q_WRP(q, q->prod) != Q_WRP(q, q->cons);
124e86d1aa8SWill Deacon }
125e86d1aa8SWill Deacon
queue_empty(struct arm_smmu_ll_queue * q)126e86d1aa8SWill Deacon static bool queue_empty(struct arm_smmu_ll_queue *q)
127e86d1aa8SWill Deacon {
128e86d1aa8SWill Deacon return Q_IDX(q, q->prod) == Q_IDX(q, q->cons) &&
129e86d1aa8SWill Deacon Q_WRP(q, q->prod) == Q_WRP(q, q->cons);
130e86d1aa8SWill Deacon }
131e86d1aa8SWill Deacon
queue_consumed(struct arm_smmu_ll_queue * q,u32 prod)132e86d1aa8SWill Deacon static bool queue_consumed(struct arm_smmu_ll_queue *q, u32 prod)
133e86d1aa8SWill Deacon {
134e86d1aa8SWill Deacon return ((Q_WRP(q, q->cons) == Q_WRP(q, prod)) &&
135e86d1aa8SWill Deacon (Q_IDX(q, q->cons) > Q_IDX(q, prod))) ||
136e86d1aa8SWill Deacon ((Q_WRP(q, q->cons) != Q_WRP(q, prod)) &&
137e86d1aa8SWill Deacon (Q_IDX(q, q->cons) <= Q_IDX(q, prod)));
138e86d1aa8SWill Deacon }
139e86d1aa8SWill Deacon
queue_sync_cons_out(struct arm_smmu_queue * q)140e86d1aa8SWill Deacon static void queue_sync_cons_out(struct arm_smmu_queue *q)
141e86d1aa8SWill Deacon {
142e86d1aa8SWill Deacon /*
143e86d1aa8SWill Deacon * Ensure that all CPU accesses (reads and writes) to the queue
144e86d1aa8SWill Deacon * are complete before we update the cons pointer.
145e86d1aa8SWill Deacon */
146a76a3777SZhou Wang __iomb();
147e86d1aa8SWill Deacon writel_relaxed(q->llq.cons, q->cons_reg);
148e86d1aa8SWill Deacon }
149e86d1aa8SWill Deacon
queue_inc_cons(struct arm_smmu_ll_queue * q)150e86d1aa8SWill Deacon static void queue_inc_cons(struct arm_smmu_ll_queue *q)
151e86d1aa8SWill Deacon {
152e86d1aa8SWill Deacon u32 cons = (Q_WRP(q, q->cons) | Q_IDX(q, q->cons)) + 1;
153e86d1aa8SWill Deacon q->cons = Q_OVF(q->cons) | Q_WRP(q, cons) | Q_IDX(q, cons);
154e86d1aa8SWill Deacon }
155e86d1aa8SWill Deacon
queue_sync_cons_ovf(struct arm_smmu_queue * q)15667ea0b7cSTomas Krcka static void queue_sync_cons_ovf(struct arm_smmu_queue *q)
15767ea0b7cSTomas Krcka {
15867ea0b7cSTomas Krcka struct arm_smmu_ll_queue *llq = &q->llq;
15967ea0b7cSTomas Krcka
16067ea0b7cSTomas Krcka if (likely(Q_OVF(llq->prod) == Q_OVF(llq->cons)))
16167ea0b7cSTomas Krcka return;
16267ea0b7cSTomas Krcka
16367ea0b7cSTomas Krcka llq->cons = Q_OVF(llq->prod) | Q_WRP(llq, llq->cons) |
16467ea0b7cSTomas Krcka Q_IDX(llq, llq->cons);
16567ea0b7cSTomas Krcka queue_sync_cons_out(q);
16667ea0b7cSTomas Krcka }
16767ea0b7cSTomas Krcka
queue_sync_prod_in(struct arm_smmu_queue * q)168e86d1aa8SWill Deacon static int queue_sync_prod_in(struct arm_smmu_queue *q)
169e86d1aa8SWill Deacon {
170a76a3777SZhou Wang u32 prod;
171e86d1aa8SWill Deacon int ret = 0;
172a76a3777SZhou Wang
173a76a3777SZhou Wang /*
174a76a3777SZhou Wang * We can't use the _relaxed() variant here, as we must prevent
175a76a3777SZhou Wang * speculative reads of the queue before we have determined that
176a76a3777SZhou Wang * prod has indeed moved.
177a76a3777SZhou Wang */
178a76a3777SZhou Wang prod = readl(q->prod_reg);
179e86d1aa8SWill Deacon
180e86d1aa8SWill Deacon if (Q_OVF(prod) != Q_OVF(q->llq.prod))
181e86d1aa8SWill Deacon ret = -EOVERFLOW;
182e86d1aa8SWill Deacon
183e86d1aa8SWill Deacon q->llq.prod = prod;
184e86d1aa8SWill Deacon return ret;
185e86d1aa8SWill Deacon }
186e86d1aa8SWill Deacon
queue_inc_prod_n(struct arm_smmu_ll_queue * q,int n)187e86d1aa8SWill Deacon static u32 queue_inc_prod_n(struct arm_smmu_ll_queue *q, int n)
188e86d1aa8SWill Deacon {
189e86d1aa8SWill Deacon u32 prod = (Q_WRP(q, q->prod) | Q_IDX(q, q->prod)) + n;
190e86d1aa8SWill Deacon return Q_OVF(q->prod) | Q_WRP(q, prod) | Q_IDX(q, prod);
191e86d1aa8SWill Deacon }
192e86d1aa8SWill Deacon
queue_poll_init(struct arm_smmu_device * smmu,struct arm_smmu_queue_poll * qp)193e86d1aa8SWill Deacon static void queue_poll_init(struct arm_smmu_device *smmu,
194e86d1aa8SWill Deacon struct arm_smmu_queue_poll *qp)
195e86d1aa8SWill Deacon {
196e86d1aa8SWill Deacon qp->delay = 1;
197e86d1aa8SWill Deacon qp->spin_cnt = 0;
198e86d1aa8SWill Deacon qp->wfe = !!(smmu->features & ARM_SMMU_FEAT_SEV);
199e86d1aa8SWill Deacon qp->timeout = ktime_add_us(ktime_get(), ARM_SMMU_POLL_TIMEOUT_US);
200e86d1aa8SWill Deacon }
201e86d1aa8SWill Deacon
queue_poll(struct arm_smmu_queue_poll * qp)202e86d1aa8SWill Deacon static int queue_poll(struct arm_smmu_queue_poll *qp)
203e86d1aa8SWill Deacon {
204e86d1aa8SWill Deacon if (ktime_compare(ktime_get(), qp->timeout) > 0)
205e86d1aa8SWill Deacon return -ETIMEDOUT;
206e86d1aa8SWill Deacon
207e86d1aa8SWill Deacon if (qp->wfe) {
208e86d1aa8SWill Deacon wfe();
209e86d1aa8SWill Deacon } else if (++qp->spin_cnt < ARM_SMMU_POLL_SPIN_COUNT) {
210e86d1aa8SWill Deacon cpu_relax();
211e86d1aa8SWill Deacon } else {
212e86d1aa8SWill Deacon udelay(qp->delay);
213e86d1aa8SWill Deacon qp->delay *= 2;
214e86d1aa8SWill Deacon qp->spin_cnt = 0;
215e86d1aa8SWill Deacon }
216e86d1aa8SWill Deacon
217e86d1aa8SWill Deacon return 0;
218e86d1aa8SWill Deacon }
219e86d1aa8SWill Deacon
queue_write(__le64 * dst,u64 * src,size_t n_dwords)220e86d1aa8SWill Deacon static void queue_write(__le64 *dst, u64 *src, size_t n_dwords)
221e86d1aa8SWill Deacon {
222e86d1aa8SWill Deacon int i;
223e86d1aa8SWill Deacon
224e86d1aa8SWill Deacon for (i = 0; i < n_dwords; ++i)
225e86d1aa8SWill Deacon *dst++ = cpu_to_le64(*src++);
226e86d1aa8SWill Deacon }
227e86d1aa8SWill Deacon
queue_read(u64 * dst,__le64 * src,size_t n_dwords)228376cdf66SJean-Philippe Brucker static void queue_read(u64 *dst, __le64 *src, size_t n_dwords)
229e86d1aa8SWill Deacon {
230e86d1aa8SWill Deacon int i;
231e86d1aa8SWill Deacon
232e86d1aa8SWill Deacon for (i = 0; i < n_dwords; ++i)
233e86d1aa8SWill Deacon *dst++ = le64_to_cpu(*src++);
234e86d1aa8SWill Deacon }
235e86d1aa8SWill Deacon
queue_remove_raw(struct arm_smmu_queue * q,u64 * ent)236e86d1aa8SWill Deacon static int queue_remove_raw(struct arm_smmu_queue *q, u64 *ent)
237e86d1aa8SWill Deacon {
238e86d1aa8SWill Deacon if (queue_empty(&q->llq))
239e86d1aa8SWill Deacon return -EAGAIN;
240e86d1aa8SWill Deacon
241e86d1aa8SWill Deacon queue_read(ent, Q_ENT(q, q->llq.cons), q->ent_dwords);
242e86d1aa8SWill Deacon queue_inc_cons(&q->llq);
243e86d1aa8SWill Deacon queue_sync_cons_out(q);
244e86d1aa8SWill Deacon return 0;
245e86d1aa8SWill Deacon }
246e86d1aa8SWill Deacon
247e86d1aa8SWill Deacon /* High-level queue accessors */
arm_smmu_cmdq_build_cmd(u64 * cmd,struct arm_smmu_cmdq_ent * ent)248e86d1aa8SWill Deacon static int arm_smmu_cmdq_build_cmd(u64 *cmd, struct arm_smmu_cmdq_ent *ent)
249e86d1aa8SWill Deacon {
250e86d1aa8SWill Deacon memset(cmd, 0, 1 << CMDQ_ENT_SZ_SHIFT);
251e86d1aa8SWill Deacon cmd[0] |= FIELD_PREP(CMDQ_0_OP, ent->opcode);
252e86d1aa8SWill Deacon
253e86d1aa8SWill Deacon switch (ent->opcode) {
254e86d1aa8SWill Deacon case CMDQ_OP_TLBI_EL2_ALL:
255e86d1aa8SWill Deacon case CMDQ_OP_TLBI_NSNH_ALL:
256e86d1aa8SWill Deacon break;
257e86d1aa8SWill Deacon case CMDQ_OP_PREFETCH_CFG:
258e86d1aa8SWill Deacon cmd[0] |= FIELD_PREP(CMDQ_PREFETCH_0_SID, ent->prefetch.sid);
259e86d1aa8SWill Deacon break;
260e86d1aa8SWill Deacon case CMDQ_OP_CFGI_CD:
261e86d1aa8SWill Deacon cmd[0] |= FIELD_PREP(CMDQ_CFGI_0_SSID, ent->cfgi.ssid);
262df561f66SGustavo A. R. Silva fallthrough;
263e86d1aa8SWill Deacon case CMDQ_OP_CFGI_STE:
264e86d1aa8SWill Deacon cmd[0] |= FIELD_PREP(CMDQ_CFGI_0_SID, ent->cfgi.sid);
265e86d1aa8SWill Deacon cmd[1] |= FIELD_PREP(CMDQ_CFGI_1_LEAF, ent->cfgi.leaf);
266e86d1aa8SWill Deacon break;
267e86d1aa8SWill Deacon case CMDQ_OP_CFGI_CD_ALL:
268e86d1aa8SWill Deacon cmd[0] |= FIELD_PREP(CMDQ_CFGI_0_SID, ent->cfgi.sid);
269e86d1aa8SWill Deacon break;
270e86d1aa8SWill Deacon case CMDQ_OP_CFGI_ALL:
271e86d1aa8SWill Deacon /* Cover the entire SID range */
272e86d1aa8SWill Deacon cmd[1] |= FIELD_PREP(CMDQ_CFGI_1_RANGE, 31);
273e86d1aa8SWill Deacon break;
274e86d1aa8SWill Deacon case CMDQ_OP_TLBI_NH_VA:
2759111aebfSJean-Philippe Brucker cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_VMID, ent->tlbi.vmid);
2769111aebfSJean-Philippe Brucker fallthrough;
2779111aebfSJean-Philippe Brucker case CMDQ_OP_TLBI_EL2_VA:
278e86d1aa8SWill Deacon cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_NUM, ent->tlbi.num);
279e86d1aa8SWill Deacon cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_SCALE, ent->tlbi.scale);
280e86d1aa8SWill Deacon cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_ASID, ent->tlbi.asid);
281e86d1aa8SWill Deacon cmd[1] |= FIELD_PREP(CMDQ_TLBI_1_LEAF, ent->tlbi.leaf);
282e86d1aa8SWill Deacon cmd[1] |= FIELD_PREP(CMDQ_TLBI_1_TTL, ent->tlbi.ttl);
283e86d1aa8SWill Deacon cmd[1] |= FIELD_PREP(CMDQ_TLBI_1_TG, ent->tlbi.tg);
284e86d1aa8SWill Deacon cmd[1] |= ent->tlbi.addr & CMDQ_TLBI_1_VA_MASK;
285e86d1aa8SWill Deacon break;
286e86d1aa8SWill Deacon case CMDQ_OP_TLBI_S2_IPA:
287e86d1aa8SWill Deacon cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_NUM, ent->tlbi.num);
288e86d1aa8SWill Deacon cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_SCALE, ent->tlbi.scale);
289e86d1aa8SWill Deacon cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_VMID, ent->tlbi.vmid);
290e86d1aa8SWill Deacon cmd[1] |= FIELD_PREP(CMDQ_TLBI_1_LEAF, ent->tlbi.leaf);
291e86d1aa8SWill Deacon cmd[1] |= FIELD_PREP(CMDQ_TLBI_1_TTL, ent->tlbi.ttl);
292e86d1aa8SWill Deacon cmd[1] |= FIELD_PREP(CMDQ_TLBI_1_TG, ent->tlbi.tg);
293e86d1aa8SWill Deacon cmd[1] |= ent->tlbi.addr & CMDQ_TLBI_1_IPA_MASK;
294e86d1aa8SWill Deacon break;
295e86d1aa8SWill Deacon case CMDQ_OP_TLBI_NH_ASID:
296e86d1aa8SWill Deacon cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_ASID, ent->tlbi.asid);
297df561f66SGustavo A. R. Silva fallthrough;
298e86d1aa8SWill Deacon case CMDQ_OP_TLBI_S12_VMALL:
299e86d1aa8SWill Deacon cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_VMID, ent->tlbi.vmid);
300e86d1aa8SWill Deacon break;
3019111aebfSJean-Philippe Brucker case CMDQ_OP_TLBI_EL2_ASID:
3029111aebfSJean-Philippe Brucker cmd[0] |= FIELD_PREP(CMDQ_TLBI_0_ASID, ent->tlbi.asid);
3039111aebfSJean-Philippe Brucker break;
304e86d1aa8SWill Deacon case CMDQ_OP_ATC_INV:
305e86d1aa8SWill Deacon cmd[0] |= FIELD_PREP(CMDQ_0_SSV, ent->substream_valid);
306e86d1aa8SWill Deacon cmd[0] |= FIELD_PREP(CMDQ_ATC_0_GLOBAL, ent->atc.global);
307e86d1aa8SWill Deacon cmd[0] |= FIELD_PREP(CMDQ_ATC_0_SSID, ent->atc.ssid);
308e86d1aa8SWill Deacon cmd[0] |= FIELD_PREP(CMDQ_ATC_0_SID, ent->atc.sid);
309e86d1aa8SWill Deacon cmd[1] |= FIELD_PREP(CMDQ_ATC_1_SIZE, ent->atc.size);
310e86d1aa8SWill Deacon cmd[1] |= ent->atc.addr & CMDQ_ATC_1_ADDR_MASK;
311e86d1aa8SWill Deacon break;
312e86d1aa8SWill Deacon case CMDQ_OP_PRI_RESP:
313e86d1aa8SWill Deacon cmd[0] |= FIELD_PREP(CMDQ_0_SSV, ent->substream_valid);
314e86d1aa8SWill Deacon cmd[0] |= FIELD_PREP(CMDQ_PRI_0_SSID, ent->pri.ssid);
315e86d1aa8SWill Deacon cmd[0] |= FIELD_PREP(CMDQ_PRI_0_SID, ent->pri.sid);
316e86d1aa8SWill Deacon cmd[1] |= FIELD_PREP(CMDQ_PRI_1_GRPID, ent->pri.grpid);
317e86d1aa8SWill Deacon switch (ent->pri.resp) {
318e86d1aa8SWill Deacon case PRI_RESP_DENY:
319e86d1aa8SWill Deacon case PRI_RESP_FAIL:
320e86d1aa8SWill Deacon case PRI_RESP_SUCC:
321e86d1aa8SWill Deacon break;
322e86d1aa8SWill Deacon default:
323e86d1aa8SWill Deacon return -EINVAL;
324e86d1aa8SWill Deacon }
325e86d1aa8SWill Deacon cmd[1] |= FIELD_PREP(CMDQ_PRI_1_RESP, ent->pri.resp);
326e86d1aa8SWill Deacon break;
327395ad89dSJean-Philippe Brucker case CMDQ_OP_RESUME:
328395ad89dSJean-Philippe Brucker cmd[0] |= FIELD_PREP(CMDQ_RESUME_0_SID, ent->resume.sid);
329395ad89dSJean-Philippe Brucker cmd[0] |= FIELD_PREP(CMDQ_RESUME_0_RESP, ent->resume.resp);
330395ad89dSJean-Philippe Brucker cmd[1] |= FIELD_PREP(CMDQ_RESUME_1_STAG, ent->resume.stag);
331395ad89dSJean-Philippe Brucker break;
332e86d1aa8SWill Deacon case CMDQ_OP_CMD_SYNC:
333e86d1aa8SWill Deacon if (ent->sync.msiaddr) {
334e86d1aa8SWill Deacon cmd[0] |= FIELD_PREP(CMDQ_SYNC_0_CS, CMDQ_SYNC_0_CS_IRQ);
335e86d1aa8SWill Deacon cmd[1] |= ent->sync.msiaddr & CMDQ_SYNC_1_MSIADDR_MASK;
336e86d1aa8SWill Deacon } else {
337e86d1aa8SWill Deacon cmd[0] |= FIELD_PREP(CMDQ_SYNC_0_CS, CMDQ_SYNC_0_CS_SEV);
338e86d1aa8SWill Deacon }
339e86d1aa8SWill Deacon cmd[0] |= FIELD_PREP(CMDQ_SYNC_0_MSH, ARM_SMMU_SH_ISH);
340e86d1aa8SWill Deacon cmd[0] |= FIELD_PREP(CMDQ_SYNC_0_MSIATTR, ARM_SMMU_MEMATTR_OIWB);
341e86d1aa8SWill Deacon break;
342e86d1aa8SWill Deacon default:
343e86d1aa8SWill Deacon return -ENOENT;
344e86d1aa8SWill Deacon }
345e86d1aa8SWill Deacon
346e86d1aa8SWill Deacon return 0;
347e86d1aa8SWill Deacon }
348e86d1aa8SWill Deacon
arm_smmu_get_cmdq(struct arm_smmu_device * smmu,struct arm_smmu_cmdq_ent * ent)3498639cc83SZhen Lei static struct arm_smmu_cmdq *arm_smmu_get_cmdq(struct arm_smmu_device *smmu,
350e86d1aa8SWill Deacon struct arm_smmu_cmdq_ent *ent)
3518639cc83SZhen Lei {
3528639cc83SZhen Lei struct arm_smmu_cmdq *cmdq = NULL;
3538639cc83SZhen Lei
3548639cc83SZhen Lei if (smmu->impl_ops && smmu->impl_ops->get_secondary_cmdq)
3558639cc83SZhen Lei cmdq = smmu->impl_ops->get_secondary_cmdq(smmu, ent);
3568639cc83SZhen Lei
357e86d1aa8SWill Deacon return cmdq ?: &smmu->cmdq;
358e86d1aa8SWill Deacon }
359e86d1aa8SWill Deacon
arm_smmu_cmdq_needs_busy_polling(struct arm_smmu_device * smmu,struct arm_smmu_cmdq * cmdq)360e86d1aa8SWill Deacon static bool arm_smmu_cmdq_needs_busy_polling(struct arm_smmu_device *smmu,
361e86d1aa8SWill Deacon struct arm_smmu_cmdq *cmdq)
362e86d1aa8SWill Deacon {
363e86d1aa8SWill Deacon if (cmdq == &smmu->cmdq)
364e86d1aa8SWill Deacon return false;
365bd07a20aSBarry Song
366e86d1aa8SWill Deacon return smmu->options & ARM_SMMU_OPT_TEGRA241_CMDQV;
367e86d1aa8SWill Deacon }
368e86d1aa8SWill Deacon
arm_smmu_cmdq_build_sync_cmd(u64 * cmd,struct arm_smmu_device * smmu,struct arm_smmu_cmdq * cmdq,u32 prod)369e86d1aa8SWill Deacon static void arm_smmu_cmdq_build_sync_cmd(u64 *cmd, struct arm_smmu_device *smmu,
370e86d1aa8SWill Deacon struct arm_smmu_cmdq *cmdq, u32 prod)
371e86d1aa8SWill Deacon {
372e86d1aa8SWill Deacon struct arm_smmu_queue *q = &cmdq->q;
3732cbeaf3fSZhen Lei struct arm_smmu_cmdq_ent ent = {
3742cbeaf3fSZhen Lei .opcode = CMDQ_OP_CMD_SYNC,
375e86d1aa8SWill Deacon };
376d56d5162SBixuan Cui
377e86d1aa8SWill Deacon /*
378e86d1aa8SWill Deacon * Beware that Hi16xx adds an extra 32 bits of goodness to its MSI
379e86d1aa8SWill Deacon * payload, so the write will zero the entire command on that platform.
380e86d1aa8SWill Deacon */
381e86d1aa8SWill Deacon if (smmu->options & ARM_SMMU_OPT_MSIPOLL) {
382e86d1aa8SWill Deacon ent.sync.msiaddr = q->base_dma + Q_IDX(&q->llq, prod) *
383e86d1aa8SWill Deacon q->ent_dwords * 8;
384e86d1aa8SWill Deacon }
385e86d1aa8SWill Deacon
386e86d1aa8SWill Deacon arm_smmu_cmdq_build_cmd(cmd, &ent);
387e86d1aa8SWill Deacon if (arm_smmu_cmdq_needs_busy_polling(smmu, cmdq))
388e86d1aa8SWill Deacon u64p_replace_bits(cmd, CMDQ_SYNC_0_CS_NONE, CMDQ_SYNC_0_CS);
389e86d1aa8SWill Deacon }
390e86d1aa8SWill Deacon
__arm_smmu_cmdq_skip_err(struct arm_smmu_device * smmu,struct arm_smmu_cmdq * cmdq)391e86d1aa8SWill Deacon void __arm_smmu_cmdq_skip_err(struct arm_smmu_device *smmu,
392e86d1aa8SWill Deacon struct arm_smmu_cmdq *cmdq)
393e86d1aa8SWill Deacon {
394e86d1aa8SWill Deacon static const char * const cerror_str[] = {
395e86d1aa8SWill Deacon [CMDQ_ERR_CERROR_NONE_IDX] = "No error",
396e86d1aa8SWill Deacon [CMDQ_ERR_CERROR_ILL_IDX] = "Illegal command",
3975a1ab5c0SGustavo A. R. Silva [CMDQ_ERR_CERROR_ABT_IDX] = "Abort on command fetch",
398e86d1aa8SWill Deacon [CMDQ_ERR_CERROR_ATC_INV_IDX] = "ATC invalidate timeout",
399e86d1aa8SWill Deacon };
400e86d1aa8SWill Deacon struct arm_smmu_queue *q = &cmdq->q;
401e86d1aa8SWill Deacon
402e86d1aa8SWill Deacon int i;
403e86d1aa8SWill Deacon u64 cmd[CMDQ_ENT_DWORDS];
404e86d1aa8SWill Deacon u32 cons = readl_relaxed(q->cons_reg);
405e86d1aa8SWill Deacon u32 idx = FIELD_GET(CMDQ_CONS_ERR, cons);
406e86d1aa8SWill Deacon struct arm_smmu_cmdq_ent cmd_sync = {
407e86d1aa8SWill Deacon .opcode = CMDQ_OP_CMD_SYNC,
408e86d1aa8SWill Deacon };
409e86d1aa8SWill Deacon
410e86d1aa8SWill Deacon dev_err(smmu->dev, "CMDQ error (cons 0x%08x): %s\n", cons,
411e86d1aa8SWill Deacon idx < ARRAY_SIZE(cerror_str) ? cerror_str[idx] : "Unknown");
412e86d1aa8SWill Deacon
413e86d1aa8SWill Deacon switch (idx) {
414e86d1aa8SWill Deacon case CMDQ_ERR_CERROR_ABT_IDX:
415e86d1aa8SWill Deacon dev_err(smmu->dev, "retrying command fetch\n");
416e86d1aa8SWill Deacon return;
417e86d1aa8SWill Deacon case CMDQ_ERR_CERROR_NONE_IDX:
418e86d1aa8SWill Deacon return;
419e86d1aa8SWill Deacon case CMDQ_ERR_CERROR_ATC_INV_IDX:
420e86d1aa8SWill Deacon /*
421e86d1aa8SWill Deacon * ATC Invalidation Completion timeout. CONS is still pointing
422e86d1aa8SWill Deacon * at the CMD_SYNC. Attempt to complete other pending commands
42359d9bd72SZhen Lei * by repeating the CMD_SYNC, though we might well end up back
424e86d1aa8SWill Deacon * here since the ATC invalidation may still be pending.
425e86d1aa8SWill Deacon */
426e86d1aa8SWill Deacon return;
427e86d1aa8SWill Deacon case CMDQ_ERR_CERROR_ILL_IDX:
4282cbeaf3fSZhen Lei default:
4292cbeaf3fSZhen Lei break;
4302cbeaf3fSZhen Lei }
4312cbeaf3fSZhen Lei
4322cbeaf3fSZhen Lei /*
433e86d1aa8SWill Deacon * We may have concurrent producers, so we need to be careful
434e86d1aa8SWill Deacon * not to touch any of the shadow cmdq state.
435e86d1aa8SWill Deacon */
436e86d1aa8SWill Deacon queue_read(cmd, Q_ENT(q, cons), q->ent_dwords);
437e86d1aa8SWill Deacon dev_err(smmu->dev, "skipping command in error state:\n");
438e86d1aa8SWill Deacon for (i = 0; i < ARRAY_SIZE(cmd); ++i)
439e86d1aa8SWill Deacon dev_err(smmu->dev, "\t0x%016llx\n", (unsigned long long)cmd[i]);
440e86d1aa8SWill Deacon
441e86d1aa8SWill Deacon /* Convert the erroneous command into a CMD_SYNC */
442e86d1aa8SWill Deacon arm_smmu_cmdq_build_cmd(cmd, &cmd_sync);
443e86d1aa8SWill Deacon if (arm_smmu_cmdq_needs_busy_polling(smmu, cmdq))
444e86d1aa8SWill Deacon u64p_replace_bits(cmd, CMDQ_SYNC_0_CS_NONE, CMDQ_SYNC_0_CS);
445e86d1aa8SWill Deacon
446e86d1aa8SWill Deacon queue_write(Q_ENT(q, cons), cmd, q->ent_dwords);
447e86d1aa8SWill Deacon }
448e86d1aa8SWill Deacon
arm_smmu_cmdq_skip_err(struct arm_smmu_device * smmu)449e86d1aa8SWill Deacon static void arm_smmu_cmdq_skip_err(struct arm_smmu_device *smmu)
450e86d1aa8SWill Deacon {
451e86d1aa8SWill Deacon __arm_smmu_cmdq_skip_err(smmu, &smmu->cmdq);
452e86d1aa8SWill Deacon }
453e86d1aa8SWill Deacon
454e86d1aa8SWill Deacon /*
455e86d1aa8SWill Deacon * Command queue locking.
456e86d1aa8SWill Deacon * This is a form of bastardised rwlock with the following major changes:
457e86d1aa8SWill Deacon *
458e86d1aa8SWill Deacon * - The only LOCK routines are exclusive_trylock() and shared_lock().
459e86d1aa8SWill Deacon * Neither have barrier semantics, and instead provide only a control
460e86d1aa8SWill Deacon * dependency.
461e86d1aa8SWill Deacon *
462e86d1aa8SWill Deacon * - The UNLOCK routines are supplemented with shared_tryunlock(), which
463e86d1aa8SWill Deacon * fails if the caller appears to be the last lock holder (yes, this is
464e86d1aa8SWill Deacon * racy). All successful UNLOCK routines have RELEASE semantics.
465e86d1aa8SWill Deacon */
arm_smmu_cmdq_shared_lock(struct arm_smmu_cmdq * cmdq)466e86d1aa8SWill Deacon static void arm_smmu_cmdq_shared_lock(struct arm_smmu_cmdq *cmdq)
467e86d1aa8SWill Deacon {
468e86d1aa8SWill Deacon int val;
469e86d1aa8SWill Deacon
470e86d1aa8SWill Deacon /*
471e86d1aa8SWill Deacon * We can try to avoid the cmpxchg() loop by simply incrementing the
472e86d1aa8SWill Deacon * lock counter. When held in exclusive state, the lock counter is set
473e86d1aa8SWill Deacon * to INT_MIN so these increments won't hurt as the value will remain
474e86d1aa8SWill Deacon * negative.
475e86d1aa8SWill Deacon */
476e86d1aa8SWill Deacon if (atomic_fetch_inc_relaxed(&cmdq->lock) >= 0)
477e86d1aa8SWill Deacon return;
478e86d1aa8SWill Deacon
479e86d1aa8SWill Deacon do {
480e86d1aa8SWill Deacon val = atomic_cond_read_relaxed(&cmdq->lock, VAL >= 0);
481e86d1aa8SWill Deacon } while (atomic_cmpxchg_relaxed(&cmdq->lock, val, val + 1) != val);
482e86d1aa8SWill Deacon }
483e86d1aa8SWill Deacon
arm_smmu_cmdq_shared_unlock(struct arm_smmu_cmdq * cmdq)484e86d1aa8SWill Deacon static void arm_smmu_cmdq_shared_unlock(struct arm_smmu_cmdq *cmdq)
485e86d1aa8SWill Deacon {
486e86d1aa8SWill Deacon (void)atomic_dec_return_release(&cmdq->lock);
487e86d1aa8SWill Deacon }
488e86d1aa8SWill Deacon
arm_smmu_cmdq_shared_tryunlock(struct arm_smmu_cmdq * cmdq)489e86d1aa8SWill Deacon static bool arm_smmu_cmdq_shared_tryunlock(struct arm_smmu_cmdq *cmdq)
490e86d1aa8SWill Deacon {
491e86d1aa8SWill Deacon if (atomic_read(&cmdq->lock) == 1)
492e86d1aa8SWill Deacon return false;
493e86d1aa8SWill Deacon
494e86d1aa8SWill Deacon arm_smmu_cmdq_shared_unlock(cmdq);
495e86d1aa8SWill Deacon return true;
496e86d1aa8SWill Deacon }
497e86d1aa8SWill Deacon
498e86d1aa8SWill Deacon #define arm_smmu_cmdq_exclusive_trylock_irqsave(cmdq, flags) \
499e86d1aa8SWill Deacon ({ \
500e86d1aa8SWill Deacon bool __ret; \
501e86d1aa8SWill Deacon local_irq_save(flags); \
502e86d1aa8SWill Deacon __ret = !atomic_cmpxchg_relaxed(&cmdq->lock, 0, INT_MIN); \
503e86d1aa8SWill Deacon if (!__ret) \
504e86d1aa8SWill Deacon local_irq_restore(flags); \
505e86d1aa8SWill Deacon __ret; \
506e86d1aa8SWill Deacon })
507e86d1aa8SWill Deacon
508e86d1aa8SWill Deacon #define arm_smmu_cmdq_exclusive_unlock_irqrestore(cmdq, flags) \
509e86d1aa8SWill Deacon ({ \
510e86d1aa8SWill Deacon atomic_set_release(&cmdq->lock, 0); \
511e86d1aa8SWill Deacon local_irq_restore(flags); \
512e86d1aa8SWill Deacon })
513e86d1aa8SWill Deacon
514e86d1aa8SWill Deacon
515e86d1aa8SWill Deacon /*
516e86d1aa8SWill Deacon * Command queue insertion.
517e86d1aa8SWill Deacon * This is made fiddly by our attempts to achieve some sort of scalability
518e86d1aa8SWill Deacon * since there is one queue shared amongst all of the CPUs in the system. If
519e86d1aa8SWill Deacon * you like mixed-size concurrency, dependency ordering and relaxed atomics,
520e86d1aa8SWill Deacon * then you'll *love* this monstrosity.
521e86d1aa8SWill Deacon *
522e86d1aa8SWill Deacon * The basic idea is to split the queue up into ranges of commands that are
523e86d1aa8SWill Deacon * owned by a given CPU; the owner may not have written all of the commands
524e86d1aa8SWill Deacon * itself, but is responsible for advancing the hardware prod pointer when
525e86d1aa8SWill Deacon * the time comes. The algorithm is roughly:
526e86d1aa8SWill Deacon *
527e86d1aa8SWill Deacon * 1. Allocate some space in the queue. At this point we also discover
528e86d1aa8SWill Deacon * whether the head of the queue is currently owned by another CPU,
529e86d1aa8SWill Deacon * or whether we are the owner.
530e86d1aa8SWill Deacon *
531e86d1aa8SWill Deacon * 2. Write our commands into our allocated slots in the queue.
532e86d1aa8SWill Deacon *
533e86d1aa8SWill Deacon * 3. Mark our slots as valid in arm_smmu_cmdq.valid_map.
534e86d1aa8SWill Deacon *
535e86d1aa8SWill Deacon * 4. If we are an owner:
536e86d1aa8SWill Deacon * a. Wait for the previous owner to finish.
537e86d1aa8SWill Deacon * b. Mark the queue head as unowned, which tells us the range
538e86d1aa8SWill Deacon * that we are responsible for publishing.
539e86d1aa8SWill Deacon * c. Wait for all commands in our owned range to become valid.
540e86d1aa8SWill Deacon * d. Advance the hardware prod pointer.
541e86d1aa8SWill Deacon * e. Tell the next owner we've finished.
542e86d1aa8SWill Deacon *
543e86d1aa8SWill Deacon * 5. If we are inserting a CMD_SYNC (we may or may not have been an
544e86d1aa8SWill Deacon * owner), then we need to stick around until it has completed:
545e86d1aa8SWill Deacon * a. If we have MSIs, the SMMU can write back into the CMD_SYNC
546e86d1aa8SWill Deacon * to clear the first 4 bytes.
547e86d1aa8SWill Deacon * b. Otherwise, we spin waiting for the hardware cons pointer to
548e86d1aa8SWill Deacon * advance past our command.
549e86d1aa8SWill Deacon *
550e86d1aa8SWill Deacon * The devil is in the details, particularly the use of locking for handling
551e86d1aa8SWill Deacon * SYNC completion and freeing up space in the queue before we think that it is
552e86d1aa8SWill Deacon * full.
553e86d1aa8SWill Deacon */
__arm_smmu_cmdq_poll_set_valid_map(struct arm_smmu_cmdq * cmdq,u32 sprod,u32 eprod,bool set)554e86d1aa8SWill Deacon static void __arm_smmu_cmdq_poll_set_valid_map(struct arm_smmu_cmdq *cmdq,
555e86d1aa8SWill Deacon u32 sprod, u32 eprod, bool set)
556e86d1aa8SWill Deacon {
557e86d1aa8SWill Deacon u32 swidx, sbidx, ewidx, ebidx;
558e86d1aa8SWill Deacon struct arm_smmu_ll_queue llq = {
559e86d1aa8SWill Deacon .max_n_shift = cmdq->q.llq.max_n_shift,
560e86d1aa8SWill Deacon .prod = sprod,
561e86d1aa8SWill Deacon };
562e86d1aa8SWill Deacon
563e86d1aa8SWill Deacon ewidx = BIT_WORD(Q_IDX(&llq, eprod));
564e86d1aa8SWill Deacon ebidx = Q_IDX(&llq, eprod) % BITS_PER_LONG;
565e86d1aa8SWill Deacon
566e86d1aa8SWill Deacon while (llq.prod != eprod) {
567e86d1aa8SWill Deacon unsigned long mask;
568e86d1aa8SWill Deacon atomic_long_t *ptr;
569e86d1aa8SWill Deacon u32 limit = BITS_PER_LONG;
570e86d1aa8SWill Deacon
571e86d1aa8SWill Deacon swidx = BIT_WORD(Q_IDX(&llq, llq.prod));
572e86d1aa8SWill Deacon sbidx = Q_IDX(&llq, llq.prod) % BITS_PER_LONG;
573e86d1aa8SWill Deacon
574e86d1aa8SWill Deacon ptr = &cmdq->valid_map[swidx];
575e86d1aa8SWill Deacon
576e86d1aa8SWill Deacon if ((swidx == ewidx) && (sbidx < ebidx))
577e86d1aa8SWill Deacon limit = ebidx;
578e86d1aa8SWill Deacon
579e86d1aa8SWill Deacon mask = GENMASK(limit - 1, sbidx);
580e86d1aa8SWill Deacon
581e86d1aa8SWill Deacon /*
582e86d1aa8SWill Deacon * The valid bit is the inverse of the wrap bit. This means
583e86d1aa8SWill Deacon * that a zero-initialised queue is invalid and, after marking
584e86d1aa8SWill Deacon * all entries as valid, they become invalid again when we
585e86d1aa8SWill Deacon * wrap.
586e86d1aa8SWill Deacon */
587e86d1aa8SWill Deacon if (set) {
588e86d1aa8SWill Deacon atomic_long_xor(mask, ptr);
589e86d1aa8SWill Deacon } else { /* Poll */
590e86d1aa8SWill Deacon unsigned long valid;
591e86d1aa8SWill Deacon
592e86d1aa8SWill Deacon valid = (ULONG_MAX + !!Q_WRP(&llq, llq.prod)) & mask;
593e86d1aa8SWill Deacon atomic_long_cond_read_relaxed(ptr, (VAL & mask) == valid);
594e86d1aa8SWill Deacon }
595e86d1aa8SWill Deacon
596e86d1aa8SWill Deacon llq.prod = queue_inc_prod_n(&llq, limit - sbidx);
597e86d1aa8SWill Deacon }
598e86d1aa8SWill Deacon }
5998639cc83SZhen Lei
600e86d1aa8SWill Deacon /* Mark all entries in the range [sprod, eprod) as valid */
arm_smmu_cmdq_set_valid_map(struct arm_smmu_cmdq * cmdq,u32 sprod,u32 eprod)601e86d1aa8SWill Deacon static void arm_smmu_cmdq_set_valid_map(struct arm_smmu_cmdq *cmdq,
602e86d1aa8SWill Deacon u32 sprod, u32 eprod)
603e86d1aa8SWill Deacon {
604e86d1aa8SWill Deacon __arm_smmu_cmdq_poll_set_valid_map(cmdq, sprod, eprod, true);
605e86d1aa8SWill Deacon }
606e86d1aa8SWill Deacon
607e86d1aa8SWill Deacon /* Wait for all entries in the range [sprod, eprod) to become valid */
arm_smmu_cmdq_poll_valid_map(struct arm_smmu_cmdq * cmdq,u32 sprod,u32 eprod)608e86d1aa8SWill Deacon static void arm_smmu_cmdq_poll_valid_map(struct arm_smmu_cmdq *cmdq,
609e86d1aa8SWill Deacon u32 sprod, u32 eprod)
610e86d1aa8SWill Deacon {
611e86d1aa8SWill Deacon __arm_smmu_cmdq_poll_set_valid_map(cmdq, sprod, eprod, false);
612e86d1aa8SWill Deacon }
613e86d1aa8SWill Deacon
614e86d1aa8SWill Deacon /* Wait for the command queue to become non-full */
arm_smmu_cmdq_poll_until_not_full(struct arm_smmu_device * smmu,struct arm_smmu_cmdq * cmdq,struct arm_smmu_ll_queue * llq)6158639cc83SZhen Lei static int arm_smmu_cmdq_poll_until_not_full(struct arm_smmu_device *smmu,
616e86d1aa8SWill Deacon struct arm_smmu_cmdq *cmdq,
617e86d1aa8SWill Deacon struct arm_smmu_ll_queue *llq)
618e86d1aa8SWill Deacon {
619e86d1aa8SWill Deacon unsigned long flags;
620e86d1aa8SWill Deacon struct arm_smmu_queue_poll qp;
621e86d1aa8SWill Deacon int ret = 0;
622e86d1aa8SWill Deacon
623e86d1aa8SWill Deacon /*
624e86d1aa8SWill Deacon * Try to update our copy of cons by grabbing exclusive cmdq access. If
625e86d1aa8SWill Deacon * that fails, spin until somebody else updates it for us.
626e86d1aa8SWill Deacon */
627e86d1aa8SWill Deacon if (arm_smmu_cmdq_exclusive_trylock_irqsave(cmdq, flags)) {
628e86d1aa8SWill Deacon WRITE_ONCE(cmdq->q.llq.cons, readl_relaxed(cmdq->q.cons_reg));
629e86d1aa8SWill Deacon arm_smmu_cmdq_exclusive_unlock_irqrestore(cmdq, flags);
630e86d1aa8SWill Deacon llq->val = READ_ONCE(cmdq->q.llq.val);
631e86d1aa8SWill Deacon return 0;
632e86d1aa8SWill Deacon }
633e86d1aa8SWill Deacon
6348639cc83SZhen Lei queue_poll_init(smmu, &qp);
635e86d1aa8SWill Deacon do {
636e86d1aa8SWill Deacon llq->val = READ_ONCE(cmdq->q.llq.val);
637e86d1aa8SWill Deacon if (!queue_full(llq))
638e86d1aa8SWill Deacon break;
639e86d1aa8SWill Deacon
640e86d1aa8SWill Deacon ret = queue_poll(&qp);
641e86d1aa8SWill Deacon } while (!ret);
642e86d1aa8SWill Deacon
643e86d1aa8SWill Deacon return ret;
644e86d1aa8SWill Deacon }
645e86d1aa8SWill Deacon
646e86d1aa8SWill Deacon /*
647e86d1aa8SWill Deacon * Wait until the SMMU signals a CMD_SYNC completion MSI.
648e86d1aa8SWill Deacon * Must be called with the cmdq lock held in some capacity.
649e86d1aa8SWill Deacon */
__arm_smmu_cmdq_poll_until_msi(struct arm_smmu_device * smmu,struct arm_smmu_cmdq * cmdq,struct arm_smmu_ll_queue * llq)650e86d1aa8SWill Deacon static int __arm_smmu_cmdq_poll_until_msi(struct arm_smmu_device *smmu,
651e86d1aa8SWill Deacon struct arm_smmu_cmdq *cmdq,
652e86d1aa8SWill Deacon struct arm_smmu_ll_queue *llq)
653e86d1aa8SWill Deacon {
654e86d1aa8SWill Deacon int ret = 0;
655e86d1aa8SWill Deacon struct arm_smmu_queue_poll qp;
656e86d1aa8SWill Deacon u32 *cmd = (u32 *)(Q_ENT(&cmdq->q, llq->prod));
6578639cc83SZhen Lei
658e86d1aa8SWill Deacon queue_poll_init(smmu, &qp);
659e86d1aa8SWill Deacon
660e86d1aa8SWill Deacon /*
661e86d1aa8SWill Deacon * The MSI won't generate an event, since it's being written back
6628639cc83SZhen Lei * into the command queue.
663e86d1aa8SWill Deacon */
664e86d1aa8SWill Deacon qp.wfe = false;
665e86d1aa8SWill Deacon smp_cond_load_relaxed(cmd, !VAL || (ret = queue_poll(&qp)));
666e86d1aa8SWill Deacon llq->cons = ret ? llq->prod : queue_inc_prod_n(llq, 1);
667e86d1aa8SWill Deacon return ret;
668e86d1aa8SWill Deacon }
669e86d1aa8SWill Deacon
670e86d1aa8SWill Deacon /*
671e86d1aa8SWill Deacon * Wait until the SMMU cons index passes llq->prod.
672e86d1aa8SWill Deacon * Must be called with the cmdq lock held in some capacity.
673e86d1aa8SWill Deacon */
__arm_smmu_cmdq_poll_until_consumed(struct arm_smmu_device * smmu,struct arm_smmu_cmdq * cmdq,struct arm_smmu_ll_queue * llq)674e86d1aa8SWill Deacon static int __arm_smmu_cmdq_poll_until_consumed(struct arm_smmu_device *smmu,
675e86d1aa8SWill Deacon struct arm_smmu_cmdq *cmdq,
676e86d1aa8SWill Deacon struct arm_smmu_ll_queue *llq)
677e86d1aa8SWill Deacon {
678e86d1aa8SWill Deacon struct arm_smmu_queue_poll qp;
679e86d1aa8SWill Deacon u32 prod = llq->prod;
680e86d1aa8SWill Deacon int ret = 0;
681e86d1aa8SWill Deacon
682e86d1aa8SWill Deacon queue_poll_init(smmu, &qp);
683e86d1aa8SWill Deacon llq->val = READ_ONCE(cmdq->q.llq.val);
684e86d1aa8SWill Deacon do {
685e86d1aa8SWill Deacon if (queue_consumed(llq, prod))
686e86d1aa8SWill Deacon break;
687e86d1aa8SWill Deacon
688e86d1aa8SWill Deacon ret = queue_poll(&qp);
689e86d1aa8SWill Deacon
690e86d1aa8SWill Deacon /*
691e86d1aa8SWill Deacon * This needs to be a readl() so that our subsequent call
692e86d1aa8SWill Deacon * to arm_smmu_cmdq_shared_tryunlock() can fail accurately.
693e86d1aa8SWill Deacon *
694e86d1aa8SWill Deacon * Specifically, we need to ensure that we observe all
695e86d1aa8SWill Deacon * shared_lock()s by other CMD_SYNCs that share our owner,
696e86d1aa8SWill Deacon * so that a failing call to tryunlock() means that we're
697e86d1aa8SWill Deacon * the last one out and therefore we can safely advance
698e86d1aa8SWill Deacon * cmdq->q.llq.cons. Roughly speaking:
699e86d1aa8SWill Deacon *
700e86d1aa8SWill Deacon * CPU 0 CPU1 CPU2 (us)
701e86d1aa8SWill Deacon *
702e86d1aa8SWill Deacon * if (sync)
703e86d1aa8SWill Deacon * shared_lock();
704e86d1aa8SWill Deacon *
705e86d1aa8SWill Deacon * dma_wmb();
706bd07a20aSBarry Song * set_valid_map();
707e86d1aa8SWill Deacon *
708e86d1aa8SWill Deacon * if (owner) {
709e86d1aa8SWill Deacon * poll_valid_map();
710e86d1aa8SWill Deacon * <control dependency>
711e86d1aa8SWill Deacon * writel(prod_reg);
712e86d1aa8SWill Deacon *
713e86d1aa8SWill Deacon * readl(cons_reg);
714e86d1aa8SWill Deacon * tryunlock();
715e86d1aa8SWill Deacon *
716e86d1aa8SWill Deacon * Requires us to see CPU 0's shared_lock() acquisition.
717e86d1aa8SWill Deacon */
718e86d1aa8SWill Deacon llq->cons = readl(cmdq->q.cons_reg);
719e86d1aa8SWill Deacon } while (!ret);
720e86d1aa8SWill Deacon
721e86d1aa8SWill Deacon return ret;
722e86d1aa8SWill Deacon }
723e86d1aa8SWill Deacon
arm_smmu_cmdq_poll_until_sync(struct arm_smmu_device * smmu,struct arm_smmu_cmdq * cmdq,struct arm_smmu_ll_queue * llq)724e86d1aa8SWill Deacon static int arm_smmu_cmdq_poll_until_sync(struct arm_smmu_device *smmu,
725e86d1aa8SWill Deacon struct arm_smmu_cmdq *cmdq,
726e86d1aa8SWill Deacon struct arm_smmu_ll_queue *llq)
727e86d1aa8SWill Deacon {
728e86d1aa8SWill Deacon if (smmu->options & ARM_SMMU_OPT_MSIPOLL &&
729e86d1aa8SWill Deacon !arm_smmu_cmdq_needs_busy_polling(smmu, cmdq))
730e86d1aa8SWill Deacon return __arm_smmu_cmdq_poll_until_msi(smmu, cmdq, llq);
731e86d1aa8SWill Deacon
732e86d1aa8SWill Deacon return __arm_smmu_cmdq_poll_until_consumed(smmu, cmdq, llq);
733e86d1aa8SWill Deacon }
734e86d1aa8SWill Deacon
arm_smmu_cmdq_write_entries(struct arm_smmu_cmdq * cmdq,u64 * cmds,u32 prod,int n)735e86d1aa8SWill Deacon static void arm_smmu_cmdq_write_entries(struct arm_smmu_cmdq *cmdq, u64 *cmds,
736e86d1aa8SWill Deacon u32 prod, int n)
737e86d1aa8SWill Deacon {
738e86d1aa8SWill Deacon int i;
739e86d1aa8SWill Deacon struct arm_smmu_ll_queue llq = {
740e86d1aa8SWill Deacon .max_n_shift = cmdq->q.llq.max_n_shift,
741e86d1aa8SWill Deacon .prod = prod,
742e86d1aa8SWill Deacon };
743e86d1aa8SWill Deacon
744e86d1aa8SWill Deacon for (i = 0; i < n; ++i) {
745e86d1aa8SWill Deacon u64 *cmd = &cmds[i * CMDQ_ENT_DWORDS];
746e86d1aa8SWill Deacon
747e86d1aa8SWill Deacon prod = queue_inc_prod_n(&llq, i);
748e86d1aa8SWill Deacon queue_write(Q_ENT(&cmdq->q, prod), cmd, CMDQ_ENT_DWORDS);
749e86d1aa8SWill Deacon }
750e86d1aa8SWill Deacon }
751e86d1aa8SWill Deacon
7528639cc83SZhen Lei /*
7535c08c5acSJohn Garry * This is the actual insertion function, and provides the following
754e86d1aa8SWill Deacon * ordering guarantees to callers:
755e86d1aa8SWill Deacon *
7565c08c5acSJohn Garry * - There is a dma_wmb() before publishing any commands to the queue.
7575c08c5acSJohn Garry * This can be relied upon to order prior writes to data structures
758e86d1aa8SWill Deacon * in memory (such as a CD or an STE) before the command.
759e86d1aa8SWill Deacon *
760e86d1aa8SWill Deacon * - On completion of a CMD_SYNC, there is a control dependency.
761e86d1aa8SWill Deacon * This can be relied upon to order subsequent writes to memory (e.g.
762e86d1aa8SWill Deacon * freeing an IOVA) after completion of the CMD_SYNC.
763e86d1aa8SWill Deacon *
764e86d1aa8SWill Deacon * - Command insertion is totally ordered, so if two CPUs each race to
765e86d1aa8SWill Deacon * insert their own list of commands then all of the commands from one
766e86d1aa8SWill Deacon * CPU will appear before any of the commands from the other CPU.
767e86d1aa8SWill Deacon */
arm_smmu_cmdq_issue_cmdlist(struct arm_smmu_device * smmu,struct arm_smmu_cmdq * cmdq,u64 * cmds,int n,bool sync)768e86d1aa8SWill Deacon static int arm_smmu_cmdq_issue_cmdlist(struct arm_smmu_device *smmu,
769e86d1aa8SWill Deacon struct arm_smmu_cmdq *cmdq,
770e86d1aa8SWill Deacon u64 *cmds, int n, bool sync)
771e86d1aa8SWill Deacon {
772e86d1aa8SWill Deacon u64 cmd_sync[CMDQ_ENT_DWORDS];
773e86d1aa8SWill Deacon u32 prod;
774e86d1aa8SWill Deacon unsigned long flags;
775e86d1aa8SWill Deacon bool owner;
776e86d1aa8SWill Deacon struct arm_smmu_ll_queue llq, head;
777e86d1aa8SWill Deacon int ret = 0;
778e86d1aa8SWill Deacon
779e86d1aa8SWill Deacon llq.max_n_shift = cmdq->q.llq.max_n_shift;
780e86d1aa8SWill Deacon
781e86d1aa8SWill Deacon /* 1. Allocate some space in the queue */
782e86d1aa8SWill Deacon local_irq_save(flags);
783e86d1aa8SWill Deacon llq.val = READ_ONCE(cmdq->q.llq.val);
784e86d1aa8SWill Deacon do {
785e86d1aa8SWill Deacon u64 old;
786e86d1aa8SWill Deacon
787e86d1aa8SWill Deacon while (!queue_has_space(&llq, n + sync)) {
788e86d1aa8SWill Deacon local_irq_restore(flags);
789e86d1aa8SWill Deacon if (arm_smmu_cmdq_poll_until_not_full(smmu, cmdq, &llq))
790e86d1aa8SWill Deacon dev_err_ratelimited(smmu->dev, "CMDQ timeout\n");
791e86d1aa8SWill Deacon local_irq_save(flags);
7928639cc83SZhen Lei }
793e86d1aa8SWill Deacon
794e86d1aa8SWill Deacon head.cons = llq.cons;
795e86d1aa8SWill Deacon head.prod = queue_inc_prod_n(&llq, n + sync) |
796e86d1aa8SWill Deacon CMDQ_PROD_OWNED_FLAG;
797e86d1aa8SWill Deacon
798e86d1aa8SWill Deacon old = cmpxchg_relaxed(&cmdq->q.llq.val, llq.val, head.val);
799e86d1aa8SWill Deacon if (old == llq.val)
800e86d1aa8SWill Deacon break;
801e86d1aa8SWill Deacon
802e86d1aa8SWill Deacon llq.val = old;
803e86d1aa8SWill Deacon } while (1);
804e86d1aa8SWill Deacon owner = !(llq.prod & CMDQ_PROD_OWNED_FLAG);
805e86d1aa8SWill Deacon head.prod &= ~CMDQ_PROD_OWNED_FLAG;
806e86d1aa8SWill Deacon llq.prod &= ~CMDQ_PROD_OWNED_FLAG;
807e86d1aa8SWill Deacon
808e86d1aa8SWill Deacon /*
809e86d1aa8SWill Deacon * 2. Write our commands into the queue
810e86d1aa8SWill Deacon * Dependency ordering from the cmpxchg() loop above.
811e86d1aa8SWill Deacon */
812e86d1aa8SWill Deacon arm_smmu_cmdq_write_entries(cmdq, cmds, llq.prod, n);
813e86d1aa8SWill Deacon if (sync) {
814e86d1aa8SWill Deacon prod = queue_inc_prod_n(&llq, n);
815e86d1aa8SWill Deacon arm_smmu_cmdq_build_sync_cmd(cmd_sync, smmu, cmdq, prod);
816e86d1aa8SWill Deacon queue_write(Q_ENT(&cmdq->q, prod), cmd_sync, CMDQ_ENT_DWORDS);
817e86d1aa8SWill Deacon
818e86d1aa8SWill Deacon /*
819e86d1aa8SWill Deacon * In order to determine completion of our CMD_SYNC, we must
820e86d1aa8SWill Deacon * ensure that the queue can't wrap twice without us noticing.
821e86d1aa8SWill Deacon * We achieve that by taking the cmdq lock as shared before
822e86d1aa8SWill Deacon * marking our slot as valid.
823e86d1aa8SWill Deacon */
824e86d1aa8SWill Deacon arm_smmu_cmdq_shared_lock(cmdq);
825e86d1aa8SWill Deacon }
826e86d1aa8SWill Deacon
827e86d1aa8SWill Deacon /* 3. Mark our slots as valid, ensuring commands are visible first */
828e86d1aa8SWill Deacon dma_wmb();
829e86d1aa8SWill Deacon arm_smmu_cmdq_set_valid_map(cmdq, llq.prod, head.prod);
830e86d1aa8SWill Deacon
831e86d1aa8SWill Deacon /* 4. If we are the owner, take control of the SMMU hardware */
832e86d1aa8SWill Deacon if (owner) {
833e86d1aa8SWill Deacon /* a. Wait for previous owner to finish */
834e86d1aa8SWill Deacon atomic_cond_read_relaxed(&cmdq->owner_prod, VAL == llq.prod);
835e86d1aa8SWill Deacon
836e86d1aa8SWill Deacon /* b. Stop gathering work by clearing the owned flag */
837e86d1aa8SWill Deacon prod = atomic_fetch_andnot_relaxed(CMDQ_PROD_OWNED_FLAG,
838e86d1aa8SWill Deacon &cmdq->q.llq.atomic.prod);
839e86d1aa8SWill Deacon prod &= ~CMDQ_PROD_OWNED_FLAG;
840e86d1aa8SWill Deacon
841e86d1aa8SWill Deacon /*
842e86d1aa8SWill Deacon * c. Wait for any gathered work to be written to the queue.
843e86d1aa8SWill Deacon * Note that we read our own entries so that we have the control
844e86d1aa8SWill Deacon * dependency required by (d).
845e86d1aa8SWill Deacon */
846e86d1aa8SWill Deacon arm_smmu_cmdq_poll_valid_map(cmdq, llq.prod, prod);
847e86d1aa8SWill Deacon
848e86d1aa8SWill Deacon /*
849e86d1aa8SWill Deacon * d. Advance the hardware prod pointer
850e86d1aa8SWill Deacon * Control dependency ordering from the entries becoming valid.
851e86d1aa8SWill Deacon */
852e86d1aa8SWill Deacon writel_relaxed(prod, cmdq->q.prod_reg);
853e86d1aa8SWill Deacon
854e86d1aa8SWill Deacon /*
855e86d1aa8SWill Deacon * e. Tell the next owner we're done
856e86d1aa8SWill Deacon * Make sure we've updated the hardware first, so that we don't
857e86d1aa8SWill Deacon * race to update prod and potentially move it backwards.
858e86d1aa8SWill Deacon */
859e86d1aa8SWill Deacon atomic_set_release(&cmdq->owner_prod, prod);
860e86d1aa8SWill Deacon }
861e86d1aa8SWill Deacon
862e86d1aa8SWill Deacon /* 5. If we are inserting a CMD_SYNC, we must wait for it to complete */
863e86d1aa8SWill Deacon if (sync) {
864e86d1aa8SWill Deacon llq.prod = queue_inc_prod_n(&llq, n);
8654537f6f1SZhen Lei ret = arm_smmu_cmdq_poll_until_sync(smmu, cmdq, &llq);
8664537f6f1SZhen Lei if (ret) {
8674537f6f1SZhen Lei dev_err_ratelimited(smmu->dev,
868e86d1aa8SWill Deacon "CMD_SYNC timeout at 0x%08x [hwprod 0x%08x, hwcons 0x%08x]\n",
869e86d1aa8SWill Deacon llq.prod,
870e86d1aa8SWill Deacon readl_relaxed(cmdq->q.prod_reg),
87159d9bd72SZhen Lei readl_relaxed(cmdq->q.cons_reg));
872e86d1aa8SWill Deacon }
873e86d1aa8SWill Deacon
874e86d1aa8SWill Deacon /*
875e86d1aa8SWill Deacon * Try to unlock the cmdq lock. This will fail if we're the last
876e86d1aa8SWill Deacon * reader, in which case we can safely update cmdq->q.llq.cons
8774537f6f1SZhen Lei */
878e86d1aa8SWill Deacon if (!arm_smmu_cmdq_shared_tryunlock(cmdq)) {
879e86d1aa8SWill Deacon WRITE_ONCE(cmdq->q.llq.cons, llq.cons);
8804537f6f1SZhen Lei arm_smmu_cmdq_shared_unlock(cmdq);
8814537f6f1SZhen Lei }
882e86d1aa8SWill Deacon }
8834537f6f1SZhen Lei
8844537f6f1SZhen Lei local_irq_restore(flags);
8854537f6f1SZhen Lei return ret;
8864537f6f1SZhen Lei }
8874537f6f1SZhen Lei
__arm_smmu_cmdq_issue_cmd(struct arm_smmu_device * smmu,struct arm_smmu_cmdq_ent * ent,bool sync)8884537f6f1SZhen Lei static int __arm_smmu_cmdq_issue_cmd(struct arm_smmu_device *smmu,
8894537f6f1SZhen Lei struct arm_smmu_cmdq_ent *ent,
890e86d1aa8SWill Deacon bool sync)
891e86d1aa8SWill Deacon {
892e86d1aa8SWill Deacon u64 cmd[CMDQ_ENT_DWORDS];
893e86d1aa8SWill Deacon
894e86d1aa8SWill Deacon if (unlikely(arm_smmu_cmdq_build_cmd(cmd, ent))) {
895e86d1aa8SWill Deacon dev_warn(smmu->dev, "ignoring unknown CMDQ opcode 0x%x\n",
89659d9bd72SZhen Lei ent->opcode);
89759d9bd72SZhen Lei return -EINVAL;
898309a15cbSRobin Murphy }
899309a15cbSRobin Murphy
900309a15cbSRobin Murphy return arm_smmu_cmdq_issue_cmdlist(
901309a15cbSRobin Murphy smmu, arm_smmu_get_cmdq(smmu, ent), cmd, 1, sync);
902309a15cbSRobin Murphy }
903309a15cbSRobin Murphy
arm_smmu_cmdq_issue_cmd(struct arm_smmu_device * smmu,struct arm_smmu_cmdq_ent * ent)904e86d1aa8SWill Deacon static int arm_smmu_cmdq_issue_cmd(struct arm_smmu_device *smmu,
905e86d1aa8SWill Deacon struct arm_smmu_cmdq_ent *ent)
906e86d1aa8SWill Deacon {
907e86d1aa8SWill Deacon return __arm_smmu_cmdq_issue_cmd(smmu, ent, false);
90859d9bd72SZhen Lei }
90959d9bd72SZhen Lei
arm_smmu_cmdq_issue_cmd_with_sync(struct arm_smmu_device * smmu,struct arm_smmu_cmdq_ent * ent)91059d9bd72SZhen Lei static int arm_smmu_cmdq_issue_cmd_with_sync(struct arm_smmu_device *smmu,
91159d9bd72SZhen Lei struct arm_smmu_cmdq_ent *ent)
91259d9bd72SZhen Lei {
91359d9bd72SZhen Lei return __arm_smmu_cmdq_issue_cmd(smmu, ent, true);
91459d9bd72SZhen Lei }
91559d9bd72SZhen Lei
arm_smmu_cmdq_batch_init(struct arm_smmu_device * smmu,struct arm_smmu_cmdq_batch * cmds,struct arm_smmu_cmdq_ent * ent)916e86d1aa8SWill Deacon static void arm_smmu_cmdq_batch_init(struct arm_smmu_device *smmu,
917e86d1aa8SWill Deacon struct arm_smmu_cmdq_batch *cmds,
918e86d1aa8SWill Deacon struct arm_smmu_cmdq_ent *ent)
919e86d1aa8SWill Deacon {
920e86d1aa8SWill Deacon cmds->num = 0;
921e86d1aa8SWill Deacon cmds->cmdq = arm_smmu_get_cmdq(smmu, ent);
922e86d1aa8SWill Deacon }
923e86d1aa8SWill Deacon
arm_smmu_cmdq_batch_add(struct arm_smmu_device * smmu,struct arm_smmu_cmdq_batch * cmds,struct arm_smmu_cmdq_ent * cmd)924e86d1aa8SWill Deacon static void arm_smmu_cmdq_batch_add(struct arm_smmu_device *smmu,
925b554e396SLu Baolu struct arm_smmu_cmdq_batch *cmds,
926395ad89dSJean-Philippe Brucker struct arm_smmu_cmdq_ent *cmd)
927395ad89dSJean-Philippe Brucker {
928395ad89dSJean-Philippe Brucker bool unsupported_cmd = !arm_smmu_cmdq_supports_cmd(cmds->cmdq, cmd);
929395ad89dSJean-Philippe Brucker bool force_sync = (cmds->num == CMDQ_BATCH_ENTRIES - 1) &&
930395ad89dSJean-Philippe Brucker (smmu->options & ARM_SMMU_OPT_CMDQ_FORCE_SYNC);
931395ad89dSJean-Philippe Brucker int index;
932b554e396SLu Baolu
933b554e396SLu Baolu if (force_sync || unsupported_cmd) {
934b554e396SLu Baolu arm_smmu_cmdq_issue_cmdlist(smmu, cmds->cmdq, cmds->cmds,
935395ad89dSJean-Philippe Brucker cmds->num, true);
936395ad89dSJean-Philippe Brucker arm_smmu_cmdq_batch_init(smmu, cmds, cmd);
937395ad89dSJean-Philippe Brucker }
938395ad89dSJean-Philippe Brucker
939395ad89dSJean-Philippe Brucker if (cmds->num == CMDQ_BATCH_ENTRIES) {
940395ad89dSJean-Philippe Brucker arm_smmu_cmdq_issue_cmdlist(smmu, cmds->cmdq, cmds->cmds,
941395ad89dSJean-Philippe Brucker cmds->num, false);
942395ad89dSJean-Philippe Brucker arm_smmu_cmdq_batch_init(smmu, cmds, cmd);
943395ad89dSJean-Philippe Brucker }
944395ad89dSJean-Philippe Brucker
945395ad89dSJean-Philippe Brucker index = cmds->num * CMDQ_ENT_DWORDS;
946395ad89dSJean-Philippe Brucker if (unlikely(arm_smmu_cmdq_build_cmd(&cmds->cmds[index], cmd))) {
947b554e396SLu Baolu dev_warn(smmu->dev, "ignoring unknown CMDQ opcode 0x%x\n",
948395ad89dSJean-Philippe Brucker cmd->opcode);
949395ad89dSJean-Philippe Brucker return;
950395ad89dSJean-Philippe Brucker }
951395ad89dSJean-Philippe Brucker
952395ad89dSJean-Philippe Brucker cmds->num++;
953395ad89dSJean-Philippe Brucker }
954395ad89dSJean-Philippe Brucker
arm_smmu_cmdq_batch_submit(struct arm_smmu_device * smmu,struct arm_smmu_cmdq_batch * cmds)955395ad89dSJean-Philippe Brucker static int arm_smmu_cmdq_batch_submit(struct arm_smmu_device *smmu,
956395ad89dSJean-Philippe Brucker struct arm_smmu_cmdq_batch *cmds)
957395ad89dSJean-Philippe Brucker {
958395ad89dSJean-Philippe Brucker return arm_smmu_cmdq_issue_cmdlist(smmu, cmds->cmdq, cmds->cmds,
959e86d1aa8SWill Deacon cmds->num, true);
9603e630336SJean-Philippe Brucker }
9613e630336SJean-Philippe Brucker
arm_smmu_page_response(struct device * dev,struct iopf_fault * unused,struct iommu_page_response * resp)9623e630336SJean-Philippe Brucker static void arm_smmu_page_response(struct device *dev, struct iopf_fault *unused,
9639111aebfSJean-Philippe Brucker struct iommu_page_response *resp)
9649111aebfSJean-Philippe Brucker {
9653e630336SJean-Philippe Brucker struct arm_smmu_cmdq_ent cmd = {0};
9663e630336SJean-Philippe Brucker struct arm_smmu_master *master = dev_iommu_priv_get(dev);
9673e630336SJean-Philippe Brucker int sid = master->streams[0].id;
9684537f6f1SZhen Lei
9693e630336SJean-Philippe Brucker if (WARN_ON(!master->stall_enabled))
9703e630336SJean-Philippe Brucker return;
9717da51af9SJason Gunthorpe
9727da51af9SJason Gunthorpe cmd.opcode = CMDQ_OP_RESUME;
9737da51af9SJason Gunthorpe cmd.resume.sid = sid;
9747da51af9SJason Gunthorpe cmd.resume.stag = resp->grpid;
9757da51af9SJason Gunthorpe switch (resp->code) {
97656e1a4ccSJason Gunthorpe case IOMMU_PAGE_RESP_INVALID:
97756e1a4ccSJason Gunthorpe case IOMMU_PAGE_RESP_FAILURE:
9787da51af9SJason Gunthorpe cmd.resume.resp = CMDQ_RESUME_0_RESP_ABORT;
979de31c355SJason Gunthorpe break;
9807da51af9SJason Gunthorpe case IOMMU_PAGE_RESP_SUCCESS:
981de31c355SJason Gunthorpe cmd.resume.resp = CMDQ_RESUME_0_RESP_RETRY;
982de31c355SJason Gunthorpe break;
9837da51af9SJason Gunthorpe default:
9847da51af9SJason Gunthorpe break;
985de31c355SJason Gunthorpe }
9867da51af9SJason Gunthorpe
9877da51af9SJason Gunthorpe arm_smmu_cmdq_issue_cmd(master->smmu, &cmd);
9887da51af9SJason Gunthorpe /*
989de31c355SJason Gunthorpe * Don't send a SYNC, it doesn't do anything for RESUME or PRI_RESP.
9907da51af9SJason Gunthorpe * RESUME consumption guarantees that the stalled transaction will be
9917da51af9SJason Gunthorpe * terminated... at some point in the future. PRI_RESP is fire and
992de31c355SJason Gunthorpe * forget.
9937da51af9SJason Gunthorpe */
9947da51af9SJason Gunthorpe }
9957da51af9SJason Gunthorpe
9967da51af9SJason Gunthorpe /* Context descriptor manipulation functions */
arm_smmu_tlb_inv_asid(struct arm_smmu_device * smmu,u16 asid)997de31c355SJason Gunthorpe void arm_smmu_tlb_inv_asid(struct arm_smmu_device *smmu, u16 asid)
998ce26ea9eSJason Gunthorpe {
999ce26ea9eSJason Gunthorpe struct arm_smmu_cmdq_ent cmd = {
1000ce26ea9eSJason Gunthorpe .opcode = smmu->features & ARM_SMMU_FEAT_E2H ?
1001ce26ea9eSJason Gunthorpe CMDQ_OP_TLBI_EL2_ASID : CMDQ_OP_TLBI_NH_ASID,
1002ce26ea9eSJason Gunthorpe .tlbi.asid = asid,
1003ce26ea9eSJason Gunthorpe };
1004ce26ea9eSJason Gunthorpe
1005ce26ea9eSJason Gunthorpe arm_smmu_cmdq_issue_cmd_with_sync(smmu, &cmd);
10067da51af9SJason Gunthorpe }
10077da51af9SJason Gunthorpe
10087da51af9SJason Gunthorpe /*
10097da51af9SJason Gunthorpe * Based on the value of ent report which bits of the STE the HW will access. It
1010de31c355SJason Gunthorpe * would be nice if this was complete according to the spec, but minimally it
10117da51af9SJason Gunthorpe * has to capture the bits this driver uses.
1012de31c355SJason Gunthorpe */
10137da51af9SJason Gunthorpe VISIBLE_IF_KUNIT
arm_smmu_get_ste_used(const __le64 * ent,__le64 * used_bits)10147da51af9SJason Gunthorpe void arm_smmu_get_ste_used(const __le64 *ent, __le64 *used_bits)
10157da51af9SJason Gunthorpe {
1016de31c355SJason Gunthorpe unsigned int cfg = FIELD_GET(STRTAB_STE_0_CFG, le64_to_cpu(ent[0]));
10177da51af9SJason Gunthorpe
10187da51af9SJason Gunthorpe used_bits[0] = cpu_to_le64(STRTAB_STE_0_V);
10197da51af9SJason Gunthorpe if (!(ent[0] & cpu_to_le64(STRTAB_STE_0_V)))
1020de31c355SJason Gunthorpe return;
10217da51af9SJason Gunthorpe
1022da55da5aSJason Gunthorpe used_bits[0] |= cpu_to_le64(STRTAB_STE_0_CFG);
10237da51af9SJason Gunthorpe
10247da51af9SJason Gunthorpe /* S1 translates */
10257da51af9SJason Gunthorpe if (cfg & BIT(0)) {
10267da51af9SJason Gunthorpe used_bits[0] |= cpu_to_le64(STRTAB_STE_0_S1FMT |
10277da51af9SJason Gunthorpe STRTAB_STE_0_S1CTXPTR_MASK |
10287da51af9SJason Gunthorpe STRTAB_STE_0_S1CDMAX);
10297da51af9SJason Gunthorpe used_bits[1] |=
1030de31c355SJason Gunthorpe cpu_to_le64(STRTAB_STE_1_S1DSS | STRTAB_STE_1_S1CIR |
1031de31c355SJason Gunthorpe STRTAB_STE_1_S1COR | STRTAB_STE_1_S1CSH |
1032de31c355SJason Gunthorpe STRTAB_STE_1_S1STALLD | STRTAB_STE_1_STRW |
10337da51af9SJason Gunthorpe STRTAB_STE_1_EATS);
1034de31c355SJason Gunthorpe used_bits[2] |= cpu_to_le64(STRTAB_STE_2_S2VMID);
1035de31c355SJason Gunthorpe
10367da51af9SJason Gunthorpe /*
10377da51af9SJason Gunthorpe * See 13.5 Summary of attribute/permission configuration fields
10387da51af9SJason Gunthorpe * for the SHCFG behavior.
1039de31c355SJason Gunthorpe */
1040de31c355SJason Gunthorpe if (FIELD_GET(STRTAB_STE_1_S1DSS, le64_to_cpu(ent[1])) ==
10417da51af9SJason Gunthorpe STRTAB_STE_1_S1DSS_BYPASS)
1042de31c355SJason Gunthorpe used_bits[1] |= cpu_to_le64(STRTAB_STE_1_SHCFG);
10437da51af9SJason Gunthorpe }
10447da51af9SJason Gunthorpe
10457da51af9SJason Gunthorpe /* S2 translates */
10467da51af9SJason Gunthorpe if (cfg & BIT(1)) {
10477da51af9SJason Gunthorpe used_bits[1] |=
1048de31c355SJason Gunthorpe cpu_to_le64(STRTAB_STE_1_EATS | STRTAB_STE_1_SHCFG);
10497da51af9SJason Gunthorpe used_bits[2] |=
10507da51af9SJason Gunthorpe cpu_to_le64(STRTAB_STE_2_S2VMID | STRTAB_STE_2_VTCR |
1051de31c355SJason Gunthorpe STRTAB_STE_2_S2AA64 | STRTAB_STE_2_S2ENDI |
1052de31c355SJason Gunthorpe STRTAB_STE_2_S2PTW | STRTAB_STE_2_S2S |
10537da51af9SJason Gunthorpe STRTAB_STE_2_S2R);
10547da51af9SJason Gunthorpe used_bits[3] |= cpu_to_le64(STRTAB_STE_3_S2TTB_MASK);
10557da51af9SJason Gunthorpe }
10567da51af9SJason Gunthorpe
1057de31c355SJason Gunthorpe if (cfg == STRTAB_STE_0_CFG_BYPASS)
10587da51af9SJason Gunthorpe used_bits[1] |= cpu_to_le64(STRTAB_STE_1_SHCFG);
10597da51af9SJason Gunthorpe }
10607da51af9SJason Gunthorpe EXPORT_SYMBOL_IF_KUNIT(arm_smmu_get_ste_used);
10617da51af9SJason Gunthorpe
10627da51af9SJason Gunthorpe /*
1063de31c355SJason Gunthorpe * Figure out if we can do a hitless update of entry to become target. Returns a
1064de31c355SJason Gunthorpe * bit mask where 1 indicates that qword needs to be set disruptively.
10657da51af9SJason Gunthorpe * unused_update is an intermediate value of entry that has unused bits set to
10667da51af9SJason Gunthorpe * their new values.
10677da51af9SJason Gunthorpe */
arm_smmu_entry_qword_diff(struct arm_smmu_entry_writer * writer,const __le64 * entry,const __le64 * target,__le64 * unused_update)10687da51af9SJason Gunthorpe static u8 arm_smmu_entry_qword_diff(struct arm_smmu_entry_writer *writer,
10697da51af9SJason Gunthorpe const __le64 *entry, const __le64 *target,
10707da51af9SJason Gunthorpe __le64 *unused_update)
1071de31c355SJason Gunthorpe {
1072de31c355SJason Gunthorpe __le64 target_used[NUM_ENTRY_QWORDS] = {};
10737da51af9SJason Gunthorpe __le64 cur_used[NUM_ENTRY_QWORDS] = {};
10747da51af9SJason Gunthorpe u8 used_qword_diff = 0;
10757da51af9SJason Gunthorpe unsigned int i;
10767da51af9SJason Gunthorpe
10777da51af9SJason Gunthorpe writer->ops->get_used(entry, cur_used);
1078de31c355SJason Gunthorpe writer->ops->get_used(target, target_used);
10797da51af9SJason Gunthorpe
10807da51af9SJason Gunthorpe for (i = 0; i != NUM_ENTRY_QWORDS; i++) {
10817da51af9SJason Gunthorpe /*
10827da51af9SJason Gunthorpe * Check that masks are up to date, the make functions are not
10837da51af9SJason Gunthorpe * allowed to set a bit to 1 if the used function doesn't say it
10847da51af9SJason Gunthorpe * is used.
10857da51af9SJason Gunthorpe */
10867da51af9SJason Gunthorpe WARN_ON_ONCE(target[i] & ~target_used[i]);
10877da51af9SJason Gunthorpe
10887da51af9SJason Gunthorpe /* Bits can change because they are not currently being used */
10897da51af9SJason Gunthorpe unused_update[i] = (entry[i] & cur_used[i]) |
10907da51af9SJason Gunthorpe (target[i] & ~cur_used[i]);
10917da51af9SJason Gunthorpe /*
10927da51af9SJason Gunthorpe * Each bit indicates that a used bit in a qword needs to be
10937da51af9SJason Gunthorpe * changed after unused_update is applied.
10947da51af9SJason Gunthorpe */
10957da51af9SJason Gunthorpe if ((unused_update[i] & target_used[i]) != target[i])
10967da51af9SJason Gunthorpe used_qword_diff |= 1 << i;
10977da51af9SJason Gunthorpe }
10987da51af9SJason Gunthorpe return used_qword_diff;
10997da51af9SJason Gunthorpe }
11007da51af9SJason Gunthorpe
entry_set(struct arm_smmu_entry_writer * writer,__le64 * entry,const __le64 * target,unsigned int start,unsigned int len)11017da51af9SJason Gunthorpe static bool entry_set(struct arm_smmu_entry_writer *writer, __le64 *entry,
11027da51af9SJason Gunthorpe const __le64 *target, unsigned int start,
11037da51af9SJason Gunthorpe unsigned int len)
11047da51af9SJason Gunthorpe {
11057da51af9SJason Gunthorpe bool changed = false;
11067da51af9SJason Gunthorpe unsigned int i;
11077da51af9SJason Gunthorpe
110856e1a4ccSJason Gunthorpe for (i = start; len != 0; len--, i++) {
110956e1a4ccSJason Gunthorpe if (entry[i] != target[i]) {
111056e1a4ccSJason Gunthorpe WRITE_ONCE(entry[i], target[i]);
11117da51af9SJason Gunthorpe changed = true;
1112de31c355SJason Gunthorpe }
11137da51af9SJason Gunthorpe }
11147da51af9SJason Gunthorpe
11157da51af9SJason Gunthorpe if (changed)
1116de31c355SJason Gunthorpe writer->ops->sync(writer);
11177da51af9SJason Gunthorpe return changed;
11187da51af9SJason Gunthorpe }
11197da51af9SJason Gunthorpe
1120de31c355SJason Gunthorpe /*
1121de31c355SJason Gunthorpe * Update the STE/CD to the target configuration. The transition from the
1122de31c355SJason Gunthorpe * current entry to the target entry takes place over multiple steps that
1123de31c355SJason Gunthorpe * attempts to make the transition hitless if possible. This function takes care
11247da51af9SJason Gunthorpe * not to create a situation where the HW can perceive a corrupted entry. HW is
11257da51af9SJason Gunthorpe * only required to have a 64 bit atomicity with stores from the CPU, while
11267da51af9SJason Gunthorpe * entries are many 64 bit values big.
11277da51af9SJason Gunthorpe *
11287da51af9SJason Gunthorpe * The difference between the current value and the target value is analyzed to
11297da51af9SJason Gunthorpe * determine which of three updates are required - disruptive, hitless or no
11307da51af9SJason Gunthorpe * change.
11317da51af9SJason Gunthorpe *
1132de31c355SJason Gunthorpe * In the most general disruptive case we can make any update in three steps:
1133de31c355SJason Gunthorpe * - Disrupting the entry (V=0)
1134de31c355SJason Gunthorpe * - Fill now unused qwords, execpt qword 0 which contains V
1135de31c355SJason Gunthorpe * - Make qword 0 have the final value and valid (V=1) with a single 64
1136de31c355SJason Gunthorpe * bit store
11377da51af9SJason Gunthorpe *
11387da51af9SJason Gunthorpe * However this disrupts the HW while it is happening. There are several
11397da51af9SJason Gunthorpe * interesting cases where a STE/CD can be updated without disturbing the HW
11407da51af9SJason Gunthorpe * because only a small number of bits are changing (S1DSS, CONFIG, etc) or
11417da51af9SJason Gunthorpe * because the used bits don't intersect. We can detect this by calculating how
11427da51af9SJason Gunthorpe * many 64 bit values need update after adjusting the unused bits and skip the
1143de31c355SJason Gunthorpe * V=0 process. This relies on the IGNORED behavior described in the
1144de31c355SJason Gunthorpe * specification.
1145de31c355SJason Gunthorpe */
1146de31c355SJason Gunthorpe VISIBLE_IF_KUNIT
arm_smmu_write_entry(struct arm_smmu_entry_writer * writer,__le64 * entry,const __le64 * target)11477da51af9SJason Gunthorpe void arm_smmu_write_entry(struct arm_smmu_entry_writer *writer, __le64 *entry,
11487da51af9SJason Gunthorpe const __le64 *target)
11497da51af9SJason Gunthorpe {
11507da51af9SJason Gunthorpe __le64 unused_update[NUM_ENTRY_QWORDS];
11517da51af9SJason Gunthorpe u8 used_qword_diff;
11527da51af9SJason Gunthorpe
11537da51af9SJason Gunthorpe used_qword_diff =
1154de31c355SJason Gunthorpe arm_smmu_entry_qword_diff(writer, entry, target, unused_update);
11557da51af9SJason Gunthorpe if (hweight8(used_qword_diff) == 1) {
11567da51af9SJason Gunthorpe /*
1157da55da5aSJason Gunthorpe * Only one qword needs its used bits to be changed. This is a
11587da51af9SJason Gunthorpe * hitless update, update all bits the current STE/CD is
115924503148SMichael Shavit * ignoring to their new values, then update a single "critical
1160e86d1aa8SWill Deacon * qword" to change the STE/CD and finally 0 out any bits that
1161e86d1aa8SWill Deacon * are now unused in the target configuration.
1162e86d1aa8SWill Deacon */
1163fac95671SJohn Garry unsigned int critical_qword_index = ffs(used_qword_diff) - 1;
116424503148SMichael Shavit
1165e86d1aa8SWill Deacon /*
1166e86d1aa8SWill Deacon * Skip writing unused bits in the critical qword since we'll be
1167e86d1aa8SWill Deacon * writing it in the next step anyways. This can save a sync
1168e86d1aa8SWill Deacon * when the only change is in that qword.
1169e86d1aa8SWill Deacon */
1170e86d1aa8SWill Deacon unused_update[critical_qword_index] =
1171e86d1aa8SWill Deacon entry[critical_qword_index];
1172e86d1aa8SWill Deacon entry_set(writer, entry, unused_update, 0, NUM_ENTRY_QWORDS);
1173fac95671SJohn Garry entry_set(writer, entry, target, critical_qword_index, 1);
1174cdf315f9SJean-Philippe Brucker entry_set(writer, entry, target, 0, NUM_ENTRY_QWORDS);
1175cdf315f9SJean-Philippe Brucker } else if (used_qword_diff) {
1176e86d1aa8SWill Deacon /*
1177e86d1aa8SWill Deacon * At least two qwords need their inuse bits to be changed. This
1178e86d1aa8SWill Deacon * requires a breaking update, zero the V bit, write all qwords
1179e86d1aa8SWill Deacon * but 0, then set qword 0
1180e86d1aa8SWill Deacon */
1181e86d1aa8SWill Deacon unused_update[0] = 0;
1182e86d1aa8SWill Deacon entry_set(writer, entry, unused_update, 0, 1);
1183e86d1aa8SWill Deacon entry_set(writer, entry, target, 1, NUM_ENTRY_QWORDS - 1);
1184e86d1aa8SWill Deacon entry_set(writer, entry, target, 0, 1);
1185e86d1aa8SWill Deacon } else {
1186e86d1aa8SWill Deacon /*
1187e86d1aa8SWill Deacon * No inuse bit changed. Sanity check that all unused bits are 0
1188e86d1aa8SWill Deacon * in the entry. The target was already sanity checked by
1189e86d1aa8SWill Deacon * compute_qword_diff().
1190e86d1aa8SWill Deacon */
1191e86d1aa8SWill Deacon WARN_ON_ONCE(
1192e86d1aa8SWill Deacon entry_set(writer, entry, target, 0, NUM_ENTRY_QWORDS));
1193e86d1aa8SWill Deacon }
1194e86d1aa8SWill Deacon }
1195e86d1aa8SWill Deacon EXPORT_SYMBOL_IF_KUNIT(arm_smmu_write_entry);
1196e86d1aa8SWill Deacon
arm_smmu_sync_cd(struct arm_smmu_master * master,int ssid,bool leaf)1197e86d1aa8SWill Deacon static void arm_smmu_sync_cd(struct arm_smmu_master *master,
1198e86d1aa8SWill Deacon int ssid, bool leaf)
1199e86d1aa8SWill Deacon {
1200e86d1aa8SWill Deacon size_t i;
1201e86d1aa8SWill Deacon struct arm_smmu_cmdq_batch cmds;
1202e86d1aa8SWill Deacon struct arm_smmu_device *smmu = master->smmu;
12037b87c93cSJason Gunthorpe struct arm_smmu_cmdq_ent cmd = {
1204e86d1aa8SWill Deacon .opcode = CMDQ_OP_CFGI_CD,
1205e86d1aa8SWill Deacon .cfgi = {
1206e86d1aa8SWill Deacon .ssid = ssid,
1207e9d1e4ffSJason Gunthorpe .leaf = leaf,
1208e8e4398dSJason Gunthorpe },
1209e86d1aa8SWill Deacon };
1210e86d1aa8SWill Deacon
1211475918e9SMichael Shavit arm_smmu_cmdq_batch_init(smmu, &cmds, &cmd);
1212e86d1aa8SWill Deacon for (i = 0; i < master->num_streams; i++) {
1213b2f4c0fcSJason Gunthorpe cmd.cfgi.sid = master->streams[i].id;
1214b2f4c0fcSJason Gunthorpe arm_smmu_cmdq_batch_add(smmu, &cmds, &cmd);
1215e86d1aa8SWill Deacon }
1216e86d1aa8SWill Deacon
1217e8e4398dSJason Gunthorpe arm_smmu_cmdq_batch_submit(smmu, &cmds);
1218e8e4398dSJason Gunthorpe }
1219e86d1aa8SWill Deacon
arm_smmu_write_cd_l1_desc(struct arm_smmu_cdtab_l1 * dst,dma_addr_t l2ptr_dma)1220b2f4c0fcSJason Gunthorpe static void arm_smmu_write_cd_l1_desc(struct arm_smmu_cdtab_l1 *dst,
1221b2f4c0fcSJason Gunthorpe dma_addr_t l2ptr_dma)
1222b2f4c0fcSJason Gunthorpe {
1223b2f4c0fcSJason Gunthorpe u64 val = (l2ptr_dma & CTXDESC_L1_DESC_L2PTR_MASK) | CTXDESC_L1_DESC_V;
1224b2f4c0fcSJason Gunthorpe
1225b2f4c0fcSJason Gunthorpe /* The HW has 64 bit atomicity with stores to the L2 CD table */
122685f2fb6eSJason Gunthorpe WRITE_ONCE(dst->l2ptr, cpu_to_le64(val));
1227b2f4c0fcSJason Gunthorpe }
1228b2f4c0fcSJason Gunthorpe
arm_smmu_cd_l1_get_desc(const struct arm_smmu_cdtab_l1 * src)1229b2f4c0fcSJason Gunthorpe static dma_addr_t arm_smmu_cd_l1_get_desc(const struct arm_smmu_cdtab_l1 *src)
1230b2f4c0fcSJason Gunthorpe {
1231b2f4c0fcSJason Gunthorpe return le64_to_cpu(src->l2ptr) & CTXDESC_L1_DESC_L2PTR_MASK;
12327b87c93cSJason Gunthorpe }
12337b87c93cSJason Gunthorpe
arm_smmu_get_cd_ptr(struct arm_smmu_master * master,u32 ssid)12347b87c93cSJason Gunthorpe struct arm_smmu_cd *arm_smmu_get_cd_ptr(struct arm_smmu_master *master,
1235b2f4c0fcSJason Gunthorpe u32 ssid)
1236b2f4c0fcSJason Gunthorpe {
1237b2f4c0fcSJason Gunthorpe struct arm_smmu_cdtab_l2 *l2;
1238b2f4c0fcSJason Gunthorpe struct arm_smmu_ctx_desc_cfg *cd_table = &master->cd_table;
1239b2f4c0fcSJason Gunthorpe
1240b2f4c0fcSJason Gunthorpe if (!arm_smmu_cdtab_allocated(cd_table))
1241b2f4c0fcSJason Gunthorpe return NULL;
1242b2f4c0fcSJason Gunthorpe
1243b2f4c0fcSJason Gunthorpe if (cd_table->s1fmt == STRTAB_STE_0_S1FMT_LINEAR)
1244475918e9SMichael Shavit return &cd_table->linear.table[ssid];
1245e86d1aa8SWill Deacon
1246b2f4c0fcSJason Gunthorpe l2 = cd_table->l2.l2ptrs[arm_smmu_cdtab_l1_idx(ssid)];
1247b2f4c0fcSJason Gunthorpe if (!l2)
1248e86d1aa8SWill Deacon return NULL;
1249e86d1aa8SWill Deacon return &l2->cds[arm_smmu_cdtab_l2_idx(ssid)];
1250e86d1aa8SWill Deacon }
1251475918e9SMichael Shavit
arm_smmu_alloc_cd_ptr(struct arm_smmu_master * master,u32 ssid)1252e86d1aa8SWill Deacon static struct arm_smmu_cd *arm_smmu_alloc_cd_ptr(struct arm_smmu_master *master,
1253e86d1aa8SWill Deacon u32 ssid)
125424503148SMichael Shavit {
1255e86d1aa8SWill Deacon struct arm_smmu_ctx_desc_cfg *cd_table = &master->cd_table;
1256b2f4c0fcSJason Gunthorpe struct arm_smmu_device *smmu = master->smmu;
1257b2f4c0fcSJason Gunthorpe
1258e86d1aa8SWill Deacon might_sleep();
1259e86d1aa8SWill Deacon iommu_group_mutex_assert(master->dev);
126078a5fbe8SJason Gunthorpe
126178a5fbe8SJason Gunthorpe if (!arm_smmu_cdtab_allocated(cd_table)) {
126278a5fbe8SJason Gunthorpe if (arm_smmu_alloc_cd_tables(master))
126378a5fbe8SJason Gunthorpe return NULL;
126478a5fbe8SJason Gunthorpe }
126556e1a4ccSJason Gunthorpe
126656e1a4ccSJason Gunthorpe if (cd_table->s1fmt == STRTAB_STE_0_S1FMT_64K_L2) {
1267e86d1aa8SWill Deacon unsigned int idx = arm_smmu_cdtab_l1_idx(ssid);
126878a5fbe8SJason Gunthorpe struct arm_smmu_cdtab_l2 **l2ptr = &cd_table->l2.l2ptrs[idx];
126978a5fbe8SJason Gunthorpe
127078a5fbe8SJason Gunthorpe if (!*l2ptr) {
127178a5fbe8SJason Gunthorpe dma_addr_t l2ptr_dma;
1272e86d1aa8SWill Deacon
1273e86d1aa8SWill Deacon *l2ptr = dma_alloc_coherent(smmu->dev, sizeof(**l2ptr),
127478a5fbe8SJason Gunthorpe &l2ptr_dma, GFP_KERNEL);
127578a5fbe8SJason Gunthorpe if (!*l2ptr)
1276e86d1aa8SWill Deacon return NULL;
127778a5fbe8SJason Gunthorpe
127878a5fbe8SJason Gunthorpe arm_smmu_write_cd_l1_desc(&cd_table->l2.l1tab[idx],
127978a5fbe8SJason Gunthorpe l2ptr_dma);
128078a5fbe8SJason Gunthorpe /* An invalid L1CD can be cached */
128178a5fbe8SJason Gunthorpe arm_smmu_sync_cd(master, ssid, false);
128278a5fbe8SJason Gunthorpe }
128378a5fbe8SJason Gunthorpe }
128478a5fbe8SJason Gunthorpe return arm_smmu_get_cd_ptr(master, ssid);
1285da55da5aSJason Gunthorpe }
1286e86d1aa8SWill Deacon
128778a5fbe8SJason Gunthorpe struct arm_smmu_cd_writer {
128878a5fbe8SJason Gunthorpe struct arm_smmu_entry_writer writer;
128978a5fbe8SJason Gunthorpe unsigned int ssid;
129078a5fbe8SJason Gunthorpe };
129178a5fbe8SJason Gunthorpe
129278a5fbe8SJason Gunthorpe VISIBLE_IF_KUNIT
arm_smmu_get_cd_used(const __le64 * ent,__le64 * used_bits)129378a5fbe8SJason Gunthorpe void arm_smmu_get_cd_used(const __le64 *ent, __le64 *used_bits)
129478a5fbe8SJason Gunthorpe {
129578a5fbe8SJason Gunthorpe used_bits[0] = cpu_to_le64(CTXDESC_CD_0_V);
129678a5fbe8SJason Gunthorpe if (!(ent[0] & cpu_to_le64(CTXDESC_CD_0_V)))
129778a5fbe8SJason Gunthorpe return;
129878a5fbe8SJason Gunthorpe memset(used_bits, 0xFF, sizeof(struct arm_smmu_cd));
129978a5fbe8SJason Gunthorpe
1300e9d1e4ffSJason Gunthorpe /*
130178a5fbe8SJason Gunthorpe * If EPD0 is set by the make function it means
130278a5fbe8SJason Gunthorpe * T0SZ/TG0/IR0/OR0/SH0/TTB0 are IGNORED
130378a5fbe8SJason Gunthorpe */
1304be7c90deSJason Gunthorpe if (ent[0] & cpu_to_le64(CTXDESC_CD_0_TCR_EPD0)) {
1305be7c90deSJason Gunthorpe used_bits[0] &= ~cpu_to_le64(
130678a5fbe8SJason Gunthorpe CTXDESC_CD_0_TCR_T0SZ | CTXDESC_CD_0_TCR_TG0 |
130778a5fbe8SJason Gunthorpe CTXDESC_CD_0_TCR_IRGN0 | CTXDESC_CD_0_TCR_ORGN0 |
130878a5fbe8SJason Gunthorpe CTXDESC_CD_0_TCR_SH0);
130978a5fbe8SJason Gunthorpe used_bits[1] &= ~cpu_to_le64(CTXDESC_CD_1_TTB0_MASK);
131078a5fbe8SJason Gunthorpe }
131178a5fbe8SJason Gunthorpe }
131278a5fbe8SJason Gunthorpe EXPORT_SYMBOL_IF_KUNIT(arm_smmu_get_cd_used);
131378a5fbe8SJason Gunthorpe
arm_smmu_cd_writer_sync_entry(struct arm_smmu_entry_writer * writer)1314be7c90deSJason Gunthorpe static void arm_smmu_cd_writer_sync_entry(struct arm_smmu_entry_writer *writer)
1315be7c90deSJason Gunthorpe {
1316be7c90deSJason Gunthorpe struct arm_smmu_cd_writer *cd_writer =
1317be7c90deSJason Gunthorpe container_of(writer, struct arm_smmu_cd_writer, writer);
1318be7c90deSJason Gunthorpe
1319be7c90deSJason Gunthorpe arm_smmu_sync_cd(writer->master, cd_writer->ssid, true);
1320be7c90deSJason Gunthorpe }
132178a5fbe8SJason Gunthorpe
132278a5fbe8SJason Gunthorpe static const struct arm_smmu_entry_writer_ops arm_smmu_cd_writer_ops = {
132378a5fbe8SJason Gunthorpe .sync = arm_smmu_cd_writer_sync_entry,
1324e9d1e4ffSJason Gunthorpe .get_used = arm_smmu_get_cd_used,
1325e9d1e4ffSJason Gunthorpe };
1326e9d1e4ffSJason Gunthorpe
arm_smmu_write_cd_entry(struct arm_smmu_master * master,int ssid,struct arm_smmu_cd * cdptr,const struct arm_smmu_cd * target)1327e9d1e4ffSJason Gunthorpe void arm_smmu_write_cd_entry(struct arm_smmu_master *master, int ssid,
1328e9d1e4ffSJason Gunthorpe struct arm_smmu_cd *cdptr,
132904905c17SJason Gunthorpe const struct arm_smmu_cd *target)
133004905c17SJason Gunthorpe {
133104905c17SJason Gunthorpe bool target_valid = target->data[0] & cpu_to_le64(CTXDESC_CD_0_V);
133204905c17SJason Gunthorpe bool cur_valid = cdptr->data[0] & cpu_to_le64(CTXDESC_CD_0_V);
1333e9d1e4ffSJason Gunthorpe struct arm_smmu_cd_writer cd_writer = {
1334e9d1e4ffSJason Gunthorpe .writer = {
1335e9d1e4ffSJason Gunthorpe .ops = &arm_smmu_cd_writer_ops,
1336e9d1e4ffSJason Gunthorpe .master = master,
133704905c17SJason Gunthorpe },
133804905c17SJason Gunthorpe .ssid = ssid,
133904905c17SJason Gunthorpe };
134004905c17SJason Gunthorpe
134104905c17SJason Gunthorpe if (ssid != IOMMU_NO_PASID && cur_valid != target_valid) {
1342e86d1aa8SWill Deacon if (cur_valid)
1343e86d1aa8SWill Deacon master->cd_table.used_ssids--;
1344e86d1aa8SWill Deacon else
134504905c17SJason Gunthorpe master->cd_table.used_ssids++;
1346e9d1e4ffSJason Gunthorpe }
134704905c17SJason Gunthorpe
1348e86d1aa8SWill Deacon arm_smmu_write_entry(&cd_writer.writer, cdptr->data, target->data);
1349e9d1e4ffSJason Gunthorpe }
1350e9d1e4ffSJason Gunthorpe
arm_smmu_make_s1_cd(struct arm_smmu_cd * target,struct arm_smmu_master * master,struct arm_smmu_domain * smmu_domain)1351e9d1e4ffSJason Gunthorpe void arm_smmu_make_s1_cd(struct arm_smmu_cd *target,
1352e9d1e4ffSJason Gunthorpe struct arm_smmu_master *master,
1353e9d1e4ffSJason Gunthorpe struct arm_smmu_domain *smmu_domain)
1354e9d1e4ffSJason Gunthorpe {
135525c776ddSKunkun Jiang struct arm_smmu_ctx_desc *cd = &smmu_domain->cd;
135625c776ddSKunkun Jiang const struct io_pgtable_cfg *pgtbl_cfg =
135725c776ddSKunkun Jiang &io_pgtable_ops_to_pgtable(smmu_domain->pgtbl_ops)->cfg;
135825c776ddSKunkun Jiang typeof(&pgtbl_cfg->arm_lpae_s1_cfg.tcr) tcr =
135925c776ddSKunkun Jiang &pgtbl_cfg->arm_lpae_s1_cfg.tcr;
136025c776ddSKunkun Jiang
136104905c17SJason Gunthorpe memset(target, 0, sizeof(*target));
136204905c17SJason Gunthorpe
136304905c17SJason Gunthorpe target->data[0] = cpu_to_le64(
1364e86d1aa8SWill Deacon FIELD_PREP(CTXDESC_CD_0_TCR_T0SZ, tcr->tsz) |
1365da55da5aSJason Gunthorpe FIELD_PREP(CTXDESC_CD_0_TCR_TG0, tcr->tg) |
1366e86d1aa8SWill Deacon FIELD_PREP(CTXDESC_CD_0_TCR_IRGN0, tcr->irgn) |
1367af8f0b83SJason Gunthorpe FIELD_PREP(CTXDESC_CD_0_TCR_ORGN0, tcr->orgn) |
1368af8f0b83SJason Gunthorpe FIELD_PREP(CTXDESC_CD_0_TCR_SH0, tcr->sh) |
1369af8f0b83SJason Gunthorpe #ifdef __BIG_ENDIAN
1370af8f0b83SJason Gunthorpe CTXDESC_CD_0_ENDI |
1371af8f0b83SJason Gunthorpe #endif
1372af8f0b83SJason Gunthorpe CTXDESC_CD_0_TCR_EPD1 |
1373af8f0b83SJason Gunthorpe CTXDESC_CD_0_V |
1374af8f0b83SJason Gunthorpe FIELD_PREP(CTXDESC_CD_0_TCR_IPS, tcr->ips) |
1375af8f0b83SJason Gunthorpe CTXDESC_CD_0_AA64 |
1376af8f0b83SJason Gunthorpe (master->stall_enabled ? CTXDESC_CD_0_S : 0) |
1377af8f0b83SJason Gunthorpe CTXDESC_CD_0_R |
1378e86d1aa8SWill Deacon CTXDESC_CD_0_A |
1379e86d1aa8SWill Deacon CTXDESC_CD_0_ASET |
138010e4968cSMichael Shavit FIELD_PREP(CTXDESC_CD_0_ASID, cd->asid)
1381e86d1aa8SWill Deacon );
1382e86d1aa8SWill Deacon
1383e86d1aa8SWill Deacon /* To enable dirty flag update, set both Access flag and dirty state update */
1384e86d1aa8SWill Deacon if (pgtbl_cfg->quirks & IO_PGTABLE_QUIRK_ARM_HD)
138524503148SMichael Shavit target->data[0] |= cpu_to_le64(CTXDESC_CD_0_TCR_HA |
1386475918e9SMichael Shavit CTXDESC_CD_0_TCR_HD);
1387e86d1aa8SWill Deacon
1388475918e9SMichael Shavit target->data[1] = cpu_to_le64(pgtbl_cfg->arm_lpae_s1_cfg.ttbr &
1389475918e9SMichael Shavit CTXDESC_CD_1_TTB0_MASK);
1390e86d1aa8SWill Deacon target->data[3] = cpu_to_le64(pgtbl_cfg->arm_lpae_s1_cfg.mair);
1391e86d1aa8SWill Deacon }
1392e86d1aa8SWill Deacon EXPORT_SYMBOL_IF_KUNIT(arm_smmu_make_s1_cd);
1393475918e9SMichael Shavit
arm_smmu_clear_cd(struct arm_smmu_master * master,ioasid_t ssid)1394475918e9SMichael Shavit void arm_smmu_clear_cd(struct arm_smmu_master *master, ioasid_t ssid)
1395e86d1aa8SWill Deacon {
1396e86d1aa8SWill Deacon struct arm_smmu_cd target = {};
1397e86d1aa8SWill Deacon struct arm_smmu_cd *cdptr;
1398475918e9SMichael Shavit
1399475918e9SMichael Shavit if (!arm_smmu_cdtab_allocated(&master->cd_table))
1400e86d1aa8SWill Deacon return;
1401e86d1aa8SWill Deacon cdptr = arm_smmu_get_cd_ptr(master, ssid);
1402475918e9SMichael Shavit if (WARN_ON(!cdptr))
1403475918e9SMichael Shavit return;
1404e86d1aa8SWill Deacon arm_smmu_write_cd_entry(master, ssid, cdptr, &target);
1405475918e9SMichael Shavit }
1406e86d1aa8SWill Deacon
arm_smmu_alloc_cd_tables(struct arm_smmu_master * master)1407e86d1aa8SWill Deacon static int arm_smmu_alloc_cd_tables(struct arm_smmu_master *master)
1408475918e9SMichael Shavit {
1409e86d1aa8SWill Deacon int ret;
1410e86d1aa8SWill Deacon size_t l1size;
1411475918e9SMichael Shavit size_t max_contexts;
1412e86d1aa8SWill Deacon struct arm_smmu_device *smmu = master->smmu;
1413475918e9SMichael Shavit struct arm_smmu_ctx_desc_cfg *cd_table = &master->cd_table;
1414e86d1aa8SWill Deacon
1415e86d1aa8SWill Deacon cd_table->s1cdmax = master->ssid_bits;
1416e86d1aa8SWill Deacon max_contexts = 1 << cd_table->s1cdmax;
1417e86d1aa8SWill Deacon
1418e86d1aa8SWill Deacon if (!(smmu->features & ARM_SMMU_FEAT_2_LVL_CDTAB) ||
1419e86d1aa8SWill Deacon max_contexts <= CTXDESC_L2_ENTRIES) {
1420e86d1aa8SWill Deacon cd_table->s1fmt = STRTAB_STE_0_S1FMT_LINEAR;
1421e86d1aa8SWill Deacon cd_table->linear.num_ents = max_contexts;
1422475918e9SMichael Shavit
1423475918e9SMichael Shavit l1size = max_contexts * sizeof(struct arm_smmu_cd),
1424475918e9SMichael Shavit cd_table->linear.table = dma_alloc_coherent(smmu->dev, l1size,
1425e86d1aa8SWill Deacon &cd_table->cdtab_dma,
1426e86d1aa8SWill Deacon GFP_KERNEL);
1427e86d1aa8SWill Deacon if (!cd_table->linear.table)
1428e86d1aa8SWill Deacon return -ENOMEM;
142910e4968cSMichael Shavit } else {
1430e86d1aa8SWill Deacon cd_table->s1fmt = STRTAB_STE_0_S1FMT_64K_L2;
1431e86d1aa8SWill Deacon cd_table->l2.num_l1_ents =
1432e86d1aa8SWill Deacon DIV_ROUND_UP(max_contexts, CTXDESC_L2_ENTRIES);
143310e4968cSMichael Shavit
1434475918e9SMichael Shavit cd_table->l2.l2ptrs = kcalloc(cd_table->l2.num_l1_ents,
1435e86d1aa8SWill Deacon sizeof(*cd_table->l2.l2ptrs),
1436475918e9SMichael Shavit GFP_KERNEL);
1437e86d1aa8SWill Deacon if (!cd_table->l2.l2ptrs)
1438e86d1aa8SWill Deacon return -ENOMEM;
1439475918e9SMichael Shavit
1440475918e9SMichael Shavit l1size = cd_table->l2.num_l1_ents * sizeof(struct arm_smmu_cdtab_l1);
1441e86d1aa8SWill Deacon cd_table->l2.l1tab = dma_alloc_coherent(smmu->dev, l1size,
1442e86d1aa8SWill Deacon &cd_table->cdtab_dma,
1443e86d1aa8SWill Deacon GFP_KERNEL);
1444475918e9SMichael Shavit if (!cd_table->l2.l2ptrs) {
1445475918e9SMichael Shavit ret = -ENOMEM;
1446e86d1aa8SWill Deacon goto err_free_l2ptrs;
1447475918e9SMichael Shavit }
1448475918e9SMichael Shavit }
1449e86d1aa8SWill Deacon return 0;
1450475918e9SMichael Shavit
1451e86d1aa8SWill Deacon err_free_l2ptrs:
1452475918e9SMichael Shavit kfree(cd_table->l2.l2ptrs);
1453e86d1aa8SWill Deacon cd_table->l2.l2ptrs = NULL;
1454e86d1aa8SWill Deacon return ret;
1455475918e9SMichael Shavit }
1456475918e9SMichael Shavit
arm_smmu_free_cd_tables(struct arm_smmu_master * master)1457475918e9SMichael Shavit static void arm_smmu_free_cd_tables(struct arm_smmu_master *master)
1458e86d1aa8SWill Deacon {
1459e86d1aa8SWill Deacon int i;
1460e86d1aa8SWill Deacon struct arm_smmu_device *smmu = master->smmu;
1461a4d75360SJason Gunthorpe struct arm_smmu_ctx_desc_cfg *cd_table = &master->cd_table;
1462e86d1aa8SWill Deacon
1463e86d1aa8SWill Deacon if (cd_table->s1fmt != STRTAB_STE_0_S1FMT_LINEAR) {
1464e86d1aa8SWill Deacon for (i = 0; i < cd_table->l2.num_l1_ents; i++) {
1465a4d75360SJason Gunthorpe if (!cd_table->l2.l2ptrs[i])
1466a4d75360SJason Gunthorpe continue;
1467e86d1aa8SWill Deacon
14687b87c93cSJason Gunthorpe dma_free_coherent(smmu->dev,
1469e86d1aa8SWill Deacon sizeof(*cd_table->l2.l2ptrs[i]),
1470e86d1aa8SWill Deacon cd_table->l2.l2ptrs[i],
1471e86d1aa8SWill Deacon arm_smmu_cd_l1_get_desc(&cd_table->l2.l1tab[i]));
1472de31c355SJason Gunthorpe }
1473de31c355SJason Gunthorpe kfree(cd_table->l2.l2ptrs);
1474de31c355SJason Gunthorpe
1475de31c355SJason Gunthorpe dma_free_coherent(smmu->dev,
1476de31c355SJason Gunthorpe cd_table->l2.num_l1_ents *
1477de31c355SJason Gunthorpe sizeof(struct arm_smmu_cdtab_l1),
1478e86d1aa8SWill Deacon cd_table->l2.l1tab, cd_table->cdtab_dma);
1479de31c355SJason Gunthorpe } else {
1480de31c355SJason Gunthorpe dma_free_coherent(smmu->dev,
1481e86d1aa8SWill Deacon cd_table->linear.num_ents *
1482e86d1aa8SWill Deacon sizeof(struct arm_smmu_cd),
1483e86d1aa8SWill Deacon cd_table->linear.table, cd_table->cdtab_dma);
1484de31c355SJason Gunthorpe }
1485e86d1aa8SWill Deacon }
1486e86d1aa8SWill Deacon
1487e86d1aa8SWill Deacon /* Stream table manipulation functions */
arm_smmu_write_strtab_l1_desc(struct arm_smmu_strtab_l1 * dst,dma_addr_t l2ptr_dma)1488e86d1aa8SWill Deacon static void arm_smmu_write_strtab_l1_desc(struct arm_smmu_strtab_l1 *dst,
1489de31c355SJason Gunthorpe dma_addr_t l2ptr_dma)
1490e86d1aa8SWill Deacon {
1491e86d1aa8SWill Deacon u64 val = 0;
1492de31c355SJason Gunthorpe
1493de31c355SJason Gunthorpe val |= FIELD_PREP(STRTAB_L1_DESC_SPAN, STRTAB_SPLIT + 1);
1494de31c355SJason Gunthorpe val |= l2ptr_dma & STRTAB_L1_DESC_L2PTR_MASK;
1495de31c355SJason Gunthorpe
1496de31c355SJason Gunthorpe /* The HW has 64 bit atomicity with stores to the L2 STE table */
1497de31c355SJason Gunthorpe WRITE_ONCE(dst->l2ptr, cpu_to_le64(val));
1498de31c355SJason Gunthorpe }
1499de31c355SJason Gunthorpe
1500de31c355SJason Gunthorpe struct arm_smmu_ste_writer {
1501de31c355SJason Gunthorpe struct arm_smmu_entry_writer writer;
1502de31c355SJason Gunthorpe u32 sid;
1503de31c355SJason Gunthorpe };
1504de31c355SJason Gunthorpe
arm_smmu_ste_writer_sync_entry(struct arm_smmu_entry_writer * writer)1505de31c355SJason Gunthorpe static void arm_smmu_ste_writer_sync_entry(struct arm_smmu_entry_writer *writer)
1506de31c355SJason Gunthorpe {
1507de31c355SJason Gunthorpe struct arm_smmu_ste_writer *ste_writer =
1508de31c355SJason Gunthorpe container_of(writer, struct arm_smmu_ste_writer, writer);
1509de31c355SJason Gunthorpe struct arm_smmu_cmdq_ent cmd = {
1510de31c355SJason Gunthorpe .opcode = CMDQ_OP_CFGI_STE,
1511de31c355SJason Gunthorpe .cfgi = {
1512de31c355SJason Gunthorpe .sid = ste_writer->sid,
1513de31c355SJason Gunthorpe .leaf = true,
1514de31c355SJason Gunthorpe },
1515de31c355SJason Gunthorpe };
1516de31c355SJason Gunthorpe
1517de31c355SJason Gunthorpe arm_smmu_cmdq_issue_cmd_with_sync(writer->master->smmu, &cmd);
1518de31c355SJason Gunthorpe }
1519de31c355SJason Gunthorpe
1520de31c355SJason Gunthorpe static const struct arm_smmu_entry_writer_ops arm_smmu_ste_writer_ops = {
1521de31c355SJason Gunthorpe .sync = arm_smmu_ste_writer_sync_entry,
1522e86d1aa8SWill Deacon .get_used = arm_smmu_get_ste_used,
1523e86d1aa8SWill Deacon };
152456e1a4ccSJason Gunthorpe
arm_smmu_write_ste(struct arm_smmu_master * master,u32 sid,struct arm_smmu_ste * ste,const struct arm_smmu_ste * target)152556e1a4ccSJason Gunthorpe static void arm_smmu_write_ste(struct arm_smmu_master *master, u32 sid,
1526e86d1aa8SWill Deacon struct arm_smmu_ste *ste,
15277686aa5fSJason Gunthorpe const struct arm_smmu_ste *target)
15287686aa5fSJason Gunthorpe {
15297686aa5fSJason Gunthorpe struct arm_smmu_device *smmu = master->smmu;
15307686aa5fSJason Gunthorpe struct arm_smmu_ste_writer ste_writer = {
15317686aa5fSJason Gunthorpe .writer = {
1532da55da5aSJason Gunthorpe .ops = &arm_smmu_ste_writer_ops,
15337686aa5fSJason Gunthorpe .master = master,
153456e1a4ccSJason Gunthorpe },
153556e1a4ccSJason Gunthorpe .sid = sid,
1536ec9098d6SMostafa Saleh };
15377686aa5fSJason Gunthorpe
15387686aa5fSJason Gunthorpe arm_smmu_write_entry(&ste_writer.writer, ste->data, target->data);
15397686aa5fSJason Gunthorpe
15407686aa5fSJason Gunthorpe /* It's likely that we'll want to use the new STE soon */
15417686aa5fSJason Gunthorpe if (!(smmu->options & ARM_SMMU_OPT_SKIP_PREFETCH)) {
1542ec9098d6SMostafa Saleh struct arm_smmu_cmdq_ent
1543ec9098d6SMostafa Saleh prefetch_cmd = { .opcode = CMDQ_OP_PREFETCH_CFG,
1544ec9098d6SMostafa Saleh .prefetch = {
1545ec9098d6SMostafa Saleh .sid = sid,
15467686aa5fSJason Gunthorpe } };
1547da55da5aSJason Gunthorpe
15487686aa5fSJason Gunthorpe arm_smmu_cmdq_issue_cmd(smmu, &prefetch_cmd);
154956e1a4ccSJason Gunthorpe }
155056e1a4ccSJason Gunthorpe }
1551ce26ea9eSJason Gunthorpe
1552ce26ea9eSJason Gunthorpe VISIBLE_IF_KUNIT
arm_smmu_make_abort_ste(struct arm_smmu_ste * target)1553efe15df0SJason Gunthorpe void arm_smmu_make_abort_ste(struct arm_smmu_ste *target)
1554efe15df0SJason Gunthorpe {
155512a48fe9SJason Gunthorpe memset(target, 0, sizeof(*target));
1556e86d1aa8SWill Deacon target->data[0] = cpu_to_le64(
1557efe15df0SJason Gunthorpe STRTAB_STE_0_V |
1558efe15df0SJason Gunthorpe FIELD_PREP(STRTAB_STE_0_CFG, STRTAB_STE_0_CFG_ABORT));
1559efe15df0SJason Gunthorpe }
1560efe15df0SJason Gunthorpe EXPORT_SYMBOL_IF_KUNIT(arm_smmu_make_abort_ste);
1561efe15df0SJason Gunthorpe
1562efe15df0SJason Gunthorpe VISIBLE_IF_KUNIT
arm_smmu_make_bypass_ste(struct arm_smmu_device * smmu,struct arm_smmu_ste * target)1563efe15df0SJason Gunthorpe void arm_smmu_make_bypass_ste(struct arm_smmu_device *smmu,
1564e86d1aa8SWill Deacon struct arm_smmu_ste *target)
1565efe15df0SJason Gunthorpe {
1566ce26ea9eSJason Gunthorpe memset(target, 0, sizeof(*target));
1567e86d1aa8SWill Deacon target->data[0] = cpu_to_le64(
1568e86d1aa8SWill Deacon STRTAB_STE_0_V |
1569e86d1aa8SWill Deacon FIELD_PREP(STRTAB_STE_0_CFG, STRTAB_STE_0_CFG_BYPASS));
1570efe15df0SJason Gunthorpe
1571efe15df0SJason Gunthorpe if (smmu->features & ARM_SMMU_FEAT_ATTR_TYPES_OVR)
1572efe15df0SJason Gunthorpe target->data[1] = cpu_to_le64(FIELD_PREP(STRTAB_STE_1_SHCFG,
1573efe15df0SJason Gunthorpe STRTAB_STE_1_SHCFG_INCOMING));
1574efe15df0SJason Gunthorpe }
15757497f421SJason Gunthorpe EXPORT_SYMBOL_IF_KUNIT(arm_smmu_make_bypass_ste);
1576e86d1aa8SWill Deacon
1577ce26ea9eSJason Gunthorpe VISIBLE_IF_KUNIT
arm_smmu_make_cdtable_ste(struct arm_smmu_ste * target,struct arm_smmu_master * master,bool ats_enabled,unsigned int s1dss)1578ce26ea9eSJason Gunthorpe void arm_smmu_make_cdtable_ste(struct arm_smmu_ste *target,
1579ce26ea9eSJason Gunthorpe struct arm_smmu_master *master, bool ats_enabled,
1580ce26ea9eSJason Gunthorpe unsigned int s1dss)
1581ce26ea9eSJason Gunthorpe {
1582efe15df0SJason Gunthorpe struct arm_smmu_ctx_desc_cfg *cd_table = &master->cd_table;
1583efe15df0SJason Gunthorpe struct arm_smmu_device *smmu = master->smmu;
1584efe15df0SJason Gunthorpe
1585efe15df0SJason Gunthorpe memset(target, 0, sizeof(*target));
1586efe15df0SJason Gunthorpe target->data[0] = cpu_to_le64(
1587efe15df0SJason Gunthorpe STRTAB_STE_0_V |
1588efe15df0SJason Gunthorpe FIELD_PREP(STRTAB_STE_0_CFG, STRTAB_STE_0_CFG_S1_TRANS) |
1589efe15df0SJason Gunthorpe FIELD_PREP(STRTAB_STE_0_S1FMT, cd_table->s1fmt) |
1590efe15df0SJason Gunthorpe (cd_table->cdtab_dma & STRTAB_STE_0_S1CTXPTR_MASK) |
1591efe15df0SJason Gunthorpe FIELD_PREP(STRTAB_STE_0_S1CDMAX, cd_table->s1cdmax));
1592efe15df0SJason Gunthorpe
1593efe15df0SJason Gunthorpe target->data[1] = cpu_to_le64(
1594efe15df0SJason Gunthorpe FIELD_PREP(STRTAB_STE_1_S1DSS, s1dss) |
1595efe15df0SJason Gunthorpe FIELD_PREP(STRTAB_STE_1_S1CIR, STRTAB_STE_1_S1C_CACHE_WBRA) |
1596e86d1aa8SWill Deacon FIELD_PREP(STRTAB_STE_1_S1COR, STRTAB_STE_1_S1C_CACHE_WBRA) |
1597efe15df0SJason Gunthorpe FIELD_PREP(STRTAB_STE_1_S1CSH, ARM_SMMU_SH_ISH) |
1598efe15df0SJason Gunthorpe ((smmu->features & ARM_SMMU_FEAT_STALLS &&
1599efe15df0SJason Gunthorpe !master->stall_enabled) ?
1600efe15df0SJason Gunthorpe STRTAB_STE_1_S1STALLD :
1601efe15df0SJason Gunthorpe 0) |
1602efe15df0SJason Gunthorpe FIELD_PREP(STRTAB_STE_1_EATS,
1603efe15df0SJason Gunthorpe ats_enabled ? STRTAB_STE_1_EATS_TRANS : 0));
1604e86d1aa8SWill Deacon
1605da55da5aSJason Gunthorpe if ((smmu->features & ARM_SMMU_FEAT_ATTR_TYPES_OVR) &&
1606e86d1aa8SWill Deacon s1dss == STRTAB_STE_1_S1DSS_BYPASS)
160756e1a4ccSJason Gunthorpe target->data[1] |= cpu_to_le64(FIELD_PREP(
160856e1a4ccSJason Gunthorpe STRTAB_STE_1_SHCFG, STRTAB_STE_1_SHCFG_INCOMING));
1609efe15df0SJason Gunthorpe
16107497f421SJason Gunthorpe if (smmu->features & ARM_SMMU_FEAT_E2H) {
16117497f421SJason Gunthorpe /*
1612efe15df0SJason Gunthorpe * To support BTM the streamworld needs to match the
1613efe15df0SJason Gunthorpe * configuration of the CPU so that the ASID broadcasts are
161471b0aa10SJason Gunthorpe * properly matched. This means either S/NS-EL2-E2H (hypervisor)
161571b0aa10SJason Gunthorpe * or NS-EL1 (guest). Since an SVA domain can be installed in a
161671b0aa10SJason Gunthorpe * PASID this should always use a BTM compatible configuration
161771b0aa10SJason Gunthorpe * if the HW supports it.
161871b0aa10SJason Gunthorpe */
1619ec9098d6SMostafa Saleh target->data[1] |= cpu_to_le64(
1620efe15df0SJason Gunthorpe FIELD_PREP(STRTAB_STE_1_STRW, STRTAB_STE_1_STRW_EL2));
1621efe15df0SJason Gunthorpe } else {
1622efe15df0SJason Gunthorpe target->data[1] |= cpu_to_le64(
1623efe15df0SJason Gunthorpe FIELD_PREP(STRTAB_STE_1_STRW, STRTAB_STE_1_STRW_NSEL1));
1624efe15df0SJason Gunthorpe
1625efe15df0SJason Gunthorpe /*
1626efe15df0SJason Gunthorpe * VMID 0 is reserved for stage-2 bypass EL1 STEs, see
1627efe15df0SJason Gunthorpe * arm_smmu_domain_alloc_id()
16287497f421SJason Gunthorpe */
1629ec9098d6SMostafa Saleh target->data[2] =
1630ec9098d6SMostafa Saleh cpu_to_le64(FIELD_PREP(STRTAB_STE_2_S2VMID, 0));
1631ec9098d6SMostafa Saleh }
1632efe15df0SJason Gunthorpe }
1633efe15df0SJason Gunthorpe EXPORT_SYMBOL_IF_KUNIT(arm_smmu_make_cdtable_ste);
163471b0aa10SJason Gunthorpe
163571b0aa10SJason Gunthorpe VISIBLE_IF_KUNIT
arm_smmu_make_s2_domain_ste(struct arm_smmu_ste * target,struct arm_smmu_master * master,struct arm_smmu_domain * smmu_domain,bool ats_enabled)163671b0aa10SJason Gunthorpe void arm_smmu_make_s2_domain_ste(struct arm_smmu_ste *target,
163771b0aa10SJason Gunthorpe struct arm_smmu_master *master,
163871b0aa10SJason Gunthorpe struct arm_smmu_domain *smmu_domain,
163971b0aa10SJason Gunthorpe bool ats_enabled)
164071b0aa10SJason Gunthorpe {
1641efe15df0SJason Gunthorpe struct arm_smmu_s2_cfg *s2_cfg = &smmu_domain->s2_cfg;
1642e86d1aa8SWill Deacon const struct io_pgtable_cfg *pgtbl_cfg =
164371b0aa10SJason Gunthorpe &io_pgtable_ops_to_pgtable(smmu_domain->pgtbl_ops)->cfg;
1644efe15df0SJason Gunthorpe typeof(&pgtbl_cfg->arm_lpae_s2_cfg.vtcr) vtcr =
1645e86d1aa8SWill Deacon &pgtbl_cfg->arm_lpae_s2_cfg.vtcr;
1646e86d1aa8SWill Deacon u64 vtcr_val;
1647e86d1aa8SWill Deacon struct arm_smmu_device *smmu = master->smmu;
1648efe15df0SJason Gunthorpe
1649e86d1aa8SWill Deacon memset(target, 0, sizeof(*target));
1650e86d1aa8SWill Deacon target->data[0] = cpu_to_le64(
165171b0aa10SJason Gunthorpe STRTAB_STE_0_V |
165271b0aa10SJason Gunthorpe FIELD_PREP(STRTAB_STE_0_CFG, STRTAB_STE_0_CFG_S2_TRANS));
1653e86d1aa8SWill Deacon
1654da55da5aSJason Gunthorpe target->data[1] = cpu_to_le64(
1655e86d1aa8SWill Deacon FIELD_PREP(STRTAB_STE_1_EATS,
16567686aa5fSJason Gunthorpe ats_enabled ? STRTAB_STE_1_EATS_TRANS : 0));
16577686aa5fSJason Gunthorpe
16587686aa5fSJason Gunthorpe if (smmu->features & ARM_SMMU_FEAT_ATTR_TYPES_OVR)
16597686aa5fSJason Gunthorpe target->data[1] |= cpu_to_le64(FIELD_PREP(STRTAB_STE_1_SHCFG,
1660734554fdSRobin Murphy STRTAB_STE_1_SHCFG_INCOMING));
16617686aa5fSJason Gunthorpe
1662e86d1aa8SWill Deacon vtcr_val = FIELD_PREP(STRTAB_STE_2_VTCR_S2T0SZ, vtcr->tsz) |
1663e86d1aa8SWill Deacon FIELD_PREP(STRTAB_STE_2_VTCR_S2SL0, vtcr->sl) |
1664e86d1aa8SWill Deacon FIELD_PREP(STRTAB_STE_2_VTCR_S2IR0, vtcr->irgn) |
1665e86d1aa8SWill Deacon FIELD_PREP(STRTAB_STE_2_VTCR_S2OR0, vtcr->orgn) |
16667686aa5fSJason Gunthorpe FIELD_PREP(STRTAB_STE_2_VTCR_S2SH0, vtcr->sh) |
166757b89048SJason Gunthorpe FIELD_PREP(STRTAB_STE_2_VTCR_S2TG, vtcr->tg) |
1668e86d1aa8SWill Deacon FIELD_PREP(STRTAB_STE_2_VTCR_S2PS, vtcr->ps);
1669e86d1aa8SWill Deacon target->data[2] = cpu_to_le64(
1670e86d1aa8SWill Deacon FIELD_PREP(STRTAB_STE_2_S2VMID, s2_cfg->vmid) |
1671e86d1aa8SWill Deacon FIELD_PREP(STRTAB_STE_2_VTCR, vtcr_val) |
1672e86d1aa8SWill Deacon STRTAB_STE_2_S2AA64 |
1673e86d1aa8SWill Deacon #ifdef __BIG_ENDIAN
1674e86d1aa8SWill Deacon STRTAB_STE_2_S2ENDI |
1675a4d75360SJason Gunthorpe #endif
1676e86d1aa8SWill Deacon STRTAB_STE_2_S2PTW |
1677e86d1aa8SWill Deacon (master->stall_enabled ? STRTAB_STE_2_S2S : 0) |
1678e86d1aa8SWill Deacon STRTAB_STE_2_S2R);
1679e86d1aa8SWill Deacon
1680e86d1aa8SWill Deacon target->data[3] = cpu_to_le64(pgtbl_cfg->arm_lpae_s2_cfg.vttbr &
1681e86d1aa8SWill Deacon STRTAB_STE_3_S2TTB_MASK);
1682e86d1aa8SWill Deacon }
1683e86d1aa8SWill Deacon EXPORT_SYMBOL_IF_KUNIT(arm_smmu_make_s2_domain_ste);
1684e86d1aa8SWill Deacon
1685a4d75360SJason Gunthorpe /*
1686e86d1aa8SWill Deacon * This can safely directly manipulate the STE memory without a sync sequence
1687e86d1aa8SWill Deacon * because the STE table has not been installed in the SMMU yet.
1688e86d1aa8SWill Deacon */
arm_smmu_init_initial_stes(struct arm_smmu_ste * strtab,unsigned int nent)1689e86d1aa8SWill Deacon static void arm_smmu_init_initial_stes(struct arm_smmu_ste *strtab,
1690e86d1aa8SWill Deacon unsigned int nent)
1691e86d1aa8SWill Deacon {
1692e86d1aa8SWill Deacon unsigned int i;
1693e86d1aa8SWill Deacon
1694734554fdSRobin Murphy for (i = 0; i < nent; ++i) {
1695a4d75360SJason Gunthorpe arm_smmu_make_abort_ste(strtab);
1696e86d1aa8SWill Deacon strtab++;
1697e86d1aa8SWill Deacon }
1698e86d1aa8SWill Deacon }
1699cdf315f9SJean-Philippe Brucker
arm_smmu_init_l2_strtab(struct arm_smmu_device * smmu,u32 sid)1700cdf315f9SJean-Philippe Brucker static int arm_smmu_init_l2_strtab(struct arm_smmu_device *smmu, u32 sid)
1701cdf315f9SJean-Philippe Brucker {
1702cdf315f9SJean-Philippe Brucker dma_addr_t l2ptr_dma;
1703cdf315f9SJean-Philippe Brucker struct arm_smmu_strtab_cfg *cfg = &smmu->strtab_cfg;
1704cdf315f9SJean-Philippe Brucker struct arm_smmu_strtab_l2 **l2table;
1705cdf315f9SJean-Philippe Brucker
1706cdf315f9SJean-Philippe Brucker l2table = &cfg->l2.l2ptrs[arm_smmu_strtab_l1_idx(sid)];
1707cdf315f9SJean-Philippe Brucker if (*l2table)
1708cdf315f9SJean-Philippe Brucker return 0;
1709cdf315f9SJean-Philippe Brucker
1710cdf315f9SJean-Philippe Brucker *l2table = dmam_alloc_coherent(smmu->dev, sizeof(**l2table),
1711cdf315f9SJean-Philippe Brucker &l2ptr_dma, GFP_KERNEL);
1712cdf315f9SJean-Philippe Brucker if (!*l2table) {
1713cdf315f9SJean-Philippe Brucker dev_err(smmu->dev,
1714cdf315f9SJean-Philippe Brucker "failed to allocate l2 stream table for SID %u\n",
1715cdf315f9SJean-Philippe Brucker sid);
1716cdf315f9SJean-Philippe Brucker return -ENOMEM;
1717cdf315f9SJean-Philippe Brucker }
1718cdf315f9SJean-Philippe Brucker
1719cdf315f9SJean-Philippe Brucker arm_smmu_init_initial_stes((*l2table)->stes,
1720cdf315f9SJean-Philippe Brucker ARRAY_SIZE((*l2table)->stes));
1721e86d1aa8SWill Deacon arm_smmu_write_strtab_l1_desc(&cfg->l2.l1tab[arm_smmu_strtab_l1_idx(sid)],
1722395ad89dSJean-Philippe Brucker l2ptr_dma);
1723395ad89dSJean-Philippe Brucker return 0;
17243dfa64aeSLu Baolu }
1725395ad89dSJean-Philippe Brucker
arm_smmu_streams_cmp_key(const void * lhs,const struct rb_node * rhs)1726395ad89dSJean-Philippe Brucker static int arm_smmu_streams_cmp_key(const void *lhs, const struct rb_node *rhs)
1727395ad89dSJean-Philippe Brucker {
1728395ad89dSJean-Philippe Brucker struct arm_smmu_stream *stream_rhs =
17293f02a9dcSLu Baolu rb_entry(rhs, struct arm_smmu_stream, node);
1730395ad89dSJean-Philippe Brucker const u32 *sid_lhs = lhs;
1731395ad89dSJean-Philippe Brucker
1732395ad89dSJean-Philippe Brucker if (*sid_lhs < stream_rhs->id)
1733395ad89dSJean-Philippe Brucker return -1;
1734395ad89dSJean-Philippe Brucker if (*sid_lhs > stream_rhs->id)
1735395ad89dSJean-Philippe Brucker return 1;
1736395ad89dSJean-Philippe Brucker return 0;
1737395ad89dSJean-Philippe Brucker }
1738395ad89dSJean-Philippe Brucker
arm_smmu_streams_cmp_node(struct rb_node * lhs,const struct rb_node * rhs)1739395ad89dSJean-Philippe Brucker static int arm_smmu_streams_cmp_node(struct rb_node *lhs,
1740395ad89dSJean-Philippe Brucker const struct rb_node *rhs)
1741395ad89dSJean-Philippe Brucker {
1742395ad89dSJean-Philippe Brucker return arm_smmu_streams_cmp_key(
1743395ad89dSJean-Philippe Brucker &rb_entry(lhs, struct arm_smmu_stream, node)->id, rhs);
1744395ad89dSJean-Philippe Brucker }
1745395ad89dSJean-Philippe Brucker
174666014df7SLu Baolu static struct arm_smmu_master *
arm_smmu_find_master(struct arm_smmu_device * smmu,u32 sid)174766014df7SLu Baolu arm_smmu_find_master(struct arm_smmu_device *smmu, u32 sid)
174866014df7SLu Baolu {
1749395ad89dSJean-Philippe Brucker struct rb_node *node;
1750395ad89dSJean-Philippe Brucker
1751395ad89dSJean-Philippe Brucker lockdep_assert_held(&smmu->streams_mutex);
1752395ad89dSJean-Philippe Brucker
1753395ad89dSJean-Philippe Brucker node = rb_find(&sid, &smmu->streams, arm_smmu_streams_cmp_key);
1754395ad89dSJean-Philippe Brucker if (!node)
1755395ad89dSJean-Philippe Brucker return NULL;
1756395ad89dSJean-Philippe Brucker return rb_entry(node, struct arm_smmu_stream, node)->master;
1757395ad89dSJean-Philippe Brucker }
1758395ad89dSJean-Philippe Brucker
1759395ad89dSJean-Philippe Brucker /* IRQ and event handlers */
arm_smmu_handle_evt(struct arm_smmu_device * smmu,u64 * evt)1760395ad89dSJean-Philippe Brucker static int arm_smmu_handle_evt(struct arm_smmu_device *smmu, u64 *evt)
1761395ad89dSJean-Philippe Brucker {
1762395ad89dSJean-Philippe Brucker int ret = 0;
1763395ad89dSJean-Philippe Brucker u32 perm = 0;
1764395ad89dSJean-Philippe Brucker struct arm_smmu_master *master;
1765395ad89dSJean-Philippe Brucker bool ssid_valid = evt[0] & EVTQ_0_SSV;
1766395ad89dSJean-Philippe Brucker u32 sid = FIELD_GET(EVTQ_0_SID, evt[0]);
1767395ad89dSJean-Philippe Brucker struct iopf_fault fault_evt = { };
1768395ad89dSJean-Philippe Brucker struct iommu_fault *flt = &fault_evt.fault;
1769395ad89dSJean-Philippe Brucker
1770395ad89dSJean-Philippe Brucker switch (FIELD_GET(EVTQ_0_ID, evt[0])) {
1771395ad89dSJean-Philippe Brucker case EVT_ID_TRANSLATION_FAULT:
1772395ad89dSJean-Philippe Brucker case EVT_ID_ADDR_SIZE_FAULT:
1773395ad89dSJean-Philippe Brucker case EVT_ID_ACCESS_FAULT:
1774395ad89dSJean-Philippe Brucker case EVT_ID_PERMISSION_FAULT:
1775395ad89dSJean-Philippe Brucker break;
1776395ad89dSJean-Philippe Brucker default:
1777395ad89dSJean-Philippe Brucker return -EOPNOTSUPP;
1778395ad89dSJean-Philippe Brucker }
1779395ad89dSJean-Philippe Brucker
1780*b58b133eSPranjal Shrivastava if (!(evt[1] & EVTQ_1_STALL))
1781395ad89dSJean-Philippe Brucker return -EOPNOTSUPP;
1782395ad89dSJean-Philippe Brucker
1783395ad89dSJean-Philippe Brucker if (evt[1] & EVTQ_1_RnW)
1784395ad89dSJean-Philippe Brucker perm |= IOMMU_FAULT_PERM_READ;
1785395ad89dSJean-Philippe Brucker else
1786e86d1aa8SWill Deacon perm |= IOMMU_FAULT_PERM_WRITE;
1787e86d1aa8SWill Deacon
1788395ad89dSJean-Philippe Brucker if (evt[1] & EVTQ_1_InD)
1789e86d1aa8SWill Deacon perm |= IOMMU_FAULT_PERM_EXEC;
1790e86d1aa8SWill Deacon
1791e86d1aa8SWill Deacon if (evt[1] & EVTQ_1_PnU)
17929cff922bSJean-Philippe Brucker perm |= IOMMU_FAULT_PERM_PRIV;
17939cff922bSJean-Philippe Brucker
1794e86d1aa8SWill Deacon flt->type = IOMMU_FAULT_PAGE_REQ;
1795e86d1aa8SWill Deacon flt->prm = (struct iommu_fault_page_request) {
1796e86d1aa8SWill Deacon .flags = IOMMU_FAULT_PAGE_REQUEST_LAST_PAGE,
1797e86d1aa8SWill Deacon .grpid = FIELD_GET(EVTQ_1_STAG, evt[1]),
1798e86d1aa8SWill Deacon .perm = perm,
1799e86d1aa8SWill Deacon .addr = FIELD_GET(EVTQ_2_ADDR, evt[2]),
1800395ad89dSJean-Philippe Brucker };
18019cff922bSJean-Philippe Brucker
1802395ad89dSJean-Philippe Brucker if (ssid_valid) {
1803395ad89dSJean-Philippe Brucker flt->prm.flags |= IOMMU_FAULT_PAGE_REQUEST_PASID_VALID;
1804e86d1aa8SWill Deacon flt->prm.pasid = FIELD_GET(EVTQ_0_SSID, evt[0]);
1805e86d1aa8SWill Deacon }
1806e86d1aa8SWill Deacon
1807e86d1aa8SWill Deacon mutex_lock(&smmu->streams_mutex);
1808e86d1aa8SWill Deacon master = arm_smmu_find_master(smmu, sid);
180930de2b54SZhou Guanghui if (!master) {
1810e86d1aa8SWill Deacon ret = -EINVAL;
1811e86d1aa8SWill Deacon goto out_unlock;
1812e86d1aa8SWill Deacon }
1813e86d1aa8SWill Deacon
1814e86d1aa8SWill Deacon ret = iommu_report_device_fault(master->dev, &fault_evt);
1815e86d1aa8SWill Deacon out_unlock:
1816e86d1aa8SWill Deacon mutex_unlock(&smmu->streams_mutex);
1817e86d1aa8SWill Deacon return ret;
1818e86d1aa8SWill Deacon }
1819e86d1aa8SWill Deacon
arm_smmu_evtq_thread(int irq,void * dev)1820e86d1aa8SWill Deacon static irqreturn_t arm_smmu_evtq_thread(int irq, void *dev)
182167ea0b7cSTomas Krcka {
1822e86d1aa8SWill Deacon int i, ret;
1823e86d1aa8SWill Deacon struct arm_smmu_device *smmu = dev;
1824e86d1aa8SWill Deacon struct arm_smmu_queue *q = &smmu->evtq.q;
1825e86d1aa8SWill Deacon struct arm_smmu_ll_queue *llq = &q->llq;
1826e86d1aa8SWill Deacon static DEFINE_RATELIMIT_STATE(rs, DEFAULT_RATELIMIT_INTERVAL,
1827e86d1aa8SWill Deacon DEFAULT_RATELIMIT_BURST);
1828e86d1aa8SWill Deacon u64 evt[EVTQ_ENT_DWORDS];
1829e86d1aa8SWill Deacon
1830e86d1aa8SWill Deacon do {
1831e86d1aa8SWill Deacon while (!queue_remove_raw(q, evt)) {
1832e86d1aa8SWill Deacon u8 id = FIELD_GET(EVTQ_0_ID, evt[0]);
183342987801SJacob Pan
1834e86d1aa8SWill Deacon ret = arm_smmu_handle_evt(smmu, evt);
1835e86d1aa8SWill Deacon if (!ret || !__ratelimit(&rs))
1836e86d1aa8SWill Deacon continue;
1837e86d1aa8SWill Deacon
1838e86d1aa8SWill Deacon dev_info(smmu->dev, "event 0x%02x received:\n", id);
1839e86d1aa8SWill Deacon for (i = 0; i < ARRAY_SIZE(evt); ++i)
1840e86d1aa8SWill Deacon dev_info(smmu->dev, "\t0x%016llx\n",
1841e86d1aa8SWill Deacon (unsigned long long)evt[i]);
1842e86d1aa8SWill Deacon
1843e86d1aa8SWill Deacon cond_resched();
1844e86d1aa8SWill Deacon }
1845e86d1aa8SWill Deacon
1846e86d1aa8SWill Deacon /*
1847e86d1aa8SWill Deacon * Not much we can do on overflow, so scream and pretend we're
1848e86d1aa8SWill Deacon * trying harder.
1849e86d1aa8SWill Deacon */
1850e86d1aa8SWill Deacon if (queue_sync_prod_in(q) == -EOVERFLOW)
1851e86d1aa8SWill Deacon dev_err(smmu->dev, "EVTQ overflow detected -- events lost\n");
1852e86d1aa8SWill Deacon } while (!queue_empty(llq));
1853e86d1aa8SWill Deacon
1854e86d1aa8SWill Deacon /* Sync our overflow flag, as we believe we're up to speed */
1855e86d1aa8SWill Deacon queue_sync_cons_ovf(q);
1856e86d1aa8SWill Deacon return IRQ_HANDLED;
1857e86d1aa8SWill Deacon }
1858e86d1aa8SWill Deacon
arm_smmu_handle_ppr(struct arm_smmu_device * smmu,u64 * evt)1859e86d1aa8SWill Deacon static void arm_smmu_handle_ppr(struct arm_smmu_device *smmu, u64 *evt)
1860e86d1aa8SWill Deacon {
1861e86d1aa8SWill Deacon u32 sid, ssid;
1862e86d1aa8SWill Deacon u16 grpid;
1863e86d1aa8SWill Deacon bool ssv, last;
1864e86d1aa8SWill Deacon
1865e86d1aa8SWill Deacon sid = FIELD_GET(PRIQ_0_SID, evt[0]);
1866e86d1aa8SWill Deacon ssv = FIELD_GET(PRIQ_0_SSID_V, evt[0]);
1867e86d1aa8SWill Deacon ssid = ssv ? FIELD_GET(PRIQ_0_SSID, evt[0]) : IOMMU_NO_PASID;
1868e86d1aa8SWill Deacon last = FIELD_GET(PRIQ_0_PRG_LAST, evt[0]);
1869e86d1aa8SWill Deacon grpid = FIELD_GET(PRIQ_1_PRG_IDX, evt[1]);
1870e86d1aa8SWill Deacon
1871e86d1aa8SWill Deacon dev_info(smmu->dev, "unexpected PRI request received:\n");
1872e86d1aa8SWill Deacon dev_info(smmu->dev,
1873e86d1aa8SWill Deacon "\tsid 0x%08x.0x%05x: [%u%s] %sprivileged %s%s%s access at iova 0x%016llx\n",
1874e86d1aa8SWill Deacon sid, ssid, grpid, last ? "L" : "",
1875e86d1aa8SWill Deacon evt[0] & PRIQ_0_PERM_PRIV ? "" : "un",
1876e86d1aa8SWill Deacon evt[0] & PRIQ_0_PERM_READ ? "R" : "",
1877e86d1aa8SWill Deacon evt[0] & PRIQ_0_PERM_WRITE ? "W" : "",
1878e86d1aa8SWill Deacon evt[0] & PRIQ_0_PERM_EXEC ? "X" : "",
187967ea0b7cSTomas Krcka evt[1] & PRIQ_1_ADDR_MASK);
1880e86d1aa8SWill Deacon
1881e86d1aa8SWill Deacon if (last) {
1882e86d1aa8SWill Deacon struct arm_smmu_cmdq_ent cmd = {
1883e86d1aa8SWill Deacon .opcode = CMDQ_OP_PRI_RESP,
1884e86d1aa8SWill Deacon .substream_valid = ssv,
1885e86d1aa8SWill Deacon .pri = {
1886e86d1aa8SWill Deacon .sid = sid,
1887e86d1aa8SWill Deacon .ssid = ssid,
1888e86d1aa8SWill Deacon .grpid = grpid,
1889e86d1aa8SWill Deacon .resp = PRI_RESP_DENY,
1890e86d1aa8SWill Deacon },
1891e86d1aa8SWill Deacon };
1892e86d1aa8SWill Deacon
1893e86d1aa8SWill Deacon arm_smmu_cmdq_issue_cmd(smmu, &cmd);
1894e86d1aa8SWill Deacon }
1895e86d1aa8SWill Deacon }
1896e86d1aa8SWill Deacon
arm_smmu_priq_thread(int irq,void * dev)1897e86d1aa8SWill Deacon static irqreturn_t arm_smmu_priq_thread(int irq, void *dev)
1898e86d1aa8SWill Deacon {
1899e86d1aa8SWill Deacon struct arm_smmu_device *smmu = dev;
1900e86d1aa8SWill Deacon struct arm_smmu_queue *q = &smmu->priq.q;
1901e86d1aa8SWill Deacon struct arm_smmu_ll_queue *llq = &q->llq;
1902e86d1aa8SWill Deacon u64 evt[PRIQ_ENT_DWORDS];
1903e86d1aa8SWill Deacon
1904e86d1aa8SWill Deacon do {
1905e86d1aa8SWill Deacon while (!queue_remove_raw(q, evt))
1906e86d1aa8SWill Deacon arm_smmu_handle_ppr(smmu, evt);
1907e86d1aa8SWill Deacon
1908e86d1aa8SWill Deacon if (queue_sync_prod_in(q) == -EOVERFLOW)
1909e86d1aa8SWill Deacon dev_err(smmu->dev, "PRIQ overflow detected -- requests lost\n");
1910e86d1aa8SWill Deacon } while (!queue_empty(llq));
1911e86d1aa8SWill Deacon
1912e86d1aa8SWill Deacon /* Sync our overflow flag, as we believe we're up to speed */
1913e86d1aa8SWill Deacon queue_sync_cons_ovf(q);
1914e86d1aa8SWill Deacon return IRQ_HANDLED;
1915e86d1aa8SWill Deacon }
1916e86d1aa8SWill Deacon
1917e86d1aa8SWill Deacon static int arm_smmu_device_disable(struct arm_smmu_device *smmu);
1918e86d1aa8SWill Deacon
arm_smmu_gerror_handler(int irq,void * dev)1919e86d1aa8SWill Deacon static irqreturn_t arm_smmu_gerror_handler(int irq, void *dev)
1920e86d1aa8SWill Deacon {
1921e86d1aa8SWill Deacon u32 gerror, gerrorn, active;
1922e86d1aa8SWill Deacon struct arm_smmu_device *smmu = dev;
1923e86d1aa8SWill Deacon
1924e86d1aa8SWill Deacon gerror = readl_relaxed(smmu->base + ARM_SMMU_GERROR);
1925e86d1aa8SWill Deacon gerrorn = readl_relaxed(smmu->base + ARM_SMMU_GERRORN);
1926e86d1aa8SWill Deacon
1927e86d1aa8SWill Deacon active = gerror ^ gerrorn;
1928e86d1aa8SWill Deacon if (!(active & GERROR_ERR_MASK))
1929e86d1aa8SWill Deacon return IRQ_NONE; /* No errors pending */
1930e86d1aa8SWill Deacon
1931e86d1aa8SWill Deacon dev_warn(smmu->dev,
1932e86d1aa8SWill Deacon "unexpected global error reported (0x%08x), this could be serious\n",
1933e86d1aa8SWill Deacon active);
1934e86d1aa8SWill Deacon
1935e86d1aa8SWill Deacon if (active & GERROR_SFM_ERR) {
1936e86d1aa8SWill Deacon dev_err(smmu->dev, "device has entered Service Failure Mode!\n");
1937e86d1aa8SWill Deacon arm_smmu_device_disable(smmu);
1938e86d1aa8SWill Deacon }
1939e86d1aa8SWill Deacon
1940e86d1aa8SWill Deacon if (active & GERROR_MSI_GERROR_ABT_ERR)
1941e86d1aa8SWill Deacon dev_warn(smmu->dev, "GERROR MSI write aborted\n");
1942e86d1aa8SWill Deacon
1943e86d1aa8SWill Deacon if (active & GERROR_MSI_PRIQ_ABT_ERR)
1944e86d1aa8SWill Deacon dev_warn(smmu->dev, "PRIQ MSI write aborted\n");
1945e86d1aa8SWill Deacon
1946e86d1aa8SWill Deacon if (active & GERROR_MSI_EVTQ_ABT_ERR)
1947e86d1aa8SWill Deacon dev_warn(smmu->dev, "EVTQ MSI write aborted\n");
1948e86d1aa8SWill Deacon
1949e86d1aa8SWill Deacon if (active & GERROR_MSI_CMDQ_ABT_ERR)
1950e86d1aa8SWill Deacon dev_warn(smmu->dev, "CMDQ MSI write aborted\n");
1951e86d1aa8SWill Deacon
1952e86d1aa8SWill Deacon if (active & GERROR_PRIQ_ABT_ERR)
1953e86d1aa8SWill Deacon dev_err(smmu->dev, "PRIQ write aborted -- events may have been lost\n");
1954e86d1aa8SWill Deacon
1955e86d1aa8SWill Deacon if (active & GERROR_EVTQ_ABT_ERR)
1956e86d1aa8SWill Deacon dev_err(smmu->dev, "EVTQ write aborted -- events may have been lost\n");
1957e86d1aa8SWill Deacon
19582f7e8c55SJean-Philippe Brucker if (active & GERROR_CMDQ_ERR)
19592f7e8c55SJean-Philippe Brucker arm_smmu_cmdq_skip_err(smmu);
19602f7e8c55SJean-Philippe Brucker
19612f7e8c55SJean-Philippe Brucker writel(gerror, smmu->base + ARM_SMMU_GERRORN);
19622f7e8c55SJean-Philippe Brucker return IRQ_HANDLED;
19632f7e8c55SJean-Philippe Brucker }
19642f7e8c55SJean-Philippe Brucker
arm_smmu_combined_irq_thread(int irq,void * dev)19652f7e8c55SJean-Philippe Brucker static irqreturn_t arm_smmu_combined_irq_thread(int irq, void *dev)
19662f7e8c55SJean-Philippe Brucker {
19672f7e8c55SJean-Philippe Brucker struct arm_smmu_device *smmu = dev;
19682f7e8c55SJean-Philippe Brucker
19692f7e8c55SJean-Philippe Brucker arm_smmu_evtq_thread(irq, dev);
19702f7e8c55SJean-Philippe Brucker if (smmu->features & ARM_SMMU_FEAT_PRI)
19712f7e8c55SJean-Philippe Brucker arm_smmu_priq_thread(irq, dev);
1972e86d1aa8SWill Deacon
1973e86d1aa8SWill Deacon return IRQ_HANDLED;
197442987801SJacob Pan }
1975e86d1aa8SWill Deacon
arm_smmu_combined_irq_handler(int irq,void * dev)1976e86d1aa8SWill Deacon static irqreturn_t arm_smmu_combined_irq_handler(int irq, void *dev)
1977e86d1aa8SWill Deacon {
1978e86d1aa8SWill Deacon arm_smmu_gerror_handler(irq, dev);
1979e86d1aa8SWill Deacon return IRQ_WAKE_THREAD;
1980e86d1aa8SWill Deacon }
1981e86d1aa8SWill Deacon
1982e86d1aa8SWill Deacon static void
arm_smmu_atc_inv_to_cmd(int ssid,unsigned long iova,size_t size,struct arm_smmu_cmdq_ent * cmd)1983e86d1aa8SWill Deacon arm_smmu_atc_inv_to_cmd(int ssid, unsigned long iova, size_t size,
1984e86d1aa8SWill Deacon struct arm_smmu_cmdq_ent *cmd)
1985e86d1aa8SWill Deacon {
1986e86d1aa8SWill Deacon size_t log2_span;
1987e86d1aa8SWill Deacon size_t span_mask;
1988e86d1aa8SWill Deacon /* ATC invalidates are always on 4096-bytes pages */
1989e86d1aa8SWill Deacon size_t inval_grain_shift = 12;
1990e86d1aa8SWill Deacon unsigned long page_start, page_end;
1991e86d1aa8SWill Deacon
1992e86d1aa8SWill Deacon /*
1993e86d1aa8SWill Deacon * ATS and PASID:
1994e86d1aa8SWill Deacon *
1995e86d1aa8SWill Deacon * If substream_valid is clear, the PCIe TLP is sent without a PASID
1996e86d1aa8SWill Deacon * prefix. In that case all ATC entries within the address range are
1997e86d1aa8SWill Deacon * invalidated, including those that were requested with a PASID! There
1998e86d1aa8SWill Deacon * is no way to invalidate only entries without PASID.
1999e86d1aa8SWill Deacon *
2000e86d1aa8SWill Deacon * When using STRTAB_STE_1_S1DSS_SSID0 (reserving CD 0 for non-PASID
2001e86d1aa8SWill Deacon * traffic), translation requests without PASID create ATC entries
2002e86d1aa8SWill Deacon * without PASID, which must be invalidated with substream_valid clear.
2003e86d1aa8SWill Deacon * This has the unpleasant side-effect of invalidating all PASID-tagged
2004e86d1aa8SWill Deacon * ATC entries within the address range.
2005e86d1aa8SWill Deacon */
2006e86d1aa8SWill Deacon *cmd = (struct arm_smmu_cmdq_ent) {
2007e86d1aa8SWill Deacon .opcode = CMDQ_OP_ATC_INV,
2008e86d1aa8SWill Deacon .substream_valid = (ssid != IOMMU_NO_PASID),
2009e86d1aa8SWill Deacon .atc.ssid = ssid,
2010e86d1aa8SWill Deacon };
2011e86d1aa8SWill Deacon
2012e86d1aa8SWill Deacon if (!size) {
2013e86d1aa8SWill Deacon cmd->atc.size = ATC_INV_SIZE_ALL;
2014e86d1aa8SWill Deacon return;
20151d5f34f0SJason Gunthorpe }
20161d5f34f0SJason Gunthorpe
2017e86d1aa8SWill Deacon page_start = iova >> inval_grain_shift;
2018e86d1aa8SWill Deacon page_end = (iova + size - 1) >> inval_grain_shift;
2019e86d1aa8SWill Deacon
202093f9f795SZhen Lei /*
2021e86d1aa8SWill Deacon * In an ATS Invalidate Request, the address must be aligned on the
20221d5f34f0SJason Gunthorpe * range size, which must be a power of two number of page sizes. We
2023e86d1aa8SWill Deacon * thus have to choose between grossly over-invalidating the region, or
202493f9f795SZhen Lei * splitting the invalidation into multiple commands. For simplicity
2025cdf315f9SJean-Philippe Brucker * we'll go with the first solution, but should refine it in the future
2026cdf315f9SJean-Philippe Brucker * if multiple commands are shown to be more efficient.
2027eff19474SZhen Lei *
2028e86d1aa8SWill Deacon * Find the smallest power of two that covers the range. The most
2029e86d1aa8SWill Deacon * significant differing bit between the start and end addresses,
2030eff19474SZhen Lei * fls(start ^ end), indicates the required span. For example:
2031e86d1aa8SWill Deacon *
2032e86d1aa8SWill Deacon * We want to invalidate pages [8; 11]. This is already the ideal range:
2033d38c28dbSJason Gunthorpe * x = 0b1000 ^ 0b1011 = 0b11
2034d38c28dbSJason Gunthorpe * span = 1 << fls(x) = 4
2035e86d1aa8SWill Deacon *
2036ad10dce6SJason Gunthorpe * To invalidate pages [7; 10], we need to invalidate [0; 15]:
2037e86d1aa8SWill Deacon * x = 0b0111 ^ 0b1010 = 0b1101
2038e86d1aa8SWill Deacon * span = 1 << fls(x) = 16
2039e86d1aa8SWill Deacon */
2040fac95671SJohn Garry log2_span = fls_long(page_start ^ page_end);
2041e86d1aa8SWill Deacon span_mask = (1ULL << log2_span) - 1;
2042e86d1aa8SWill Deacon
2043e86d1aa8SWill Deacon page_start &= ~span_mask;
2044e86d1aa8SWill Deacon
2045e86d1aa8SWill Deacon cmd->atc.addr = page_start << inval_grain_shift;
2046e86d1aa8SWill Deacon cmd->atc.size = log2_span;
2047e86d1aa8SWill Deacon }
2048e86d1aa8SWill Deacon
arm_smmu_atc_inv_master(struct arm_smmu_master * master,ioasid_t ssid)2049e86d1aa8SWill Deacon static int arm_smmu_atc_inv_master(struct arm_smmu_master *master,
2050e86d1aa8SWill Deacon ioasid_t ssid)
2051e86d1aa8SWill Deacon {
2052e86d1aa8SWill Deacon int i;
2053e86d1aa8SWill Deacon struct arm_smmu_cmdq_ent cmd;
2054e86d1aa8SWill Deacon struct arm_smmu_cmdq_batch cmds;
2055e86d1aa8SWill Deacon
2056e86d1aa8SWill Deacon arm_smmu_atc_inv_to_cmd(ssid, 0, 0, &cmd);
2057e86d1aa8SWill Deacon
2058e86d1aa8SWill Deacon arm_smmu_cmdq_batch_init(master->smmu, &cmds, &cmd);
2059e86d1aa8SWill Deacon for (i = 0; i < master->num_streams; i++) {
2060e86d1aa8SWill Deacon cmd.atc.sid = master->streams[i].id;
2061e86d1aa8SWill Deacon arm_smmu_cmdq_batch_add(master->smmu, &cmds, &cmd);
2062fac95671SJohn Garry }
2063fac95671SJohn Garry
2064e86d1aa8SWill Deacon return arm_smmu_cmdq_batch_submit(master->smmu, &cmds);
2065ad10dce6SJason Gunthorpe }
2066ad10dce6SJason Gunthorpe
arm_smmu_atc_inv_domain(struct arm_smmu_domain * smmu_domain,unsigned long iova,size_t size)2067ad10dce6SJason Gunthorpe int arm_smmu_atc_inv_domain(struct arm_smmu_domain *smmu_domain,
2068ad10dce6SJason Gunthorpe unsigned long iova, size_t size)
2069e86d1aa8SWill Deacon {
2070e86d1aa8SWill Deacon struct arm_smmu_master_domain *master_domain;
2071e86d1aa8SWill Deacon int i;
2072d38c28dbSJason Gunthorpe unsigned long flags;
207364efb3deSJason Gunthorpe struct arm_smmu_cmdq_ent cmd = {
2074cdf315f9SJean-Philippe Brucker .opcode = CMDQ_OP_ATC_INV,
2075cdf315f9SJean-Philippe Brucker };
2076e86d1aa8SWill Deacon struct arm_smmu_cmdq_batch cmds;
2077e86d1aa8SWill Deacon
2078e86d1aa8SWill Deacon if (!(smmu_domain->smmu->features & ARM_SMMU_FEAT_ATS))
2079e86d1aa8SWill Deacon return 0;
2080e86d1aa8SWill Deacon
2081e86d1aa8SWill Deacon /*
2082e86d1aa8SWill Deacon * Ensure that we've completed prior invalidation of the main TLBs
2083e86d1aa8SWill Deacon * before we read 'nr_ats_masters' in case of a concurrent call to
2084e86d1aa8SWill Deacon * arm_smmu_enable_ats():
2085e86d1aa8SWill Deacon *
2086e86d1aa8SWill Deacon * // unmap() // arm_smmu_enable_ats()
2087e86d1aa8SWill Deacon * TLBI+SYNC atomic_inc(&nr_ats_masters);
2088e86d1aa8SWill Deacon * smp_mb(); [...]
2089e86d1aa8SWill Deacon * atomic_read(&nr_ats_masters); pci_enable_ats() // writel()
2090e86d1aa8SWill Deacon *
2091e86d1aa8SWill Deacon * Ensures that we always see the incremented 'nr_ats_masters' count if
2092e86d1aa8SWill Deacon * ATS was enabled at the PCI device before completion of the TLBI.
2093e86d1aa8SWill Deacon */
2094e86d1aa8SWill Deacon smp_mb();
2095e86d1aa8SWill Deacon if (!atomic_read(&smmu_domain->nr_ats_masters))
2096e86d1aa8SWill Deacon return 0;
2097e86d1aa8SWill Deacon
20983e630336SJean-Philippe Brucker arm_smmu_cmdq_batch_init(smmu_domain->smmu, &cmds, &cmd);
2099987a878eSMichael Shavit
21003e630336SJean-Philippe Brucker spin_lock_irqsave(&smmu_domain->devices_lock, flags);
21013e630336SJean-Philippe Brucker list_for_each_entry(master_domain, &smmu_domain->devices,
21023e630336SJean-Philippe Brucker devices_elm) {
21034537f6f1SZhen Lei struct arm_smmu_master *master = master_domain->master;
21043e630336SJean-Philippe Brucker
210564efb3deSJason Gunthorpe if (!master->ats_enabled)
2106e86d1aa8SWill Deacon continue;
2107e86d1aa8SWill Deacon
2108eba8d2f8SJean-Philippe Brucker arm_smmu_atc_inv_to_cmd(master_domain->ssid, iova, size, &cmd);
2109eba8d2f8SJean-Philippe Brucker
2110eba8d2f8SJean-Philippe Brucker for (i = 0; i < master->num_streams; i++) {
2111e86d1aa8SWill Deacon cmd.atc.sid = master->streams[i].id;
2112e86d1aa8SWill Deacon arm_smmu_cmdq_batch_add(smmu_domain->smmu, &cmds, &cmd);
2113e86d1aa8SWill Deacon }
2114eba8d2f8SJean-Philippe Brucker }
2115e86d1aa8SWill Deacon spin_unlock_irqrestore(&smmu_domain->devices_lock, flags);
2116fac95671SJohn Garry
2117e86d1aa8SWill Deacon return arm_smmu_cmdq_batch_submit(smmu_domain->smmu, &cmds);
2118e86d1aa8SWill Deacon }
2119e86d1aa8SWill Deacon
2120e86d1aa8SWill Deacon /* IO_PGTABLE API */
arm_smmu_tlb_inv_context(void * cookie)2121e86d1aa8SWill Deacon static void arm_smmu_tlb_inv_context(void *cookie)
2122e86d1aa8SWill Deacon {
2123e86d1aa8SWill Deacon struct arm_smmu_domain *smmu_domain = cookie;
2124e86d1aa8SWill Deacon struct arm_smmu_device *smmu = smmu_domain->smmu;
2125eb6c9764SRobin Murphy struct arm_smmu_cmdq_ent cmd;
2126eb6c9764SRobin Murphy
2127e86d1aa8SWill Deacon /*
2128eba8d2f8SJean-Philippe Brucker * NOTE: when io-pgtable is in non-strict mode, we may get here with
2129e86d1aa8SWill Deacon * PTEs previously cleared by unmaps on the current CPU not yet visible
21306833b8f2SRobin Murphy * to the SMMU. We are relying on the dma_wmb() implicit during cmd
2131eb6c9764SRobin Murphy * insertion to guarantee those are observed before the TLBI. Do be
2132eb6c9764SRobin Murphy * careful, 007.
2133eb6c9764SRobin Murphy */
2134eb6c9764SRobin Murphy if (smmu_domain->stage == ARM_SMMU_DOMAIN_S1) {
2135eb6c9764SRobin Murphy arm_smmu_tlb_inv_asid(smmu, smmu_domain->cd.asid);
2136eb6c9764SRobin Murphy } else {
21376833b8f2SRobin Murphy cmd.opcode = CMDQ_OP_TLBI_S12_VMALL;
21386833b8f2SRobin Murphy cmd.tlbi.vmid = smmu_domain->s2_cfg.vmid;
2139eba8d2f8SJean-Philippe Brucker arm_smmu_cmdq_issue_cmd_with_sync(smmu, &cmd);
2140eb6c9764SRobin Murphy }
2141eb6c9764SRobin Murphy arm_smmu_atc_inv_domain(smmu_domain, 0, 0);
2142e86d1aa8SWill Deacon }
2143e86d1aa8SWill Deacon
__arm_smmu_tlb_inv_range(struct arm_smmu_cmdq_ent * cmd,unsigned long iova,size_t size,size_t granule,struct arm_smmu_domain * smmu_domain)2144fac95671SJohn Garry static void __arm_smmu_tlb_inv_range(struct arm_smmu_cmdq_ent *cmd,
2145fac95671SJohn Garry unsigned long iova, size_t size,
2146e86d1aa8SWill Deacon size_t granule,
2147e86d1aa8SWill Deacon struct arm_smmu_domain *smmu_domain)
2148e86d1aa8SWill Deacon {
2149e86d1aa8SWill Deacon struct arm_smmu_device *smmu = smmu_domain->smmu;
2150e86d1aa8SWill Deacon unsigned long end = iova + size, num_pages = 0, tg = 0;
2151e86d1aa8SWill Deacon size_t inv_range = granule;
2152e86d1aa8SWill Deacon struct arm_smmu_cmdq_batch cmds;
2153e86d1aa8SWill Deacon
2154e86d1aa8SWill Deacon if (!size)
2155e86d1aa8SWill Deacon return;
2156e86d1aa8SWill Deacon
2157e86d1aa8SWill Deacon if (smmu->features & ARM_SMMU_FEAT_RANGE_INV) {
2158e86d1aa8SWill Deacon /* Get the leaf page size */
2159eba8d2f8SJean-Philippe Brucker tg = __ffs(smmu_domain->domain.pgsize_bitmap);
2160e86d1aa8SWill Deacon
2161e86d1aa8SWill Deacon num_pages = size >> tg;
2162e86d1aa8SWill Deacon
2163eba8d2f8SJean-Philippe Brucker /* Convert page size of 12,14,16 (log2) to 1,2,3 */
2164e86d1aa8SWill Deacon cmd->tlbi.tg = (tg - 10) / 2;
2165e86d1aa8SWill Deacon
2166e86d1aa8SWill Deacon /*
2167e86d1aa8SWill Deacon * Determine what level the granule is at. For non-leaf, both
2168e86d1aa8SWill Deacon * io-pgtable and SVA pass a nominal last-level granule because
2169e86d1aa8SWill Deacon * they don't know what level(s) actually apply, so ignore that
2170e86d1aa8SWill Deacon * and leave TTL=0. However for various errata reasons we still
2171e86d1aa8SWill Deacon * want to use a range command, so avoid the SVA corner case
2172eba8d2f8SJean-Philippe Brucker * where both scale and num could be 0 as well.
2173eba8d2f8SJean-Philippe Brucker */
2174e86d1aa8SWill Deacon if (cmd->tlbi.leaf)
2175e86d1aa8SWill Deacon cmd->tlbi.ttl = 4 - ((ilog2(granule) - 3) / (tg - 3));
2176e86d1aa8SWill Deacon else if ((num_pages & CMDQ_TLBI_RANGE_NUM_MAX) == 1)
2177eba8d2f8SJean-Philippe Brucker num_pages++;
2178eba8d2f8SJean-Philippe Brucker }
2179eba8d2f8SJean-Philippe Brucker
2180eba8d2f8SJean-Philippe Brucker arm_smmu_cmdq_batch_init(smmu, &cmds, cmd);
2181eba8d2f8SJean-Philippe Brucker
2182eba8d2f8SJean-Philippe Brucker while (iova < end) {
2183eba8d2f8SJean-Philippe Brucker if (smmu->features & ARM_SMMU_FEAT_RANGE_INV) {
2184eba8d2f8SJean-Philippe Brucker /*
2185eba8d2f8SJean-Philippe Brucker * On each iteration of the loop, the range is 5 bits
2186eba8d2f8SJean-Philippe Brucker * worth of the aligned size remaining.
2187eba8d2f8SJean-Philippe Brucker * The range in pages is:
2188eba8d2f8SJean-Philippe Brucker *
2189eba8d2f8SJean-Philippe Brucker * range = (num_pages & (0x1f << __ffs(num_pages)))
21909111aebfSJean-Philippe Brucker */
21919111aebfSJean-Philippe Brucker unsigned long scale, num;
2192987a878eSMichael Shavit
2193eba8d2f8SJean-Philippe Brucker /* Determine the power of 2 multiple number of pages */
2194eba8d2f8SJean-Philippe Brucker scale = __ffs(num_pages);
2195eba8d2f8SJean-Philippe Brucker cmd->tlbi.scale = scale;
2196eba8d2f8SJean-Philippe Brucker
2197eba8d2f8SJean-Philippe Brucker /* Determine how many chunks of 2^scale size we have */
2198e86d1aa8SWill Deacon num = (num_pages >> scale) & CMDQ_TLBI_RANGE_NUM_MAX;
2199e86d1aa8SWill Deacon cmd->tlbi.num = num - 1;
2200e86d1aa8SWill Deacon
2201e86d1aa8SWill Deacon /* range is num * 2^scale * pgsize */
2202e86d1aa8SWill Deacon inv_range = num << (scale + tg);
220364efb3deSJason Gunthorpe
2204e86d1aa8SWill Deacon /* Clear out the lower order bits for the next iteration */
2205e86d1aa8SWill Deacon num_pages -= num << scale;
220651d113c3SJean-Philippe Brucker }
220751d113c3SJean-Philippe Brucker
220851d113c3SJean-Philippe Brucker cmd->tlbi.addr = iova;
220951d113c3SJean-Philippe Brucker arm_smmu_cmdq_batch_add(smmu, &cmds, cmd);
221051d113c3SJean-Philippe Brucker iova += inv_range;
22119111aebfSJean-Philippe Brucker }
22129111aebfSJean-Philippe Brucker arm_smmu_cmdq_batch_submit(smmu, &cmds);
221351d113c3SJean-Philippe Brucker }
221451d113c3SJean-Philippe Brucker
arm_smmu_tlb_inv_range_domain(unsigned long iova,size_t size,size_t granule,bool leaf,struct arm_smmu_domain * smmu_domain)221551d113c3SJean-Philippe Brucker static void arm_smmu_tlb_inv_range_domain(unsigned long iova, size_t size,
221651d113c3SJean-Philippe Brucker size_t granule, bool leaf,
221751d113c3SJean-Philippe Brucker struct arm_smmu_domain *smmu_domain)
221851d113c3SJean-Philippe Brucker {
221951d113c3SJean-Philippe Brucker struct arm_smmu_cmdq_ent cmd = {
222051d113c3SJean-Philippe Brucker .tlbi = {
222151d113c3SJean-Philippe Brucker .leaf = leaf,
2222e86d1aa8SWill Deacon },
2223e86d1aa8SWill Deacon };
2224e86d1aa8SWill Deacon
2225e86d1aa8SWill Deacon if (smmu_domain->stage == ARM_SMMU_DOMAIN_S1) {
2226e86d1aa8SWill Deacon cmd.opcode = smmu_domain->smmu->features & ARM_SMMU_FEAT_E2H ?
2227e86d1aa8SWill Deacon CMDQ_OP_TLBI_EL2_VA : CMDQ_OP_TLBI_NH_VA;
2228e86d1aa8SWill Deacon cmd.tlbi.asid = smmu_domain->cd.asid;
2229e86d1aa8SWill Deacon } else {
2230e86d1aa8SWill Deacon cmd.opcode = CMDQ_OP_TLBI_S2_IPA;
2231e86d1aa8SWill Deacon cmd.tlbi.vmid = smmu_domain->s2_cfg.vmid;
2232e86d1aa8SWill Deacon }
2233e86d1aa8SWill Deacon __arm_smmu_tlb_inv_range(&cmd, iova, size, granule, smmu_domain);
2234e86d1aa8SWill Deacon
2235eba8d2f8SJean-Philippe Brucker /*
2236e86d1aa8SWill Deacon * Unfortunately, this can't be leaf-only since we may have
2237e86d1aa8SWill Deacon * zapped an entire table.
2238e86d1aa8SWill Deacon */
2239e86d1aa8SWill Deacon arm_smmu_atc_inv_domain(smmu_domain, iova, size);
2240e86d1aa8SWill Deacon }
2241e86d1aa8SWill Deacon
arm_smmu_tlb_inv_range_asid(unsigned long iova,size_t size,int asid,size_t granule,bool leaf,struct arm_smmu_domain * smmu_domain)2242e86d1aa8SWill Deacon void arm_smmu_tlb_inv_range_asid(unsigned long iova, size_t size, int asid,
2243e86d1aa8SWill Deacon size_t granule, bool leaf,
224425c776ddSKunkun Jiang struct arm_smmu_domain *smmu_domain)
224525c776ddSKunkun Jiang {
224625c776ddSKunkun Jiang struct arm_smmu_cmdq_ent cmd = {
224725c776ddSKunkun Jiang .opcode = smmu_domain->smmu->features & ARM_SMMU_FEAT_E2H ?
224825c776ddSKunkun Jiang CMDQ_OP_TLBI_EL2_VA : CMDQ_OP_TLBI_NH_VA,
224925c776ddSKunkun Jiang .tlbi = {
225025c776ddSKunkun Jiang .asid = asid,
2251e86d1aa8SWill Deacon .leaf = leaf,
2252359ad157SRobin Murphy },
2253e86d1aa8SWill Deacon };
2254df198b37SRobin Murphy
2255df198b37SRobin Murphy __arm_smmu_tlb_inv_range(&cmd, iova, size, granule, smmu_domain);
2256e86d1aa8SWill Deacon }
2257e86d1aa8SWill Deacon
arm_smmu_tlb_inv_page_nosync(struct iommu_iotlb_gather * gather,unsigned long iova,size_t granule,void * cookie)2258df198b37SRobin Murphy static void arm_smmu_tlb_inv_page_nosync(struct iommu_iotlb_gather *gather,
2259df198b37SRobin Murphy unsigned long iova, size_t granule,
2260e86d1aa8SWill Deacon void *cookie)
22614a20ce0fSRobin Murphy {
2262e86d1aa8SWill Deacon struct arm_smmu_domain *smmu_domain = cookie;
226325c776ddSKunkun Jiang struct iommu_domain *domain = &smmu_domain->domain;
226425c776ddSKunkun Jiang
2265e86d1aa8SWill Deacon iommu_iotlb_gather_add_page(domain, gather, iova, granule);
2266e86d1aa8SWill Deacon }
2267e86d1aa8SWill Deacon
arm_smmu_tlb_inv_walk(unsigned long iova,size_t size,size_t granule,void * cookie)2268e86d1aa8SWill Deacon static void arm_smmu_tlb_inv_walk(unsigned long iova, size_t size,
2269e86d1aa8SWill Deacon size_t granule, void *cookie)
2270d7b2d2baSJason Gunthorpe {
2271d7b2d2baSJason Gunthorpe arm_smmu_tlb_inv_range_domain(iova, size, granule, false, cookie);
2272d7b2d2baSJason Gunthorpe }
2273d7b2d2baSJason Gunthorpe
2274d7b2d2baSJason Gunthorpe static const struct iommu_flush_ops arm_smmu_flush_ops = {
2275d7b2d2baSJason Gunthorpe .tlb_flush_all = arm_smmu_tlb_inv_context,
2276d7b2d2baSJason Gunthorpe .tlb_flush_walk = arm_smmu_tlb_inv_walk,
2277d7b2d2baSJason Gunthorpe .tlb_add_page = arm_smmu_tlb_inv_page_nosync,
2278d7b2d2baSJason Gunthorpe };
2279d7b2d2baSJason Gunthorpe
arm_smmu_dbm_capable(struct arm_smmu_device * smmu)2280d7b2d2baSJason Gunthorpe static bool arm_smmu_dbm_capable(struct arm_smmu_device *smmu)
2281d7b2d2baSJason Gunthorpe {
2282d7b2d2baSJason Gunthorpe u32 features = (ARM_SMMU_FEAT_HD | ARM_SMMU_FEAT_COHERENCY);
2283d7b2d2baSJason Gunthorpe
2284d7b2d2baSJason Gunthorpe return (smmu->features & features) == features;
2285327e10b4SJason Gunthorpe }
2286327e10b4SJason Gunthorpe
2287327e10b4SJason Gunthorpe /* IOMMU API */
arm_smmu_capable(struct device * dev,enum iommu_cap cap)2288e86d1aa8SWill Deacon static bool arm_smmu_capable(struct device *dev, enum iommu_cap cap)
2289e86d1aa8SWill Deacon {
2290e86d1aa8SWill Deacon struct arm_smmu_master *master = dev_iommu_priv_get(dev);
2291e86d1aa8SWill Deacon
2292e86d1aa8SWill Deacon switch (cap) {
2293e86d1aa8SWill Deacon case IOMMU_CAP_CACHE_COHERENCY:
2294d7b2d2baSJason Gunthorpe /* Assume that a coherent TCU implies coherent TBUs */
2295d7b2d2baSJason Gunthorpe return master->smmu->features & ARM_SMMU_FEAT_COHERENCY;
2296d7b2d2baSJason Gunthorpe case IOMMU_CAP_NOEXEC:
2297e86d1aa8SWill Deacon case IOMMU_CAP_DEFERRED_FLUSH:
2298327e10b4SJason Gunthorpe return true;
2299327e10b4SJason Gunthorpe case IOMMU_CAP_DIRTY_TRACKING:
2300327e10b4SJason Gunthorpe return arm_smmu_dbm_capable(master->smmu);
2301327e10b4SJason Gunthorpe default:
2302eb054d67SJoao Martins return false;
2303327e10b4SJason Gunthorpe }
2304327e10b4SJason Gunthorpe }
2305327e10b4SJason Gunthorpe
arm_smmu_domain_alloc(void)2306327e10b4SJason Gunthorpe struct arm_smmu_domain *arm_smmu_domain_alloc(void)
2307327e10b4SJason Gunthorpe {
2308e86d1aa8SWill Deacon struct arm_smmu_domain *smmu_domain;
2309e86d1aa8SWill Deacon
2310e86d1aa8SWill Deacon smmu_domain = kzalloc(sizeof(*smmu_domain), GFP_KERNEL);
2311d7b2d2baSJason Gunthorpe if (!smmu_domain)
2312e86d1aa8SWill Deacon return ERR_PTR(-ENOMEM);
2313e86d1aa8SWill Deacon
2314e86d1aa8SWill Deacon mutex_init(&smmu_domain->init_mutex);
2315e86d1aa8SWill Deacon INIT_LIST_HEAD(&smmu_domain->devices);
2316e86d1aa8SWill Deacon spin_lock_init(&smmu_domain->devices_lock);
2317e86d1aa8SWill Deacon
231810e4968cSMichael Shavit return smmu_domain;
2319e86d1aa8SWill Deacon }
23203f1ce8e8SJean-Philippe Brucker
arm_smmu_domain_alloc_paging(struct device * dev)23213f1ce8e8SJean-Philippe Brucker static struct iommu_domain *arm_smmu_domain_alloc_paging(struct device *dev)
2322d38c28dbSJason Gunthorpe {
23233f1ce8e8SJean-Philippe Brucker struct arm_smmu_domain *smmu_domain;
2324e86d1aa8SWill Deacon
2325e86d1aa8SWill Deacon /*
2326e86d1aa8SWill Deacon * Allocate the domain and initialise some of its data structures.
23271672730cSDawei Li * We can't really do anything meaningful until we've added a
2328e86d1aa8SWill Deacon * master.
2329e86d1aa8SWill Deacon */
2330e86d1aa8SWill Deacon smmu_domain = arm_smmu_domain_alloc();
2331e86d1aa8SWill Deacon if (IS_ERR(smmu_domain))
2332e86d1aa8SWill Deacon return ERR_CAST(smmu_domain);
2333d8cd2006SJason Gunthorpe
233404905c17SJason Gunthorpe if (dev) {
2335e86d1aa8SWill Deacon struct arm_smmu_master *master = dev_iommu_priv_get(dev);
2336e86d1aa8SWill Deacon int ret;
2337d3867e71SMostafa Saleh
2338987a878eSMichael Shavit ret = arm_smmu_domain_finalise(smmu_domain, master->smmu, 0);
2339e86d1aa8SWill Deacon if (ret) {
23403f1ce8e8SJean-Philippe Brucker kfree(smmu_domain);
23413f1ce8e8SJean-Philippe Brucker return ERR_PTR(ret);
2342d38c28dbSJason Gunthorpe }
2343e86d1aa8SWill Deacon }
2344987a878eSMichael Shavit return &smmu_domain->domain;
23453f1ce8e8SJean-Philippe Brucker }
2346e86d1aa8SWill Deacon
arm_smmu_domain_free_paging(struct iommu_domain * domain)2347e86d1aa8SWill Deacon static void arm_smmu_domain_free_paging(struct iommu_domain *domain)
2348e86d1aa8SWill Deacon {
2349d8cd2006SJason Gunthorpe struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
235004905c17SJason Gunthorpe struct arm_smmu_device *smmu = smmu_domain->smmu;
2351e86d1aa8SWill Deacon
2352e86d1aa8SWill Deacon free_io_pgtable_ops(smmu_domain->pgtbl_ops);
2353e86d1aa8SWill Deacon
2354e86d1aa8SWill Deacon /* Free the ASID or VMID */
23551672730cSDawei Li if (smmu_domain->stage == ARM_SMMU_DOMAIN_S1) {
23561672730cSDawei Li /* Prevent SVA from touching the CD while we're freeing it */
23571672730cSDawei Li mutex_lock(&arm_smmu_asid_lock);
2358e86d1aa8SWill Deacon xa_erase(&arm_smmu_asid_xa, smmu_domain->cd.asid);
2359e86d1aa8SWill Deacon mutex_unlock(&arm_smmu_asid_lock);
2360e86d1aa8SWill Deacon } else {
2361e86d1aa8SWill Deacon struct arm_smmu_s2_cfg *cfg = &smmu_domain->s2_cfg;
2362e86d1aa8SWill Deacon if (cfg->vmid)
2363e86d1aa8SWill Deacon ida_free(&smmu->vmid_map, cfg->vmid);
2364e86d1aa8SWill Deacon }
2365d8cd2006SJason Gunthorpe
2366eb054d67SJoao Martins kfree(smmu_domain);
2367e86d1aa8SWill Deacon }
2368e86d1aa8SWill Deacon
arm_smmu_domain_finalise_s1(struct arm_smmu_device * smmu,struct arm_smmu_domain * smmu_domain)2369e86d1aa8SWill Deacon static int arm_smmu_domain_finalise_s1(struct arm_smmu_device *smmu,
2370e86d1aa8SWill Deacon struct arm_smmu_domain *smmu_domain)
2371e86d1aa8SWill Deacon {
2372d8cd2006SJason Gunthorpe int ret;
237304905c17SJason Gunthorpe u32 asid = 0;
2374eb054d67SJoao Martins struct arm_smmu_ctx_desc *cd = &smmu_domain->cd;
2375e86d1aa8SWill Deacon
2376e86d1aa8SWill Deacon /* Prevent SVA from modifying the ASID until it is written to the CD */
2377e86d1aa8SWill Deacon mutex_lock(&arm_smmu_asid_lock);
2378e86d1aa8SWill Deacon ret = xa_alloc(&arm_smmu_asid_xa, &asid, smmu_domain,
2379e86d1aa8SWill Deacon XA_LIMIT(1, (1 << smmu->asid_bits) - 1), GFP_KERNEL);
2380e86d1aa8SWill Deacon cd->asid = (u16)asid;
2381e86d1aa8SWill Deacon mutex_unlock(&arm_smmu_asid_lock);
2382eb054d67SJoao Martins return ret;
2383eb054d67SJoao Martins }
2384eb054d67SJoao Martins
arm_smmu_domain_finalise_s2(struct arm_smmu_device * smmu,struct arm_smmu_domain * smmu_domain)2385eb054d67SJoao Martins static int arm_smmu_domain_finalise_s2(struct arm_smmu_device *smmu,
2386eb054d67SJoao Martins struct arm_smmu_domain *smmu_domain)
2387eb054d67SJoao Martins {
2388eb054d67SJoao Martins int vmid;
2389e86d1aa8SWill Deacon struct arm_smmu_s2_cfg *cfg = &smmu_domain->s2_cfg;
2390eb054d67SJoao Martins
2391eb054d67SJoao Martins /* Reserve VMID 0 for stage-2 bypass STEs */
2392eb054d67SJoao Martins vmid = ida_alloc_range(&smmu->vmid_map, 1, (1 << smmu->vmid_bits) - 1,
2393eb054d67SJoao Martins GFP_KERNEL);
2394eb054d67SJoao Martins if (vmid < 0)
2395eb054d67SJoao Martins return vmid;
2396eb054d67SJoao Martins
2397eb054d67SJoao Martins cfg->vmid = (u16)vmid;
2398e86d1aa8SWill Deacon return 0;
2399e86d1aa8SWill Deacon }
2400e86d1aa8SWill Deacon
arm_smmu_domain_finalise(struct arm_smmu_domain * smmu_domain,struct arm_smmu_device * smmu,u32 flags)2401eb054d67SJoao Martins static int arm_smmu_domain_finalise(struct arm_smmu_domain *smmu_domain,
2402e86d1aa8SWill Deacon struct arm_smmu_device *smmu, u32 flags)
2403eb054d67SJoao Martins {
2404eb054d67SJoao Martins int ret;
2405eb054d67SJoao Martins enum io_pgtable_fmt fmt;
2406eb054d67SJoao Martins struct io_pgtable_cfg pgtbl_cfg;
2407e86d1aa8SWill Deacon struct io_pgtable_ops *pgtbl_ops;
2408e86d1aa8SWill Deacon int (*finalise_stage_fn)(struct arm_smmu_device *smmu,
2409e86d1aa8SWill Deacon struct arm_smmu_domain *smmu_domain);
2410e86d1aa8SWill Deacon bool enable_dirty = flags & IOMMU_HWPT_ALLOC_DIRTY_TRACKING;
2411e86d1aa8SWill Deacon
2412e86d1aa8SWill Deacon /* Restrict the stage to what we can actually support */
2413e86d1aa8SWill Deacon if (!(smmu->features & ARM_SMMU_FEAT_TRANS_S1))
2414e86d1aa8SWill Deacon smmu_domain->stage = ARM_SMMU_DOMAIN_S2;
2415e86d1aa8SWill Deacon if (!(smmu->features & ARM_SMMU_FEAT_TRANS_S2))
2416e86d1aa8SWill Deacon smmu_domain->stage = ARM_SMMU_DOMAIN_S1;
2417e86d1aa8SWill Deacon
2418d8cd2006SJason Gunthorpe pgtbl_cfg = (struct io_pgtable_cfg) {
2419d8cd2006SJason Gunthorpe .pgsize_bitmap = smmu->pgsize_bitmap,
2420d8cd2006SJason Gunthorpe .coherent_walk = smmu->features & ARM_SMMU_FEAT_COHERENCY,
2421eb054d67SJoao Martins .tlb = &arm_smmu_flush_ops,
2422eb054d67SJoao Martins .iommu_dev = smmu->dev,
2423e86d1aa8SWill Deacon };
242404905c17SJason Gunthorpe
2425e86d1aa8SWill Deacon switch (smmu_domain->stage) {
2426e86d1aa8SWill Deacon case ARM_SMMU_DOMAIN_S1: {
2427e86d1aa8SWill Deacon unsigned long ias = (smmu->features &
2428e86d1aa8SWill Deacon ARM_SMMU_FEAT_VAX) ? 52 : 48;
2429e86d1aa8SWill Deacon
2430e86d1aa8SWill Deacon pgtbl_cfg.ias = min_t(unsigned long, ias, VA_BITS);
2431d8cd2006SJason Gunthorpe pgtbl_cfg.oas = smmu->ias;
2432e86d1aa8SWill Deacon if (enable_dirty)
2433e86d1aa8SWill Deacon pgtbl_cfg.quirks |= IO_PGTABLE_QUIRK_ARM_HD;
2434e86d1aa8SWill Deacon fmt = ARM_64_LPAE_S1;
243557b89048SJason Gunthorpe finalise_stage_fn = arm_smmu_domain_finalise_s1;
243657b89048SJason Gunthorpe break;
2437e86d1aa8SWill Deacon }
2438e86d1aa8SWill Deacon case ARM_SMMU_DOMAIN_S2:
2439e86d1aa8SWill Deacon if (enable_dirty)
2440e86d1aa8SWill Deacon return -EOPNOTSUPP;
244157b89048SJason Gunthorpe pgtbl_cfg.ias = smmu->ias;
2442e86d1aa8SWill Deacon pgtbl_cfg.oas = smmu->oas;
2443e86d1aa8SWill Deacon fmt = ARM_64_LPAE_S2;
244457b89048SJason Gunthorpe finalise_stage_fn = arm_smmu_domain_finalise_s2;
244557b89048SJason Gunthorpe break;
244657b89048SJason Gunthorpe default:
2447e86d1aa8SWill Deacon return -EINVAL;
2448e86d1aa8SWill Deacon }
244957b89048SJason Gunthorpe
245057b89048SJason Gunthorpe pgtbl_ops = alloc_io_pgtable_ops(fmt, &pgtbl_cfg, smmu_domain);
2451e86d1aa8SWill Deacon if (!pgtbl_ops)
2452e86d1aa8SWill Deacon return -ENOMEM;
2453e86d1aa8SWill Deacon
245465547275SJason Gunthorpe smmu_domain->domain.pgsize_bitmap = pgtbl_cfg.pgsize_bitmap;
245565547275SJason Gunthorpe smmu_domain->domain.geometry.aperture_end = (1UL << pgtbl_cfg.ias) - 1;
2456e86d1aa8SWill Deacon smmu_domain->domain.geometry.force_aperture = true;
2457e86d1aa8SWill Deacon if (enable_dirty && smmu_domain->stage == ARM_SMMU_DOMAIN_S1)
2458e86d1aa8SWill Deacon smmu_domain->domain.dirty_ops = &arm_smmu_dirty_ops;
2459e86d1aa8SWill Deacon
246085f2fb6eSJason Gunthorpe ret = finalise_stage_fn(smmu, smmu_domain);
246185f2fb6eSJason Gunthorpe if (ret < 0) {
246285f2fb6eSJason Gunthorpe free_io_pgtable_ops(pgtbl_ops);
24638ee9175cSJason Gunthorpe return ret;
24648ee9175cSJason Gunthorpe }
24658ee9175cSJason Gunthorpe
246685f2fb6eSJason Gunthorpe smmu_domain->pgtbl_ops = pgtbl_ops;
2467cdf315f9SJean-Philippe Brucker smmu_domain->smmu = smmu;
2468cdf315f9SJean-Philippe Brucker return 0;
246957b89048SJason Gunthorpe }
247057b89048SJason Gunthorpe
2471e86d1aa8SWill Deacon static struct arm_smmu_ste *
arm_smmu_get_step_for_sid(struct arm_smmu_device * smmu,u32 sid)2472e86d1aa8SWill Deacon arm_smmu_get_step_for_sid(struct arm_smmu_device *smmu, u32 sid)
2473e86d1aa8SWill Deacon {
2474cdf315f9SJean-Philippe Brucker struct arm_smmu_strtab_cfg *cfg = &smmu->strtab_cfg;
2475e86d1aa8SWill Deacon
2476e86d1aa8SWill Deacon if (smmu->features & ARM_SMMU_FEAT_2_LVL_STRTAB) {
2477e86d1aa8SWill Deacon /* Two-level walk */
2478e86d1aa8SWill Deacon return &cfg->l2.l2ptrs[arm_smmu_strtab_l1_idx(sid)]
247965547275SJason Gunthorpe ->stes[arm_smmu_strtab_l2_idx(sid)];
2480e86d1aa8SWill Deacon } else {
2481e86d1aa8SWill Deacon /* Simple linear lookup */
2482e86d1aa8SWill Deacon return &cfg->linear.table[sid];
2483e86d1aa8SWill Deacon }
2484e86d1aa8SWill Deacon }
2485e86d1aa8SWill Deacon
arm_smmu_install_ste_for_dev(struct arm_smmu_master * master,const struct arm_smmu_ste * target)2486e86d1aa8SWill Deacon static void arm_smmu_install_ste_for_dev(struct arm_smmu_master *master,
2487e86d1aa8SWill Deacon const struct arm_smmu_ste *target)
2488e86d1aa8SWill Deacon {
2489e86d1aa8SWill Deacon int i, j;
2490e86d1aa8SWill Deacon struct arm_smmu_device *smmu = master->smmu;
2491e86d1aa8SWill Deacon
2492e86d1aa8SWill Deacon master->cd_table.in_ste =
2493e86d1aa8SWill Deacon FIELD_GET(STRTAB_STE_0_CFG, le64_to_cpu(target->data[0])) ==
2494e86d1aa8SWill Deacon STRTAB_STE_0_CFG_S1_TRANS;
2495e86d1aa8SWill Deacon master->ste_ats_enabled =
2496e86d1aa8SWill Deacon FIELD_GET(STRTAB_STE_1_EATS, le64_to_cpu(target->data[1])) ==
2497e86d1aa8SWill Deacon STRTAB_STE_1_EATS_TRANS;
24987497f421SJason Gunthorpe
2499e86d1aa8SWill Deacon for (i = 0; i < master->num_streams; ++i) {
2500e86d1aa8SWill Deacon u32 sid = master->streams[i].id;
2501e86d1aa8SWill Deacon struct arm_smmu_ste *step =
2502e86d1aa8SWill Deacon arm_smmu_get_step_for_sid(smmu, sid);
2503e86d1aa8SWill Deacon
2504e86d1aa8SWill Deacon /* Bridged PCI devices may end up with duplicated IDs */
2505e86d1aa8SWill Deacon for (j = 0; j < i; j++)
2506e86d1aa8SWill Deacon if (master->streams[j].id == sid)
2507e86d1aa8SWill Deacon break;
250886e5ca09SJason Gunthorpe if (j < i)
250986e5ca09SJason Gunthorpe continue;
251086e5ca09SJason Gunthorpe
25111d5f34f0SJason Gunthorpe arm_smmu_write_ste(master, sid, step, target);
2512e86d1aa8SWill Deacon }
2513e86d1aa8SWill Deacon }
2514e86d1aa8SWill Deacon
arm_smmu_ats_supported(struct arm_smmu_master * master)2515e86d1aa8SWill Deacon static bool arm_smmu_ats_supported(struct arm_smmu_master *master)
2516e86d1aa8SWill Deacon {
2517e86d1aa8SWill Deacon struct device *dev = master->dev;
2518e86d1aa8SWill Deacon struct arm_smmu_device *smmu = master->smmu;
2519e86d1aa8SWill Deacon struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
2520e86d1aa8SWill Deacon
2521e86d1aa8SWill Deacon if (!(smmu->features & ARM_SMMU_FEAT_ATS))
2522e86d1aa8SWill Deacon return false;
2523e86d1aa8SWill Deacon
2524e86d1aa8SWill Deacon if (!(fwspec->flags & IOMMU_FWSPEC_PCI_RC_ATS))
2525e86d1aa8SWill Deacon return false;
2526e86d1aa8SWill Deacon
2527e86d1aa8SWill Deacon return dev_is_pci(dev) && pci_ats_supported(to_pci_dev(dev));
2528e86d1aa8SWill Deacon }
2529e86d1aa8SWill Deacon
arm_smmu_enable_ats(struct arm_smmu_master * master)2530e86d1aa8SWill Deacon static void arm_smmu_enable_ats(struct arm_smmu_master *master)
2531e86d1aa8SWill Deacon {
2532e86d1aa8SWill Deacon size_t stu;
2533e86d1aa8SWill Deacon struct pci_dev *pdev;
2534e86d1aa8SWill Deacon struct arm_smmu_device *smmu = master->smmu;
2535e86d1aa8SWill Deacon
2536e86d1aa8SWill Deacon /* Smallest Translation Unit: log2 of the smallest supported granule */
2537e86d1aa8SWill Deacon stu = __ffs(smmu->pgsize_bitmap);
2538e86d1aa8SWill Deacon pdev = to_pci_dev(master->dev);
2539e86d1aa8SWill Deacon
2540e86d1aa8SWill Deacon /*
2541e86d1aa8SWill Deacon * ATC invalidation of PASID 0 causes the entire ATC to be flushed.
2542e86d1aa8SWill Deacon */
2543e86d1aa8SWill Deacon arm_smmu_atc_inv_master(master, IOMMU_NO_PASID);
2544e86d1aa8SWill Deacon if (pci_enable_ats(pdev, stu))
2545e86d1aa8SWill Deacon dev_err(master->dev, "Failed to enable ATS (STU %zu)\n", stu);
2546e86d1aa8SWill Deacon }
2547e86d1aa8SWill Deacon
arm_smmu_enable_pasid(struct arm_smmu_master * master)2548e86d1aa8SWill Deacon static int arm_smmu_enable_pasid(struct arm_smmu_master *master)
2549e86d1aa8SWill Deacon {
2550e86d1aa8SWill Deacon int ret;
2551e86d1aa8SWill Deacon int features;
2552e86d1aa8SWill Deacon int num_pasids;
2553e86d1aa8SWill Deacon struct pci_dev *pdev;
2554e86d1aa8SWill Deacon
2555e86d1aa8SWill Deacon if (!dev_is_pci(master->dev))
2556e86d1aa8SWill Deacon return -ENODEV;
2557e86d1aa8SWill Deacon
2558e86d1aa8SWill Deacon pdev = to_pci_dev(master->dev);
2559e86d1aa8SWill Deacon
2560e86d1aa8SWill Deacon features = pci_pasid_features(pdev);
2561e86d1aa8SWill Deacon if (features < 0)
2562e86d1aa8SWill Deacon return features;
2563ad10dce6SJason Gunthorpe
2564ad10dce6SJason Gunthorpe num_pasids = pci_max_pasids(pdev);
256564efb3deSJason Gunthorpe if (num_pasids <= 0)
256664efb3deSJason Gunthorpe return num_pasids;
2567ad10dce6SJason Gunthorpe
2568ad10dce6SJason Gunthorpe ret = pci_enable_pasid(pdev, features);
2569ad10dce6SJason Gunthorpe if (ret) {
2570ad10dce6SJason Gunthorpe dev_err(&pdev->dev, "Failed to enable PASID\n");
2571ad10dce6SJason Gunthorpe return ret;
2572ad10dce6SJason Gunthorpe }
2573ad10dce6SJason Gunthorpe
257464efb3deSJason Gunthorpe master->ssid_bits = min_t(u8, ilog2(num_pasids),
257564efb3deSJason Gunthorpe master->smmu->ssid_bits);
2576ad10dce6SJason Gunthorpe return 0;
2577ad10dce6SJason Gunthorpe }
2578ad10dce6SJason Gunthorpe
arm_smmu_disable_pasid(struct arm_smmu_master * master)2579ad10dce6SJason Gunthorpe static void arm_smmu_disable_pasid(struct arm_smmu_master *master)
2580ad10dce6SJason Gunthorpe {
25817497f421SJason Gunthorpe struct pci_dev *pdev;
25827497f421SJason Gunthorpe
25837497f421SJason Gunthorpe if (!dev_is_pci(master->dev))
25847497f421SJason Gunthorpe return;
25857497f421SJason Gunthorpe
25867497f421SJason Gunthorpe pdev = to_pci_dev(master->dev);
25877497f421SJason Gunthorpe
2588e86d1aa8SWill Deacon if (!pdev->pasid_enabled)
25897497f421SJason Gunthorpe return;
25907497f421SJason Gunthorpe
25917497f421SJason Gunthorpe master->ssid_bits = 0;
259249db2ed2SJason Gunthorpe pci_disable_pasid(pdev);
259349db2ed2SJason Gunthorpe }
25947497f421SJason Gunthorpe
25957497f421SJason Gunthorpe static struct arm_smmu_master_domain *
arm_smmu_find_master_domain(struct arm_smmu_domain * smmu_domain,struct arm_smmu_master * master,ioasid_t ssid)25967497f421SJason Gunthorpe arm_smmu_find_master_domain(struct arm_smmu_domain *smmu_domain,
25977497f421SJason Gunthorpe struct arm_smmu_master *master,
25987497f421SJason Gunthorpe ioasid_t ssid)
25991d5f34f0SJason Gunthorpe {
26001d5f34f0SJason Gunthorpe struct arm_smmu_master_domain *master_domain;
26017497f421SJason Gunthorpe
26027497f421SJason Gunthorpe lockdep_assert_held(&smmu_domain->devices_lock);
2603ad10dce6SJason Gunthorpe
2604e86d1aa8SWill Deacon list_for_each_entry(master_domain, &smmu_domain->devices,
2605e86d1aa8SWill Deacon devices_elm) {
26067497f421SJason Gunthorpe if (master_domain->master == master &&
2607e86d1aa8SWill Deacon master_domain->ssid == ssid)
2608e86d1aa8SWill Deacon return master_domain;
2609e86d1aa8SWill Deacon }
26101d5f34f0SJason Gunthorpe return NULL;
2611ad10dce6SJason Gunthorpe }
2612ad10dce6SJason Gunthorpe
2613ad10dce6SJason Gunthorpe /*
26147497f421SJason Gunthorpe * If the domain uses the smmu_domain->devices list return the arm_smmu_domain
26157497f421SJason Gunthorpe * structure, otherwise NULL. These domains track attached devices so they can
2616ad10dce6SJason Gunthorpe * issue invalidations.
2617e86d1aa8SWill Deacon */
26187497f421SJason Gunthorpe static struct arm_smmu_domain *
to_smmu_domain_devices(struct iommu_domain * domain)2619e86d1aa8SWill Deacon to_smmu_domain_devices(struct iommu_domain *domain)
26207497f421SJason Gunthorpe {
26217497f421SJason Gunthorpe /* The domain can be NULL only when processing the first attach */
26227497f421SJason Gunthorpe if (!domain)
26237497f421SJason Gunthorpe return NULL;
2624ce26ea9eSJason Gunthorpe if ((domain->type & __IOMMU_DOMAIN_PAGING) ||
26251d5f34f0SJason Gunthorpe domain->type == IOMMU_DOMAIN_SVA)
26267497f421SJason Gunthorpe return to_smmu_domain(domain);
26277497f421SJason Gunthorpe return NULL;
26287497f421SJason Gunthorpe }
26297497f421SJason Gunthorpe
arm_smmu_remove_master_domain(struct arm_smmu_master * master,struct iommu_domain * domain,ioasid_t ssid)26307497f421SJason Gunthorpe static void arm_smmu_remove_master_domain(struct arm_smmu_master *master,
26317497f421SJason Gunthorpe struct iommu_domain *domain,
26327497f421SJason Gunthorpe ioasid_t ssid)
26337497f421SJason Gunthorpe {
26347497f421SJason Gunthorpe struct arm_smmu_domain *smmu_domain = to_smmu_domain_devices(domain);
26357497f421SJason Gunthorpe struct arm_smmu_master_domain *master_domain;
26367497f421SJason Gunthorpe unsigned long flags;
26377497f421SJason Gunthorpe
26387497f421SJason Gunthorpe if (!smmu_domain)
26397497f421SJason Gunthorpe return;
26407497f421SJason Gunthorpe
26417497f421SJason Gunthorpe spin_lock_irqsave(&smmu_domain->devices_lock, flags);
26427497f421SJason Gunthorpe master_domain = arm_smmu_find_master_domain(smmu_domain, master, ssid);
26437497f421SJason Gunthorpe if (master_domain) {
26447497f421SJason Gunthorpe list_del(&master_domain->devices_elm);
26457497f421SJason Gunthorpe kfree(master_domain);
26467497f421SJason Gunthorpe if (master->ats_enabled)
26477497f421SJason Gunthorpe atomic_dec(&smmu_domain->nr_ats_masters);
26487497f421SJason Gunthorpe }
26497497f421SJason Gunthorpe spin_unlock_irqrestore(&smmu_domain->devices_lock, flags);
26507497f421SJason Gunthorpe }
26517497f421SJason Gunthorpe
26527497f421SJason Gunthorpe struct arm_smmu_attach_state {
26537497f421SJason Gunthorpe /* Inputs */
26547497f421SJason Gunthorpe struct iommu_domain *old_domain;
26557497f421SJason Gunthorpe struct arm_smmu_master *master;
26567497f421SJason Gunthorpe bool cd_needs_ats;
26577497f421SJason Gunthorpe ioasid_t ssid;
26587497f421SJason Gunthorpe /* Resulting state */
26597497f421SJason Gunthorpe bool ats_enabled;
26607497f421SJason Gunthorpe };
26617497f421SJason Gunthorpe
26627497f421SJason Gunthorpe /*
26637497f421SJason Gunthorpe * Start the sequence to attach a domain to a master. The sequence contains three
26647497f421SJason Gunthorpe * steps:
26657497f421SJason Gunthorpe * arm_smmu_attach_prepare()
2666ce26ea9eSJason Gunthorpe * arm_smmu_install_ste_for_dev()
26677497f421SJason Gunthorpe * arm_smmu_attach_commit()
26687497f421SJason Gunthorpe *
26697497f421SJason Gunthorpe * If prepare succeeds then the sequence must be completed. The STE installed
26707497f421SJason Gunthorpe * must set the STE.EATS field according to state.ats_enabled.
26717497f421SJason Gunthorpe *
26727497f421SJason Gunthorpe * If the device supports ATS then this determines if EATS should be enabled
26737497f421SJason Gunthorpe * in the STE, and starts sequencing EATS disable if required.
26747497f421SJason Gunthorpe *
26757497f421SJason Gunthorpe * The change of the EATS in the STE and the PCI ATS config space is managed by
26767497f421SJason Gunthorpe * this sequence to be in the right order so that if PCI ATS is enabled then
26777497f421SJason Gunthorpe * STE.ETAS is enabled.
2678ce26ea9eSJason Gunthorpe *
26797497f421SJason Gunthorpe * new_domain can be a non-paging domain. In this case ATS will not be enabled,
2680ce26ea9eSJason Gunthorpe * and invalidations won't be tracked.
26817497f421SJason Gunthorpe */
arm_smmu_attach_prepare(struct arm_smmu_attach_state * state,struct iommu_domain * new_domain)26827497f421SJason Gunthorpe static int arm_smmu_attach_prepare(struct arm_smmu_attach_state *state,
26837497f421SJason Gunthorpe struct iommu_domain *new_domain)
26847497f421SJason Gunthorpe {
26851d5f34f0SJason Gunthorpe struct arm_smmu_master *master = state->master;
26867497f421SJason Gunthorpe struct arm_smmu_master_domain *master_domain;
26877497f421SJason Gunthorpe struct arm_smmu_domain *smmu_domain =
26887497f421SJason Gunthorpe to_smmu_domain_devices(new_domain);
26897497f421SJason Gunthorpe unsigned long flags;
26907497f421SJason Gunthorpe
26917497f421SJason Gunthorpe /*
26927497f421SJason Gunthorpe * arm_smmu_share_asid() must not see two domains pointing to the same
26937497f421SJason Gunthorpe * arm_smmu_master_domain contents otherwise it could randomly write one
26947497f421SJason Gunthorpe * or the other to the CD.
26957497f421SJason Gunthorpe */
26967497f421SJason Gunthorpe lockdep_assert_held(&arm_smmu_asid_lock);
26977497f421SJason Gunthorpe
26987497f421SJason Gunthorpe if (smmu_domain || state->cd_needs_ats) {
26997497f421SJason Gunthorpe /*
27007497f421SJason Gunthorpe * The SMMU does not support enabling ATS with bypass/abort.
27017497f421SJason Gunthorpe * When the STE is in bypass (STE.Config[2:0] == 0b100), ATS
27027497f421SJason Gunthorpe * Translation Requests and Translated transactions are denied
27037497f421SJason Gunthorpe * as though ATS is disabled for the stream (STE.EATS == 0b00),
27047497f421SJason Gunthorpe * causing F_BAD_ATS_TREQ and F_TRANSL_FORBIDDEN events
27057497f421SJason Gunthorpe * (IHI0070Ea 5.2 Stream Table Entry). Thus ATS can only be
27067497f421SJason Gunthorpe * enabled if we have arm_smmu_domain, those always have page
27077497f421SJason Gunthorpe * tables.
27087497f421SJason Gunthorpe */
27097497f421SJason Gunthorpe state->ats_enabled = arm_smmu_ats_supported(master);
27107497f421SJason Gunthorpe }
27117497f421SJason Gunthorpe
27127497f421SJason Gunthorpe if (smmu_domain) {
27137497f421SJason Gunthorpe master_domain = kzalloc(sizeof(*master_domain), GFP_KERNEL);
27147497f421SJason Gunthorpe if (!master_domain)
27157497f421SJason Gunthorpe return -ENOMEM;
27167497f421SJason Gunthorpe master_domain->master = master;
27177497f421SJason Gunthorpe master_domain->ssid = state->ssid;
27187497f421SJason Gunthorpe
27197497f421SJason Gunthorpe /*
27207497f421SJason Gunthorpe * During prepare we want the current smmu_domain and new
27217497f421SJason Gunthorpe * smmu_domain to be in the devices list before we change any
27227497f421SJason Gunthorpe * HW. This ensures that both domains will send ATS
27237497f421SJason Gunthorpe * invalidations to the master until we are done.
27247497f421SJason Gunthorpe *
27257497f421SJason Gunthorpe * It is tempting to make this list only track masters that are
27267497f421SJason Gunthorpe * using ATS, but arm_smmu_share_asid() also uses this to change
27277497f421SJason Gunthorpe * the ASID of a domain, unrelated to ATS.
27287497f421SJason Gunthorpe *
27297497f421SJason Gunthorpe * Notice if we are re-attaching the same domain then the list
27307497f421SJason Gunthorpe * will have two identical entries and commit will remove only
27317497f421SJason Gunthorpe * one of them.
27327497f421SJason Gunthorpe */
27331d5f34f0SJason Gunthorpe spin_lock_irqsave(&smmu_domain->devices_lock, flags);
27347497f421SJason Gunthorpe if (state->ats_enabled)
27357497f421SJason Gunthorpe atomic_inc(&smmu_domain->nr_ats_masters);
27367497f421SJason Gunthorpe list_add(&master_domain->devices_elm, &smmu_domain->devices);
27377497f421SJason Gunthorpe spin_unlock_irqrestore(&smmu_domain->devices_lock, flags);
27387497f421SJason Gunthorpe }
27391d5f34f0SJason Gunthorpe
27401d5f34f0SJason Gunthorpe if (!state->ats_enabled && master->ats_enabled) {
27411d5f34f0SJason Gunthorpe pci_disable_ats(to_pci_dev(master->dev));
27421d5f34f0SJason Gunthorpe /*
27437497f421SJason Gunthorpe * This is probably overkill, but the config write for disabling
27447497f421SJason Gunthorpe * ATS should complete before the STE is configured to generate
27457497f421SJason Gunthorpe * UR to avoid AER noise.
27461d5f34f0SJason Gunthorpe */
2747e86d1aa8SWill Deacon wmb();
2748e86d1aa8SWill Deacon }
2749e86d1aa8SWill Deacon return 0;
2750e86d1aa8SWill Deacon }
2751e86d1aa8SWill Deacon
275265547275SJason Gunthorpe /*
2753e86d1aa8SWill Deacon * Commit is done after the STE/CD are configured with the EATS setting. It
2754e86d1aa8SWill Deacon * completes synchronizing the PCI device's ATC and finishes manipulating the
2755e86d1aa8SWill Deacon * smmu_domain->devices list.
27567497f421SJason Gunthorpe */
arm_smmu_attach_commit(struct arm_smmu_attach_state * state)27577497f421SJason Gunthorpe static void arm_smmu_attach_commit(struct arm_smmu_attach_state *state)
27581d5f34f0SJason Gunthorpe {
27597497f421SJason Gunthorpe struct arm_smmu_master *master = state->master;
2760e86d1aa8SWill Deacon
276113abe4faSJason Gunthorpe lockdep_assert_held(&arm_smmu_asid_lock);
2762e86d1aa8SWill Deacon
2763e86d1aa8SWill Deacon if (state->ats_enabled && !master->ats_enabled) {
2764e86d1aa8SWill Deacon arm_smmu_enable_ats(master);
2765e86d1aa8SWill Deacon } else if (state->ats_enabled && master->ats_enabled) {
27667497f421SJason Gunthorpe /*
2767e86d1aa8SWill Deacon * The translation has changed, flush the ATC. At this point the
2768e86d1aa8SWill Deacon * SMMU is translating for the new domain and both the old&new
2769e86d1aa8SWill Deacon * domain will issue invalidations.
2770e86d1aa8SWill Deacon */
2771e86d1aa8SWill Deacon arm_smmu_atc_inv_master(master, state->ssid);
2772eb054d67SJoao Martins } else if (!state->ats_enabled && master->ats_enabled) {
277310e4968cSMichael Shavit /* ATS is being switched off, invalidate the entire ATC */
2774f4a14773SNicolin Chen arm_smmu_atc_inv_master(master, IOMMU_NO_PASID);
277510e4968cSMichael Shavit }
277610e4968cSMichael Shavit master->ats_enabled = state->ats_enabled;
277710e4968cSMichael Shavit
277810e4968cSMichael Shavit arm_smmu_remove_master_domain(master, state->old_domain, state->ssid);
2779e86d1aa8SWill Deacon }
278013abe4faSJason Gunthorpe
arm_smmu_attach_dev(struct iommu_domain * domain,struct device * dev)278113abe4faSJason Gunthorpe static int arm_smmu_attach_dev(struct iommu_domain *domain, struct device *dev)
278213abe4faSJason Gunthorpe {
278313abe4faSJason Gunthorpe int ret = 0;
2784be7c90deSJason Gunthorpe struct arm_smmu_ste target;
2785be7c90deSJason Gunthorpe struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
278613abe4faSJason Gunthorpe struct arm_smmu_device *smmu;
27878c153645SJean-Philippe Brucker struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
27889f7c6891SJason Gunthorpe struct arm_smmu_attach_state state = {
27899f7c6891SJason Gunthorpe .old_domain = iommu_get_domain_for_dev(dev),
27909f7c6891SJason Gunthorpe .ssid = IOMMU_NO_PASID,
27919f7c6891SJason Gunthorpe };
27928c153645SJean-Philippe Brucker struct arm_smmu_master *master;
27939f7c6891SJason Gunthorpe struct arm_smmu_cd *cdptr;
27949f7c6891SJason Gunthorpe
27957497f421SJason Gunthorpe if (!fwspec)
27967497f421SJason Gunthorpe return -ENOENT;
27977497f421SJason Gunthorpe
27987497f421SJason Gunthorpe state.master = master = dev_iommu_priv_get(dev);
27997497f421SJason Gunthorpe smmu = master->smmu;
2800e86d1aa8SWill Deacon
280165547275SJason Gunthorpe mutex_lock(&smmu_domain->init_mutex);
2802e9d1e4ffSJason Gunthorpe
2803e9d1e4ffSJason Gunthorpe if (!smmu_domain->smmu) {
2804e86d1aa8SWill Deacon ret = arm_smmu_domain_finalise(smmu_domain, smmu, 0);
2805e9d1e4ffSJason Gunthorpe } else if (smmu_domain->smmu != smmu)
2806e9d1e4ffSJason Gunthorpe ret = -EINVAL;
2807e9d1e4ffSJason Gunthorpe
2808ce26ea9eSJason Gunthorpe mutex_unlock(&smmu_domain->init_mutex);
2809ce26ea9eSJason Gunthorpe if (ret)
2810d2e053d7SJason Gunthorpe return ret;
281165547275SJason Gunthorpe
2812e9d1e4ffSJason Gunthorpe if (smmu_domain->stage == ARM_SMMU_DOMAIN_S1) {
281365547275SJason Gunthorpe cdptr = arm_smmu_alloc_cd_ptr(master, IOMMU_NO_PASID);
28147497f421SJason Gunthorpe if (!cdptr)
28157497f421SJason Gunthorpe return -ENOMEM;
2816d2e053d7SJason Gunthorpe } else if (arm_smmu_ssids_in_use(&master->cd_table))
2817af8f0b83SJason Gunthorpe return -EBUSY;
281865547275SJason Gunthorpe
281910e4968cSMichael Shavit /*
282010e4968cSMichael Shavit * Prevent arm_smmu_share_asid() from trying to change the ASID
28217497f421SJason Gunthorpe * of either the old or new domain while we are working on it.
28229f7c6891SJason Gunthorpe * This allows the STE and the smmu_domain->devices list to
282313abe4faSJason Gunthorpe * be inconsistent during this routine.
2824e86d1aa8SWill Deacon */
2825e86d1aa8SWill Deacon mutex_lock(&arm_smmu_asid_lock);
2826f3b273b7SJason Gunthorpe
2827f3b273b7SJason Gunthorpe ret = arm_smmu_attach_prepare(&state, domain);
2828f3b273b7SJason Gunthorpe if (ret) {
2829f3b273b7SJason Gunthorpe mutex_unlock(&arm_smmu_asid_lock);
2830f3b273b7SJason Gunthorpe return ret;
2831f3b273b7SJason Gunthorpe }
2832f3b273b7SJason Gunthorpe
2833f3b273b7SJason Gunthorpe switch (smmu_domain->stage) {
2834f3b273b7SJason Gunthorpe case ARM_SMMU_DOMAIN_S1: {
2835f3b273b7SJason Gunthorpe struct arm_smmu_cd target_cd;
2836f3b273b7SJason Gunthorpe
2837eb054d67SJoao Martins arm_smmu_make_s1_cd(&target_cd, master, smmu_domain);
2838f3b273b7SJason Gunthorpe arm_smmu_write_cd_entry(master, IOMMU_NO_PASID, cdptr,
2839f3b273b7SJason Gunthorpe &target_cd);
2840f3b273b7SJason Gunthorpe arm_smmu_make_cdtable_ste(&target, master, state.ats_enabled,
2841f3b273b7SJason Gunthorpe STRTAB_STE_1_S1DSS_SSID0);
2842f3b273b7SJason Gunthorpe arm_smmu_install_ste_for_dev(master, &target);
2843f3b273b7SJason Gunthorpe break;
2844f3b273b7SJason Gunthorpe }
2845f3b273b7SJason Gunthorpe case ARM_SMMU_DOMAIN_S2:
2846f3b273b7SJason Gunthorpe arm_smmu_make_s2_domain_ste(&target, master, smmu_domain,
2847f3b273b7SJason Gunthorpe state.ats_enabled);
2848f3b273b7SJason Gunthorpe arm_smmu_install_ste_for_dev(master, &target);
2849f3b273b7SJason Gunthorpe arm_smmu_clear_cd(master, IOMMU_NO_PASID);
2850f3b273b7SJason Gunthorpe break;
2851f3b273b7SJason Gunthorpe }
2852f3b273b7SJason Gunthorpe
2853f3b273b7SJason Gunthorpe arm_smmu_attach_commit(&state);
2854f3b273b7SJason Gunthorpe mutex_unlock(&arm_smmu_asid_lock);
2855f3b273b7SJason Gunthorpe return 0;
28568ee9175cSJason Gunthorpe }
28578ee9175cSJason Gunthorpe
arm_smmu_s1_set_dev_pasid(struct iommu_domain * domain,struct device * dev,ioasid_t id)28588ee9175cSJason Gunthorpe static int arm_smmu_s1_set_dev_pasid(struct iommu_domain *domain,
28598ee9175cSJason Gunthorpe struct device *dev, ioasid_t id)
28608ee9175cSJason Gunthorpe {
28618ee9175cSJason Gunthorpe struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
28628ee9175cSJason Gunthorpe struct arm_smmu_master *master = dev_iommu_priv_get(dev);
28638ee9175cSJason Gunthorpe struct arm_smmu_device *smmu = master->smmu;
28648ee9175cSJason Gunthorpe struct arm_smmu_cd target_cd;
28658ee9175cSJason Gunthorpe int ret = 0;
28668ee9175cSJason Gunthorpe
28678ee9175cSJason Gunthorpe mutex_lock(&smmu_domain->init_mutex);
28688ee9175cSJason Gunthorpe if (!smmu_domain->smmu)
28698ee9175cSJason Gunthorpe ret = arm_smmu_domain_finalise(smmu_domain, smmu, 0);
28708ee9175cSJason Gunthorpe else if (smmu_domain->smmu != smmu)
28718ee9175cSJason Gunthorpe ret = -EINVAL;
28728ee9175cSJason Gunthorpe mutex_unlock(&smmu_domain->init_mutex);
28738ee9175cSJason Gunthorpe if (ret)
28748ee9175cSJason Gunthorpe return ret;
28758ee9175cSJason Gunthorpe
28768ee9175cSJason Gunthorpe if (smmu_domain->stage != ARM_SMMU_DOMAIN_S1)
28778ee9175cSJason Gunthorpe return -EINVAL;
28788ee9175cSJason Gunthorpe
28798ee9175cSJason Gunthorpe /*
28808ee9175cSJason Gunthorpe * We can read cd.asid outside the lock because arm_smmu_set_pasid()
288185f2fb6eSJason Gunthorpe * will fix it
288285f2fb6eSJason Gunthorpe */
2883f3b273b7SJason Gunthorpe arm_smmu_make_s1_cd(&target_cd, master, smmu_domain);
288485f2fb6eSJason Gunthorpe return arm_smmu_set_pasid(master, to_smmu_domain(domain), id,
28858ee9175cSJason Gunthorpe &target_cd);
288649db2ed2SJason Gunthorpe }
288749db2ed2SJason Gunthorpe
arm_smmu_update_ste(struct arm_smmu_master * master,struct iommu_domain * sid_domain,bool ats_enabled)288849db2ed2SJason Gunthorpe static void arm_smmu_update_ste(struct arm_smmu_master *master,
288949db2ed2SJason Gunthorpe struct iommu_domain *sid_domain,
289049db2ed2SJason Gunthorpe bool ats_enabled)
289149db2ed2SJason Gunthorpe {
289249db2ed2SJason Gunthorpe unsigned int s1dss = STRTAB_STE_1_S1DSS_TERMINATE;
289349db2ed2SJason Gunthorpe struct arm_smmu_ste ste;
289485f2fb6eSJason Gunthorpe
289549db2ed2SJason Gunthorpe if (master->cd_table.in_ste && master->ste_ats_enabled == ats_enabled)
289685f2fb6eSJason Gunthorpe return;
289785f2fb6eSJason Gunthorpe
289885f2fb6eSJason Gunthorpe if (sid_domain->type == IOMMU_DOMAIN_IDENTITY)
2899d38c28dbSJason Gunthorpe s1dss = STRTAB_STE_1_S1DSS_BYPASS;
2900d38c28dbSJason Gunthorpe else
2901d38c28dbSJason Gunthorpe WARN_ON(sid_domain->type != IOMMU_DOMAIN_BLOCKED);
29028ee9175cSJason Gunthorpe
29038ee9175cSJason Gunthorpe /*
29048ee9175cSJason Gunthorpe * Change the STE into a cdtable one with SID IDENTITY/BLOCKED behavior
29058ee9175cSJason Gunthorpe * using s1dss if necessary. If the cd_table is already installed then
290685f2fb6eSJason Gunthorpe * the S1DSS is correct and this will just update the EATS. Otherwise it
290785f2fb6eSJason Gunthorpe * installs the entire thing. This will be hitless.
290885f2fb6eSJason Gunthorpe */
290985f2fb6eSJason Gunthorpe arm_smmu_make_cdtable_ste(&ste, master, ats_enabled, s1dss);
291049db2ed2SJason Gunthorpe arm_smmu_install_ste_for_dev(master, &ste);
291149db2ed2SJason Gunthorpe }
291249db2ed2SJason Gunthorpe
arm_smmu_set_pasid(struct arm_smmu_master * master,struct arm_smmu_domain * smmu_domain,ioasid_t pasid,struct arm_smmu_cd * cd)291349db2ed2SJason Gunthorpe int arm_smmu_set_pasid(struct arm_smmu_master *master,
291449db2ed2SJason Gunthorpe struct arm_smmu_domain *smmu_domain, ioasid_t pasid,
291549db2ed2SJason Gunthorpe struct arm_smmu_cd *cd)
2916f3b273b7SJason Gunthorpe {
2917f3b273b7SJason Gunthorpe struct iommu_domain *sid_domain = iommu_get_domain_for_dev(master->dev);
2918f3b273b7SJason Gunthorpe struct arm_smmu_attach_state state = {
2919f3b273b7SJason Gunthorpe .master = master,
2920f3b273b7SJason Gunthorpe /*
2921f3b273b7SJason Gunthorpe * For now the core code prevents calling this when a domain is
2922f3b273b7SJason Gunthorpe * already attached, no need to set old_domain.
2923f3b273b7SJason Gunthorpe */
292485f2fb6eSJason Gunthorpe .ssid = pasid,
29258ee9175cSJason Gunthorpe };
292649db2ed2SJason Gunthorpe struct arm_smmu_cd *cdptr;
292749db2ed2SJason Gunthorpe int ret;
292849db2ed2SJason Gunthorpe
292949db2ed2SJason Gunthorpe /* The core code validates pasid */
293049db2ed2SJason Gunthorpe
293149db2ed2SJason Gunthorpe if (smmu_domain->smmu != master->smmu)
293285f2fb6eSJason Gunthorpe return -EINVAL;
293385f2fb6eSJason Gunthorpe
2934d38c28dbSJason Gunthorpe if (!master->cd_table.in_ste &&
2935d38c28dbSJason Gunthorpe sid_domain->type != IOMMU_DOMAIN_IDENTITY &&
293685f2fb6eSJason Gunthorpe sid_domain->type != IOMMU_DOMAIN_BLOCKED)
2937d38c28dbSJason Gunthorpe return -EINVAL;
2938d38c28dbSJason Gunthorpe
2939d38c28dbSJason Gunthorpe cdptr = arm_smmu_alloc_cd_ptr(master, pasid);
2940d38c28dbSJason Gunthorpe if (!cdptr)
2941d38c28dbSJason Gunthorpe return -ENOMEM;
294249db2ed2SJason Gunthorpe
294385f2fb6eSJason Gunthorpe mutex_lock(&arm_smmu_asid_lock);
294449db2ed2SJason Gunthorpe ret = arm_smmu_attach_prepare(&state, &smmu_domain->domain);
294549db2ed2SJason Gunthorpe if (ret)
294649db2ed2SJason Gunthorpe goto out_unlock;
294749db2ed2SJason Gunthorpe
29488ee9175cSJason Gunthorpe /*
29498ee9175cSJason Gunthorpe * We don't want to obtain to the asid_lock too early, so fix up the
29508ee9175cSJason Gunthorpe * caller set ASID under the lock in case it changed.
29518ee9175cSJason Gunthorpe */
29528ee9175cSJason Gunthorpe cd->data[0] &= ~cpu_to_le64(CTXDESC_CD_0_ASID);
29538ee9175cSJason Gunthorpe cd->data[0] |= cpu_to_le64(
29548ee9175cSJason Gunthorpe FIELD_PREP(CTXDESC_CD_0_ASID, smmu_domain->cd.asid));
29558ee9175cSJason Gunthorpe
29568ee9175cSJason Gunthorpe arm_smmu_write_cd_entry(master, pasid, cdptr, cd);
29578ee9175cSJason Gunthorpe arm_smmu_update_ste(master, sid_domain, state.ats_enabled);
29588ee9175cSJason Gunthorpe
29598ee9175cSJason Gunthorpe arm_smmu_attach_commit(&state);
29608ee9175cSJason Gunthorpe
296185f2fb6eSJason Gunthorpe out_unlock:
296285f2fb6eSJason Gunthorpe mutex_unlock(&arm_smmu_asid_lock);
2963ce26ea9eSJason Gunthorpe return ret;
2964ce26ea9eSJason Gunthorpe }
2965ce26ea9eSJason Gunthorpe
arm_smmu_remove_dev_pasid(struct device * dev,ioasid_t pasid,struct iommu_domain * domain)2966ce26ea9eSJason Gunthorpe static void arm_smmu_remove_dev_pasid(struct device *dev, ioasid_t pasid,
296712dacfb5SJason Gunthorpe struct iommu_domain *domain)
296812dacfb5SJason Gunthorpe {
29697497f421SJason Gunthorpe struct arm_smmu_master *master = dev_iommu_priv_get(dev);
29707497f421SJason Gunthorpe struct arm_smmu_domain *smmu_domain;
29717497f421SJason Gunthorpe
29721d5f34f0SJason Gunthorpe smmu_domain = to_smmu_domain(domain);
29737497f421SJason Gunthorpe
297412dacfb5SJason Gunthorpe mutex_lock(&arm_smmu_asid_lock);
297512dacfb5SJason Gunthorpe arm_smmu_clear_cd(master, pasid);
297612dacfb5SJason Gunthorpe if (master->ats_enabled)
297712dacfb5SJason Gunthorpe arm_smmu_atc_inv_master(master, pasid);
297812dacfb5SJason Gunthorpe arm_smmu_remove_master_domain(master, &smmu_domain->domain, pasid);
297912dacfb5SJason Gunthorpe mutex_unlock(&arm_smmu_asid_lock);
298012dacfb5SJason Gunthorpe
2981ce26ea9eSJason Gunthorpe /*
2982ce26ea9eSJason Gunthorpe * When the last user of the CD table goes away downgrade the STE back
2983ce26ea9eSJason Gunthorpe * to a non-cd_table one.
2984ce26ea9eSJason Gunthorpe */
2985ce26ea9eSJason Gunthorpe if (!arm_smmu_ssids_in_use(&master->cd_table)) {
2986ce26ea9eSJason Gunthorpe struct iommu_domain *sid_domain =
2987ce26ea9eSJason Gunthorpe iommu_get_domain_for_dev(master->dev);
2988ce26ea9eSJason Gunthorpe
2989ce26ea9eSJason Gunthorpe if (sid_domain->type == IOMMU_DOMAIN_IDENTITY ||
2990ce26ea9eSJason Gunthorpe sid_domain->type == IOMMU_DOMAIN_BLOCKED)
2991ce26ea9eSJason Gunthorpe sid_domain->ops->attach_dev(sid_domain, dev);
29927497f421SJason Gunthorpe }
2993ce26ea9eSJason Gunthorpe }
2994ce26ea9eSJason Gunthorpe
arm_smmu_attach_dev_ste(struct iommu_domain * domain,struct device * dev,struct arm_smmu_ste * ste,unsigned int s1dss)2995ce26ea9eSJason Gunthorpe static void arm_smmu_attach_dev_ste(struct iommu_domain *domain,
2996ce26ea9eSJason Gunthorpe struct device *dev,
299712dacfb5SJason Gunthorpe struct arm_smmu_ste *ste,
29987497f421SJason Gunthorpe unsigned int s1dss)
299912dacfb5SJason Gunthorpe {
300012dacfb5SJason Gunthorpe struct arm_smmu_master *master = dev_iommu_priv_get(dev);
300112dacfb5SJason Gunthorpe struct arm_smmu_attach_state state = {
300212dacfb5SJason Gunthorpe .master = master,
300312dacfb5SJason Gunthorpe .old_domain = iommu_get_domain_for_dev(dev),
300412dacfb5SJason Gunthorpe .ssid = IOMMU_NO_PASID,
300512dacfb5SJason Gunthorpe };
3006af8f0b83SJason Gunthorpe
300712dacfb5SJason Gunthorpe /*
300812dacfb5SJason Gunthorpe * Do not allow any ASID to be changed while are working on the STE,
300912dacfb5SJason Gunthorpe * otherwise we could miss invalidations.
301012dacfb5SJason Gunthorpe */
301112dacfb5SJason Gunthorpe mutex_lock(&arm_smmu_asid_lock);
301212dacfb5SJason Gunthorpe
3013ec9098d6SMostafa Saleh /*
301412dacfb5SJason Gunthorpe * If the CD table is not in use we can use the provided STE, otherwise
3015ec9098d6SMostafa Saleh * we use a cdtable STE with the provided S1DSS.
3016ce26ea9eSJason Gunthorpe */
3017ce26ea9eSJason Gunthorpe if (arm_smmu_ssids_in_use(&master->cd_table)) {
301812dacfb5SJason Gunthorpe /*
301912dacfb5SJason Gunthorpe * If a CD table has to be present then we need to run with ATS
302012dacfb5SJason Gunthorpe * on even though the RID will fail ATS queries with UR. This is
302112dacfb5SJason Gunthorpe * because we have no idea what the PASID's need.
302212dacfb5SJason Gunthorpe */
302312dacfb5SJason Gunthorpe state.cd_needs_ats = true;
302412dacfb5SJason Gunthorpe arm_smmu_attach_prepare(&state, domain);
302512dacfb5SJason Gunthorpe arm_smmu_make_cdtable_ste(ste, master, state.ats_enabled, s1dss);
302612dacfb5SJason Gunthorpe } else {
302712dacfb5SJason Gunthorpe arm_smmu_attach_prepare(&state, domain);
302812dacfb5SJason Gunthorpe }
3029352bd64cSJason Gunthorpe arm_smmu_install_ste_for_dev(master, ste);
3030352bd64cSJason Gunthorpe arm_smmu_attach_commit(&state);
3031352bd64cSJason Gunthorpe mutex_unlock(&arm_smmu_asid_lock);
3032352bd64cSJason Gunthorpe
3033352bd64cSJason Gunthorpe /*
3034352bd64cSJason Gunthorpe * This has to be done after removing the master from the
3035ce26ea9eSJason Gunthorpe * arm_smmu_domain->devices to avoid races updating the same context
3036ce26ea9eSJason Gunthorpe * descriptor from arm_smmu_share_asid().
3037ce26ea9eSJason Gunthorpe */
3038352bd64cSJason Gunthorpe arm_smmu_clear_cd(master, IOMMU_NO_PASID);
3039352bd64cSJason Gunthorpe }
3040352bd64cSJason Gunthorpe
arm_smmu_attach_dev_identity(struct iommu_domain * domain,struct device * dev)3041352bd64cSJason Gunthorpe static int arm_smmu_attach_dev_identity(struct iommu_domain *domain,
3042352bd64cSJason Gunthorpe struct device *dev)
3043352bd64cSJason Gunthorpe {
3044352bd64cSJason Gunthorpe struct arm_smmu_ste ste;
3045352bd64cSJason Gunthorpe struct arm_smmu_master *master = dev_iommu_priv_get(dev);
3046352bd64cSJason Gunthorpe
3047352bd64cSJason Gunthorpe arm_smmu_make_bypass_ste(master->smmu, &ste);
3048352bd64cSJason Gunthorpe arm_smmu_attach_dev_ste(domain, dev, &ste, STRTAB_STE_1_S1DSS_BYPASS);
304952acd7d8SShameer Kolothum return 0;
305052acd7d8SShameer Kolothum }
305152acd7d8SShameer Kolothum
305252acd7d8SShameer Kolothum static const struct iommu_domain_ops arm_smmu_identity_ops = {
305352acd7d8SShameer Kolothum .attach_dev = arm_smmu_attach_dev_identity,
305452acd7d8SShameer Kolothum };
3055eb054d67SJoao Martins
305652acd7d8SShameer Kolothum static struct iommu_domain arm_smmu_identity_domain = {
305752acd7d8SShameer Kolothum .type = IOMMU_DOMAIN_IDENTITY,
305852acd7d8SShameer Kolothum .ops = &arm_smmu_identity_ops,
3059eb054d67SJoao Martins };
3060eb054d67SJoao Martins
arm_smmu_attach_dev_blocked(struct iommu_domain * domain,struct device * dev)3061eb054d67SJoao Martins static int arm_smmu_attach_dev_blocked(struct iommu_domain *domain,
306252acd7d8SShameer Kolothum struct device *dev)
306352acd7d8SShameer Kolothum {
306452acd7d8SShameer Kolothum struct arm_smmu_ste ste;
306552acd7d8SShameer Kolothum
306652acd7d8SShameer Kolothum arm_smmu_make_abort_ste(&ste);
306752acd7d8SShameer Kolothum arm_smmu_attach_dev_ste(domain, dev, &ste,
306852acd7d8SShameer Kolothum STRTAB_STE_1_S1DSS_TERMINATE);
306952acd7d8SShameer Kolothum return 0;
3070eb054d67SJoao Martins }
307152acd7d8SShameer Kolothum
307252acd7d8SShameer Kolothum static const struct iommu_domain_ops arm_smmu_blocked_ops = {
307352acd7d8SShameer Kolothum .attach_dev = arm_smmu_attach_dev_blocked,
307452acd7d8SShameer Kolothum };
307552acd7d8SShameer Kolothum
307652acd7d8SShameer Kolothum static struct iommu_domain arm_smmu_blocked_domain = {
307752acd7d8SShameer Kolothum .type = IOMMU_DOMAIN_BLOCKED,
307852acd7d8SShameer Kolothum .ops = &arm_smmu_blocked_ops,
307952acd7d8SShameer Kolothum };
30809eec3f9bSXiang Chen
30819eec3f9bSXiang Chen static struct iommu_domain *
arm_smmu_domain_alloc_user(struct device * dev,u32 flags,struct iommu_domain * parent,const struct iommu_user_data * user_data)30829eec3f9bSXiang Chen arm_smmu_domain_alloc_user(struct device *dev, u32 flags,
3083e86d1aa8SWill Deacon struct iommu_domain *parent,
3084e86d1aa8SWill Deacon const struct iommu_user_data *user_data)
3085e86d1aa8SWill Deacon {
3086e86d1aa8SWill Deacon struct arm_smmu_master *master = dev_iommu_priv_get(dev);
3087e86d1aa8SWill Deacon const u32 PAGING_FLAGS = IOMMU_HWPT_ALLOC_DIRTY_TRACKING;
3088e86d1aa8SWill Deacon struct arm_smmu_domain *smmu_domain;
30899eec3f9bSXiang Chen int ret;
3090e86d1aa8SWill Deacon
3091e86d1aa8SWill Deacon if (flags & ~PAGING_FLAGS)
309259103c79SXiang Chen return ERR_PTR(-EOPNOTSUPP);
309359103c79SXiang Chen if (parent || user_data)
309459103c79SXiang Chen return ERR_PTR(-EOPNOTSUPP);
3095e86d1aa8SWill Deacon
3096e86d1aa8SWill Deacon smmu_domain = arm_smmu_domain_alloc();
3097e86d1aa8SWill Deacon if (IS_ERR(smmu_domain))
3098e86d1aa8SWill Deacon return ERR_CAST(smmu_domain);
3099e86d1aa8SWill Deacon
3100e86d1aa8SWill Deacon smmu_domain->domain.type = IOMMU_DOMAIN_UNMANAGED;
3101e86d1aa8SWill Deacon smmu_domain->domain.ops = arm_smmu_ops.default_domain_ops;
310259103c79SXiang Chen ret = arm_smmu_domain_finalise(smmu_domain, master->smmu, flags);
3103e86d1aa8SWill Deacon if (ret)
3104e86d1aa8SWill Deacon goto err_free;
3105e86d1aa8SWill Deacon return &smmu_domain->domain;
3106e86d1aa8SWill Deacon
3107e86d1aa8SWill Deacon err_free:
3108e86d1aa8SWill Deacon kfree(smmu_domain);
3109e86d1aa8SWill Deacon return ERR_PTR(ret);
3110e86d1aa8SWill Deacon }
3111e86d1aa8SWill Deacon
arm_smmu_map_pages(struct iommu_domain * domain,unsigned long iova,phys_addr_t paddr,size_t pgsize,size_t pgcount,int prot,gfp_t gfp,size_t * mapped)3112e86d1aa8SWill Deacon static int arm_smmu_map_pages(struct iommu_domain *domain, unsigned long iova,
3113e86d1aa8SWill Deacon phys_addr_t paddr, size_t pgsize, size_t pgcount,
3114e86d1aa8SWill Deacon int prot, gfp_t gfp, size_t *mapped)
3115e86d1aa8SWill Deacon {
3116e86d1aa8SWill Deacon struct io_pgtable_ops *ops = to_smmu_domain(domain)->pgtbl_ops;
3117e86d1aa8SWill Deacon
31186cc7e5a9SXiang Chen if (!ops)
31196cc7e5a9SXiang Chen return -ENODEV;
31206cc7e5a9SXiang Chen
3121eba8d2f8SJean-Philippe Brucker return ops->map_pages(ops, iova, paddr, pgsize, pgcount, prot, gfp, mapped);
31227060377cSWill Deacon }
3123e86d1aa8SWill Deacon
arm_smmu_unmap_pages(struct iommu_domain * domain,unsigned long iova,size_t pgsize,size_t pgcount,struct iommu_iotlb_gather * gather)3124e86d1aa8SWill Deacon static size_t arm_smmu_unmap_pages(struct iommu_domain *domain, unsigned long iova,
3125e86d1aa8SWill Deacon size_t pgsize, size_t pgcount,
3126e86d1aa8SWill Deacon struct iommu_iotlb_gather *gather)
3127e86d1aa8SWill Deacon {
3128e86d1aa8SWill Deacon struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
3129e86d1aa8SWill Deacon struct io_pgtable_ops *ops = smmu_domain->pgtbl_ops;
3130e86d1aa8SWill Deacon
3131e86d1aa8SWill Deacon if (!ops)
3132e86d1aa8SWill Deacon return 0;
3133e86d1aa8SWill Deacon
3134e86d1aa8SWill Deacon return ops->unmap_pages(ops, iova, pgsize, pgcount, gather);
3135e86d1aa8SWill Deacon }
3136e86d1aa8SWill Deacon
arm_smmu_flush_iotlb_all(struct iommu_domain * domain)3137e86d1aa8SWill Deacon static void arm_smmu_flush_iotlb_all(struct iommu_domain *domain)
3138e86d1aa8SWill Deacon {
3139e86d1aa8SWill Deacon struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
3140e86d1aa8SWill Deacon
3141e86d1aa8SWill Deacon if (smmu_domain->smmu)
3142e86d1aa8SWill Deacon arm_smmu_tlb_inv_context(smmu_domain);
3143e86d1aa8SWill Deacon }
3144e86d1aa8SWill Deacon
arm_smmu_iotlb_sync(struct iommu_domain * domain,struct iommu_iotlb_gather * gather)3145e86d1aa8SWill Deacon static void arm_smmu_iotlb_sync(struct iommu_domain *domain,
3146e86d1aa8SWill Deacon struct iommu_iotlb_gather *gather)
3147e86d1aa8SWill Deacon {
3148e86d1aa8SWill Deacon struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
3149e86d1aa8SWill Deacon
3150e86d1aa8SWill Deacon if (!gather->pgsize)
3151e86d1aa8SWill Deacon return;
3152e86d1aa8SWill Deacon
3153e86d1aa8SWill Deacon arm_smmu_tlb_inv_range_domain(gather->start,
3154e86d1aa8SWill Deacon gather->end - gather->start + 1,
3155e86d1aa8SWill Deacon gather->pgsize, true, smmu_domain);
3156e86d1aa8SWill Deacon }
3157e86d1aa8SWill Deacon
315804e2afd1SShameer Kolothum static phys_addr_t
arm_smmu_iova_to_phys(struct iommu_domain * domain,dma_addr_t iova)315904e2afd1SShameer Kolothum arm_smmu_iova_to_phys(struct iommu_domain *domain, dma_addr_t iova)
316004e2afd1SShameer Kolothum {
316104e2afd1SShameer Kolothum struct io_pgtable_ops *ops = to_smmu_domain(domain)->pgtbl_ops;
316204e2afd1SShameer Kolothum
316304e2afd1SShameer Kolothum if (!ops)
316404e2afd1SShameer Kolothum return 0;
316504e2afd1SShameer Kolothum
316604e2afd1SShameer Kolothum return ops->iova_to_phys(ops, iova);
316704e2afd1SShameer Kolothum }
316804e2afd1SShameer Kolothum
316904e2afd1SShameer Kolothum static struct platform_driver arm_smmu_driver;
317004e2afd1SShameer Kolothum
3171cdf315f9SJean-Philippe Brucker static
arm_smmu_get_by_fwnode(struct fwnode_handle * fwnode)3172cdf315f9SJean-Philippe Brucker struct arm_smmu_device *arm_smmu_get_by_fwnode(struct fwnode_handle *fwnode)
3173cdf315f9SJean-Philippe Brucker {
3174cdf315f9SJean-Philippe Brucker struct device *dev = driver_find_device_by_fwnode(&arm_smmu_driver.driver,
3175cdf315f9SJean-Philippe Brucker fwnode);
3176cdf315f9SJean-Philippe Brucker put_device(dev);
3177cdf315f9SJean-Philippe Brucker return dev ? dev_get_drvdata(dev) : NULL;
3178cdf315f9SJean-Philippe Brucker }
3179cdf315f9SJean-Philippe Brucker
arm_smmu_sid_in_range(struct arm_smmu_device * smmu,u32 sid)3180cdf315f9SJean-Philippe Brucker static bool arm_smmu_sid_in_range(struct arm_smmu_device *smmu, u32 sid)
3181cdf315f9SJean-Philippe Brucker {
3182cdf315f9SJean-Philippe Brucker if (smmu->features & ARM_SMMU_FEAT_2_LVL_STRTAB)
3183cdf315f9SJean-Philippe Brucker return arm_smmu_strtab_l1_idx(sid) < smmu->strtab_cfg.l2.num_l1_ents;
3184cdf315f9SJean-Philippe Brucker return sid < smmu->strtab_cfg.linear.num_ents;
3185cdf315f9SJean-Philippe Brucker }
3186cdf315f9SJean-Philippe Brucker
arm_smmu_init_sid_strtab(struct arm_smmu_device * smmu,u32 sid)3187cdf315f9SJean-Philippe Brucker static int arm_smmu_init_sid_strtab(struct arm_smmu_device *smmu, u32 sid)
3188cdf315f9SJean-Philippe Brucker {
3189cdf315f9SJean-Philippe Brucker /* Check the SIDs are in range of the SMMU and our stream table */
3190cdf315f9SJean-Philippe Brucker if (!arm_smmu_sid_in_range(smmu, sid))
3191cdf315f9SJean-Philippe Brucker return -ERANGE;
3192cdf315f9SJean-Philippe Brucker
3193cdf315f9SJean-Philippe Brucker /* Ensure l2 strtab is initialised */
319404e2afd1SShameer Kolothum if (smmu->features & ARM_SMMU_FEAT_2_LVL_STRTAB)
3195cdf315f9SJean-Philippe Brucker return arm_smmu_init_l2_strtab(smmu, sid);
3196cdf315f9SJean-Philippe Brucker
3197cdf315f9SJean-Philippe Brucker return 0;
3198cdf315f9SJean-Philippe Brucker }
3199cdf315f9SJean-Philippe Brucker
arm_smmu_insert_master(struct arm_smmu_device * smmu,struct arm_smmu_master * master)3200cdf315f9SJean-Philippe Brucker static int arm_smmu_insert_master(struct arm_smmu_device *smmu,
3201cdf315f9SJean-Philippe Brucker struct arm_smmu_master *master)
3202cdf315f9SJean-Philippe Brucker {
3203cdf315f9SJean-Philippe Brucker int i;
3204cdf315f9SJean-Philippe Brucker int ret = 0;
3205cdf315f9SJean-Philippe Brucker struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(master->dev);
3206cdf315f9SJean-Philippe Brucker
3207cdf315f9SJean-Philippe Brucker master->streams = kcalloc(fwspec->num_ids, sizeof(*master->streams),
3208cdf315f9SJean-Philippe Brucker GFP_KERNEL);
3209cdf315f9SJean-Philippe Brucker if (!master->streams)
3210cdf315f9SJean-Philippe Brucker return -ENOMEM;
3211cdf315f9SJean-Philippe Brucker master->num_streams = fwspec->num_ids;
3212cdf315f9SJean-Philippe Brucker
3213cdf315f9SJean-Philippe Brucker mutex_lock(&smmu->streams_mutex);
3214cdf315f9SJean-Philippe Brucker for (i = 0; i < fwspec->num_ids; i++) {
3215cdf315f9SJean-Philippe Brucker struct arm_smmu_stream *new_stream = &master->streams[i];
3216cdf315f9SJean-Philippe Brucker u32 sid = fwspec->ids[i];
3217cdf315f9SJean-Philippe Brucker
3218cdf315f9SJean-Philippe Brucker new_stream->id = sid;
3219cdf315f9SJean-Philippe Brucker new_stream->master = master;
3220cdf315f9SJean-Philippe Brucker
3221cdf315f9SJean-Philippe Brucker ret = arm_smmu_init_sid_strtab(smmu, sid);
3222cdf315f9SJean-Philippe Brucker if (ret)
3223cdf315f9SJean-Philippe Brucker break;
3224cdf315f9SJean-Philippe Brucker
3225cdf315f9SJean-Philippe Brucker /* Insert into SID tree */
3226cdf315f9SJean-Philippe Brucker if (rb_find_add(&new_stream->node, &smmu->streams,
3227cdf315f9SJean-Philippe Brucker arm_smmu_streams_cmp_node)) {
3228cdf315f9SJean-Philippe Brucker dev_warn(master->dev, "stream %u already in tree\n",
3229cdf315f9SJean-Philippe Brucker sid);
3230cdf315f9SJean-Philippe Brucker ret = -EINVAL;
3231cdf315f9SJean-Philippe Brucker break;
3232cdf315f9SJean-Philippe Brucker }
3233cdf315f9SJean-Philippe Brucker }
3234cdf315f9SJean-Philippe Brucker
3235cdf315f9SJean-Philippe Brucker if (ret) {
3236cdf315f9SJean-Philippe Brucker for (i--; i >= 0; i--)
3237cdf315f9SJean-Philippe Brucker rb_erase(&master->streams[i].node, &smmu->streams);
3238cdf315f9SJean-Philippe Brucker kfree(master->streams);
3239cdf315f9SJean-Philippe Brucker }
3240cdf315f9SJean-Philippe Brucker mutex_unlock(&smmu->streams_mutex);
3241cdf315f9SJean-Philippe Brucker
3242cdf315f9SJean-Philippe Brucker return ret;
3243cdf315f9SJean-Philippe Brucker }
3244cdf315f9SJean-Philippe Brucker
arm_smmu_remove_master(struct arm_smmu_master * master)3245cdf315f9SJean-Philippe Brucker static void arm_smmu_remove_master(struct arm_smmu_master *master)
3246cdf315f9SJean-Philippe Brucker {
3247cdf315f9SJean-Philippe Brucker int i;
3248cdf315f9SJean-Philippe Brucker struct arm_smmu_device *smmu = master->smmu;
3249cdf315f9SJean-Philippe Brucker struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(master->dev);
3250e86d1aa8SWill Deacon
3251e86d1aa8SWill Deacon if (!smmu || !master->streams)
3252cdf315f9SJean-Philippe Brucker return;
3253e86d1aa8SWill Deacon
3254e86d1aa8SWill Deacon mutex_lock(&smmu->streams_mutex);
3255e86d1aa8SWill Deacon for (i = 0; i < fwspec->num_ids; i++)
3256e86d1aa8SWill Deacon rb_erase(&master->streams[i].node, &smmu->streams);
3257e86d1aa8SWill Deacon mutex_unlock(&smmu->streams_mutex);
3258e86d1aa8SWill Deacon
3259e86d1aa8SWill Deacon kfree(master->streams);
3260e86d1aa8SWill Deacon }
3261e86d1aa8SWill Deacon
arm_smmu_probe_device(struct device * dev)3262e86d1aa8SWill Deacon static struct iommu_device *arm_smmu_probe_device(struct device *dev)
3263e86d1aa8SWill Deacon {
3264e86d1aa8SWill Deacon int ret;
3265e86d1aa8SWill Deacon struct arm_smmu_device *smmu;
3266e86d1aa8SWill Deacon struct arm_smmu_master *master;
3267e86d1aa8SWill Deacon struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
3268e86d1aa8SWill Deacon
3269e86d1aa8SWill Deacon if (WARN_ON_ONCE(dev_iommu_priv_get(dev)))
3270e86d1aa8SWill Deacon return ERR_PTR(-EBUSY);
3271e86d1aa8SWill Deacon
3272cdf315f9SJean-Philippe Brucker smmu = arm_smmu_get_by_fwnode(fwspec->iommu_fwnode);
3273e86d1aa8SWill Deacon if (!smmu)
3274e86d1aa8SWill Deacon return ERR_PTR(-ENODEV);
3275e86d1aa8SWill Deacon
3276434b73e6SJean-Philippe Brucker master = kzalloc(sizeof(*master), GFP_KERNEL);
3277434b73e6SJean-Philippe Brucker if (!master)
3278e86d1aa8SWill Deacon return ERR_PTR(-ENOMEM);
3279e86d1aa8SWill Deacon
3280e86d1aa8SWill Deacon master->dev = dev;
3281e86d1aa8SWill Deacon master->smmu = smmu;
3282e86d1aa8SWill Deacon dev_iommu_priv_set(dev, master);
3283e86d1aa8SWill Deacon
3284e86d1aa8SWill Deacon ret = arm_smmu_insert_master(smmu, master);
3285e86d1aa8SWill Deacon if (ret)
3286e86d1aa8SWill Deacon goto err_free_master;
3287e86d1aa8SWill Deacon
3288e86d1aa8SWill Deacon device_property_read_u32(dev, "pasid-num-bits", &master->ssid_bits);
3289e86d1aa8SWill Deacon master->ssid_bits = min(smmu->ssid_bits, master->ssid_bits);
3290e86d1aa8SWill Deacon
3291e86d1aa8SWill Deacon /*
3292e86d1aa8SWill Deacon * Note that PASID must be enabled before, and disabled after ATS:
3293395ad89dSJean-Philippe Brucker * PCI Express Base 4.0r1.0 - 10.5.1.3 ATS Control Register
3294395ad89dSJean-Philippe Brucker *
3295395ad89dSJean-Philippe Brucker * Behavior is undefined if this bit is Set and the value of the PASID
3296395ad89dSJean-Philippe Brucker * Enable, Execute Requested Enable, or Privileged Mode Requested bits
3297395ad89dSJean-Philippe Brucker * are changed.
3298e86d1aa8SWill Deacon */
3299e86d1aa8SWill Deacon arm_smmu_enable_pasid(master);
3300e86d1aa8SWill Deacon
3301e86d1aa8SWill Deacon if (!(smmu->features & ARM_SMMU_FEAT_2_LVL_CDTAB))
3302e86d1aa8SWill Deacon master->ssid_bits = min_t(u8, master->ssid_bits,
3303e86d1aa8SWill Deacon CTXDESC_LINEAR_CDMAX);
3304e86d1aa8SWill Deacon
3305e86d1aa8SWill Deacon if ((smmu->features & ARM_SMMU_FEAT_STALLS &&
3306e86d1aa8SWill Deacon device_property_read_bool(dev, "dma-can-stall")) ||
33074d26ba67SRobin Murphy smmu->features & ARM_SMMU_FEAT_STALL_FORCE)
3308e86d1aa8SWill Deacon master->stall_enabled = true;
3309395ad89dSJean-Philippe Brucker
3310395ad89dSJean-Philippe Brucker if (dev_is_pci(dev)) {
33118c73c32cSJason Gunthorpe unsigned int stu = __ffs(smmu->pgsize_bitmap);
33128c73c32cSJason Gunthorpe
3313734554fdSRobin Murphy pci_prepare_ats(to_pci_dev(dev), stu);
3314d36464f4SJason Gunthorpe }
3315734554fdSRobin Murphy
3316734554fdSRobin Murphy return &smmu->iommu;
33178c73c32cSJason Gunthorpe
3318e86d1aa8SWill Deacon err_free_master:
3319cdf315f9SJean-Philippe Brucker kfree(master);
332010e4968cSMichael Shavit return ERR_PTR(ret);
332110e4968cSMichael Shavit }
3322e86d1aa8SWill Deacon
arm_smmu_release_device(struct device * dev)3323e86d1aa8SWill Deacon static void arm_smmu_release_device(struct device *dev)
3324e86d1aa8SWill Deacon {
3325eb054d67SJoao Martins struct arm_smmu_master *master = dev_iommu_priv_get(dev);
3326eb054d67SJoao Martins
3327eb054d67SJoao Martins if (WARN_ON(arm_smmu_master_sva_enabled(master)))
3328eb054d67SJoao Martins iopf_queue_remove_device(master->smmu->evtq.iopf, dev);
3329eb054d67SJoao Martins
3330eb054d67SJoao Martins /* Put the STE back to what arm_smmu_init_strtab() sets */
3331eb054d67SJoao Martins if (dev->iommu->require_direct)
3332eb054d67SJoao Martins arm_smmu_attach_dev_identity(&arm_smmu_identity_domain, dev);
3333eb054d67SJoao Martins else
3334eb054d67SJoao Martins arm_smmu_attach_dev_blocked(&arm_smmu_blocked_domain, dev);
3335eb054d67SJoao Martins
3336eb054d67SJoao Martins arm_smmu_disable_pasid(master);
3337eb054d67SJoao Martins arm_smmu_remove_master(master);
3338eb054d67SJoao Martins if (arm_smmu_cdtab_allocated(&master->cd_table))
3339eb054d67SJoao Martins arm_smmu_free_cd_tables(master);
3340eb054d67SJoao Martins kfree(master);
3341eb054d67SJoao Martins }
3342eb054d67SJoao Martins
arm_smmu_read_and_clear_dirty(struct iommu_domain * domain,unsigned long iova,size_t size,unsigned long flags,struct iommu_dirty_bitmap * dirty)3343eb054d67SJoao Martins static int arm_smmu_read_and_clear_dirty(struct iommu_domain *domain,
3344eb054d67SJoao Martins unsigned long iova, size_t size,
3345eb054d67SJoao Martins unsigned long flags,
3346e86d1aa8SWill Deacon struct iommu_dirty_bitmap *dirty)
3347e86d1aa8SWill Deacon {
3348e86d1aa8SWill Deacon struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
3349e86d1aa8SWill Deacon struct io_pgtable_ops *ops = smmu_domain->pgtbl_ops;
3350e86d1aa8SWill Deacon
3351e86d1aa8SWill Deacon return ops->read_and_clear_dirty(ops, iova, size, flags, dirty);
3352e86d1aa8SWill Deacon }
3353e86d1aa8SWill Deacon
arm_smmu_set_dirty_tracking(struct iommu_domain * domain,bool enabled)3354e86d1aa8SWill Deacon static int arm_smmu_set_dirty_tracking(struct iommu_domain *domain,
3355e86d1aa8SWill Deacon bool enabled)
3356e86d1aa8SWill Deacon {
3357e86d1aa8SWill Deacon /*
3358e86d1aa8SWill Deacon * Always enabled and the dirty bitmap is cleared prior to
3359e86d1aa8SWill Deacon * set_dirty_tracking().
3360e86d1aa8SWill Deacon */
3361e86d1aa8SWill Deacon return 0;
3362e86d1aa8SWill Deacon }
33637e147547SChristoph Hellwig
arm_smmu_device_group(struct device * dev)3364e86d1aa8SWill Deacon static struct iommu_group *arm_smmu_device_group(struct device *dev)
3365e86d1aa8SWill Deacon {
3366e86d1aa8SWill Deacon struct iommu_group *group;
3367e86d1aa8SWill Deacon
3368e86d1aa8SWill Deacon /*
33697e147547SChristoph Hellwig * We don't support devices sharing stream IDs other than PCI RID
3370e86d1aa8SWill Deacon * aliases, since the necessary ID-to-device lookup becomes rather
3371e86d1aa8SWill Deacon * impractical given a potential sparse 32-bit stream ID space.
33729fde0083SJason Gunthorpe */
3373e86d1aa8SWill Deacon if (dev_is_pci(dev))
33747e147547SChristoph Hellwig group = pci_device_group(dev);
3375e86d1aa8SWill Deacon else
3376e86d1aa8SWill Deacon group = generic_device_group(dev);
3377e86d1aa8SWill Deacon
3378b42a905bSKrzysztof Kozlowski return group;
3379b42a905bSKrzysztof Kozlowski }
3380e86d1aa8SWill Deacon
arm_smmu_enable_nesting(struct iommu_domain * domain)3381e86d1aa8SWill Deacon static int arm_smmu_enable_nesting(struct iommu_domain *domain)
3382e86d1aa8SWill Deacon {
3383e86d1aa8SWill Deacon struct arm_smmu_domain *smmu_domain = to_smmu_domain(domain);
3384e86d1aa8SWill Deacon int ret = 0;
3385e86d1aa8SWill Deacon
3386e86d1aa8SWill Deacon mutex_lock(&smmu_domain->init_mutex);
3387e86d1aa8SWill Deacon if (smmu_domain->smmu)
3388e86d1aa8SWill Deacon ret = -EPERM;
3389e86d1aa8SWill Deacon else
3390e86d1aa8SWill Deacon smmu_domain->stage = ARM_SMMU_DOMAIN_S2;
33910251d010SLu Baolu mutex_unlock(&smmu_domain->init_mutex);
3392e86d1aa8SWill Deacon
3393e86d1aa8SWill Deacon return ret;
3394e86d1aa8SWill Deacon }
3395e86d1aa8SWill Deacon
arm_smmu_of_xlate(struct device * dev,const struct of_phandle_args * args)3396e86d1aa8SWill Deacon static int arm_smmu_of_xlate(struct device *dev,
3397e86d1aa8SWill Deacon const struct of_phandle_args *args)
3398e86d1aa8SWill Deacon {
3399e86d1aa8SWill Deacon return iommu_fwspec_add_ids(dev, args->args, 1);
3400f534d98bSJean-Philippe Brucker }
3401f534d98bSJean-Philippe Brucker
arm_smmu_get_resv_regions(struct device * dev,struct list_head * head)3402f534d98bSJean-Philippe Brucker static void arm_smmu_get_resv_regions(struct device *dev,
3403395ad89dSJean-Philippe Brucker struct list_head *head)
3404395ad89dSJean-Philippe Brucker {
3405469b7b8aSChristoph Hellwig struct iommu_resv_region *region;
3406f534d98bSJean-Philippe Brucker int prot = IOMMU_WRITE | IOMMU_NOEXEC | IOMMU_MMIO;
3407f534d98bSJean-Philippe Brucker
3408f534d98bSJean-Philippe Brucker region = iommu_alloc_resv_region(MSI_IOVA_BASE, MSI_IOVA_LENGTH,
3409395ad89dSJean-Philippe Brucker prot, IOMMU_RESV_SW_MSI, GFP_KERNEL);
3410469b7b8aSChristoph Hellwig if (!region)
3411469b7b8aSChristoph Hellwig return;
3412469b7b8aSChristoph Hellwig
3413469b7b8aSChristoph Hellwig list_add_tail(®ion->list, head);
3414395ad89dSJean-Philippe Brucker
3415395ad89dSJean-Philippe Brucker iommu_dma_get_resv_regions(dev, head);
3416f534d98bSJean-Philippe Brucker }
3417469b7b8aSChristoph Hellwig
arm_smmu_dev_enable_feature(struct device * dev,enum iommu_dev_features feat)3418469b7b8aSChristoph Hellwig static int arm_smmu_dev_enable_feature(struct device *dev,
3419469b7b8aSChristoph Hellwig enum iommu_dev_features feat)
3420469b7b8aSChristoph Hellwig {
3421395ad89dSJean-Philippe Brucker struct arm_smmu_master *master = dev_iommu_priv_get(dev);
3422f534d98bSJean-Philippe Brucker
3423f534d98bSJean-Philippe Brucker if (!master)
3424f534d98bSJean-Philippe Brucker return -ENODEV;
3425f534d98bSJean-Philippe Brucker
3426f534d98bSJean-Philippe Brucker switch (feat) {
3427f534d98bSJean-Philippe Brucker case IOMMU_DEV_FEAT_IOPF:
3428f534d98bSJean-Philippe Brucker if (!arm_smmu_master_iopf_supported(master))
3429f534d98bSJean-Philippe Brucker return -EINVAL;
3430395ad89dSJean-Philippe Brucker if (master->iopf_enabled)
3431395ad89dSJean-Philippe Brucker return -EBUSY;
3432469b7b8aSChristoph Hellwig master->iopf_enabled = true;
3433f534d98bSJean-Philippe Brucker return 0;
3434f534d98bSJean-Philippe Brucker case IOMMU_DEV_FEAT_SVA:
3435f534d98bSJean-Philippe Brucker if (!arm_smmu_master_sva_supported(master))
3436395ad89dSJean-Philippe Brucker return -EINVAL;
3437469b7b8aSChristoph Hellwig if (arm_smmu_master_sva_enabled(master))
3438469b7b8aSChristoph Hellwig return -EBUSY;
3439395ad89dSJean-Philippe Brucker return arm_smmu_master_enable_sva(master);
3440395ad89dSJean-Philippe Brucker default:
3441395ad89dSJean-Philippe Brucker return -EINVAL;
3442395ad89dSJean-Philippe Brucker }
3443f534d98bSJean-Philippe Brucker }
3444469b7b8aSChristoph Hellwig
arm_smmu_dev_disable_feature(struct device * dev,enum iommu_dev_features feat)3445469b7b8aSChristoph Hellwig static int arm_smmu_dev_disable_feature(struct device *dev,
3446395ad89dSJean-Philippe Brucker enum iommu_dev_features feat)
3447f534d98bSJean-Philippe Brucker {
3448f534d98bSJean-Philippe Brucker struct arm_smmu_master *master = dev_iommu_priv_get(dev);
3449f534d98bSJean-Philippe Brucker
3450f534d98bSJean-Philippe Brucker if (!master)
3451f534d98bSJean-Philippe Brucker return -EINVAL;
345224b6c779SYicong Yang
345324b6c779SYicong Yang switch (feat) {
345424b6c779SYicong Yang case IOMMU_DEV_FEAT_IOPF:
345524b6c779SYicong Yang if (!master->iopf_enabled)
345624b6c779SYicong Yang return -EINVAL;
345724b6c779SYicong Yang if (master->sva_enabled)
345824b6c779SYicong Yang return -EBUSY;
345924b6c779SYicong Yang master->iopf_enabled = false;
346024b6c779SYicong Yang return 0;
346124b6c779SYicong Yang case IOMMU_DEV_FEAT_SVA:
346224b6c779SYicong Yang if (!arm_smmu_master_sva_enabled(master))
346324b6c779SYicong Yang return -EINVAL;
346424b6c779SYicong Yang return arm_smmu_master_disable_sva(master);
346524b6c779SYicong Yang default:
346624b6c779SYicong Yang return -EINVAL;
346724b6c779SYicong Yang }
346824b6c779SYicong Yang }
346924b6c779SYicong Yang
347024b6c779SYicong Yang /*
347124b6c779SYicong Yang * HiSilicon PCIe tune and trace device can be used to trace TLP headers on the
3472e86d1aa8SWill Deacon * PCIe link and save the data to memory by DMA. The hardware is restricted to
347312dacfb5SJason Gunthorpe * use identity mapping only.
3474352bd64cSJason Gunthorpe */
3475e86d1aa8SWill Deacon #define IS_HISI_PTT_DEVICE(pdev) ((pdev)->vendor == PCI_VENDOR_ID_HUAWEI && \
3476327e10b4SJason Gunthorpe (pdev)->device == 0xa12e)
3477678d79b9SJason Gunthorpe
arm_smmu_def_domain_type(struct device * dev)347852acd7d8SShameer Kolothum static int arm_smmu_def_domain_type(struct device *dev)
3479e86d1aa8SWill Deacon {
3480e86d1aa8SWill Deacon if (dev_is_pci(dev)) {
3481e86d1aa8SWill Deacon struct pci_dev *pdev = to_pci_dev(dev);
3482e86d1aa8SWill Deacon
3483e86d1aa8SWill Deacon if (IS_HISI_PTT_DEVICE(pdev))
3484386fa64fSLu Baolu return IOMMU_DOMAIN_IDENTITY;
3485f534d98bSJean-Philippe Brucker }
3486f534d98bSJean-Philippe Brucker
3487395ad89dSJean-Philippe Brucker return 0;
348824b6c779SYicong Yang }
3489e86d1aa8SWill Deacon
3490c0aec668SRobin Murphy static struct iommu_ops arm_smmu_ops = {
34919a630a4bSLu Baolu .identity_domain = &arm_smmu_identity_domain,
34929a630a4bSLu Baolu .blocked_domain = &arm_smmu_blocked_domain,
3493f3b273b7SJason Gunthorpe .capable = arm_smmu_capable,
34949a630a4bSLu Baolu .domain_alloc_paging = arm_smmu_domain_alloc_paging,
34959a630a4bSLu Baolu .domain_alloc_sva = arm_smmu_sva_domain_alloc,
34969a630a4bSLu Baolu .domain_alloc_user = arm_smmu_domain_alloc_user,
34979a630a4bSLu Baolu .probe_device = arm_smmu_probe_device,
34989a630a4bSLu Baolu .release_device = arm_smmu_release_device,
34999a630a4bSLu Baolu .device_group = arm_smmu_device_group,
3500d7b2d2baSJason Gunthorpe .of_xlate = arm_smmu_of_xlate,
35019a630a4bSLu Baolu .get_resv_regions = arm_smmu_get_resv_regions,
3502e86d1aa8SWill Deacon .remove_dev_pasid = arm_smmu_remove_dev_pasid,
3503e86d1aa8SWill Deacon .dev_enable_feat = arm_smmu_dev_enable_feature,
3504eb054d67SJoao Martins .dev_disable_feat = arm_smmu_dev_disable_feature,
3505eb054d67SJoao Martins .page_response = arm_smmu_page_response,
3506eb054d67SJoao Martins .def_domain_type = arm_smmu_def_domain_type,
3507eb054d67SJoao Martins .pgsize_bitmap = -1UL, /* Restricted during device attach */
3508eb054d67SJoao Martins .owner = THIS_MODULE,
3509e86d1aa8SWill Deacon .default_domain_ops = &(const struct iommu_domain_ops) {
3510e86d1aa8SWill Deacon .attach_dev = arm_smmu_attach_dev,
3511e86d1aa8SWill Deacon .set_dev_pasid = arm_smmu_s1_set_dev_pasid,
351286d2d921SRobin Murphy .map_pages = arm_smmu_map_pages,
3513e86d1aa8SWill Deacon .unmap_pages = arm_smmu_unmap_pages,
3514e86d1aa8SWill Deacon .flush_iotlb_all = arm_smmu_flush_iotlb_all,
3515e86d1aa8SWill Deacon .iotlb_sync = arm_smmu_iotlb_sync,
3516e86d1aa8SWill Deacon .iova_to_phys = arm_smmu_iova_to_phys,
3517e86d1aa8SWill Deacon .enable_nesting = arm_smmu_enable_nesting,
3518e86d1aa8SWill Deacon .free = arm_smmu_domain_free_paging,
3519e86d1aa8SWill Deacon }
3520e86d1aa8SWill Deacon };
3521e86d1aa8SWill Deacon
3522e86d1aa8SWill Deacon static struct iommu_dirty_ops arm_smmu_dirty_ops = {
3523e86d1aa8SWill Deacon .read_and_clear_dirty = arm_smmu_read_and_clear_dirty,
3524e86d1aa8SWill Deacon .set_dirty_tracking = arm_smmu_set_dirty_tracking,
3525e86d1aa8SWill Deacon };
3526e86d1aa8SWill Deacon
3527e86d1aa8SWill Deacon /* Probing and initialisation functions */
arm_smmu_init_one_queue(struct arm_smmu_device * smmu,struct arm_smmu_queue * q,void __iomem * page,unsigned long prod_off,unsigned long cons_off,size_t dwords,const char * name)3528e86d1aa8SWill Deacon int arm_smmu_init_one_queue(struct arm_smmu_device *smmu,
3529e86d1aa8SWill Deacon struct arm_smmu_queue *q, void __iomem *page,
3530e86d1aa8SWill Deacon unsigned long prod_off, unsigned long cons_off,
3531e86d1aa8SWill Deacon size_t dwords, const char *name)
3532e86d1aa8SWill Deacon {
3533e86d1aa8SWill Deacon size_t qsz;
3534e86d1aa8SWill Deacon
3535e86d1aa8SWill Deacon do {
3536e86d1aa8SWill Deacon qsz = ((1 << q->llq.max_n_shift) * dwords) << 3;
3537e86d1aa8SWill Deacon q->base = dmam_alloc_coherent(smmu->dev, qsz, &q->base_dma,
3538e86d1aa8SWill Deacon GFP_KERNEL);
3539e86d1aa8SWill Deacon if (q->base || qsz < PAGE_SIZE)
3540e86d1aa8SWill Deacon break;
354186d2d921SRobin Murphy
354286d2d921SRobin Murphy q->llq.max_n_shift--;
3543e86d1aa8SWill Deacon } while (1);
3544e86d1aa8SWill Deacon
3545e86d1aa8SWill Deacon if (!q->base) {
3546e86d1aa8SWill Deacon dev_err(smmu->dev,
3547e86d1aa8SWill Deacon "failed to allocate queue (0x%zx bytes) for %s\n",
3548e86d1aa8SWill Deacon qsz, name);
3549e86d1aa8SWill Deacon return -ENOMEM;
3550e86d1aa8SWill Deacon }
3551e86d1aa8SWill Deacon
3552e86d1aa8SWill Deacon if (!WARN_ON(q->base_dma & (qsz - 1))) {
3553e86d1aa8SWill Deacon dev_info(smmu->dev, "allocated %u entries for %s\n",
3554e86d1aa8SWill Deacon 1 << q->llq.max_n_shift, name);
3555e86d1aa8SWill Deacon }
3556e86d1aa8SWill Deacon
3557e86d1aa8SWill Deacon q->prod_reg = page + prod_off;
3558e86d1aa8SWill Deacon q->cons_reg = page + cons_off;
3559e86d1aa8SWill Deacon q->ent_dwords = dwords;
3560e86d1aa8SWill Deacon
3561fcdeb8c3SChristophe JAILLET q->q_base = Q_BASE_RWA;
3562fcdeb8c3SChristophe JAILLET q->q_base |= q->base_dma & Q_BASE_ADDR_MASK;
3563fcdeb8c3SChristophe JAILLET q->q_base |= FIELD_PREP(Q_BASE_LOG2SIZE, q->llq.max_n_shift);
3564fcdeb8c3SChristophe JAILLET
3565e86d1aa8SWill Deacon q->llq.prod = q->llq.cons = 0;
3566fcdeb8c3SChristophe JAILLET return 0;
3567e86d1aa8SWill Deacon }
3568e86d1aa8SWill Deacon
arm_smmu_cmdq_init(struct arm_smmu_device * smmu,struct arm_smmu_cmdq * cmdq)3569e86d1aa8SWill Deacon int arm_smmu_cmdq_init(struct arm_smmu_device *smmu,
3570e86d1aa8SWill Deacon struct arm_smmu_cmdq *cmdq)
3571e86d1aa8SWill Deacon {
3572e86d1aa8SWill Deacon unsigned int nents = 1 << cmdq->q.llq.max_n_shift;
3573e86d1aa8SWill Deacon
357486d2d921SRobin Murphy atomic_set(&cmdq->owner_prod, 0);
357586d2d921SRobin Murphy atomic_set(&cmdq->lock, 0);
357686d2d921SRobin Murphy
3577e86d1aa8SWill Deacon cmdq->valid_map = (atomic_long_t *)devm_bitmap_zalloc(smmu->dev, nents,
3578e86d1aa8SWill Deacon GFP_KERNEL);
3579e86d1aa8SWill Deacon if (!cmdq->valid_map)
3580e86d1aa8SWill Deacon return -ENOMEM;
3581e86d1aa8SWill Deacon
3582e86d1aa8SWill Deacon return 0;
3583e86d1aa8SWill Deacon }
3584e86d1aa8SWill Deacon
arm_smmu_init_queues(struct arm_smmu_device * smmu)358586d2d921SRobin Murphy static int arm_smmu_init_queues(struct arm_smmu_device *smmu)
358686d2d921SRobin Murphy {
358786d2d921SRobin Murphy int ret;
3588e86d1aa8SWill Deacon
3589e86d1aa8SWill Deacon /* cmdq */
3590e86d1aa8SWill Deacon ret = arm_smmu_init_one_queue(smmu, &smmu->cmdq.q, smmu->base,
3591395ad89dSJean-Philippe Brucker ARM_SMMU_CMDQ_PROD, ARM_SMMU_CMDQ_CONS,
3592395ad89dSJean-Philippe Brucker CMDQ_ENT_DWORDS, "cmdq");
3593395ad89dSJean-Philippe Brucker if (ret)
3594395ad89dSJean-Philippe Brucker return ret;
3595395ad89dSJean-Philippe Brucker
3596395ad89dSJean-Philippe Brucker ret = arm_smmu_cmdq_init(smmu, &smmu->cmdq);
3597395ad89dSJean-Philippe Brucker if (ret)
3598e86d1aa8SWill Deacon return ret;
3599e86d1aa8SWill Deacon
3600e86d1aa8SWill Deacon /* evtq */
3601e86d1aa8SWill Deacon ret = arm_smmu_init_one_queue(smmu, &smmu->evtq.q, smmu->page1,
360286d2d921SRobin Murphy ARM_SMMU_EVTQ_PROD, ARM_SMMU_EVTQ_CONS,
360386d2d921SRobin Murphy EVTQ_ENT_DWORDS, "evtq");
360486d2d921SRobin Murphy if (ret)
3605e86d1aa8SWill Deacon return ret;
3606e86d1aa8SWill Deacon
3607e86d1aa8SWill Deacon if ((smmu->features & ARM_SMMU_FEAT_SVA) &&
3608e86d1aa8SWill Deacon (smmu->features & ARM_SMMU_FEAT_STALLS)) {
3609e86d1aa8SWill Deacon smmu->evtq.iopf = iopf_queue_alloc(dev_name(smmu->dev));
3610e86d1aa8SWill Deacon if (!smmu->evtq.iopf)
3611e86d1aa8SWill Deacon return -ENOMEM;
3612e86d1aa8SWill Deacon }
3613e86d1aa8SWill Deacon
3614e86d1aa8SWill Deacon /* priq */
3615e86d1aa8SWill Deacon if (!(smmu->features & ARM_SMMU_FEAT_PRI))
3616e86d1aa8SWill Deacon return 0;
3617e86d1aa8SWill Deacon
3618e86d1aa8SWill Deacon return arm_smmu_init_one_queue(smmu, &smmu->priq.q, smmu->page1,
3619e86d1aa8SWill Deacon ARM_SMMU_PRIQ_PROD, ARM_SMMU_PRIQ_CONS,
3620e86d1aa8SWill Deacon PRIQ_ENT_DWORDS, "priq");
3621e86d1aa8SWill Deacon }
3622e86d1aa8SWill Deacon
arm_smmu_init_strtab_2lvl(struct arm_smmu_device * smmu)3623e86d1aa8SWill Deacon static int arm_smmu_init_strtab_2lvl(struct arm_smmu_device *smmu)
3624e86d1aa8SWill Deacon {
3625e86d1aa8SWill Deacon u32 l1size;
3626e86d1aa8SWill Deacon struct arm_smmu_strtab_cfg *cfg = &smmu->strtab_cfg;
3627e86d1aa8SWill Deacon unsigned int last_sid_idx =
3628e86d1aa8SWill Deacon arm_smmu_strtab_l1_idx((1 << smmu->sid_bits) - 1);
3629e86d1aa8SWill Deacon
3630e86d1aa8SWill Deacon /* Calculate the L1 size, capped to the SIDSIZE. */
3631dc898eb8SZenghui Yu cfg->l2.num_l1_ents = min(last_sid_idx + 1, STRTAB_MAX_L1_ENTRIES);
3632e86d1aa8SWill Deacon if (cfg->l2.num_l1_ents <= last_sid_idx)
3633e86d1aa8SWill Deacon dev_warn(smmu->dev,
3634e86d1aa8SWill Deacon "2-level strtab only covers %u/%u bits of SID\n",
3635e86d1aa8SWill Deacon ilog2(cfg->l2.num_l1_ents * STRTAB_NUM_L2_STES),
3636e86d1aa8SWill Deacon smmu->sid_bits);
3637e86d1aa8SWill Deacon
3638e86d1aa8SWill Deacon l1size = cfg->l2.num_l1_ents * sizeof(struct arm_smmu_strtab_l1);
3639e86d1aa8SWill Deacon cfg->l2.l1tab = dmam_alloc_coherent(smmu->dev, l1size, &cfg->l2.l1_dma,
3640e86d1aa8SWill Deacon GFP_KERNEL);
3641e86d1aa8SWill Deacon if (!cfg->l2.l1tab) {
3642c84c5ab7SJason Gunthorpe dev_err(smmu->dev,
3643c84c5ab7SJason Gunthorpe "failed to allocate l1 stream table (%u bytes)\n",
3644c84c5ab7SJason Gunthorpe l1size);
3645c84c5ab7SJason Gunthorpe return -ENOMEM;
3646c84c5ab7SJason Gunthorpe }
3647c84c5ab7SJason Gunthorpe
3648e86d1aa8SWill Deacon cfg->l2.l2ptrs = devm_kcalloc(smmu->dev, cfg->l2.num_l1_ents,
3649e86d1aa8SWill Deacon sizeof(*cfg->l2.l2ptrs), GFP_KERNEL);
3650e86d1aa8SWill Deacon if (!cfg->l2.l2ptrs)
3651e86d1aa8SWill Deacon return -ENOMEM;
3652e86d1aa8SWill Deacon
3653e86d1aa8SWill Deacon return 0;
3654e86d1aa8SWill Deacon }
3655e86d1aa8SWill Deacon
arm_smmu_init_strtab_linear(struct arm_smmu_device * smmu)3656e86d1aa8SWill Deacon static int arm_smmu_init_strtab_linear(struct arm_smmu_device *smmu)
3657e86d1aa8SWill Deacon {
3658e86d1aa8SWill Deacon u32 size;
3659e86d1aa8SWill Deacon struct arm_smmu_strtab_cfg *cfg = &smmu->strtab_cfg;
3660e86d1aa8SWill Deacon
3661e86d1aa8SWill Deacon size = (1 << smmu->sid_bits) * sizeof(struct arm_smmu_ste);
3662e86d1aa8SWill Deacon cfg->linear.table = dmam_alloc_coherent(smmu->dev, size,
3663e86d1aa8SWill Deacon &cfg->linear.ste_dma,
3664e86d1aa8SWill Deacon GFP_KERNEL);
3665e86d1aa8SWill Deacon if (!cfg->linear.table) {
3666e86d1aa8SWill Deacon dev_err(smmu->dev,
3667e86d1aa8SWill Deacon "failed to allocate linear stream table (%u bytes)\n",
3668e86d1aa8SWill Deacon size);
3669e86d1aa8SWill Deacon return -ENOMEM;
3670e86d1aa8SWill Deacon }
3671e86d1aa8SWill Deacon cfg->linear.num_ents = 1 << smmu->sid_bits;
3672e86d1aa8SWill Deacon
3673e86d1aa8SWill Deacon arm_smmu_init_initial_stes(cfg->linear.table, cfg->linear.num_ents);
3674734554fdSRobin Murphy return 0;
3675e86d1aa8SWill Deacon }
3676e86d1aa8SWill Deacon
arm_smmu_init_strtab(struct arm_smmu_device * smmu)3677e86d1aa8SWill Deacon static int arm_smmu_init_strtab(struct arm_smmu_device *smmu)
3678e86d1aa8SWill Deacon {
3679e86d1aa8SWill Deacon int ret;
3680e86d1aa8SWill Deacon
3681e86d1aa8SWill Deacon if (smmu->features & ARM_SMMU_FEAT_2_LVL_STRTAB)
3682e86d1aa8SWill Deacon ret = arm_smmu_init_strtab_2lvl(smmu);
3683e86d1aa8SWill Deacon else
3684e86d1aa8SWill Deacon ret = arm_smmu_init_strtab_linear(smmu);
3685e86d1aa8SWill Deacon if (ret)
3686e86d1aa8SWill Deacon return ret;
3687e86d1aa8SWill Deacon
3688e86d1aa8SWill Deacon ida_init(&smmu->vmid_map);
3689e86d1aa8SWill Deacon
3690e86d1aa8SWill Deacon return 0;
3691e86d1aa8SWill Deacon }
3692e86d1aa8SWill Deacon
arm_smmu_init_structures(struct arm_smmu_device * smmu)3693e86d1aa8SWill Deacon static int arm_smmu_init_structures(struct arm_smmu_device *smmu)
3694e86d1aa8SWill Deacon {
3695e86d1aa8SWill Deacon int ret;
36961672730cSDawei Li
36971672730cSDawei Li mutex_init(&smmu->streams_mutex);
3698e86d1aa8SWill Deacon smmu->streams = RB_ROOT;
3699e86d1aa8SWill Deacon
3700e86d1aa8SWill Deacon ret = arm_smmu_init_queues(smmu);
3701e86d1aa8SWill Deacon if (ret)
3702e86d1aa8SWill Deacon return ret;
3703e86d1aa8SWill Deacon
3704e86d1aa8SWill Deacon ret = arm_smmu_init_strtab(smmu);
3705cdf315f9SJean-Philippe Brucker if (ret)
3706cdf315f9SJean-Philippe Brucker return ret;
3707cdf315f9SJean-Philippe Brucker
3708e86d1aa8SWill Deacon if (smmu->impl_ops && smmu->impl_ops->init_structures)
3709e86d1aa8SWill Deacon return smmu->impl_ops->init_structures(smmu);
3710e86d1aa8SWill Deacon
3711e86d1aa8SWill Deacon return 0;
3712e86d1aa8SWill Deacon }
3713e86d1aa8SWill Deacon
arm_smmu_write_reg_sync(struct arm_smmu_device * smmu,u32 val,unsigned int reg_off,unsigned int ack_off)3714e86d1aa8SWill Deacon static int arm_smmu_write_reg_sync(struct arm_smmu_device *smmu, u32 val,
3715e86d1aa8SWill Deacon unsigned int reg_off, unsigned int ack_off)
3716e86d1aa8SWill Deacon {
3717e86d1aa8SWill Deacon u32 reg;
3718e86d1aa8SWill Deacon
3719e86d1aa8SWill Deacon writel_relaxed(val, smmu->base + reg_off);
3720e86d1aa8SWill Deacon return readl_relaxed_poll_timeout(smmu->base + ack_off, reg, reg == val,
3721e86d1aa8SWill Deacon 1, ARM_SMMU_POLL_TIMEOUT_US);
3722e86d1aa8SWill Deacon }
3723e86d1aa8SWill Deacon
3724e86d1aa8SWill Deacon /* GBPA is "special" */
arm_smmu_update_gbpa(struct arm_smmu_device * smmu,u32 set,u32 clr)3725e86d1aa8SWill Deacon static int arm_smmu_update_gbpa(struct arm_smmu_device *smmu, u32 set, u32 clr)
3726e86d1aa8SWill Deacon {
3727e86d1aa8SWill Deacon int ret;
3728e86d1aa8SWill Deacon u32 reg, __iomem *gbpa = smmu->base + ARM_SMMU_GBPA;
3729e86d1aa8SWill Deacon
3730e86d1aa8SWill Deacon ret = readl_relaxed_poll_timeout(gbpa, reg, !(reg & GBPA_UPDATE),
3731e86d1aa8SWill Deacon 1, ARM_SMMU_POLL_TIMEOUT_US);
3732e86d1aa8SWill Deacon if (ret)
3733e86d1aa8SWill Deacon return ret;
3734e86d1aa8SWill Deacon
3735e86d1aa8SWill Deacon reg &= ~clr;
3736e86d1aa8SWill Deacon reg |= set;
3737e86d1aa8SWill Deacon writel_relaxed(reg | GBPA_UPDATE, gbpa);
3738e86d1aa8SWill Deacon ret = readl_relaxed_poll_timeout(gbpa, reg, !(reg & GBPA_UPDATE),
3739e86d1aa8SWill Deacon 1, ARM_SMMU_POLL_TIMEOUT_US);
3740e86d1aa8SWill Deacon
3741e86d1aa8SWill Deacon if (ret)
3742e86d1aa8SWill Deacon dev_err(smmu->dev, "GBPA not responding to update\n");
3743e86d1aa8SWill Deacon return ret;
3744e86d1aa8SWill Deacon }
3745e86d1aa8SWill Deacon
arm_smmu_free_msis(void * data)3746e86d1aa8SWill Deacon static void arm_smmu_free_msis(void *data)
3747e86d1aa8SWill Deacon {
3748e86d1aa8SWill Deacon struct device *dev = data;
3749e86d1aa8SWill Deacon
375014fd06c7SThomas Gleixner platform_device_msi_free_irqs_all(dev);
375114fd06c7SThomas Gleixner }
3752e86d1aa8SWill Deacon
arm_smmu_write_msi_msg(struct msi_desc * desc,struct msi_msg * msg)3753e86d1aa8SWill Deacon static void arm_smmu_write_msi_msg(struct msi_desc *desc, struct msi_msg *msg)
3754e86d1aa8SWill Deacon {
3755e86d1aa8SWill Deacon phys_addr_t doorbell;
3756e86d1aa8SWill Deacon struct device *dev = msi_desc_to_dev(desc);
3757e86d1aa8SWill Deacon struct arm_smmu_device *smmu = dev_get_drvdata(dev);
3758e86d1aa8SWill Deacon phys_addr_t *cfg = arm_smmu_msi_cfg[desc->msi_index];
3759dba27c7fSThomas Gleixner
3760e86d1aa8SWill Deacon doorbell = (((u64)msg->address_hi) << 32) | msg->address_lo;
3761e86d1aa8SWill Deacon doorbell &= MSI_CFG0_ADDR_MASK;
3762e86d1aa8SWill Deacon
3763e86d1aa8SWill Deacon writeq_relaxed(doorbell, smmu->base + cfg[0]);
3764e86d1aa8SWill Deacon writel_relaxed(msg->data, smmu->base + cfg[1]);
3765e86d1aa8SWill Deacon writel_relaxed(ARM_SMMU_MEMATTR_DEVICE_nGnRE, smmu->base + cfg[2]);
3766e86d1aa8SWill Deacon }
3767e86d1aa8SWill Deacon
arm_smmu_setup_msis(struct arm_smmu_device * smmu)3768e86d1aa8SWill Deacon static void arm_smmu_setup_msis(struct arm_smmu_device *smmu)
3769e86d1aa8SWill Deacon {
3770e86d1aa8SWill Deacon int ret, nvec = ARM_SMMU_MAX_MSIS;
3771e86d1aa8SWill Deacon struct device *dev = smmu->dev;
3772e86d1aa8SWill Deacon
3773e86d1aa8SWill Deacon /* Clear the MSI address regs */
3774e86d1aa8SWill Deacon writeq_relaxed(0, smmu->base + ARM_SMMU_GERROR_IRQ_CFG0);
3775e86d1aa8SWill Deacon writeq_relaxed(0, smmu->base + ARM_SMMU_EVTQ_IRQ_CFG0);
3776e86d1aa8SWill Deacon
3777e86d1aa8SWill Deacon if (smmu->features & ARM_SMMU_FEAT_PRI)
3778e86d1aa8SWill Deacon writeq_relaxed(0, smmu->base + ARM_SMMU_PRIQ_IRQ_CFG0);
3779e86d1aa8SWill Deacon else
3780e86d1aa8SWill Deacon nvec--;
3781e86d1aa8SWill Deacon
3782e86d1aa8SWill Deacon if (!(smmu->features & ARM_SMMU_FEAT_MSI))
3783e86d1aa8SWill Deacon return;
3784e86d1aa8SWill Deacon
3785e86d1aa8SWill Deacon if (!dev->msi.domain) {
378634fff628SThomas Gleixner dev_info(smmu->dev, "msi_domain absent - falling back to wired irqs\n");
3787e86d1aa8SWill Deacon return;
3788e86d1aa8SWill Deacon }
3789e86d1aa8SWill Deacon
3790e86d1aa8SWill Deacon /* Allocate MSIs for evtq, gerror and priq. Ignore cmdq */
3791e86d1aa8SWill Deacon ret = platform_device_msi_init_and_alloc_irqs(dev, nvec, arm_smmu_write_msi_msg);
379214fd06c7SThomas Gleixner if (ret) {
3793e86d1aa8SWill Deacon dev_warn(dev, "failed to allocate MSIs - falling back to wired irqs\n");
3794e86d1aa8SWill Deacon return;
3795e86d1aa8SWill Deacon }
3796e86d1aa8SWill Deacon
3797e86d1aa8SWill Deacon smmu->evtq.q.irq = msi_get_virq(dev, EVTQ_MSI_INDEX);
3798065afdc9SThomas Gleixner smmu->gerr_irq = msi_get_virq(dev, GERROR_MSI_INDEX);
3799065afdc9SThomas Gleixner smmu->priq.q.irq = msi_get_virq(dev, PRIQ_MSI_INDEX);
3800065afdc9SThomas Gleixner
3801e86d1aa8SWill Deacon /* Add callback to free MSIs on teardown */
3802e86d1aa8SWill Deacon devm_add_action_or_reset(dev, arm_smmu_free_msis, dev);
380380fea979SAleksandr Aprelkov }
3804e86d1aa8SWill Deacon
arm_smmu_setup_unique_irqs(struct arm_smmu_device * smmu)3805e86d1aa8SWill Deacon static void arm_smmu_setup_unique_irqs(struct arm_smmu_device *smmu)
3806e86d1aa8SWill Deacon {
3807e86d1aa8SWill Deacon int irq, ret;
3808e86d1aa8SWill Deacon
3809e86d1aa8SWill Deacon arm_smmu_setup_msis(smmu);
3810e86d1aa8SWill Deacon
3811e86d1aa8SWill Deacon /* Request interrupt lines */
3812e86d1aa8SWill Deacon irq = smmu->evtq.q.irq;
3813e86d1aa8SWill Deacon if (irq) {
3814e86d1aa8SWill Deacon ret = devm_request_threaded_irq(smmu->dev, irq, NULL,
3815e86d1aa8SWill Deacon arm_smmu_evtq_thread,
3816e86d1aa8SWill Deacon IRQF_ONESHOT,
3817e86d1aa8SWill Deacon "arm-smmu-v3-evtq", smmu);
3818e86d1aa8SWill Deacon if (ret < 0)
3819e86d1aa8SWill Deacon dev_warn(smmu->dev, "failed to enable evtq irq\n");
3820e86d1aa8SWill Deacon } else {
3821e86d1aa8SWill Deacon dev_warn(smmu->dev, "no evtq irq - events will not be reported!\n");
3822e86d1aa8SWill Deacon }
3823e86d1aa8SWill Deacon
3824e86d1aa8SWill Deacon irq = smmu->gerr_irq;
3825e86d1aa8SWill Deacon if (irq) {
3826e86d1aa8SWill Deacon ret = devm_request_irq(smmu->dev, irq, arm_smmu_gerror_handler,
3827e86d1aa8SWill Deacon 0, "arm-smmu-v3-gerror", smmu);
3828e86d1aa8SWill Deacon if (ret < 0)
3829e86d1aa8SWill Deacon dev_warn(smmu->dev, "failed to enable gerror irq\n");
3830e86d1aa8SWill Deacon } else {
3831e86d1aa8SWill Deacon dev_warn(smmu->dev, "no gerr irq - errors will not be reported!\n");
3832e86d1aa8SWill Deacon }
3833e86d1aa8SWill Deacon
3834e86d1aa8SWill Deacon if (smmu->features & ARM_SMMU_FEAT_PRI) {
3835e86d1aa8SWill Deacon irq = smmu->priq.q.irq;
3836e86d1aa8SWill Deacon if (irq) {
3837e86d1aa8SWill Deacon ret = devm_request_threaded_irq(smmu->dev, irq, NULL,
3838e86d1aa8SWill Deacon arm_smmu_priq_thread,
3839e86d1aa8SWill Deacon IRQF_ONESHOT,
3840e86d1aa8SWill Deacon "arm-smmu-v3-priq",
3841e86d1aa8SWill Deacon smmu);
3842e86d1aa8SWill Deacon if (ret < 0)
3843e86d1aa8SWill Deacon dev_warn(smmu->dev,
3844e86d1aa8SWill Deacon "failed to enable priq irq\n");
3845e86d1aa8SWill Deacon } else {
3846e86d1aa8SWill Deacon dev_warn(smmu->dev, "no priq irq - PRI will be broken\n");
3847e86d1aa8SWill Deacon }
3848e86d1aa8SWill Deacon }
3849e86d1aa8SWill Deacon }
3850e86d1aa8SWill Deacon
arm_smmu_setup_irqs(struct arm_smmu_device * smmu)3851e86d1aa8SWill Deacon static int arm_smmu_setup_irqs(struct arm_smmu_device *smmu)
3852e86d1aa8SWill Deacon {
3853e86d1aa8SWill Deacon int ret, irq;
3854e86d1aa8SWill Deacon u32 irqen_flags = IRQ_CTRL_EVTQ_IRQEN | IRQ_CTRL_GERROR_IRQEN;
3855e86d1aa8SWill Deacon
3856e86d1aa8SWill Deacon /* Disable IRQs first */
3857e86d1aa8SWill Deacon ret = arm_smmu_write_reg_sync(smmu, 0, ARM_SMMU_IRQ_CTRL,
3858e86d1aa8SWill Deacon ARM_SMMU_IRQ_CTRLACK);
3859e86d1aa8SWill Deacon if (ret) {
3860e86d1aa8SWill Deacon dev_err(smmu->dev, "failed to disable irqs\n");
3861e86d1aa8SWill Deacon return ret;
3862e86d1aa8SWill Deacon }
3863e86d1aa8SWill Deacon
3864e86d1aa8SWill Deacon irq = smmu->combined_irq;
3865e86d1aa8SWill Deacon if (irq) {
3866e86d1aa8SWill Deacon /*
3867e86d1aa8SWill Deacon * Cavium ThunderX2 implementation doesn't support unique irq
3868e86d1aa8SWill Deacon * lines. Use a single irq line for all the SMMUv3 interrupts.
3869e86d1aa8SWill Deacon */
3870e86d1aa8SWill Deacon ret = devm_request_threaded_irq(smmu->dev, irq,
3871e86d1aa8SWill Deacon arm_smmu_combined_irq_handler,
3872e86d1aa8SWill Deacon arm_smmu_combined_irq_thread,
3873e86d1aa8SWill Deacon IRQF_ONESHOT,
3874e86d1aa8SWill Deacon "arm-smmu-v3-combined-irq", smmu);
3875e86d1aa8SWill Deacon if (ret < 0)
3876e86d1aa8SWill Deacon dev_warn(smmu->dev, "failed to enable combined irq\n");
3877e86d1aa8SWill Deacon } else
3878e86d1aa8SWill Deacon arm_smmu_setup_unique_irqs(smmu);
3879e86d1aa8SWill Deacon
3880e86d1aa8SWill Deacon if (smmu->features & ARM_SMMU_FEAT_PRI)
3881e86d1aa8SWill Deacon irqen_flags |= IRQ_CTRL_PRIQ_IRQEN;
3882e86d1aa8SWill Deacon
3883e86d1aa8SWill Deacon /* Enable interrupt generation on the SMMU */
3884e86d1aa8SWill Deacon ret = arm_smmu_write_reg_sync(smmu, irqen_flags,
3885e86d1aa8SWill Deacon ARM_SMMU_IRQ_CTRL, ARM_SMMU_IRQ_CTRLACK);
3886e86d1aa8SWill Deacon if (ret)
3887e86d1aa8SWill Deacon dev_warn(smmu->dev, "failed to enable irqs\n");
3888e86d1aa8SWill Deacon
3889e86d1aa8SWill Deacon return 0;
3890e86d1aa8SWill Deacon }
3891e86d1aa8SWill Deacon
arm_smmu_device_disable(struct arm_smmu_device * smmu)3892e86d1aa8SWill Deacon static int arm_smmu_device_disable(struct arm_smmu_device *smmu)
3893e86d1aa8SWill Deacon {
3894e86d1aa8SWill Deacon int ret;
3895e86d1aa8SWill Deacon
3896e86d1aa8SWill Deacon ret = arm_smmu_write_reg_sync(smmu, 0, ARM_SMMU_CR0, ARM_SMMU_CR0ACK);
3897e86d1aa8SWill Deacon if (ret)
3898e86d1aa8SWill Deacon dev_err(smmu->dev, "failed to clear cr0\n");
3899e86d1aa8SWill Deacon
3900e86d1aa8SWill Deacon return ret;
3901e86d1aa8SWill Deacon }
3902e86d1aa8SWill Deacon
arm_smmu_write_strtab(struct arm_smmu_device * smmu)3903e86d1aa8SWill Deacon static void arm_smmu_write_strtab(struct arm_smmu_device *smmu)
3904734554fdSRobin Murphy {
3905e86d1aa8SWill Deacon struct arm_smmu_strtab_cfg *cfg = &smmu->strtab_cfg;
3906e86d1aa8SWill Deacon dma_addr_t dma;
3907e86d1aa8SWill Deacon u32 reg;
3908e86d1aa8SWill Deacon
3909e86d1aa8SWill Deacon if (smmu->features & ARM_SMMU_FEAT_2_LVL_STRTAB) {
3910e86d1aa8SWill Deacon reg = FIELD_PREP(STRTAB_BASE_CFG_FMT,
3911e86d1aa8SWill Deacon STRTAB_BASE_CFG_FMT_2LVL) |
3912e86d1aa8SWill Deacon FIELD_PREP(STRTAB_BASE_CFG_LOG2SIZE,
3913e86d1aa8SWill Deacon ilog2(cfg->l2.num_l1_ents) + STRTAB_SPLIT) |
3914e86d1aa8SWill Deacon FIELD_PREP(STRTAB_BASE_CFG_SPLIT, STRTAB_SPLIT);
3915e86d1aa8SWill Deacon dma = cfg->l2.l1_dma;
3916e86d1aa8SWill Deacon } else {
3917e86d1aa8SWill Deacon reg = FIELD_PREP(STRTAB_BASE_CFG_FMT,
3918e86d1aa8SWill Deacon STRTAB_BASE_CFG_FMT_LINEAR) |
3919e86d1aa8SWill Deacon FIELD_PREP(STRTAB_BASE_CFG_LOG2SIZE, smmu->sid_bits);
3920e86d1aa8SWill Deacon dma = cfg->linear.ste_dma;
3921e86d1aa8SWill Deacon }
3922e86d1aa8SWill Deacon writeq_relaxed((dma & STRTAB_BASE_ADDR_MASK) | STRTAB_BASE_RA,
3923e86d1aa8SWill Deacon smmu->base + ARM_SMMU_STRTAB_BASE);
3924e86d1aa8SWill Deacon writel_relaxed(reg, smmu->base + ARM_SMMU_STRTAB_BASE_CFG);
3925e86d1aa8SWill Deacon }
3926e86d1aa8SWill Deacon
arm_smmu_device_reset(struct arm_smmu_device * smmu)3927e86d1aa8SWill Deacon static int arm_smmu_device_reset(struct arm_smmu_device *smmu)
3928e86d1aa8SWill Deacon {
3929e86d1aa8SWill Deacon int ret;
3930e86d1aa8SWill Deacon u32 reg, enables;
39319111aebfSJean-Philippe Brucker struct arm_smmu_cmdq_ent cmd;
39329111aebfSJean-Philippe Brucker
39339111aebfSJean-Philippe Brucker /* Clear CR0 and sync (disables SMMU and queue processing) */
39349111aebfSJean-Philippe Brucker reg = readl_relaxed(smmu->base + ARM_SMMU_CR0);
39359111aebfSJean-Philippe Brucker if (reg & CR0_SMMUEN) {
3936e86d1aa8SWill Deacon dev_warn(smmu->dev, "SMMU currently enabled! Resetting...\n");
3937e86d1aa8SWill Deacon arm_smmu_update_gbpa(smmu, GBPA_ABORT, 0);
3938e86d1aa8SWill Deacon }
3939e86d1aa8SWill Deacon
3940e86d1aa8SWill Deacon ret = arm_smmu_device_disable(smmu);
3941e86d1aa8SWill Deacon if (ret)
3942e86d1aa8SWill Deacon return ret;
3943e86d1aa8SWill Deacon
3944e86d1aa8SWill Deacon /* CR1 (table and queue memory attributes) */
3945e86d1aa8SWill Deacon reg = FIELD_PREP(CR1_TABLE_SH, ARM_SMMU_SH_ISH) |
3946e86d1aa8SWill Deacon FIELD_PREP(CR1_TABLE_OC, CR1_CACHE_WB) |
3947e86d1aa8SWill Deacon FIELD_PREP(CR1_TABLE_IC, CR1_CACHE_WB) |
3948e86d1aa8SWill Deacon FIELD_PREP(CR1_QUEUE_SH, ARM_SMMU_SH_ISH) |
3949e86d1aa8SWill Deacon FIELD_PREP(CR1_QUEUE_OC, CR1_CACHE_WB) |
3950e86d1aa8SWill Deacon FIELD_PREP(CR1_QUEUE_IC, CR1_CACHE_WB);
3951e86d1aa8SWill Deacon writel_relaxed(reg, smmu->base + ARM_SMMU_CR1);
3952e86d1aa8SWill Deacon
3953e86d1aa8SWill Deacon /* CR2 (random crap) */
3954e86d1aa8SWill Deacon reg = CR2_PTM | CR2_RECINVSID;
3955e86d1aa8SWill Deacon
3956e86d1aa8SWill Deacon if (smmu->features & ARM_SMMU_FEAT_E2H)
3957e86d1aa8SWill Deacon reg |= CR2_E2H;
3958e86d1aa8SWill Deacon
39594537f6f1SZhen Lei writel_relaxed(reg, smmu->base + ARM_SMMU_CR2);
3960e86d1aa8SWill Deacon
3961e86d1aa8SWill Deacon /* Stream table */
3962e86d1aa8SWill Deacon arm_smmu_write_strtab(smmu);
3963e86d1aa8SWill Deacon
39644537f6f1SZhen Lei /* Command queue */
3965e86d1aa8SWill Deacon writeq_relaxed(smmu->cmdq.q.q_base, smmu->base + ARM_SMMU_CMDQ_BASE);
3966e86d1aa8SWill Deacon writel_relaxed(smmu->cmdq.q.llq.prod, smmu->base + ARM_SMMU_CMDQ_PROD);
3967e86d1aa8SWill Deacon writel_relaxed(smmu->cmdq.q.llq.cons, smmu->base + ARM_SMMU_CMDQ_CONS);
39684537f6f1SZhen Lei
3969e86d1aa8SWill Deacon enables = CR0_CMDQEN;
3970e86d1aa8SWill Deacon ret = arm_smmu_write_reg_sync(smmu, enables, ARM_SMMU_CR0,
3971e86d1aa8SWill Deacon ARM_SMMU_CR0ACK);
397286d2d921SRobin Murphy if (ret) {
397386d2d921SRobin Murphy dev_err(smmu->dev, "failed to enable command queue\n");
3974e86d1aa8SWill Deacon return ret;
3975e86d1aa8SWill Deacon }
3976e86d1aa8SWill Deacon
3977e86d1aa8SWill Deacon /* Invalidate any cached configuration */
3978e86d1aa8SWill Deacon cmd.opcode = CMDQ_OP_CFGI_ALL;
3979e86d1aa8SWill Deacon arm_smmu_cmdq_issue_cmd_with_sync(smmu, &cmd);
3980e86d1aa8SWill Deacon
3981e86d1aa8SWill Deacon /* Invalidate any stale TLB entries */
3982e86d1aa8SWill Deacon if (smmu->features & ARM_SMMU_FEAT_HYP) {
3983e86d1aa8SWill Deacon cmd.opcode = CMDQ_OP_TLBI_EL2_ALL;
3984e86d1aa8SWill Deacon arm_smmu_cmdq_issue_cmd_with_sync(smmu, &cmd);
3985e86d1aa8SWill Deacon }
3986e86d1aa8SWill Deacon
3987e86d1aa8SWill Deacon cmd.opcode = CMDQ_OP_TLBI_NSNH_ALL;
398886d2d921SRobin Murphy arm_smmu_cmdq_issue_cmd_with_sync(smmu, &cmd);
3989e86d1aa8SWill Deacon
399086d2d921SRobin Murphy /* Event queue */
3991e86d1aa8SWill Deacon writeq_relaxed(smmu->evtq.q.q_base, smmu->base + ARM_SMMU_EVTQ_BASE);
3992e86d1aa8SWill Deacon writel_relaxed(smmu->evtq.q.llq.prod, smmu->page1 + ARM_SMMU_EVTQ_PROD);
3993e86d1aa8SWill Deacon writel_relaxed(smmu->evtq.q.llq.cons, smmu->page1 + ARM_SMMU_EVTQ_CONS);
3994e86d1aa8SWill Deacon
3995e86d1aa8SWill Deacon enables |= CR0_EVTQEN;
3996e86d1aa8SWill Deacon ret = arm_smmu_write_reg_sync(smmu, enables, ARM_SMMU_CR0,
3997e86d1aa8SWill Deacon ARM_SMMU_CR0ACK);
3998e86d1aa8SWill Deacon if (ret) {
3999e86d1aa8SWill Deacon dev_err(smmu->dev, "failed to enable event queue\n");
4000e86d1aa8SWill Deacon return ret;
4001e86d1aa8SWill Deacon }
4002e86d1aa8SWill Deacon
4003e86d1aa8SWill Deacon /* PRI queue */
4004e86d1aa8SWill Deacon if (smmu->features & ARM_SMMU_FEAT_PRI) {
4005e86d1aa8SWill Deacon writeq_relaxed(smmu->priq.q.q_base,
4006e86d1aa8SWill Deacon smmu->base + ARM_SMMU_PRIQ_BASE);
4007e86d1aa8SWill Deacon writel_relaxed(smmu->priq.q.llq.prod,
4008e86d1aa8SWill Deacon smmu->page1 + ARM_SMMU_PRIQ_PROD);
4009e86d1aa8SWill Deacon writel_relaxed(smmu->priq.q.llq.cons,
4010e86d1aa8SWill Deacon smmu->page1 + ARM_SMMU_PRIQ_CONS);
4011e86d1aa8SWill Deacon
4012e86d1aa8SWill Deacon enables |= CR0_PRIQEN;
4013e86d1aa8SWill Deacon ret = arm_smmu_write_reg_sync(smmu, enables, ARM_SMMU_CR0,
4014e86d1aa8SWill Deacon ARM_SMMU_CR0ACK);
4015e86d1aa8SWill Deacon if (ret) {
4016e86d1aa8SWill Deacon dev_err(smmu->dev, "failed to enable PRI queue\n");
4017e86d1aa8SWill Deacon return ret;
4018e86d1aa8SWill Deacon }
4019e86d1aa8SWill Deacon }
4020734554fdSRobin Murphy
4021e86d1aa8SWill Deacon if (smmu->features & ARM_SMMU_FEAT_ATS) {
4022e86d1aa8SWill Deacon enables |= CR0_ATSCHK;
4023e86d1aa8SWill Deacon ret = arm_smmu_write_reg_sync(smmu, enables, ARM_SMMU_CR0,
4024e86d1aa8SWill Deacon ARM_SMMU_CR0ACK);
4025e86d1aa8SWill Deacon if (ret) {
4026e86d1aa8SWill Deacon dev_err(smmu->dev, "failed to enable ATS check\n");
4027e86d1aa8SWill Deacon return ret;
4028e86d1aa8SWill Deacon }
4029e86d1aa8SWill Deacon }
4030e86d1aa8SWill Deacon
4031e86d1aa8SWill Deacon ret = arm_smmu_setup_irqs(smmu);
4032f322e8afSRobin Murphy if (ret) {
4033f322e8afSRobin Murphy dev_err(smmu->dev, "failed to setup irqs\n");
4034309a15cbSRobin Murphy return ret;
4035f322e8afSRobin Murphy }
4036f322e8afSRobin Murphy
4037f322e8afSRobin Murphy if (is_kdump_kernel())
4038f322e8afSRobin Murphy enables &= ~(CR0_EVTQEN | CR0_PRIQEN);
4039f322e8afSRobin Murphy
4040f322e8afSRobin Murphy /* Enable the SMMU interface */
4041f322e8afSRobin Murphy enables |= CR0_SMMUEN;
4042f322e8afSRobin Murphy ret = arm_smmu_write_reg_sync(smmu, enables, ARM_SMMU_CR0,
4043f322e8afSRobin Murphy ARM_SMMU_CR0ACK);
4044f322e8afSRobin Murphy if (ret) {
4045f322e8afSRobin Murphy dev_err(smmu->dev, "failed to enable SMMU interface\n");
4046f322e8afSRobin Murphy return ret;
4047f322e8afSRobin Murphy }
4048f322e8afSRobin Murphy
4049f322e8afSRobin Murphy if (smmu->impl_ops && smmu->impl_ops->device_reset) {
4050f322e8afSRobin Murphy ret = smmu->impl_ops->device_reset(smmu);
4051f322e8afSRobin Murphy if (ret) {
4052f322e8afSRobin Murphy dev_err(smmu->dev, "failed to reset impl\n");
4053f322e8afSRobin Murphy return ret;
40540bfbfc52SRobin Murphy }
40550bfbfc52SRobin Murphy }
40560bfbfc52SRobin Murphy
4057f322e8afSRobin Murphy return 0;
4058309a15cbSRobin Murphy }
4059309a15cbSRobin Murphy
4060309a15cbSRobin Murphy #define IIDR_IMPLEMENTER_ARM 0x43b
4061309a15cbSRobin Murphy #define IIDR_PRODUCTID_ARM_MMU_600 0x483
40620bfbfc52SRobin Murphy #define IIDR_PRODUCTID_ARM_MMU_700 0x487
40630bfbfc52SRobin Murphy
arm_smmu_device_iidr_probe(struct arm_smmu_device * smmu)4064309a15cbSRobin Murphy static void arm_smmu_device_iidr_probe(struct arm_smmu_device *smmu)
4065f322e8afSRobin Murphy {
4066f322e8afSRobin Murphy u32 reg;
4067f322e8afSRobin Murphy unsigned int implementer, productid, variant, revision;
4068f322e8afSRobin Murphy
4069f322e8afSRobin Murphy reg = readl_relaxed(smmu->base + ARM_SMMU_IIDR);
40702f8d6178SJean-Philippe Brucker implementer = FIELD_GET(IIDR_IMPLEMENTER, reg);
40712f8d6178SJean-Philippe Brucker productid = FIELD_GET(IIDR_PRODUCTID, reg);
40722f8d6178SJean-Philippe Brucker variant = FIELD_GET(IIDR_VARIANT, reg);
40732f8d6178SJean-Philippe Brucker revision = FIELD_GET(IIDR_REVISION, reg);
40742f8d6178SJean-Philippe Brucker
40752f8d6178SJean-Philippe Brucker switch (implementer) {
40762f8d6178SJean-Philippe Brucker case IIDR_IMPLEMENTER_ARM:
40772f8d6178SJean-Philippe Brucker switch (productid) {
40782f8d6178SJean-Philippe Brucker case IIDR_PRODUCTID_ARM_MMU_600:
40792f8d6178SJean-Philippe Brucker /* Arm erratum 1076982 */
40802f8d6178SJean-Philippe Brucker if (variant == 0 && revision <= 2)
40812f8d6178SJean-Philippe Brucker smmu->features &= ~ARM_SMMU_FEAT_SEV;
40822f8d6178SJean-Philippe Brucker /* Arm erratum 1209401 */
40832f8d6178SJean-Philippe Brucker if (variant < 2)
40842f8d6178SJean-Philippe Brucker smmu->features &= ~ARM_SMMU_FEAT_NESTING;
40852f8d6178SJean-Philippe Brucker break;
40862f8d6178SJean-Philippe Brucker case IIDR_PRODUCTID_ARM_MMU_700:
40872f8d6178SJean-Philippe Brucker /* Arm erratum 2812531 */
40882f8d6178SJean-Philippe Brucker smmu->features &= ~ARM_SMMU_FEAT_BTM;
40892f8d6178SJean-Philippe Brucker smmu->options |= ARM_SMMU_OPT_CMDQ_FORCE_SYNC;
40902f8d6178SJean-Philippe Brucker /* Arm errata 2268618, 2812531 */
40912f8d6178SJean-Philippe Brucker smmu->features &= ~ARM_SMMU_FEAT_NESTING;
4092e86d1aa8SWill Deacon break;
4093e86d1aa8SWill Deacon }
4094e86d1aa8SWill Deacon break;
4095e86d1aa8SWill Deacon }
4096e86d1aa8SWill Deacon }
4097e86d1aa8SWill Deacon
arm_smmu_get_httu(struct arm_smmu_device * smmu,u32 reg)4098e86d1aa8SWill Deacon static void arm_smmu_get_httu(struct arm_smmu_device *smmu, u32 reg)
4099e86d1aa8SWill Deacon {
4100e86d1aa8SWill Deacon u32 fw_features = smmu->features & (ARM_SMMU_FEAT_HA | ARM_SMMU_FEAT_HD);
4101e86d1aa8SWill Deacon u32 hw_features = 0;
4102e86d1aa8SWill Deacon
4103e86d1aa8SWill Deacon switch (FIELD_GET(IDR0_HTTU, reg)) {
4104e86d1aa8SWill Deacon case IDR0_HTTU_ACCESS_DIRTY:
4105e86d1aa8SWill Deacon hw_features |= ARM_SMMU_FEAT_HD;
4106e86d1aa8SWill Deacon fallthrough;
4107e86d1aa8SWill Deacon case IDR0_HTTU_ACCESS:
4108e86d1aa8SWill Deacon hw_features |= ARM_SMMU_FEAT_HA;
4109e86d1aa8SWill Deacon }
4110e86d1aa8SWill Deacon
4111e86d1aa8SWill Deacon if (smmu->dev->of_node)
4112e86d1aa8SWill Deacon smmu->features |= hw_features;
4113e86d1aa8SWill Deacon else if (hw_features != fw_features)
4114e86d1aa8SWill Deacon /* ACPI IORT sets the HTTU bits */
4115e86d1aa8SWill Deacon dev_warn(smmu->dev,
4116e86d1aa8SWill Deacon "IDR0.HTTU features(0x%x) overridden by FW configuration (0x%x)\n",
4117e86d1aa8SWill Deacon hw_features, fw_features);
4118e86d1aa8SWill Deacon }
4119e86d1aa8SWill Deacon
arm_smmu_device_hw_probe(struct arm_smmu_device * smmu)4120e86d1aa8SWill Deacon static int arm_smmu_device_hw_probe(struct arm_smmu_device *smmu)
4121e86d1aa8SWill Deacon {
4122e86d1aa8SWill Deacon u32 reg;
4123e86d1aa8SWill Deacon bool coherent = smmu->features & ARM_SMMU_FEAT_COHERENCY;
4124e86d1aa8SWill Deacon
4125e86d1aa8SWill Deacon /* IDR0 */
4126e86d1aa8SWill Deacon reg = readl_relaxed(smmu->base + ARM_SMMU_IDR0);
4127e86d1aa8SWill Deacon
4128e86d1aa8SWill Deacon /* 2-level structures */
4129e86d1aa8SWill Deacon if (FIELD_GET(IDR0_ST_LVL, reg) == IDR0_ST_LVL_2LVL)
4130e86d1aa8SWill Deacon smmu->features |= ARM_SMMU_FEAT_2_LVL_STRTAB;
4131e86d1aa8SWill Deacon
4132e86d1aa8SWill Deacon if (reg & IDR0_CD2L)
4133e86d1aa8SWill Deacon smmu->features |= ARM_SMMU_FEAT_2_LVL_CDTAB;
4134e86d1aa8SWill Deacon
4135e86d1aa8SWill Deacon /*
4136e86d1aa8SWill Deacon * Translation table endianness.
4137e86d1aa8SWill Deacon * We currently require the same endianness as the CPU, but this
4138e86d1aa8SWill Deacon * could be changed later by adding a new IO_PGTABLE_QUIRK.
4139e86d1aa8SWill Deacon */
4140bd07a20aSBarry Song switch (FIELD_GET(IDR0_TTENDIAN, reg)) {
4141e86d1aa8SWill Deacon case IDR0_TTENDIAN_MIXED:
4142bd07a20aSBarry Song smmu->features |= ARM_SMMU_FEAT_TT_LE | ARM_SMMU_FEAT_TT_BE;
4143bd07a20aSBarry Song break;
4144bd07a20aSBarry Song #ifdef __BIG_ENDIAN
4145e86d1aa8SWill Deacon case IDR0_TTENDIAN_BE:
41469111aebfSJean-Philippe Brucker smmu->features |= ARM_SMMU_FEAT_TT_BE;
4147e86d1aa8SWill Deacon break;
41489111aebfSJean-Philippe Brucker #else
41499111aebfSJean-Philippe Brucker case IDR0_TTENDIAN_LE:
41509111aebfSJean-Philippe Brucker smmu->features |= ARM_SMMU_FEAT_TT_LE;
4151e86d1aa8SWill Deacon break;
41522f8d6178SJean-Philippe Brucker #endif
41532f8d6178SJean-Philippe Brucker default:
4154e86d1aa8SWill Deacon dev_err(smmu->dev, "unknown/unsupported TT endianness!\n");
4155e86d1aa8SWill Deacon return -ENXIO;
4156e86d1aa8SWill Deacon }
4157e86d1aa8SWill Deacon
4158e86d1aa8SWill Deacon /* Boolean feature flags */
4159e86d1aa8SWill Deacon if (IS_ENABLED(CONFIG_PCI_PRI) && reg & IDR0_PRI)
4160e86d1aa8SWill Deacon smmu->features |= ARM_SMMU_FEAT_PRI;
4161e86d1aa8SWill Deacon
4162e86d1aa8SWill Deacon if (IS_ENABLED(CONFIG_PCI_ATS) && reg & IDR0_ATS)
4163e86d1aa8SWill Deacon smmu->features |= ARM_SMMU_FEAT_ATS;
4164e86d1aa8SWill Deacon
4165df561f66SGustavo A. R. Silva if (reg & IDR0_SEV)
4166e86d1aa8SWill Deacon smmu->features |= ARM_SMMU_FEAT_SEV;
4167e86d1aa8SWill Deacon
4168e86d1aa8SWill Deacon if (reg & IDR0_MSI) {
4169e86d1aa8SWill Deacon smmu->features |= ARM_SMMU_FEAT_MSI;
4170e86d1aa8SWill Deacon if (coherent && !disable_msipolling)
4171e86d1aa8SWill Deacon smmu->options |= ARM_SMMU_OPT_MSIPOLL;
4172e86d1aa8SWill Deacon }
4173e86d1aa8SWill Deacon
4174e86d1aa8SWill Deacon if (reg & IDR0_HYP) {
4175e86d1aa8SWill Deacon smmu->features |= ARM_SMMU_FEAT_HYP;
4176e86d1aa8SWill Deacon if (cpus_have_cap(ARM64_HAS_VIRT_HOST_EXTN))
4177e86d1aa8SWill Deacon smmu->features |= ARM_SMMU_FEAT_E2H;
4178e86d1aa8SWill Deacon }
4179e86d1aa8SWill Deacon
4180e86d1aa8SWill Deacon arm_smmu_get_httu(smmu, reg);
4181e86d1aa8SWill Deacon
4182e86d1aa8SWill Deacon /*
4183e86d1aa8SWill Deacon * The coherency feature as set by FW is used in preference to the ID
4184e86d1aa8SWill Deacon * register, but warn on mismatch.
4185df561f66SGustavo A. R. Silva */
4186e86d1aa8SWill Deacon if (!!(reg & IDR0_COHACC) != coherent)
4187e86d1aa8SWill Deacon dev_warn(smmu->dev, "IDR0.COHACC overridden by FW configuration (%s)\n",
4188e86d1aa8SWill Deacon coherent ? "true" : "false");
4189e86d1aa8SWill Deacon
4190e86d1aa8SWill Deacon switch (FIELD_GET(IDR0_STALL_MODEL, reg)) {
4191e86d1aa8SWill Deacon case IDR0_STALL_MODEL_FORCE:
4192e86d1aa8SWill Deacon smmu->features |= ARM_SMMU_FEAT_STALL_FORCE;
4193e86d1aa8SWill Deacon fallthrough;
4194e86d1aa8SWill Deacon case IDR0_STALL_MODEL_STALL:
4195e86d1aa8SWill Deacon smmu->features |= ARM_SMMU_FEAT_STALLS;
4196e86d1aa8SWill Deacon }
4197e86d1aa8SWill Deacon
4198e86d1aa8SWill Deacon if (reg & IDR0_S1P)
4199e86d1aa8SWill Deacon smmu->features |= ARM_SMMU_FEAT_TRANS_S1;
4200e86d1aa8SWill Deacon
4201e86d1aa8SWill Deacon if (reg & IDR0_S2P)
4202e86d1aa8SWill Deacon smmu->features |= ARM_SMMU_FEAT_TRANS_S2;
4203e86d1aa8SWill Deacon
4204ec9098d6SMostafa Saleh if (!(reg & (IDR0_S1P | IDR0_S2P))) {
4205ec9098d6SMostafa Saleh dev_err(smmu->dev, "no translation support!\n");
4206ec9098d6SMostafa Saleh return -ENXIO;
4207e86d1aa8SWill Deacon }
4208e86d1aa8SWill Deacon
4209e86d1aa8SWill Deacon /* We only support the AArch64 table format at present */
4210e86d1aa8SWill Deacon switch (FIELD_GET(IDR0_TTF, reg)) {
4211e86d1aa8SWill Deacon case IDR0_TTF_AARCH32_64:
4212e86d1aa8SWill Deacon smmu->ias = 40;
4213e86d1aa8SWill Deacon fallthrough;
4214e86d1aa8SWill Deacon case IDR0_TTF_AARCH64:
4215e86d1aa8SWill Deacon break;
4216e86d1aa8SWill Deacon default:
4217e86d1aa8SWill Deacon dev_err(smmu->dev, "AArch64 table format not supported!\n");
4218e86d1aa8SWill Deacon return -ENXIO;
4219e86d1aa8SWill Deacon }
4220e86d1aa8SWill Deacon
4221e86d1aa8SWill Deacon /* ASID/VMID sizes */
4222e86d1aa8SWill Deacon smmu->asid_bits = reg & IDR0_ASID16 ? 16 : 8;
4223e86d1aa8SWill Deacon smmu->vmid_bits = reg & IDR0_VMID16 ? 16 : 8;
4224e86d1aa8SWill Deacon
4225e86d1aa8SWill Deacon /* IDR1 */
4226e86d1aa8SWill Deacon reg = readl_relaxed(smmu->base + ARM_SMMU_IDR1);
4227e86d1aa8SWill Deacon if (reg & (IDR1_TABLES_PRESET | IDR1_QUEUES_PRESET | IDR1_REL)) {
4228e86d1aa8SWill Deacon dev_err(smmu->dev, "embedded implementation not supported\n");
4229e86d1aa8SWill Deacon return -ENXIO;
42301adf3cc2SLu Baolu }
4231e86d1aa8SWill Deacon
4232e86d1aa8SWill Deacon if (reg & IDR1_ATTR_TYPES_OVR)
4233e86d1aa8SWill Deacon smmu->features |= ARM_SMMU_FEAT_ATTR_TYPES_OVR;
4234e86d1aa8SWill Deacon
4235e86d1aa8SWill Deacon /* Queue sizes, capped to ensure natural alignment */
4236e86d1aa8SWill Deacon smmu->cmdq.q.llq.max_n_shift = min_t(u32, CMDQ_MAX_SZ_SHIFT,
4237e86d1aa8SWill Deacon FIELD_GET(IDR1_CMDQS, reg));
4238e86d1aa8SWill Deacon if (smmu->cmdq.q.llq.max_n_shift <= ilog2(CMDQ_BATCH_ENTRIES)) {
4239e86d1aa8SWill Deacon /*
4240e86d1aa8SWill Deacon * We don't support splitting up batches, so one batch of
4241e86d1aa8SWill Deacon * commands plus an extra sync needs to fit inside the command
4242e86d1aa8SWill Deacon * queue. There's also no way we can handle the weird alignment
4243e86d1aa8SWill Deacon * restrictions on the base pointer for a unit-length queue.
4244e86d1aa8SWill Deacon */
4245e86d1aa8SWill Deacon dev_err(smmu->dev, "command queue size <= %d entries not supported\n",
4246e86d1aa8SWill Deacon CMDQ_BATCH_ENTRIES);
4247e86d1aa8SWill Deacon return -ENXIO;
4248e86d1aa8SWill Deacon }
4249e86d1aa8SWill Deacon
4250e86d1aa8SWill Deacon smmu->evtq.q.llq.max_n_shift = min_t(u32, EVTQ_MAX_SZ_SHIFT,
4251e86d1aa8SWill Deacon FIELD_GET(IDR1_EVTQS, reg));
4252e86d1aa8SWill Deacon smmu->priq.q.llq.max_n_shift = min_t(u32, PRIQ_MAX_SZ_SHIFT,
4253e86d1aa8SWill Deacon FIELD_GET(IDR1_PRIQS, reg));
4254e86d1aa8SWill Deacon
4255e86d1aa8SWill Deacon /* SID/SSID sizes */
4256e86d1aa8SWill Deacon smmu->ssid_bits = FIELD_GET(IDR1_SSIDSIZE, reg);
4257e86d1aa8SWill Deacon smmu->sid_bits = FIELD_GET(IDR1_SIDSIZE, reg);
4258e86d1aa8SWill Deacon smmu->iommu.max_pasids = 1UL << smmu->ssid_bits;
4259e86d1aa8SWill Deacon
4260e86d1aa8SWill Deacon /*
4261e86d1aa8SWill Deacon * If the SMMU supports fewer bits than would fill a single L2 stream
4262e86d1aa8SWill Deacon * table, use a linear table instead.
4263e86d1aa8SWill Deacon */
4264e86d1aa8SWill Deacon if (smmu->sid_bits <= STRTAB_SPLIT)
4265e86d1aa8SWill Deacon smmu->features &= ~ARM_SMMU_FEAT_2_LVL_STRTAB;
4266e86d1aa8SWill Deacon
4267e86d1aa8SWill Deacon /* IDR3 */
4268e86d1aa8SWill Deacon reg = readl_relaxed(smmu->base + ARM_SMMU_IDR3);
4269e86d1aa8SWill Deacon if (FIELD_GET(IDR3_RIL, reg))
4270e86d1aa8SWill Deacon smmu->features |= ARM_SMMU_FEAT_RANGE_INV;
4271e86d1aa8SWill Deacon
4272e86d1aa8SWill Deacon /* IDR5 */
4273e86d1aa8SWill Deacon reg = readl_relaxed(smmu->base + ARM_SMMU_IDR5);
4274e86d1aa8SWill Deacon
4275e86d1aa8SWill Deacon /* Maximum number of outstanding stalls */
4276e86d1aa8SWill Deacon smmu->evtq.max_stalls = FIELD_GET(IDR5_STALL_MAX, reg);
4277e86d1aa8SWill Deacon
4278e86d1aa8SWill Deacon /* Page sizes */
4279e86d1aa8SWill Deacon if (reg & IDR5_GRAN64K)
4280e86d1aa8SWill Deacon smmu->pgsize_bitmap |= SZ_64K | SZ_512M;
4281e86d1aa8SWill Deacon if (reg & IDR5_GRAN16K)
4282e86d1aa8SWill Deacon smmu->pgsize_bitmap |= SZ_16K | SZ_32M;
4283e86d1aa8SWill Deacon if (reg & IDR5_GRAN4K)
4284e86d1aa8SWill Deacon smmu->pgsize_bitmap |= SZ_4K | SZ_2M | SZ_1G;
4285e86d1aa8SWill Deacon
4286df561f66SGustavo A. R. Silva /* Input address size */
4287e86d1aa8SWill Deacon if (FIELD_GET(IDR5_VAX, reg) == IDR5_VAX_52_BIT)
4288e86d1aa8SWill Deacon smmu->features |= ARM_SMMU_FEAT_VAX;
4289e86d1aa8SWill Deacon
4290e86d1aa8SWill Deacon /* Output address size */
4291e86d1aa8SWill Deacon switch (FIELD_GET(IDR5_OAS, reg)) {
4292e86d1aa8SWill Deacon case IDR5_OAS_32_BIT:
4293e86d1aa8SWill Deacon smmu->oas = 32;
4294e86d1aa8SWill Deacon break;
4295e86d1aa8SWill Deacon case IDR5_OAS_36_BIT:
4296e86d1aa8SWill Deacon smmu->oas = 36;
4297e86d1aa8SWill Deacon break;
4298e86d1aa8SWill Deacon case IDR5_OAS_40_BIT:
4299e86d1aa8SWill Deacon smmu->oas = 40;
4300e86d1aa8SWill Deacon break;
4301e86d1aa8SWill Deacon case IDR5_OAS_42_BIT:
4302e86d1aa8SWill Deacon smmu->oas = 42;
43031d9777b9SRobin Murphy break;
43041d9777b9SRobin Murphy case IDR5_OAS_44_BIT:
43051d9777b9SRobin Murphy smmu->oas = 44;
43061d9777b9SRobin Murphy break;
4307f322e8afSRobin Murphy case IDR5_OAS_52_BIT:
4308f322e8afSRobin Murphy smmu->oas = 52;
4309d744f9e6SJean-Philippe Brucker smmu->pgsize_bitmap |= 1ULL << 42; /* 4TB */
4310d744f9e6SJean-Philippe Brucker break;
4311d744f9e6SJean-Philippe Brucker default:
4312e86d1aa8SWill Deacon dev_info(smmu->dev,
4313e86d1aa8SWill Deacon "unknown output address size. Truncating to 48-bit\n");
4314e86d1aa8SWill Deacon fallthrough;
4315e86d1aa8SWill Deacon case IDR5_OAS_48_BIT:
4316e86d1aa8SWill Deacon smmu->oas = 48;
4317e86d1aa8SWill Deacon }
4318e86d1aa8SWill Deacon
4319e86d1aa8SWill Deacon if (arm_smmu_ops.pgsize_bitmap == -1UL)
4320e86d1aa8SWill Deacon arm_smmu_ops.pgsize_bitmap = smmu->pgsize_bitmap;
4321e86d1aa8SWill Deacon else
4322e86d1aa8SWill Deacon arm_smmu_ops.pgsize_bitmap |= smmu->pgsize_bitmap;
4323e86d1aa8SWill Deacon
4324e86d1aa8SWill Deacon /* Set the DMA mask for our table walker */
4325e86d1aa8SWill Deacon if (dma_set_mask_and_coherent(smmu->dev, DMA_BIT_MASK(smmu->oas)))
4326e86d1aa8SWill Deacon dev_warn(smmu->dev,
4327e86d1aa8SWill Deacon "failed to set DMA mask for table walker\n");
4328e86d1aa8SWill Deacon
4329e86d1aa8SWill Deacon smmu->ias = max(smmu->ias, smmu->oas);
4330e86d1aa8SWill Deacon
4331e86d1aa8SWill Deacon if ((smmu->features & ARM_SMMU_FEAT_TRANS_S1) &&
4332e86d1aa8SWill Deacon (smmu->features & ARM_SMMU_FEAT_TRANS_S2))
4333e86d1aa8SWill Deacon smmu->features |= ARM_SMMU_FEAT_NESTING;
4334e86d1aa8SWill Deacon
4335e86d1aa8SWill Deacon arm_smmu_device_iidr_probe(smmu);
4336e86d1aa8SWill Deacon
4337e86d1aa8SWill Deacon if (arm_smmu_sva_supported(smmu))
4338e86d1aa8SWill Deacon smmu->features |= ARM_SMMU_FEAT_SVA;
4339e86d1aa8SWill Deacon
4340e86d1aa8SWill Deacon dev_info(smmu->dev, "ias %lu-bit, oas %lu-bit (features 0x%08x)\n",
4341e86d1aa8SWill Deacon smmu->ias, smmu->oas, smmu->features);
4342e86d1aa8SWill Deacon return 0;
4343e86d1aa8SWill Deacon }
4344e86d1aa8SWill Deacon
4345e86d1aa8SWill Deacon #ifdef CONFIG_ACPI
4346e86d1aa8SWill Deacon #ifdef CONFIG_TEGRA241_CMDQV
acpi_smmu_dsdt_probe_tegra241_cmdqv(struct acpi_iort_node * node,struct arm_smmu_device * smmu)4347e86d1aa8SWill Deacon static void acpi_smmu_dsdt_probe_tegra241_cmdqv(struct acpi_iort_node *node,
4348e86d1aa8SWill Deacon struct arm_smmu_device *smmu)
43492f8d6178SJean-Philippe Brucker {
43502f8d6178SJean-Philippe Brucker const char *uid = kasprintf(GFP_KERNEL, "%u", node->identifier);
43512f8d6178SJean-Philippe Brucker struct acpi_device *adev;
43522f8d6178SJean-Philippe Brucker
43532f8d6178SJean-Philippe Brucker /* Look for an NVDA200C node whose _UID matches the SMMU node ID */
43542f8d6178SJean-Philippe Brucker adev = acpi_dev_get_first_match_dev("NVDA200C", uid, -1);
43552f8d6178SJean-Philippe Brucker if (adev) {
43562f8d6178SJean-Philippe Brucker /* Tegra241 CMDQV driver is responsible for put_device() */
4357e86d1aa8SWill Deacon smmu->impl_dev = &adev->dev;
4358e86d1aa8SWill Deacon smmu->options |= ARM_SMMU_OPT_TEGRA241_CMDQV;
4359e86d1aa8SWill Deacon dev_info(smmu->dev, "found companion CMDQV device: %s\n",
4360e86d1aa8SWill Deacon dev_name(smmu->impl_dev));
4361e86d1aa8SWill Deacon }
4362e86d1aa8SWill Deacon kfree(uid);
4363e86d1aa8SWill Deacon }
4364e86d1aa8SWill Deacon #else
acpi_smmu_dsdt_probe_tegra241_cmdqv(struct acpi_iort_node * node,struct arm_smmu_device * smmu)4365e86d1aa8SWill Deacon static void acpi_smmu_dsdt_probe_tegra241_cmdqv(struct acpi_iort_node *node,
4366e86d1aa8SWill Deacon struct arm_smmu_device *smmu)
4367e86d1aa8SWill Deacon {
4368e86d1aa8SWill Deacon }
4369e86d1aa8SWill Deacon #endif
4370e86d1aa8SWill Deacon
acpi_smmu_iort_probe_model(struct acpi_iort_node * node,struct arm_smmu_device * smmu)4371e86d1aa8SWill Deacon static int acpi_smmu_iort_probe_model(struct acpi_iort_node *node,
4372e86d1aa8SWill Deacon struct arm_smmu_device *smmu)
4373e86d1aa8SWill Deacon {
4374e86d1aa8SWill Deacon struct acpi_iort_smmu_v3 *iort_smmu =
4375e86d1aa8SWill Deacon (struct acpi_iort_smmu_v3 *)node->node_data;
4376e86d1aa8SWill Deacon
4377e86d1aa8SWill Deacon switch (iort_smmu->model) {
4378e86d1aa8SWill Deacon case ACPI_IORT_SMMU_V3_CAVIUM_CN99XX:
4379e86d1aa8SWill Deacon smmu->options |= ARM_SMMU_OPT_PAGE0_REGS_ONLY;
4380e86d1aa8SWill Deacon break;
4381e86d1aa8SWill Deacon case ACPI_IORT_SMMU_V3_HISILICON_HI161X:
4382e86d1aa8SWill Deacon smmu->options |= ARM_SMMU_OPT_SKIP_PREFETCH;
4383e86d1aa8SWill Deacon break;
4384e86d1aa8SWill Deacon case ACPI_IORT_SMMU_V3_GENERIC:
4385e86d1aa8SWill Deacon /*
4386e86d1aa8SWill Deacon * Tegra241 implementation stores its SMMU options and impl_dev
4387e86d1aa8SWill Deacon * in DSDT. Thus, go through the ACPI tables unconditionally.
4388e86d1aa8SWill Deacon */
4389e86d1aa8SWill Deacon acpi_smmu_dsdt_probe_tegra241_cmdqv(node, smmu);
4390e86d1aa8SWill Deacon break;
4391e86d1aa8SWill Deacon }
4392e86d1aa8SWill Deacon
4393e86d1aa8SWill Deacon dev_notice(smmu->dev, "option mask 0x%x\n", smmu->options);
4394e86d1aa8SWill Deacon return 0;
4395e86d1aa8SWill Deacon }
4396e86d1aa8SWill Deacon
arm_smmu_device_acpi_probe(struct platform_device * pdev,struct arm_smmu_device * smmu)4397e86d1aa8SWill Deacon static int arm_smmu_device_acpi_probe(struct platform_device *pdev,
4398e86d1aa8SWill Deacon struct arm_smmu_device *smmu)
4399e86d1aa8SWill Deacon {
4400932bc8c7SZhen Lei struct acpi_iort_smmu_v3 *iort_smmu;
4401e86d1aa8SWill Deacon struct device *dev = smmu->dev;
4402e86d1aa8SWill Deacon struct acpi_iort_node *node;
4403e86d1aa8SWill Deacon
4404e86d1aa8SWill Deacon node = *(struct acpi_iort_node **)dev_get_platdata(dev);
44059bdbdaa3SShameer Kolothum
44069bdbdaa3SShameer Kolothum /* Retrieve SMMUv3 specific data */
44079bdbdaa3SShameer Kolothum iort_smmu = (struct acpi_iort_smmu_v3 *)node->node_data;
44089bdbdaa3SShameer Kolothum
44099bdbdaa3SShameer Kolothum if (iort_smmu->flags & ACPI_IORT_SMMU_V3_COHACC_OVERRIDE)
44109bdbdaa3SShameer Kolothum smmu->features |= ARM_SMMU_FEAT_COHERENCY;
44119bdbdaa3SShameer Kolothum
44129bdbdaa3SShameer Kolothum switch (FIELD_GET(ACPI_IORT_SMMU_V3_HTTU_OVERRIDE, iort_smmu->flags)) {
44139bdbdaa3SShameer Kolothum case IDR0_HTTU_ACCESS_DIRTY:
44149bdbdaa3SShameer Kolothum smmu->features |= ARM_SMMU_FEAT_HD;
44159bdbdaa3SShameer Kolothum fallthrough;
44169bdbdaa3SShameer Kolothum case IDR0_HTTU_ACCESS:
44179bdbdaa3SShameer Kolothum smmu->features |= ARM_SMMU_FEAT_HA;
44189bdbdaa3SShameer Kolothum }
44199bdbdaa3SShameer Kolothum
44209bdbdaa3SShameer Kolothum return acpi_smmu_iort_probe_model(node, smmu);
44219bdbdaa3SShameer Kolothum }
44229bdbdaa3SShameer Kolothum #else
arm_smmu_device_acpi_probe(struct platform_device * pdev,struct arm_smmu_device * smmu)44239bdbdaa3SShameer Kolothum static inline int arm_smmu_device_acpi_probe(struct platform_device *pdev,
44249bdbdaa3SShameer Kolothum struct arm_smmu_device *smmu)
44259bdbdaa3SShameer Kolothum {
44267686aa5fSJason Gunthorpe return -ENODEV;
44277686aa5fSJason Gunthorpe }
44287686aa5fSJason Gunthorpe #endif
44297686aa5fSJason Gunthorpe
arm_smmu_device_dt_probe(struct platform_device * pdev,struct arm_smmu_device * smmu)4430ec9098d6SMostafa Saleh static int arm_smmu_device_dt_probe(struct platform_device *pdev,
44317686aa5fSJason Gunthorpe struct arm_smmu_device *smmu)
44329bdbdaa3SShameer Kolothum {
44339bdbdaa3SShameer Kolothum struct device *dev = &pdev->dev;
44349bdbdaa3SShameer Kolothum u32 cells;
44359bdbdaa3SShameer Kolothum int ret = -EINVAL;
44369bdbdaa3SShameer Kolothum
44379bdbdaa3SShameer Kolothum if (of_property_read_u32(dev->of_node, "#iommu-cells", &cells))
4438e86d1aa8SWill Deacon dev_err(dev, "missing #iommu-cells property\n");
4439e86d1aa8SWill Deacon else if (cells != 1)
4440e86d1aa8SWill Deacon dev_err(dev, "invalid #iommu-cells value (%d)\n", cells);
4441e86d1aa8SWill Deacon else
4442e86d1aa8SWill Deacon ret = 0;
4443e86d1aa8SWill Deacon
4444e86d1aa8SWill Deacon parse_driver_options(smmu);
4445e86d1aa8SWill Deacon
4446e86d1aa8SWill Deacon if (of_dma_is_coherent(dev->of_node))
4447affa9095SZhen Lei smmu->features |= ARM_SMMU_FEAT_COHERENCY;
4448e86d1aa8SWill Deacon
4449e86d1aa8SWill Deacon return ret;
4450e86d1aa8SWill Deacon }
4451e86d1aa8SWill Deacon
arm_smmu_resource_size(struct arm_smmu_device * smmu)4452e86d1aa8SWill Deacon static unsigned long arm_smmu_resource_size(struct arm_smmu_device *smmu)
4453e86d1aa8SWill Deacon {
4454e86d1aa8SWill Deacon if (smmu->options & ARM_SMMU_OPT_PAGE0_REGS_ONLY)
4455e86d1aa8SWill Deacon return SZ_64K;
4456734554fdSRobin Murphy else
4457734554fdSRobin Murphy return SZ_128K;
4458e86d1aa8SWill Deacon }
4459e86d1aa8SWill Deacon
arm_smmu_ioremap(struct device * dev,resource_size_t start,resource_size_t size)4460e86d1aa8SWill Deacon static void __iomem *arm_smmu_ioremap(struct device *dev, resource_size_t start,
4461b131fa8cSYang Yingliang resource_size_t size)
4462b131fa8cSYang Yingliang {
4463e86d1aa8SWill Deacon struct resource res = DEFINE_RES_MEM(start, size);
4464e86d1aa8SWill Deacon
4465e86d1aa8SWill Deacon return devm_ioremap_resource(dev, &res);
4466e86d1aa8SWill Deacon }
4467e86d1aa8SWill Deacon
arm_smmu_rmr_install_bypass_ste(struct arm_smmu_device * smmu)4468e86d1aa8SWill Deacon static void arm_smmu_rmr_install_bypass_ste(struct arm_smmu_device *smmu)
4469e86d1aa8SWill Deacon {
4470e86d1aa8SWill Deacon struct list_head rmr_list;
4471e86d1aa8SWill Deacon struct iommu_resv_region *e;
4472e86d1aa8SWill Deacon
4473e86d1aa8SWill Deacon INIT_LIST_HEAD(&rmr_list);
4474e86d1aa8SWill Deacon iort_get_rmr_sids(dev_fwnode(smmu->dev), &rmr_list);
4475e86d1aa8SWill Deacon
4476e86d1aa8SWill Deacon list_for_each_entry(e, &rmr_list, list) {
4477e86d1aa8SWill Deacon struct iommu_iort_rmr_data *rmr;
4478e86d1aa8SWill Deacon int ret, i;
4479e86d1aa8SWill Deacon
4480e86d1aa8SWill Deacon rmr = container_of(e, struct iommu_iort_rmr_data, rr);
4481e86d1aa8SWill Deacon for (i = 0; i < rmr->num_sids; i++) {
4482e86d1aa8SWill Deacon ret = arm_smmu_init_sid_strtab(smmu, rmr->sids[i]);
4483e86d1aa8SWill Deacon if (ret) {
4484e86d1aa8SWill Deacon dev_err(smmu->dev, "RMR SID(0x%x) bypass failed\n",
4485e86d1aa8SWill Deacon rmr->sids[i]);
4486e86d1aa8SWill Deacon continue;
4487e86d1aa8SWill Deacon }
4488e86d1aa8SWill Deacon
4489e86d1aa8SWill Deacon /*
4490e86d1aa8SWill Deacon * STE table is not programmed to HW, see
4491e86d1aa8SWill Deacon * arm_smmu_initial_bypass_stes()
4492e86d1aa8SWill Deacon */
4493e86d1aa8SWill Deacon arm_smmu_make_bypass_ste(smmu,
4494e86d1aa8SWill Deacon arm_smmu_get_step_for_sid(smmu, rmr->sids[i]));
4495e86d1aa8SWill Deacon }
4496e86d1aa8SWill Deacon }
4497e86d1aa8SWill Deacon
4498e86d1aa8SWill Deacon iort_put_rmr_sids(dev_fwnode(smmu->dev), &rmr_list);
4499e86d1aa8SWill Deacon }
4500e86d1aa8SWill Deacon
arm_smmu_impl_remove(void * data)4501e86d1aa8SWill Deacon static void arm_smmu_impl_remove(void *data)
4502e86d1aa8SWill Deacon {
4503e86d1aa8SWill Deacon struct arm_smmu_device *smmu = data;
4504e86d1aa8SWill Deacon
4505e86d1aa8SWill Deacon if (smmu->impl_ops && smmu->impl_ops->device_remove)
4506e86d1aa8SWill Deacon smmu->impl_ops->device_remove(smmu);
4507e86d1aa8SWill Deacon }
4508e86d1aa8SWill Deacon
4509e86d1aa8SWill Deacon /*
4510e86d1aa8SWill Deacon * Probe all the compiled in implementations. Each one checks to see if it
4511e86d1aa8SWill Deacon * matches this HW and if so returns a devm_krealloc'd arm_smmu_device which
4512e86d1aa8SWill Deacon * replaces the callers. Otherwise the original is returned or ERR_PTR.
4513e86d1aa8SWill Deacon */
arm_smmu_impl_probe(struct arm_smmu_device * smmu)4514e86d1aa8SWill Deacon static struct arm_smmu_device *arm_smmu_impl_probe(struct arm_smmu_device *smmu)
4515e86d1aa8SWill Deacon {
4516e86d1aa8SWill Deacon struct arm_smmu_device *new_smmu = ERR_PTR(-ENODEV);
45179bdbdaa3SShameer Kolothum int ret;
45189bdbdaa3SShameer Kolothum
45199bdbdaa3SShameer Kolothum if (smmu->impl_dev && (smmu->options & ARM_SMMU_OPT_TEGRA241_CMDQV))
4520e86d1aa8SWill Deacon new_smmu = tegra241_cmdqv_probe(smmu);
4521734554fdSRobin Murphy
4522e86d1aa8SWill Deacon if (new_smmu == ERR_PTR(-ENODEV))
4523e86d1aa8SWill Deacon return smmu;
4524e86d1aa8SWill Deacon if (IS_ERR(new_smmu))
4525e86d1aa8SWill Deacon return new_smmu;
4526e86d1aa8SWill Deacon
4527e86d1aa8SWill Deacon ret = devm_add_action_or_reset(new_smmu->dev, arm_smmu_impl_remove,
4528e86d1aa8SWill Deacon new_smmu);
4529e86d1aa8SWill Deacon if (ret)
4530e86d1aa8SWill Deacon return ERR_PTR(ret);
45312d471b20SRobin Murphy return new_smmu;
4532e86d1aa8SWill Deacon }
4533e86d1aa8SWill Deacon
arm_smmu_device_probe(struct platform_device * pdev)4534249c9dc6SAmey Narkhede static int arm_smmu_device_probe(struct platform_device *pdev)
4535249c9dc6SAmey Narkhede {
4536e86d1aa8SWill Deacon int irq, ret;
4537e86d1aa8SWill Deacon struct resource *res;
45382efbd29bSRobin Murphy resource_size_t ioaddr;
45392efbd29bSRobin Murphy struct arm_smmu_device *smmu;
45402efbd29bSRobin Murphy struct device *dev = &pdev->dev;
454166c7076fSUwe Kleine-König
4542e86d1aa8SWill Deacon smmu = devm_kzalloc(dev, sizeof(*smmu), GFP_KERNEL);
4543e86d1aa8SWill Deacon if (!smmu)
4544e86d1aa8SWill Deacon return -ENOMEM;
4545e86d1aa8SWill Deacon smmu->dev = dev;
4546e86d1aa8SWill Deacon
4547e86d1aa8SWill Deacon if (dev->of_node) {
4548395ad89dSJean-Philippe Brucker ret = arm_smmu_device_dt_probe(pdev, smmu);
45491672730cSDawei Li } else {
4550e86d1aa8SWill Deacon ret = arm_smmu_device_acpi_probe(pdev, smmu);
4551e86d1aa8SWill Deacon }
4552e86d1aa8SWill Deacon if (ret)
4553e86d1aa8SWill Deacon return ret;
455432ea2c57SVladimir Oltean
455532ea2c57SVladimir Oltean smmu = arm_smmu_impl_probe(smmu);
455632ea2c57SVladimir Oltean if (IS_ERR(smmu))
4557e86d1aa8SWill Deacon return PTR_ERR(smmu);
4558e86d1aa8SWill Deacon
4559e86d1aa8SWill Deacon /* Base address */
4560e86d1aa8SWill Deacon res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
4561e86d1aa8SWill Deacon if (!res)
4562e86d1aa8SWill Deacon return -EINVAL;
4563e86d1aa8SWill Deacon if (resource_size(res) < arm_smmu_resource_size(smmu)) {
4564e86d1aa8SWill Deacon dev_err(dev, "MMIO region too small (%pr)\n", res);
456532784a95SJean-Philippe Brucker return -EINVAL;
456632784a95SJean-Philippe Brucker }
456732784a95SJean-Philippe Brucker ioaddr = res->start;
456832784a95SJean-Philippe Brucker
456932784a95SJean-Philippe Brucker /*
457032784a95SJean-Philippe Brucker * Don't map the IMPLEMENTATION DEFINED regions, since they may contain
4571e86d1aa8SWill Deacon * the PMCG registers which are reserved by the PMU driver.
4572e86d1aa8SWill Deacon */
4573e86d1aa8SWill Deacon smmu->base = arm_smmu_ioremap(dev, ioaddr, ARM_SMMU_REG_SZ);
4574e86d1aa8SWill Deacon if (IS_ERR(smmu->base))
4575e86d1aa8SWill Deacon return PTR_ERR(smmu->base);
4576e86d1aa8SWill Deacon
4577e86d1aa8SWill Deacon if (arm_smmu_resource_size(smmu) > SZ_64K) {
457866c7076fSUwe Kleine-König smmu->page1 = arm_smmu_ioremap(dev, ioaddr + SZ_64K,
4579e86d1aa8SWill Deacon ARM_SMMU_REG_SZ);
4580e86d1aa8SWill Deacon if (IS_ERR(smmu->page1))
458132784a95SJean-Philippe Brucker return PTR_ERR(smmu->page1);
458232784a95SJean-Philippe Brucker } else {
4583e86d1aa8SWill Deacon smmu->page1 = smmu->base;
4584e86d1aa8SWill Deacon }
4585e86d1aa8SWill Deacon
4586e86d1aa8SWill Deacon /* Interrupt lines */
4587e86d1aa8SWill Deacon
4588 irq = platform_get_irq_byname_optional(pdev, "combined");
4589 if (irq > 0)
4590 smmu->combined_irq = irq;
4591 else {
4592 irq = platform_get_irq_byname_optional(pdev, "eventq");
4593 if (irq > 0)
4594 smmu->evtq.q.irq = irq;
4595
4596 irq = platform_get_irq_byname_optional(pdev, "priq");
4597 if (irq > 0)
4598 smmu->priq.q.irq = irq;
4599
4600 irq = platform_get_irq_byname_optional(pdev, "gerror");
4601 if (irq > 0)
4602 smmu->gerr_irq = irq;
4603 }
4604 /* Probe the h/w */
4605 ret = arm_smmu_device_hw_probe(smmu);
4606 if (ret)
4607 return ret;
4608
4609 /* Initialise in-memory data structures */
4610 ret = arm_smmu_init_structures(smmu);
4611 if (ret)
4612 return ret;
4613
4614 /* Record our private device structure */
4615 platform_set_drvdata(pdev, smmu);
4616
4617 /* Check for RMRs and install bypass STEs if any */
4618 arm_smmu_rmr_install_bypass_ste(smmu);
4619
4620 /* Reset the device */
4621 ret = arm_smmu_device_reset(smmu);
4622 if (ret)
4623 return ret;
4624
4625 /* And we're up. Go go go! */
4626 ret = iommu_device_sysfs_add(&smmu->iommu, dev, NULL,
4627 "smmu3.%pa", &ioaddr);
4628 if (ret)
4629 return ret;
4630
4631 ret = iommu_device_register(&smmu->iommu, &arm_smmu_ops, dev);
4632 if (ret) {
4633 dev_err(dev, "Failed to register iommu\n");
4634 iommu_device_sysfs_remove(&smmu->iommu);
4635 return ret;
4636 }
4637
4638 return 0;
4639 }
4640
arm_smmu_device_remove(struct platform_device * pdev)4641 static void arm_smmu_device_remove(struct platform_device *pdev)
4642 {
4643 struct arm_smmu_device *smmu = platform_get_drvdata(pdev);
4644
4645 iommu_device_unregister(&smmu->iommu);
4646 iommu_device_sysfs_remove(&smmu->iommu);
4647 arm_smmu_device_disable(smmu);
4648 iopf_queue_free(smmu->evtq.iopf);
4649 ida_destroy(&smmu->vmid_map);
4650 }
4651
arm_smmu_device_shutdown(struct platform_device * pdev)4652 static void arm_smmu_device_shutdown(struct platform_device *pdev)
4653 {
4654 struct arm_smmu_device *smmu = platform_get_drvdata(pdev);
4655
4656 arm_smmu_device_disable(smmu);
4657 }
4658
4659 static const struct of_device_id arm_smmu_of_match[] = {
4660 { .compatible = "arm,smmu-v3", },
4661 { },
4662 };
4663 MODULE_DEVICE_TABLE(of, arm_smmu_of_match);
4664
arm_smmu_driver_unregister(struct platform_driver * drv)4665 static void arm_smmu_driver_unregister(struct platform_driver *drv)
4666 {
4667 arm_smmu_sva_notifier_synchronize();
4668 platform_driver_unregister(drv);
4669 }
4670
4671 static struct platform_driver arm_smmu_driver = {
4672 .driver = {
4673 .name = "arm-smmu-v3",
4674 .of_match_table = arm_smmu_of_match,
4675 .suppress_bind_attrs = true,
4676 },
4677 .probe = arm_smmu_device_probe,
4678 .remove_new = arm_smmu_device_remove,
4679 .shutdown = arm_smmu_device_shutdown,
4680 };
4681 module_driver(arm_smmu_driver, platform_driver_register,
4682 arm_smmu_driver_unregister);
4683
4684 MODULE_DESCRIPTION("IOMMU API for ARM architected SMMUv3 implementations");
4685 MODULE_AUTHOR("Will Deacon <will@kernel.org>");
4686 MODULE_ALIAS("platform:arm-smmu-v3");
4687 MODULE_LICENSE("GPL v2");
4688