1 // SPDX-License-Identifier: GPL-2.0-only
2 /* Copyright (C) 2021-2024 NVIDIA CORPORATION & AFFILIATES. */
3
4 #define dev_fmt(fmt) "tegra241_cmdqv: " fmt
5
6 #include <linux/debugfs.h>
7 #include <linux/dma-mapping.h>
8 #include <linux/interrupt.h>
9 #include <linux/iommu.h>
10 #include <linux/iommufd.h>
11 #include <linux/iopoll.h>
12 #include <linux/platform_device.h>
13 #include <uapi/linux/iommufd.h>
14
15 #include "arm-smmu-v3.h"
16
17 /* CMDQV register page base and size defines */
18 #define TEGRA241_CMDQV_CONFIG_BASE (0)
19 #define TEGRA241_CMDQV_CONFIG_SIZE (SZ_64K)
20 #define TEGRA241_VCMDQ_PAGE0_BASE (TEGRA241_CMDQV_CONFIG_BASE + SZ_64K)
21 #define TEGRA241_VCMDQ_PAGE1_BASE (TEGRA241_VCMDQ_PAGE0_BASE + SZ_64K)
22 #define TEGRA241_VINTF_PAGE_BASE (TEGRA241_VCMDQ_PAGE1_BASE + SZ_64K)
23
24 /* CMDQV global base regs */
25 #define TEGRA241_CMDQV_CONFIG 0x0000
26 #define CMDQV_EN BIT(0)
27
28 #define TEGRA241_CMDQV_PARAM 0x0004
29 #define CMDQV_NUM_SID_PER_VM_LOG2 GENMASK(15, 12)
30 #define CMDQV_NUM_VINTF_LOG2 GENMASK(11, 8)
31 #define CMDQV_NUM_VCMDQ_LOG2 GENMASK(7, 4)
32 #define CMDQV_VER GENMASK(3, 0)
33
34 #define TEGRA241_CMDQV_STATUS 0x0008
35 #define CMDQV_ENABLED BIT(0)
36
37 #define TEGRA241_CMDQV_VINTF_ERR_MAP 0x0014
38 #define TEGRA241_CMDQV_VINTF_INT_MASK 0x001C
39 #define TEGRA241_CMDQV_CMDQ_ERR_MAP(m) (0x0024 + 0x4*(m))
40
41 #define TEGRA241_CMDQV_CMDQ_ALLOC(q) (0x0200 + 0x4*(q))
42 #define CMDQV_CMDQ_ALLOC_VINTF GENMASK(20, 15)
43 #define CMDQV_CMDQ_ALLOC_LVCMDQ GENMASK(7, 1)
44 #define CMDQV_CMDQ_ALLOCATED BIT(0)
45
46 /* VINTF base regs */
47 #define TEGRA241_VINTF(v) (0x1000 + 0x100*(v))
48
49 #define TEGRA241_VINTF_CONFIG 0x0000
50 #define VINTF_HYP_OWN BIT(17)
51 #define VINTF_VMID GENMASK(16, 1)
52 #define VINTF_EN BIT(0)
53
54 #define TEGRA241_VINTF_STATUS 0x0004
55 #define VINTF_STATUS GENMASK(3, 1)
56 #define VINTF_ENABLED BIT(0)
57
58 #define TEGRA241_VINTF_SID_MATCH(s) (0x0040 + 0x4*(s))
59 #define TEGRA241_VINTF_SID_REPLACE(s) (0x0080 + 0x4*(s))
60
61 #define TEGRA241_VINTF_LVCMDQ_ERR_MAP_64(m) \
62 (0x00C0 + 0x8*(m))
63 #define LVCMDQ_ERR_MAP_NUM_64 2
64
65 /* VCMDQ base regs */
66 /* -- PAGE0 -- */
67 #define TEGRA241_VCMDQ_PAGE0(q) (TEGRA241_VCMDQ_PAGE0_BASE + 0x80*(q))
68
69 #define TEGRA241_VCMDQ_CONS 0x00000
70 #define VCMDQ_CONS_ERR GENMASK(30, 24)
71
72 #define TEGRA241_VCMDQ_PROD 0x00004
73
74 #define TEGRA241_VCMDQ_CONFIG 0x00008
75 #define VCMDQ_EN BIT(0)
76
77 #define TEGRA241_VCMDQ_STATUS 0x0000C
78 #define VCMDQ_ENABLED BIT(0)
79
80 #define TEGRA241_VCMDQ_GERROR 0x00010
81 #define TEGRA241_VCMDQ_GERRORN 0x00014
82
83 /* -- PAGE1 -- */
84 #define TEGRA241_VCMDQ_PAGE1(q) (TEGRA241_VCMDQ_PAGE1_BASE + 0x80*(q))
85 #define VCMDQ_ADDR GENMASK(47, 5)
86 #define VCMDQ_LOG2SIZE GENMASK(4, 0)
87
88 #define TEGRA241_VCMDQ_BASE 0x00000
89 #define TEGRA241_VCMDQ_CONS_INDX_BASE 0x00008
90
91 /* VINTF logical-VCMDQ pages */
92 #define TEGRA241_VINTFi_PAGE0(i) (TEGRA241_VINTF_PAGE_BASE + SZ_128K*(i))
93 #define TEGRA241_VINTFi_PAGE1(i) (TEGRA241_VINTFi_PAGE0(i) + SZ_64K)
94 #define TEGRA241_VINTFi_LVCMDQ_PAGE0(i, q) \
95 (TEGRA241_VINTFi_PAGE0(i) + 0x80*(q))
96 #define TEGRA241_VINTFi_LVCMDQ_PAGE1(i, q) \
97 (TEGRA241_VINTFi_PAGE1(i) + 0x80*(q))
98
99 /* MMIO helpers */
100 #define REG_CMDQV(_cmdqv, _regname) \
101 ((_cmdqv)->base + TEGRA241_CMDQV_##_regname)
102 #define REG_VINTF(_vintf, _regname) \
103 ((_vintf)->base + TEGRA241_VINTF_##_regname)
104 #define REG_VCMDQ_PAGE0(_vcmdq, _regname) \
105 ((_vcmdq)->page0 + TEGRA241_VCMDQ_##_regname)
106 #define REG_VCMDQ_PAGE1(_vcmdq, _regname) \
107 ((_vcmdq)->page1 + TEGRA241_VCMDQ_##_regname)
108
109
110 static bool disable_cmdqv;
111 module_param(disable_cmdqv, bool, 0444);
112 MODULE_PARM_DESC(disable_cmdqv,
113 "This allows to disable CMDQV HW and use default SMMU internal CMDQ.");
114
115 static bool bypass_vcmdq;
116 module_param(bypass_vcmdq, bool, 0444);
117 MODULE_PARM_DESC(bypass_vcmdq,
118 "This allows to bypass VCMDQ for debugging use or perf comparison.");
119
120 /**
121 * struct tegra241_vcmdq - Virtual Command Queue
122 * @core: Embedded iommufd_hw_queue structure
123 * @idx: Global index in the CMDQV
124 * @lidx: Local index in the VINTF
125 * @enabled: Enable status
126 * @cmdqv: Parent CMDQV pointer
127 * @vintf: Parent VINTF pointer
128 * @prev: Previous LVCMDQ to depend on
129 * @cmdq: Command Queue struct
130 * @page0: MMIO Page0 base address
131 * @page1: MMIO Page1 base address
132 */
133 struct tegra241_vcmdq {
134 struct iommufd_hw_queue core;
135
136 u16 idx;
137 u16 lidx;
138
139 bool enabled;
140
141 struct tegra241_cmdqv *cmdqv;
142 struct tegra241_vintf *vintf;
143 struct tegra241_vcmdq *prev;
144 struct arm_smmu_cmdq cmdq;
145
146 void __iomem *page0;
147 void __iomem *page1;
148 };
149 #define hw_queue_to_vcmdq(v) container_of(v, struct tegra241_vcmdq, core)
150
151 /**
152 * struct tegra241_vintf - Virtual Interface
153 * @vsmmu: Embedded arm_vsmmu structure
154 * @idx: Global index in the CMDQV
155 * @enabled: Enable status
156 * @hyp_own: Owned by hypervisor (in-kernel)
157 * @cmdqv: Parent CMDQV pointer
158 * @lvcmdqs: List of logical VCMDQ pointers
159 * @lvcmdq_mutex: Lock to serialize user-allocated lvcmdqs
160 * @base: MMIO base address
161 * @mmap_offset: Offset argument for mmap() syscall
162 * @sids: Stream ID mapping resources
163 */
164 struct tegra241_vintf {
165 struct arm_vsmmu vsmmu;
166
167 u16 idx;
168
169 bool enabled;
170 bool hyp_own;
171
172 struct tegra241_cmdqv *cmdqv;
173 struct tegra241_vcmdq **lvcmdqs;
174 struct mutex lvcmdq_mutex; /* user space race */
175
176 void __iomem *base;
177 unsigned long mmap_offset;
178
179 struct ida sids;
180 };
181 #define viommu_to_vintf(v) container_of(v, struct tegra241_vintf, vsmmu.core)
182
183 /**
184 * struct tegra241_vintf_sid - Virtual Interface Stream ID Mapping
185 * @core: Embedded iommufd_vdevice structure, holding virtual Stream ID
186 * @vintf: Parent VINTF pointer
187 * @sid: Physical Stream ID
188 * @idx: Mapping index in the VINTF
189 */
190 struct tegra241_vintf_sid {
191 struct iommufd_vdevice core;
192 struct tegra241_vintf *vintf;
193 u32 sid;
194 u8 idx;
195 };
196 #define vdev_to_vsid(v) container_of(v, struct tegra241_vintf_sid, core)
197
198 /**
199 * struct tegra241_cmdqv - CMDQ-V for SMMUv3
200 * @smmu: SMMUv3 device
201 * @dev: CMDQV device
202 * @base: MMIO base address
203 * @base_phys: MMIO physical base address, for mmap
204 * @irq: IRQ number
205 * @num_vintfs: Total number of VINTFs
206 * @num_vcmdqs: Total number of VCMDQs
207 * @num_lvcmdqs_per_vintf: Number of logical VCMDQs per VINTF
208 * @num_sids_per_vintf: Total number of SID mappings per VINTF
209 * @vintf_ids: VINTF id allocator
210 * @vintfs: List of VINTFs
211 */
212 struct tegra241_cmdqv {
213 struct arm_smmu_device smmu;
214 struct device *dev;
215
216 void __iomem *base;
217 phys_addr_t base_phys;
218 int irq;
219
220 /* CMDQV Hardware Params */
221 u16 num_vintfs;
222 u16 num_vcmdqs;
223 u16 num_lvcmdqs_per_vintf;
224 u16 num_sids_per_vintf;
225
226 struct ida vintf_ids;
227
228 struct tegra241_vintf **vintfs;
229 };
230
231 /* Config and Polling Helpers */
232
tegra241_cmdqv_write_config(struct tegra241_cmdqv * cmdqv,void __iomem * addr_config,void __iomem * addr_status,u32 regval,const char * header,bool * out_enabled)233 static inline int tegra241_cmdqv_write_config(struct tegra241_cmdqv *cmdqv,
234 void __iomem *addr_config,
235 void __iomem *addr_status,
236 u32 regval, const char *header,
237 bool *out_enabled)
238 {
239 bool en = regval & BIT(0);
240 int ret;
241
242 writel(regval, addr_config);
243 ret = readl_poll_timeout(addr_status, regval,
244 en ? regval & BIT(0) : !(regval & BIT(0)),
245 1, ARM_SMMU_POLL_TIMEOUT_US);
246 if (ret)
247 dev_err(cmdqv->dev, "%sfailed to %sable, STATUS=0x%08X\n",
248 header, en ? "en" : "dis", regval);
249 if (out_enabled)
250 WRITE_ONCE(*out_enabled, regval & BIT(0));
251 return ret;
252 }
253
cmdqv_write_config(struct tegra241_cmdqv * cmdqv,u32 regval)254 static inline int cmdqv_write_config(struct tegra241_cmdqv *cmdqv, u32 regval)
255 {
256 return tegra241_cmdqv_write_config(cmdqv,
257 REG_CMDQV(cmdqv, CONFIG),
258 REG_CMDQV(cmdqv, STATUS),
259 regval, "CMDQV: ", NULL);
260 }
261
vintf_write_config(struct tegra241_vintf * vintf,u32 regval)262 static inline int vintf_write_config(struct tegra241_vintf *vintf, u32 regval)
263 {
264 char header[16];
265
266 snprintf(header, 16, "VINTF%u: ", vintf->idx);
267 return tegra241_cmdqv_write_config(vintf->cmdqv,
268 REG_VINTF(vintf, CONFIG),
269 REG_VINTF(vintf, STATUS),
270 regval, header, &vintf->enabled);
271 }
272
lvcmdq_error_header(struct tegra241_vcmdq * vcmdq,char * header,int hlen)273 static inline char *lvcmdq_error_header(struct tegra241_vcmdq *vcmdq,
274 char *header, int hlen)
275 {
276 WARN_ON(hlen < 64);
277 if (WARN_ON(!vcmdq->vintf))
278 return "";
279 snprintf(header, hlen, "VINTF%u: VCMDQ%u/LVCMDQ%u: ",
280 vcmdq->vintf->idx, vcmdq->idx, vcmdq->lidx);
281 return header;
282 }
283
vcmdq_write_config(struct tegra241_vcmdq * vcmdq,u32 regval)284 static inline int vcmdq_write_config(struct tegra241_vcmdq *vcmdq, u32 regval)
285 {
286 char header[64], *h = lvcmdq_error_header(vcmdq, header, 64);
287
288 return tegra241_cmdqv_write_config(vcmdq->cmdqv,
289 REG_VCMDQ_PAGE0(vcmdq, CONFIG),
290 REG_VCMDQ_PAGE0(vcmdq, STATUS),
291 regval, h, &vcmdq->enabled);
292 }
293
294 /* ISR Functions */
295
tegra241_vintf_user_handle_error(struct tegra241_vintf * vintf)296 static void tegra241_vintf_user_handle_error(struct tegra241_vintf *vintf)
297 {
298 struct iommufd_viommu *viommu = &vintf->vsmmu.core;
299 struct iommu_vevent_tegra241_cmdqv vevent_data;
300 int i;
301
302 for (i = 0; i < LVCMDQ_ERR_MAP_NUM_64; i++) {
303 u64 err = readq_relaxed(REG_VINTF(vintf, LVCMDQ_ERR_MAP_64(i)));
304
305 vevent_data.lvcmdq_err_map[i] = cpu_to_le64(err);
306 }
307
308 iommufd_viommu_report_event(viommu, IOMMU_VEVENTQ_TYPE_TEGRA241_CMDQV,
309 &vevent_data, sizeof(vevent_data));
310 }
311
tegra241_vintf0_handle_error(struct tegra241_vintf * vintf)312 static void tegra241_vintf0_handle_error(struct tegra241_vintf *vintf)
313 {
314 int i;
315
316 for (i = 0; i < LVCMDQ_ERR_MAP_NUM_64; i++) {
317 u64 map = readq_relaxed(REG_VINTF(vintf, LVCMDQ_ERR_MAP_64(i)));
318
319 while (map) {
320 unsigned long lidx = __ffs64(map);
321 struct tegra241_vcmdq *vcmdq = vintf->lvcmdqs[lidx];
322 u32 gerror = readl_relaxed(REG_VCMDQ_PAGE0(vcmdq, GERROR));
323
324 __arm_smmu_cmdq_skip_err(&vintf->cmdqv->smmu, &vcmdq->cmdq);
325 writel(gerror, REG_VCMDQ_PAGE0(vcmdq, GERRORN));
326 map &= ~BIT_ULL(lidx);
327 }
328 }
329 }
330
tegra241_cmdqv_isr(int irq,void * devid)331 static irqreturn_t tegra241_cmdqv_isr(int irq, void *devid)
332 {
333 struct tegra241_cmdqv *cmdqv = (struct tegra241_cmdqv *)devid;
334 void __iomem *reg_vintf_map = REG_CMDQV(cmdqv, VINTF_ERR_MAP);
335 char err_str[256];
336 u64 vintf_map;
337
338 /* Use readl_relaxed() as register addresses are not 64-bit aligned */
339 vintf_map = (u64)readl_relaxed(reg_vintf_map + 0x4) << 32 |
340 (u64)readl_relaxed(reg_vintf_map);
341
342 snprintf(err_str, sizeof(err_str),
343 "vintf_map: %016llx, vcmdq_map %08x:%08x:%08x:%08x", vintf_map,
344 readl_relaxed(REG_CMDQV(cmdqv, CMDQ_ERR_MAP(3))),
345 readl_relaxed(REG_CMDQV(cmdqv, CMDQ_ERR_MAP(2))),
346 readl_relaxed(REG_CMDQV(cmdqv, CMDQ_ERR_MAP(1))),
347 readl_relaxed(REG_CMDQV(cmdqv, CMDQ_ERR_MAP(0))));
348
349 dev_warn(cmdqv->dev, "unexpected error reported. %s\n", err_str);
350
351 /* Handle VINTF0 and its LVCMDQs */
352 if (vintf_map & BIT_ULL(0)) {
353 tegra241_vintf0_handle_error(cmdqv->vintfs[0]);
354 vintf_map &= ~BIT_ULL(0);
355 }
356
357 /* Handle other user VINTFs and their LVCMDQs */
358 while (vintf_map) {
359 unsigned long idx = __ffs64(vintf_map);
360
361 tegra241_vintf_user_handle_error(cmdqv->vintfs[idx]);
362 vintf_map &= ~BIT_ULL(idx);
363 }
364
365 return IRQ_HANDLED;
366 }
367
368 /* Command Queue Function */
369
tegra241_guest_vcmdq_supports_cmd(struct arm_smmu_cmdq_ent * ent)370 static bool tegra241_guest_vcmdq_supports_cmd(struct arm_smmu_cmdq_ent *ent)
371 {
372 switch (ent->opcode) {
373 case CMDQ_OP_TLBI_NH_ASID:
374 case CMDQ_OP_TLBI_NH_VA:
375 case CMDQ_OP_ATC_INV:
376 return true;
377 default:
378 return false;
379 }
380 }
381
382 static struct arm_smmu_cmdq *
tegra241_cmdqv_get_cmdq(struct arm_smmu_device * smmu,struct arm_smmu_cmdq_ent * ent)383 tegra241_cmdqv_get_cmdq(struct arm_smmu_device *smmu,
384 struct arm_smmu_cmdq_ent *ent)
385 {
386 struct tegra241_cmdqv *cmdqv =
387 container_of(smmu, struct tegra241_cmdqv, smmu);
388 struct tegra241_vintf *vintf = cmdqv->vintfs[0];
389 struct tegra241_vcmdq *vcmdq;
390 u16 lidx;
391
392 if (READ_ONCE(bypass_vcmdq))
393 return NULL;
394
395 /* Use SMMU CMDQ if VINTF0 is uninitialized */
396 if (!READ_ONCE(vintf->enabled))
397 return NULL;
398
399 /*
400 * Select a LVCMDQ to use. Here we use a temporal solution to
401 * balance out traffic on cmdq issuing: each cmdq has its own
402 * lock, if all cpus issue cmdlist using the same cmdq, only
403 * one CPU at a time can enter the process, while the others
404 * will be spinning at the same lock.
405 */
406 lidx = raw_smp_processor_id() % cmdqv->num_lvcmdqs_per_vintf;
407 vcmdq = vintf->lvcmdqs[lidx];
408 if (!vcmdq || !READ_ONCE(vcmdq->enabled))
409 return NULL;
410
411 /* Unsupported CMD goes for smmu->cmdq pathway */
412 if (!arm_smmu_cmdq_supports_cmd(&vcmdq->cmdq, ent))
413 return NULL;
414 return &vcmdq->cmdq;
415 }
416
417 /* HW Reset Functions */
418
419 /*
420 * When a guest-owned VCMDQ is disabled, if the guest did not enqueue a CMD_SYNC
421 * following an ATC_INV command at the end of the guest queue while this ATC_INV
422 * is timed out, the TIMEOUT will not be reported until this VCMDQ gets assigned
423 * to the next VM, which will be a false alarm potentially causing some unwanted
424 * behavior in the new VM. Thus, a guest-owned VCMDQ must flush the TIMEOUT when
425 * it gets disabled. This can be done by just issuing a CMD_SYNC to SMMU CMDQ.
426 */
tegra241_vcmdq_hw_flush_timeout(struct tegra241_vcmdq * vcmdq)427 static void tegra241_vcmdq_hw_flush_timeout(struct tegra241_vcmdq *vcmdq)
428 {
429 struct arm_smmu_device *smmu = &vcmdq->cmdqv->smmu;
430 u64 cmd_sync[CMDQ_ENT_DWORDS] = {};
431
432 cmd_sync[0] = FIELD_PREP(CMDQ_0_OP, CMDQ_OP_CMD_SYNC) |
433 FIELD_PREP(CMDQ_SYNC_0_CS, CMDQ_SYNC_0_CS_NONE);
434
435 /*
436 * It does not hurt to insert another CMD_SYNC, taking advantage of the
437 * arm_smmu_cmdq_issue_cmdlist() that waits for the CMD_SYNC completion.
438 */
439 arm_smmu_cmdq_issue_cmdlist(smmu, &smmu->cmdq, cmd_sync, 1, true);
440 }
441
442 /* This function is for LVCMDQ, so @vcmdq must not be unmapped yet */
tegra241_vcmdq_hw_deinit(struct tegra241_vcmdq * vcmdq)443 static void tegra241_vcmdq_hw_deinit(struct tegra241_vcmdq *vcmdq)
444 {
445 char header[64], *h = lvcmdq_error_header(vcmdq, header, 64);
446 u32 gerrorn, gerror;
447
448 if (vcmdq_write_config(vcmdq, 0)) {
449 dev_err(vcmdq->cmdqv->dev,
450 "%sGERRORN=0x%X, GERROR=0x%X, CONS=0x%X\n", h,
451 readl_relaxed(REG_VCMDQ_PAGE0(vcmdq, GERRORN)),
452 readl_relaxed(REG_VCMDQ_PAGE0(vcmdq, GERROR)),
453 readl_relaxed(REG_VCMDQ_PAGE0(vcmdq, CONS)));
454 }
455 tegra241_vcmdq_hw_flush_timeout(vcmdq);
456
457 writel_relaxed(0, REG_VCMDQ_PAGE0(vcmdq, PROD));
458 writel_relaxed(0, REG_VCMDQ_PAGE0(vcmdq, CONS));
459 writeq_relaxed(0, REG_VCMDQ_PAGE1(vcmdq, BASE));
460 writeq_relaxed(0, REG_VCMDQ_PAGE1(vcmdq, CONS_INDX_BASE));
461
462 gerrorn = readl_relaxed(REG_VCMDQ_PAGE0(vcmdq, GERRORN));
463 gerror = readl_relaxed(REG_VCMDQ_PAGE0(vcmdq, GERROR));
464 if (gerror != gerrorn) {
465 dev_warn(vcmdq->cmdqv->dev,
466 "%suncleared error detected, resetting\n", h);
467 writel(gerror, REG_VCMDQ_PAGE0(vcmdq, GERRORN));
468 }
469
470 dev_dbg(vcmdq->cmdqv->dev, "%sdeinited\n", h);
471 }
472
473 /* This function is for LVCMDQ, so @vcmdq must be mapped prior */
tegra241_vcmdq_hw_init(struct tegra241_vcmdq * vcmdq)474 static int tegra241_vcmdq_hw_init(struct tegra241_vcmdq *vcmdq)
475 {
476 char header[64], *h = lvcmdq_error_header(vcmdq, header, 64);
477 int ret;
478
479 /* Reset VCMDQ */
480 tegra241_vcmdq_hw_deinit(vcmdq);
481
482 /* Configure and enable VCMDQ */
483 writeq_relaxed(vcmdq->cmdq.q.q_base, REG_VCMDQ_PAGE1(vcmdq, BASE));
484
485 ret = vcmdq_write_config(vcmdq, VCMDQ_EN);
486 if (ret) {
487 dev_err(vcmdq->cmdqv->dev,
488 "%sGERRORN=0x%X, GERROR=0x%X, CONS=0x%X\n", h,
489 readl_relaxed(REG_VCMDQ_PAGE0(vcmdq, GERRORN)),
490 readl_relaxed(REG_VCMDQ_PAGE0(vcmdq, GERROR)),
491 readl_relaxed(REG_VCMDQ_PAGE0(vcmdq, CONS)));
492 return ret;
493 }
494
495 dev_dbg(vcmdq->cmdqv->dev, "%sinited\n", h);
496 return 0;
497 }
498
499 /* Unmap a global VCMDQ from the pre-assigned LVCMDQ */
tegra241_vcmdq_unmap_lvcmdq(struct tegra241_vcmdq * vcmdq)500 static void tegra241_vcmdq_unmap_lvcmdq(struct tegra241_vcmdq *vcmdq)
501 {
502 u32 regval = readl(REG_CMDQV(vcmdq->cmdqv, CMDQ_ALLOC(vcmdq->idx)));
503 char header[64], *h = lvcmdq_error_header(vcmdq, header, 64);
504
505 writel(regval & ~CMDQV_CMDQ_ALLOCATED,
506 REG_CMDQV(vcmdq->cmdqv, CMDQ_ALLOC(vcmdq->idx)));
507 dev_dbg(vcmdq->cmdqv->dev, "%sunmapped\n", h);
508 }
509
tegra241_vintf_hw_deinit(struct tegra241_vintf * vintf)510 static void tegra241_vintf_hw_deinit(struct tegra241_vintf *vintf)
511 {
512 u16 lidx = vintf->cmdqv->num_lvcmdqs_per_vintf;
513 int sidx;
514
515 /* HW requires to unmap LVCMDQs in descending order */
516 while (lidx--) {
517 if (vintf->lvcmdqs && vintf->lvcmdqs[lidx]) {
518 tegra241_vcmdq_hw_deinit(vintf->lvcmdqs[lidx]);
519 tegra241_vcmdq_unmap_lvcmdq(vintf->lvcmdqs[lidx]);
520 }
521 }
522 vintf_write_config(vintf, 0);
523 for (sidx = 0; sidx < vintf->cmdqv->num_sids_per_vintf; sidx++) {
524 writel(0, REG_VINTF(vintf, SID_MATCH(sidx)));
525 writel(0, REG_VINTF(vintf, SID_REPLACE(sidx)));
526 }
527 }
528
529 /* Map a global VCMDQ to the pre-assigned LVCMDQ */
tegra241_vcmdq_map_lvcmdq(struct tegra241_vcmdq * vcmdq)530 static void tegra241_vcmdq_map_lvcmdq(struct tegra241_vcmdq *vcmdq)
531 {
532 u32 regval = readl(REG_CMDQV(vcmdq->cmdqv, CMDQ_ALLOC(vcmdq->idx)));
533 char header[64], *h = lvcmdq_error_header(vcmdq, header, 64);
534
535 writel(regval | CMDQV_CMDQ_ALLOCATED,
536 REG_CMDQV(vcmdq->cmdqv, CMDQ_ALLOC(vcmdq->idx)));
537 dev_dbg(vcmdq->cmdqv->dev, "%smapped\n", h);
538 }
539
tegra241_vintf_hw_init(struct tegra241_vintf * vintf,bool hyp_own)540 static int tegra241_vintf_hw_init(struct tegra241_vintf *vintf, bool hyp_own)
541 {
542 u32 regval;
543 u16 lidx;
544 int ret;
545
546 /* Reset VINTF */
547 tegra241_vintf_hw_deinit(vintf);
548
549 /* Configure and enable VINTF */
550 /*
551 * Note that HYP_OWN bit is wired to zero when running in guest kernel,
552 * whether enabling it here or not, as !HYP_OWN cmdq HWs only support a
553 * restricted set of supported commands.
554 */
555 regval = FIELD_PREP(VINTF_HYP_OWN, hyp_own) |
556 FIELD_PREP(VINTF_VMID, vintf->vsmmu.vmid);
557 writel(regval, REG_VINTF(vintf, CONFIG));
558
559 ret = vintf_write_config(vintf, regval | VINTF_EN);
560 if (ret)
561 return ret;
562 /*
563 * As being mentioned above, HYP_OWN bit is wired to zero for a guest
564 * kernel, so read it back from HW to ensure that reflects in hyp_own
565 */
566 vintf->hyp_own = !!(VINTF_HYP_OWN & readl(REG_VINTF(vintf, CONFIG)));
567
568 /* HW requires to map LVCMDQs in ascending order */
569 for (lidx = 0; lidx < vintf->cmdqv->num_lvcmdqs_per_vintf; lidx++) {
570 if (vintf->lvcmdqs && vintf->lvcmdqs[lidx]) {
571 tegra241_vcmdq_map_lvcmdq(vintf->lvcmdqs[lidx]);
572 ret = tegra241_vcmdq_hw_init(vintf->lvcmdqs[lidx]);
573 if (ret) {
574 tegra241_vintf_hw_deinit(vintf);
575 return ret;
576 }
577 }
578 }
579
580 return 0;
581 }
582
tegra241_cmdqv_hw_reset(struct arm_smmu_device * smmu)583 static int tegra241_cmdqv_hw_reset(struct arm_smmu_device *smmu)
584 {
585 struct tegra241_cmdqv *cmdqv =
586 container_of(smmu, struct tegra241_cmdqv, smmu);
587 u16 qidx, lidx, idx;
588 u32 regval;
589 int ret;
590
591 /* Reset CMDQV */
592 regval = readl_relaxed(REG_CMDQV(cmdqv, CONFIG));
593 ret = cmdqv_write_config(cmdqv, regval & ~CMDQV_EN);
594 if (ret)
595 return ret;
596 ret = cmdqv_write_config(cmdqv, regval | CMDQV_EN);
597 if (ret)
598 return ret;
599
600 /* Assign preallocated global VCMDQs to each VINTF as LVCMDQs */
601 for (idx = 0, qidx = 0; idx < cmdqv->num_vintfs; idx++) {
602 for (lidx = 0; lidx < cmdqv->num_lvcmdqs_per_vintf; lidx++) {
603 regval = FIELD_PREP(CMDQV_CMDQ_ALLOC_VINTF, idx);
604 regval |= FIELD_PREP(CMDQV_CMDQ_ALLOC_LVCMDQ, lidx);
605 writel_relaxed(regval,
606 REG_CMDQV(cmdqv, CMDQ_ALLOC(qidx++)));
607 }
608 }
609
610 return tegra241_vintf_hw_init(cmdqv->vintfs[0], true);
611 }
612
613 /* VCMDQ Resource Helpers */
614
tegra241_vcmdq_alloc_smmu_cmdq(struct tegra241_vcmdq * vcmdq)615 static int tegra241_vcmdq_alloc_smmu_cmdq(struct tegra241_vcmdq *vcmdq)
616 {
617 struct arm_smmu_device *smmu = &vcmdq->cmdqv->smmu;
618 struct arm_smmu_cmdq *cmdq = &vcmdq->cmdq;
619 struct arm_smmu_queue *q = &cmdq->q;
620 char name[16];
621 u32 regval;
622 int ret;
623
624 snprintf(name, 16, "vcmdq%u", vcmdq->idx);
625
626 /* Cap queue size to SMMU's IDR1.CMDQS and ensure natural alignment */
627 regval = readl_relaxed(smmu->base + ARM_SMMU_IDR1);
628 q->llq.max_n_shift =
629 min_t(u32, CMDQ_MAX_SZ_SHIFT, FIELD_GET(IDR1_CMDQS, regval));
630
631 /* Use the common helper to init the VCMDQ, and then... */
632 ret = arm_smmu_init_one_queue(smmu, q, vcmdq->page0,
633 TEGRA241_VCMDQ_PROD, TEGRA241_VCMDQ_CONS,
634 CMDQ_ENT_DWORDS, name);
635 if (ret)
636 return ret;
637
638 /* ...override q_base to write VCMDQ_BASE registers */
639 q->q_base = q->base_dma & VCMDQ_ADDR;
640 q->q_base |= FIELD_PREP(VCMDQ_LOG2SIZE, q->llq.max_n_shift);
641
642 if (!vcmdq->vintf->hyp_own)
643 cmdq->supports_cmd = tegra241_guest_vcmdq_supports_cmd;
644
645 return arm_smmu_cmdq_init(smmu, cmdq);
646 }
647
648 /* VINTF Logical VCMDQ Resource Helpers */
649
tegra241_vintf_deinit_lvcmdq(struct tegra241_vintf * vintf,u16 lidx)650 static void tegra241_vintf_deinit_lvcmdq(struct tegra241_vintf *vintf, u16 lidx)
651 {
652 vintf->lvcmdqs[lidx] = NULL;
653 }
654
tegra241_vintf_init_lvcmdq(struct tegra241_vintf * vintf,u16 lidx,struct tegra241_vcmdq * vcmdq)655 static int tegra241_vintf_init_lvcmdq(struct tegra241_vintf *vintf, u16 lidx,
656 struct tegra241_vcmdq *vcmdq)
657 {
658 struct tegra241_cmdqv *cmdqv = vintf->cmdqv;
659 u16 idx = vintf->idx;
660
661 vcmdq->idx = idx * cmdqv->num_lvcmdqs_per_vintf + lidx;
662 vcmdq->lidx = lidx;
663 vcmdq->cmdqv = cmdqv;
664 vcmdq->vintf = vintf;
665 vcmdq->page0 = cmdqv->base + TEGRA241_VINTFi_LVCMDQ_PAGE0(idx, lidx);
666 vcmdq->page1 = cmdqv->base + TEGRA241_VINTFi_LVCMDQ_PAGE1(idx, lidx);
667
668 vintf->lvcmdqs[lidx] = vcmdq;
669 return 0;
670 }
671
tegra241_vintf_free_lvcmdq(struct tegra241_vintf * vintf,u16 lidx)672 static void tegra241_vintf_free_lvcmdq(struct tegra241_vintf *vintf, u16 lidx)
673 {
674 struct tegra241_vcmdq *vcmdq = vintf->lvcmdqs[lidx];
675 char header[64];
676
677 /* Note that the lvcmdq queue memory space is managed by devres */
678
679 tegra241_vintf_deinit_lvcmdq(vintf, lidx);
680
681 dev_dbg(vintf->cmdqv->dev,
682 "%sdeallocated\n", lvcmdq_error_header(vcmdq, header, 64));
683 /* Guest-owned VCMDQ is free-ed with hw_queue by iommufd core */
684 if (vcmdq->vintf->hyp_own)
685 kfree(vcmdq);
686 }
687
688 static struct tegra241_vcmdq *
tegra241_vintf_alloc_lvcmdq(struct tegra241_vintf * vintf,u16 lidx)689 tegra241_vintf_alloc_lvcmdq(struct tegra241_vintf *vintf, u16 lidx)
690 {
691 struct tegra241_cmdqv *cmdqv = vintf->cmdqv;
692 struct tegra241_vcmdq *vcmdq;
693 char header[64];
694 int ret;
695
696 vcmdq = kzalloc_obj(*vcmdq);
697 if (!vcmdq)
698 return ERR_PTR(-ENOMEM);
699
700 ret = tegra241_vintf_init_lvcmdq(vintf, lidx, vcmdq);
701 if (ret)
702 goto free_vcmdq;
703
704 /* Build an arm_smmu_cmdq for each LVCMDQ */
705 ret = tegra241_vcmdq_alloc_smmu_cmdq(vcmdq);
706 if (ret)
707 goto deinit_lvcmdq;
708
709 dev_dbg(cmdqv->dev,
710 "%sallocated\n", lvcmdq_error_header(vcmdq, header, 64));
711 return vcmdq;
712
713 deinit_lvcmdq:
714 tegra241_vintf_deinit_lvcmdq(vintf, lidx);
715 free_vcmdq:
716 kfree(vcmdq);
717 return ERR_PTR(ret);
718 }
719
720 /* VINTF Resource Helpers */
721
tegra241_cmdqv_deinit_vintf(struct tegra241_cmdqv * cmdqv,u16 idx)722 static void tegra241_cmdqv_deinit_vintf(struct tegra241_cmdqv *cmdqv, u16 idx)
723 {
724 kfree(cmdqv->vintfs[idx]->lvcmdqs);
725 ida_free(&cmdqv->vintf_ids, idx);
726 cmdqv->vintfs[idx] = NULL;
727 }
728
tegra241_cmdqv_init_vintf(struct tegra241_cmdqv * cmdqv,u16 max_idx,struct tegra241_vintf * vintf)729 static int tegra241_cmdqv_init_vintf(struct tegra241_cmdqv *cmdqv, u16 max_idx,
730 struct tegra241_vintf *vintf)
731 {
732
733 u16 idx;
734 int ret;
735
736 ret = ida_alloc_max(&cmdqv->vintf_ids, max_idx, GFP_KERNEL);
737 if (ret < 0)
738 return ret;
739 idx = ret;
740
741 vintf->idx = idx;
742 vintf->cmdqv = cmdqv;
743 vintf->base = cmdqv->base + TEGRA241_VINTF(idx);
744
745 vintf->lvcmdqs = kzalloc_objs(*vintf->lvcmdqs,
746 cmdqv->num_lvcmdqs_per_vintf);
747 if (!vintf->lvcmdqs) {
748 ida_free(&cmdqv->vintf_ids, idx);
749 return -ENOMEM;
750 }
751
752 cmdqv->vintfs[idx] = vintf;
753 return ret;
754 }
755
756 /* Remove Helpers */
757
tegra241_cmdqv_remove_vintf(struct tegra241_cmdqv * cmdqv,u16 idx)758 static void tegra241_cmdqv_remove_vintf(struct tegra241_cmdqv *cmdqv, u16 idx)
759 {
760 struct tegra241_vintf *vintf = cmdqv->vintfs[idx];
761 u16 lidx;
762
763 tegra241_vintf_hw_deinit(vintf);
764
765 /* Remove LVCMDQ resources */
766 for (lidx = 0; lidx < vintf->cmdqv->num_lvcmdqs_per_vintf; lidx++)
767 if (vintf->lvcmdqs[lidx])
768 tegra241_vintf_free_lvcmdq(vintf, lidx);
769
770 dev_dbg(cmdqv->dev, "VINTF%u: deallocated\n", vintf->idx);
771 tegra241_cmdqv_deinit_vintf(cmdqv, idx);
772 if (!vintf->hyp_own) {
773 mutex_destroy(&vintf->lvcmdq_mutex);
774 ida_destroy(&vintf->sids);
775 /* Guest-owned VINTF is free-ed with viommu by iommufd core */
776 } else {
777 kfree(vintf);
778 }
779 }
780
tegra241_cmdqv_remove(struct arm_smmu_device * smmu)781 static void tegra241_cmdqv_remove(struct arm_smmu_device *smmu)
782 {
783 struct tegra241_cmdqv *cmdqv =
784 container_of(smmu, struct tegra241_cmdqv, smmu);
785 u16 idx;
786
787 /* Remove VINTF resources */
788 for (idx = 0; idx < cmdqv->num_vintfs; idx++) {
789 if (cmdqv->vintfs[idx]) {
790 /* Only vintf0 should remain at this stage */
791 WARN_ON(idx > 0);
792 tegra241_cmdqv_remove_vintf(cmdqv, idx);
793 }
794 }
795
796 /* Remove cmdqv resources */
797 ida_destroy(&cmdqv->vintf_ids);
798
799 if (cmdqv->irq > 0)
800 free_irq(cmdqv->irq, cmdqv);
801 iounmap(cmdqv->base);
802 kfree(cmdqv->vintfs);
803 put_device(cmdqv->dev); /* smmu->impl_dev */
804 }
805
806 static int
807 tegra241_cmdqv_init_vintf_user(struct arm_vsmmu *vsmmu,
808 const struct iommu_user_data *user_data);
809
tegra241_cmdqv_hw_info(struct arm_smmu_device * smmu,u32 * length,enum iommu_hw_info_type * type)810 static void *tegra241_cmdqv_hw_info(struct arm_smmu_device *smmu, u32 *length,
811 enum iommu_hw_info_type *type)
812 {
813 struct tegra241_cmdqv *cmdqv =
814 container_of(smmu, struct tegra241_cmdqv, smmu);
815 struct iommu_hw_info_tegra241_cmdqv *info;
816 u32 regval;
817
818 if (*type != IOMMU_HW_INFO_TYPE_TEGRA241_CMDQV)
819 return ERR_PTR(-EOPNOTSUPP);
820
821 info = kzalloc_obj(*info);
822 if (!info)
823 return ERR_PTR(-ENOMEM);
824
825 regval = readl_relaxed(REG_CMDQV(cmdqv, PARAM));
826 info->log2vcmdqs = ilog2(cmdqv->num_lvcmdqs_per_vintf);
827 info->log2vsids = ilog2(cmdqv->num_sids_per_vintf);
828 info->version = FIELD_GET(CMDQV_VER, regval);
829
830 *length = sizeof(*info);
831 *type = IOMMU_HW_INFO_TYPE_TEGRA241_CMDQV;
832 return info;
833 }
834
tegra241_cmdqv_get_vintf_size(enum iommu_viommu_type viommu_type)835 static size_t tegra241_cmdqv_get_vintf_size(enum iommu_viommu_type viommu_type)
836 {
837 if (viommu_type != IOMMU_VIOMMU_TYPE_TEGRA241_CMDQV)
838 return 0;
839 return VIOMMU_STRUCT_SIZE(struct tegra241_vintf, vsmmu.core);
840 }
841
842 static struct arm_smmu_impl_ops tegra241_cmdqv_impl_ops = {
843 /* For in-kernel use */
844 .get_secondary_cmdq = tegra241_cmdqv_get_cmdq,
845 .device_reset = tegra241_cmdqv_hw_reset,
846 .device_remove = tegra241_cmdqv_remove,
847 /* For user-space use */
848 .hw_info = tegra241_cmdqv_hw_info,
849 .get_viommu_size = tegra241_cmdqv_get_vintf_size,
850 .vsmmu_init = tegra241_cmdqv_init_vintf_user,
851 };
852
853 /* Probe Functions */
854
tegra241_cmdqv_init_structures(struct arm_smmu_device * smmu)855 static int tegra241_cmdqv_init_structures(struct arm_smmu_device *smmu)
856 {
857 struct tegra241_cmdqv *cmdqv =
858 container_of(smmu, struct tegra241_cmdqv, smmu);
859 struct tegra241_vintf *vintf;
860 int lidx;
861 int ret;
862
863 vintf = kzalloc_obj(*vintf);
864 if (!vintf)
865 return -ENOMEM;
866
867 /* Init VINTF0 for in-kernel use */
868 ret = tegra241_cmdqv_init_vintf(cmdqv, 0, vintf);
869 if (ret) {
870 dev_err(cmdqv->dev, "failed to init vintf0: %d\n", ret);
871 return ret;
872 }
873
874 /* Preallocate logical VCMDQs to VINTF0 */
875 for (lidx = 0; lidx < cmdqv->num_lvcmdqs_per_vintf; lidx++) {
876 struct tegra241_vcmdq *vcmdq;
877
878 vcmdq = tegra241_vintf_alloc_lvcmdq(vintf, lidx);
879 if (IS_ERR(vcmdq))
880 return PTR_ERR(vcmdq);
881 }
882
883 /* Now, we are ready to run all the impl ops */
884 smmu->impl_ops = &tegra241_cmdqv_impl_ops;
885 return 0;
886 }
887
888 #ifdef CONFIG_IOMMU_DEBUGFS
889 static struct dentry *cmdqv_debugfs_dir;
890 #endif
891
892 static struct arm_smmu_device *
__tegra241_cmdqv_probe(struct arm_smmu_device * smmu,struct resource * res,int irq)893 __tegra241_cmdqv_probe(struct arm_smmu_device *smmu, struct resource *res,
894 int irq)
895 {
896 static const struct arm_smmu_impl_ops init_ops = {
897 .init_structures = tegra241_cmdqv_init_structures,
898 .device_remove = tegra241_cmdqv_remove,
899 };
900 struct tegra241_cmdqv *cmdqv = NULL;
901 struct arm_smmu_device *new_smmu;
902 void __iomem *base;
903 u32 regval;
904 int ret;
905
906 static_assert(offsetof(struct tegra241_cmdqv, smmu) == 0);
907
908 base = ioremap(res->start, resource_size(res));
909 if (!base) {
910 dev_err(smmu->dev, "failed to ioremap\n");
911 return NULL;
912 }
913
914 regval = readl(base + TEGRA241_CMDQV_CONFIG);
915 if (disable_cmdqv) {
916 dev_info(smmu->dev, "Detected disable_cmdqv=true\n");
917 writel(regval & ~CMDQV_EN, base + TEGRA241_CMDQV_CONFIG);
918 goto iounmap;
919 }
920
921 cmdqv = devm_krealloc(smmu->dev, smmu, sizeof(*cmdqv), GFP_KERNEL);
922 if (!cmdqv)
923 goto iounmap;
924 new_smmu = &cmdqv->smmu;
925
926 cmdqv->irq = irq;
927 cmdqv->base = base;
928 cmdqv->dev = smmu->impl_dev;
929 cmdqv->base_phys = res->start;
930
931 if (cmdqv->irq > 0) {
932 ret = request_threaded_irq(irq, NULL, tegra241_cmdqv_isr,
933 IRQF_ONESHOT, "tegra241-cmdqv",
934 cmdqv);
935 if (ret) {
936 dev_err(cmdqv->dev, "failed to request irq (%d): %d\n",
937 cmdqv->irq, ret);
938 goto iounmap;
939 }
940 }
941
942 regval = readl_relaxed(REG_CMDQV(cmdqv, PARAM));
943 cmdqv->num_vintfs = 1 << FIELD_GET(CMDQV_NUM_VINTF_LOG2, regval);
944 cmdqv->num_vcmdqs = 1 << FIELD_GET(CMDQV_NUM_VCMDQ_LOG2, regval);
945 cmdqv->num_lvcmdqs_per_vintf = cmdqv->num_vcmdqs / cmdqv->num_vintfs;
946 cmdqv->num_sids_per_vintf =
947 1 << FIELD_GET(CMDQV_NUM_SID_PER_VM_LOG2, regval);
948
949 cmdqv->vintfs =
950 kzalloc_objs(*cmdqv->vintfs, cmdqv->num_vintfs);
951 if (!cmdqv->vintfs)
952 goto free_irq;
953
954 ida_init(&cmdqv->vintf_ids);
955
956 #ifdef CONFIG_IOMMU_DEBUGFS
957 if (!cmdqv_debugfs_dir) {
958 cmdqv_debugfs_dir =
959 debugfs_create_dir("tegra241_cmdqv", iommu_debugfs_dir);
960 debugfs_create_bool("bypass_vcmdq", 0644, cmdqv_debugfs_dir,
961 &bypass_vcmdq);
962 }
963 #endif
964
965 /* Provide init-level ops only, until tegra241_cmdqv_init_structures */
966 new_smmu->impl_ops = &init_ops;
967
968 return new_smmu;
969
970 free_irq:
971 if (cmdqv->irq > 0)
972 free_irq(cmdqv->irq, cmdqv);
973 iounmap:
974 iounmap(base);
975 return NULL;
976 }
977
tegra241_cmdqv_probe(struct arm_smmu_device * smmu)978 struct arm_smmu_device *tegra241_cmdqv_probe(struct arm_smmu_device *smmu)
979 {
980 struct platform_device *pdev = to_platform_device(smmu->impl_dev);
981 struct arm_smmu_device *new_smmu;
982 struct resource *res;
983 int irq;
984
985 res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
986 if (!res) {
987 dev_err(&pdev->dev, "no memory resource found for CMDQV\n");
988 goto out_fallback;
989 }
990
991 irq = platform_get_irq_optional(pdev, 0);
992 if (irq <= 0)
993 dev_warn(&pdev->dev,
994 "no interrupt. errors will not be reported\n");
995
996 new_smmu = __tegra241_cmdqv_probe(smmu, res, irq);
997 if (new_smmu)
998 return new_smmu;
999
1000 out_fallback:
1001 dev_info(smmu->impl_dev, "Falling back to standard SMMU CMDQ\n");
1002 smmu->options &= ~ARM_SMMU_OPT_TEGRA241_CMDQV;
1003 put_device(smmu->impl_dev);
1004 return ERR_PTR(-ENODEV);
1005 }
1006
1007 /* User space VINTF and VCMDQ Functions */
1008
tegra241_vintf_get_vcmdq_size(struct iommufd_viommu * viommu,enum iommu_hw_queue_type queue_type)1009 static size_t tegra241_vintf_get_vcmdq_size(struct iommufd_viommu *viommu,
1010 enum iommu_hw_queue_type queue_type)
1011 {
1012 if (queue_type != IOMMU_HW_QUEUE_TYPE_TEGRA241_CMDQV)
1013 return 0;
1014 return HW_QUEUE_STRUCT_SIZE(struct tegra241_vcmdq, core);
1015 }
1016
tegra241_vcmdq_hw_init_user(struct tegra241_vcmdq * vcmdq)1017 static int tegra241_vcmdq_hw_init_user(struct tegra241_vcmdq *vcmdq)
1018 {
1019 char header[64];
1020
1021 /* Reset VCMDQ */
1022 tegra241_vcmdq_hw_deinit(vcmdq);
1023
1024 /* Configure the vcmdq only; User space does the enabling */
1025 writeq_relaxed(vcmdq->cmdq.q.q_base, REG_VCMDQ_PAGE1(vcmdq, BASE));
1026
1027 dev_dbg(vcmdq->cmdqv->dev, "%sinited at host PA 0x%llx size 0x%lx\n",
1028 lvcmdq_error_header(vcmdq, header, 64),
1029 vcmdq->cmdq.q.q_base & VCMDQ_ADDR,
1030 1UL << (vcmdq->cmdq.q.q_base & VCMDQ_LOG2SIZE));
1031 return 0;
1032 }
1033
1034 static void
tegra241_vintf_destroy_lvcmdq_user(struct iommufd_hw_queue * hw_queue)1035 tegra241_vintf_destroy_lvcmdq_user(struct iommufd_hw_queue *hw_queue)
1036 {
1037 struct tegra241_vcmdq *vcmdq = hw_queue_to_vcmdq(hw_queue);
1038
1039 mutex_lock(&vcmdq->vintf->lvcmdq_mutex);
1040 tegra241_vcmdq_hw_deinit(vcmdq);
1041 tegra241_vcmdq_unmap_lvcmdq(vcmdq);
1042 tegra241_vintf_free_lvcmdq(vcmdq->vintf, vcmdq->lidx);
1043 if (vcmdq->prev)
1044 iommufd_hw_queue_undepend(vcmdq, vcmdq->prev, core);
1045 mutex_unlock(&vcmdq->vintf->lvcmdq_mutex);
1046 }
1047
tegra241_vintf_alloc_lvcmdq_user(struct iommufd_hw_queue * hw_queue,u32 lidx,phys_addr_t base_addr_pa)1048 static int tegra241_vintf_alloc_lvcmdq_user(struct iommufd_hw_queue *hw_queue,
1049 u32 lidx, phys_addr_t base_addr_pa)
1050 {
1051 struct tegra241_vintf *vintf = viommu_to_vintf(hw_queue->viommu);
1052 struct tegra241_vcmdq *vcmdq = hw_queue_to_vcmdq(hw_queue);
1053 struct tegra241_cmdqv *cmdqv = vintf->cmdqv;
1054 struct arm_smmu_device *smmu = &cmdqv->smmu;
1055 struct tegra241_vcmdq *prev = NULL;
1056 u32 log2size, max_n_shift;
1057 char header[64];
1058 int ret;
1059
1060 if (hw_queue->type != IOMMU_HW_QUEUE_TYPE_TEGRA241_CMDQV)
1061 return -EOPNOTSUPP;
1062 if (lidx >= cmdqv->num_lvcmdqs_per_vintf)
1063 return -EINVAL;
1064
1065 mutex_lock(&vintf->lvcmdq_mutex);
1066
1067 if (vintf->lvcmdqs[lidx]) {
1068 ret = -EEXIST;
1069 goto unlock;
1070 }
1071
1072 /*
1073 * HW requires to map LVCMDQs in ascending order, so reject if the
1074 * previous lvcmdqs is not allocated yet.
1075 */
1076 if (lidx) {
1077 prev = vintf->lvcmdqs[lidx - 1];
1078 if (!prev) {
1079 ret = -EIO;
1080 goto unlock;
1081 }
1082 }
1083
1084 /*
1085 * hw_queue->length must be a power of 2, in range of
1086 * [ 32, 2 ^ (idr[1].CMDQS + CMDQ_ENT_SZ_SHIFT) ]
1087 */
1088 max_n_shift = FIELD_GET(IDR1_CMDQS,
1089 readl_relaxed(smmu->base + ARM_SMMU_IDR1));
1090 if (!is_power_of_2(hw_queue->length) || hw_queue->length < 32 ||
1091 hw_queue->length > (1 << (max_n_shift + CMDQ_ENT_SZ_SHIFT))) {
1092 ret = -EINVAL;
1093 goto unlock;
1094 }
1095 log2size = ilog2(hw_queue->length) - CMDQ_ENT_SZ_SHIFT;
1096
1097 /* base_addr_pa must be aligned to hw_queue->length */
1098 if (base_addr_pa & ~VCMDQ_ADDR ||
1099 base_addr_pa & (hw_queue->length - 1)) {
1100 ret = -EINVAL;
1101 goto unlock;
1102 }
1103
1104 /*
1105 * HW requires to unmap LVCMDQs in descending order, so destroy() must
1106 * follow this rule. Set a dependency on its previous LVCMDQ so iommufd
1107 * core will help enforce it.
1108 */
1109 if (prev) {
1110 ret = iommufd_hw_queue_depend(vcmdq, prev, core);
1111 if (ret)
1112 goto unlock;
1113 }
1114 vcmdq->prev = prev;
1115
1116 ret = tegra241_vintf_init_lvcmdq(vintf, lidx, vcmdq);
1117 if (ret)
1118 goto undepend_vcmdq;
1119
1120 dev_dbg(cmdqv->dev, "%sallocated\n",
1121 lvcmdq_error_header(vcmdq, header, 64));
1122
1123 tegra241_vcmdq_map_lvcmdq(vcmdq);
1124
1125 vcmdq->cmdq.q.q_base = base_addr_pa & VCMDQ_ADDR;
1126 vcmdq->cmdq.q.q_base |= log2size;
1127
1128 ret = tegra241_vcmdq_hw_init_user(vcmdq);
1129 if (ret)
1130 goto unmap_lvcmdq;
1131
1132 hw_queue->destroy = &tegra241_vintf_destroy_lvcmdq_user;
1133 mutex_unlock(&vintf->lvcmdq_mutex);
1134 return 0;
1135
1136 unmap_lvcmdq:
1137 tegra241_vcmdq_unmap_lvcmdq(vcmdq);
1138 tegra241_vintf_deinit_lvcmdq(vintf, lidx);
1139 undepend_vcmdq:
1140 if (vcmdq->prev)
1141 iommufd_hw_queue_undepend(vcmdq, vcmdq->prev, core);
1142 unlock:
1143 mutex_unlock(&vintf->lvcmdq_mutex);
1144 return ret;
1145 }
1146
tegra241_cmdqv_destroy_vintf_user(struct iommufd_viommu * viommu)1147 static void tegra241_cmdqv_destroy_vintf_user(struct iommufd_viommu *viommu)
1148 {
1149 struct tegra241_vintf *vintf = viommu_to_vintf(viommu);
1150
1151 if (vintf->mmap_offset)
1152 iommufd_viommu_destroy_mmap(&vintf->vsmmu.core,
1153 vintf->mmap_offset);
1154 tegra241_cmdqv_remove_vintf(vintf->cmdqv, vintf->idx);
1155 }
1156
tegra241_vintf_destroy_vsid(struct iommufd_vdevice * vdev)1157 static void tegra241_vintf_destroy_vsid(struct iommufd_vdevice *vdev)
1158 {
1159 struct tegra241_vintf_sid *vsid = vdev_to_vsid(vdev);
1160 struct tegra241_vintf *vintf = vsid->vintf;
1161
1162 writel(0, REG_VINTF(vintf, SID_MATCH(vsid->idx)));
1163 writel(0, REG_VINTF(vintf, SID_REPLACE(vsid->idx)));
1164 ida_free(&vintf->sids, vsid->idx);
1165 dev_dbg(vintf->cmdqv->dev,
1166 "VINTF%u: deallocated SID_REPLACE%d for pSID=%x\n", vintf->idx,
1167 vsid->idx, vsid->sid);
1168 }
1169
tegra241_vintf_init_vsid(struct iommufd_vdevice * vdev)1170 static int tegra241_vintf_init_vsid(struct iommufd_vdevice *vdev)
1171 {
1172 struct device *dev = iommufd_vdevice_to_device(vdev);
1173 struct arm_smmu_master *master = dev_iommu_priv_get(dev);
1174 struct tegra241_vintf *vintf = viommu_to_vintf(vdev->viommu);
1175 struct tegra241_vintf_sid *vsid = vdev_to_vsid(vdev);
1176 struct arm_smmu_stream *stream = &master->streams[0];
1177 u64 virt_sid = vdev->virt_id;
1178 int sidx;
1179
1180 if (virt_sid > UINT_MAX)
1181 return -EINVAL;
1182
1183 WARN_ON_ONCE(master->num_streams != 1);
1184
1185 /* Find an empty pair of SID_REPLACE and SID_MATCH */
1186 sidx = ida_alloc_max(&vintf->sids, vintf->cmdqv->num_sids_per_vintf - 1,
1187 GFP_KERNEL);
1188 if (sidx < 0)
1189 return sidx;
1190
1191 writel(stream->id, REG_VINTF(vintf, SID_REPLACE(sidx)));
1192 writel(virt_sid << 1 | 0x1, REG_VINTF(vintf, SID_MATCH(sidx)));
1193 dev_dbg(vintf->cmdqv->dev,
1194 "VINTF%u: allocated SID_REPLACE%d for pSID=%x, vSID=%x\n",
1195 vintf->idx, sidx, stream->id, (u32)virt_sid);
1196
1197 vsid->idx = sidx;
1198 vsid->vintf = vintf;
1199 vsid->sid = stream->id;
1200
1201 vdev->destroy = &tegra241_vintf_destroy_vsid;
1202 return 0;
1203 }
1204
1205 static struct iommufd_viommu_ops tegra241_cmdqv_viommu_ops = {
1206 .destroy = tegra241_cmdqv_destroy_vintf_user,
1207 .alloc_domain_nested = arm_vsmmu_alloc_domain_nested,
1208 /* Non-accelerated commands will be still handled by the kernel */
1209 .cache_invalidate = arm_vsmmu_cache_invalidate,
1210 .vdevice_size = VDEVICE_STRUCT_SIZE(struct tegra241_vintf_sid, core),
1211 .vdevice_init = tegra241_vintf_init_vsid,
1212 .get_hw_queue_size = tegra241_vintf_get_vcmdq_size,
1213 .hw_queue_init_phys = tegra241_vintf_alloc_lvcmdq_user,
1214 };
1215
1216 static int
tegra241_cmdqv_init_vintf_user(struct arm_vsmmu * vsmmu,const struct iommu_user_data * user_data)1217 tegra241_cmdqv_init_vintf_user(struct arm_vsmmu *vsmmu,
1218 const struct iommu_user_data *user_data)
1219 {
1220 struct tegra241_cmdqv *cmdqv =
1221 container_of(vsmmu->smmu, struct tegra241_cmdqv, smmu);
1222 struct tegra241_vintf *vintf = viommu_to_vintf(&vsmmu->core);
1223 struct iommu_viommu_tegra241_cmdqv data;
1224 phys_addr_t page0_base;
1225 int ret;
1226
1227 /*
1228 * Unsupported type should be rejected by tegra241_cmdqv_get_vintf_size.
1229 * Seeing one here indicates a kernel bug or some data corruption.
1230 */
1231 if (WARN_ON(vsmmu->core.type != IOMMU_VIOMMU_TYPE_TEGRA241_CMDQV))
1232 return -EOPNOTSUPP;
1233
1234 if (!user_data)
1235 return -EINVAL;
1236
1237 ret = iommu_copy_struct_from_user(&data, user_data,
1238 IOMMU_VIOMMU_TYPE_TEGRA241_CMDQV,
1239 out_vintf_mmap_length);
1240 if (ret)
1241 return ret;
1242
1243 ret = tegra241_cmdqv_init_vintf(cmdqv, cmdqv->num_vintfs - 1, vintf);
1244 if (ret < 0) {
1245 dev_err(cmdqv->dev, "no more available vintf\n");
1246 return ret;
1247 }
1248
1249 /*
1250 * Initialize the user-owned VINTF without a LVCMDQ, as it cannot pre-
1251 * allocate a LVCMDQ until user space wants one, for security reasons.
1252 * It is different than the kernel-owned VINTF0, which had pre-assigned
1253 * and pre-allocated global VCMDQs that would be mapped to the LVCMDQs
1254 * by the tegra241_vintf_hw_init() call.
1255 */
1256 ret = tegra241_vintf_hw_init(vintf, false);
1257 if (ret)
1258 goto deinit_vintf;
1259
1260 page0_base = cmdqv->base_phys + TEGRA241_VINTFi_PAGE0(vintf->idx);
1261 ret = iommufd_viommu_alloc_mmap(&vintf->vsmmu.core, page0_base, SZ_64K,
1262 &vintf->mmap_offset);
1263 if (ret)
1264 goto hw_deinit_vintf;
1265
1266 data.out_vintf_mmap_length = SZ_64K;
1267 data.out_vintf_mmap_offset = vintf->mmap_offset;
1268 ret = iommu_copy_struct_to_user(user_data, &data,
1269 IOMMU_VIOMMU_TYPE_TEGRA241_CMDQV,
1270 out_vintf_mmap_length);
1271 if (ret)
1272 goto free_mmap;
1273
1274 ida_init(&vintf->sids);
1275 mutex_init(&vintf->lvcmdq_mutex);
1276
1277 dev_dbg(cmdqv->dev, "VINTF%u: allocated with vmid (%d)\n", vintf->idx,
1278 vintf->vsmmu.vmid);
1279
1280 vsmmu->core.ops = &tegra241_cmdqv_viommu_ops;
1281 return 0;
1282
1283 free_mmap:
1284 iommufd_viommu_destroy_mmap(&vintf->vsmmu.core, vintf->mmap_offset);
1285 hw_deinit_vintf:
1286 tegra241_vintf_hw_deinit(vintf);
1287 deinit_vintf:
1288 tegra241_cmdqv_deinit_vintf(cmdqv, vintf->idx);
1289 return ret;
1290 }
1291
1292 MODULE_IMPORT_NS("IOMMUFD");
1293