1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * IOMMU API for RISC-V IOMMU implementations.
4 *
5 * Copyright © 2022-2024 Rivos Inc.
6 * Copyright © 2023 FORTH-ICS/CARV
7 *
8 * Authors
9 * Tomasz Jeznach <tjeznach@rivosinc.com>
10 * Nick Kossifidis <mick@ics.forth.gr>
11 */
12
13 #define pr_fmt(fmt) "riscv-iommu: " fmt
14
15 #include <linux/acpi.h>
16 #include <linux/acpi_rimt.h>
17 #include <linux/compiler.h>
18 #include <linux/crash_dump.h>
19 #include <linux/init.h>
20 #include <linux/iommu.h>
21 #include <linux/iopoll.h>
22 #include <linux/kernel.h>
23 #include <linux/pci.h>
24 #include <linux/generic_pt/iommu.h>
25
26 #include "../iommu-pages.h"
27 #include "iommu-bits.h"
28 #include "iommu.h"
29
30 /* Timeouts in [us] */
31 #define RISCV_IOMMU_QCSR_TIMEOUT 150000
32 #define RISCV_IOMMU_QUEUE_TIMEOUT 150000
33 #define RISCV_IOMMU_DDTP_TIMEOUT 10000000
34 #define RISCV_IOMMU_IOTINVAL_TIMEOUT 90000000
35
36 /* Number of entries per CMD/FLT queue, should be <= INT_MAX */
37 #define RISCV_IOMMU_DEF_CQ_COUNT 8192
38 #define RISCV_IOMMU_DEF_FQ_COUNT 4096
39
40 /* RISC-V IOMMU PPN <> PHYS address conversions, PHYS <=> PPN[53:10] */
41 #define phys_to_ppn(pa) (((pa) >> 2) & (((1ULL << 44) - 1) << 10))
42 #define ppn_to_phys(pn) (((pn) << 2) & (((1ULL << 44) - 1) << 12))
43
44 #define dev_to_iommu(dev) \
45 iommu_get_iommu_dev(dev, struct riscv_iommu_device, iommu)
46
47 /* IOMMU PSCID allocation namespace. */
48 static DEFINE_IDA(riscv_iommu_pscids);
49 #define RISCV_IOMMU_MAX_PSCID (BIT(20) - 1)
50
51 /* Device resource-managed allocations */
52 struct riscv_iommu_devres {
53 void *addr;
54 };
55
riscv_iommu_devres_pages_release(struct device * dev,void * res)56 static void riscv_iommu_devres_pages_release(struct device *dev, void *res)
57 {
58 struct riscv_iommu_devres *devres = res;
59
60 iommu_free_pages(devres->addr);
61 }
62
riscv_iommu_devres_pages_match(struct device * dev,void * res,void * p)63 static int riscv_iommu_devres_pages_match(struct device *dev, void *res, void *p)
64 {
65 struct riscv_iommu_devres *devres = res;
66 struct riscv_iommu_devres *target = p;
67
68 return devres->addr == target->addr;
69 }
70
riscv_iommu_get_pages(struct riscv_iommu_device * iommu,unsigned int size)71 static void *riscv_iommu_get_pages(struct riscv_iommu_device *iommu,
72 unsigned int size)
73 {
74 struct riscv_iommu_devres *devres;
75 void *addr;
76
77 addr = iommu_alloc_pages_node_sz(dev_to_node(iommu->dev),
78 GFP_KERNEL_ACCOUNT, size);
79 if (unlikely(!addr))
80 return NULL;
81
82 devres = devres_alloc(riscv_iommu_devres_pages_release,
83 sizeof(struct riscv_iommu_devres), GFP_KERNEL);
84
85 if (unlikely(!devres)) {
86 iommu_free_pages(addr);
87 return NULL;
88 }
89
90 devres->addr = addr;
91
92 devres_add(iommu->dev, devres);
93
94 return addr;
95 }
96
riscv_iommu_free_pages(struct riscv_iommu_device * iommu,void * addr)97 static void riscv_iommu_free_pages(struct riscv_iommu_device *iommu, void *addr)
98 {
99 struct riscv_iommu_devres devres = { .addr = addr };
100
101 devres_release(iommu->dev, riscv_iommu_devres_pages_release,
102 riscv_iommu_devres_pages_match, &devres);
103 }
104
105 /*
106 * Hardware queue allocation and management.
107 */
108
109 /* Setup queue base, control registers and default queue length */
110 #define RISCV_IOMMU_QUEUE_INIT(q, name) do { \
111 struct riscv_iommu_queue *_q = q; \
112 _q->qid = RISCV_IOMMU_INTR_ ## name; \
113 _q->qbr = RISCV_IOMMU_REG_ ## name ## B; \
114 _q->qcr = RISCV_IOMMU_REG_ ## name ## CSR; \
115 _q->mask = _q->mask ?: (RISCV_IOMMU_DEF_ ## name ## _COUNT) - 1;\
116 } while (0)
117
118 /* Note: offsets are the same for all queues */
119 #define Q_HEAD(q) ((q)->qbr + (RISCV_IOMMU_REG_CQH - RISCV_IOMMU_REG_CQB))
120 #define Q_TAIL(q) ((q)->qbr + (RISCV_IOMMU_REG_CQT - RISCV_IOMMU_REG_CQB))
121 #define Q_ITEM(q, index) ((q)->mask & (index))
122 #define Q_IPSR(q) BIT((q)->qid)
123
124 /*
125 * Discover queue ring buffer hardware configuration, allocate in-memory
126 * ring buffer or use fixed I/O memory location, configure queue base register.
127 * Must be called before hardware queue is enabled.
128 *
129 * @queue - data structure, configured with RISCV_IOMMU_QUEUE_INIT()
130 * @entry_size - queue single element size in bytes.
131 */
riscv_iommu_queue_alloc(struct riscv_iommu_device * iommu,struct riscv_iommu_queue * queue,size_t entry_size)132 static int riscv_iommu_queue_alloc(struct riscv_iommu_device *iommu,
133 struct riscv_iommu_queue *queue,
134 size_t entry_size)
135 {
136 unsigned int logsz;
137 u64 qb, rb;
138
139 /*
140 * Use WARL base register property to discover maximum allowed
141 * number of entries and optional fixed IO address for queue location.
142 */
143 riscv_iommu_writeq(iommu, queue->qbr, RISCV_IOMMU_QUEUE_LOG2SZ_FIELD);
144 qb = riscv_iommu_readq(iommu, queue->qbr);
145
146 /*
147 * Calculate and verify hardware supported queue length, as reported
148 * by the field LOG2SZ, where max queue length is equal to 2^(LOG2SZ + 1).
149 * Update queue size based on hardware supported value.
150 */
151 logsz = ilog2(queue->mask);
152 if (logsz > FIELD_GET(RISCV_IOMMU_QUEUE_LOG2SZ_FIELD, qb))
153 logsz = FIELD_GET(RISCV_IOMMU_QUEUE_LOG2SZ_FIELD, qb);
154
155 /*
156 * Use WARL base register property to discover an optional fixed IO
157 * address for queue ring buffer location. Otherwise allocate contiguous
158 * system memory.
159 */
160 if (FIELD_GET(RISCV_IOMMU_PPN_FIELD, qb)) {
161 const size_t queue_size = entry_size << (logsz + 1);
162
163 queue->phys = PFN_PHYS(FIELD_GET(RISCV_IOMMU_PPN_FIELD, qb));
164 queue->base = devm_ioremap(iommu->dev, queue->phys, queue_size);
165 } else {
166 do {
167 const size_t queue_size = entry_size << (logsz + 1);
168
169 queue->base = riscv_iommu_get_pages(
170 iommu, max(queue_size, SZ_4K));
171 queue->phys = __pa(queue->base);
172 } while (!queue->base && logsz-- > 0);
173 }
174
175 if (!queue->base)
176 return -ENOMEM;
177
178 qb = phys_to_ppn(queue->phys) |
179 FIELD_PREP(RISCV_IOMMU_QUEUE_LOG2SZ_FIELD, logsz);
180
181 /* Update base register and read back to verify hw accepted our write */
182 riscv_iommu_writeq(iommu, queue->qbr, qb);
183 rb = riscv_iommu_readq(iommu, queue->qbr);
184 if (rb != qb) {
185 dev_err(iommu->dev, "queue #%u allocation failed\n", queue->qid);
186 return -ENODEV;
187 }
188
189 /* Update actual queue mask */
190 queue->mask = (2U << logsz) - 1;
191
192 dev_dbg(iommu->dev, "queue #%u allocated 2^%u entries",
193 queue->qid, logsz + 1);
194
195 return 0;
196 }
197
198 /* Check interrupt queue status, IPSR */
riscv_iommu_queue_ipsr(int irq,void * data)199 static irqreturn_t riscv_iommu_queue_ipsr(int irq, void *data)
200 {
201 struct riscv_iommu_queue *queue = (struct riscv_iommu_queue *)data;
202
203 if (riscv_iommu_readl(queue->iommu, RISCV_IOMMU_REG_IPSR) & Q_IPSR(queue))
204 return IRQ_WAKE_THREAD;
205
206 return IRQ_NONE;
207 }
208
riscv_iommu_queue_vec(struct riscv_iommu_device * iommu,int n)209 static int riscv_iommu_queue_vec(struct riscv_iommu_device *iommu, int n)
210 {
211 /* Reuse ICVEC.CIV mask for all interrupt vectors mapping. */
212 return (iommu->icvec >> (n * 4)) & RISCV_IOMMU_ICVEC_CIV;
213 }
214
215 /*
216 * Enable queue processing in the hardware, register interrupt handler.
217 *
218 * @queue - data structure, already allocated with riscv_iommu_queue_alloc()
219 * @irq_handler - threaded interrupt handler.
220 */
riscv_iommu_queue_enable(struct riscv_iommu_device * iommu,struct riscv_iommu_queue * queue,irq_handler_t irq_handler)221 static int riscv_iommu_queue_enable(struct riscv_iommu_device *iommu,
222 struct riscv_iommu_queue *queue,
223 irq_handler_t irq_handler)
224 {
225 const unsigned int irq = iommu->irqs[riscv_iommu_queue_vec(iommu, queue->qid)];
226 u32 csr;
227 int rc;
228
229 if (queue->iommu)
230 return -EBUSY;
231
232 /* Polling not implemented */
233 if (!irq)
234 return -ENODEV;
235
236 queue->iommu = iommu;
237 rc = request_threaded_irq(irq, riscv_iommu_queue_ipsr, irq_handler,
238 IRQF_ONESHOT | IRQF_SHARED,
239 dev_name(iommu->dev), queue);
240 if (rc) {
241 queue->iommu = NULL;
242 return rc;
243 }
244
245 /* Empty queue before enabling it */
246 if (queue->qid == RISCV_IOMMU_INTR_CQ)
247 riscv_iommu_writel(queue->iommu, Q_TAIL(queue), 0);
248 else
249 riscv_iommu_writel(queue->iommu, Q_HEAD(queue), 0);
250
251 /*
252 * Enable queue with interrupts, clear any memory fault if any.
253 * Wait for the hardware to acknowledge request and activate queue
254 * processing.
255 * Note: All CSR bitfields are in the same offsets for all queues.
256 */
257 riscv_iommu_writel(iommu, queue->qcr,
258 RISCV_IOMMU_QUEUE_ENABLE |
259 RISCV_IOMMU_QUEUE_INTR_ENABLE |
260 RISCV_IOMMU_QUEUE_MEM_FAULT);
261
262 riscv_iommu_readl_timeout(iommu, queue->qcr,
263 csr, !(csr & RISCV_IOMMU_QUEUE_BUSY),
264 10, RISCV_IOMMU_QCSR_TIMEOUT);
265
266 if (RISCV_IOMMU_QUEUE_ACTIVE != (csr & (RISCV_IOMMU_QUEUE_ACTIVE |
267 RISCV_IOMMU_QUEUE_BUSY |
268 RISCV_IOMMU_QUEUE_MEM_FAULT))) {
269 /* Best effort to stop and disable failing hardware queue. */
270 riscv_iommu_writel(iommu, queue->qcr, 0);
271 free_irq(irq, queue);
272 queue->iommu = NULL;
273 dev_err(iommu->dev, "queue #%u failed to start\n", queue->qid);
274 return -EBUSY;
275 }
276
277 /* Clear any pending interrupt flag. */
278 riscv_iommu_writel(iommu, RISCV_IOMMU_REG_IPSR, Q_IPSR(queue));
279
280 return 0;
281 }
282
283 /*
284 * Disable queue. Wait for the hardware to acknowledge request and
285 * stop processing enqueued requests. Report errors but continue.
286 */
riscv_iommu_queue_disable(struct riscv_iommu_queue * queue)287 static void riscv_iommu_queue_disable(struct riscv_iommu_queue *queue)
288 {
289 struct riscv_iommu_device *iommu = queue->iommu;
290 u32 csr;
291
292 if (!iommu)
293 return;
294
295 free_irq(iommu->irqs[riscv_iommu_queue_vec(iommu, queue->qid)], queue);
296 riscv_iommu_writel(iommu, queue->qcr, 0);
297 riscv_iommu_readl_timeout(iommu, queue->qcr,
298 csr, !(csr & RISCV_IOMMU_QUEUE_BUSY),
299 10, RISCV_IOMMU_QCSR_TIMEOUT);
300
301 if (csr & (RISCV_IOMMU_QUEUE_ACTIVE | RISCV_IOMMU_QUEUE_BUSY))
302 dev_err(iommu->dev, "fail to disable hardware queue #%u, csr 0x%x\n",
303 queue->qid, csr);
304
305 queue->iommu = NULL;
306 }
307
308 /*
309 * Returns number of available valid queue entries and the first item index.
310 * Update shadow producer index if necessary.
311 */
riscv_iommu_queue_consume(struct riscv_iommu_queue * queue,unsigned int * index)312 static int riscv_iommu_queue_consume(struct riscv_iommu_queue *queue,
313 unsigned int *index)
314 {
315 unsigned int head = atomic_read(&queue->head);
316 unsigned int tail = atomic_read(&queue->tail);
317 unsigned int last = Q_ITEM(queue, tail);
318 int available = (int)(tail - head);
319
320 *index = head;
321
322 if (available > 0)
323 return available;
324
325 /* read hardware producer index, check reserved register bits are not set. */
326 if (riscv_iommu_readl_timeout(queue->iommu, Q_TAIL(queue),
327 tail, (tail & ~queue->mask) == 0,
328 0, RISCV_IOMMU_QUEUE_TIMEOUT)) {
329 dev_err_once(queue->iommu->dev,
330 "Hardware error: queue access timeout\n");
331 return 0;
332 }
333
334 if (tail == last)
335 return 0;
336
337 /* update shadow producer index */
338 return (int)(atomic_add_return((tail - last) & queue->mask, &queue->tail) - head);
339 }
340
341 /*
342 * Release processed queue entries, should match riscv_iommu_queue_consume() calls.
343 */
riscv_iommu_queue_release(struct riscv_iommu_queue * queue,int count)344 static void riscv_iommu_queue_release(struct riscv_iommu_queue *queue, int count)
345 {
346 const unsigned int head = atomic_add_return(count, &queue->head);
347
348 riscv_iommu_writel(queue->iommu, Q_HEAD(queue), Q_ITEM(queue, head));
349 }
350
351 /* Return actual consumer index based on hardware reported queue head index. */
riscv_iommu_queue_cons(struct riscv_iommu_queue * queue)352 static unsigned int riscv_iommu_queue_cons(struct riscv_iommu_queue *queue)
353 {
354 const unsigned int cons = atomic_read(&queue->head);
355 const unsigned int last = Q_ITEM(queue, cons);
356 unsigned int head;
357
358 if (riscv_iommu_readl_timeout(queue->iommu, Q_HEAD(queue), head,
359 !(head & ~queue->mask),
360 0, RISCV_IOMMU_QUEUE_TIMEOUT))
361 return cons;
362
363 return cons + ((head - last) & queue->mask);
364 }
365
366 /* Wait for submitted item to be processed. */
riscv_iommu_queue_wait(struct riscv_iommu_queue * queue,unsigned int index,unsigned int timeout_us)367 static int riscv_iommu_queue_wait(struct riscv_iommu_queue *queue,
368 unsigned int index,
369 unsigned int timeout_us)
370 {
371 unsigned int cons = atomic_read(&queue->head);
372 unsigned int flags = RISCV_IOMMU_CQCSR_CQMF | RISCV_IOMMU_CQCSR_CMD_TO |
373 RISCV_IOMMU_CQCSR_CMD_ILL;
374
375 /* Already processed by the consumer */
376 if ((int)(cons - index) > 0)
377 return 0;
378
379 /* Monitor consumer index */
380 return readx_poll_timeout(riscv_iommu_queue_cons, queue, cons,
381 (riscv_iommu_readl(queue->iommu, queue->qcr) & flags) ||
382 (int)(cons - index) > 0, 0, timeout_us);
383 }
384
riscv_iommu_queue_wait_for_space(struct riscv_iommu_queue * queue,unsigned int last)385 static int riscv_iommu_queue_wait_for_space(struct riscv_iommu_queue *queue,
386 unsigned int last)
387 {
388 unsigned int head;
389 unsigned int tail;
390 unsigned int hw_head;
391 unsigned long flags;
392 int ret;
393
394 ret = riscv_iommu_readl_timeout(queue->iommu, Q_HEAD(queue), hw_head,
395 !(hw_head & ~queue->mask) && hw_head != last,
396 0, RISCV_IOMMU_QUEUE_TIMEOUT);
397 if (ret)
398 return ret;
399
400 raw_spin_lock_irqsave(&queue->lock, flags);
401 head = atomic_read(&queue->head);
402 tail = atomic_read(&queue->tail);
403 if ((tail - head) >= queue->mask) {
404 last = Q_ITEM(queue, head);
405 /*
406 * Re-read hw_head under the lock so that it is consistent with
407 * the freshly computed 'last'. Using the pre-lock snapshot
408 * could produce a stale value that wraps around relative to the
409 * new 'last', advancing the shadow head past entries that have
410 * not yet been consumed by the hardware.
411 */
412 hw_head = riscv_iommu_readl(queue->iommu, Q_HEAD(queue));
413 if (!(hw_head & ~queue->mask) && hw_head != last)
414 atomic_add((hw_head - last) & queue->mask, &queue->head);
415 }
416 raw_spin_unlock_irqrestore(&queue->lock, flags);
417
418 return 0;
419 }
420
421 /* Enqueue an entry and publish it to the hardware queue. */
riscv_iommu_queue_send(struct riscv_iommu_queue * queue,void * entry,size_t entry_size,unsigned int * out_prod)422 static int riscv_iommu_queue_send(struct riscv_iommu_queue *queue,
423 void *entry, size_t entry_size,
424 unsigned int *out_prod)
425 {
426 unsigned int prod;
427 unsigned int head;
428 unsigned long flags;
429 int ret;
430
431 /* 1. Wait for space availability and reserve the next slot. */
432 for (;;) {
433 raw_spin_lock_irqsave(&queue->lock, flags);
434
435 prod = atomic_read(&queue->tail);
436 head = atomic_read(&queue->head);
437
438 if ((prod - head) < queue->mask)
439 break;
440
441 head = Q_ITEM(queue, head);
442 raw_spin_unlock_irqrestore(&queue->lock, flags);
443
444 ret = riscv_iommu_queue_wait_for_space(queue, head);
445 if (ret)
446 goto err_busy;
447 }
448
449 /* 2. Store entry in the ring buffer. */
450 memcpy(queue->base + Q_ITEM(queue, prod) * entry_size, entry, entry_size);
451
452 /* 3. Make sure the entry is visible before updating the queue tail. */
453 dma_wmb();
454 riscv_iommu_writel(queue->iommu, Q_TAIL(queue), Q_ITEM(queue, prod + 1));
455
456 /*
457 * 4. Make sure the doorbell write to the device has finished before
458 * updating the shadow tail index in normal memory. 'fence o, w'
459 */
460 #ifdef CONFIG_MMIOWB
461 mmiowb();
462 #endif
463 atomic_set(&queue->tail, prod + 1);
464 atomic_set(&queue->prod, prod + 1);
465
466 if (out_prod)
467 *out_prod = prod;
468
469 raw_spin_unlock_irqrestore(&queue->lock, flags);
470 return 0;
471
472 err_busy:
473 /* Report the failure and continue; full RAS recovery is not implemented. */
474 dev_err_once(queue->iommu->dev, "Hardware error: command enqueue failed\n");
475 return ret;
476 }
477
478 /*
479 * IOMMU Command queue chapter 3.1
480 */
481
482 /* Command queue interrupt handler thread function */
riscv_iommu_cmdq_process(int irq,void * data)483 static irqreturn_t riscv_iommu_cmdq_process(int irq, void *data)
484 {
485 const struct riscv_iommu_queue *queue = (struct riscv_iommu_queue *)data;
486 unsigned int ctrl;
487
488 /* Clear MF/CQ errors, complete error recovery to be implemented. */
489 ctrl = riscv_iommu_readl(queue->iommu, queue->qcr);
490 if (ctrl & (RISCV_IOMMU_CQCSR_CQMF | RISCV_IOMMU_CQCSR_CMD_TO |
491 RISCV_IOMMU_CQCSR_CMD_ILL | RISCV_IOMMU_CQCSR_FENCE_W_IP)) {
492 riscv_iommu_writel(queue->iommu, queue->qcr, ctrl);
493 dev_warn(queue->iommu->dev,
494 "Queue #%u error; fault:%d timeout:%d illegal:%d fence_w_ip:%d\n",
495 queue->qid,
496 !!(ctrl & RISCV_IOMMU_CQCSR_CQMF),
497 !!(ctrl & RISCV_IOMMU_CQCSR_CMD_TO),
498 !!(ctrl & RISCV_IOMMU_CQCSR_CMD_ILL),
499 !!(ctrl & RISCV_IOMMU_CQCSR_FENCE_W_IP));
500 }
501
502 /* Placeholder for command queue interrupt notifiers */
503
504 /* Clear command interrupt pending. */
505 riscv_iommu_writel(queue->iommu, RISCV_IOMMU_REG_IPSR, Q_IPSR(queue));
506
507 return IRQ_HANDLED;
508 }
509
510 /* Send command to the IOMMU command queue */
riscv_iommu_cmd_send(struct riscv_iommu_device * iommu,struct riscv_iommu_command * cmd)511 static void riscv_iommu_cmd_send(struct riscv_iommu_device *iommu,
512 struct riscv_iommu_command *cmd)
513 {
514 riscv_iommu_queue_send(&iommu->cmdq, cmd, sizeof(*cmd), NULL);
515 }
516
517 /* Send IOFENCE.C command and wait for all scheduled commands to complete. */
riscv_iommu_cmd_sync(struct riscv_iommu_device * iommu,unsigned int timeout_us)518 static void riscv_iommu_cmd_sync(struct riscv_iommu_device *iommu,
519 unsigned int timeout_us)
520 {
521 struct riscv_iommu_command cmd;
522 unsigned int prod;
523 int ret;
524
525 riscv_iommu_cmd_iofence(&cmd);
526 ret = riscv_iommu_queue_send(&iommu->cmdq, &cmd, sizeof(cmd), &prod);
527 if (ret)
528 return;
529
530 if (!timeout_us)
531 return;
532
533 if (riscv_iommu_queue_wait(&iommu->cmdq, prod, timeout_us))
534 dev_err_once(iommu->dev,
535 "Hardware error: command execution timeout\n");
536 }
537
538 /*
539 * IOMMU Fault/Event queue chapter 3.2
540 */
541
riscv_iommu_fault(struct riscv_iommu_device * iommu,struct riscv_iommu_fq_record * event)542 static void riscv_iommu_fault(struct riscv_iommu_device *iommu,
543 struct riscv_iommu_fq_record *event)
544 {
545 unsigned int err = FIELD_GET(RISCV_IOMMU_FQ_HDR_CAUSE, event->hdr);
546 unsigned int devid = FIELD_GET(RISCV_IOMMU_FQ_HDR_DID, event->hdr);
547
548 /* Placeholder for future fault handling implementation, report only. */
549 if (err)
550 dev_warn_ratelimited(iommu->dev,
551 "Fault %d devid: 0x%x iotval: %llx iotval2: %llx\n",
552 err, devid, event->iotval, event->iotval2);
553 }
554
555 /* Fault queue interrupt handler thread function */
riscv_iommu_fltq_process(int irq,void * data)556 static irqreturn_t riscv_iommu_fltq_process(int irq, void *data)
557 {
558 struct riscv_iommu_queue *queue = (struct riscv_iommu_queue *)data;
559 struct riscv_iommu_device *iommu = queue->iommu;
560 struct riscv_iommu_fq_record *events;
561 unsigned int ctrl, idx;
562 int cnt, len;
563
564 events = (struct riscv_iommu_fq_record *)queue->base;
565
566 /* Clear fault interrupt pending and process all received fault events. */
567 riscv_iommu_writel(iommu, RISCV_IOMMU_REG_IPSR, Q_IPSR(queue));
568
569 do {
570 cnt = riscv_iommu_queue_consume(queue, &idx);
571 for (len = 0; len < cnt; idx++, len++)
572 riscv_iommu_fault(iommu, &events[Q_ITEM(queue, idx)]);
573 riscv_iommu_queue_release(queue, cnt);
574 } while (cnt > 0);
575
576 /* Clear MF/OF errors, complete error recovery to be implemented. */
577 ctrl = riscv_iommu_readl(iommu, queue->qcr);
578 if (ctrl & (RISCV_IOMMU_FQCSR_FQMF | RISCV_IOMMU_FQCSR_FQOF)) {
579 riscv_iommu_writel(iommu, queue->qcr, ctrl);
580 dev_warn(iommu->dev,
581 "Queue #%u error; memory fault:%d overflow:%d\n",
582 queue->qid,
583 !!(ctrl & RISCV_IOMMU_FQCSR_FQMF),
584 !!(ctrl & RISCV_IOMMU_FQCSR_FQOF));
585 }
586
587 return IRQ_HANDLED;
588 }
589
590 /* Lookup and initialize device context info structure. */
riscv_iommu_get_dc(struct riscv_iommu_device * iommu,unsigned int devid)591 static struct riscv_iommu_dc *riscv_iommu_get_dc(struct riscv_iommu_device *iommu,
592 unsigned int devid)
593 {
594 const bool base_format = !(iommu->caps & RISCV_IOMMU_CAPABILITIES_MSI_FLAT);
595 unsigned int depth;
596 unsigned long ddt, old, new;
597 void *ptr;
598 u8 ddi_bits[3] = { 0 };
599 u64 *ddtp = NULL;
600
601 /* Make sure the mode is valid */
602 if (iommu->ddt_mode < RISCV_IOMMU_DDTP_IOMMU_MODE_1LVL ||
603 iommu->ddt_mode > RISCV_IOMMU_DDTP_IOMMU_MODE_3LVL)
604 return NULL;
605
606 /*
607 * Device id partitioning for base format:
608 * DDI[0]: bits 0 - 6 (1st level) (7 bits)
609 * DDI[1]: bits 7 - 15 (2nd level) (9 bits)
610 * DDI[2]: bits 16 - 23 (3rd level) (8 bits)
611 *
612 * For extended format:
613 * DDI[0]: bits 0 - 5 (1st level) (6 bits)
614 * DDI[1]: bits 6 - 14 (2nd level) (9 bits)
615 * DDI[2]: bits 15 - 23 (3rd level) (9 bits)
616 */
617 if (base_format) {
618 ddi_bits[0] = 7;
619 ddi_bits[1] = 7 + 9;
620 ddi_bits[2] = 7 + 9 + 8;
621 } else {
622 ddi_bits[0] = 6;
623 ddi_bits[1] = 6 + 9;
624 ddi_bits[2] = 6 + 9 + 9;
625 }
626
627 /* Make sure device id is within range */
628 depth = iommu->ddt_mode - RISCV_IOMMU_DDTP_IOMMU_MODE_1LVL;
629 if (devid >= (1 << ddi_bits[depth]))
630 return NULL;
631
632 /* Get to the level of the non-leaf node that holds the device context */
633 for (ddtp = iommu->ddt_root; depth-- > 0;) {
634 const int split = ddi_bits[depth];
635 /*
636 * Each non-leaf node is 64bits wide and on each level
637 * nodes are indexed by DDI[depth].
638 */
639 ddtp += (devid >> split) & 0x1FF;
640
641 /*
642 * Check if this node has been populated and if not
643 * allocate a new level and populate it.
644 */
645 do {
646 ddt = READ_ONCE(*(unsigned long *)ddtp);
647 if (ddt & RISCV_IOMMU_DDTE_V) {
648 ddtp = __va(ppn_to_phys(ddt));
649 break;
650 }
651
652 ptr = riscv_iommu_get_pages(iommu, SZ_4K);
653 if (!ptr)
654 return NULL;
655
656 new = phys_to_ppn(__pa(ptr)) | RISCV_IOMMU_DDTE_V;
657 old = cmpxchg_relaxed((unsigned long *)ddtp, ddt, new);
658
659 if (old == ddt) {
660 ddtp = (u64 *)ptr;
661 break;
662 }
663
664 /* Race setting DDT detected, re-read and retry. */
665 riscv_iommu_free_pages(iommu, ptr);
666 } while (1);
667 }
668
669 /*
670 * Grab the node that matches DDI[depth], note that when using base
671 * format the device context is 4 * 64bits, and the extended format
672 * is 8 * 64bits, hence the (3 - base_format) below.
673 */
674 ddtp += (devid & ((64 << base_format) - 1)) << (3 - base_format);
675
676 return (struct riscv_iommu_dc *)ddtp;
677 }
678
679 /*
680 * This is best effort IOMMU translation shutdown flow.
681 * Disable IOMMU without waiting for hardware response.
682 */
riscv_iommu_disable(struct riscv_iommu_device * iommu)683 void riscv_iommu_disable(struct riscv_iommu_device *iommu)
684 {
685 riscv_iommu_writeq(iommu, RISCV_IOMMU_REG_DDTP,
686 FIELD_PREP(RISCV_IOMMU_DDTP_IOMMU_MODE,
687 RISCV_IOMMU_DDTP_IOMMU_MODE_BARE));
688 riscv_iommu_writel(iommu, RISCV_IOMMU_REG_CQCSR, 0);
689 riscv_iommu_writel(iommu, RISCV_IOMMU_REG_FQCSR, 0);
690 riscv_iommu_writel(iommu, RISCV_IOMMU_REG_PQCSR, 0);
691 }
692
693 #define riscv_iommu_read_ddtp(iommu) ({ \
694 u64 ddtp; \
695 riscv_iommu_readq_timeout((iommu), RISCV_IOMMU_REG_DDTP, ddtp, \
696 !(ddtp & RISCV_IOMMU_DDTP_BUSY), 10, \
697 RISCV_IOMMU_DDTP_TIMEOUT); \
698 ddtp; })
699
riscv_iommu_iodir_alloc(struct riscv_iommu_device * iommu)700 static int riscv_iommu_iodir_alloc(struct riscv_iommu_device *iommu)
701 {
702 u64 ddtp;
703 unsigned int mode;
704
705 ddtp = riscv_iommu_read_ddtp(iommu);
706 if (ddtp & RISCV_IOMMU_DDTP_BUSY)
707 return -EBUSY;
708
709 /*
710 * It is optional for the hardware to report a fixed address for device
711 * directory root page when DDT.MODE is OFF or BARE.
712 */
713 mode = FIELD_GET(RISCV_IOMMU_DDTP_IOMMU_MODE, ddtp);
714 if (mode == RISCV_IOMMU_DDTP_IOMMU_MODE_BARE ||
715 mode == RISCV_IOMMU_DDTP_IOMMU_MODE_OFF) {
716 /* Use WARL to discover hardware fixed DDT PPN */
717 riscv_iommu_writeq(iommu, RISCV_IOMMU_REG_DDTP,
718 FIELD_PREP(RISCV_IOMMU_DDTP_IOMMU_MODE, mode));
719 ddtp = riscv_iommu_read_ddtp(iommu);
720 if (ddtp & RISCV_IOMMU_DDTP_BUSY)
721 return -EBUSY;
722
723 iommu->ddt_phys = ppn_to_phys(ddtp);
724 if (iommu->ddt_phys)
725 iommu->ddt_root = devm_ioremap(iommu->dev,
726 iommu->ddt_phys, PAGE_SIZE);
727 if (iommu->ddt_root)
728 memset(iommu->ddt_root, 0, PAGE_SIZE);
729 }
730
731 if (!iommu->ddt_root) {
732 iommu->ddt_root = riscv_iommu_get_pages(iommu, SZ_4K);
733 iommu->ddt_phys = __pa(iommu->ddt_root);
734 }
735
736 if (!iommu->ddt_root)
737 return -ENOMEM;
738
739 return 0;
740 }
741
742 /*
743 * Discover supported DDT modes starting from requested value,
744 * configure DDTP register with accepted mode and root DDT address.
745 * Accepted iommu->ddt_mode is updated on success.
746 */
riscv_iommu_iodir_set_mode(struct riscv_iommu_device * iommu,unsigned int ddtp_mode)747 static int riscv_iommu_iodir_set_mode(struct riscv_iommu_device *iommu,
748 unsigned int ddtp_mode)
749 {
750 struct device *dev = iommu->dev;
751 u64 ddtp, rq_ddtp;
752 unsigned int mode, rq_mode = ddtp_mode;
753 struct riscv_iommu_command cmd;
754
755 ddtp = riscv_iommu_read_ddtp(iommu);
756 if (ddtp & RISCV_IOMMU_DDTP_BUSY)
757 return -EBUSY;
758
759 /* Disallow state transition from xLVL to xLVL. */
760 mode = FIELD_GET(RISCV_IOMMU_DDTP_IOMMU_MODE, ddtp);
761 if (mode != RISCV_IOMMU_DDTP_IOMMU_MODE_BARE &&
762 mode != RISCV_IOMMU_DDTP_IOMMU_MODE_OFF &&
763 rq_mode != RISCV_IOMMU_DDTP_IOMMU_MODE_BARE &&
764 rq_mode != RISCV_IOMMU_DDTP_IOMMU_MODE_OFF)
765 return -EINVAL;
766
767 do {
768 rq_ddtp = FIELD_PREP(RISCV_IOMMU_DDTP_IOMMU_MODE, rq_mode);
769 if (rq_mode > RISCV_IOMMU_DDTP_IOMMU_MODE_BARE)
770 rq_ddtp |= phys_to_ppn(iommu->ddt_phys);
771
772 riscv_iommu_writeq(iommu, RISCV_IOMMU_REG_DDTP, rq_ddtp);
773 ddtp = riscv_iommu_read_ddtp(iommu);
774 if (ddtp & RISCV_IOMMU_DDTP_BUSY) {
775 dev_err(dev, "timeout when setting ddtp (ddt mode: %u, read: %llx)\n",
776 rq_mode, ddtp);
777 return -EBUSY;
778 }
779
780 /* Verify IOMMU hardware accepts new DDTP config. */
781 mode = FIELD_GET(RISCV_IOMMU_DDTP_IOMMU_MODE, ddtp);
782
783 if (rq_mode == mode)
784 break;
785
786 /* Hardware mandatory DDTP mode has not been accepted. */
787 if (rq_mode < RISCV_IOMMU_DDTP_IOMMU_MODE_1LVL && rq_ddtp != ddtp) {
788 dev_err(dev, "DDTP update failed hw: %llx vs %llx\n",
789 ddtp, rq_ddtp);
790 return -EINVAL;
791 }
792
793 /*
794 * Mode field is WARL, an IOMMU may support a subset of
795 * directory table levels in which case if we tried to set
796 * an unsupported number of levels we'll readback either
797 * a valid xLVL or off/bare. If we got off/bare, try again
798 * with a smaller xLVL.
799 */
800 if (mode < RISCV_IOMMU_DDTP_IOMMU_MODE_1LVL &&
801 rq_mode > RISCV_IOMMU_DDTP_IOMMU_MODE_1LVL) {
802 dev_dbg(dev, "DDTP hw mode %u vs %u\n", mode, rq_mode);
803 rq_mode--;
804 continue;
805 }
806
807 /*
808 * We tried all supported modes and IOMMU hardware failed to
809 * accept new settings, something went very wrong since off/bare
810 * and at least one xLVL must be supported.
811 */
812 dev_err(dev, "DDTP hw mode %u, failed to set %u\n",
813 mode, ddtp_mode);
814 return -EINVAL;
815 } while (1);
816
817 iommu->ddt_mode = mode;
818 if (mode != ddtp_mode)
819 dev_dbg(dev, "DDTP hw mode %u, requested %u\n", mode, ddtp_mode);
820
821 /* Invalidate device context cache */
822 riscv_iommu_cmd_iodir_inval_ddt(&cmd);
823 riscv_iommu_cmd_send(iommu, &cmd);
824
825 /* Invalidate address translation cache */
826 riscv_iommu_cmd_inval_vma(&cmd);
827 riscv_iommu_cmd_send(iommu, &cmd);
828
829 /* IOFENCE.C */
830 riscv_iommu_cmd_sync(iommu, RISCV_IOMMU_IOTINVAL_TIMEOUT);
831
832 return 0;
833 }
834
835 /* This struct contains protection domain specific IOMMU driver data. */
836 struct riscv_iommu_domain {
837 union {
838 struct iommu_domain domain;
839 struct pt_iommu_riscv_64 riscvpt;
840 };
841 struct list_head bonds;
842 spinlock_t lock; /* protect bonds list updates. */
843 int pscid;
844 };
845 PT_IOMMU_CHECK_DOMAIN(struct riscv_iommu_domain, riscvpt.iommu, domain);
846
847 #define iommu_domain_to_riscv(iommu_domain) \
848 container_of(iommu_domain, struct riscv_iommu_domain, domain)
849
850 /* Private IOMMU data for managed devices, dev_iommu_priv_* */
851 struct riscv_iommu_info {
852 struct riscv_iommu_domain *domain;
853 };
854
855 /*
856 * Linkage between an iommu_domain and attached devices.
857 *
858 * Protection domain requiring IOATC and DevATC translation cache invalidations,
859 * should be linked to attached devices using a riscv_iommu_bond structure.
860 * Devices should be linked to the domain before first use and unlinked after
861 * the translations from the referenced protection domain can no longer be used.
862 * Blocking and identity domains are not tracked here, as the IOMMU hardware
863 * does not cache negative and/or identity (BARE mode) translations, and DevATC
864 * is disabled for those protection domains.
865 *
866 * The device pointer and IOMMU data remain stable in the bond struct after
867 * _probe_device() where it's attached to the managed IOMMU, up to the
868 * completion of the _release_device() call. The release of the bond structure
869 * is synchronized with the device release.
870 */
871 struct riscv_iommu_bond {
872 struct list_head list;
873 struct rcu_head rcu;
874 struct device *dev;
875 };
876
riscv_iommu_bond_link(struct riscv_iommu_domain * domain,struct device * dev)877 static int riscv_iommu_bond_link(struct riscv_iommu_domain *domain,
878 struct device *dev)
879 {
880 struct riscv_iommu_device *iommu = dev_to_iommu(dev);
881 struct riscv_iommu_bond *bond;
882 struct list_head *bonds;
883
884 bond = kzalloc_obj(*bond);
885 if (!bond)
886 return -ENOMEM;
887 bond->dev = dev;
888
889 /*
890 * List of devices attached to the domain is arranged based on
891 * managed IOMMU device.
892 */
893
894 spin_lock(&domain->lock);
895 list_for_each(bonds, &domain->bonds)
896 if (dev_to_iommu(list_entry(bonds, struct riscv_iommu_bond, list)->dev) == iommu)
897 break;
898 list_add_rcu(&bond->list, bonds);
899 spin_unlock(&domain->lock);
900
901 /* Synchronize with riscv_iommu_iotlb_inval() sequence. See comment below. */
902 smp_mb();
903
904 return 0;
905 }
906
riscv_iommu_bond_unlink(struct riscv_iommu_domain * domain,struct device * dev)907 static void riscv_iommu_bond_unlink(struct riscv_iommu_domain *domain,
908 struct device *dev)
909 {
910 struct riscv_iommu_device *iommu = dev_to_iommu(dev);
911 struct riscv_iommu_bond *bond, *found = NULL;
912 struct riscv_iommu_command cmd;
913 int count = 0;
914
915 if (!domain)
916 return;
917
918 spin_lock(&domain->lock);
919 list_for_each_entry(bond, &domain->bonds, list) {
920 if (found && count)
921 break;
922 else if (bond->dev == dev)
923 found = bond;
924 else if (dev_to_iommu(bond->dev) == iommu)
925 count++;
926 }
927 if (found)
928 list_del_rcu(&found->list);
929 spin_unlock(&domain->lock);
930 kfree_rcu(found, rcu);
931
932 /*
933 * If this was the last bond between this domain and the IOMMU
934 * invalidate all cached entries for domain's PSCID.
935 */
936 if (!count) {
937 riscv_iommu_cmd_inval_vma(&cmd);
938 riscv_iommu_cmd_inval_set_pscid(&cmd, domain->pscid);
939 riscv_iommu_cmd_send(iommu, &cmd);
940
941 riscv_iommu_cmd_sync(iommu, RISCV_IOMMU_IOTINVAL_TIMEOUT);
942 }
943 }
944
945 struct riscv_iommu_tlbi {
946 u64 start;
947 u64 last;
948 bool non_leaf;
949 struct {
950 bool use_global;
951 u8 stride_lg2;
952 unsigned int num;
953 } single;
954 struct {
955 u8 sz_lg2;
956 u64 addr;
957 } range;
958 };
959
riscv_iommu_tlbi_calc(struct riscv_iommu_tlbi * tlbi,struct iommu_iotlb_gather * gather)960 static void riscv_iommu_tlbi_calc(struct riscv_iommu_tlbi *tlbi,
961 struct iommu_iotlb_gather *gather)
962 {
963 u8 combined = gather->pt.leaf_levels_bitmap |
964 gather->pt.table_levels_bitmap;
965 u64 num;
966
967 tlbi->non_leaf = gather->pt.table_levels_bitmap != 0;
968 tlbi->start = gather->start;
969 tlbi->last = gather->end;
970
971 /* No level information available */
972 if (!combined) {
973 tlbi->single.use_global = true;
974 tlbi->range.sz_lg2 = 0;
975 return;
976 }
977
978 /*
979 * Calculate the smallest NAPOT range containing [start, last].
980 * NAPOT encoding requires a power-of-two sized, naturally aligned
981 * range. Over-invalidation is always safe.
982 */
983 tlbi->range.sz_lg2 = fls64(tlbi->start ^ tlbi->last);
984 if (unlikely(tlbi->range.sz_lg2 >= 64)) {
985 tlbi->single.use_global = true;
986 tlbi->range.sz_lg2 = 0;
987 return;
988 }
989 tlbi->range.addr = tlbi->start & ~(BIT_U64(tlbi->range.sz_lg2) - 1);
990
991 /*
992 * Calculate stride from the lowest changed level. RISC-V uses 4KiB
993 * granule with 9 bits per level.
994 */
995 tlbi->single.stride_lg2 = 9 * __ffs(combined) + 12;
996 num = (tlbi->last - tlbi->start + 1) >> tlbi->single.stride_lg2;
997 if (!num || num > 512) {
998 tlbi->single.use_global = true;
999 } else {
1000 tlbi->single.num = num;
1001 tlbi->single.use_global = false;
1002 }
1003 }
1004
riscv_iommu_iotlb_inval_iommu(struct riscv_iommu_device * iommu,int pscid,struct riscv_iommu_tlbi * tlbi)1005 static void riscv_iommu_iotlb_inval_iommu(struct riscv_iommu_device *iommu,
1006 int pscid,
1007 struct riscv_iommu_tlbi *tlbi)
1008 {
1009 bool use_nl = tlbi->non_leaf &&
1010 (iommu->caps & RISCV_IOMMU_CAPABILITIES_NL);
1011 struct riscv_iommu_command cmd;
1012 unsigned int i;
1013
1014 riscv_iommu_cmd_inval_vma(&cmd);
1015 riscv_iommu_cmd_inval_set_pscid(&cmd, pscid);
1016
1017 /*
1018 * If non-leaf entries were changed and the IOMMU doesn't
1019 * support NL, we must fall back to global invalidation (AV=0).
1020 */
1021 if (tlbi->non_leaf && !use_nl)
1022 goto global;
1023
1024 if (iommu->caps & RISCV_IOMMU_CAPABILITIES_S &&
1025 tlbi->range.sz_lg2 >= 13) {
1026 riscv_iommu_cmd_inval_set_napot(&cmd, tlbi->range.addr,
1027 tlbi->range.sz_lg2);
1028 if (use_nl)
1029 riscv_iommu_cmd_inval_set_nl(&cmd);
1030 riscv_iommu_cmd_send(iommu, &cmd);
1031 } else {
1032 unsigned long iova;
1033
1034 if (tlbi->single.use_global)
1035 goto global;
1036
1037 iova = tlbi->start;
1038 for (i = 0; i < tlbi->single.num; i++) {
1039 riscv_iommu_cmd_inval_set_addr(&cmd, iova);
1040 if (use_nl)
1041 riscv_iommu_cmd_inval_set_nl(&cmd);
1042 riscv_iommu_cmd_send(iommu, &cmd);
1043 iova += 1ULL << tlbi->single.stride_lg2;
1044 }
1045 }
1046 return;
1047 global:
1048 riscv_iommu_cmd_send(iommu, &cmd);
1049 }
1050
riscv_iommu_iotlb_inval(struct riscv_iommu_domain * domain,struct iommu_iotlb_gather * gather)1051 static void riscv_iommu_iotlb_inval(struct riscv_iommu_domain *domain,
1052 struct iommu_iotlb_gather *gather)
1053 {
1054 struct riscv_iommu_device *iommu, *prev;
1055 struct riscv_iommu_bond *bond;
1056 struct riscv_iommu_tlbi tlbi;
1057
1058 riscv_iommu_tlbi_calc(&tlbi, gather);
1059
1060 /*
1061 * For each IOMMU linked with this protection domain (via bonds->dev),
1062 * an IOTLB invaliation command will be submitted and executed.
1063 *
1064 * Possbile race with domain attach flow is handled by sequencing
1065 * bond creation - riscv_iommu_bond_link(), and device directory
1066 * update - riscv_iommu_iodir_update().
1067 *
1068 * PTE Update / IOTLB Inval Device attach & directory update
1069 * -------------------------- --------------------------
1070 * update page table entries add dev to the bond list
1071 * FENCE RW,RW FENCE RW,RW
1072 * For all IOMMUs: (can be empty) Update FSC/PSCID
1073 * FENCE IOW,IOW FENCE IOW,IOW
1074 * IOTLB.INVAL IODIR.INVAL
1075 * IOFENCE.C
1076 *
1077 * If bond list is not updated with new device, directory context will
1078 * be configured with already valid page table content. If an IOMMU is
1079 * linked to the protection domain it will receive invalidation
1080 * requests for updated page table entries.
1081 */
1082 smp_mb();
1083
1084 rcu_read_lock();
1085
1086 prev = NULL;
1087 list_for_each_entry_rcu(bond, &domain->bonds, list) {
1088 iommu = dev_to_iommu(bond->dev);
1089
1090 /*
1091 * IOTLB invalidation request can be safely omitted if already sent
1092 * to the IOMMU for the same PSCID, and with domain->bonds list
1093 * arranged based on the device's IOMMU, it's sufficient to check
1094 * last device the invalidation was sent to.
1095 */
1096 if (iommu == prev)
1097 continue;
1098
1099 riscv_iommu_iotlb_inval_iommu(iommu, domain->pscid, &tlbi);
1100 prev = iommu;
1101 }
1102
1103 prev = NULL;
1104 list_for_each_entry_rcu(bond, &domain->bonds, list) {
1105 iommu = dev_to_iommu(bond->dev);
1106 if (iommu == prev)
1107 continue;
1108
1109 riscv_iommu_cmd_sync(iommu, RISCV_IOMMU_IOTINVAL_TIMEOUT);
1110 prev = iommu;
1111 }
1112 rcu_read_unlock();
1113 }
1114
1115 #define RISCV_IOMMU_FSC_BARE 0
1116 /*
1117 * This function sends IOTINVAL commands as required by the RISC-V
1118 * IOMMU specification (Section 6.3.1 and 6.3.2 in 1.0 spec version)
1119 * after modifying DDT or PDT entries
1120 */
riscv_iommu_iodir_iotinval(struct riscv_iommu_device * iommu,bool inval_pdt,unsigned long iohgatp,struct riscv_iommu_dc * dc,struct riscv_iommu_pc * pc)1121 static void riscv_iommu_iodir_iotinval(struct riscv_iommu_device *iommu,
1122 bool inval_pdt, unsigned long iohgatp,
1123 struct riscv_iommu_dc *dc,
1124 struct riscv_iommu_pc *pc)
1125 {
1126 struct riscv_iommu_command cmd;
1127
1128 riscv_iommu_cmd_inval_vma(&cmd);
1129
1130 if (FIELD_GET(RISCV_IOMMU_DC_IOHGATP_MODE, iohgatp) ==
1131 RISCV_IOMMU_DC_IOHGATP_MODE_BARE) {
1132 if (inval_pdt) {
1133 /*
1134 * IOTINVAL.VMA with GV=AV=0, and PSCV=1, and
1135 * PSCID=PC.PSCID
1136 */
1137 riscv_iommu_cmd_inval_set_pscid(&cmd,
1138 FIELD_GET(RISCV_IOMMU_PC_TA_PSCID, pc->ta));
1139 } else {
1140 if (!FIELD_GET(RISCV_IOMMU_DC_TC_PDTV, dc->tc) &&
1141 FIELD_GET(RISCV_IOMMU_DC_FSC_MODE, dc->fsc) !=
1142 RISCV_IOMMU_DC_FSC_MODE_BARE) {
1143 /*
1144 * DC.tc.PDTV == 0 && DC.fsc.MODE != Bare
1145 * IOTINVAL.VMA with GV=AV=0, and PSCV=1, and
1146 * PSCID=DC.ta.PSCID
1147 */
1148 riscv_iommu_cmd_inval_set_pscid(&cmd,
1149 FIELD_GET(RISCV_IOMMU_DC_TA_PSCID, dc->ta));
1150 }
1151 /* else: IOTINVAL.VMA with GV=AV=PSCV=0 */
1152 }
1153 } else {
1154 riscv_iommu_cmd_inval_set_gscid(&cmd,
1155 FIELD_GET(RISCV_IOMMU_DC_IOHGATP_GSCID, iohgatp));
1156
1157 if (inval_pdt) {
1158 /*
1159 * IOTINVAL.VMA with GV=1, AV=0, and PSCV=1, and
1160 * GSCID=DC.iohgatp.GSCID, PSCID=PC.PSCID
1161 */
1162 riscv_iommu_cmd_inval_set_pscid(&cmd,
1163 FIELD_GET(RISCV_IOMMU_PC_TA_PSCID, pc->ta));
1164 }
1165 /*
1166 * else: IOTINVAL.VMA with GV=1,AV=PSCV=0,and
1167 * GSCID=DC.iohgatp.GSCID
1168 *
1169 * IOTINVAL.GVMA with GV=1,AV=0,and
1170 * GSCID=DC.iohgatp.GSCID
1171 * TODO: For now, the Second-Stage feature have not yet been merged,
1172 * also issue IOTINVAL.GVMA once second-stage support is merged.
1173 */
1174 }
1175 riscv_iommu_cmd_send(iommu, &cmd);
1176 }
1177 /*
1178 * Update IODIR for the device.
1179 *
1180 * During the execution of riscv_iommu_probe_device(), IODIR entries are
1181 * allocated for the device's identifiers. Device context invalidation
1182 * becomes necessary only if one of the updated entries was previously
1183 * marked as valid, given that invalid device context entries are not
1184 * cached by the IOMMU hardware.
1185 * In this implementation, updating a valid device context while the
1186 * device is not quiesced might be disruptive, potentially causing
1187 * interim translation faults.
1188 */
riscv_iommu_iodir_update(struct riscv_iommu_device * iommu,struct device * dev,u64 fsc,u64 ta)1189 static void riscv_iommu_iodir_update(struct riscv_iommu_device *iommu,
1190 struct device *dev, u64 fsc, u64 ta)
1191 {
1192 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
1193 struct riscv_iommu_dc *dc;
1194 struct riscv_iommu_command cmd;
1195 bool sync_required = false;
1196 u64 tc;
1197 int i;
1198
1199 for (i = 0; i < fwspec->num_ids; i++) {
1200 dc = riscv_iommu_get_dc(iommu, fwspec->ids[i]);
1201 tc = READ_ONCE(dc->tc);
1202 if (!(tc & RISCV_IOMMU_DC_TC_V))
1203 continue;
1204
1205 WRITE_ONCE(dc->tc, tc & ~RISCV_IOMMU_DC_TC_V);
1206
1207 /* Invalidate device context cached values */
1208 riscv_iommu_cmd_iodir_inval_ddt(&cmd);
1209 riscv_iommu_cmd_iodir_set_did(&cmd, fwspec->ids[i]);
1210 riscv_iommu_cmd_send(iommu, &cmd);
1211 /*
1212 * For now, the SVA and PASID features have not yet been merged, the
1213 * default configuration is inval_pdt=false and pc=NULL.
1214 */
1215 riscv_iommu_iodir_iotinval(iommu, false, dc->iohgatp, dc, NULL);
1216 sync_required = true;
1217 }
1218
1219 if (sync_required)
1220 riscv_iommu_cmd_sync(iommu, RISCV_IOMMU_IOTINVAL_TIMEOUT);
1221
1222 /*
1223 * For device context with DC_TC_PDTV = 0, translation attributes valid bit
1224 * is stored as DC_TC_V bit (both sharing the same location at BIT(0)).
1225 */
1226 for (i = 0; i < fwspec->num_ids; i++) {
1227 dc = riscv_iommu_get_dc(iommu, fwspec->ids[i]);
1228 tc = READ_ONCE(dc->tc);
1229 tc |= ta & RISCV_IOMMU_DC_TC_V;
1230
1231 WRITE_ONCE(dc->fsc, fsc);
1232 WRITE_ONCE(dc->ta, ta & RISCV_IOMMU_PC_TA_PSCID);
1233 /* Update device context, write TC.V as the last step. */
1234 dma_wmb();
1235 WRITE_ONCE(dc->tc, tc);
1236
1237 /* Invalidate device context after update */
1238 riscv_iommu_cmd_iodir_inval_ddt(&cmd);
1239 riscv_iommu_cmd_iodir_set_did(&cmd, fwspec->ids[i]);
1240 riscv_iommu_cmd_send(iommu, &cmd);
1241 /*
1242 * For now, the SVA and PASID features have not yet been merged, the
1243 * default configuration is inval_pdt=false and pc=NULL.
1244 */
1245 riscv_iommu_iodir_iotinval(iommu, false, dc->iohgatp, dc, NULL);
1246 }
1247
1248 riscv_iommu_cmd_sync(iommu, RISCV_IOMMU_IOTINVAL_TIMEOUT);
1249 }
1250
1251 /*
1252 * IOVA page translation tree management.
1253 */
1254
riscv_iommu_iotlb_flush_all(struct iommu_domain * iommu_domain)1255 static void riscv_iommu_iotlb_flush_all(struct iommu_domain *iommu_domain)
1256 {
1257 struct riscv_iommu_domain *domain = iommu_domain_to_riscv(iommu_domain);
1258 struct iommu_iotlb_gather gather = {
1259 .start = 0,
1260 .end = ULONG_MAX,
1261 .pt.leaf_levels_bitmap = 0xFF,
1262 .pt.table_levels_bitmap = 0xFE,
1263 };
1264
1265 riscv_iommu_iotlb_inval(domain, &gather);
1266 }
1267
riscv_iommu_iotlb_sync(struct iommu_domain * iommu_domain,struct iommu_iotlb_gather * gather)1268 static void riscv_iommu_iotlb_sync(struct iommu_domain *iommu_domain,
1269 struct iommu_iotlb_gather *gather)
1270 {
1271 struct riscv_iommu_domain *domain = iommu_domain_to_riscv(iommu_domain);
1272
1273 riscv_iommu_iotlb_inval(domain, gather);
1274 iommu_put_pages_list(&gather->freelist);
1275 }
1276
riscv_iommu_free_paging_domain(struct iommu_domain * iommu_domain)1277 static void riscv_iommu_free_paging_domain(struct iommu_domain *iommu_domain)
1278 {
1279 struct riscv_iommu_domain *domain = iommu_domain_to_riscv(iommu_domain);
1280
1281 WARN_ON(!list_empty(&domain->bonds));
1282
1283 if ((int)domain->pscid > 0)
1284 ida_free(&riscv_iommu_pscids, domain->pscid);
1285
1286 pt_iommu_deinit(&domain->riscvpt.iommu);
1287 kfree(domain);
1288 }
1289
riscv_iommu_pt_supported(struct riscv_iommu_device * iommu,int pgd_mode)1290 static bool riscv_iommu_pt_supported(struct riscv_iommu_device *iommu, int pgd_mode)
1291 {
1292 switch (pgd_mode) {
1293 case RISCV_IOMMU_DC_FSC_IOSATP_MODE_SV39:
1294 return iommu->caps & RISCV_IOMMU_CAPABILITIES_SV39;
1295
1296 case RISCV_IOMMU_DC_FSC_IOSATP_MODE_SV48:
1297 return iommu->caps & RISCV_IOMMU_CAPABILITIES_SV48;
1298
1299 case RISCV_IOMMU_DC_FSC_IOSATP_MODE_SV57:
1300 return iommu->caps & RISCV_IOMMU_CAPABILITIES_SV57;
1301 }
1302 return false;
1303 }
1304
riscv_iommu_attach_paging_domain(struct iommu_domain * iommu_domain,struct device * dev,struct iommu_domain * old)1305 static int riscv_iommu_attach_paging_domain(struct iommu_domain *iommu_domain,
1306 struct device *dev,
1307 struct iommu_domain *old)
1308 {
1309 struct riscv_iommu_domain *domain = iommu_domain_to_riscv(iommu_domain);
1310 struct riscv_iommu_device *iommu = dev_to_iommu(dev);
1311 struct riscv_iommu_info *info = dev_iommu_priv_get(dev);
1312 struct pt_iommu_riscv_64_hw_info pt_info;
1313 u64 fsc, ta;
1314
1315 pt_iommu_riscv_64_hw_info(&domain->riscvpt, &pt_info);
1316
1317 if (!riscv_iommu_pt_supported(iommu, pt_info.fsc_iosatp_mode))
1318 return -ENODEV;
1319
1320 fsc = FIELD_PREP(RISCV_IOMMU_PC_FSC_MODE, pt_info.fsc_iosatp_mode) |
1321 FIELD_PREP(RISCV_IOMMU_PC_FSC_PPN, pt_info.ppn);
1322 ta = FIELD_PREP(RISCV_IOMMU_PC_TA_PSCID, domain->pscid) |
1323 RISCV_IOMMU_PC_TA_V;
1324
1325 if (riscv_iommu_bond_link(domain, dev))
1326 return -ENOMEM;
1327
1328 riscv_iommu_iodir_update(iommu, dev, fsc, ta);
1329 riscv_iommu_bond_unlink(info->domain, dev);
1330 info->domain = domain;
1331
1332 return 0;
1333 }
1334
1335 static const struct iommu_domain_ops riscv_iommu_paging_domain_ops = {
1336 IOMMU_PT_DOMAIN_OPS(riscv_64),
1337 .attach_dev = riscv_iommu_attach_paging_domain,
1338 .free = riscv_iommu_free_paging_domain,
1339 .iotlb_sync = riscv_iommu_iotlb_sync,
1340 .flush_iotlb_all = riscv_iommu_iotlb_flush_all,
1341 };
1342
riscv_iommu_alloc_paging_domain(struct device * dev)1343 static struct iommu_domain *riscv_iommu_alloc_paging_domain(struct device *dev)
1344 {
1345 struct pt_iommu_riscv_64_cfg cfg = {};
1346 struct riscv_iommu_domain *domain;
1347 struct riscv_iommu_device *iommu;
1348 int ret;
1349
1350 iommu = dev_to_iommu(dev);
1351 if (iommu->caps & RISCV_IOMMU_CAPABILITIES_SV57) {
1352 cfg.common.hw_max_vasz_lg2 = 57;
1353 } else if (iommu->caps & RISCV_IOMMU_CAPABILITIES_SV48) {
1354 cfg.common.hw_max_vasz_lg2 = 48;
1355 } else if (iommu->caps & RISCV_IOMMU_CAPABILITIES_SV39) {
1356 cfg.common.hw_max_vasz_lg2 = 39;
1357 } else {
1358 dev_err(dev, "cannot find supported page table mode\n");
1359 return ERR_PTR(-ENODEV);
1360 }
1361 cfg.common.hw_max_oasz_lg2 = 56;
1362
1363 domain = kzalloc_obj(*domain);
1364 if (!domain)
1365 return ERR_PTR(-ENOMEM);
1366
1367 INIT_LIST_HEAD_RCU(&domain->bonds);
1368 spin_lock_init(&domain->lock);
1369 /*
1370 * 6.4 IOMMU capabilities [..] IOMMU implementations must support the
1371 * Svnapot standard extension for NAPOT Translation Contiguity.
1372 */
1373 cfg.common.features = BIT(PT_FEAT_SIGN_EXTEND) |
1374 BIT(PT_FEAT_FLUSH_RANGE) |
1375 BIT(PT_FEAT_RISCV_SVNAPOT_64K) |
1376 BIT(PT_FEAT_DETAILED_GATHER);
1377 if (iommu->caps & RISCV_IOMMU_CAPABILITIES_SVPBMT)
1378 cfg.common.features |= BIT(PT_FEAT_RISCV_SVPBMT);
1379 domain->riscvpt.iommu.nid = dev_to_node(iommu->dev);
1380 domain->domain.ops = &riscv_iommu_paging_domain_ops;
1381
1382 domain->pscid = ida_alloc_range(&riscv_iommu_pscids, 1,
1383 RISCV_IOMMU_MAX_PSCID, GFP_KERNEL);
1384 if (domain->pscid < 0) {
1385 riscv_iommu_free_paging_domain(&domain->domain);
1386 return ERR_PTR(-ENOMEM);
1387 }
1388
1389 ret = pt_iommu_riscv_64_init(&domain->riscvpt, &cfg, GFP_KERNEL);
1390 if (ret) {
1391 riscv_iommu_free_paging_domain(&domain->domain);
1392 return ERR_PTR(ret);
1393 }
1394 return &domain->domain;
1395 }
1396
riscv_iommu_attach_blocking_domain(struct iommu_domain * iommu_domain,struct device * dev,struct iommu_domain * old)1397 static int riscv_iommu_attach_blocking_domain(struct iommu_domain *iommu_domain,
1398 struct device *dev,
1399 struct iommu_domain *old)
1400 {
1401 struct riscv_iommu_device *iommu = dev_to_iommu(dev);
1402 struct riscv_iommu_info *info = dev_iommu_priv_get(dev);
1403
1404 /* Make device context invalid, translation requests will fault w/ #258 */
1405 riscv_iommu_iodir_update(iommu, dev, RISCV_IOMMU_FSC_BARE, 0);
1406 riscv_iommu_bond_unlink(info->domain, dev);
1407 info->domain = NULL;
1408
1409 return 0;
1410 }
1411
1412 static struct iommu_domain riscv_iommu_blocking_domain = {
1413 .type = IOMMU_DOMAIN_BLOCKED,
1414 .ops = &(const struct iommu_domain_ops) {
1415 .attach_dev = riscv_iommu_attach_blocking_domain,
1416 }
1417 };
1418
riscv_iommu_attach_identity_domain(struct iommu_domain * iommu_domain,struct device * dev,struct iommu_domain * old)1419 static int riscv_iommu_attach_identity_domain(struct iommu_domain *iommu_domain,
1420 struct device *dev,
1421 struct iommu_domain *old)
1422 {
1423 struct riscv_iommu_device *iommu = dev_to_iommu(dev);
1424 struct riscv_iommu_info *info = dev_iommu_priv_get(dev);
1425
1426 riscv_iommu_iodir_update(iommu, dev, RISCV_IOMMU_FSC_BARE, RISCV_IOMMU_PC_TA_V);
1427 riscv_iommu_bond_unlink(info->domain, dev);
1428 info->domain = NULL;
1429
1430 return 0;
1431 }
1432
1433 static struct iommu_domain riscv_iommu_identity_domain = {
1434 .type = IOMMU_DOMAIN_IDENTITY,
1435 .ops = &(const struct iommu_domain_ops) {
1436 .attach_dev = riscv_iommu_attach_identity_domain,
1437 }
1438 };
1439
riscv_iommu_device_group(struct device * dev)1440 static struct iommu_group *riscv_iommu_device_group(struct device *dev)
1441 {
1442 if (dev_is_pci(dev))
1443 return pci_device_group(dev);
1444 return generic_device_group(dev);
1445 }
1446
riscv_iommu_of_xlate(struct device * dev,const struct of_phandle_args * args)1447 static int riscv_iommu_of_xlate(struct device *dev, const struct of_phandle_args *args)
1448 {
1449 return iommu_fwspec_add_ids(dev, args->args, 1);
1450 }
1451
riscv_iommu_probe_device(struct device * dev)1452 static struct iommu_device *riscv_iommu_probe_device(struct device *dev)
1453 {
1454 struct iommu_fwspec *fwspec = dev_iommu_fwspec_get(dev);
1455 struct riscv_iommu_device *iommu;
1456 struct riscv_iommu_info *info;
1457 struct riscv_iommu_dc *dc;
1458 u64 tc;
1459 int i;
1460
1461 if (!fwspec || !fwspec->iommu_fwnode->dev || !fwspec->num_ids)
1462 return ERR_PTR(-ENODEV);
1463
1464 iommu = dev_get_drvdata(fwspec->iommu_fwnode->dev);
1465 if (!iommu)
1466 return ERR_PTR(-ENODEV);
1467
1468 /*
1469 * IOMMU hardware operating in fail-over BARE mode will provide
1470 * identity translation for all connected devices anyway...
1471 */
1472 if (iommu->ddt_mode <= RISCV_IOMMU_DDTP_IOMMU_MODE_BARE)
1473 return ERR_PTR(-ENODEV);
1474
1475 info = kzalloc_obj(*info);
1476 if (!info)
1477 return ERR_PTR(-ENOMEM);
1478 /*
1479 * Allocate and pre-configure device context entries in
1480 * the device directory. Do not mark the context valid yet.
1481 */
1482 tc = 0;
1483 for (i = 0; i < fwspec->num_ids; i++) {
1484 dc = riscv_iommu_get_dc(iommu, fwspec->ids[i]);
1485 if (!dc) {
1486 kfree(info);
1487 return ERR_PTR(-ENODEV);
1488 }
1489 if (READ_ONCE(dc->tc) & RISCV_IOMMU_DC_TC_V)
1490 dev_warn(dev, "already attached to IOMMU device directory\n");
1491 WRITE_ONCE(dc->tc, tc);
1492 }
1493
1494 dev_iommu_priv_set(dev, info);
1495
1496 return &iommu->iommu;
1497 }
1498
riscv_iommu_release_device(struct device * dev)1499 static void riscv_iommu_release_device(struct device *dev)
1500 {
1501 struct riscv_iommu_info *info = dev_iommu_priv_get(dev);
1502
1503 kfree_rcu_mightsleep(info);
1504 }
1505
1506 static const struct iommu_ops riscv_iommu_ops = {
1507 .of_xlate = riscv_iommu_of_xlate,
1508 .identity_domain = &riscv_iommu_identity_domain,
1509 .blocked_domain = &riscv_iommu_blocking_domain,
1510 .release_domain = &riscv_iommu_blocking_domain,
1511 .domain_alloc_paging = riscv_iommu_alloc_paging_domain,
1512 .device_group = riscv_iommu_device_group,
1513 .probe_device = riscv_iommu_probe_device,
1514 .release_device = riscv_iommu_release_device,
1515 };
1516
riscv_iommu_init_check(struct riscv_iommu_device * iommu)1517 static int riscv_iommu_init_check(struct riscv_iommu_device *iommu)
1518 {
1519 u64 ddtp;
1520
1521 /*
1522 * Make sure the IOMMU is switched off or in pass-through mode during
1523 * regular boot flow and disable translation when we boot into a kexec
1524 * kernel and the previous kernel left them enabled.
1525 */
1526 ddtp = riscv_iommu_readq(iommu, RISCV_IOMMU_REG_DDTP);
1527 if (ddtp & RISCV_IOMMU_DDTP_BUSY)
1528 return -EBUSY;
1529
1530 if (FIELD_GET(RISCV_IOMMU_DDTP_IOMMU_MODE, ddtp) >
1531 RISCV_IOMMU_DDTP_IOMMU_MODE_BARE) {
1532 if (!is_kdump_kernel())
1533 return -EBUSY;
1534 riscv_iommu_disable(iommu);
1535 }
1536
1537 /* Configure accesses to in-memory data structures for CPU-native byte order. */
1538 if (IS_ENABLED(CONFIG_CPU_BIG_ENDIAN) !=
1539 !!(iommu->fctl & RISCV_IOMMU_FCTL_BE)) {
1540 if (!(iommu->caps & RISCV_IOMMU_CAPABILITIES_END))
1541 return -EINVAL;
1542 riscv_iommu_writel(iommu, RISCV_IOMMU_REG_FCTL,
1543 iommu->fctl ^ RISCV_IOMMU_FCTL_BE);
1544 iommu->fctl = riscv_iommu_readl(iommu, RISCV_IOMMU_REG_FCTL);
1545 if (IS_ENABLED(CONFIG_CPU_BIG_ENDIAN) !=
1546 !!(iommu->fctl & RISCV_IOMMU_FCTL_BE))
1547 return -EINVAL;
1548 }
1549
1550 /*
1551 * Distribute interrupt vectors, always use first vector for CIV.
1552 * At least one interrupt is required. Read back and verify.
1553 */
1554 if (!iommu->irqs_count)
1555 return -EINVAL;
1556
1557 iommu->icvec = FIELD_PREP(RISCV_IOMMU_ICVEC_FIV, 1 % iommu->irqs_count) |
1558 FIELD_PREP(RISCV_IOMMU_ICVEC_PIV, 2 % iommu->irqs_count) |
1559 FIELD_PREP(RISCV_IOMMU_ICVEC_PMIV, 3 % iommu->irqs_count);
1560 riscv_iommu_writeq(iommu, RISCV_IOMMU_REG_ICVEC, iommu->icvec);
1561 iommu->icvec = riscv_iommu_readq(iommu, RISCV_IOMMU_REG_ICVEC);
1562 if (max3(FIELD_GET(RISCV_IOMMU_ICVEC_CIV, iommu->icvec),
1563 FIELD_GET(RISCV_IOMMU_ICVEC_FIV, iommu->icvec),
1564 max(FIELD_GET(RISCV_IOMMU_ICVEC_PIV, iommu->icvec),
1565 FIELD_GET(RISCV_IOMMU_ICVEC_PMIV, iommu->icvec))) >= iommu->irqs_count)
1566 return -EINVAL;
1567
1568 return 0;
1569 }
1570
riscv_iommu_remove(struct riscv_iommu_device * iommu)1571 void riscv_iommu_remove(struct riscv_iommu_device *iommu)
1572 {
1573 iommu_device_unregister(&iommu->iommu);
1574 iommu_device_sysfs_remove(&iommu->iommu);
1575 riscv_iommu_iodir_set_mode(iommu, RISCV_IOMMU_DDTP_IOMMU_MODE_OFF);
1576 riscv_iommu_queue_disable(&iommu->cmdq);
1577 riscv_iommu_queue_disable(&iommu->fltq);
1578 }
1579
riscv_iommu_init(struct riscv_iommu_device * iommu)1580 int riscv_iommu_init(struct riscv_iommu_device *iommu)
1581 {
1582 int rc;
1583
1584 RISCV_IOMMU_QUEUE_INIT(&iommu->cmdq, CQ);
1585 raw_spin_lock_init(&iommu->cmdq.lock);
1586 RISCV_IOMMU_QUEUE_INIT(&iommu->fltq, FQ);
1587
1588 rc = riscv_iommu_init_check(iommu);
1589 if (rc)
1590 return dev_err_probe(iommu->dev, rc, "unexpected device state\n");
1591
1592 rc = riscv_iommu_iodir_alloc(iommu);
1593 if (rc)
1594 return rc;
1595
1596 rc = riscv_iommu_queue_alloc(iommu, &iommu->cmdq,
1597 sizeof(struct riscv_iommu_command));
1598 if (rc)
1599 return rc;
1600
1601 rc = riscv_iommu_queue_alloc(iommu, &iommu->fltq,
1602 sizeof(struct riscv_iommu_fq_record));
1603 if (rc)
1604 return rc;
1605
1606 rc = riscv_iommu_queue_enable(iommu, &iommu->cmdq, riscv_iommu_cmdq_process);
1607 if (rc)
1608 return rc;
1609
1610 rc = riscv_iommu_queue_enable(iommu, &iommu->fltq, riscv_iommu_fltq_process);
1611 if (rc)
1612 goto err_queue_disable;
1613
1614 rc = riscv_iommu_iodir_set_mode(iommu, RISCV_IOMMU_DDTP_IOMMU_MODE_MAX);
1615 if (rc)
1616 goto err_queue_disable;
1617
1618 rc = iommu_device_sysfs_add(&iommu->iommu, NULL, NULL, "riscv-iommu@%s",
1619 dev_name(iommu->dev));
1620 if (rc) {
1621 dev_err_probe(iommu->dev, rc, "cannot register sysfs interface\n");
1622 goto err_iodir_off;
1623 }
1624
1625 if (!acpi_disabled) {
1626 rc = rimt_iommu_register(iommu->dev);
1627 if (rc) {
1628 dev_err_probe(iommu->dev, rc, "cannot register iommu with RIMT\n");
1629 goto err_remove_sysfs;
1630 }
1631 }
1632
1633 rc = iommu_device_register(&iommu->iommu, &riscv_iommu_ops, iommu->dev);
1634 if (rc) {
1635 dev_err_probe(iommu->dev, rc, "cannot register iommu interface\n");
1636 goto err_remove_sysfs;
1637 }
1638
1639 return 0;
1640
1641 err_remove_sysfs:
1642 iommu_device_sysfs_remove(&iommu->iommu);
1643 err_iodir_off:
1644 riscv_iommu_iodir_set_mode(iommu, RISCV_IOMMU_DDTP_IOMMU_MODE_OFF);
1645 err_queue_disable:
1646 riscv_iommu_queue_disable(&iommu->fltq);
1647 riscv_iommu_queue_disable(&iommu->cmdq);
1648 return rc;
1649 }
1650
1651 MODULE_IMPORT_NS("GENERIC_PT_IOMMU");
1652