1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause
3 *
4 * Copyright (c) 2026 Ruslan Bukin <br@bsdpad.com>
5 *
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions
8 * are met:
9 * 1. Redistributions of source code must retain the above copyright
10 * notice, this list of conditions and the following disclaimer.
11 * 2. Redistributions in binary form must reproduce the above copyright
12 * notice, this list of conditions and the following disclaimer in the
13 * documentation and/or other materials provided with the distribution.
14 *
15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
16 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
18 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
25 * SUCH DAMAGE.
26 */
27
28 #include <sys/param.h>
29 #include <sys/bitstring.h>
30 #include <sys/bus.h>
31 #include <sys/kernel.h>
32 #include <sys/malloc.h>
33 #include <sys/mutex.h>
34 #include <sys/rman.h>
35 #include <sys/lock.h>
36 #include <sys/sysctl.h>
37 #include <sys/tree.h>
38 #include <sys/taskqueue.h>
39 #include <sys/refcount.h>
40 #include <vm/vm.h>
41 #include <vm/vm_page.h>
42
43 #include <sys/bus.h>
44 #include <sys/conf.h>
45 #include <sys/rman.h>
46
47 #include <dev/pci/pcireg.h>
48 #include <dev/pci/pcivar.h>
49
50 #include <machine/bus.h>
51 #include <machine/resource.h>
52
53 #include <dev/iommu/iommu.h>
54 #include <riscv/iommu/iommu_pmap.h>
55 #include <riscv/iommu/iommu.h>
56
57 #include "iommu_if.h"
58
59 #define dprintf(fmt, ...)
60
61 MALLOC_DEFINE(M_IOMMU, "RISCV_IOMMU", "RISC-V IOMMU");
62
63 #define RD4(sc, reg) bus_read_4(sc->res[0], (reg))
64 #define WR4(sc, reg, val) bus_write_4(sc->res[0], (reg), (val))
65 #define RD8(sc, reg) bus_read_8(sc->res[0], (reg))
66 #define WR8(sc, reg, val) bus_write_8(sc->res[0], (reg), (val))
67
68 #define CQ_ENTRY_DWORDS 2 /* 16-byte */
69 #define CQ_ENTRY_COUNT 8192 /* Amount of 16-byte entries. */
70 #define FQ_ENTRY_DWORDS 4 /* 32-byte */
71 #define FQ_ENTRY_COUNT 8192 /* Amount of 32-byte entries. */
72 #define PQ_ENTRY_DWORDS 2 /* 16-byte */
73 #define PQ_ENTRY_COUNT 8192 /* Amount of 16-byte entries. */
74
75 #define DDT_NON_LEAF_DWORDS 1
76 #define DDT_DC_STD_DWORDS 4 /* Standard-format DC. */
77 #define DDT_DC_EXT_DWORDS 8 /* Extended-format DC. */
78 #define DDT_L1_DID_BITS 9 /* All formats. */
79
80 #define QUEUE_ALIGN (1024 * 1024) /* TODO */
81 #define QUEUE_HEAD(q) ((q)->csr + RISCV_IOMMU_CQH - RISCV_IOMMU_CQB)
82 #define QUEUE_TAIL(q) ((q)->csr + RISCV_IOMMU_CQT - RISCV_IOMMU_CQB)
83 #define QUEUE_IPSR(q) (1 << (q)->idx)
84
85 #define PHYS_TO_PPN(p) ((p) >> 12)
86
87 struct riscv_iommu_fq_event {
88 uint16_t cause_id;
89 char *descr;
90 };
91
92 static struct riscv_iommu_fq_event fq_events[] = {
93 { FQ_CAUSE_INST_FAULT, "Instruction access fault" },
94 { FQ_CAUSE_RD_ADDR_MISALIGNED, "Read address misaligned" },
95 { FQ_CAUSE_RD_FAULT, "Read access fault" },
96 { FQ_CAUSE_WR_ADDR_MISALIGNED, "Write/AMO address misaligned" },
97 { FQ_CAUSE_WR_FAULT, "Write/AMO access fault" },
98 { FQ_CAUSE_INST_FAULT_S, "Instruction page fault" },
99 { FQ_CAUSE_RD_FAULT_S, "Read page fault" },
100 { FQ_CAUSE_WR_FAULT_S, "Write/AMO page fault" },
101 { FQ_CAUSE_INST_FAULT_VS, "Instruction guest page fault" },
102 { FQ_CAUSE_RD_FAULT_VS, "Read guest-page fault" },
103 { FQ_CAUSE_WR_FAULT_VS, "Write/AMO guest-page fault" },
104 { FQ_CAUSE_DMA_DISABLED, "All inbound transactions disallowed" },
105 { FQ_CAUSE_DDT_LOAD_FAULT, "DDT entry load access fault" },
106 { FQ_CAUSE_DDT_INVALID, "DDT entry not valid" },
107 { FQ_CAUSE_DDT_MISCONFIGURED, "DDT entry misconfigured" },
108 { FQ_CAUSE_TR_TYPE_DISALLOWED, "Transaction type disallowed" },
109 { FQ_CAUSE_MSI_LOAD_FAULT, "MSI PTE load access fault" },
110 { FQ_CAUSE_MSI_INVALID, "MSI PTE not valid" },
111 { FQ_CAUSE_MSI_MISCONFIGURED, "MSI PTE misconfigured" },
112 { FQ_CAUSE_MRIF_FAULT, "MRIF access fault" },
113 { FQ_CAUSE_PDT_LOAD_FAULT, "PDT entry load access fault" },
114 { FQ_CAUSE_PDT_INVALID, "PDT entry not valid" },
115 { FQ_CAUSE_PDT_MISCONFIGURED, "PDT entry misconfigured" },
116 { FQ_CAUSE_DDT_CORRUPTED, "DDT data corruption" },
117 { FQ_CAUSE_PDT_CORRUPTED, "PDT data corruption" },
118 { FQ_CAUSE_MSI_PT_CORRUPTED, "MSI PT data corruption" },
119 { FQ_CAUSE_MRIF_CORRUPTED, "MSI MRIF data corruption" },
120 { FQ_CAUSE_INTERNAL_DP_ERROR, "Internal data path error" },
121 { FQ_CAUSE_MSI_WR_FAULT, "IOMMU MSI write access fault" },
122 { FQ_CAUSE_PT_CORRUPTED, "1st/2nd-stage PT data corruption" },
123 { 0, NULL },
124 };
125
126 static void
riscv_iommu_init_pscids(struct riscv_iommu_softc * sc)127 riscv_iommu_init_pscids(struct riscv_iommu_softc *sc)
128 {
129
130 sc->pscid_set_size = (1 << sc->pscid_bits);
131 sc->pscid_set = bit_alloc(sc->pscid_set_size, M_IOMMU, M_WAITOK);
132 mtx_init(&sc->pscid_set_mutex, "pscid set", NULL, MTX_SPIN);
133 }
134
135 static int
riscv_iommu_pscid_alloc(struct riscv_iommu_softc * sc,int * new_pscid)136 riscv_iommu_pscid_alloc(struct riscv_iommu_softc *sc, int *new_pscid)
137 {
138
139 mtx_lock_spin(&sc->pscid_set_mutex);
140 bit_ffc(sc->pscid_set, sc->pscid_set_size, new_pscid);
141 if (*new_pscid == -1) {
142 mtx_unlock_spin(&sc->pscid_set_mutex);
143 return (ENOMEM);
144 }
145 bit_set(sc->pscid_set, *new_pscid);
146 mtx_unlock_spin(&sc->pscid_set_mutex);
147
148 return (0);
149 }
150
151 static void
riscv_iommu_pscid_free(struct riscv_iommu_softc * sc,int pscid)152 riscv_iommu_pscid_free(struct riscv_iommu_softc *sc, int pscid)
153 {
154
155 mtx_lock_spin(&sc->pscid_set_mutex);
156 bit_clear(sc->pscid_set, pscid);
157 mtx_unlock_spin(&sc->pscid_set_mutex);
158 }
159
160 static uint32_t
riscv_iommu_q_inc_tail(struct riscv_iommu_queue * q)161 riscv_iommu_q_inc_tail(struct riscv_iommu_queue *q)
162 {
163
164 return ((q->lc.tail + 1) & q->mask);
165 }
166
167 static uint32_t
riscv_iommu_q_inc_head(struct riscv_iommu_queue * q)168 riscv_iommu_q_inc_head(struct riscv_iommu_queue *q)
169 {
170
171 return ((q->lc.head + 1) & q->mask);
172 }
173
174 static int
riscv_iommu_q_has_space(struct riscv_iommu_queue * q)175 riscv_iommu_q_has_space(struct riscv_iommu_queue *q)
176 {
177
178 if (riscv_iommu_q_inc_tail(q) != q->lc.head)
179 return (1);
180
181 return (0);
182 }
183
184 static int
riscv_iommu_q_empty(struct riscv_iommu_queue * q)185 riscv_iommu_q_empty(struct riscv_iommu_queue *q)
186 {
187
188 if (q->lc.tail == q->lc.head)
189 return (1);
190
191 return (0);
192 }
193
194 static int
riscv_iommu_dequeue(struct riscv_iommu_softc * sc,struct riscv_iommu_queue * q,void * data)195 riscv_iommu_dequeue(struct riscv_iommu_softc *sc, struct riscv_iommu_queue *q,
196 void *data)
197 {
198 void *entry_addr;
199
200 q->lc.val = RD8(sc, q->head_off);
201 entry_addr = (void *)((uint64_t)q->vaddr + q->lc.head * q->entry_size);
202 memcpy(data, entry_addr, q->entry_size);
203 q->lc.head = riscv_iommu_q_inc_head(q);
204 WR4(sc, q->head_off, q->lc.head);
205
206 return (0);
207 }
208
209 static int
riscv_iommu_enqueue(struct riscv_iommu_softc * sc,struct riscv_iommu_queue * q,void * data)210 riscv_iommu_enqueue(struct riscv_iommu_softc *sc, struct riscv_iommu_queue *q,
211 void *data)
212 {
213 void *entry_addr;
214
215 RISCV_IOMMU_LOCK(sc);
216
217 /* Ensure that a space is available. */
218 do {
219 q->lc.head = RD4(sc, q->head_off);
220 } while (riscv_iommu_q_has_space(q) == 0);
221
222 /* Write the command to the current tail entry. */
223 entry_addr = (void *)((uint64_t)q->vaddr + q->lc.tail * q->entry_size);
224 memcpy(entry_addr, data, q->entry_size);
225
226 /* Increment tail index. */
227 q->lc.tail = riscv_iommu_q_inc_tail(q);
228 WR4(sc, q->tail_off, q->lc.tail);
229
230 RISCV_IOMMU_UNLOCK(sc);
231
232 return (0);
233 }
234
235 static void
riscv_iommu_sync(struct riscv_iommu_softc * sc,struct riscv_iommu_queue * q)236 riscv_iommu_sync(struct riscv_iommu_softc *sc, struct riscv_iommu_queue *q)
237 {
238 struct riscv_iommu_command cmd;
239 uint64_t reg;
240
241 bzero(&cmd, sizeof(struct riscv_iommu_command));
242 reg = COMMAND_OPCODE_IOFENCE;
243 reg |= FUNC_IOFENCE_FUNC_C | FUNC_IOFENCE_PR | FUNC_IOFENCE_PW;
244 cmd.dword0 = reg;
245
246 riscv_iommu_enqueue(sc, &sc->cq, (void *)&cmd);
247
248 /*
249 * FUNC_IOFENCE_WSI does not seem to be implemented in QEMU,
250 * so ensure all requests are processed in polling mode;
251 */
252 do {
253 q->lc.head = RD4(sc, q->head_off);
254 } while (riscv_iommu_q_empty(q) == 0);
255 }
256
257 static int
riscv_iommu_inval_ddt(struct riscv_iommu_softc * sc)258 riscv_iommu_inval_ddt(struct riscv_iommu_softc *sc)
259 {
260 struct riscv_iommu_command cmd;
261 uint64_t reg;
262
263 bzero(&cmd, sizeof(struct riscv_iommu_command));
264 reg = COMMAND_OPCODE_IODIR;
265 reg |= FUNC_IODIR_INVAL_DDT;
266 cmd.dword0 = reg;
267
268 riscv_iommu_enqueue(sc, &sc->cq, (void *)&cmd);
269
270 return (0);
271 }
272
273 static int
riscv_iommu_inval_ddt_did(struct riscv_iommu_softc * sc,int did)274 riscv_iommu_inval_ddt_did(struct riscv_iommu_softc *sc, int did)
275 {
276 struct riscv_iommu_command cmd;
277 uint64_t reg;
278
279 bzero(&cmd, sizeof(struct riscv_iommu_command));
280 reg = COMMAND_OPCODE_IODIR;
281 reg |= FUNC_IODIR_INVAL_DDT;
282 reg |= FUNC_IODIR_DV;
283 reg |= (uint64_t)did << FUNC_IODIR_DID_S;
284 cmd.dword0 = reg;
285
286 riscv_iommu_enqueue(sc, &sc->cq, (void *)&cmd);
287
288 return (0);
289 }
290
291 /* Invalidate entire address space. */
292 static int
riscv_iommu_inval_vma(struct riscv_iommu_softc * sc)293 riscv_iommu_inval_vma(struct riscv_iommu_softc *sc)
294 {
295 struct riscv_iommu_command cmd;
296 uint64_t reg;
297
298 bzero(&cmd, sizeof(struct riscv_iommu_command));
299 reg = COMMAND_OPCODE_IOTINVAL;
300 reg |= FUNC_IOTINVAL_VMA;
301 cmd.dword0 = reg;
302
303 riscv_iommu_enqueue(sc, &sc->cq, (void *)&cmd);
304
305 return (0);
306 }
307
308 static int
riscv_iommu_inval_vma_page(struct riscv_iommu_softc * sc,vm_offset_t addr,int pscid)309 riscv_iommu_inval_vma_page(struct riscv_iommu_softc *sc, vm_offset_t addr,
310 int pscid)
311 {
312 struct riscv_iommu_command cmd;
313 uint64_t reg;
314
315 bzero(&cmd, sizeof(struct riscv_iommu_command));
316 reg = COMMAND_OPCODE_IOTINVAL;
317 reg |= FUNC_IOTINVAL_VMA;
318 reg |= FUNC_IOTINVAL_AV;
319 reg |= FUNC_IOTINVAL_PSCV;
320 reg |= pscid << FUNC_IOTINVAL_PSCID_S;
321 cmd.dword0 = reg;
322 cmd.dword1 = PHYS_TO_PPN(addr) << FUNC_IOTINVAL_ADDR_S;
323
324 riscv_iommu_enqueue(sc, &sc->cq, (void *)&cmd);
325
326 return (0);
327 }
328
329 static int
riscv_iommu_inval_vma_pscid(struct riscv_iommu_softc * sc,int pscid)330 riscv_iommu_inval_vma_pscid(struct riscv_iommu_softc *sc, int pscid)
331 {
332 struct riscv_iommu_command cmd;
333 uint64_t reg;
334
335 bzero(&cmd, sizeof(struct riscv_iommu_command));
336 reg = COMMAND_OPCODE_IOTINVAL;
337 reg |= FUNC_IOTINVAL_VMA;
338 reg |= FUNC_IOTINVAL_PSCV;
339 reg |= pscid << FUNC_IOTINVAL_PSCID_S;
340 cmd.dword0 = reg;
341
342 riscv_iommu_enqueue(sc, &sc->cq, (void *)&cmd);
343
344 return (0);
345 }
346
347 static int
riscv_iommu_set_mode(struct riscv_iommu_softc * sc)348 riscv_iommu_set_mode(struct riscv_iommu_softc *sc)
349 {
350 struct riscv_iommu_ddt *ddt;
351 uint64_t reg;
352 uint64_t base;
353
354 reg = RD8(sc, RISCV_IOMMU_DDTP);
355 if (reg & DDTP_BUSY)
356 return (ENXIO);
357
358 ddt = &sc->ddt;
359 base = ddt->base | (sc->iommu_mode << DDTP_IOMMU_MODE_S);
360 WR8(sc, RISCV_IOMMU_DDTP, base);
361
362 reg = RD8(sc, RISCV_IOMMU_DDTP);
363 if (reg != base) {
364 device_printf(sc->dev, "could not set mode\n");
365 return (ENXIO);
366 }
367
368 riscv_iommu_inval_ddt(sc);
369 riscv_iommu_inval_vma(sc);
370
371 return (0);
372 }
373
374 static int
riscv_iommu_enable_queue(struct riscv_iommu_softc * sc,struct riscv_iommu_queue * q)375 riscv_iommu_enable_queue(struct riscv_iommu_softc *sc,
376 struct riscv_iommu_queue *q)
377 {
378 uint32_t reg;
379 int timeout;
380
381 if (q == &sc->cq)
382 WR4(sc, QUEUE_TAIL(q), 0);
383 else
384 WR4(sc, QUEUE_HEAD(q), 0);
385
386 reg = CQCSR_CQEN | CQCSR_CIE | CQCSR_CQMF;
387 WR4(sc, q->csr, reg);
388
389 timeout = 1000;
390 do {
391 reg = RD4(sc, RISCV_IOMMU_CQCSR);
392 if ((reg & CQCSR_BUSY) == 0)
393 break;
394 DELAY(10);
395 } while (timeout--);
396
397 if (timeout <= 0) {
398 device_printf(sc->dev, "could not enable command queue\n");
399 return (-1);
400 }
401
402 if ((reg & CQCSR_CQON) == 0) {
403 device_printf(sc->dev, "could not activate command queue\n");
404 return (-1);
405 }
406
407 /* RW1C interrupt pending bit. */
408 WR4(sc, RISCV_IOMMU_IPSR, QUEUE_IPSR(q));
409
410 return (0);
411 }
412
413 static int
riscv_iommu_init_queue(struct riscv_iommu_softc * sc,struct riscv_iommu_queue * q,uint64_t base,uint32_t dwords)414 riscv_iommu_init_queue(struct riscv_iommu_softc *sc,
415 struct riscv_iommu_queue *q, uint64_t base, uint32_t dwords)
416 {
417 uint64_t reg;
418 int sz;
419
420 q->entry_size = dwords * 8;
421 sz = (1 << q->size_log2) * q->entry_size;
422
423 /* Set up the command circular buffer */
424 q->vaddr = contigmalloc(sz, M_IOMMU, M_WAITOK | M_ZERO, 0,
425 (1ul << 48) - 1, QUEUE_ALIGN, 0);
426 if (q->vaddr == NULL) {
427 device_printf(sc->dev, "failed to allocate %d bytes\n", sz);
428 return (-1);
429 }
430
431 q->mask = (1 << q->size_log2) - 1;
432 q->head_off = (uint32_t)base - RISCV_IOMMU_CQB + RISCV_IOMMU_CQH;
433 q->tail_off = (uint32_t)base - RISCV_IOMMU_CQB + RISCV_IOMMU_CQT;
434 q->paddr = vtophys(q->vaddr);
435 q->base = (sc->cq.size_log2 - 1) << CQB_LOG2SZ_1_S;
436 q->base |= PHYS_TO_PPN(q->paddr) << CQB_PPN_S;
437 WR8(sc, base, q->base);
438
439 /* Verify it sticks. */
440 reg = RD8(sc, base);
441 if (reg != q->base) {
442 device_printf(sc->dev, "could not init queue\n");
443 return (ENXIO);
444 }
445
446 return (0);
447 }
448
449 static int
riscv_iommu_init_queues(struct riscv_iommu_softc * sc)450 riscv_iommu_init_queues(struct riscv_iommu_softc *sc)
451 {
452 int error;
453
454 sc->cq.size_log2 = ilog2(CQ_ENTRY_COUNT);
455 sc->fq.size_log2 = ilog2(FQ_ENTRY_COUNT);
456 sc->pq.size_log2 = ilog2(PQ_ENTRY_COUNT);
457
458 sc->cq.csr = RISCV_IOMMU_CQCSR;
459 sc->fq.csr = RISCV_IOMMU_FQCSR;
460 sc->pq.csr = RISCV_IOMMU_PQCSR;
461
462 sc->cq.idx = 0;
463 sc->fq.idx = 1;
464 sc->pq.idx = 3;
465
466 /* Command queue (CQ). */
467 error = riscv_iommu_init_queue(sc, &sc->cq, RISCV_IOMMU_CQB,
468 CQ_ENTRY_DWORDS);
469 if (error)
470 return (error);
471
472 /* Fault queue (FQ). */
473 error = riscv_iommu_init_queue(sc, &sc->fq, RISCV_IOMMU_FQB,
474 FQ_ENTRY_DWORDS);
475 if (error)
476 return (error);
477
478 /* Page request queue (PQ). */
479 error = riscv_iommu_init_queue(sc, &sc->pq, RISCV_IOMMU_PQB,
480 PQ_ENTRY_DWORDS);
481 if (error)
482 return (error);
483
484 error = riscv_iommu_enable_queue(sc, &sc->cq);
485 if (error)
486 return (error);
487
488 error = riscv_iommu_enable_queue(sc, &sc->fq);
489 if (error)
490 return (error);
491
492 error = riscv_iommu_enable_queue(sc, &sc->pq);
493 if (error)
494 return (error);
495
496 return (0);
497 }
498
499 static int
riscv_iommu_init_pagedir(struct riscv_iommu_softc * sc)500 riscv_iommu_init_pagedir(struct riscv_iommu_softc *sc)
501 {
502
503 return (0);
504 }
505
506 static void
riscv_iommu_print_fault(struct riscv_iommu_softc * sc,struct riscv_iommu_fq_record * rec)507 riscv_iommu_print_fault(struct riscv_iommu_softc *sc,
508 struct riscv_iommu_fq_record *rec)
509 {
510 struct riscv_iommu_fq_event *ev;
511 uint16_t cause_id;
512 uint16_t ttyp;
513 uint32_t did;
514 uint32_t pid;
515 bool pv, priv;
516 int i;
517
518 cause_id = (rec->hdr & FQR_HDR_CAUSE_M) >> FQR_HDR_CAUSE_S;
519 ttyp = (rec->hdr & FQR_HDR_TTYP_M) >> FQR_HDR_TTYP_S;
520 did = (rec->hdr & FQR_HDR_DID_M) >> FQR_HDR_DID_S;
521 pid = (rec->hdr & FQR_HDR_PID_M) >> FQR_HDR_PID_S;
522 pv = (rec->hdr & FQR_HDR_PV) ? 1 : 0;
523 priv = (rec->hdr & FQR_HDR_PRIV) ? 1 : 0;
524
525 ev = NULL;
526 for (i = 0; fq_events[i].cause_id != 0; i++) {
527 if (fq_events[i].cause_id == cause_id) {
528 ev = &fq_events[i];
529 break;
530 }
531 }
532
533 if (ev == NULL) {
534 device_printf(sc->dev, "Fault: unknown fault 0x%x received\n",
535 cause_id);
536 return;
537 }
538
539 device_printf(sc->dev, "Fault: event 0x%x received: %s\n",
540 ev->cause_id, ev->descr);
541 device_printf(sc->dev, " hdr 0x%lx\n", rec->hdr);
542 device_printf(sc->dev, " iotval 0x%lx\n", rec->iotval);
543 device_printf(sc->dev, " iotval2 0x%lx\n", rec->iotval2);
544 device_printf(sc->dev, " ttyp 0x%x did 0x%x pid 0x%x pv %d priv %d"
545 "\n", ttyp, did, pid, pv, priv);
546 }
547
548 static int
riscv_cq_intr(void * arg)549 riscv_cq_intr(void *arg)
550 {
551 struct riscv_iommu_softc *sc;
552 struct riscv_iommu_queue *q;
553 uint32_t reg;
554
555 sc = arg;
556 q = &sc->cq;
557
558 reg = RD4(sc, q->csr);
559 printf("%s: pending %x\n", __func__, reg);
560
561 /* Clear pending bit. */
562 WR4(sc, RISCV_IOMMU_IPSR, IPSR_CIP);
563
564 return (FILTER_HANDLED);
565 }
566
567 static int
riscv_fq_intr(void * arg)568 riscv_fq_intr(void *arg)
569 {
570 struct riscv_iommu_fq_record rec;
571 struct riscv_iommu_softc *sc;
572 struct riscv_iommu_queue *q;
573 uint32_t reg;
574
575 sc = arg;
576 q = &sc->fq;
577
578 reg = RD4(sc, q->csr);
579 printf("%s: pending %x\n", __func__, reg);
580
581 /* Clear pending bit. */
582 WR4(sc, RISCV_IOMMU_IPSR, IPSR_FIP);
583
584 do {
585 riscv_iommu_dequeue(sc, q, &rec);
586 riscv_iommu_print_fault(sc, &rec);
587 } while (!riscv_iommu_q_empty(q));
588
589 return (FILTER_HANDLED);
590 }
591
592 static int
riscv_pm_intr(void * arg)593 riscv_pm_intr(void *arg)
594 {
595 struct riscv_iommu_softc *sc;
596
597 sc = arg;
598
599 printf("%s\n", __func__);
600
601 /* Clear pending bit. */
602 WR4(sc, RISCV_IOMMU_IPSR, IPSR_PMIP);
603
604 return (FILTER_HANDLED);
605 }
606
607 static int
riscv_pq_intr(void * arg)608 riscv_pq_intr(void *arg)
609 {
610 struct riscv_iommu_softc *sc;
611 struct riscv_iommu_queue *q;
612 uint32_t reg;
613
614 sc = arg;
615 q = &sc->pq;
616
617 reg = RD4(sc, q->csr);
618 printf("%s: pending %x\n", __func__, reg);
619
620 /* Clear pending bit. */
621 WR4(sc, RISCV_IOMMU_IPSR, IPSR_PIP);
622
623 return (FILTER_HANDLED);
624 }
625
626 static int
riscv_iommu_init_ddt_linear(struct riscv_iommu_softc * sc)627 riscv_iommu_init_ddt_linear(struct riscv_iommu_softc *sc)
628 {
629 struct riscv_iommu_ddt *ddt;
630 uint64_t size;
631 uint64_t reg;
632
633 ddt = &sc->ddt;
634 ddt->num_top_entries = (1 << sc->l0_did_bits);
635
636 size = ddt->num_top_entries * (sc->dc_dwords << 3);
637
638 if (bootverbose)
639 device_printf(sc->dev, "linear ddt size %ld, num_top_entries "
640 "%d\n", size, ddt->num_top_entries);
641
642 ddt->vaddr = contigmalloc(size, M_IOMMU,
643 M_WAITOK | M_ZERO, /* flags */
644 0, /* low */
645 (1ul << 48) - 1, /* high */
646 size, /* alignment */
647 0); /* boundary */
648 if (ddt->vaddr == NULL) {
649 device_printf(sc->dev, "failed to allocate ddt\n");
650 return (ENXIO);
651 }
652
653 reg = vtophys(ddt->vaddr);
654 if (bootverbose)
655 device_printf(sc->dev, "ddt base %p size %lx\n", ddt->vaddr,
656 size);
657 ddt->base = PHYS_TO_PPN(reg) << DDTP_PPN_S;
658
659 return (0);
660 }
661
662 static int
riscv_iommu_init_ddt_2lvl(struct riscv_iommu_softc * sc)663 riscv_iommu_init_ddt_2lvl(struct riscv_iommu_softc *sc)
664 {
665 struct riscv_iommu_ddt *ddt;
666 uint64_t size;
667 uint64_t reg;
668 uint64_t sz;
669
670 ddt = &sc->ddt;
671 ddt->num_top_entries = (1 << DDT_L1_DID_BITS);
672
673 size = ddt->num_top_entries * (DDT_NON_LEAF_DWORDS << 3);
674
675 if (bootverbose)
676 device_printf(sc->dev, "%s: size %lu, l1 entries %d, size "
677 "%lu\n", __func__, size, ddt->num_top_entries, size);
678
679 ddt->vaddr = contigmalloc(size, M_IOMMU,
680 M_WAITOK | M_ZERO, /* flags */
681 0, /* low */
682 (1ul << 48) - 1, /* high */
683 size, /* alignment */
684 0); /* boundary */
685 if (ddt->vaddr == NULL) {
686 device_printf(sc->dev, "Failed to allocate 2lvl ddt.\n");
687 return (ENOMEM);
688 }
689
690 sz = ddt->num_top_entries * sizeof(struct l1_desc);
691 ddt->l1 = malloc(sz, M_IOMMU, M_WAITOK | M_ZERO);
692
693 reg = vtophys(ddt->vaddr);
694 if (bootverbose)
695 device_printf(sc->dev, "ddt base %p size %lx\n", ddt->vaddr,
696 size);
697 ddt->base = PHYS_TO_PPN(reg) << DDTP_PPN_S;
698
699 return (0);
700 }
701
702 static int
riscv_iommu_init_l0_directory(struct riscv_iommu_softc * sc,int sid)703 riscv_iommu_init_l0_directory(struct riscv_iommu_softc *sc, int sid)
704 {
705 struct riscv_iommu_ddt *ddt;
706 struct l1_desc *l1_desc;
707 uint64_t *l1e;
708 uint64_t val;
709 size_t size;
710 int i;
711
712 ddt = &sc->ddt;
713 l1_desc = &ddt->l1[sid >> sc->l0_did_bits];
714 if (l1_desc->va) {
715 /* Already allocated. */
716 return (0);
717 }
718
719 size = (1 << sc->l0_did_bits) * (sc->dc_dwords << 3);
720
721 l1_desc->va = contigmalloc(size, M_IOMMU,
722 M_WAITOK | M_ZERO, /* flags */
723 0, /* low */
724 (1ul << 48) - 1, /* high */
725 size, /* alignment */
726 0); /* boundary */
727 if (l1_desc->va == NULL) {
728 device_printf(sc->dev, "failed to allocate l0 directory\n");
729 return (ENXIO);
730 }
731
732 l1_desc->pa = vtophys(l1_desc->va);
733
734 i = sid >> sc->l0_did_bits;
735 l1e = (void *)((uint64_t)ddt->vaddr + DDT_NON_LEAF_DWORDS * 8 * i);
736
737 /* Install the L1 entry. */
738 val = PHYS_TO_PPN(l1_desc->pa) << DC_NON_LEAF_ENTRY_PPN_S;
739 val |= DC_NON_LEAF_ENTRY_VALID;
740 *l1e = val;
741
742 return (0);
743 }
744
745 static void *
riscv_iommu_get_dc_addr(struct riscv_iommu_softc * sc,int did)746 riscv_iommu_get_dc_addr(struct riscv_iommu_softc *sc, int did)
747 {
748 struct riscv_iommu_ddt *ddt;
749 struct l1_desc *l1_desc;
750 uintptr_t l0_base;
751 void *addr;
752 int l0_offs;
753 int l1_idx;
754
755 ddt = &sc->ddt;
756
757 l0_offs = sc->dc_dwords * 8 * (did & ((1 << sc->l0_did_bits) - 1));
758
759 if (sc->iommu_mode == DDTP_IOMMU_MODE_2LVL) {
760 l1_idx = (did >> sc->l0_did_bits) &
761 ((1 << DDT_L1_DID_BITS) - 1);
762 l1_desc = &ddt->l1[l1_idx];
763 l0_base = (uintptr_t)l1_desc->va;
764 } else
765 l0_base = (uintptr_t)ddt->vaddr;
766
767 addr = (void *)(l0_base + l0_offs);
768
769 dprintf("ddt vaddr %p addr %p\n", ddt->vaddr, addr);
770
771 return (addr);
772 }
773
774 static int
riscv_iommu_init_dc(struct riscv_iommu_softc * sc,struct riscv_iommu_domain * domain,int did,bool bypass)775 riscv_iommu_init_dc(struct riscv_iommu_softc *sc,
776 struct riscv_iommu_domain *domain, int did, bool bypass)
777 {
778 struct riscv_iommu_dc_base *dc_base;
779 struct riscv_iommu_dc *dc;
780 struct riscv_iommu_pmap *p;
781
782 dc = riscv_iommu_get_dc_addr(sc, did);
783 dc_base = &dc->base;
784
785 device_printf(sc->dev, "address translation for device id"
786 " 0x%x is %s.\n", did, bypass ? "bypassed" : "enabled");
787
788 p = &domain->p;
789
790 bzero(dc_base, sizeof(struct riscv_iommu_dc_base));
791 if (bypass == false)
792 dc_base->fsc = p->pm_satp;
793 dc_base->ta = (domain->pscid << DC_TA_PSCID_S) | DC_TA_V;
794
795 riscv_iommu_inval_ddt_did(sc, did);
796 riscv_iommu_sync(sc, &sc->cq);
797 dc_base->tc |= DC_TC_V;
798 riscv_iommu_inval_ddt_did(sc, did);
799 riscv_iommu_inval_vma(sc);
800 riscv_iommu_sync(sc, &sc->cq);
801
802 return (0);
803 }
804
805 static void
riscv_iommu_deinit_dc(struct riscv_iommu_softc * sc,int did)806 riscv_iommu_deinit_dc(struct riscv_iommu_softc *sc, int did)
807 {
808 struct riscv_iommu_dc_base *dc_base;
809 struct riscv_iommu_dc *dc;
810
811 dc = riscv_iommu_get_dc_addr(sc, did);
812 dc_base = &dc->base;
813 dc_base->tc &= ~DC_TC_V;
814
815 riscv_iommu_inval_ddt_did(sc, did);
816 riscv_iommu_sync(sc, &sc->cq);
817 }
818
819 static int
riscv_iommu_setup_interrupts(struct riscv_iommu_softc * sc)820 riscv_iommu_setup_interrupts(struct riscv_iommu_softc *sc)
821 {
822 device_t dev;
823 int error;
824
825 dev = sc->dev;
826
827 if (sc->res[1] == NULL || sc->res[2] == NULL ||
828 sc->res[3] == NULL || sc->res[4] == NULL) {
829 device_printf(dev, "Warning: no interrupt resources "
830 "provided.\n");
831 return (ENXIO);
832 }
833
834 error = bus_setup_intr(dev, sc->res[1], INTR_TYPE_MISC,
835 riscv_cq_intr, NULL, sc, &sc->intr_cookie[0]);
836 if (error) {
837 device_printf(dev, "Couldn't setup cq interrupt handler\n");
838 return (ENXIO);
839 }
840
841 error = bus_setup_intr(dev, sc->res[2], INTR_TYPE_MISC,
842 riscv_fq_intr, NULL, sc, &sc->intr_cookie[1]);
843 if (error) {
844 device_printf(dev, "Couldn't setup fq interrupt handler\n");
845 return (ENXIO);
846 }
847
848 error = bus_setup_intr(dev, sc->res[3], INTR_TYPE_MISC,
849 riscv_pm_intr, NULL, sc, &sc->intr_cookie[2]);
850 if (error) {
851 device_printf(dev, "Couldn't setup pm interrupt handler\n");
852 return (ENXIO);
853 }
854
855 error = bus_setup_intr(dev, sc->res[4], INTR_TYPE_MISC,
856 riscv_pq_intr, NULL, sc, &sc->intr_cookie[3]);
857 if (error) {
858 device_printf(dev, "Couldn't setup pq interrupt handler\n");
859 return (ENXIO);
860 }
861
862 WR8(sc, RISCV_IOMMU_ICVEC, 0 << 0 | 1 << 4 | 2 << 8 | 3 << 12);
863
864 return (0);
865 }
866
867 int
riscv_iommu_attach(device_t dev)868 riscv_iommu_attach(device_t dev)
869 {
870 struct riscv_iommu_softc *sc;
871 uint64_t caps;
872 int error;
873
874 sc = device_get_softc(dev);
875
876 caps = bus_read_8(sc->res[0], RISCV_IOMMU_CAPABILITIES);
877 if (bootverbose)
878 device_printf(sc->dev, "IOMMU Capabilities: %lx\n", caps);
879
880 device_printf(sc->dev, "Device-Context structure is %s.\n",
881 caps & CAPABILITIES_MSI_FLAT ?
882 "64-bytes (ext format)" : "32-bytes (std format)");
883
884 if (caps & CAPABILITIES_MSI_FLAT) {
885 sc->dc_dwords = DDT_DC_EXT_DWORDS;
886 sc->l0_did_bits = 6;
887 } else {
888 sc->dc_dwords = DDT_DC_STD_DWORDS;
889 sc->l0_did_bits = 7;
890 }
891
892 if (caps & CAPABILITIES_SV48)
893 sc->pm_mode = PMAP_MODE_SV48;
894 else if (caps & CAPABILITIES_SV39)
895 sc->pm_mode = PMAP_MODE_SV39;
896 else {
897 device_printf(sc->dev, "Unsupported virtual memory system\n");
898 return (ENXIO);
899 }
900
901 mtx_init(&sc->mtx, device_get_nameunit(sc->dev), "riscv_iommu",
902 MTX_DEF);
903
904 WR4(sc, RISCV_IOMMU_FCTL, FCTL_WSI);
905
906 error = riscv_iommu_setup_interrupts(sc);
907 if (error) {
908 device_printf(sc->dev, "Could not setup interrupts. "
909 "Continuing with no interrupts support.");
910 }
911
912 error = riscv_iommu_init_pagedir(sc);
913 if (error)
914 return (error);
915
916 error = riscv_iommu_init_queues(sc);
917 if (error)
918 return (error);
919
920 sc->iommu_mode = DDTP_IOMMU_MODE_2LVL;
921
922 switch (sc->iommu_mode) {
923 case DDTP_IOMMU_MODE_1LVL:
924 error = riscv_iommu_init_ddt_linear(sc);
925 break;
926 case DDTP_IOMMU_MODE_2LVL:
927 error = riscv_iommu_init_ddt_2lvl(sc);
928 break;
929 default:
930 error = ENXIO;
931 }
932 if (error)
933 return (error);
934
935 sc->pscid_bits = 8;
936
937 riscv_iommu_init_pscids(sc);
938 if (error)
939 return (error);
940
941 error = riscv_iommu_set_mode(sc);
942 if (error)
943 return (error);
944
945 return (0);
946 }
947
948 static int
riscv_iommu_set_buswide(device_t dev,struct riscv_iommu_domain * domain,struct riscv_iommu_ctx * ctx)949 riscv_iommu_set_buswide(device_t dev, struct riscv_iommu_domain *domain,
950 struct riscv_iommu_ctx *ctx)
951 {
952 struct riscv_iommu_softc *sc;
953 int i;
954
955 sc = device_get_softc(dev);
956
957 printf("%s\n", __func__);
958
959 for (i = 0; i < PCI_SLOTMAX; i++)
960 riscv_iommu_init_dc(sc, domain, (ctx->did | i),
961 ctx->bypass);
962
963 return (0);
964 }
965
966 static int
riscv_iommu_pci_get_did(device_t child,uintptr_t * xref0,u_int * did0)967 riscv_iommu_pci_get_did(device_t child, uintptr_t *xref0, u_int *did0)
968 {
969 struct pci_id_ofw_iommu pi;
970 int err;
971
972 dprintf("%s\n", __func__);
973
974 err = pci_get_id(child, PCI_ID_OFW_IOMMU, (uintptr_t *)&pi);
975 if (err == 0) {
976 if (did0)
977 *did0 = pi.id;
978 if (xref0)
979 *xref0 = pi.xref;
980 }
981
982 return (err);
983 }
984
985 static int
riscv_iommu_find(device_t dev,device_t child)986 riscv_iommu_find(device_t dev, device_t child)
987 {
988 struct riscv_iommu_softc *sc;
989 uintptr_t xref;
990 int err;
991
992 dprintf("%s\n", __func__);
993
994 sc = device_get_softc(dev);
995
996 err = riscv_iommu_pci_get_did(child, &xref, NULL);
997 if (err)
998 return (ENOENT);
999
1000 /* Check if xref is ours. */
1001 dprintf("xref %lx sc->xref %lx\n", xref, sc->xref);
1002 if (xref != sc->xref)
1003 return (EFAULT);
1004
1005 return (0);
1006 }
1007
1008 struct riscv_iommu_ctx *
riscv_iommu_ctx_lookup_by_did(device_t dev,u_int did)1009 riscv_iommu_ctx_lookup_by_did(device_t dev, u_int did)
1010 {
1011 struct riscv_iommu_softc *sc;
1012 struct riscv_iommu_domain *domain;
1013 struct riscv_iommu_unit *unit;
1014 struct riscv_iommu_ctx *ctx;
1015
1016 dprintf("%s\n", __func__);
1017 sc = device_get_softc(dev);
1018
1019 unit = &sc->unit;
1020
1021 LIST_FOREACH(domain, &unit->domain_list, next) {
1022 LIST_FOREACH(ctx, &domain->ctx_list, next) {
1023 if (ctx->did == did) {
1024 refcount_acquire(&ctx->refcnt);
1025 return (ctx);
1026 }
1027 }
1028 }
1029
1030 return (NULL);
1031 }
1032
1033 static struct iommu_ctx *
riscv_iommu_ctx_lookup(device_t dev,device_t child)1034 riscv_iommu_ctx_lookup(device_t dev, device_t child)
1035 {
1036 struct iommu_unit *iommu __diagused;
1037 struct riscv_iommu_softc *sc;
1038 struct riscv_iommu_domain *domain;
1039 struct riscv_iommu_unit *unit;
1040 struct riscv_iommu_ctx *ctx;
1041
1042 dprintf("%s\n", __func__);
1043 sc = device_get_softc(dev);
1044
1045 unit = &sc->unit;
1046 iommu = &unit->iommu;
1047
1048 IOMMU_ASSERT_LOCKED(iommu);
1049
1050 LIST_FOREACH(domain, &unit->domain_list, next) {
1051 IOMMU_DOMAIN_LOCK(&domain->iodom);
1052 LIST_FOREACH(ctx, &domain->ctx_list, next) {
1053 if (ctx->dev == child) {
1054 refcount_acquire(&ctx->refcnt);
1055 IOMMU_DOMAIN_UNLOCK(&domain->iodom);
1056 return (&ctx->ioctx);
1057 }
1058 }
1059 IOMMU_DOMAIN_UNLOCK(&domain->iodom);
1060 }
1061
1062 return (NULL);
1063 }
1064
1065 static int
riscv_iommu_unmap(device_t dev,struct iommu_domain * iodom,vm_offset_t va,bus_size_t size)1066 riscv_iommu_unmap(device_t dev, struct iommu_domain *iodom,
1067 vm_offset_t va, bus_size_t size)
1068 {
1069 struct riscv_iommu_domain *domain;
1070 struct riscv_iommu_softc *sc;
1071 int err;
1072 int i;
1073
1074 sc = device_get_softc(dev);
1075
1076 domain = (struct riscv_iommu_domain *)iodom;
1077
1078 err = 0;
1079
1080 dprintf("%s: %lx, %ld, domain %d\n", __func__, va, size, domain->pscid);
1081
1082 for (i = 0; i < size; i += PAGE_SIZE) {
1083 if (iommu_pmap_remove(&domain->p, va) == 0) {
1084 /* pmap entry removed, invalidate TLB. */
1085 riscv_iommu_inval_vma_page(sc, va, domain->pscid);
1086 } else {
1087 err = ENOENT;
1088 break;
1089 }
1090 va += PAGE_SIZE;
1091 }
1092
1093 riscv_iommu_sync(sc, &sc->cq);
1094
1095 return (err);
1096 }
1097
1098 static int
riscv_iommu_map(device_t dev,struct iommu_domain * iodom,vm_offset_t va,vm_page_t * ma,vm_size_t size,vm_prot_t prot)1099 riscv_iommu_map(device_t dev, struct iommu_domain *iodom,
1100 vm_offset_t va, vm_page_t *ma, vm_size_t size,
1101 vm_prot_t prot)
1102 {
1103 struct riscv_iommu_domain *domain;
1104 struct riscv_iommu_softc *sc;
1105 vm_paddr_t pa;
1106 int error;
1107 int i;
1108
1109 sc = device_get_softc(dev);
1110
1111 domain = (struct riscv_iommu_domain *)iodom;
1112
1113 for (i = 0; size > 0; size -= PAGE_SIZE) {
1114 pa = VM_PAGE_TO_PHYS(ma[i++]);
1115 dprintf("%s: %lx -> %lx, %ld, domain %d\n", __func__, va, pa,
1116 size, domain->pscid);
1117 error = iommu_pmap_enter(&domain->p, va, pa, prot, 0);
1118 if (error)
1119 return (error);
1120 riscv_iommu_inval_vma_page(sc, va, domain->pscid);
1121 va += PAGE_SIZE;
1122 }
1123
1124 riscv_iommu_sync(sc, &sc->cq);
1125
1126 return (0);
1127 }
1128
1129 static struct iommu_domain *
riscv_iommu_domain_alloc(device_t dev,struct iommu_unit * iommu)1130 riscv_iommu_domain_alloc(device_t dev, struct iommu_unit *iommu)
1131 {
1132 struct iommu_domain *iodom;
1133 struct riscv_iommu_domain *domain;
1134 struct riscv_iommu_unit *unit;
1135 struct riscv_iommu_softc *sc;
1136 int new_pscid;
1137 int va_bits;
1138 int error;
1139
1140 sc = device_get_softc(dev);
1141
1142 dprintf("%s\n", __func__);
1143
1144 unit = (struct riscv_iommu_unit *)iommu;
1145
1146 error = riscv_iommu_pscid_alloc(sc, &new_pscid);
1147 if (error) {
1148 device_printf(sc->dev,
1149 "Could not allocate PSCID for a new domain.\n");
1150 return (NULL);
1151 }
1152
1153 domain = malloc(sizeof(*domain), M_IOMMU, M_WAITOK | M_ZERO);
1154 domain->pscid = (uint16_t)new_pscid;
1155
1156 iommu_pmap_pinit(&domain->p, sc->pm_mode);
1157
1158 riscv_iommu_inval_vma_pscid(sc, domain->pscid);
1159
1160 LIST_INIT(&domain->ctx_list);
1161
1162 IOMMU_LOCK(iommu);
1163 LIST_INSERT_HEAD(&unit->domain_list, domain, next);
1164 IOMMU_UNLOCK(iommu);
1165
1166 iodom = &domain->iodom;
1167
1168 va_bits = sc->pm_mode == PMAP_MODE_SV48 ? 48 : 39;
1169
1170 /* Avoid sign-extension. */
1171 va_bits -= 1;
1172
1173 iodom->end = (1ULL << va_bits) - 1;
1174
1175 return (iodom);
1176 }
1177
1178 static void
riscv_iommu_domain_free(device_t dev,struct iommu_domain * iodom)1179 riscv_iommu_domain_free(device_t dev, struct iommu_domain *iodom)
1180 {
1181 struct riscv_iommu_domain *domain;
1182 struct riscv_iommu_softc *sc;
1183
1184 sc = device_get_softc(dev);
1185
1186 dprintf("%s\n", __func__);
1187
1188 domain = (struct riscv_iommu_domain *)iodom;
1189
1190 LIST_REMOVE(domain, next);
1191
1192 iommu_pmap_remove_pages(&domain->p);
1193 iommu_pmap_release(&domain->p);
1194
1195 riscv_iommu_inval_vma_pscid(sc, domain->pscid);
1196 riscv_iommu_pscid_free(sc, domain->pscid);
1197
1198 free(domain, M_IOMMU);
1199 }
1200
1201 static struct iommu_ctx *
riscv_iommu_ctx_alloc(device_t dev,struct iommu_domain * iodom,device_t child,bool disabled)1202 riscv_iommu_ctx_alloc(device_t dev, struct iommu_domain *iodom, device_t child,
1203 bool disabled)
1204 {
1205 struct riscv_iommu_domain *domain;
1206 struct riscv_iommu_ctx *ctx;
1207
1208 dprintf("%s\n", __func__);
1209
1210 domain = (struct riscv_iommu_domain *)iodom;
1211
1212 ctx = malloc(sizeof(struct riscv_iommu_ctx), M_IOMMU,
1213 M_WAITOK | M_ZERO);
1214 ctx->dev = child;
1215 ctx->domain = domain;
1216 refcount_init(&ctx->refcnt, 1);
1217 if (disabled)
1218 ctx->bypass = true;
1219
1220 IOMMU_DOMAIN_LOCK(iodom);
1221 LIST_INSERT_HEAD(&domain->ctx_list, ctx, next);
1222 IOMMU_DOMAIN_UNLOCK(iodom);
1223
1224 return (&ctx->ioctx);
1225 }
1226
1227 static int
riscv_iommu_ctx_init(device_t dev,struct iommu_ctx * ioctx)1228 riscv_iommu_ctx_init(device_t dev, struct iommu_ctx *ioctx)
1229 {
1230 struct riscv_iommu_domain *domain;
1231 struct iommu_domain *iodom;
1232 struct riscv_iommu_softc *sc;
1233 struct riscv_iommu_ctx *ctx;
1234 devclass_t pci_class;
1235 u_int did;
1236 int error;
1237
1238 ctx = (struct riscv_iommu_ctx *)ioctx;
1239
1240 dprintf("%s\n", __func__);
1241
1242 sc = device_get_softc(dev);
1243
1244 domain = ctx->domain;
1245 iodom = (struct iommu_domain *)domain;
1246
1247 pci_class = devclass_find("pci");
1248 if (device_get_devclass(device_get_parent(ctx->dev)) == pci_class) {
1249 error = riscv_iommu_pci_get_did(ctx->dev, NULL, &did);
1250 if (error)
1251 return (error);
1252
1253 ioctx->rid = pci_get_rid(dev);
1254 ctx->did = did;
1255 ctx->vendor = pci_get_vendor(ctx->dev);
1256 ctx->device = pci_get_device(ctx->dev);
1257 }
1258
1259 if (sc->iommu_mode == DDTP_IOMMU_MODE_2LVL) {
1260 error = riscv_iommu_init_l0_directory(sc, ctx->did);
1261 if (error)
1262 return (error);
1263 }
1264 riscv_iommu_init_dc(sc, domain, ctx->did, ctx->bypass);
1265
1266 if (device_get_devclass(device_get_parent(ctx->dev)) == pci_class)
1267 if (iommu_is_buswide_ctx(iodom->iommu, pci_get_bus(ctx->dev)))
1268 riscv_iommu_set_buswide(dev, domain, ctx);
1269
1270 return (0);
1271 }
1272
1273 static bool
riscv_iommu_ctx_free(device_t dev,struct iommu_ctx * ioctx)1274 riscv_iommu_ctx_free(device_t dev, struct iommu_ctx *ioctx)
1275 {
1276 struct riscv_iommu_softc *sc;
1277 struct riscv_iommu_ctx *ctx;
1278
1279 dprintf("%s\n", __func__);
1280
1281 IOMMU_ASSERT_LOCKED(ioctx->domain->iommu);
1282
1283 sc = device_get_softc(dev);
1284
1285 ctx = (struct riscv_iommu_ctx *)ioctx;
1286 if (refcount_release(&ctx->refcnt)) {
1287 riscv_iommu_deinit_dc(sc, ctx->did);
1288 LIST_REMOVE(ctx, next);
1289 free(ctx, M_IOMMU);
1290 return (true);
1291 }
1292
1293 return (false);
1294 }
1295
1296 #ifdef FDT
1297 static int
riscv_iommu_ofw_md_data(device_t dev,struct iommu_ctx * ioctx,pcell_t * cells,int ncells)1298 riscv_iommu_ofw_md_data(device_t dev, struct iommu_ctx *ioctx, pcell_t *cells,
1299 int ncells)
1300 {
1301 struct riscv_iommu_ctx *ctx;
1302
1303 printf("%s\n", __func__);
1304 ctx = (struct riscv_iommu_ctx *)ioctx;
1305
1306 if (ncells != 1)
1307 return (-1);
1308
1309 ctx->did = cells[0];
1310
1311 return (0);
1312 }
1313 #endif
1314
1315 static int
riscv_iommu_read_ivar(device_t dev,device_t child,int which,uintptr_t * result)1316 riscv_iommu_read_ivar(device_t dev, device_t child, int which,
1317 uintptr_t *result)
1318 {
1319 struct riscv_iommu_softc *sc;
1320
1321 sc = device_get_softc(dev);
1322
1323 device_printf(sc->dev, "%s\n", __func__);
1324
1325 return (ENOENT);
1326 }
1327
1328 static device_method_t riscv_iommu_methods[] = {
1329 /* IOMMU interface */
1330 DEVMETHOD(iommu_find, riscv_iommu_find),
1331 DEVMETHOD(iommu_map, riscv_iommu_map),
1332 DEVMETHOD(iommu_unmap, riscv_iommu_unmap),
1333 DEVMETHOD(iommu_domain_alloc, riscv_iommu_domain_alloc),
1334 DEVMETHOD(iommu_domain_free, riscv_iommu_domain_free),
1335 DEVMETHOD(iommu_ctx_alloc, riscv_iommu_ctx_alloc),
1336 DEVMETHOD(iommu_ctx_init, riscv_iommu_ctx_init),
1337 DEVMETHOD(iommu_ctx_free, riscv_iommu_ctx_free),
1338 DEVMETHOD(iommu_ctx_lookup, riscv_iommu_ctx_lookup),
1339 #ifdef FDT
1340 DEVMETHOD(iommu_ofw_md_data, riscv_iommu_ofw_md_data),
1341 #endif
1342
1343 /* Bus interface */
1344 DEVMETHOD(bus_read_ivar, riscv_iommu_read_ivar),
1345
1346 /* End */
1347 DEVMETHOD_END
1348 };
1349
1350 DEFINE_CLASS_0(riscv_iommu, riscv_iommu_driver, riscv_iommu_methods,
1351 sizeof(struct riscv_iommu_softc));
1352