1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause 3 * 4 * Copyright (c) 2026 Ruslan Bukin <br@bsdpad.com> 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 16 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 25 * SUCH DAMAGE. 26 */ 27 28 #ifndef _RISCV_IOMMU_IOMMU_H_ 29 #define _RISCV_IOMMU_IOMMU_H_ 30 31 #define RISCV_IOMMU_CAPABILITIES 0x0000 32 #define CAPABILITIES_VERSION_S 0 33 #define CAPABILITIES_VERSION_M (0xff << CAPABILITIES_VERSION_S) 34 #define CAPABILITIES_SV32 (1 << 8) 35 #define CAPABILITIES_SV39 (1 << 9) 36 #define CAPABILITIES_SV48 (1 << 10) 37 #define CAPABILITIES_SV57 (1 << 11) 38 #define CAPABILITIES_SVPBMT (1 << 15) 39 #define CAPABILITIES_SV32X4 (1 << 16) 40 #define CAPABILITIES_SV39X4 (1 << 17) 41 #define CAPABILITIES_SV48X4 (1 << 18) 42 #define CAPABILITIES_SV57X4 (1 << 19) 43 #define CAPABILITIES_AMO_MRIF (1 << 21) 44 #define CAPABILITIES_MSI_FLAT (1 << 22) 45 #define CAPABILITIES_MSI_MRIF (1 << 23) 46 #define CAPABILITIES_AMO_HWAD (1 << 24) 47 #define CAPABILITIES_ATS (1 << 25) 48 #define CAPABILITIES_T2GPA (1 << 26) 49 #define CAPABILITIES_END (1 << 27) 50 #define CAPABILITIES_IGS_S 28 51 #define CAPABILITIES_IGS_M (0x3 << CAPABILITIES_IGS_S) 52 #define CAPABILITIES_HPM (1 << 30) 53 #define CAPABILITIES_DBG (1 << 31) 54 #define CAPABILITIES_PAS_S 32ULL 55 #define CAPABILITIES_PAS_M (0x3f << CAPABILITIES_PAS_S) 56 #define CAPABILITIES_PD8 (1ULL << 38) 57 #define CAPABILITIES_PD17 (1ULL << 39) 58 #define CAPABILITIES_PD20 (1ULL << 40) 59 #define RISCV_IOMMU_FCTL 0x0008 60 #define FCTL_BE (1 << 0) /* Big-endian */ 61 #define FCTL_WSI (1 << 1) /* Wire-signalled Ints. */ 62 #define FCTL_GXL (1 << 2) /* Guest physical addresses */ 63 #define RISCV_IOMMU_DDTP 0x0010 64 #define DDTP_IOMMU_MODE_S 0 65 #define DDTP_IOMMU_MODE_OFF (0 << DDTP_IOMMU_MODE_S) 66 #define DDTP_IOMMU_MODE_BARE (1 << DDTP_IOMMU_MODE_S) 67 #define DDTP_IOMMU_MODE_1LVL (2 << DDTP_IOMMU_MODE_S) 68 #define DDTP_IOMMU_MODE_2LVL (3 << DDTP_IOMMU_MODE_S) 69 #define DDTP_IOMMU_MODE_3LVL (4 << DDTP_IOMMU_MODE_S) 70 #define DDTP_BUSY (1 << 4) 71 #define DDTP_PPN_S 10 72 #define DDTP_PPN_M (0xfffffffffffULL << DDTP_PPN_S) 73 #define RISCV_IOMMU_CQB 0x18 /* Command queue base. */ 74 #define CQB_LOG2SZ_1_S 0 75 #define CQB_LOG2SZ_1_M (0x3f << CQB_LOG2SZ_1_S) 76 #define CQB_PPN_S 10 77 #define CQB_PPN_M (0xfffffffffffULL << CQB_PPN_S) 78 #define RISCV_IOMMU_CQH 0x20 79 #define RISCV_IOMMU_CQT 0x24 80 #define RISCV_IOMMU_FQB 0x28 /* Fault queue base. */ 81 #define RISCV_IOMMU_FQH 0x30 82 #define RISCV_IOMMU_FQT 0x34 83 #define RISCV_IOMMU_PQB 0x38 /* Page queue base. */ 84 #define RISCV_IOMMU_PQH 0x40 85 #define RISCV_IOMMU_PQT 0x44 86 #define RISCV_IOMMU_CQCSR 0x48 87 #define CQCSR_BUSY (1 << 17) /* Write is observed */ 88 #define CQCSR_CQON (1 << 16) /* Active */ 89 #define CQCSR_FENCE_W_IP (1 << 11) /* iofence.c completed */ 90 #define CQCSR_CMD_ILL (1 << 10) /* Illegal command */ 91 #define CQCSR_CMD_TO (1 << 9) /* Timeout */ 92 #define CQCSR_CQMF (1 << 8) /* Memory Fault */ 93 #define CQCSR_CIE (1 << 1) /* Interrupt Enable */ 94 #define CQCSR_CQEN (1 << 0) /* Enable */ 95 #define RISCV_IOMMU_FQCSR 0x4C 96 #define FQCSR_BUSY (1 << 17) /* Write is observed */ 97 #define FQCSR_FQON (1 << 16) /* Active */ 98 #define FQCSR_FQOF (1 << 9) /* Overflow */ 99 #define FQCSR_FQMF (1 << 8) /* Memory Fault */ 100 #define FQCSR_FIE (1 << 1) /* Interrupt Enable */ 101 #define FQCSR_FQEN (1 << 0) /* Enable */ 102 #define RISCV_IOMMU_PQCSR 0x50 103 #define PQCSR_BUSY (1 << 17) /* Write is observed */ 104 #define PQCSR_PQON (1 << 16) /* Active */ 105 #define PQCSR_PQOF (1 << 9) /* Overflow */ 106 #define PQCSR_PQMF (1 << 8) /* Memory Fault */ 107 #define PQCSR_PIE (1 << 1) /* Interrupt Enable */ 108 #define PQCSR_PQEN (1 << 0) /* Enable */ 109 #define RISCV_IOMMU_IPSR 0x54 110 #define IPSR_CIP (1 << 0) /* Command queue interrupt pending */ 111 #define IPSR_FIP (1 << 1) /* Fault queue interrupt pending */ 112 #define IPSR_PMIP (1 << 2) /* Performance monitoring int pend */ 113 #define IPSR_PIP (1 << 3) /* Page queue interrupt pending */ 114 115 #define RISCV_IOMMU_IOCOUNTOVF 0x0058 116 #define RISCV_IOMMU_IOCOUNTINH 0x005C 117 118 #define RISCV_IOMMU_IOHPMCYCLES 0x0060 119 #define RISCV_IOMMU_IOHPMCTR_BASE 0x0068 120 #define RISCV_IOMMU_IOHPMCTR(_n) (RISCV_IOMMU_IOHPMCTR_BASE + ((_n) * 0x8)) 121 #define RISCV_IOMMU_IOHPMEVT_BASE 0x0160 122 #define RISCV_IOMMU_IOHPMEVT(_n) (RISCV_IOMMU_IOHPMEVT_BASE + ((_n) * 0x8)) 123 #define RISCV_IOMMU_TR_REQ_IOVA 0x0258 124 #define RISCV_IOMMU_TR_REQ_CTL 0x0260 125 #define RISCV_IOMMU_TR_RESPONSE 0x0268 126 #define RISCV_IOMMU_ICVEC 0x02F8 127 128 #define RISCV_IOMMU_LOCK(_sc) mtx_lock(&(_sc)->mtx) 129 #define RISCV_IOMMU_UNLOCK(_sc) mtx_unlock(&(_sc)->mtx) 130 131 DECLARE_CLASS(riscv_iommu_driver); 132 133 MALLOC_DECLARE(M_IOMMU); 134 135 struct riscv_iommu_unit { 136 struct iommu_unit iommu; 137 LIST_HEAD(, riscv_iommu_domain) domain_list; 138 LIST_ENTRY(riscv_iommu_unit) next; 139 device_t dev; 140 intptr_t xref; 141 }; 142 143 struct riscv_iommu_domain { 144 struct iommu_domain iodom; 145 LIST_HEAD(, riscv_iommu_ctx) ctx_list; 146 LIST_ENTRY(riscv_iommu_domain) next; 147 u_int entries_cnt; 148 struct riscv_iommu_cd *cd; 149 struct riscv_iommu_pmap p; 150 uint16_t pscid; 151 }; 152 153 struct riscv_iommu_ctx { 154 struct iommu_ctx ioctx; 155 struct riscv_iommu_domain *domain; 156 LIST_ENTRY(riscv_iommu_ctx) next; 157 device_t dev; 158 bool bypass; 159 int did; 160 uint16_t vendor; 161 uint16_t device; 162 u_int refcnt; 163 }; 164 165 struct riscv_iommu_queue_local_copy { 166 union { 167 uint64_t val; 168 struct { 169 uint32_t head; 170 uint32_t tail; 171 }; 172 }; 173 }; 174 175 struct riscv_iommu_queue { 176 struct riscv_iommu_queue_local_copy lc; 177 vm_paddr_t paddr; 178 void *vaddr; 179 uint64_t mask; 180 uint32_t head_off; 181 uint32_t tail_off; 182 int size_log2; 183 uint64_t base; 184 uint64_t csr; 185 int idx; 186 uint8_t entry_size; 187 }; 188 189 struct l1_desc { 190 uint8_t span; 191 void *va; 192 vm_paddr_t pa; 193 }; 194 195 /* Base-format device-context. */ 196 struct riscv_iommu_dc_base { 197 uint64_t tc; /* Translation control */ 198 #define DC_TC_V (1 << 0) 199 #define DC_TC_EN_ATS (1 << 1) 200 #define DC_TC_EN_PRI (1 << 2) 201 #define DC_TC_T2GPA (1 << 3) 202 #define DC_TC_DTF (1 << 4) 203 #define DC_TC_PDTV (1 << 5) 204 #define DC_TC_PRPR (1 << 6) 205 #define DC_TC_GADE (1 << 7) 206 #define DC_TC_SADE (1 << 8) 207 #define DC_TC_DPE (1 << 9) 208 #define DC_TC_SBE (1 << 10) 209 #define DC_TC_SXL (1 << 11) 210 uint64_t iohgatp; /* IO Hyp guest address translation */ 211 uint64_t ta; /* Translation attributes */ 212 #define DC_TA_V (1 << 0) 213 #define DC_TA_ENS (1 << 1) 214 #define DC_TA_SUM (1 << 2) 215 #define DC_TA_PSCID_S 12 216 #define DC_TA_PSCID_M (0xfffff << DC_TA_PSCID_S) 217 uint64_t fsc; /* First-stage-context */ 218 }; 219 220 /* Extended-format device-context. */ 221 struct riscv_iommu_dc { 222 struct riscv_iommu_dc_base base; 223 uint64_t msiptp; /* MSI page table pointer */ 224 uint64_t msi_addr_mask; 225 uint64_t msi_addr_pattern; 226 uint64_t _reserved; 227 }; 228 229 #define DC_NON_LEAF_ENTRY_PPN_S 10 230 #define DC_NON_LEAF_ENTRY_VALID (1 << 0) 231 232 struct riscv_iommu_ddt { 233 void *vaddr; 234 uint64_t base; 235 uint32_t base_cfg; 236 uint32_t num_top_entries; 237 struct l1_desc *l1; 238 struct riscv_iommu_dc *dc; 239 }; 240 241 struct riscv_iommu_softc { 242 device_t dev; 243 intptr_t xref; 244 struct riscv_iommu_unit unit; 245 struct resource *res[5]; 246 void *intr_cookie[4]; 247 struct riscv_iommu_queue cq; 248 struct riscv_iommu_queue fq; 249 struct riscv_iommu_queue pq; 250 struct riscv_iommu_ddt ddt; 251 struct mtx mtx; 252 uint32_t l0_did_bits; 253 uint32_t dc_dwords; 254 255 /* PSCID management. */ 256 bitstr_t *pscid_set; 257 int pscid_set_size; 258 struct mtx pscid_set_mutex; 259 uint32_t pscid_bits; 260 261 enum pmap_mode pm_mode; 262 int iommu_mode; 263 }; 264 265 /* 266 * Command queue request. 267 */ 268 struct riscv_iommu_command { 269 uint64_t dword0; 270 uint64_t dword1; 271 }; 272 273 enum riscv_iommu_fq_causes { 274 FQ_CAUSE_INST_FAULT = 1, 275 FQ_CAUSE_RD_ADDR_MISALIGNED = 4, 276 FQ_CAUSE_RD_FAULT = 5, 277 FQ_CAUSE_WR_ADDR_MISALIGNED = 6, 278 FQ_CAUSE_WR_FAULT = 7, 279 FQ_CAUSE_INST_FAULT_S = 12, 280 FQ_CAUSE_RD_FAULT_S = 13, 281 FQ_CAUSE_WR_FAULT_S = 15, 282 FQ_CAUSE_INST_FAULT_VS = 20, 283 FQ_CAUSE_RD_FAULT_VS = 21, 284 FQ_CAUSE_WR_FAULT_VS = 23, 285 FQ_CAUSE_DMA_DISABLED = 256, 286 FQ_CAUSE_DDT_LOAD_FAULT = 257, 287 FQ_CAUSE_DDT_INVALID = 258, 288 FQ_CAUSE_DDT_MISCONFIGURED = 259, 289 FQ_CAUSE_TR_TYPE_DISALLOWED = 260, 290 FQ_CAUSE_MSI_LOAD_FAULT = 261, 291 FQ_CAUSE_MSI_INVALID = 262, 292 FQ_CAUSE_MSI_MISCONFIGURED = 263, 293 FQ_CAUSE_MRIF_FAULT = 264, 294 FQ_CAUSE_PDT_LOAD_FAULT = 265, 295 FQ_CAUSE_PDT_INVALID = 266, 296 FQ_CAUSE_PDT_MISCONFIGURED = 267, 297 FQ_CAUSE_DDT_CORRUPTED = 268, 298 FQ_CAUSE_PDT_CORRUPTED = 269, 299 FQ_CAUSE_MSI_PT_CORRUPTED = 270, 300 FQ_CAUSE_MRIF_CORRUPTED = 271, 301 FQ_CAUSE_INTERNAL_DP_ERROR = 272, 302 FQ_CAUSE_MSI_WR_FAULT = 273, 303 FQ_CAUSE_PT_CORRUPTED = 274, 304 }; 305 306 /* 307 * Fault queue record. 308 */ 309 struct riscv_iommu_fq_record { 310 uint64_t hdr; 311 #define FQR_HDR_CAUSE_S 0 312 #define FQR_HDR_CAUSE_M (0xfff << FQR_HDR_CAUSE_S) 313 #define FQR_HDR_PID_S 12 314 #define FQR_HDR_PID_M (0xfffffULL << FQR_HDR_PID_S) 315 #define FQR_HDR_PV (1ULL << 32) 316 #define FQR_HDR_PRIV (1ULL << 33) 317 #define FQR_HDR_TTYP_S 34ULL 318 #define FQR_HDR_TTYP_M (0x3fULL << FQR_HDR_TTYP_S) 319 #define FQR_HDR_DID_S 40ULL 320 #define FQR_HDR_DID_M (0xffffffULL << FQR_HDR_DID_S) 321 uint32_t custom; 322 uint32_t reserved; 323 uint64_t iotval; 324 uint64_t iotval2; 325 }; 326 327 #define COMMAND_OPCODE_S 0 328 #define COMMAND_OPCODE_IOTINVAL (1 << COMMAND_OPCODE_S) 329 #define COMMAND_OPCODE_IOFENCE (2 << COMMAND_OPCODE_S) 330 #define COMMAND_OPCODE_IODIR (3 << COMMAND_OPCODE_S) 331 #define COMMAND_OPCODE_ATS (4 << COMMAND_OPCODE_S) 332 #define COMMAND_OPCODE_FUNC_S 7 333 #define COMMAND_OPCODE_FUNC_M (0x3 << COMMAND_OPCODE_FUNC_S) 334 #define FUNC_IODIR_INVAL_DDT (0 << COMMAND_OPCODE_FUNC_S) 335 #define FUNC_IODIR_INVAL_PDT (1 << COMMAND_OPCODE_FUNC_S) 336 #define FUNC_IODIR_PID_S 12 337 #define FUNC_IODIR_DV (1ULL << 33) /* DID Valid */ 338 #define FUNC_IODIR_DID_S 40ULL 339 /* dword0 */ 340 #define FUNC_IOTINVAL_VMA (0 << COMMAND_OPCODE_FUNC_S) 341 #define FUNC_IOTINVAL_GVMA (1 << COMMAND_OPCODE_FUNC_S) 342 #define FUNC_IOTINVAL_AV (1 << 10) /* Address Valid */ 343 #define FUNC_IOTINVAL_PSCID_S 12 /* Process-Soft-Context ID */ 344 #define FUNC_IOTINVAL_PSCV (1ULL << 32) /* PSCID Valid */ 345 #define FUNC_IOTINVAL_GV (1ULL << 33) /* GSCID Valid */ 346 #define FUNC_IOTINVAL_GSCID_S 44 /* Guest-Soft-Context ID */ 347 /* dword1 */ 348 #define FUNC_IOTINVAL_ADDR_S 10 349 #define FUNC_IOFENCE_FUNC_C (0 << 7) 350 #define FUNC_IOFENCE_AV (1 << 10) 351 #define FUNC_IOFENCE_WSI (1 << 11) 352 #define FUNC_IOFENCE_PR (1 << 12) 353 #define FUNC_IOFENCE_PW (1 << 13) 354 #define FUNC_IOFENCE_DATA_S 32ULL 355 356 int riscv_iommu_attach(device_t dev); 357 struct riscv_iommu_ctx *riscv_iommu_ctx_lookup_by_did(device_t dev, u_int did); 358 359 #endif /* _RISCV_IOMMU_IOMMU_H_ */ 360