1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause 3 * 4 * Copyright (C) 2018 Alexandru Elisei <alexandru.elisei@gmail.com> 5 * Copyright (C) 2020-2022 Andrew Turner 6 * Copyright (C) 2023 Arm Ltd 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 17 * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS ``AS IS'' AND 18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20 * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE 21 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 27 * SUCH DAMAGE. 28 */ 29 30 #include <sys/cdefs.h> 31 32 #include <sys/types.h> 33 #include <sys/errno.h> 34 #include <sys/systm.h> 35 #include <sys/bitstring.h> 36 #include <sys/bus.h> 37 #include <sys/kernel.h> 38 #include <sys/lock.h> 39 #include <sys/malloc.h> 40 #include <sys/module.h> 41 #include <sys/mutex.h> 42 #include <sys/rman.h> 43 #include <sys/smp.h> 44 45 #include <vm/vm.h> 46 #include <vm/pmap.h> 47 48 #include <dev/ofw/openfirm.h> 49 50 #include <machine/armreg.h> 51 #include <machine/atomic.h> 52 #include <machine/bus.h> 53 #include <machine/cpufunc.h> 54 #include <machine/cpu.h> 55 #include <machine/machdep.h> 56 #include <machine/param.h> 57 #include <machine/pmap.h> 58 #include <machine/vmparam.h> 59 #include <machine/intr.h> 60 #include <machine/vmm.h> 61 #include <machine/vmm_dev.h> 62 #include <machine/vmm_instruction_emul.h> 63 64 #include <arm/arm/gic_common.h> 65 #include <arm64/arm64/gic_v3_reg.h> 66 #include <arm64/arm64/gic_v3_var.h> 67 68 #include <arm64/vmm/hyp.h> 69 #include <arm64/vmm/mmu.h> 70 #include <arm64/vmm/arm64.h> 71 #include <arm64/vmm/vmm_handlers.h> 72 73 #include "vgic.h" 74 #include "vgic_v3.h" 75 #include "vgic_v3_reg.h" 76 77 #include "vgic_if.h" 78 79 #define VGIC_SGI_NUM (GIC_LAST_SGI - GIC_FIRST_SGI + 1) 80 #define VGIC_PPI_NUM (GIC_LAST_PPI - GIC_FIRST_PPI + 1) 81 #define VGIC_SPI_NUM (GIC_LAST_SPI - GIC_FIRST_SPI + 1) 82 #define VGIC_PRV_I_NUM (VGIC_SGI_NUM + VGIC_PPI_NUM) 83 #define VGIC_SHR_I_NUM (VGIC_SPI_NUM) 84 85 MALLOC_DEFINE(M_VGIC_V3, "ARM VMM VGIC V3", "ARM VMM VGIC V3"); 86 87 /* TODO: Move to softc */ 88 struct vgic_v3_virt_features { 89 uint8_t min_prio; 90 size_t ich_lr_num; 91 size_t ich_apr_num; 92 }; 93 94 struct vgic_v3_irq { 95 /* List of IRQs that are active or pending */ 96 TAILQ_ENTRY(vgic_v3_irq) act_pend_list; 97 struct mtx irq_spinmtx; 98 uint64_t mpidr; 99 int target_vcpu; 100 uint32_t irq; 101 bool active; 102 bool pending; 103 bool enabled; 104 bool level; 105 bool on_aplist; 106 uint8_t priority; 107 uint8_t config; 108 #define VGIC_CONFIG_MASK 0x2 109 #define VGIC_CONFIG_LEVEL 0x0 110 #define VGIC_CONFIG_EDGE 0x2 111 }; 112 113 /* Global data not needed by EL2 */ 114 struct vgic_v3 { 115 struct mtx dist_mtx; 116 uint64_t dist_start; 117 size_t dist_end; 118 119 uint64_t redist_start; 120 size_t redist_end; 121 122 uint32_t gicd_ctlr; /* Distributor Control Register */ 123 124 struct vgic_v3_irq *irqs; 125 }; 126 127 /* Per-CPU data not needed by EL2 */ 128 struct vgic_v3_cpu { 129 /* 130 * We need a mutex for accessing the list registers because they are 131 * modified asynchronously by the virtual timer. 132 * 133 * Note that the mutex *MUST* be a spin mutex because an interrupt can 134 * be injected by a callout callback function, thereby modifying the 135 * list registers from a context where sleeping is forbidden. 136 */ 137 struct mtx lr_mtx; 138 139 struct vgic_v3_irq private_irqs[VGIC_PRV_I_NUM]; 140 TAILQ_HEAD(, vgic_v3_irq) irq_act_pend; 141 u_int ich_lr_used; 142 }; 143 144 /* How many IRQs we support (SGIs + PPIs + SPIs). Not including LPIs */ 145 #define VGIC_NIRQS 1023 146 /* Pretend to be an Arm design */ 147 #define VGIC_IIDR 0x43b 148 149 static vgic_inject_irq_t vgic_v3_inject_irq; 150 static vgic_inject_msi_t vgic_v3_inject_msi; 151 152 static int vgic_v3_max_cpu_count(device_t dev, struct hyp *hyp); 153 154 #define INJECT_IRQ(hyp, vcpuid, irqid, level) \ 155 vgic_v3_inject_irq(NULL, (hyp), (vcpuid), (irqid), (level)) 156 157 typedef void (register_read)(struct hypctx *, u_int, uint64_t *, void *); 158 typedef void (register_write)(struct hypctx *, u_int, u_int, u_int, 159 uint64_t, void *); 160 161 #define VGIC_8_BIT (1 << 0) 162 /* (1 << 1) is reserved for 16 bit accesses */ 163 #define VGIC_32_BIT (1 << 2) 164 #define VGIC_64_BIT (1 << 3) 165 166 struct vgic_register { 167 u_int start; /* Start within a memory region */ 168 u_int end; 169 u_int size; 170 u_int flags; 171 register_read *read; 172 register_write *write; 173 }; 174 175 #define VGIC_REGISTER_RANGE(reg_start, reg_end, reg_size, reg_flags, readf, \ 176 writef) \ 177 { \ 178 .start = (reg_start), \ 179 .end = (reg_end), \ 180 .size = (reg_size), \ 181 .flags = (reg_flags), \ 182 .read = (readf), \ 183 .write = (writef), \ 184 } 185 186 #define VGIC_REGISTER_RANGE_RAZ_WI(reg_start, reg_end, reg_size, reg_flags) \ 187 VGIC_REGISTER_RANGE(reg_start, reg_end, reg_size, reg_flags, \ 188 gic_zero_read, gic_ignore_write) 189 190 #define VGIC_REGISTER(start_addr, reg_size, reg_flags, readf, writef) \ 191 VGIC_REGISTER_RANGE(start_addr, (start_addr) + (reg_size), \ 192 reg_size, reg_flags, readf, writef) 193 194 #define VGIC_REGISTER_RAZ_WI(start_addr, reg_size, reg_flags) \ 195 VGIC_REGISTER_RANGE_RAZ_WI(start_addr, \ 196 (start_addr) + (reg_size), reg_size, reg_flags) 197 198 static register_read gic_pidr2_read; 199 static register_read gic_zero_read; 200 static register_write gic_ignore_write; 201 202 /* GICD_CTLR */ 203 static register_read dist_ctlr_read; 204 static register_write dist_ctlr_write; 205 /* GICD_TYPER */ 206 static register_read dist_typer_read; 207 /* GICD_IIDR */ 208 static register_read dist_iidr_read; 209 /* GICD_STATUSR - RAZ/WI as we don't report errors (yet) */ 210 /* GICD_SETSPI_NSR & GICD_CLRSPI_NSR */ 211 static register_write dist_setclrspi_nsr_write; 212 /* GICD_SETSPI_SR - RAZ/WI */ 213 /* GICD_CLRSPI_SR - RAZ/WI */ 214 /* GICD_IGROUPR - RAZ/WI as GICD_CTLR.ARE == 1 */ 215 /* GICD_ISENABLER */ 216 static register_read dist_isenabler_read; 217 static register_write dist_isenabler_write; 218 /* GICD_ICENABLER */ 219 static register_read dist_icenabler_read; 220 static register_write dist_icenabler_write; 221 /* GICD_ISPENDR */ 222 static register_read dist_ispendr_read; 223 static register_write dist_ispendr_write; 224 /* GICD_ICPENDR */ 225 static register_read dist_icpendr_read; 226 static register_write dist_icpendr_write; 227 /* GICD_ISACTIVER */ 228 static register_read dist_isactiver_read; 229 static register_write dist_isactiver_write; 230 /* GICD_ICACTIVER */ 231 static register_read dist_icactiver_read; 232 static register_write dist_icactiver_write; 233 /* GICD_IPRIORITYR */ 234 static register_read dist_ipriorityr_read; 235 static register_write dist_ipriorityr_write; 236 /* GICD_ITARGETSR - RAZ/WI as GICD_CTLR.ARE == 1 */ 237 /* GICD_ICFGR */ 238 static register_read dist_icfgr_read; 239 static register_write dist_icfgr_write; 240 /* GICD_IGRPMODR - RAZ/WI from non-secure mode */ 241 /* GICD_NSACR - RAZ/WI from non-secure mode */ 242 /* GICD_SGIR - RAZ/WI as GICD_CTLR.ARE == 1 */ 243 /* GICD_CPENDSGIR - RAZ/WI as GICD_CTLR.ARE == 1 */ 244 /* GICD_SPENDSGIR - RAZ/WI as GICD_CTLR.ARE == 1 */ 245 /* GICD_IROUTER */ 246 static register_read dist_irouter_read; 247 static register_write dist_irouter_write; 248 249 static struct vgic_register dist_registers[] = { 250 VGIC_REGISTER(GICD_CTLR, 4, VGIC_32_BIT, dist_ctlr_read, 251 dist_ctlr_write), 252 VGIC_REGISTER(GICD_TYPER, 4, VGIC_32_BIT, dist_typer_read, 253 gic_ignore_write), 254 VGIC_REGISTER(GICD_IIDR, 4, VGIC_32_BIT, dist_iidr_read, 255 gic_ignore_write), 256 VGIC_REGISTER_RAZ_WI(GICD_STATUSR, 4, VGIC_32_BIT), 257 VGIC_REGISTER(GICD_SETSPI_NSR, 4, VGIC_32_BIT, gic_zero_read, 258 dist_setclrspi_nsr_write), 259 VGIC_REGISTER(GICD_CLRSPI_NSR, 4, VGIC_32_BIT, gic_zero_read, 260 dist_setclrspi_nsr_write), 261 VGIC_REGISTER_RAZ_WI(GICD_SETSPI_SR, 4, VGIC_32_BIT), 262 VGIC_REGISTER_RAZ_WI(GICD_CLRSPI_SR, 4, VGIC_32_BIT), 263 VGIC_REGISTER_RANGE_RAZ_WI(GICD_IGROUPR(0), GICD_IGROUPR(1024), 4, 264 VGIC_32_BIT), 265 266 VGIC_REGISTER_RAZ_WI(GICD_ISENABLER(0), 4, VGIC_32_BIT), 267 VGIC_REGISTER_RANGE(GICD_ISENABLER(32), GICD_ISENABLER(1024), 4, 268 VGIC_32_BIT, dist_isenabler_read, dist_isenabler_write), 269 270 VGIC_REGISTER_RAZ_WI(GICD_ICENABLER(0), 4, VGIC_32_BIT), 271 VGIC_REGISTER_RANGE(GICD_ICENABLER(32), GICD_ICENABLER(1024), 4, 272 VGIC_32_BIT, dist_icenabler_read, dist_icenabler_write), 273 274 VGIC_REGISTER_RAZ_WI(GICD_ISPENDR(0), 4, VGIC_32_BIT), 275 VGIC_REGISTER_RANGE(GICD_ISPENDR(32), GICD_ISPENDR(1024), 4, 276 VGIC_32_BIT, dist_ispendr_read, dist_ispendr_write), 277 278 VGIC_REGISTER_RAZ_WI(GICD_ICPENDR(0), 4, VGIC_32_BIT), 279 VGIC_REGISTER_RANGE(GICD_ICPENDR(32), GICD_ICPENDR(1024), 4, 280 VGIC_32_BIT, dist_icpendr_read, dist_icpendr_write), 281 282 VGIC_REGISTER_RAZ_WI(GICD_ISACTIVER(0), 4, VGIC_32_BIT), 283 VGIC_REGISTER_RANGE(GICD_ISACTIVER(32), GICD_ISACTIVER(1024), 4, 284 VGIC_32_BIT, dist_isactiver_read, dist_isactiver_write), 285 286 VGIC_REGISTER_RAZ_WI(GICD_ICACTIVER(0), 4, VGIC_32_BIT), 287 VGIC_REGISTER_RANGE(GICD_ICACTIVER(32), GICD_ICACTIVER(1024), 4, 288 VGIC_32_BIT, dist_icactiver_read, dist_icactiver_write), 289 290 VGIC_REGISTER_RANGE_RAZ_WI(GICD_IPRIORITYR(0), GICD_IPRIORITYR(32), 4, 291 VGIC_32_BIT | VGIC_8_BIT), 292 VGIC_REGISTER_RANGE(GICD_IPRIORITYR(32), GICD_IPRIORITYR(1024), 4, 293 VGIC_32_BIT | VGIC_8_BIT, dist_ipriorityr_read, 294 dist_ipriorityr_write), 295 296 VGIC_REGISTER_RANGE_RAZ_WI(GICD_ITARGETSR(0), GICD_ITARGETSR(1024), 4, 297 VGIC_32_BIT | VGIC_8_BIT), 298 299 VGIC_REGISTER_RANGE_RAZ_WI(GICD_ICFGR(0), GICD_ICFGR(32), 4, 300 VGIC_32_BIT), 301 VGIC_REGISTER_RANGE(GICD_ICFGR(32), GICD_ICFGR(1024), 4, 302 VGIC_32_BIT, dist_icfgr_read, dist_icfgr_write), 303 /* 304 VGIC_REGISTER_RANGE(GICD_IGRPMODR(0), GICD_IGRPMODR(1024), 4, 305 VGIC_32_BIT, dist_igrpmodr_read, dist_igrpmodr_write), 306 VGIC_REGISTER_RANGE(GICD_NSACR(0), GICD_NSACR(1024), 4, 307 VGIC_32_BIT, dist_nsacr_read, dist_nsacr_write), 308 */ 309 VGIC_REGISTER_RAZ_WI(GICD_SGIR, 4, VGIC_32_BIT), 310 /* 311 VGIC_REGISTER_RANGE(GICD_CPENDSGIR(0), GICD_CPENDSGIR(1024), 4, 312 VGIC_32_BIT | VGIC_8_BIT, dist_cpendsgir_read, 313 dist_cpendsgir_write), 314 VGIC_REGISTER_RANGE(GICD_SPENDSGIR(0), GICD_SPENDSGIR(1024), 4, 315 VGIC_32_BIT | VGIC_8_BIT, dist_spendsgir_read, 316 dist_spendsgir_write), 317 */ 318 VGIC_REGISTER_RANGE(GICD_IROUTER(32), GICD_IROUTER(1024), 8, 319 VGIC_64_BIT | VGIC_32_BIT, dist_irouter_read, dist_irouter_write), 320 321 VGIC_REGISTER_RANGE_RAZ_WI(GICD_PIDR4, GICD_PIDR2, 4, VGIC_32_BIT), 322 VGIC_REGISTER(GICD_PIDR2, 4, VGIC_32_BIT, gic_pidr2_read, 323 gic_ignore_write), 324 VGIC_REGISTER_RANGE_RAZ_WI(GICD_PIDR2 + 4, GICD_SIZE, 4, VGIC_32_BIT), 325 }; 326 327 /* GICR_CTLR - Ignore writes as no bits can be set */ 328 static register_read redist_ctlr_read; 329 /* GICR_IIDR */ 330 static register_read redist_iidr_read; 331 /* GICR_TYPER */ 332 static register_read redist_typer_read; 333 /* GICR_STATUSR - RAZ/WI as we don't report errors (yet) */ 334 /* GICR_WAKER - RAZ/WI from non-secure mode */ 335 /* GICR_SETLPIR - RAZ/WI as no LPIs are supported */ 336 /* GICR_CLRLPIR - RAZ/WI as no LPIs are supported */ 337 /* GICR_PROPBASER - RAZ/WI as no LPIs are supported */ 338 /* GICR_PENDBASER - RAZ/WI as no LPIs are supported */ 339 /* GICR_INVLPIR - RAZ/WI as no LPIs are supported */ 340 /* GICR_INVALLR - RAZ/WI as no LPIs are supported */ 341 /* GICR_SYNCR - RAZ/WI as no LPIs are supported */ 342 343 static struct vgic_register redist_rd_registers[] = { 344 VGIC_REGISTER(GICR_CTLR, 4, VGIC_32_BIT, redist_ctlr_read, 345 gic_ignore_write), 346 VGIC_REGISTER(GICR_IIDR, 4, VGIC_32_BIT, redist_iidr_read, 347 gic_ignore_write), 348 VGIC_REGISTER(GICR_TYPER, 8, VGIC_64_BIT | VGIC_32_BIT, 349 redist_typer_read, gic_ignore_write), 350 VGIC_REGISTER_RAZ_WI(GICR_STATUSR, 4, VGIC_32_BIT), 351 VGIC_REGISTER_RAZ_WI(GICR_WAKER, 4, VGIC_32_BIT), 352 VGIC_REGISTER_RAZ_WI(GICR_SETLPIR, 8, VGIC_64_BIT | VGIC_32_BIT), 353 VGIC_REGISTER_RAZ_WI(GICR_CLRLPIR, 8, VGIC_64_BIT | VGIC_32_BIT), 354 VGIC_REGISTER_RAZ_WI(GICR_PROPBASER, 8, VGIC_64_BIT | VGIC_32_BIT), 355 VGIC_REGISTER_RAZ_WI(GICR_PENDBASER, 8, VGIC_64_BIT | VGIC_32_BIT), 356 VGIC_REGISTER_RAZ_WI(GICR_INVLPIR, 8, VGIC_64_BIT | VGIC_32_BIT), 357 VGIC_REGISTER_RAZ_WI(GICR_INVALLR, 8, VGIC_64_BIT | VGIC_32_BIT), 358 VGIC_REGISTER_RAZ_WI(GICR_SYNCR, 4, VGIC_32_BIT), 359 360 /* These are identical to the dist registers */ 361 VGIC_REGISTER_RANGE_RAZ_WI(GICD_PIDR4, GICD_PIDR2, 4, VGIC_32_BIT), 362 VGIC_REGISTER(GICD_PIDR2, 4, VGIC_32_BIT, gic_pidr2_read, 363 gic_ignore_write), 364 VGIC_REGISTER_RANGE_RAZ_WI(GICD_PIDR2 + 4, GICD_SIZE, 4, 365 VGIC_32_BIT), 366 }; 367 368 /* GICR_IGROUPR0 - RAZ/WI from non-secure mode */ 369 /* GICR_ISENABLER0 */ 370 static register_read redist_ienabler0_read; 371 static register_write redist_isenabler0_write; 372 /* GICR_ICENABLER0 */ 373 static register_write redist_icenabler0_write; 374 /* GICR_ISPENDR0 */ 375 static register_read redist_ipendr0_read; 376 static register_write redist_ispendr0_write; 377 /* GICR_ICPENDR0 */ 378 static register_write redist_icpendr0_write; 379 /* GICR_ISACTIVER0 */ 380 static register_read redist_iactiver0_read; 381 static register_write redist_isactiver0_write; 382 /* GICR_ICACTIVER0 */ 383 static register_write redist_icactiver0_write; 384 /* GICR_IPRIORITYR */ 385 static register_read redist_ipriorityr_read; 386 static register_write redist_ipriorityr_write; 387 /* GICR_ICFGR0 - RAZ/WI from non-secure mode */ 388 /* GICR_ICFGR1 */ 389 static register_read redist_icfgr1_read; 390 static register_write redist_icfgr1_write; 391 /* GICR_IGRPMODR0 - RAZ/WI from non-secure mode */ 392 /* GICR_NSCAR - RAZ/WI from non-secure mode */ 393 394 static struct vgic_register redist_sgi_registers[] = { 395 VGIC_REGISTER_RAZ_WI(GICR_IGROUPR0, 4, VGIC_32_BIT), 396 VGIC_REGISTER(GICR_ISENABLER0, 4, VGIC_32_BIT, redist_ienabler0_read, 397 redist_isenabler0_write), 398 VGIC_REGISTER(GICR_ICENABLER0, 4, VGIC_32_BIT, redist_ienabler0_read, 399 redist_icenabler0_write), 400 VGIC_REGISTER(GICR_ISPENDR0, 4, VGIC_32_BIT, redist_ipendr0_read, 401 redist_ispendr0_write), 402 VGIC_REGISTER(GICR_ICPENDR0, 4, VGIC_32_BIT, redist_ipendr0_read, 403 redist_icpendr0_write), 404 VGIC_REGISTER(GICR_ISACTIVER0, 4, VGIC_32_BIT, redist_iactiver0_read, 405 redist_isactiver0_write), 406 VGIC_REGISTER(GICR_ICACTIVER0, 4, VGIC_32_BIT, redist_iactiver0_read, 407 redist_icactiver0_write), 408 VGIC_REGISTER_RANGE(GICR_IPRIORITYR(0), GICR_IPRIORITYR(32), 4, 409 VGIC_32_BIT | VGIC_8_BIT, redist_ipriorityr_read, 410 redist_ipriorityr_write), 411 VGIC_REGISTER_RAZ_WI(GICR_ICFGR0, 4, VGIC_32_BIT), 412 VGIC_REGISTER(GICR_ICFGR1, 4, VGIC_32_BIT, redist_icfgr1_read, 413 redist_icfgr1_write), 414 VGIC_REGISTER_RAZ_WI(GICR_IGRPMODR0, 4, VGIC_32_BIT), 415 VGIC_REGISTER_RAZ_WI(GICR_NSACR, 4, VGIC_32_BIT), 416 }; 417 418 static struct vgic_v3_virt_features virt_features; 419 420 static struct vgic_v3_irq *vgic_v3_get_irq(struct hyp *, int, uint32_t); 421 static void vgic_v3_release_irq(struct vgic_v3_irq *); 422 423 /* TODO: Move to a common file */ 424 static int 425 mpidr_to_vcpu(struct hyp *hyp, uint64_t mpidr) 426 { 427 struct vm *vm; 428 struct hypctx *hypctx; 429 430 vm = hyp->vm; 431 for (int i = 0; i < vm_get_maxcpus(vm); i++) { 432 hypctx = hyp->ctx[i]; 433 if (hypctx != NULL && (hypctx->vmpidr_el2 & GICD_AFF) == mpidr) 434 return (i); 435 } 436 return (-1); 437 } 438 439 static void 440 vgic_v3_vminit(device_t dev, struct hyp *hyp) 441 { 442 struct vgic_v3 *vgic; 443 444 hyp->vgic = malloc(sizeof(*hyp->vgic), M_VGIC_V3, 445 M_WAITOK | M_ZERO); 446 vgic = hyp->vgic; 447 448 /* 449 * Configure the Distributor control register. The register resets to an 450 * architecturally UNKNOWN value, so we reset to 0 to disable all 451 * functionality controlled by the register. 452 * 453 * The exception is GICD_CTLR.DS, which is RA0/WI when the Distributor 454 * supports one security state (ARM GIC Architecture Specification for 455 * GICv3 and GICv4, p. 4-464) 456 */ 457 vgic->gicd_ctlr = 0; 458 459 mtx_init(&vgic->dist_mtx, "VGICv3 Distributor lock", NULL, 460 MTX_SPIN); 461 } 462 463 static void 464 vgic_v3_cpuinit(device_t dev, struct hypctx *hypctx) 465 { 466 struct vgic_v3_cpu *vgic_cpu; 467 struct vgic_v3_irq *irq; 468 int i, irqid; 469 470 hypctx->vgic_cpu = malloc(sizeof(*hypctx->vgic_cpu), 471 M_VGIC_V3, M_WAITOK | M_ZERO); 472 vgic_cpu = hypctx->vgic_cpu; 473 474 mtx_init(&vgic_cpu->lr_mtx, "VGICv3 ICH_LR_EL2 lock", NULL, MTX_SPIN); 475 476 /* Set the SGI and PPI state */ 477 for (irqid = 0; irqid < VGIC_PRV_I_NUM; irqid++) { 478 irq = &vgic_cpu->private_irqs[irqid]; 479 480 mtx_init(&irq->irq_spinmtx, "VGIC IRQ spinlock", NULL, 481 MTX_SPIN); 482 irq->irq = irqid; 483 irq->mpidr = hypctx->vmpidr_el2 & GICD_AFF; 484 irq->target_vcpu = vcpu_vcpuid(hypctx->vcpu); 485 MPASS(irq->target_vcpu >= 0); 486 487 if (irqid < VGIC_SGI_NUM) { 488 /* SGIs */ 489 irq->enabled = true; 490 irq->config = VGIC_CONFIG_EDGE; 491 } else { 492 /* PPIs */ 493 irq->config = VGIC_CONFIG_LEVEL; 494 } 495 irq->priority = 0; 496 } 497 498 /* 499 * Configure the Interrupt Controller Hyp Control Register. 500 * 501 * ICH_HCR_EL2_En: enable virtual CPU interface. 502 * 503 * Maintenance interrupts are disabled. 504 */ 505 hypctx->vgic_v3_regs.ich_hcr_el2 = ICH_HCR_EL2_En; 506 507 /* 508 * Configure the Interrupt Controller Virtual Machine Control Register. 509 * 510 * ICH_VMCR_EL2_VPMR: lowest priority mask for the VCPU interface 511 * ICH_VMCR_EL2_VBPR1_NO_PREEMPTION: disable interrupt preemption for 512 * Group 1 interrupts 513 * ICH_VMCR_EL2_VBPR0_NO_PREEMPTION: disable interrupt preemption for 514 * Group 0 interrupts 515 * ~ICH_VMCR_EL2_VEOIM: writes to EOI registers perform priority drop 516 * and interrupt deactivation. 517 * ICH_VMCR_EL2_VENG0: virtual Group 0 interrupts enabled. 518 * ICH_VMCR_EL2_VENG1: virtual Group 1 interrupts enabled. 519 */ 520 hypctx->vgic_v3_regs.ich_vmcr_el2 = 521 (virt_features.min_prio << ICH_VMCR_EL2_VPMR_SHIFT) | 522 ICH_VMCR_EL2_VBPR1_NO_PREEMPTION | ICH_VMCR_EL2_VBPR0_NO_PREEMPTION; 523 hypctx->vgic_v3_regs.ich_vmcr_el2 &= ~ICH_VMCR_EL2_VEOIM; 524 hypctx->vgic_v3_regs.ich_vmcr_el2 |= ICH_VMCR_EL2_VENG0 | 525 ICH_VMCR_EL2_VENG1; 526 527 hypctx->vgic_v3_regs.ich_lr_num = virt_features.ich_lr_num; 528 for (i = 0; i < hypctx->vgic_v3_regs.ich_lr_num; i++) 529 hypctx->vgic_v3_regs.ich_lr_el2[i] = 0UL; 530 vgic_cpu->ich_lr_used = 0; 531 TAILQ_INIT(&vgic_cpu->irq_act_pend); 532 533 hypctx->vgic_v3_regs.ich_apr_num = virt_features.ich_apr_num; 534 } 535 536 static void 537 vgic_v3_cpucleanup(device_t dev, struct hypctx *hypctx) 538 { 539 struct vgic_v3_cpu *vgic_cpu; 540 struct vgic_v3_irq *irq; 541 int irqid; 542 543 vgic_cpu = hypctx->vgic_cpu; 544 for (irqid = 0; irqid < VGIC_PRV_I_NUM; irqid++) { 545 irq = &vgic_cpu->private_irqs[irqid]; 546 mtx_destroy(&irq->irq_spinmtx); 547 } 548 549 mtx_destroy(&vgic_cpu->lr_mtx); 550 free(hypctx->vgic_cpu, M_VGIC_V3); 551 } 552 553 static void 554 vgic_v3_vmcleanup(device_t dev, struct hyp *hyp) 555 { 556 mtx_destroy(&hyp->vgic->dist_mtx); 557 free(hyp->vgic, M_VGIC_V3); 558 } 559 560 static int 561 vgic_v3_max_cpu_count(device_t dev, struct hyp *hyp) 562 { 563 struct vgic_v3 *vgic; 564 size_t count; 565 int16_t max_count; 566 567 vgic = hyp->vgic; 568 max_count = vm_get_maxcpus(hyp->vm); 569 570 /* No registers, assume the maximum CPUs */ 571 if (vgic->redist_start == 0 && vgic->redist_end == 0) 572 return (max_count); 573 574 count = (vgic->redist_end - vgic->redist_start) / 575 (GICR_RD_BASE_SIZE + GICR_SGI_BASE_SIZE); 576 577 /* 578 * max_count is smaller than INT_MAX so will also limit count 579 * to a positive integer value. 580 */ 581 if (count > max_count) 582 return (max_count); 583 584 return (count); 585 } 586 587 static bool 588 vgic_v3_irq_pending(struct vgic_v3_irq *irq) 589 { 590 if ((irq->config & VGIC_CONFIG_MASK) == VGIC_CONFIG_LEVEL) { 591 return (irq->pending || irq->level); 592 } else { 593 return (irq->pending); 594 } 595 } 596 597 static bool 598 vgic_v3_queue_irq(struct hyp *hyp, struct vgic_v3_cpu *vgic_cpu, 599 int vcpuid, struct vgic_v3_irq *irq) 600 { 601 MPASS(vcpuid >= 0); 602 MPASS(vcpuid < vm_get_maxcpus(hyp->vm)); 603 604 mtx_assert(&vgic_cpu->lr_mtx, MA_OWNED); 605 mtx_assert(&irq->irq_spinmtx, MA_OWNED); 606 607 /* No need to queue the IRQ */ 608 if (!irq->level && !irq->pending) 609 return (false); 610 611 if (!irq->on_aplist) { 612 irq->on_aplist = true; 613 TAILQ_INSERT_TAIL(&vgic_cpu->irq_act_pend, irq, act_pend_list); 614 } 615 return (true); 616 } 617 618 static uint64_t 619 gic_reg_value_64(uint64_t field, uint64_t val, u_int offset, u_int size) 620 { 621 uint32_t mask; 622 623 if (offset != 0 || size != 8) { 624 mask = ((1ul << (size * 8)) - 1) << (offset * 8); 625 /* Shift the new bits to the correct place */ 626 val <<= (offset * 8); 627 /* Keep only the interesting bits */ 628 val &= mask; 629 /* Add the bits we are keeping from the old value */ 630 val |= field & ~mask; 631 } 632 633 return (val); 634 } 635 636 static void 637 gic_pidr2_read(struct hypctx *hypctx, u_int reg, uint64_t *rval, 638 void *arg) 639 { 640 *rval = GICR_PIDR2_ARCH_GICv3 << GICR_PIDR2_ARCH_SHIFT; 641 } 642 643 /* Common read-only/write-ignored helpers */ 644 static void 645 gic_zero_read(struct hypctx *hypctx, u_int reg, uint64_t *rval, 646 void *arg) 647 { 648 *rval = 0; 649 } 650 651 static void 652 gic_ignore_write(struct hypctx *hypctx, u_int reg, u_int offset, u_int size, 653 uint64_t wval, void *arg) 654 { 655 /* Nothing to do */ 656 } 657 658 static uint64_t 659 read_enabler(struct hypctx *hypctx, int n) 660 { 661 struct vgic_v3_irq *irq; 662 uint64_t ret; 663 uint32_t irq_base; 664 int i; 665 666 ret = 0; 667 irq_base = n * 32; 668 for (i = 0; i < 32; i++) { 669 irq = vgic_v3_get_irq(hypctx->hyp, vcpu_vcpuid(hypctx->vcpu), 670 irq_base + i); 671 if (irq == NULL) 672 continue; 673 674 if (!irq->enabled) 675 ret |= 1u << i; 676 vgic_v3_release_irq(irq); 677 } 678 679 return (ret); 680 } 681 682 static void 683 write_enabler(struct hypctx *hypctx,int n, bool set, uint64_t val) 684 { 685 struct vgic_v3_irq *irq; 686 uint32_t irq_base; 687 int i; 688 689 irq_base = n * 32; 690 for (i = 0; i < 32; i++) { 691 /* We only change interrupts when the appropriate bit is set */ 692 if ((val & (1u << i)) == 0) 693 continue; 694 695 /* Find the interrupt this bit represents */ 696 irq = vgic_v3_get_irq(hypctx->hyp, vcpu_vcpuid(hypctx->vcpu), 697 irq_base + i); 698 if (irq == NULL) 699 continue; 700 701 irq->enabled = set; 702 vgic_v3_release_irq(irq); 703 } 704 } 705 706 static uint64_t 707 read_pendr(struct hypctx *hypctx, int n) 708 { 709 struct vgic_v3_irq *irq; 710 uint64_t ret; 711 uint32_t irq_base; 712 int i; 713 714 ret = 0; 715 irq_base = n * 32; 716 for (i = 0; i < 32; i++) { 717 irq = vgic_v3_get_irq(hypctx->hyp, vcpu_vcpuid(hypctx->vcpu), 718 irq_base + i); 719 if (irq == NULL) 720 continue; 721 722 if (vgic_v3_irq_pending(irq)) 723 ret |= 1u << i; 724 vgic_v3_release_irq(irq); 725 } 726 727 return (ret); 728 } 729 730 static uint64_t 731 write_pendr(struct hypctx *hypctx, int n, bool set, uint64_t val) 732 { 733 struct vgic_v3_cpu *vgic_cpu; 734 struct vgic_v3_irq *irq; 735 struct hyp *hyp; 736 struct hypctx *target_hypctx; 737 uint64_t ret; 738 uint32_t irq_base; 739 int target_vcpu, i; 740 bool notify; 741 742 hyp = hypctx->hyp; 743 ret = 0; 744 irq_base = n * 32; 745 for (i = 0; i < 32; i++) { 746 /* We only change interrupts when the appropriate bit is set */ 747 if ((val & (1u << i)) == 0) 748 continue; 749 750 irq = vgic_v3_get_irq(hypctx->hyp, vcpu_vcpuid(hypctx->vcpu), 751 irq_base + i); 752 if (irq == NULL) 753 continue; 754 755 notify = false; 756 target_vcpu = irq->target_vcpu; 757 if (target_vcpu < 0) 758 goto next_irq; 759 target_hypctx = hyp->ctx[target_vcpu]; 760 if (target_hypctx == NULL) 761 goto next_irq; 762 vgic_cpu = target_hypctx->vgic_cpu; 763 764 if (!set) { 765 /* pending -> not pending */ 766 irq->pending = false; 767 } else { 768 irq->pending = true; 769 mtx_lock_spin(&vgic_cpu->lr_mtx); 770 notify = vgic_v3_queue_irq(hyp, vgic_cpu, target_vcpu, 771 irq); 772 mtx_unlock_spin(&vgic_cpu->lr_mtx); 773 } 774 next_irq: 775 vgic_v3_release_irq(irq); 776 777 if (notify) 778 vcpu_notify_event(vm_vcpu(hyp->vm, target_vcpu)); 779 } 780 781 return (ret); 782 } 783 784 static uint64_t 785 read_activer(struct hypctx *hypctx, int n) 786 { 787 struct vgic_v3_irq *irq; 788 uint64_t ret; 789 uint32_t irq_base; 790 int i; 791 792 ret = 0; 793 irq_base = n * 32; 794 for (i = 0; i < 32; i++) { 795 irq = vgic_v3_get_irq(hypctx->hyp, vcpu_vcpuid(hypctx->vcpu), 796 irq_base + i); 797 if (irq == NULL) 798 continue; 799 800 if (irq->active) 801 ret |= 1u << i; 802 vgic_v3_release_irq(irq); 803 } 804 805 return (ret); 806 } 807 808 static void 809 write_activer(struct hypctx *hypctx, u_int n, bool set, uint64_t val) 810 { 811 struct vgic_v3_cpu *vgic_cpu; 812 struct vgic_v3_irq *irq; 813 struct hyp *hyp; 814 struct hypctx *target_hypctx; 815 uint32_t irq_base; 816 int target_vcpu, i; 817 bool notify; 818 819 hyp = hypctx->hyp; 820 irq_base = n * 32; 821 for (i = 0; i < 32; i++) { 822 /* We only change interrupts when the appropriate bit is set */ 823 if ((val & (1u << i)) == 0) 824 continue; 825 826 irq = vgic_v3_get_irq(hypctx->hyp, vcpu_vcpuid(hypctx->vcpu), 827 irq_base + i); 828 if (irq == NULL) 829 continue; 830 831 notify = false; 832 target_vcpu = irq->target_vcpu; 833 if (target_vcpu < 0) 834 goto next_irq; 835 target_hypctx = hyp->ctx[target_vcpu]; 836 if (target_hypctx == NULL) 837 goto next_irq; 838 vgic_cpu = target_hypctx->vgic_cpu; 839 840 if (!set) { 841 /* active -> not active */ 842 irq->active = false; 843 } else { 844 /* not active -> active */ 845 irq->active = true; 846 mtx_lock_spin(&vgic_cpu->lr_mtx); 847 notify = vgic_v3_queue_irq(hyp, vgic_cpu, target_vcpu, 848 irq); 849 mtx_unlock_spin(&vgic_cpu->lr_mtx); 850 } 851 next_irq: 852 vgic_v3_release_irq(irq); 853 854 if (notify) 855 vcpu_notify_event(vm_vcpu(hyp->vm, target_vcpu)); 856 } 857 } 858 859 static uint64_t 860 read_priorityr(struct hypctx *hypctx, int n) 861 { 862 struct vgic_v3_irq *irq; 863 uint64_t ret; 864 uint32_t irq_base; 865 int i; 866 867 ret = 0; 868 irq_base = n * 4; 869 for (i = 0; i < 4; i++) { 870 irq = vgic_v3_get_irq(hypctx->hyp, vcpu_vcpuid(hypctx->vcpu), 871 irq_base + i); 872 if (irq == NULL) 873 continue; 874 875 ret |= ((uint64_t)irq->priority) << (i * 8); 876 vgic_v3_release_irq(irq); 877 } 878 879 return (ret); 880 } 881 882 static void 883 write_priorityr(struct hypctx *hypctx, u_int irq_base, u_int size, uint64_t val) 884 { 885 struct vgic_v3_irq *irq; 886 int i; 887 888 for (i = 0; i < size; i++) { 889 irq = vgic_v3_get_irq(hypctx->hyp, vcpu_vcpuid(hypctx->vcpu), 890 irq_base + i); 891 if (irq == NULL) 892 continue; 893 894 /* Set the priority. We support 32 priority steps (5 bits) */ 895 irq->priority = (val >> (i * 8)) & 0xf8; 896 vgic_v3_release_irq(irq); 897 } 898 } 899 900 static uint64_t 901 read_config(struct hypctx *hypctx, int n) 902 { 903 struct vgic_v3_irq *irq; 904 uint64_t ret; 905 uint32_t irq_base; 906 int i; 907 908 ret = 0; 909 irq_base = n * 16; 910 for (i = 0; i < 16; i++) { 911 irq = vgic_v3_get_irq(hypctx->hyp, vcpu_vcpuid(hypctx->vcpu), 912 irq_base + i); 913 if (irq == NULL) 914 continue; 915 916 ret |= ((uint64_t)irq->config) << (i * 2); 917 vgic_v3_release_irq(irq); 918 } 919 920 return (ret); 921 } 922 923 static void 924 write_config(struct hypctx *hypctx, int n, uint64_t val) 925 { 926 struct vgic_v3_irq *irq; 927 uint32_t irq_base; 928 int i; 929 930 irq_base = n * 16; 931 for (i = 0; i < 16; i++) { 932 /* 933 * The config can't be changed for SGIs and PPIs. SGIs have 934 * an edge-triggered behaviour, and the register is 935 * implementation defined to be read-only for PPIs. 936 */ 937 if (irq_base + i < VGIC_PRV_I_NUM) 938 continue; 939 940 irq = vgic_v3_get_irq(hypctx->hyp, vcpu_vcpuid(hypctx->vcpu), 941 irq_base + i); 942 if (irq == NULL) 943 continue; 944 945 /* Bit 0 is RES0 */ 946 irq->config = (val >> (i * 2)) & VGIC_CONFIG_MASK; 947 vgic_v3_release_irq(irq); 948 } 949 } 950 951 static uint64_t 952 read_route(struct hypctx *hypctx, int n) 953 { 954 struct vgic_v3_irq *irq; 955 uint64_t mpidr; 956 957 irq = vgic_v3_get_irq(hypctx->hyp, vcpu_vcpuid(hypctx->vcpu), n); 958 if (irq == NULL) 959 return (0); 960 961 mpidr = irq->mpidr; 962 vgic_v3_release_irq(irq); 963 964 return (mpidr); 965 } 966 967 static void 968 write_route(struct hypctx *hypctx, int n, uint64_t val, u_int offset, 969 u_int size) 970 { 971 struct vgic_v3_irq *irq; 972 973 irq = vgic_v3_get_irq(hypctx->hyp, vcpu_vcpuid(hypctx->vcpu), n); 974 if (irq == NULL) 975 return; 976 977 irq->mpidr = gic_reg_value_64(irq->mpidr, val, offset, size) & GICD_AFF; 978 irq->target_vcpu = mpidr_to_vcpu(hypctx->hyp, irq->mpidr); 979 /* 980 * If the interrupt is pending we can either use the old mpidr, or 981 * the new mpidr. To simplify this code we use the old value so we 982 * don't need to move the interrupt until the next time it is 983 * moved to the pending state. 984 */ 985 vgic_v3_release_irq(irq); 986 } 987 988 /* 989 * Distributor register handlers. 990 */ 991 /* GICD_CTLR */ 992 static void 993 dist_ctlr_read(struct hypctx *hypctx, u_int reg, uint64_t *rval, 994 void *arg) 995 { 996 struct hyp *hyp; 997 struct vgic_v3 *vgic; 998 999 hyp = hypctx->hyp; 1000 vgic = hyp->vgic; 1001 1002 mtx_lock_spin(&vgic->dist_mtx); 1003 *rval = vgic->gicd_ctlr; 1004 mtx_unlock_spin(&vgic->dist_mtx); 1005 1006 /* Writes are never pending */ 1007 *rval &= ~GICD_CTLR_RWP; 1008 } 1009 1010 static void 1011 dist_ctlr_write(struct hypctx *hypctx, u_int reg, u_int offset, u_int size, 1012 uint64_t wval, void *arg) 1013 { 1014 struct vgic_v3 *vgic; 1015 1016 MPASS(offset == 0); 1017 MPASS(size == 4); 1018 vgic = hypctx->hyp->vgic; 1019 1020 /* 1021 * GICv2 backwards compatibility is not implemented so 1022 * ARE_NS is RAO/WI. This means EnableGrp1 is RES0. 1023 * 1024 * EnableGrp1A is supported, and RWP is read-only. 1025 * 1026 * All other bits are RES0 from non-secure mode as we 1027 * implement as if we are in a system with two security 1028 * states. 1029 */ 1030 wval &= GICD_CTLR_G1A; 1031 wval |= GICD_CTLR_ARE_NS; 1032 mtx_lock_spin(&vgic->dist_mtx); 1033 vgic->gicd_ctlr = wval; 1034 /* TODO: Wake any vcpus that have interrupts pending */ 1035 mtx_unlock_spin(&vgic->dist_mtx); 1036 } 1037 1038 /* GICD_TYPER */ 1039 static void 1040 dist_typer_read(struct hypctx *hypctx, u_int reg, uint64_t *rval, 1041 void *arg) 1042 { 1043 uint32_t typer; 1044 1045 typer = (10 - 1) << GICD_TYPER_IDBITS_SHIFT; 1046 typer |= GICD_TYPER_MBIS; 1047 /* ITLinesNumber: */ 1048 typer |= howmany(VGIC_NIRQS + 1, 32) - 1; 1049 1050 *rval = typer; 1051 } 1052 1053 /* GICD_IIDR */ 1054 static void 1055 dist_iidr_read(struct hypctx *hypctx, u_int reg, uint64_t *rval, void *arg) 1056 { 1057 *rval = VGIC_IIDR; 1058 } 1059 1060 /* GICD_SETSPI_NSR & GICD_CLRSPI_NSR */ 1061 static void 1062 dist_setclrspi_nsr_write(struct hypctx *hypctx, u_int reg, u_int offset, 1063 u_int size, uint64_t wval, void *arg) 1064 { 1065 uint32_t irqid; 1066 1067 MPASS(offset == 0); 1068 MPASS(size == 4); 1069 irqid = wval & GICD_SPI_INTID_MASK; 1070 INJECT_IRQ(hypctx->hyp, vcpu_vcpuid(hypctx->vcpu), irqid, 1071 reg == GICD_SETSPI_NSR); 1072 } 1073 1074 /* GICD_ISENABLER */ 1075 static void 1076 dist_isenabler_read(struct hypctx *hypctx, u_int reg, uint64_t *rval, void *arg) 1077 { 1078 int n; 1079 1080 n = (reg - GICD_ISENABLER(0)) / 4; 1081 /* GICD_ISENABLER0 is RAZ/WI so handled separately */ 1082 MPASS(n > 0); 1083 *rval = read_enabler(hypctx, n); 1084 } 1085 1086 static void 1087 dist_isenabler_write(struct hypctx *hypctx, u_int reg, u_int offset, u_int size, 1088 uint64_t wval, void *arg) 1089 { 1090 int n; 1091 1092 MPASS(offset == 0); 1093 MPASS(size == 4); 1094 n = (reg - GICD_ISENABLER(0)) / 4; 1095 /* GICD_ISENABLER0 is RAZ/WI so handled separately */ 1096 MPASS(n > 0); 1097 write_enabler(hypctx, n, true, wval); 1098 } 1099 1100 /* GICD_ICENABLER */ 1101 static void 1102 dist_icenabler_read(struct hypctx *hypctx, u_int reg, uint64_t *rval, void *arg) 1103 { 1104 int n; 1105 1106 n = (reg - GICD_ICENABLER(0)) / 4; 1107 /* GICD_ICENABLER0 is RAZ/WI so handled separately */ 1108 MPASS(n > 0); 1109 *rval = read_enabler(hypctx, n); 1110 } 1111 1112 static void 1113 dist_icenabler_write(struct hypctx *hypctx, u_int reg, u_int offset, u_int size, 1114 uint64_t wval, void *arg) 1115 { 1116 int n; 1117 1118 MPASS(offset == 0); 1119 MPASS(size == 4); 1120 n = (reg - GICD_ISENABLER(0)) / 4; 1121 /* GICD_ICENABLER0 is RAZ/WI so handled separately */ 1122 MPASS(n > 0); 1123 write_enabler(hypctx, n, false, wval); 1124 } 1125 1126 /* GICD_ISPENDR */ 1127 static void 1128 dist_ispendr_read(struct hypctx *hypctx, u_int reg, uint64_t *rval, void *arg) 1129 { 1130 int n; 1131 1132 n = (reg - GICD_ISPENDR(0)) / 4; 1133 /* GICD_ISPENDR0 is RAZ/WI so handled separately */ 1134 MPASS(n > 0); 1135 *rval = read_pendr(hypctx, n); 1136 } 1137 1138 static void 1139 dist_ispendr_write(struct hypctx *hypctx, u_int reg, u_int offset, u_int size, 1140 uint64_t wval, void *arg) 1141 { 1142 int n; 1143 1144 MPASS(offset == 0); 1145 MPASS(size == 4); 1146 n = (reg - GICD_ISPENDR(0)) / 4; 1147 /* GICD_ISPENDR0 is RAZ/WI so handled separately */ 1148 MPASS(n > 0); 1149 write_pendr(hypctx, n, true, wval); 1150 } 1151 1152 /* GICD_ICPENDR */ 1153 static void 1154 dist_icpendr_read(struct hypctx *hypctx, u_int reg, uint64_t *rval, void *arg) 1155 { 1156 int n; 1157 1158 n = (reg - GICD_ICPENDR(0)) / 4; 1159 /* GICD_ICPENDR0 is RAZ/WI so handled separately */ 1160 MPASS(n > 0); 1161 *rval = read_pendr(hypctx, n); 1162 } 1163 1164 static void 1165 dist_icpendr_write(struct hypctx *hypctx, u_int reg, u_int offset, u_int size, 1166 uint64_t wval, void *arg) 1167 { 1168 int n; 1169 1170 MPASS(offset == 0); 1171 MPASS(size == 4); 1172 n = (reg - GICD_ICPENDR(0)) / 4; 1173 /* GICD_ICPENDR0 is RAZ/WI so handled separately */ 1174 MPASS(n > 0); 1175 write_pendr(hypctx, n, false, wval); 1176 } 1177 1178 /* GICD_ISACTIVER */ 1179 /* Affinity routing is enabled so isactiver0 is RAZ/WI */ 1180 static void 1181 dist_isactiver_read(struct hypctx *hypctx, u_int reg, uint64_t *rval, void *arg) 1182 { 1183 int n; 1184 1185 n = (reg - GICD_ISACTIVER(0)) / 4; 1186 /* GICD_ISACTIVER0 is RAZ/WI so handled separately */ 1187 MPASS(n > 0); 1188 *rval = read_activer(hypctx, n); 1189 } 1190 1191 static void 1192 dist_isactiver_write(struct hypctx *hypctx, u_int reg, u_int offset, u_int size, 1193 uint64_t wval, void *arg) 1194 { 1195 int n; 1196 1197 MPASS(offset == 0); 1198 MPASS(size == 4); 1199 n = (reg - GICD_ISACTIVER(0)) / 4; 1200 /* GICD_ISACTIVE0 is RAZ/WI so handled separately */ 1201 MPASS(n > 0); 1202 write_activer(hypctx, n, true, wval); 1203 } 1204 1205 /* GICD_ICACTIVER */ 1206 static void 1207 dist_icactiver_read(struct hypctx *hypctx, u_int reg, uint64_t *rval, 1208 void *arg) 1209 { 1210 int n; 1211 1212 n = (reg - GICD_ICACTIVER(0)) / 4; 1213 /* GICD_ICACTIVE0 is RAZ/WI so handled separately */ 1214 MPASS(n > 0); 1215 *rval = read_activer(hypctx, n); 1216 } 1217 1218 static void 1219 dist_icactiver_write(struct hypctx *hypctx, u_int reg, u_int offset, u_int size, 1220 uint64_t wval, void *arg) 1221 { 1222 int n; 1223 1224 MPASS(offset == 0); 1225 MPASS(size == 4); 1226 n = (reg - GICD_ICACTIVER(0)) / 4; 1227 /* GICD_ICACTIVE0 is RAZ/WI so handled separately */ 1228 MPASS(n > 0); 1229 write_activer(hypctx, n, false, wval); 1230 } 1231 1232 /* GICD_IPRIORITYR */ 1233 /* Affinity routing is enabled so ipriorityr0-7 is RAZ/WI */ 1234 static void 1235 dist_ipriorityr_read(struct hypctx *hypctx, u_int reg, uint64_t *rval, 1236 void *arg) 1237 { 1238 int n; 1239 1240 n = (reg - GICD_IPRIORITYR(0)) / 4; 1241 /* GICD_IPRIORITY0-7 is RAZ/WI so handled separately */ 1242 MPASS(n > 7); 1243 *rval = read_priorityr(hypctx, n); 1244 } 1245 1246 static void 1247 dist_ipriorityr_write(struct hypctx *hypctx, u_int reg, u_int offset, 1248 u_int size, uint64_t wval, void *arg) 1249 { 1250 u_int irq_base; 1251 1252 irq_base = (reg - GICD_IPRIORITYR(0)) + offset; 1253 /* GICD_IPRIORITY0-7 is RAZ/WI so handled separately */ 1254 MPASS(irq_base > 31); 1255 write_priorityr(hypctx, irq_base, size, wval); 1256 } 1257 1258 /* GICD_ICFGR */ 1259 static void 1260 dist_icfgr_read(struct hypctx *hypctx, u_int reg, uint64_t *rval, void *arg) 1261 { 1262 int n; 1263 1264 n = (reg - GICD_ICFGR(0)) / 4; 1265 /* GICD_ICFGR0-1 are RAZ/WI so handled separately */ 1266 MPASS(n > 1); 1267 *rval = read_config(hypctx, n); 1268 } 1269 1270 static void 1271 dist_icfgr_write(struct hypctx *hypctx, u_int reg, u_int offset, u_int size, 1272 uint64_t wval, void *arg) 1273 { 1274 int n; 1275 1276 MPASS(offset == 0); 1277 MPASS(size == 4); 1278 n = (reg - GICD_ICFGR(0)) / 4; 1279 /* GICD_ICFGR0-1 are RAZ/WI so handled separately */ 1280 MPASS(n > 1); 1281 write_config(hypctx, n, wval); 1282 } 1283 1284 /* GICD_IROUTER */ 1285 static void 1286 dist_irouter_read(struct hypctx *hypctx, u_int reg, uint64_t *rval, void *arg) 1287 { 1288 int n; 1289 1290 n = (reg - GICD_IROUTER(0)) / 8; 1291 /* GICD_IROUTER0-31 don't exist */ 1292 MPASS(n > 31); 1293 *rval = read_route(hypctx, n); 1294 } 1295 1296 static void 1297 dist_irouter_write(struct hypctx *hypctx, u_int reg, u_int offset, u_int size, 1298 uint64_t wval, void *arg) 1299 { 1300 int n; 1301 1302 n = (reg - GICD_IROUTER(0)) / 8; 1303 /* GICD_IROUTER0-31 don't exist */ 1304 MPASS(n > 31); 1305 write_route(hypctx, n, wval, offset, size); 1306 } 1307 1308 static bool 1309 vgic_register_read(struct hypctx *hypctx, struct vgic_register *reg_list, 1310 u_int reg_list_size, u_int reg, u_int size, uint64_t *rval, void *arg) 1311 { 1312 u_int i, offset; 1313 1314 for (i = 0; i < reg_list_size; i++) { 1315 if (reg_list[i].start <= reg && reg_list[i].end >= reg + size) { 1316 offset = reg & (reg_list[i].size - 1); 1317 reg -= offset; 1318 if ((reg_list[i].flags & size) != 0) { 1319 reg_list[i].read(hypctx, reg, rval, NULL); 1320 1321 /* Move the bits into the correct place */ 1322 *rval >>= (offset * 8); 1323 if (size < 8) { 1324 *rval &= (1ul << (size * 8)) - 1; 1325 } 1326 } else { 1327 /* 1328 * The access is an invalid size. Section 1329 * 12.1.3 "GIC memory-mapped register access" 1330 * of the GICv3 and GICv4 spec issue H 1331 * (IHI0069) lists the options. For a read 1332 * the controller returns unknown data, in 1333 * this case it is zero. 1334 */ 1335 *rval = 0; 1336 } 1337 return (true); 1338 } 1339 } 1340 return (false); 1341 } 1342 1343 static bool 1344 vgic_register_write(struct hypctx *hypctx, struct vgic_register *reg_list, 1345 u_int reg_list_size, u_int reg, u_int size, uint64_t wval, void *arg) 1346 { 1347 u_int i, offset; 1348 1349 for (i = 0; i < reg_list_size; i++) { 1350 if (reg_list[i].start <= reg && reg_list[i].end >= reg + size) { 1351 offset = reg & (reg_list[i].size - 1); 1352 reg -= offset; 1353 if ((reg_list[i].flags & size) != 0) { 1354 reg_list[i].write(hypctx, reg, offset, 1355 size, wval, NULL); 1356 } else { 1357 /* 1358 * See the comment in vgic_register_read. 1359 * For writes the controller ignores the 1360 * operation. 1361 */ 1362 } 1363 return (true); 1364 } 1365 } 1366 return (false); 1367 } 1368 1369 static int 1370 dist_read(struct vcpu *vcpu, uint64_t fault_ipa, uint64_t *rval, 1371 int size, void *arg) 1372 { 1373 struct hyp *hyp; 1374 struct hypctx *hypctx; 1375 struct vgic_v3 *vgic; 1376 uint64_t reg; 1377 1378 hypctx = vcpu_get_cookie(vcpu); 1379 hyp = hypctx->hyp; 1380 vgic = hyp->vgic; 1381 1382 /* Check the register is one of ours and is the correct size */ 1383 if (fault_ipa < vgic->dist_start || fault_ipa + size > vgic->dist_end) { 1384 return (EINVAL); 1385 } 1386 1387 reg = fault_ipa - vgic->dist_start; 1388 /* 1389 * As described in vgic_register_read an access with an invalid 1390 * alignment is read with an unknown value 1391 */ 1392 if ((reg & (size - 1)) != 0) { 1393 *rval = 0; 1394 return (0); 1395 } 1396 1397 if (vgic_register_read(hypctx, dist_registers, nitems(dist_registers), 1398 reg, size, rval, NULL)) 1399 return (0); 1400 1401 /* Reserved register addresses are RES0 so we can hardware it to 0 */ 1402 *rval = 0; 1403 1404 return (0); 1405 } 1406 1407 static int 1408 dist_write(struct vcpu *vcpu, uint64_t fault_ipa, uint64_t wval, 1409 int size, void *arg) 1410 { 1411 struct hyp *hyp; 1412 struct hypctx *hypctx; 1413 struct vgic_v3 *vgic; 1414 uint64_t reg; 1415 1416 hypctx = vcpu_get_cookie(vcpu); 1417 hyp = hypctx->hyp; 1418 vgic = hyp->vgic; 1419 1420 /* Check the register is one of ours and is the correct size */ 1421 if (fault_ipa < vgic->dist_start || fault_ipa + size > vgic->dist_end) { 1422 return (EINVAL); 1423 } 1424 1425 reg = fault_ipa - vgic->dist_start; 1426 /* 1427 * As described in vgic_register_read an access with an invalid 1428 * alignment is write ignored. 1429 */ 1430 if ((reg & (size - 1)) != 0) 1431 return (0); 1432 1433 if (vgic_register_write(hypctx, dist_registers, nitems(dist_registers), 1434 reg, size, wval, NULL)) 1435 return (0); 1436 1437 /* Reserved register addresses are RES0 so we can ignore the write */ 1438 return (0); 1439 } 1440 1441 /* 1442 * Redistributor register handlers. 1443 * 1444 * RD_base: 1445 */ 1446 /* GICR_CTLR */ 1447 static void 1448 redist_ctlr_read(struct hypctx *hypctx, u_int reg, uint64_t *rval, void *arg) 1449 { 1450 /* LPIs not supported */ 1451 *rval = 0; 1452 } 1453 1454 /* GICR_IIDR */ 1455 static void 1456 redist_iidr_read(struct hypctx *hypctx, u_int reg, uint64_t *rval, void *arg) 1457 { 1458 *rval = VGIC_IIDR; 1459 } 1460 1461 /* GICR_TYPER */ 1462 static void 1463 redist_typer_read(struct hypctx *hypctx, u_int reg, uint64_t *rval, void *arg) 1464 { 1465 uint64_t aff, gicr_typer, vmpidr_el2; 1466 bool last_vcpu; 1467 1468 last_vcpu = false; 1469 if (vcpu_vcpuid(hypctx->vcpu) == (vgic_max_cpu_count(hypctx->hyp) - 1)) 1470 last_vcpu = true; 1471 1472 vmpidr_el2 = hypctx->vmpidr_el2; 1473 MPASS(vmpidr_el2 != 0); 1474 /* 1475 * Get affinity for the current CPU. The guest CPU affinity is taken 1476 * from VMPIDR_EL2. The Redistributor corresponding to this CPU is 1477 * the Redistributor with the same affinity from GICR_TYPER. 1478 */ 1479 aff = (CPU_AFF3(vmpidr_el2) << 24) | (CPU_AFF2(vmpidr_el2) << 16) | 1480 (CPU_AFF1(vmpidr_el2) << 8) | CPU_AFF0(vmpidr_el2); 1481 1482 /* Set up GICR_TYPER. */ 1483 gicr_typer = aff << GICR_TYPER_AFF_SHIFT; 1484 /* Set the vcpu as the processsor ID */ 1485 gicr_typer |= 1486 (uint64_t)vcpu_vcpuid(hypctx->vcpu) << GICR_TYPER_CPUNUM_SHIFT; 1487 1488 if (last_vcpu) 1489 /* Mark the last Redistributor */ 1490 gicr_typer |= GICR_TYPER_LAST; 1491 1492 *rval = gicr_typer; 1493 } 1494 1495 /* 1496 * SGI_base: 1497 */ 1498 /* GICR_ISENABLER0 */ 1499 static void 1500 redist_ienabler0_read(struct hypctx *hypctx, u_int reg, uint64_t *rval, 1501 void *arg) 1502 { 1503 *rval = read_enabler(hypctx, 0); 1504 } 1505 1506 static void 1507 redist_isenabler0_write(struct hypctx *hypctx, u_int reg, u_int offset, 1508 u_int size, uint64_t wval, void *arg) 1509 { 1510 MPASS(offset == 0); 1511 MPASS(size == 4); 1512 write_enabler(hypctx, 0, true, wval); 1513 } 1514 1515 /* GICR_ICENABLER0 */ 1516 static void 1517 redist_icenabler0_write(struct hypctx *hypctx, u_int reg, u_int offset, 1518 u_int size, uint64_t wval, void *arg) 1519 { 1520 MPASS(offset == 0); 1521 MPASS(size == 4); 1522 write_enabler(hypctx, 0, false, wval); 1523 } 1524 1525 /* GICR_ISPENDR0 */ 1526 static void 1527 redist_ipendr0_read(struct hypctx *hypctx, u_int reg, uint64_t *rval, 1528 void *arg) 1529 { 1530 *rval = read_pendr(hypctx, 0); 1531 } 1532 1533 static void 1534 redist_ispendr0_write(struct hypctx *hypctx, u_int reg, u_int offset, 1535 u_int size, uint64_t wval, void *arg) 1536 { 1537 MPASS(offset == 0); 1538 MPASS(size == 4); 1539 write_pendr(hypctx, 0, true, wval); 1540 } 1541 1542 /* GICR_ICPENDR0 */ 1543 static void 1544 redist_icpendr0_write(struct hypctx *hypctx, u_int reg, u_int offset, 1545 u_int size, uint64_t wval, void *arg) 1546 { 1547 MPASS(offset == 0); 1548 MPASS(size == 4); 1549 write_pendr(hypctx, 0, false, wval); 1550 } 1551 1552 /* GICR_ISACTIVER0 */ 1553 static void 1554 redist_iactiver0_read(struct hypctx *hypctx, u_int reg, uint64_t *rval, 1555 void *arg) 1556 { 1557 *rval = read_activer(hypctx, 0); 1558 } 1559 1560 static void 1561 redist_isactiver0_write(struct hypctx *hypctx, u_int reg, u_int offset, 1562 u_int size, uint64_t wval, void *arg) 1563 { 1564 write_activer(hypctx, 0, true, wval); 1565 } 1566 1567 /* GICR_ICACTIVER0 */ 1568 static void 1569 redist_icactiver0_write(struct hypctx *hypctx, u_int reg, u_int offset, 1570 u_int size, uint64_t wval, void *arg) 1571 { 1572 write_activer(hypctx, 0, false, wval); 1573 } 1574 1575 /* GICR_IPRIORITYR */ 1576 static void 1577 redist_ipriorityr_read(struct hypctx *hypctx, u_int reg, uint64_t *rval, 1578 void *arg) 1579 { 1580 int n; 1581 1582 n = (reg - GICR_IPRIORITYR(0)) / 4; 1583 *rval = read_priorityr(hypctx, n); 1584 } 1585 1586 static void 1587 redist_ipriorityr_write(struct hypctx *hypctx, u_int reg, u_int offset, 1588 u_int size, uint64_t wval, void *arg) 1589 { 1590 u_int irq_base; 1591 1592 irq_base = (reg - GICR_IPRIORITYR(0)) + offset; 1593 write_priorityr(hypctx, irq_base, size, wval); 1594 } 1595 1596 /* GICR_ICFGR1 */ 1597 static void 1598 redist_icfgr1_read(struct hypctx *hypctx, u_int reg, uint64_t *rval, void *arg) 1599 { 1600 *rval = read_config(hypctx, 1); 1601 } 1602 1603 static void 1604 redist_icfgr1_write(struct hypctx *hypctx, u_int reg, u_int offset, u_int size, 1605 uint64_t wval, void *arg) 1606 { 1607 MPASS(offset == 0); 1608 MPASS(size == 4); 1609 write_config(hypctx, 1, wval); 1610 } 1611 1612 static int 1613 redist_read(struct vcpu *vcpu, uint64_t fault_ipa, uint64_t *rval, 1614 int size, void *arg) 1615 { 1616 struct hyp *hyp; 1617 struct hypctx *hypctx, *target_hypctx; 1618 struct vgic_v3 *vgic; 1619 uint64_t reg; 1620 int vcpuid; 1621 1622 /* Find the current vcpu ctx to get the vgic struct */ 1623 hypctx = vcpu_get_cookie(vcpu); 1624 hyp = hypctx->hyp; 1625 vgic = hyp->vgic; 1626 1627 /* Check the register is one of ours and is the correct size */ 1628 if (fault_ipa < vgic->redist_start || 1629 fault_ipa + size > vgic->redist_end) { 1630 return (EINVAL); 1631 } 1632 1633 vcpuid = (fault_ipa - vgic->redist_start) / 1634 (GICR_RD_BASE_SIZE + GICR_SGI_BASE_SIZE); 1635 if (vcpuid >= vm_get_maxcpus(hyp->vm)) { 1636 /* 1637 * This should never happen, but lets be defensive so if it 1638 * does we don't panic a non-INVARIANTS kernel. 1639 */ 1640 #ifdef INVARIANTS 1641 panic("%s: Invalid vcpuid %d", __func__, vcpuid); 1642 #else 1643 *rval = 0; 1644 return (0); 1645 #endif 1646 } 1647 1648 /* Find the target vcpu ctx for the access */ 1649 target_hypctx = hyp->ctx[vcpuid]; 1650 if (target_hypctx == NULL) { 1651 /* 1652 * The CPU has not yet started. The redistributor and CPU are 1653 * in the same power domain. As such the redistributor will 1654 * also be powered down so any access will raise an external 1655 * abort. 1656 */ 1657 raise_data_insn_abort(hypctx, fault_ipa, true, 1658 ISS_DATA_DFSC_EXT); 1659 return (0); 1660 } 1661 1662 reg = (fault_ipa - vgic->redist_start) % 1663 (GICR_RD_BASE_SIZE + GICR_SGI_BASE_SIZE); 1664 1665 /* 1666 * As described in vgic_register_read an access with an invalid 1667 * alignment is read with an unknown value 1668 */ 1669 if ((reg & (size - 1)) != 0) { 1670 *rval = 0; 1671 return (0); 1672 } 1673 1674 if (reg < GICR_RD_BASE_SIZE) { 1675 if (vgic_register_read(target_hypctx, redist_rd_registers, 1676 nitems(redist_rd_registers), reg, size, rval, NULL)) 1677 return (0); 1678 } else if (reg < (GICR_SGI_BASE + GICR_SGI_BASE_SIZE)) { 1679 if (vgic_register_read(target_hypctx, redist_sgi_registers, 1680 nitems(redist_sgi_registers), reg - GICR_SGI_BASE, size, 1681 rval, NULL)) 1682 return (0); 1683 } 1684 1685 /* Reserved register addresses are RES0 so we can hardware it to 0 */ 1686 *rval = 0; 1687 return (0); 1688 } 1689 1690 static int 1691 redist_write(struct vcpu *vcpu, uint64_t fault_ipa, uint64_t wval, 1692 int size, void *arg) 1693 { 1694 struct hyp *hyp; 1695 struct hypctx *hypctx, *target_hypctx; 1696 struct vgic_v3 *vgic; 1697 uint64_t reg; 1698 int vcpuid; 1699 1700 /* Find the current vcpu ctx to get the vgic struct */ 1701 hypctx = vcpu_get_cookie(vcpu); 1702 hyp = hypctx->hyp; 1703 vgic = hyp->vgic; 1704 1705 /* Check the register is one of ours and is the correct size */ 1706 if (fault_ipa < vgic->redist_start || 1707 fault_ipa + size > vgic->redist_end) { 1708 return (EINVAL); 1709 } 1710 1711 vcpuid = (fault_ipa - vgic->redist_start) / 1712 (GICR_RD_BASE_SIZE + GICR_SGI_BASE_SIZE); 1713 if (vcpuid >= vm_get_maxcpus(hyp->vm)) { 1714 /* 1715 * This should never happen, but lets be defensive so if it 1716 * does we don't panic a non-INVARIANTS kernel. 1717 */ 1718 #ifdef INVARIANTS 1719 panic("%s: Invalid vcpuid %d", __func__, vcpuid); 1720 #else 1721 return (0); 1722 #endif 1723 } 1724 1725 /* Find the target vcpu ctx for the access */ 1726 target_hypctx = hyp->ctx[vcpuid]; 1727 if (target_hypctx == NULL) { 1728 /* 1729 * The CPU has not yet started. The redistributor and CPU are 1730 * in the same power domain. As such the redistributor will 1731 * also be powered down so any access will raise an external 1732 * abort. 1733 */ 1734 raise_data_insn_abort(hypctx, fault_ipa, true, 1735 ISS_DATA_DFSC_EXT); 1736 return (0); 1737 } 1738 1739 reg = (fault_ipa - vgic->redist_start) % 1740 (GICR_RD_BASE_SIZE + GICR_SGI_BASE_SIZE); 1741 1742 /* 1743 * As described in vgic_register_read an access with an invalid 1744 * alignment is write ignored. 1745 */ 1746 if ((reg & (size - 1)) != 0) 1747 return (0); 1748 1749 if (reg < GICR_RD_BASE_SIZE) { 1750 if (vgic_register_write(target_hypctx, redist_rd_registers, 1751 nitems(redist_rd_registers), reg, size, wval, NULL)) 1752 return (0); 1753 } else if (reg < (GICR_SGI_BASE + GICR_SGI_BASE_SIZE)) { 1754 if (vgic_register_write(target_hypctx, redist_sgi_registers, 1755 nitems(redist_sgi_registers), reg - GICR_SGI_BASE, size, 1756 wval, NULL)) 1757 return (0); 1758 } 1759 1760 /* Reserved register addresses are RES0 so we can ignore the write */ 1761 return (0); 1762 } 1763 1764 static int 1765 vgic_v3_icc_sgi1r_read(struct vcpu *vcpu, uint64_t *rval, void *arg) 1766 { 1767 /* 1768 * TODO: Inject an unknown exception. 1769 */ 1770 *rval = 0; 1771 return (0); 1772 } 1773 1774 static int 1775 vgic_v3_icc_sgi1r_write(struct vcpu *vcpu, uint64_t rval, void *arg) 1776 { 1777 struct vm *vm; 1778 struct hyp *hyp; 1779 cpuset_t active_cpus; 1780 uint64_t mpidr, aff1, aff2, aff3; 1781 uint32_t irqid; 1782 int cpus, cpu_off, target_vcpuid, vcpuid; 1783 1784 vm = vcpu_vm(vcpu); 1785 hyp = vm_get_cookie(vm); 1786 active_cpus = vm_active_cpus(vm); 1787 vcpuid = vcpu_vcpuid(vcpu); 1788 1789 irqid = ICC_SGI1R_EL1_SGIID_VAL(rval) >> ICC_SGI1R_EL1_SGIID_SHIFT; 1790 if ((rval & ICC_SGI1R_EL1_IRM) == 0) { 1791 /* Non-zero points at no vcpus */ 1792 if (ICC_SGI1R_EL1_RS_VAL(rval) != 0) 1793 return (0); 1794 1795 aff1 = ICC_SGI1R_EL1_AFF1_VAL(rval) >> ICC_SGI1R_EL1_AFF1_SHIFT; 1796 aff2 = ICC_SGI1R_EL1_AFF2_VAL(rval) >> ICC_SGI1R_EL1_AFF2_SHIFT; 1797 aff3 = ICC_SGI1R_EL1_AFF3_VAL(rval) >> ICC_SGI1R_EL1_AFF3_SHIFT; 1798 mpidr = aff3 << MPIDR_AFF3_SHIFT | 1799 aff2 << MPIDR_AFF2_SHIFT | aff1 << MPIDR_AFF1_SHIFT; 1800 1801 cpus = ICC_SGI1R_EL1_TL_VAL(rval) >> ICC_SGI1R_EL1_TL_SHIFT; 1802 cpu_off = 0; 1803 while (cpus > 0) { 1804 if (cpus & 1) { 1805 target_vcpuid = mpidr_to_vcpu(hyp, 1806 mpidr | (cpu_off << MPIDR_AFF0_SHIFT)); 1807 if (target_vcpuid >= 0 && 1808 CPU_ISSET(target_vcpuid, &active_cpus)) { 1809 INJECT_IRQ(hyp, target_vcpuid, irqid, 1810 true); 1811 } 1812 } 1813 cpu_off++; 1814 cpus >>= 1; 1815 } 1816 } else { 1817 /* Send an IPI to all CPUs other than the current CPU */ 1818 for (target_vcpuid = 0; target_vcpuid < vm_get_maxcpus(vm); 1819 target_vcpuid++) { 1820 if (CPU_ISSET(target_vcpuid, &active_cpus) && 1821 target_vcpuid != vcpuid) { 1822 INJECT_IRQ(hyp, target_vcpuid, irqid, true); 1823 } 1824 } 1825 } 1826 1827 return (0); 1828 } 1829 1830 static void 1831 vgic_v3_mmio_init(struct hyp *hyp) 1832 { 1833 struct vgic_v3 *vgic; 1834 struct vgic_v3_irq *irq; 1835 int i; 1836 1837 /* Allocate memory for the SPIs */ 1838 vgic = hyp->vgic; 1839 vgic->irqs = malloc((VGIC_NIRQS - VGIC_PRV_I_NUM) * 1840 sizeof(*vgic->irqs), M_VGIC_V3, M_WAITOK | M_ZERO); 1841 1842 for (i = 0; i < VGIC_NIRQS - VGIC_PRV_I_NUM; i++) { 1843 irq = &vgic->irqs[i]; 1844 1845 mtx_init(&irq->irq_spinmtx, "VGIC IRQ spinlock", NULL, 1846 MTX_SPIN); 1847 1848 irq->irq = i + VGIC_PRV_I_NUM; 1849 } 1850 } 1851 1852 static void 1853 vgic_v3_mmio_destroy(struct hyp *hyp) 1854 { 1855 struct vgic_v3 *vgic; 1856 struct vgic_v3_irq *irq; 1857 int i; 1858 1859 vgic = hyp->vgic; 1860 for (i = 0; i < VGIC_NIRQS - VGIC_PRV_I_NUM; i++) { 1861 irq = &vgic->irqs[i]; 1862 1863 mtx_destroy(&irq->irq_spinmtx); 1864 } 1865 1866 free(vgic->irqs, M_VGIC_V3); 1867 } 1868 1869 static int 1870 vgic_v3_attach_to_vm(device_t dev, struct hyp *hyp, struct vm_vgic_descr *descr) 1871 { 1872 struct vm *vm; 1873 struct vgic_v3 *vgic; 1874 size_t cpu_count; 1875 1876 if (descr->ver.version != 3) 1877 return (EINVAL); 1878 1879 /* 1880 * The register bases need to be 64k aligned 1881 * The redist register space is the RD + SGI size 1882 */ 1883 if (!__is_aligned(descr->v3_regs.dist_start, PAGE_SIZE_64K) || 1884 !__is_aligned(descr->v3_regs.redist_start, PAGE_SIZE_64K) || 1885 !__is_aligned(descr->v3_regs.redist_size, 1886 GICR_RD_BASE_SIZE + GICR_SGI_BASE_SIZE)) 1887 return (EINVAL); 1888 1889 /* The dist register space is 1 64k block */ 1890 if (descr->v3_regs.dist_size != PAGE_SIZE_64K) 1891 return (EINVAL); 1892 1893 vm = hyp->vm; 1894 1895 /* 1896 * Return an error if the redist space is too large for the maximum 1897 * number of CPUs we support. 1898 */ 1899 cpu_count = descr->v3_regs.redist_size / 1900 (GICR_RD_BASE_SIZE + GICR_SGI_BASE_SIZE); 1901 if (cpu_count > vm_get_maxcpus(vm)) 1902 return (EINVAL); 1903 1904 vgic = hyp->vgic; 1905 1906 /* Set the distributor address and size for trapping guest access. */ 1907 vgic->dist_start = descr->v3_regs.dist_start; 1908 vgic->dist_end = descr->v3_regs.dist_start + descr->v3_regs.dist_size; 1909 1910 vgic->redist_start = descr->v3_regs.redist_start; 1911 vgic->redist_end = descr->v3_regs.redist_start + 1912 descr->v3_regs.redist_size; 1913 1914 vm_register_inst_handler(vm, descr->v3_regs.dist_start, 1915 descr->v3_regs.dist_size, dist_read, dist_write); 1916 vm_register_inst_handler(vm, descr->v3_regs.redist_start, 1917 descr->v3_regs.redist_size, redist_read, redist_write); 1918 1919 vm_register_reg_handler(vm, ISS_MSR_REG(ICC_SGI1R_EL1), 1920 ISS_MSR_REG_MASK, vgic_v3_icc_sgi1r_read, vgic_v3_icc_sgi1r_write, 1921 NULL); 1922 1923 vgic_v3_mmio_init(hyp); 1924 1925 hyp->vgic_attached = true; 1926 1927 return (0); 1928 } 1929 1930 static void 1931 vgic_v3_detach_from_vm(device_t dev, struct hyp *hyp) 1932 { 1933 if (hyp->vgic_attached) { 1934 hyp->vgic_attached = false; 1935 vgic_v3_mmio_destroy(hyp); 1936 } 1937 } 1938 1939 static struct vgic_v3_irq * 1940 vgic_v3_get_irq(struct hyp *hyp, int vcpuid, uint32_t irqid) 1941 { 1942 struct vgic_v3_cpu *vgic_cpu; 1943 struct vgic_v3_irq *irq; 1944 struct hypctx *hypctx; 1945 1946 if (irqid < VGIC_PRV_I_NUM) { 1947 if (vcpuid < 0 || vcpuid >= vm_get_maxcpus(hyp->vm)) 1948 return (NULL); 1949 hypctx = hyp->ctx[vcpuid]; 1950 if (hypctx == NULL) 1951 return (NULL); 1952 vgic_cpu = hypctx->vgic_cpu; 1953 irq = &vgic_cpu->private_irqs[irqid]; 1954 } else if (irqid <= GIC_LAST_SPI) { 1955 irqid -= VGIC_PRV_I_NUM; 1956 if (irqid >= VGIC_NIRQS) 1957 return (NULL); 1958 irq = &hyp->vgic->irqs[irqid]; 1959 } else if (irqid < GIC_FIRST_LPI) { 1960 return (NULL); 1961 } else { 1962 /* No support for LPIs */ 1963 return (NULL); 1964 } 1965 1966 mtx_lock_spin(&irq->irq_spinmtx); 1967 return (irq); 1968 } 1969 1970 static void 1971 vgic_v3_release_irq(struct vgic_v3_irq *irq) 1972 { 1973 1974 mtx_unlock_spin(&irq->irq_spinmtx); 1975 } 1976 1977 static bool 1978 vgic_v3_has_pending_irq(device_t dev, struct hypctx *hypctx) 1979 { 1980 struct vgic_v3_cpu *vgic_cpu; 1981 bool empty; 1982 1983 vgic_cpu = hypctx->vgic_cpu; 1984 mtx_lock_spin(&vgic_cpu->lr_mtx); 1985 empty = TAILQ_EMPTY(&vgic_cpu->irq_act_pend); 1986 mtx_unlock_spin(&vgic_cpu->lr_mtx); 1987 1988 return (!empty); 1989 } 1990 1991 static bool 1992 vgic_v3_check_irq(struct vgic_v3_irq *irq, bool level) 1993 { 1994 /* 1995 * Only inject if: 1996 * - Level-triggered IRQ: level changes low -> high 1997 * - Edge-triggered IRQ: level is high 1998 */ 1999 switch (irq->config & VGIC_CONFIG_MASK) { 2000 case VGIC_CONFIG_LEVEL: 2001 return (level != irq->level); 2002 case VGIC_CONFIG_EDGE: 2003 return (level); 2004 default: 2005 break; 2006 } 2007 2008 return (false); 2009 } 2010 2011 static int 2012 vgic_v3_inject_irq(device_t dev, struct hyp *hyp, int vcpuid, uint32_t irqid, 2013 bool level) 2014 { 2015 struct vgic_v3_cpu *vgic_cpu; 2016 struct vgic_v3_irq *irq; 2017 struct hypctx *hypctx; 2018 int target_vcpu; 2019 bool notify; 2020 2021 if (!hyp->vgic_attached) 2022 return (ENODEV); 2023 2024 KASSERT(vcpuid == -1 || irqid < VGIC_PRV_I_NUM, 2025 ("%s: SPI/LPI with vcpuid set: irq %u vcpuid %u", __func__, irqid, 2026 vcpuid)); 2027 2028 irq = vgic_v3_get_irq(hyp, vcpuid, irqid); 2029 if (irq == NULL) { 2030 eprintf("Malformed IRQ %u.\n", irqid); 2031 return (EINVAL); 2032 } 2033 2034 target_vcpu = irq->target_vcpu; 2035 KASSERT(vcpuid == -1 || vcpuid == target_vcpu, 2036 ("%s: Interrupt %u has bad cpu affinity: vcpu %d target vcpu %d", 2037 __func__, irqid, vcpuid, target_vcpu)); 2038 KASSERT(target_vcpu >= 0 && target_vcpu < vm_get_maxcpus(hyp->vm), 2039 ("%s: Interrupt %u sent to invalid vcpu %d", __func__, irqid, 2040 target_vcpu)); 2041 2042 if (vcpuid == -1) 2043 vcpuid = target_vcpu; 2044 /* TODO: Check from 0 to vm->maxcpus */ 2045 if (vcpuid < 0 || vcpuid >= vm_get_maxcpus(hyp->vm)) { 2046 vgic_v3_release_irq(irq); 2047 return (EINVAL); 2048 } 2049 2050 hypctx = hyp->ctx[vcpuid]; 2051 if (hypctx == NULL) { 2052 vgic_v3_release_irq(irq); 2053 return (EINVAL); 2054 } 2055 2056 notify = false; 2057 vgic_cpu = hypctx->vgic_cpu; 2058 2059 mtx_lock_spin(&vgic_cpu->lr_mtx); 2060 2061 if (!vgic_v3_check_irq(irq, level)) { 2062 goto out; 2063 } 2064 2065 if ((irq->config & VGIC_CONFIG_MASK) == VGIC_CONFIG_LEVEL) 2066 irq->level = level; 2067 else /* VGIC_CONFIG_EDGE */ 2068 irq->pending = true; 2069 2070 notify = vgic_v3_queue_irq(hyp, vgic_cpu, vcpuid, irq); 2071 2072 out: 2073 mtx_unlock_spin(&vgic_cpu->lr_mtx); 2074 vgic_v3_release_irq(irq); 2075 2076 if (notify) 2077 vcpu_notify_event(vm_vcpu(hyp->vm, vcpuid)); 2078 2079 return (0); 2080 } 2081 2082 static int 2083 vgic_v3_inject_msi(device_t dev, struct hyp *hyp, uint64_t msg, uint64_t addr) 2084 { 2085 struct vgic_v3 *vgic; 2086 uint64_t reg; 2087 2088 vgic = hyp->vgic; 2089 2090 /* This is a 4 byte register */ 2091 if (addr < vgic->dist_start || addr + 4 > vgic->dist_end) { 2092 return (EINVAL); 2093 } 2094 2095 reg = addr - vgic->dist_start; 2096 if (reg != GICD_SETSPI_NSR) 2097 return (EINVAL); 2098 2099 return (INJECT_IRQ(hyp, -1, msg, true)); 2100 } 2101 2102 static void 2103 vgic_v3_flush_hwstate(device_t dev, struct hypctx *hypctx) 2104 { 2105 struct vgic_v3_cpu *vgic_cpu; 2106 struct vgic_v3_irq *irq; 2107 int i; 2108 2109 vgic_cpu = hypctx->vgic_cpu; 2110 2111 /* 2112 * All Distributor writes have been executed at this point, do not 2113 * protect Distributor reads with a mutex. 2114 * 2115 * This is callled with all interrupts disabled, so there is no need for 2116 * a List Register spinlock either. 2117 */ 2118 mtx_lock_spin(&vgic_cpu->lr_mtx); 2119 2120 hypctx->vgic_v3_regs.ich_hcr_el2 &= ~ICH_HCR_EL2_UIE; 2121 2122 /* Exit early if there are no buffered interrupts */ 2123 if (TAILQ_EMPTY(&vgic_cpu->irq_act_pend)) 2124 goto out; 2125 2126 KASSERT(vgic_cpu->ich_lr_used == 0, ("%s: Used LR count not zero %u", 2127 __func__, vgic_cpu->ich_lr_used)); 2128 2129 i = 0; 2130 hypctx->vgic_v3_regs.ich_elrsr_el2 = 2131 (1u << hypctx->vgic_v3_regs.ich_lr_num) - 1; 2132 TAILQ_FOREACH(irq, &vgic_cpu->irq_act_pend, act_pend_list) { 2133 /* No free list register, stop searching for IRQs */ 2134 if (i == hypctx->vgic_v3_regs.ich_lr_num) 2135 break; 2136 2137 if (!irq->enabled) 2138 continue; 2139 2140 hypctx->vgic_v3_regs.ich_lr_el2[i] = ICH_LR_EL2_GROUP1 | 2141 ((uint64_t)irq->priority << ICH_LR_EL2_PRIO_SHIFT) | 2142 irq->irq; 2143 2144 if (irq->active) { 2145 hypctx->vgic_v3_regs.ich_lr_el2[i] |= 2146 ICH_LR_EL2_STATE_ACTIVE; 2147 } 2148 2149 #ifdef notyet 2150 /* TODO: Check why this is needed */ 2151 if ((irq->config & _MASK) == LEVEL) 2152 hypctx->vgic_v3_regs.ich_lr_el2[i] |= ICH_LR_EL2_EOI; 2153 #endif 2154 2155 if (!irq->active && vgic_v3_irq_pending(irq)) { 2156 hypctx->vgic_v3_regs.ich_lr_el2[i] |= 2157 ICH_LR_EL2_STATE_PENDING; 2158 2159 /* 2160 * This IRQ is now pending on the guest. Allow for 2161 * another edge that could cause the interrupt to 2162 * be raised again. 2163 */ 2164 if ((irq->config & VGIC_CONFIG_MASK) == 2165 VGIC_CONFIG_EDGE) { 2166 irq->pending = false; 2167 } 2168 } 2169 2170 i++; 2171 } 2172 vgic_cpu->ich_lr_used = i; 2173 2174 out: 2175 mtx_unlock_spin(&vgic_cpu->lr_mtx); 2176 } 2177 2178 static void 2179 vgic_v3_sync_hwstate(device_t dev, struct hypctx *hypctx) 2180 { 2181 struct vgic_v3_cpu *vgic_cpu; 2182 struct vgic_v3_irq *irq; 2183 uint64_t lr; 2184 int i; 2185 2186 vgic_cpu = hypctx->vgic_cpu; 2187 2188 /* Exit early if there are no buffered interrupts */ 2189 if (vgic_cpu->ich_lr_used == 0) 2190 return; 2191 2192 /* 2193 * Check on the IRQ state after running the guest. ich_lr_used and 2194 * ich_lr_el2 are only ever used within this thread so is safe to 2195 * access unlocked. 2196 */ 2197 for (i = 0; i < vgic_cpu->ich_lr_used; i++) { 2198 lr = hypctx->vgic_v3_regs.ich_lr_el2[i]; 2199 hypctx->vgic_v3_regs.ich_lr_el2[i] = 0; 2200 2201 irq = vgic_v3_get_irq(hypctx->hyp, vcpu_vcpuid(hypctx->vcpu), 2202 ICH_LR_EL2_VINTID(lr)); 2203 if (irq == NULL) 2204 continue; 2205 2206 irq->active = (lr & ICH_LR_EL2_STATE_ACTIVE) != 0; 2207 2208 if ((irq->config & VGIC_CONFIG_MASK) == VGIC_CONFIG_EDGE) { 2209 /* 2210 * If we have an edge triggered IRQ preserve the 2211 * pending bit until the IRQ has been handled. 2212 */ 2213 if ((lr & ICH_LR_EL2_STATE_PENDING) != 0) { 2214 irq->pending = true; 2215 } 2216 } else { 2217 /* 2218 * If we have a level triggerend IRQ remove the 2219 * pending bit if the IRQ has been handled. 2220 * The level is separate, so may still be high 2221 * triggering another IRQ. 2222 */ 2223 if ((lr & ICH_LR_EL2_STATE_PENDING) == 0) { 2224 irq->pending = false; 2225 } 2226 } 2227 2228 /* Lock to update irq_act_pend */ 2229 mtx_lock_spin(&vgic_cpu->lr_mtx); 2230 if (irq->active) { 2231 /* Ensure the active IRQ is at the head of the list */ 2232 TAILQ_REMOVE(&vgic_cpu->irq_act_pend, irq, 2233 act_pend_list); 2234 TAILQ_INSERT_HEAD(&vgic_cpu->irq_act_pend, irq, 2235 act_pend_list); 2236 } else if (!vgic_v3_irq_pending(irq)) { 2237 /* If pending or active remove from the list */ 2238 TAILQ_REMOVE(&vgic_cpu->irq_act_pend, irq, 2239 act_pend_list); 2240 irq->on_aplist = false; 2241 } 2242 mtx_unlock_spin(&vgic_cpu->lr_mtx); 2243 vgic_v3_release_irq(irq); 2244 } 2245 2246 hypctx->vgic_v3_regs.ich_hcr_el2 &= ~ICH_HCR_EL2_EOICOUNT_MASK; 2247 vgic_cpu->ich_lr_used = 0; 2248 } 2249 2250 static void 2251 vgic_v3_init(device_t dev) 2252 { 2253 uint64_t ich_vtr_el2; 2254 uint32_t pribits, prebits; 2255 2256 ich_vtr_el2 = vmm_read_reg(HYP_REG_ICH_VTR); 2257 2258 /* TODO: These fields are common with the vgicv2 driver */ 2259 pribits = ICH_VTR_EL2_PRIBITS(ich_vtr_el2); 2260 switch (pribits) { 2261 default: 2262 case 5: 2263 virt_features.min_prio = 0xf8; 2264 break; 2265 case 6: 2266 virt_features.min_prio = 0xfc; 2267 break; 2268 case 7: 2269 virt_features.min_prio = 0xfe; 2270 break; 2271 case 8: 2272 virt_features.min_prio = 0xff; 2273 break; 2274 } 2275 2276 prebits = ICH_VTR_EL2_PREBITS(ich_vtr_el2); 2277 switch (prebits) { 2278 default: 2279 case 5: 2280 virt_features.ich_apr_num = 1; 2281 break; 2282 case 6: 2283 virt_features.ich_apr_num = 2; 2284 break; 2285 case 7: 2286 virt_features.ich_apr_num = 4; 2287 break; 2288 } 2289 2290 virt_features.ich_lr_num = ICH_VTR_EL2_LISTREGS(ich_vtr_el2); 2291 } 2292 2293 static int 2294 vgic_v3_probe(device_t dev) 2295 { 2296 if (!gic_get_vgic(dev)) 2297 return (EINVAL); 2298 2299 /* We currently only support the GICv3 */ 2300 if (gic_get_hw_rev(dev) < 3) 2301 return (EINVAL); 2302 2303 device_set_desc(dev, "Virtual GIC v3"); 2304 return (BUS_PROBE_DEFAULT); 2305 } 2306 2307 static int 2308 vgic_v3_attach(device_t dev) 2309 { 2310 vgic_dev = dev; 2311 return (0); 2312 } 2313 2314 static int 2315 vgic_v3_detach(device_t dev) 2316 { 2317 vgic_dev = NULL; 2318 return (0); 2319 } 2320 2321 static device_method_t vgic_v3_methods[] = { 2322 /* Device interface */ 2323 DEVMETHOD(device_probe, vgic_v3_probe), 2324 DEVMETHOD(device_attach, vgic_v3_attach), 2325 DEVMETHOD(device_detach, vgic_v3_detach), 2326 2327 /* VGIC interface */ 2328 DEVMETHOD(vgic_init, vgic_v3_init), 2329 DEVMETHOD(vgic_attach_to_vm, vgic_v3_attach_to_vm), 2330 DEVMETHOD(vgic_detach_from_vm, vgic_v3_detach_from_vm), 2331 DEVMETHOD(vgic_vminit, vgic_v3_vminit), 2332 DEVMETHOD(vgic_cpuinit, vgic_v3_cpuinit), 2333 DEVMETHOD(vgic_cpucleanup, vgic_v3_cpucleanup), 2334 DEVMETHOD(vgic_vmcleanup, vgic_v3_vmcleanup), 2335 DEVMETHOD(vgic_max_cpu_count, vgic_v3_max_cpu_count), 2336 DEVMETHOD(vgic_has_pending_irq, vgic_v3_has_pending_irq), 2337 DEVMETHOD(vgic_inject_irq, vgic_v3_inject_irq), 2338 DEVMETHOD(vgic_inject_msi, vgic_v3_inject_msi), 2339 DEVMETHOD(vgic_flush_hwstate, vgic_v3_flush_hwstate), 2340 DEVMETHOD(vgic_sync_hwstate, vgic_v3_sync_hwstate), 2341 2342 /* End */ 2343 DEVMETHOD_END 2344 }; 2345 2346 /* TODO: Create a vgic base class? */ 2347 DEFINE_CLASS_0(vgic, vgic_v3_driver, vgic_v3_methods, 0); 2348 2349 DRIVER_MODULE(vgic_v3, gic, vgic_v3_driver, 0, 0); 2350