1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause 3 * 4 * Copyright (c) 2024 Ruslan Bukin <br@bsdpad.com> 5 * 6 * This software was developed by the University of Cambridge Computer 7 * Laboratory (Department of Computer Science and Technology) under Innovate 8 * UK project 105694, "Digital Security by Design (DSbD) Technology Platform 9 * Prototype". 10 * 11 * Redistribution and use in source and binary forms, with or without 12 * modification, are permitted provided that the following conditions 13 * are met: 14 * 1. Redistributions of source code must retain the above copyright 15 * notice, this list of conditions and the following disclaimer. 16 * 2. Redistributions in binary form must reproduce the above copyright 17 * notice, this list of conditions and the following disclaimer in the 18 * documentation and/or other materials provided with the distribution. 19 * 20 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 21 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 23 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 24 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 25 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 26 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 27 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 28 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 29 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 30 * SUCH DAMAGE. 31 */ 32 33 #include <sys/types.h> 34 #include <sys/errno.h> 35 #include <sys/systm.h> 36 #include <sys/bus.h> 37 #include <sys/kernel.h> 38 #include <sys/lock.h> 39 #include <sys/malloc.h> 40 #include <sys/module.h> 41 #include <sys/mutex.h> 42 #include <sys/rman.h> 43 #include <sys/smp.h> 44 45 #include <riscv/vmm/riscv.h> 46 #include <riscv/vmm/vmm_aplic.h> 47 48 #include <machine/vmm_instruction_emul.h> 49 #include <machine/vmm_dev.h> 50 51 MALLOC_DEFINE(M_APLIC, "RISC-V VMM APLIC", "RISC-V AIA APLIC"); 52 53 #define APLIC_DOMAINCFG 0x0000 54 #define DOMAINCFG_IE (1 << 8) /* Interrupt Enable. */ 55 #define DOMAINCFG_DM (1 << 2) /* Direct Mode. */ 56 #define DOMAINCFG_BE (1 << 0) /* Big-Endian. */ 57 #define APLIC_SOURCECFG(x) (0x0004 + ((x) - 1) * 4) 58 #define SOURCECFG_D (1 << 10) /* D - Delegate. */ 59 /* If D == 0. */ 60 #define SOURCECFG_SM_S (0) 61 #define SOURCECFG_SM_M (0x7 << SOURCECFG_SM_S) 62 #define SOURCECFG_SM_INACTIVE (0) /* Not delegated. */ 63 #define SOURCECFG_SM_DETACHED (1) 64 #define SOURCECFG_SM_RESERVED (2) 65 #define SOURCECFG_SM_RESERVED1 (3) 66 #define SOURCECFG_SM_EDGE1 (4) /* Rising edge. */ 67 #define SOURCECFG_SM_EDGE0 (5) /* Falling edge. */ 68 #define SOURCECFG_SM_LEVEL1 (6) /* High. */ 69 #define SOURCECFG_SM_LEVEL0 (7) /* Low. */ 70 /* If D == 1. */ 71 #define SOURCECFG_CHILD_INDEX_S (0) 72 #define SOURCECFG_CHILD_INDEX_M (0x3ff << SOURCECFG_CHILD_INDEX_S) 73 #define APLIC_SETIP 0x1c00 74 #define APLIC_SETIPNUM 0x1cdc 75 #define APLIC_CLRIP 0x1d00 76 #define APLIC_CLRIPNUM 0x1ddc 77 #define APLIC_SETIE 0x1e00 78 #define APLIC_SETIENUM 0x1edc 79 #define APLIC_CLRIE 0x1f00 80 #define APLIC_CLRIENUM 0x1fdc 81 #define APLIC_GENMSI 0x3000 82 #define APLIC_TARGET(x) (0x3004 + ((x) - 1) * 4) 83 #define TARGET_HART_S 18 84 #define TARGET_HART_M 0x3fff 85 #define APLIC_IDC(x) (0x4000 + (x) * 32) 86 #define IDC_IDELIVERY(x) (APLIC_IDC(x) + 0x0) 87 #define IDC_IFORCE(x) (APLIC_IDC(x) + 0x4) 88 #define IDC_ITHRESHOLD(x) (APLIC_IDC(x) + 0x8) 89 #define IDC_TOPI(x) (APLIC_IDC(x) + 0x18) 90 #define IDC_CLAIMI(x) (APLIC_IDC(x) + 0x1C) 91 #define CLAIMI_IRQ_S (16) 92 #define CLAIMI_IRQ_M (0x3ff << CLAIMI_IRQ_S) 93 #define CLAIMI_PRIO_S (0) 94 #define CLAIMI_PRIO_M (0xff << CLAIMI_PRIO_S) 95 96 #define APLIC_NIRQS 63 97 98 struct aplic_irq { 99 uint32_t sourcecfg; 100 uint32_t state; 101 #define APLIC_IRQ_STATE_PENDING (1 << 0) 102 #define APLIC_IRQ_STATE_ENABLED (1 << 1) 103 #define APLIC_IRQ_STATE_INPUT (1 << 2) 104 uint32_t target; 105 uint32_t target_hart; 106 }; 107 108 struct aplic { 109 uint32_t mem_start; 110 uint32_t mem_end; 111 struct mtx mtx; 112 struct aplic_irq *irqs; 113 int nirqs; 114 uint32_t domaincfg; 115 }; 116 117 static int 118 aplic_handle_sourcecfg(struct aplic *aplic, int i, bool write, uint64_t *val) 119 { 120 struct aplic_irq *irq; 121 122 if (i <= 0 || i > aplic->nirqs) 123 return (ENOENT); 124 125 mtx_lock_spin(&aplic->mtx); 126 irq = &aplic->irqs[i]; 127 if (write) 128 irq->sourcecfg = *val; 129 else 130 *val = irq->sourcecfg; 131 mtx_unlock_spin(&aplic->mtx); 132 133 return (0); 134 } 135 136 static int 137 aplic_set_enabled(struct aplic *aplic, bool write, uint64_t *val, bool enabled) 138 { 139 struct aplic_irq *irq; 140 int i; 141 142 if (!write) { 143 *val = 0; 144 return (0); 145 } 146 147 i = *val; 148 if (i <= 0 || i > aplic->nirqs) 149 return (-1); 150 151 irq = &aplic->irqs[i]; 152 153 mtx_lock_spin(&aplic->mtx); 154 if ((irq->sourcecfg & SOURCECFG_SM_M) != SOURCECFG_SM_INACTIVE) { 155 if (enabled) 156 irq->state |= APLIC_IRQ_STATE_ENABLED; 157 else 158 irq->state &= ~APLIC_IRQ_STATE_ENABLED; 159 } 160 mtx_unlock_spin(&aplic->mtx); 161 162 return (0); 163 } 164 165 static void 166 aplic_set_enabled_word(struct aplic *aplic, bool write, uint32_t word, 167 uint64_t *val, bool enabled) 168 { 169 uint64_t v; 170 int i; 171 172 if (!write) { 173 *val = 0; 174 return; 175 } 176 177 /* 178 * The write is ignored if value written is not an active interrupt 179 * source number in the domain. 180 */ 181 for (i = 0; i < 32; i++) 182 if (*val & (1u << i)) { 183 v = word * 32 + i; 184 (void)aplic_set_enabled(aplic, write, &v, enabled); 185 } 186 } 187 188 static int 189 aplic_handle_target(struct aplic *aplic, int i, bool write, uint64_t *val) 190 { 191 struct aplic_irq *irq; 192 193 mtx_lock_spin(&aplic->mtx); 194 irq = &aplic->irqs[i]; 195 if (write) { 196 irq->target = *val; 197 irq->target_hart = (irq->target >> TARGET_HART_S); 198 } else 199 *val = irq->target; 200 mtx_unlock_spin(&aplic->mtx); 201 202 return (0); 203 } 204 205 static int 206 aplic_handle_idc_claimi(struct hyp *hyp, struct aplic *aplic, int cpu_id, 207 bool write, uint64_t *val) 208 { 209 struct aplic_irq *irq; 210 bool found; 211 int i; 212 213 /* Writes to claimi are ignored. */ 214 if (write) 215 return (-1); 216 217 found = false; 218 219 mtx_lock_spin(&aplic->mtx); 220 for (i = 0; i < aplic->nirqs; i++) { 221 irq = &aplic->irqs[i]; 222 if (irq->target_hart != cpu_id) 223 continue; 224 if (irq->state & APLIC_IRQ_STATE_PENDING) { 225 *val = (i << CLAIMI_IRQ_S) | (0 << CLAIMI_PRIO_S); 226 irq->state &= ~APLIC_IRQ_STATE_PENDING; 227 found = true; 228 break; 229 } 230 } 231 mtx_unlock_spin(&aplic->mtx); 232 233 if (found == false) 234 *val = 0; 235 236 return (0); 237 } 238 239 static int 240 aplic_handle_idc(struct hyp *hyp, struct aplic *aplic, int cpu, int reg, 241 bool write, uint64_t *val) 242 { 243 int error; 244 245 switch (reg + APLIC_IDC(0)) { 246 case IDC_IDELIVERY(0): 247 case IDC_IFORCE(0): 248 case IDC_ITHRESHOLD(0): 249 case IDC_TOPI(0): 250 error = 0; 251 break; 252 case IDC_CLAIMI(0): 253 error = aplic_handle_idc_claimi(hyp, aplic, cpu, write, val); 254 break; 255 default: 256 error = ENOENT; 257 } 258 259 return (error); 260 } 261 262 static int 263 aplic_mmio_access(struct hyp *hyp, struct aplic *aplic, uint64_t reg, 264 bool write, uint64_t *val) 265 { 266 int error; 267 int cpu; 268 int r; 269 int i; 270 271 dprintf("%s: reg %lx\n", __func__, reg); 272 273 if ((reg >= APLIC_SOURCECFG(1)) && 274 (reg <= APLIC_SOURCECFG(aplic->nirqs))) { 275 i = ((reg - APLIC_SOURCECFG(1)) >> 2) + 1; 276 error = aplic_handle_sourcecfg(aplic, i, write, val); 277 return (error); 278 } 279 280 if ((reg >= APLIC_TARGET(1)) && (reg <= APLIC_TARGET(aplic->nirqs))) { 281 i = ((reg - APLIC_TARGET(1)) >> 2) + 1; 282 error = aplic_handle_target(aplic, i, write, val); 283 return (error); 284 } 285 286 if ((reg >= APLIC_IDC(0)) && (reg < APLIC_IDC(mp_ncpus))) { 287 cpu = (reg - APLIC_IDC(0)) >> 5; 288 r = (reg - APLIC_IDC(0)) % 32; 289 error = aplic_handle_idc(hyp, aplic, cpu, r, write, val); 290 return (error); 291 } 292 293 if ((reg >= APLIC_CLRIE) && (reg < (APLIC_CLRIE + aplic->nirqs * 4))) { 294 i = (reg - APLIC_CLRIE) >> 2; 295 aplic_set_enabled_word(aplic, write, i, val, false); 296 return (0); 297 } 298 299 switch (reg) { 300 case APLIC_DOMAINCFG: 301 mtx_lock_spin(&aplic->mtx); 302 if (write) 303 aplic->domaincfg = *val & DOMAINCFG_IE; 304 else 305 *val = aplic->domaincfg; 306 mtx_unlock_spin(&aplic->mtx); 307 error = 0; 308 break; 309 case APLIC_SETIENUM: 310 error = aplic_set_enabled(aplic, write, val, true); 311 break; 312 case APLIC_CLRIENUM: 313 error = aplic_set_enabled(aplic, write, val, false); 314 break; 315 default: 316 dprintf("%s: unknown reg %lx", __func__, reg); 317 error = ENOENT; 318 break; 319 }; 320 321 return (error); 322 } 323 324 static int 325 mem_read(struct vcpu *vcpu, uint64_t fault_ipa, uint64_t *rval, int size, 326 void *arg) 327 { 328 struct hypctx *hypctx; 329 struct hyp *hyp; 330 struct aplic *aplic; 331 uint64_t reg; 332 uint64_t val; 333 int error; 334 335 hypctx = vcpu_get_cookie(vcpu); 336 hyp = hypctx->hyp; 337 aplic = hyp->aplic; 338 339 dprintf("%s: fault_ipa %lx size %d\n", __func__, fault_ipa, size); 340 341 if (fault_ipa < aplic->mem_start || fault_ipa + size > aplic->mem_end) 342 return (EINVAL); 343 344 reg = fault_ipa - aplic->mem_start; 345 346 error = aplic_mmio_access(hyp, aplic, reg, false, &val); 347 if (error == 0) 348 *rval = val; 349 350 return (error); 351 } 352 353 static int 354 mem_write(struct vcpu *vcpu, uint64_t fault_ipa, uint64_t wval, int size, 355 void *arg) 356 { 357 struct hypctx *hypctx; 358 struct hyp *hyp; 359 struct aplic *aplic; 360 uint64_t reg; 361 uint64_t val; 362 int error; 363 364 hypctx = vcpu_get_cookie(vcpu); 365 hyp = hypctx->hyp; 366 aplic = hyp->aplic; 367 368 dprintf("%s: fault_ipa %lx wval %lx size %d\n", __func__, fault_ipa, 369 wval, size); 370 371 if (fault_ipa < aplic->mem_start || fault_ipa + size > aplic->mem_end) 372 return (EINVAL); 373 374 reg = fault_ipa - aplic->mem_start; 375 376 val = wval; 377 378 error = aplic_mmio_access(hyp, aplic, reg, true, &val); 379 380 return (error); 381 } 382 383 void 384 aplic_vminit(struct hyp *hyp) 385 { 386 struct aplic *aplic; 387 388 hyp->aplic = malloc(sizeof(*hyp->aplic), M_APLIC, 389 M_WAITOK | M_ZERO); 390 aplic = hyp->aplic; 391 392 mtx_init(&aplic->mtx, "APLIC lock", NULL, MTX_SPIN); 393 } 394 395 void 396 aplic_vmcleanup(struct hyp *hyp) 397 { 398 struct aplic *aplic; 399 400 aplic = hyp->aplic; 401 402 mtx_destroy(&aplic->mtx); 403 404 free(hyp->aplic, M_APLIC); 405 } 406 407 int 408 aplic_attach_to_vm(struct hyp *hyp, struct vm_aplic_descr *descr) 409 { 410 struct aplic *aplic; 411 struct vm *vm; 412 413 vm = hyp->vm; 414 415 dprintf("%s\n", __func__); 416 417 vm_register_inst_handler(vm, descr->mem_start, descr->mem_size, 418 mem_read, mem_write); 419 420 aplic = hyp->aplic; 421 aplic->nirqs = APLIC_NIRQS; 422 aplic->mem_start = descr->mem_start; 423 aplic->mem_end = descr->mem_start + descr->mem_size; 424 aplic->irqs = malloc(sizeof(struct aplic_irq) * aplic->nirqs, M_APLIC, 425 M_WAITOK | M_ZERO); 426 427 hyp->aplic_attached = true; 428 429 return (0); 430 } 431 432 void 433 aplic_detach_from_vm(struct hyp *hyp) 434 { 435 struct aplic *aplic; 436 437 aplic = hyp->aplic; 438 439 dprintf("%s\n", __func__); 440 441 if (hyp->aplic_attached) { 442 hyp->aplic_attached = false; 443 free(aplic->irqs, M_APLIC); 444 } 445 } 446 447 int 448 aplic_check_pending(struct hypctx *hypctx) 449 { 450 struct aplic_irq *irq; 451 struct aplic *aplic; 452 struct hyp *hyp; 453 int i; 454 455 hyp = hypctx->hyp; 456 aplic = hyp->aplic; 457 458 mtx_lock_spin(&aplic->mtx); 459 if ((aplic->domaincfg & DOMAINCFG_IE) == 0) { 460 mtx_unlock_spin(&aplic->mtx); 461 return (0); 462 } 463 464 for (i = 0; i < aplic->nirqs; i++) { 465 irq = &aplic->irqs[i]; 466 if (irq->target_hart != hypctx->cpu_id) 467 continue; 468 if ((irq->state & APLIC_IRQ_STATE_ENABLED) && 469 (irq->state & APLIC_IRQ_STATE_PENDING)) { 470 mtx_unlock_spin(&aplic->mtx); 471 /* Found. */ 472 return (1); 473 } 474 } 475 mtx_unlock_spin(&aplic->mtx); 476 477 return (0); 478 } 479 480 int 481 aplic_inject_irq(struct hyp *hyp, int vcpuid, uint32_t irqid, bool level) 482 { 483 struct aplic_irq *irq; 484 struct aplic *aplic; 485 bool notify; 486 int error; 487 int mask; 488 489 aplic = hyp->aplic; 490 491 error = 0; 492 493 mtx_lock_spin(&aplic->mtx); 494 if ((aplic->domaincfg & DOMAINCFG_IE) == 0) { 495 mtx_unlock_spin(&aplic->mtx); 496 return (error); 497 } 498 499 irq = &aplic->irqs[irqid]; 500 if (irq->sourcecfg & SOURCECFG_D) { 501 mtx_unlock_spin(&aplic->mtx); 502 return (error); 503 } 504 505 notify = false; 506 switch (irq->sourcecfg & SOURCECFG_SM_M) { 507 case SOURCECFG_SM_LEVEL0: 508 if (!level) 509 irq->state |= APLIC_IRQ_STATE_PENDING; 510 break; 511 case SOURCECFG_SM_LEVEL1: 512 if (level) 513 irq->state |= APLIC_IRQ_STATE_PENDING; 514 break; 515 case SOURCECFG_SM_EDGE0: 516 if (!level && (irq->state & APLIC_IRQ_STATE_INPUT)) 517 irq->state |= APLIC_IRQ_STATE_PENDING; 518 break; 519 case SOURCECFG_SM_EDGE1: 520 if (level && !(irq->state & APLIC_IRQ_STATE_INPUT)) 521 irq->state |= APLIC_IRQ_STATE_PENDING; 522 break; 523 case SOURCECFG_SM_DETACHED: 524 case SOURCECFG_SM_INACTIVE: 525 break; 526 default: 527 error = ENXIO; 528 break; 529 } 530 531 if (level) 532 irq->state |= APLIC_IRQ_STATE_INPUT; 533 else 534 irq->state &= ~APLIC_IRQ_STATE_INPUT; 535 536 mask = APLIC_IRQ_STATE_ENABLED | APLIC_IRQ_STATE_PENDING; 537 if ((irq->state & mask) == mask) 538 notify = true; 539 540 mtx_unlock_spin(&aplic->mtx); 541 542 if (notify) 543 vcpu_notify_event(vm_vcpu(hyp->vm, irq->target_hart)); 544 545 return (error); 546 } 547 548 int 549 aplic_inject_msi(struct hyp *hyp, uint64_t msg, uint64_t addr) 550 { 551 552 /* TODO. */ 553 554 return (ENXIO); 555 } 556 557 void 558 aplic_cpuinit(struct hypctx *hypctx) 559 { 560 561 } 562 563 void 564 aplic_cpucleanup(struct hypctx *hypctx) 565 { 566 567 } 568 569 void 570 aplic_flush_hwstate(struct hypctx *hypctx) 571 { 572 573 } 574 575 void 576 aplic_sync_hwstate(struct hypctx *hypctx) 577 { 578 579 } 580 581 int 582 aplic_max_cpu_count(struct hyp *hyp) 583 { 584 int16_t max_count; 585 586 max_count = vm_get_maxcpus(hyp->vm); 587 588 return (max_count); 589 } 590