1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (C) 2019 Western Digital Corporation or its affiliates. 4 * 5 * Authors: 6 * Anup Patel <anup.patel@wdc.com> 7 */ 8 9 #include <linux/bitops.h> 10 #include <linux/entry-kvm.h> 11 #include <linux/errno.h> 12 #include <linux/err.h> 13 #include <linux/kdebug.h> 14 #include <linux/module.h> 15 #include <linux/percpu.h> 16 #include <linux/vmalloc.h> 17 #include <linux/sched/signal.h> 18 #include <linux/fs.h> 19 #include <linux/kvm_host.h> 20 #include <asm/csr.h> 21 #include <asm/cacheflush.h> 22 #include <asm/kvm_vcpu_vector.h> 23 24 const struct _kvm_stats_desc kvm_vcpu_stats_desc[] = { 25 KVM_GENERIC_VCPU_STATS(), 26 STATS_DESC_COUNTER(VCPU, ecall_exit_stat), 27 STATS_DESC_COUNTER(VCPU, wfi_exit_stat), 28 STATS_DESC_COUNTER(VCPU, mmio_exit_user), 29 STATS_DESC_COUNTER(VCPU, mmio_exit_kernel), 30 STATS_DESC_COUNTER(VCPU, csr_exit_user), 31 STATS_DESC_COUNTER(VCPU, csr_exit_kernel), 32 STATS_DESC_COUNTER(VCPU, signal_exits), 33 STATS_DESC_COUNTER(VCPU, exits) 34 }; 35 36 const struct kvm_stats_header kvm_vcpu_stats_header = { 37 .name_size = KVM_STATS_NAME_SIZE, 38 .num_desc = ARRAY_SIZE(kvm_vcpu_stats_desc), 39 .id_offset = sizeof(struct kvm_stats_header), 40 .desc_offset = sizeof(struct kvm_stats_header) + KVM_STATS_NAME_SIZE, 41 .data_offset = sizeof(struct kvm_stats_header) + KVM_STATS_NAME_SIZE + 42 sizeof(kvm_vcpu_stats_desc), 43 }; 44 45 static void kvm_riscv_reset_vcpu(struct kvm_vcpu *vcpu) 46 { 47 struct kvm_vcpu_csr *csr = &vcpu->arch.guest_csr; 48 struct kvm_vcpu_csr *reset_csr = &vcpu->arch.guest_reset_csr; 49 struct kvm_cpu_context *cntx = &vcpu->arch.guest_context; 50 struct kvm_cpu_context *reset_cntx = &vcpu->arch.guest_reset_context; 51 bool loaded; 52 53 /** 54 * The preemption should be disabled here because it races with 55 * kvm_sched_out/kvm_sched_in(called from preempt notifiers) which 56 * also calls vcpu_load/put. 57 */ 58 get_cpu(); 59 loaded = (vcpu->cpu != -1); 60 if (loaded) 61 kvm_arch_vcpu_put(vcpu); 62 63 vcpu->arch.last_exit_cpu = -1; 64 65 memcpy(csr, reset_csr, sizeof(*csr)); 66 67 spin_lock(&vcpu->arch.reset_cntx_lock); 68 memcpy(cntx, reset_cntx, sizeof(*cntx)); 69 spin_unlock(&vcpu->arch.reset_cntx_lock); 70 71 kvm_riscv_vcpu_fp_reset(vcpu); 72 73 kvm_riscv_vcpu_vector_reset(vcpu); 74 75 kvm_riscv_vcpu_timer_reset(vcpu); 76 77 kvm_riscv_vcpu_aia_reset(vcpu); 78 79 bitmap_zero(vcpu->arch.irqs_pending, KVM_RISCV_VCPU_NR_IRQS); 80 bitmap_zero(vcpu->arch.irqs_pending_mask, KVM_RISCV_VCPU_NR_IRQS); 81 82 kvm_riscv_vcpu_pmu_reset(vcpu); 83 84 vcpu->arch.hfence_head = 0; 85 vcpu->arch.hfence_tail = 0; 86 memset(vcpu->arch.hfence_queue, 0, sizeof(vcpu->arch.hfence_queue)); 87 88 kvm_riscv_vcpu_sbi_sta_reset(vcpu); 89 90 /* Reset the guest CSRs for hotplug usecase */ 91 if (loaded) 92 kvm_arch_vcpu_load(vcpu, smp_processor_id()); 93 put_cpu(); 94 } 95 96 int kvm_arch_vcpu_precreate(struct kvm *kvm, unsigned int id) 97 { 98 return 0; 99 } 100 101 int kvm_arch_vcpu_create(struct kvm_vcpu *vcpu) 102 { 103 int rc; 104 struct kvm_cpu_context *cntx; 105 struct kvm_vcpu_csr *reset_csr = &vcpu->arch.guest_reset_csr; 106 107 spin_lock_init(&vcpu->arch.mp_state_lock); 108 109 /* Mark this VCPU never ran */ 110 vcpu->arch.ran_atleast_once = false; 111 vcpu->arch.mmu_page_cache.gfp_zero = __GFP_ZERO; 112 bitmap_zero(vcpu->arch.isa, RISCV_ISA_EXT_MAX); 113 114 /* Setup ISA features available to VCPU */ 115 kvm_riscv_vcpu_setup_isa(vcpu); 116 117 /* Setup vendor, arch, and implementation details */ 118 vcpu->arch.mvendorid = sbi_get_mvendorid(); 119 vcpu->arch.marchid = sbi_get_marchid(); 120 vcpu->arch.mimpid = sbi_get_mimpid(); 121 122 /* Setup VCPU hfence queue */ 123 spin_lock_init(&vcpu->arch.hfence_lock); 124 125 /* Setup reset state of shadow SSTATUS and HSTATUS CSRs */ 126 spin_lock_init(&vcpu->arch.reset_cntx_lock); 127 128 spin_lock(&vcpu->arch.reset_cntx_lock); 129 cntx = &vcpu->arch.guest_reset_context; 130 cntx->sstatus = SR_SPP | SR_SPIE; 131 cntx->hstatus = 0; 132 cntx->hstatus |= HSTATUS_VTW; 133 cntx->hstatus |= HSTATUS_SPVP; 134 cntx->hstatus |= HSTATUS_SPV; 135 spin_unlock(&vcpu->arch.reset_cntx_lock); 136 137 if (kvm_riscv_vcpu_alloc_vector_context(vcpu, cntx)) 138 return -ENOMEM; 139 140 /* By default, make CY, TM, and IR counters accessible in VU mode */ 141 reset_csr->scounteren = 0x7; 142 143 /* Setup VCPU timer */ 144 kvm_riscv_vcpu_timer_init(vcpu); 145 146 /* setup performance monitoring */ 147 kvm_riscv_vcpu_pmu_init(vcpu); 148 149 /* Setup VCPU AIA */ 150 rc = kvm_riscv_vcpu_aia_init(vcpu); 151 if (rc) 152 return rc; 153 154 /* 155 * Setup SBI extensions 156 * NOTE: This must be the last thing to be initialized. 157 */ 158 kvm_riscv_vcpu_sbi_init(vcpu); 159 160 /* Reset VCPU */ 161 kvm_riscv_reset_vcpu(vcpu); 162 163 return 0; 164 } 165 166 void kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu) 167 { 168 /** 169 * vcpu with id 0 is the designated boot cpu. 170 * Keep all vcpus with non-zero id in power-off state so that 171 * they can be brought up using SBI HSM extension. 172 */ 173 if (vcpu->vcpu_idx != 0) 174 kvm_riscv_vcpu_power_off(vcpu); 175 } 176 177 void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu) 178 { 179 /* Cleanup VCPU AIA context */ 180 kvm_riscv_vcpu_aia_deinit(vcpu); 181 182 /* Cleanup VCPU timer */ 183 kvm_riscv_vcpu_timer_deinit(vcpu); 184 185 kvm_riscv_vcpu_pmu_deinit(vcpu); 186 187 /* Free unused pages pre-allocated for G-stage page table mappings */ 188 kvm_mmu_free_memory_cache(&vcpu->arch.mmu_page_cache); 189 190 /* Free vector context space for host and guest kernel */ 191 kvm_riscv_vcpu_free_vector_context(vcpu); 192 } 193 194 int kvm_cpu_has_pending_timer(struct kvm_vcpu *vcpu) 195 { 196 return kvm_riscv_vcpu_timer_pending(vcpu); 197 } 198 199 void kvm_arch_vcpu_blocking(struct kvm_vcpu *vcpu) 200 { 201 kvm_riscv_aia_wakeon_hgei(vcpu, true); 202 } 203 204 void kvm_arch_vcpu_unblocking(struct kvm_vcpu *vcpu) 205 { 206 kvm_riscv_aia_wakeon_hgei(vcpu, false); 207 } 208 209 int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu) 210 { 211 return (kvm_riscv_vcpu_has_interrupts(vcpu, -1UL) && 212 !kvm_riscv_vcpu_stopped(vcpu) && !vcpu->arch.pause); 213 } 214 215 int kvm_arch_vcpu_should_kick(struct kvm_vcpu *vcpu) 216 { 217 return kvm_vcpu_exiting_guest_mode(vcpu) == IN_GUEST_MODE; 218 } 219 220 bool kvm_arch_vcpu_in_kernel(struct kvm_vcpu *vcpu) 221 { 222 return (vcpu->arch.guest_context.sstatus & SR_SPP) ? true : false; 223 } 224 225 vm_fault_t kvm_arch_vcpu_fault(struct kvm_vcpu *vcpu, struct vm_fault *vmf) 226 { 227 return VM_FAULT_SIGBUS; 228 } 229 230 long kvm_arch_vcpu_async_ioctl(struct file *filp, 231 unsigned int ioctl, unsigned long arg) 232 { 233 struct kvm_vcpu *vcpu = filp->private_data; 234 void __user *argp = (void __user *)arg; 235 236 if (ioctl == KVM_INTERRUPT) { 237 struct kvm_interrupt irq; 238 239 if (copy_from_user(&irq, argp, sizeof(irq))) 240 return -EFAULT; 241 242 if (irq.irq == KVM_INTERRUPT_SET) 243 return kvm_riscv_vcpu_set_interrupt(vcpu, IRQ_VS_EXT); 244 else 245 return kvm_riscv_vcpu_unset_interrupt(vcpu, IRQ_VS_EXT); 246 } 247 248 return -ENOIOCTLCMD; 249 } 250 251 long kvm_arch_vcpu_ioctl(struct file *filp, 252 unsigned int ioctl, unsigned long arg) 253 { 254 struct kvm_vcpu *vcpu = filp->private_data; 255 void __user *argp = (void __user *)arg; 256 long r = -EINVAL; 257 258 switch (ioctl) { 259 case KVM_SET_ONE_REG: 260 case KVM_GET_ONE_REG: { 261 struct kvm_one_reg reg; 262 263 r = -EFAULT; 264 if (copy_from_user(®, argp, sizeof(reg))) 265 break; 266 267 if (ioctl == KVM_SET_ONE_REG) 268 r = kvm_riscv_vcpu_set_reg(vcpu, ®); 269 else 270 r = kvm_riscv_vcpu_get_reg(vcpu, ®); 271 break; 272 } 273 case KVM_GET_REG_LIST: { 274 struct kvm_reg_list __user *user_list = argp; 275 struct kvm_reg_list reg_list; 276 unsigned int n; 277 278 r = -EFAULT; 279 if (copy_from_user(®_list, user_list, sizeof(reg_list))) 280 break; 281 n = reg_list.n; 282 reg_list.n = kvm_riscv_vcpu_num_regs(vcpu); 283 if (copy_to_user(user_list, ®_list, sizeof(reg_list))) 284 break; 285 r = -E2BIG; 286 if (n < reg_list.n) 287 break; 288 r = kvm_riscv_vcpu_copy_reg_indices(vcpu, user_list->reg); 289 break; 290 } 291 default: 292 break; 293 } 294 295 return r; 296 } 297 298 int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu, 299 struct kvm_sregs *sregs) 300 { 301 return -EINVAL; 302 } 303 304 int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu, 305 struct kvm_sregs *sregs) 306 { 307 return -EINVAL; 308 } 309 310 int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu) 311 { 312 return -EINVAL; 313 } 314 315 int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu) 316 { 317 return -EINVAL; 318 } 319 320 int kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu *vcpu, 321 struct kvm_translation *tr) 322 { 323 return -EINVAL; 324 } 325 326 int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs) 327 { 328 return -EINVAL; 329 } 330 331 int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs) 332 { 333 return -EINVAL; 334 } 335 336 void kvm_riscv_vcpu_flush_interrupts(struct kvm_vcpu *vcpu) 337 { 338 struct kvm_vcpu_csr *csr = &vcpu->arch.guest_csr; 339 unsigned long mask, val; 340 341 if (READ_ONCE(vcpu->arch.irqs_pending_mask[0])) { 342 mask = xchg_acquire(&vcpu->arch.irqs_pending_mask[0], 0); 343 val = READ_ONCE(vcpu->arch.irqs_pending[0]) & mask; 344 345 csr->hvip &= ~mask; 346 csr->hvip |= val; 347 } 348 349 /* Flush AIA high interrupts */ 350 kvm_riscv_vcpu_aia_flush_interrupts(vcpu); 351 } 352 353 void kvm_riscv_vcpu_sync_interrupts(struct kvm_vcpu *vcpu) 354 { 355 unsigned long hvip; 356 struct kvm_vcpu_arch *v = &vcpu->arch; 357 struct kvm_vcpu_csr *csr = &vcpu->arch.guest_csr; 358 359 /* Read current HVIP and VSIE CSRs */ 360 csr->vsie = csr_read(CSR_VSIE); 361 362 /* Sync-up HVIP.VSSIP bit changes does by Guest */ 363 hvip = csr_read(CSR_HVIP); 364 if ((csr->hvip ^ hvip) & (1UL << IRQ_VS_SOFT)) { 365 if (hvip & (1UL << IRQ_VS_SOFT)) { 366 if (!test_and_set_bit(IRQ_VS_SOFT, 367 v->irqs_pending_mask)) 368 set_bit(IRQ_VS_SOFT, v->irqs_pending); 369 } else { 370 if (!test_and_set_bit(IRQ_VS_SOFT, 371 v->irqs_pending_mask)) 372 clear_bit(IRQ_VS_SOFT, v->irqs_pending); 373 } 374 } 375 376 /* Sync up the HVIP.LCOFIP bit changes (only clear) by the guest */ 377 if ((csr->hvip ^ hvip) & (1UL << IRQ_PMU_OVF)) { 378 if (!(hvip & (1UL << IRQ_PMU_OVF)) && 379 !test_and_set_bit(IRQ_PMU_OVF, v->irqs_pending_mask)) 380 clear_bit(IRQ_PMU_OVF, v->irqs_pending); 381 } 382 383 /* Sync-up AIA high interrupts */ 384 kvm_riscv_vcpu_aia_sync_interrupts(vcpu); 385 386 /* Sync-up timer CSRs */ 387 kvm_riscv_vcpu_timer_sync(vcpu); 388 } 389 390 int kvm_riscv_vcpu_set_interrupt(struct kvm_vcpu *vcpu, unsigned int irq) 391 { 392 /* 393 * We only allow VS-mode software, timer, and external 394 * interrupts when irq is one of the local interrupts 395 * defined by RISC-V privilege specification. 396 */ 397 if (irq < IRQ_LOCAL_MAX && 398 irq != IRQ_VS_SOFT && 399 irq != IRQ_VS_TIMER && 400 irq != IRQ_VS_EXT && 401 irq != IRQ_PMU_OVF) 402 return -EINVAL; 403 404 set_bit(irq, vcpu->arch.irqs_pending); 405 smp_mb__before_atomic(); 406 set_bit(irq, vcpu->arch.irqs_pending_mask); 407 408 kvm_vcpu_kick(vcpu); 409 410 return 0; 411 } 412 413 int kvm_riscv_vcpu_unset_interrupt(struct kvm_vcpu *vcpu, unsigned int irq) 414 { 415 /* 416 * We only allow VS-mode software, timer, counter overflow and external 417 * interrupts when irq is one of the local interrupts 418 * defined by RISC-V privilege specification. 419 */ 420 if (irq < IRQ_LOCAL_MAX && 421 irq != IRQ_VS_SOFT && 422 irq != IRQ_VS_TIMER && 423 irq != IRQ_VS_EXT && 424 irq != IRQ_PMU_OVF) 425 return -EINVAL; 426 427 clear_bit(irq, vcpu->arch.irqs_pending); 428 smp_mb__before_atomic(); 429 set_bit(irq, vcpu->arch.irqs_pending_mask); 430 431 return 0; 432 } 433 434 bool kvm_riscv_vcpu_has_interrupts(struct kvm_vcpu *vcpu, u64 mask) 435 { 436 unsigned long ie; 437 438 ie = ((vcpu->arch.guest_csr.vsie & VSIP_VALID_MASK) 439 << VSIP_TO_HVIP_SHIFT) & (unsigned long)mask; 440 ie |= vcpu->arch.guest_csr.vsie & ~IRQ_LOCAL_MASK & 441 (unsigned long)mask; 442 if (READ_ONCE(vcpu->arch.irqs_pending[0]) & ie) 443 return true; 444 445 /* Check AIA high interrupts */ 446 return kvm_riscv_vcpu_aia_has_interrupts(vcpu, mask); 447 } 448 449 void __kvm_riscv_vcpu_power_off(struct kvm_vcpu *vcpu) 450 { 451 WRITE_ONCE(vcpu->arch.mp_state.mp_state, KVM_MP_STATE_STOPPED); 452 kvm_make_request(KVM_REQ_SLEEP, vcpu); 453 kvm_vcpu_kick(vcpu); 454 } 455 456 void kvm_riscv_vcpu_power_off(struct kvm_vcpu *vcpu) 457 { 458 spin_lock(&vcpu->arch.mp_state_lock); 459 __kvm_riscv_vcpu_power_off(vcpu); 460 spin_unlock(&vcpu->arch.mp_state_lock); 461 } 462 463 void __kvm_riscv_vcpu_power_on(struct kvm_vcpu *vcpu) 464 { 465 WRITE_ONCE(vcpu->arch.mp_state.mp_state, KVM_MP_STATE_RUNNABLE); 466 kvm_vcpu_wake_up(vcpu); 467 } 468 469 void kvm_riscv_vcpu_power_on(struct kvm_vcpu *vcpu) 470 { 471 spin_lock(&vcpu->arch.mp_state_lock); 472 __kvm_riscv_vcpu_power_on(vcpu); 473 spin_unlock(&vcpu->arch.mp_state_lock); 474 } 475 476 bool kvm_riscv_vcpu_stopped(struct kvm_vcpu *vcpu) 477 { 478 return READ_ONCE(vcpu->arch.mp_state.mp_state) == KVM_MP_STATE_STOPPED; 479 } 480 481 int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu, 482 struct kvm_mp_state *mp_state) 483 { 484 *mp_state = READ_ONCE(vcpu->arch.mp_state); 485 486 return 0; 487 } 488 489 int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu, 490 struct kvm_mp_state *mp_state) 491 { 492 int ret = 0; 493 494 spin_lock(&vcpu->arch.mp_state_lock); 495 496 switch (mp_state->mp_state) { 497 case KVM_MP_STATE_RUNNABLE: 498 WRITE_ONCE(vcpu->arch.mp_state, *mp_state); 499 break; 500 case KVM_MP_STATE_STOPPED: 501 __kvm_riscv_vcpu_power_off(vcpu); 502 break; 503 default: 504 ret = -EINVAL; 505 } 506 507 spin_unlock(&vcpu->arch.mp_state_lock); 508 509 return ret; 510 } 511 512 int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu, 513 struct kvm_guest_debug *dbg) 514 { 515 if (dbg->control & KVM_GUESTDBG_ENABLE) { 516 vcpu->guest_debug = dbg->control; 517 vcpu->arch.cfg.hedeleg &= ~BIT(EXC_BREAKPOINT); 518 } else { 519 vcpu->guest_debug = 0; 520 vcpu->arch.cfg.hedeleg |= BIT(EXC_BREAKPOINT); 521 } 522 523 return 0; 524 } 525 526 static void kvm_riscv_vcpu_setup_config(struct kvm_vcpu *vcpu) 527 { 528 const unsigned long *isa = vcpu->arch.isa; 529 struct kvm_vcpu_config *cfg = &vcpu->arch.cfg; 530 531 if (riscv_isa_extension_available(isa, SVPBMT)) 532 cfg->henvcfg |= ENVCFG_PBMTE; 533 534 if (riscv_isa_extension_available(isa, SSTC)) 535 cfg->henvcfg |= ENVCFG_STCE; 536 537 if (riscv_isa_extension_available(isa, ZICBOM)) 538 cfg->henvcfg |= (ENVCFG_CBIE | ENVCFG_CBCFE); 539 540 if (riscv_isa_extension_available(isa, ZICBOZ)) 541 cfg->henvcfg |= ENVCFG_CBZE; 542 543 if (riscv_has_extension_unlikely(RISCV_ISA_EXT_SMSTATEEN)) { 544 cfg->hstateen0 |= SMSTATEEN0_HSENVCFG; 545 if (riscv_isa_extension_available(isa, SSAIA)) 546 cfg->hstateen0 |= SMSTATEEN0_AIA_IMSIC | 547 SMSTATEEN0_AIA | 548 SMSTATEEN0_AIA_ISEL; 549 if (riscv_isa_extension_available(isa, SMSTATEEN)) 550 cfg->hstateen0 |= SMSTATEEN0_SSTATEEN0; 551 } 552 553 cfg->hedeleg = KVM_HEDELEG_DEFAULT; 554 if (vcpu->guest_debug) 555 cfg->hedeleg &= ~BIT(EXC_BREAKPOINT); 556 } 557 558 void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu) 559 { 560 struct kvm_vcpu_csr *csr = &vcpu->arch.guest_csr; 561 struct kvm_vcpu_config *cfg = &vcpu->arch.cfg; 562 563 csr_write(CSR_VSSTATUS, csr->vsstatus); 564 csr_write(CSR_VSIE, csr->vsie); 565 csr_write(CSR_VSTVEC, csr->vstvec); 566 csr_write(CSR_VSSCRATCH, csr->vsscratch); 567 csr_write(CSR_VSEPC, csr->vsepc); 568 csr_write(CSR_VSCAUSE, csr->vscause); 569 csr_write(CSR_VSTVAL, csr->vstval); 570 csr_write(CSR_HEDELEG, cfg->hedeleg); 571 csr_write(CSR_HVIP, csr->hvip); 572 csr_write(CSR_VSATP, csr->vsatp); 573 csr_write(CSR_HENVCFG, cfg->henvcfg); 574 if (IS_ENABLED(CONFIG_32BIT)) 575 csr_write(CSR_HENVCFGH, cfg->henvcfg >> 32); 576 if (riscv_has_extension_unlikely(RISCV_ISA_EXT_SMSTATEEN)) { 577 csr_write(CSR_HSTATEEN0, cfg->hstateen0); 578 if (IS_ENABLED(CONFIG_32BIT)) 579 csr_write(CSR_HSTATEEN0H, cfg->hstateen0 >> 32); 580 } 581 582 kvm_riscv_gstage_update_hgatp(vcpu); 583 584 kvm_riscv_vcpu_timer_restore(vcpu); 585 586 kvm_riscv_vcpu_host_fp_save(&vcpu->arch.host_context); 587 kvm_riscv_vcpu_guest_fp_restore(&vcpu->arch.guest_context, 588 vcpu->arch.isa); 589 kvm_riscv_vcpu_host_vector_save(&vcpu->arch.host_context); 590 kvm_riscv_vcpu_guest_vector_restore(&vcpu->arch.guest_context, 591 vcpu->arch.isa); 592 593 kvm_riscv_vcpu_aia_load(vcpu, cpu); 594 595 kvm_make_request(KVM_REQ_STEAL_UPDATE, vcpu); 596 597 vcpu->cpu = cpu; 598 } 599 600 void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu) 601 { 602 struct kvm_vcpu_csr *csr = &vcpu->arch.guest_csr; 603 604 vcpu->cpu = -1; 605 606 kvm_riscv_vcpu_aia_put(vcpu); 607 608 kvm_riscv_vcpu_guest_fp_save(&vcpu->arch.guest_context, 609 vcpu->arch.isa); 610 kvm_riscv_vcpu_host_fp_restore(&vcpu->arch.host_context); 611 612 kvm_riscv_vcpu_timer_save(vcpu); 613 kvm_riscv_vcpu_guest_vector_save(&vcpu->arch.guest_context, 614 vcpu->arch.isa); 615 kvm_riscv_vcpu_host_vector_restore(&vcpu->arch.host_context); 616 617 csr->vsstatus = csr_read(CSR_VSSTATUS); 618 csr->vsie = csr_read(CSR_VSIE); 619 csr->vstvec = csr_read(CSR_VSTVEC); 620 csr->vsscratch = csr_read(CSR_VSSCRATCH); 621 csr->vsepc = csr_read(CSR_VSEPC); 622 csr->vscause = csr_read(CSR_VSCAUSE); 623 csr->vstval = csr_read(CSR_VSTVAL); 624 csr->hvip = csr_read(CSR_HVIP); 625 csr->vsatp = csr_read(CSR_VSATP); 626 } 627 628 static void kvm_riscv_check_vcpu_requests(struct kvm_vcpu *vcpu) 629 { 630 struct rcuwait *wait = kvm_arch_vcpu_get_wait(vcpu); 631 632 if (kvm_request_pending(vcpu)) { 633 if (kvm_check_request(KVM_REQ_SLEEP, vcpu)) { 634 kvm_vcpu_srcu_read_unlock(vcpu); 635 rcuwait_wait_event(wait, 636 (!kvm_riscv_vcpu_stopped(vcpu)) && (!vcpu->arch.pause), 637 TASK_INTERRUPTIBLE); 638 kvm_vcpu_srcu_read_lock(vcpu); 639 640 if (kvm_riscv_vcpu_stopped(vcpu) || vcpu->arch.pause) { 641 /* 642 * Awaken to handle a signal, request to 643 * sleep again later. 644 */ 645 kvm_make_request(KVM_REQ_SLEEP, vcpu); 646 } 647 } 648 649 if (kvm_check_request(KVM_REQ_VCPU_RESET, vcpu)) 650 kvm_riscv_reset_vcpu(vcpu); 651 652 if (kvm_check_request(KVM_REQ_UPDATE_HGATP, vcpu)) 653 kvm_riscv_gstage_update_hgatp(vcpu); 654 655 if (kvm_check_request(KVM_REQ_FENCE_I, vcpu)) 656 kvm_riscv_fence_i_process(vcpu); 657 658 /* 659 * The generic KVM_REQ_TLB_FLUSH is same as 660 * KVM_REQ_HFENCE_GVMA_VMID_ALL 661 */ 662 if (kvm_check_request(KVM_REQ_HFENCE_GVMA_VMID_ALL, vcpu)) 663 kvm_riscv_hfence_gvma_vmid_all_process(vcpu); 664 665 if (kvm_check_request(KVM_REQ_HFENCE_VVMA_ALL, vcpu)) 666 kvm_riscv_hfence_vvma_all_process(vcpu); 667 668 if (kvm_check_request(KVM_REQ_HFENCE, vcpu)) 669 kvm_riscv_hfence_process(vcpu); 670 671 if (kvm_check_request(KVM_REQ_STEAL_UPDATE, vcpu)) 672 kvm_riscv_vcpu_record_steal_time(vcpu); 673 } 674 } 675 676 static void kvm_riscv_update_hvip(struct kvm_vcpu *vcpu) 677 { 678 struct kvm_vcpu_csr *csr = &vcpu->arch.guest_csr; 679 680 csr_write(CSR_HVIP, csr->hvip); 681 kvm_riscv_vcpu_aia_update_hvip(vcpu); 682 } 683 684 static __always_inline void kvm_riscv_vcpu_swap_in_guest_state(struct kvm_vcpu *vcpu) 685 { 686 struct kvm_vcpu_smstateen_csr *smcsr = &vcpu->arch.smstateen_csr; 687 struct kvm_vcpu_csr *csr = &vcpu->arch.guest_csr; 688 struct kvm_vcpu_config *cfg = &vcpu->arch.cfg; 689 690 vcpu->arch.host_senvcfg = csr_swap(CSR_SENVCFG, csr->senvcfg); 691 if (riscv_has_extension_unlikely(RISCV_ISA_EXT_SMSTATEEN) && 692 (cfg->hstateen0 & SMSTATEEN0_SSTATEEN0)) 693 vcpu->arch.host_sstateen0 = csr_swap(CSR_SSTATEEN0, 694 smcsr->sstateen0); 695 } 696 697 static __always_inline void kvm_riscv_vcpu_swap_in_host_state(struct kvm_vcpu *vcpu) 698 { 699 struct kvm_vcpu_smstateen_csr *smcsr = &vcpu->arch.smstateen_csr; 700 struct kvm_vcpu_csr *csr = &vcpu->arch.guest_csr; 701 struct kvm_vcpu_config *cfg = &vcpu->arch.cfg; 702 703 csr->senvcfg = csr_swap(CSR_SENVCFG, vcpu->arch.host_senvcfg); 704 if (riscv_has_extension_unlikely(RISCV_ISA_EXT_SMSTATEEN) && 705 (cfg->hstateen0 & SMSTATEEN0_SSTATEEN0)) 706 smcsr->sstateen0 = csr_swap(CSR_SSTATEEN0, 707 vcpu->arch.host_sstateen0); 708 } 709 710 /* 711 * Actually run the vCPU, entering an RCU extended quiescent state (EQS) while 712 * the vCPU is running. 713 * 714 * This must be noinstr as instrumentation may make use of RCU, and this is not 715 * safe during the EQS. 716 */ 717 static void noinstr kvm_riscv_vcpu_enter_exit(struct kvm_vcpu *vcpu) 718 { 719 kvm_riscv_vcpu_swap_in_guest_state(vcpu); 720 guest_state_enter_irqoff(); 721 __kvm_riscv_switch_to(&vcpu->arch); 722 vcpu->arch.last_exit_cpu = vcpu->cpu; 723 guest_state_exit_irqoff(); 724 kvm_riscv_vcpu_swap_in_host_state(vcpu); 725 } 726 727 int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu) 728 { 729 int ret; 730 struct kvm_cpu_trap trap; 731 struct kvm_run *run = vcpu->run; 732 733 if (!vcpu->arch.ran_atleast_once) 734 kvm_riscv_vcpu_setup_config(vcpu); 735 736 /* Mark this VCPU ran at least once */ 737 vcpu->arch.ran_atleast_once = true; 738 739 kvm_vcpu_srcu_read_lock(vcpu); 740 741 switch (run->exit_reason) { 742 case KVM_EXIT_MMIO: 743 /* Process MMIO value returned from user-space */ 744 ret = kvm_riscv_vcpu_mmio_return(vcpu, vcpu->run); 745 break; 746 case KVM_EXIT_RISCV_SBI: 747 /* Process SBI value returned from user-space */ 748 ret = kvm_riscv_vcpu_sbi_return(vcpu, vcpu->run); 749 break; 750 case KVM_EXIT_RISCV_CSR: 751 /* Process CSR value returned from user-space */ 752 ret = kvm_riscv_vcpu_csr_return(vcpu, vcpu->run); 753 break; 754 default: 755 ret = 0; 756 break; 757 } 758 if (ret) { 759 kvm_vcpu_srcu_read_unlock(vcpu); 760 return ret; 761 } 762 763 if (run->immediate_exit) { 764 kvm_vcpu_srcu_read_unlock(vcpu); 765 return -EINTR; 766 } 767 768 vcpu_load(vcpu); 769 770 kvm_sigset_activate(vcpu); 771 772 ret = 1; 773 run->exit_reason = KVM_EXIT_UNKNOWN; 774 while (ret > 0) { 775 /* Check conditions before entering the guest */ 776 ret = xfer_to_guest_mode_handle_work(vcpu); 777 if (ret) 778 continue; 779 ret = 1; 780 781 kvm_riscv_gstage_vmid_update(vcpu); 782 783 kvm_riscv_check_vcpu_requests(vcpu); 784 785 preempt_disable(); 786 787 /* Update AIA HW state before entering guest */ 788 ret = kvm_riscv_vcpu_aia_update(vcpu); 789 if (ret <= 0) { 790 preempt_enable(); 791 continue; 792 } 793 794 local_irq_disable(); 795 796 /* 797 * Ensure we set mode to IN_GUEST_MODE after we disable 798 * interrupts and before the final VCPU requests check. 799 * See the comment in kvm_vcpu_exiting_guest_mode() and 800 * Documentation/virt/kvm/vcpu-requests.rst 801 */ 802 vcpu->mode = IN_GUEST_MODE; 803 804 kvm_vcpu_srcu_read_unlock(vcpu); 805 smp_mb__after_srcu_read_unlock(); 806 807 /* 808 * We might have got VCPU interrupts updated asynchronously 809 * so update it in HW. 810 */ 811 kvm_riscv_vcpu_flush_interrupts(vcpu); 812 813 /* Update HVIP CSR for current CPU */ 814 kvm_riscv_update_hvip(vcpu); 815 816 if (kvm_riscv_gstage_vmid_ver_changed(&vcpu->kvm->arch.vmid) || 817 kvm_request_pending(vcpu) || 818 xfer_to_guest_mode_work_pending()) { 819 vcpu->mode = OUTSIDE_GUEST_MODE; 820 local_irq_enable(); 821 preempt_enable(); 822 kvm_vcpu_srcu_read_lock(vcpu); 823 continue; 824 } 825 826 /* 827 * Cleanup stale TLB enteries 828 * 829 * Note: This should be done after G-stage VMID has been 830 * updated using kvm_riscv_gstage_vmid_ver_changed() 831 */ 832 kvm_riscv_local_tlb_sanitize(vcpu); 833 834 guest_timing_enter_irqoff(); 835 836 kvm_riscv_vcpu_enter_exit(vcpu); 837 838 vcpu->mode = OUTSIDE_GUEST_MODE; 839 vcpu->stat.exits++; 840 841 /* 842 * Save SCAUSE, STVAL, HTVAL, and HTINST because we might 843 * get an interrupt between __kvm_riscv_switch_to() and 844 * local_irq_enable() which can potentially change CSRs. 845 */ 846 trap.sepc = vcpu->arch.guest_context.sepc; 847 trap.scause = csr_read(CSR_SCAUSE); 848 trap.stval = csr_read(CSR_STVAL); 849 trap.htval = csr_read(CSR_HTVAL); 850 trap.htinst = csr_read(CSR_HTINST); 851 852 /* Syncup interrupts state with HW */ 853 kvm_riscv_vcpu_sync_interrupts(vcpu); 854 855 /* 856 * We must ensure that any pending interrupts are taken before 857 * we exit guest timing so that timer ticks are accounted as 858 * guest time. Transiently unmask interrupts so that any 859 * pending interrupts are taken. 860 * 861 * There's no barrier which ensures that pending interrupts are 862 * recognised, so we just hope that the CPU takes any pending 863 * interrupts between the enable and disable. 864 */ 865 local_irq_enable(); 866 local_irq_disable(); 867 868 guest_timing_exit_irqoff(); 869 870 local_irq_enable(); 871 872 preempt_enable(); 873 874 kvm_vcpu_srcu_read_lock(vcpu); 875 876 ret = kvm_riscv_vcpu_exit(vcpu, run, &trap); 877 } 878 879 kvm_sigset_deactivate(vcpu); 880 881 vcpu_put(vcpu); 882 883 kvm_vcpu_srcu_read_unlock(vcpu); 884 885 return ret; 886 } 887