1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (C) 2019 Western Digital Corporation or its affiliates. 4 * 5 * Authors: 6 * Anup Patel <anup.patel@wdc.com> 7 */ 8 9 #include <linux/bitops.h> 10 #include <linux/entry-kvm.h> 11 #include <linux/errno.h> 12 #include <linux/err.h> 13 #include <linux/kdebug.h> 14 #include <linux/module.h> 15 #include <linux/percpu.h> 16 #include <linux/vmalloc.h> 17 #include <linux/sched/signal.h> 18 #include <linux/fs.h> 19 #include <linux/kvm_host.h> 20 #include <asm/csr.h> 21 #include <asm/cacheflush.h> 22 #include <asm/kvm_vcpu_vector.h> 23 24 #define CREATE_TRACE_POINTS 25 #include "trace.h" 26 27 const struct _kvm_stats_desc kvm_vcpu_stats_desc[] = { 28 KVM_GENERIC_VCPU_STATS(), 29 STATS_DESC_COUNTER(VCPU, ecall_exit_stat), 30 STATS_DESC_COUNTER(VCPU, wfi_exit_stat), 31 STATS_DESC_COUNTER(VCPU, mmio_exit_user), 32 STATS_DESC_COUNTER(VCPU, mmio_exit_kernel), 33 STATS_DESC_COUNTER(VCPU, csr_exit_user), 34 STATS_DESC_COUNTER(VCPU, csr_exit_kernel), 35 STATS_DESC_COUNTER(VCPU, signal_exits), 36 STATS_DESC_COUNTER(VCPU, exits) 37 }; 38 39 const struct kvm_stats_header kvm_vcpu_stats_header = { 40 .name_size = KVM_STATS_NAME_SIZE, 41 .num_desc = ARRAY_SIZE(kvm_vcpu_stats_desc), 42 .id_offset = sizeof(struct kvm_stats_header), 43 .desc_offset = sizeof(struct kvm_stats_header) + KVM_STATS_NAME_SIZE, 44 .data_offset = sizeof(struct kvm_stats_header) + KVM_STATS_NAME_SIZE + 45 sizeof(kvm_vcpu_stats_desc), 46 }; 47 48 static void kvm_riscv_reset_vcpu(struct kvm_vcpu *vcpu) 49 { 50 struct kvm_vcpu_csr *csr = &vcpu->arch.guest_csr; 51 struct kvm_vcpu_csr *reset_csr = &vcpu->arch.guest_reset_csr; 52 struct kvm_cpu_context *cntx = &vcpu->arch.guest_context; 53 struct kvm_cpu_context *reset_cntx = &vcpu->arch.guest_reset_context; 54 bool loaded; 55 56 /** 57 * The preemption should be disabled here because it races with 58 * kvm_sched_out/kvm_sched_in(called from preempt notifiers) which 59 * also calls vcpu_load/put. 60 */ 61 get_cpu(); 62 loaded = (vcpu->cpu != -1); 63 if (loaded) 64 kvm_arch_vcpu_put(vcpu); 65 66 vcpu->arch.last_exit_cpu = -1; 67 68 memcpy(csr, reset_csr, sizeof(*csr)); 69 70 spin_lock(&vcpu->arch.reset_cntx_lock); 71 memcpy(cntx, reset_cntx, sizeof(*cntx)); 72 spin_unlock(&vcpu->arch.reset_cntx_lock); 73 74 kvm_riscv_vcpu_fp_reset(vcpu); 75 76 kvm_riscv_vcpu_vector_reset(vcpu); 77 78 kvm_riscv_vcpu_timer_reset(vcpu); 79 80 kvm_riscv_vcpu_aia_reset(vcpu); 81 82 bitmap_zero(vcpu->arch.irqs_pending, KVM_RISCV_VCPU_NR_IRQS); 83 bitmap_zero(vcpu->arch.irqs_pending_mask, KVM_RISCV_VCPU_NR_IRQS); 84 85 kvm_riscv_vcpu_pmu_reset(vcpu); 86 87 vcpu->arch.hfence_head = 0; 88 vcpu->arch.hfence_tail = 0; 89 memset(vcpu->arch.hfence_queue, 0, sizeof(vcpu->arch.hfence_queue)); 90 91 kvm_riscv_vcpu_sbi_sta_reset(vcpu); 92 93 /* Reset the guest CSRs for hotplug usecase */ 94 if (loaded) 95 kvm_arch_vcpu_load(vcpu, smp_processor_id()); 96 put_cpu(); 97 } 98 99 int kvm_arch_vcpu_precreate(struct kvm *kvm, unsigned int id) 100 { 101 return 0; 102 } 103 104 int kvm_arch_vcpu_create(struct kvm_vcpu *vcpu) 105 { 106 int rc; 107 struct kvm_cpu_context *cntx; 108 struct kvm_vcpu_csr *reset_csr = &vcpu->arch.guest_reset_csr; 109 110 spin_lock_init(&vcpu->arch.mp_state_lock); 111 112 /* Mark this VCPU never ran */ 113 vcpu->arch.ran_atleast_once = false; 114 vcpu->arch.mmu_page_cache.gfp_zero = __GFP_ZERO; 115 bitmap_zero(vcpu->arch.isa, RISCV_ISA_EXT_MAX); 116 117 /* Setup ISA features available to VCPU */ 118 kvm_riscv_vcpu_setup_isa(vcpu); 119 120 /* Setup vendor, arch, and implementation details */ 121 vcpu->arch.mvendorid = sbi_get_mvendorid(); 122 vcpu->arch.marchid = sbi_get_marchid(); 123 vcpu->arch.mimpid = sbi_get_mimpid(); 124 125 /* Setup VCPU hfence queue */ 126 spin_lock_init(&vcpu->arch.hfence_lock); 127 128 /* Setup reset state of shadow SSTATUS and HSTATUS CSRs */ 129 spin_lock_init(&vcpu->arch.reset_cntx_lock); 130 131 spin_lock(&vcpu->arch.reset_cntx_lock); 132 cntx = &vcpu->arch.guest_reset_context; 133 cntx->sstatus = SR_SPP | SR_SPIE; 134 cntx->hstatus = 0; 135 cntx->hstatus |= HSTATUS_VTW; 136 cntx->hstatus |= HSTATUS_SPVP; 137 cntx->hstatus |= HSTATUS_SPV; 138 spin_unlock(&vcpu->arch.reset_cntx_lock); 139 140 if (kvm_riscv_vcpu_alloc_vector_context(vcpu, cntx)) 141 return -ENOMEM; 142 143 /* By default, make CY, TM, and IR counters accessible in VU mode */ 144 reset_csr->scounteren = 0x7; 145 146 /* Setup VCPU timer */ 147 kvm_riscv_vcpu_timer_init(vcpu); 148 149 /* setup performance monitoring */ 150 kvm_riscv_vcpu_pmu_init(vcpu); 151 152 /* Setup VCPU AIA */ 153 rc = kvm_riscv_vcpu_aia_init(vcpu); 154 if (rc) 155 return rc; 156 157 /* 158 * Setup SBI extensions 159 * NOTE: This must be the last thing to be initialized. 160 */ 161 kvm_riscv_vcpu_sbi_init(vcpu); 162 163 /* Reset VCPU */ 164 kvm_riscv_reset_vcpu(vcpu); 165 166 return 0; 167 } 168 169 void kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu) 170 { 171 /** 172 * vcpu with id 0 is the designated boot cpu. 173 * Keep all vcpus with non-zero id in power-off state so that 174 * they can be brought up using SBI HSM extension. 175 */ 176 if (vcpu->vcpu_idx != 0) 177 kvm_riscv_vcpu_power_off(vcpu); 178 } 179 180 void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu) 181 { 182 /* Cleanup VCPU AIA context */ 183 kvm_riscv_vcpu_aia_deinit(vcpu); 184 185 /* Cleanup VCPU timer */ 186 kvm_riscv_vcpu_timer_deinit(vcpu); 187 188 kvm_riscv_vcpu_pmu_deinit(vcpu); 189 190 /* Free unused pages pre-allocated for G-stage page table mappings */ 191 kvm_mmu_free_memory_cache(&vcpu->arch.mmu_page_cache); 192 193 /* Free vector context space for host and guest kernel */ 194 kvm_riscv_vcpu_free_vector_context(vcpu); 195 } 196 197 int kvm_cpu_has_pending_timer(struct kvm_vcpu *vcpu) 198 { 199 return kvm_riscv_vcpu_timer_pending(vcpu); 200 } 201 202 void kvm_arch_vcpu_blocking(struct kvm_vcpu *vcpu) 203 { 204 kvm_riscv_aia_wakeon_hgei(vcpu, true); 205 } 206 207 void kvm_arch_vcpu_unblocking(struct kvm_vcpu *vcpu) 208 { 209 kvm_riscv_aia_wakeon_hgei(vcpu, false); 210 } 211 212 int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu) 213 { 214 return (kvm_riscv_vcpu_has_interrupts(vcpu, -1UL) && 215 !kvm_riscv_vcpu_stopped(vcpu) && !vcpu->arch.pause); 216 } 217 218 int kvm_arch_vcpu_should_kick(struct kvm_vcpu *vcpu) 219 { 220 return kvm_vcpu_exiting_guest_mode(vcpu) == IN_GUEST_MODE; 221 } 222 223 bool kvm_arch_vcpu_in_kernel(struct kvm_vcpu *vcpu) 224 { 225 return (vcpu->arch.guest_context.sstatus & SR_SPP) ? true : false; 226 } 227 228 vm_fault_t kvm_arch_vcpu_fault(struct kvm_vcpu *vcpu, struct vm_fault *vmf) 229 { 230 return VM_FAULT_SIGBUS; 231 } 232 233 long kvm_arch_vcpu_async_ioctl(struct file *filp, 234 unsigned int ioctl, unsigned long arg) 235 { 236 struct kvm_vcpu *vcpu = filp->private_data; 237 void __user *argp = (void __user *)arg; 238 239 if (ioctl == KVM_INTERRUPT) { 240 struct kvm_interrupt irq; 241 242 if (copy_from_user(&irq, argp, sizeof(irq))) 243 return -EFAULT; 244 245 if (irq.irq == KVM_INTERRUPT_SET) 246 return kvm_riscv_vcpu_set_interrupt(vcpu, IRQ_VS_EXT); 247 else 248 return kvm_riscv_vcpu_unset_interrupt(vcpu, IRQ_VS_EXT); 249 } 250 251 return -ENOIOCTLCMD; 252 } 253 254 long kvm_arch_vcpu_ioctl(struct file *filp, 255 unsigned int ioctl, unsigned long arg) 256 { 257 struct kvm_vcpu *vcpu = filp->private_data; 258 void __user *argp = (void __user *)arg; 259 long r = -EINVAL; 260 261 switch (ioctl) { 262 case KVM_SET_ONE_REG: 263 case KVM_GET_ONE_REG: { 264 struct kvm_one_reg reg; 265 266 r = -EFAULT; 267 if (copy_from_user(®, argp, sizeof(reg))) 268 break; 269 270 if (ioctl == KVM_SET_ONE_REG) 271 r = kvm_riscv_vcpu_set_reg(vcpu, ®); 272 else 273 r = kvm_riscv_vcpu_get_reg(vcpu, ®); 274 break; 275 } 276 case KVM_GET_REG_LIST: { 277 struct kvm_reg_list __user *user_list = argp; 278 struct kvm_reg_list reg_list; 279 unsigned int n; 280 281 r = -EFAULT; 282 if (copy_from_user(®_list, user_list, sizeof(reg_list))) 283 break; 284 n = reg_list.n; 285 reg_list.n = kvm_riscv_vcpu_num_regs(vcpu); 286 if (copy_to_user(user_list, ®_list, sizeof(reg_list))) 287 break; 288 r = -E2BIG; 289 if (n < reg_list.n) 290 break; 291 r = kvm_riscv_vcpu_copy_reg_indices(vcpu, user_list->reg); 292 break; 293 } 294 default: 295 break; 296 } 297 298 return r; 299 } 300 301 int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu, 302 struct kvm_sregs *sregs) 303 { 304 return -EINVAL; 305 } 306 307 int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu, 308 struct kvm_sregs *sregs) 309 { 310 return -EINVAL; 311 } 312 313 int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu) 314 { 315 return -EINVAL; 316 } 317 318 int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu) 319 { 320 return -EINVAL; 321 } 322 323 int kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu *vcpu, 324 struct kvm_translation *tr) 325 { 326 return -EINVAL; 327 } 328 329 int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs) 330 { 331 return -EINVAL; 332 } 333 334 int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs) 335 { 336 return -EINVAL; 337 } 338 339 void kvm_riscv_vcpu_flush_interrupts(struct kvm_vcpu *vcpu) 340 { 341 struct kvm_vcpu_csr *csr = &vcpu->arch.guest_csr; 342 unsigned long mask, val; 343 344 if (READ_ONCE(vcpu->arch.irqs_pending_mask[0])) { 345 mask = xchg_acquire(&vcpu->arch.irqs_pending_mask[0], 0); 346 val = READ_ONCE(vcpu->arch.irqs_pending[0]) & mask; 347 348 csr->hvip &= ~mask; 349 csr->hvip |= val; 350 } 351 352 /* Flush AIA high interrupts */ 353 kvm_riscv_vcpu_aia_flush_interrupts(vcpu); 354 } 355 356 void kvm_riscv_vcpu_sync_interrupts(struct kvm_vcpu *vcpu) 357 { 358 unsigned long hvip; 359 struct kvm_vcpu_arch *v = &vcpu->arch; 360 struct kvm_vcpu_csr *csr = &vcpu->arch.guest_csr; 361 362 /* Read current HVIP and VSIE CSRs */ 363 csr->vsie = csr_read(CSR_VSIE); 364 365 /* Sync-up HVIP.VSSIP bit changes does by Guest */ 366 hvip = csr_read(CSR_HVIP); 367 if ((csr->hvip ^ hvip) & (1UL << IRQ_VS_SOFT)) { 368 if (hvip & (1UL << IRQ_VS_SOFT)) { 369 if (!test_and_set_bit(IRQ_VS_SOFT, 370 v->irqs_pending_mask)) 371 set_bit(IRQ_VS_SOFT, v->irqs_pending); 372 } else { 373 if (!test_and_set_bit(IRQ_VS_SOFT, 374 v->irqs_pending_mask)) 375 clear_bit(IRQ_VS_SOFT, v->irqs_pending); 376 } 377 } 378 379 /* Sync up the HVIP.LCOFIP bit changes (only clear) by the guest */ 380 if ((csr->hvip ^ hvip) & (1UL << IRQ_PMU_OVF)) { 381 if (!(hvip & (1UL << IRQ_PMU_OVF)) && 382 !test_and_set_bit(IRQ_PMU_OVF, v->irqs_pending_mask)) 383 clear_bit(IRQ_PMU_OVF, v->irqs_pending); 384 } 385 386 /* Sync-up AIA high interrupts */ 387 kvm_riscv_vcpu_aia_sync_interrupts(vcpu); 388 389 /* Sync-up timer CSRs */ 390 kvm_riscv_vcpu_timer_sync(vcpu); 391 } 392 393 int kvm_riscv_vcpu_set_interrupt(struct kvm_vcpu *vcpu, unsigned int irq) 394 { 395 /* 396 * We only allow VS-mode software, timer, and external 397 * interrupts when irq is one of the local interrupts 398 * defined by RISC-V privilege specification. 399 */ 400 if (irq < IRQ_LOCAL_MAX && 401 irq != IRQ_VS_SOFT && 402 irq != IRQ_VS_TIMER && 403 irq != IRQ_VS_EXT && 404 irq != IRQ_PMU_OVF) 405 return -EINVAL; 406 407 set_bit(irq, vcpu->arch.irqs_pending); 408 smp_mb__before_atomic(); 409 set_bit(irq, vcpu->arch.irqs_pending_mask); 410 411 kvm_vcpu_kick(vcpu); 412 413 return 0; 414 } 415 416 int kvm_riscv_vcpu_unset_interrupt(struct kvm_vcpu *vcpu, unsigned int irq) 417 { 418 /* 419 * We only allow VS-mode software, timer, counter overflow and external 420 * interrupts when irq is one of the local interrupts 421 * defined by RISC-V privilege specification. 422 */ 423 if (irq < IRQ_LOCAL_MAX && 424 irq != IRQ_VS_SOFT && 425 irq != IRQ_VS_TIMER && 426 irq != IRQ_VS_EXT && 427 irq != IRQ_PMU_OVF) 428 return -EINVAL; 429 430 clear_bit(irq, vcpu->arch.irqs_pending); 431 smp_mb__before_atomic(); 432 set_bit(irq, vcpu->arch.irqs_pending_mask); 433 434 return 0; 435 } 436 437 bool kvm_riscv_vcpu_has_interrupts(struct kvm_vcpu *vcpu, u64 mask) 438 { 439 unsigned long ie; 440 441 ie = ((vcpu->arch.guest_csr.vsie & VSIP_VALID_MASK) 442 << VSIP_TO_HVIP_SHIFT) & (unsigned long)mask; 443 ie |= vcpu->arch.guest_csr.vsie & ~IRQ_LOCAL_MASK & 444 (unsigned long)mask; 445 if (READ_ONCE(vcpu->arch.irqs_pending[0]) & ie) 446 return true; 447 448 /* Check AIA high interrupts */ 449 return kvm_riscv_vcpu_aia_has_interrupts(vcpu, mask); 450 } 451 452 void __kvm_riscv_vcpu_power_off(struct kvm_vcpu *vcpu) 453 { 454 WRITE_ONCE(vcpu->arch.mp_state.mp_state, KVM_MP_STATE_STOPPED); 455 kvm_make_request(KVM_REQ_SLEEP, vcpu); 456 kvm_vcpu_kick(vcpu); 457 } 458 459 void kvm_riscv_vcpu_power_off(struct kvm_vcpu *vcpu) 460 { 461 spin_lock(&vcpu->arch.mp_state_lock); 462 __kvm_riscv_vcpu_power_off(vcpu); 463 spin_unlock(&vcpu->arch.mp_state_lock); 464 } 465 466 void __kvm_riscv_vcpu_power_on(struct kvm_vcpu *vcpu) 467 { 468 WRITE_ONCE(vcpu->arch.mp_state.mp_state, KVM_MP_STATE_RUNNABLE); 469 kvm_vcpu_wake_up(vcpu); 470 } 471 472 void kvm_riscv_vcpu_power_on(struct kvm_vcpu *vcpu) 473 { 474 spin_lock(&vcpu->arch.mp_state_lock); 475 __kvm_riscv_vcpu_power_on(vcpu); 476 spin_unlock(&vcpu->arch.mp_state_lock); 477 } 478 479 bool kvm_riscv_vcpu_stopped(struct kvm_vcpu *vcpu) 480 { 481 return READ_ONCE(vcpu->arch.mp_state.mp_state) == KVM_MP_STATE_STOPPED; 482 } 483 484 int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu, 485 struct kvm_mp_state *mp_state) 486 { 487 *mp_state = READ_ONCE(vcpu->arch.mp_state); 488 489 return 0; 490 } 491 492 int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu, 493 struct kvm_mp_state *mp_state) 494 { 495 int ret = 0; 496 497 spin_lock(&vcpu->arch.mp_state_lock); 498 499 switch (mp_state->mp_state) { 500 case KVM_MP_STATE_RUNNABLE: 501 WRITE_ONCE(vcpu->arch.mp_state, *mp_state); 502 break; 503 case KVM_MP_STATE_STOPPED: 504 __kvm_riscv_vcpu_power_off(vcpu); 505 break; 506 default: 507 ret = -EINVAL; 508 } 509 510 spin_unlock(&vcpu->arch.mp_state_lock); 511 512 return ret; 513 } 514 515 int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu, 516 struct kvm_guest_debug *dbg) 517 { 518 if (dbg->control & KVM_GUESTDBG_ENABLE) { 519 vcpu->guest_debug = dbg->control; 520 vcpu->arch.cfg.hedeleg &= ~BIT(EXC_BREAKPOINT); 521 } else { 522 vcpu->guest_debug = 0; 523 vcpu->arch.cfg.hedeleg |= BIT(EXC_BREAKPOINT); 524 } 525 526 return 0; 527 } 528 529 static void kvm_riscv_vcpu_setup_config(struct kvm_vcpu *vcpu) 530 { 531 const unsigned long *isa = vcpu->arch.isa; 532 struct kvm_vcpu_config *cfg = &vcpu->arch.cfg; 533 534 if (riscv_isa_extension_available(isa, SVPBMT)) 535 cfg->henvcfg |= ENVCFG_PBMTE; 536 537 if (riscv_isa_extension_available(isa, SSTC)) 538 cfg->henvcfg |= ENVCFG_STCE; 539 540 if (riscv_isa_extension_available(isa, ZICBOM)) 541 cfg->henvcfg |= (ENVCFG_CBIE | ENVCFG_CBCFE); 542 543 if (riscv_isa_extension_available(isa, ZICBOZ)) 544 cfg->henvcfg |= ENVCFG_CBZE; 545 546 if (riscv_has_extension_unlikely(RISCV_ISA_EXT_SMSTATEEN)) { 547 cfg->hstateen0 |= SMSTATEEN0_HSENVCFG; 548 if (riscv_isa_extension_available(isa, SSAIA)) 549 cfg->hstateen0 |= SMSTATEEN0_AIA_IMSIC | 550 SMSTATEEN0_AIA | 551 SMSTATEEN0_AIA_ISEL; 552 if (riscv_isa_extension_available(isa, SMSTATEEN)) 553 cfg->hstateen0 |= SMSTATEEN0_SSTATEEN0; 554 } 555 556 cfg->hedeleg = KVM_HEDELEG_DEFAULT; 557 if (vcpu->guest_debug) 558 cfg->hedeleg &= ~BIT(EXC_BREAKPOINT); 559 } 560 561 void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu) 562 { 563 struct kvm_vcpu_csr *csr = &vcpu->arch.guest_csr; 564 struct kvm_vcpu_config *cfg = &vcpu->arch.cfg; 565 566 csr_write(CSR_VSSTATUS, csr->vsstatus); 567 csr_write(CSR_VSIE, csr->vsie); 568 csr_write(CSR_VSTVEC, csr->vstvec); 569 csr_write(CSR_VSSCRATCH, csr->vsscratch); 570 csr_write(CSR_VSEPC, csr->vsepc); 571 csr_write(CSR_VSCAUSE, csr->vscause); 572 csr_write(CSR_VSTVAL, csr->vstval); 573 csr_write(CSR_HEDELEG, cfg->hedeleg); 574 csr_write(CSR_HVIP, csr->hvip); 575 csr_write(CSR_VSATP, csr->vsatp); 576 csr_write(CSR_HENVCFG, cfg->henvcfg); 577 if (IS_ENABLED(CONFIG_32BIT)) 578 csr_write(CSR_HENVCFGH, cfg->henvcfg >> 32); 579 if (riscv_has_extension_unlikely(RISCV_ISA_EXT_SMSTATEEN)) { 580 csr_write(CSR_HSTATEEN0, cfg->hstateen0); 581 if (IS_ENABLED(CONFIG_32BIT)) 582 csr_write(CSR_HSTATEEN0H, cfg->hstateen0 >> 32); 583 } 584 585 kvm_riscv_gstage_update_hgatp(vcpu); 586 587 kvm_riscv_vcpu_timer_restore(vcpu); 588 589 kvm_riscv_vcpu_host_fp_save(&vcpu->arch.host_context); 590 kvm_riscv_vcpu_guest_fp_restore(&vcpu->arch.guest_context, 591 vcpu->arch.isa); 592 kvm_riscv_vcpu_host_vector_save(&vcpu->arch.host_context); 593 kvm_riscv_vcpu_guest_vector_restore(&vcpu->arch.guest_context, 594 vcpu->arch.isa); 595 596 kvm_riscv_vcpu_aia_load(vcpu, cpu); 597 598 kvm_make_request(KVM_REQ_STEAL_UPDATE, vcpu); 599 600 vcpu->cpu = cpu; 601 } 602 603 void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu) 604 { 605 struct kvm_vcpu_csr *csr = &vcpu->arch.guest_csr; 606 607 vcpu->cpu = -1; 608 609 kvm_riscv_vcpu_aia_put(vcpu); 610 611 kvm_riscv_vcpu_guest_fp_save(&vcpu->arch.guest_context, 612 vcpu->arch.isa); 613 kvm_riscv_vcpu_host_fp_restore(&vcpu->arch.host_context); 614 615 kvm_riscv_vcpu_timer_save(vcpu); 616 kvm_riscv_vcpu_guest_vector_save(&vcpu->arch.guest_context, 617 vcpu->arch.isa); 618 kvm_riscv_vcpu_host_vector_restore(&vcpu->arch.host_context); 619 620 csr->vsstatus = csr_read(CSR_VSSTATUS); 621 csr->vsie = csr_read(CSR_VSIE); 622 csr->vstvec = csr_read(CSR_VSTVEC); 623 csr->vsscratch = csr_read(CSR_VSSCRATCH); 624 csr->vsepc = csr_read(CSR_VSEPC); 625 csr->vscause = csr_read(CSR_VSCAUSE); 626 csr->vstval = csr_read(CSR_VSTVAL); 627 csr->hvip = csr_read(CSR_HVIP); 628 csr->vsatp = csr_read(CSR_VSATP); 629 } 630 631 static void kvm_riscv_check_vcpu_requests(struct kvm_vcpu *vcpu) 632 { 633 struct rcuwait *wait = kvm_arch_vcpu_get_wait(vcpu); 634 635 if (kvm_request_pending(vcpu)) { 636 if (kvm_check_request(KVM_REQ_SLEEP, vcpu)) { 637 kvm_vcpu_srcu_read_unlock(vcpu); 638 rcuwait_wait_event(wait, 639 (!kvm_riscv_vcpu_stopped(vcpu)) && (!vcpu->arch.pause), 640 TASK_INTERRUPTIBLE); 641 kvm_vcpu_srcu_read_lock(vcpu); 642 643 if (kvm_riscv_vcpu_stopped(vcpu) || vcpu->arch.pause) { 644 /* 645 * Awaken to handle a signal, request to 646 * sleep again later. 647 */ 648 kvm_make_request(KVM_REQ_SLEEP, vcpu); 649 } 650 } 651 652 if (kvm_check_request(KVM_REQ_VCPU_RESET, vcpu)) 653 kvm_riscv_reset_vcpu(vcpu); 654 655 if (kvm_check_request(KVM_REQ_UPDATE_HGATP, vcpu)) 656 kvm_riscv_gstage_update_hgatp(vcpu); 657 658 if (kvm_check_request(KVM_REQ_FENCE_I, vcpu)) 659 kvm_riscv_fence_i_process(vcpu); 660 661 /* 662 * The generic KVM_REQ_TLB_FLUSH is same as 663 * KVM_REQ_HFENCE_GVMA_VMID_ALL 664 */ 665 if (kvm_check_request(KVM_REQ_HFENCE_GVMA_VMID_ALL, vcpu)) 666 kvm_riscv_hfence_gvma_vmid_all_process(vcpu); 667 668 if (kvm_check_request(KVM_REQ_HFENCE_VVMA_ALL, vcpu)) 669 kvm_riscv_hfence_vvma_all_process(vcpu); 670 671 if (kvm_check_request(KVM_REQ_HFENCE, vcpu)) 672 kvm_riscv_hfence_process(vcpu); 673 674 if (kvm_check_request(KVM_REQ_STEAL_UPDATE, vcpu)) 675 kvm_riscv_vcpu_record_steal_time(vcpu); 676 } 677 } 678 679 static void kvm_riscv_update_hvip(struct kvm_vcpu *vcpu) 680 { 681 struct kvm_vcpu_csr *csr = &vcpu->arch.guest_csr; 682 683 csr_write(CSR_HVIP, csr->hvip); 684 kvm_riscv_vcpu_aia_update_hvip(vcpu); 685 } 686 687 static __always_inline void kvm_riscv_vcpu_swap_in_guest_state(struct kvm_vcpu *vcpu) 688 { 689 struct kvm_vcpu_smstateen_csr *smcsr = &vcpu->arch.smstateen_csr; 690 struct kvm_vcpu_csr *csr = &vcpu->arch.guest_csr; 691 struct kvm_vcpu_config *cfg = &vcpu->arch.cfg; 692 693 vcpu->arch.host_senvcfg = csr_swap(CSR_SENVCFG, csr->senvcfg); 694 if (riscv_has_extension_unlikely(RISCV_ISA_EXT_SMSTATEEN) && 695 (cfg->hstateen0 & SMSTATEEN0_SSTATEEN0)) 696 vcpu->arch.host_sstateen0 = csr_swap(CSR_SSTATEEN0, 697 smcsr->sstateen0); 698 } 699 700 static __always_inline void kvm_riscv_vcpu_swap_in_host_state(struct kvm_vcpu *vcpu) 701 { 702 struct kvm_vcpu_smstateen_csr *smcsr = &vcpu->arch.smstateen_csr; 703 struct kvm_vcpu_csr *csr = &vcpu->arch.guest_csr; 704 struct kvm_vcpu_config *cfg = &vcpu->arch.cfg; 705 706 csr->senvcfg = csr_swap(CSR_SENVCFG, vcpu->arch.host_senvcfg); 707 if (riscv_has_extension_unlikely(RISCV_ISA_EXT_SMSTATEEN) && 708 (cfg->hstateen0 & SMSTATEEN0_SSTATEEN0)) 709 smcsr->sstateen0 = csr_swap(CSR_SSTATEEN0, 710 vcpu->arch.host_sstateen0); 711 } 712 713 /* 714 * Actually run the vCPU, entering an RCU extended quiescent state (EQS) while 715 * the vCPU is running. 716 * 717 * This must be noinstr as instrumentation may make use of RCU, and this is not 718 * safe during the EQS. 719 */ 720 static void noinstr kvm_riscv_vcpu_enter_exit(struct kvm_vcpu *vcpu) 721 { 722 kvm_riscv_vcpu_swap_in_guest_state(vcpu); 723 guest_state_enter_irqoff(); 724 __kvm_riscv_switch_to(&vcpu->arch); 725 vcpu->arch.last_exit_cpu = vcpu->cpu; 726 guest_state_exit_irqoff(); 727 kvm_riscv_vcpu_swap_in_host_state(vcpu); 728 } 729 730 int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu) 731 { 732 int ret; 733 struct kvm_cpu_trap trap; 734 struct kvm_run *run = vcpu->run; 735 736 if (!vcpu->arch.ran_atleast_once) 737 kvm_riscv_vcpu_setup_config(vcpu); 738 739 /* Mark this VCPU ran at least once */ 740 vcpu->arch.ran_atleast_once = true; 741 742 kvm_vcpu_srcu_read_lock(vcpu); 743 744 switch (run->exit_reason) { 745 case KVM_EXIT_MMIO: 746 /* Process MMIO value returned from user-space */ 747 ret = kvm_riscv_vcpu_mmio_return(vcpu, vcpu->run); 748 break; 749 case KVM_EXIT_RISCV_SBI: 750 /* Process SBI value returned from user-space */ 751 ret = kvm_riscv_vcpu_sbi_return(vcpu, vcpu->run); 752 break; 753 case KVM_EXIT_RISCV_CSR: 754 /* Process CSR value returned from user-space */ 755 ret = kvm_riscv_vcpu_csr_return(vcpu, vcpu->run); 756 break; 757 default: 758 ret = 0; 759 break; 760 } 761 if (ret) { 762 kvm_vcpu_srcu_read_unlock(vcpu); 763 return ret; 764 } 765 766 if (!vcpu->wants_to_run) { 767 kvm_vcpu_srcu_read_unlock(vcpu); 768 return -EINTR; 769 } 770 771 vcpu_load(vcpu); 772 773 kvm_sigset_activate(vcpu); 774 775 ret = 1; 776 run->exit_reason = KVM_EXIT_UNKNOWN; 777 while (ret > 0) { 778 /* Check conditions before entering the guest */ 779 ret = xfer_to_guest_mode_handle_work(vcpu); 780 if (ret) 781 continue; 782 ret = 1; 783 784 kvm_riscv_gstage_vmid_update(vcpu); 785 786 kvm_riscv_check_vcpu_requests(vcpu); 787 788 preempt_disable(); 789 790 /* Update AIA HW state before entering guest */ 791 ret = kvm_riscv_vcpu_aia_update(vcpu); 792 if (ret <= 0) { 793 preempt_enable(); 794 continue; 795 } 796 797 local_irq_disable(); 798 799 /* 800 * Ensure we set mode to IN_GUEST_MODE after we disable 801 * interrupts and before the final VCPU requests check. 802 * See the comment in kvm_vcpu_exiting_guest_mode() and 803 * Documentation/virt/kvm/vcpu-requests.rst 804 */ 805 vcpu->mode = IN_GUEST_MODE; 806 807 kvm_vcpu_srcu_read_unlock(vcpu); 808 smp_mb__after_srcu_read_unlock(); 809 810 /* 811 * We might have got VCPU interrupts updated asynchronously 812 * so update it in HW. 813 */ 814 kvm_riscv_vcpu_flush_interrupts(vcpu); 815 816 /* Update HVIP CSR for current CPU */ 817 kvm_riscv_update_hvip(vcpu); 818 819 if (kvm_riscv_gstage_vmid_ver_changed(&vcpu->kvm->arch.vmid) || 820 kvm_request_pending(vcpu) || 821 xfer_to_guest_mode_work_pending()) { 822 vcpu->mode = OUTSIDE_GUEST_MODE; 823 local_irq_enable(); 824 preempt_enable(); 825 kvm_vcpu_srcu_read_lock(vcpu); 826 continue; 827 } 828 829 /* 830 * Cleanup stale TLB enteries 831 * 832 * Note: This should be done after G-stage VMID has been 833 * updated using kvm_riscv_gstage_vmid_ver_changed() 834 */ 835 kvm_riscv_local_tlb_sanitize(vcpu); 836 837 trace_kvm_entry(vcpu); 838 839 guest_timing_enter_irqoff(); 840 841 kvm_riscv_vcpu_enter_exit(vcpu); 842 843 vcpu->mode = OUTSIDE_GUEST_MODE; 844 vcpu->stat.exits++; 845 846 /* 847 * Save SCAUSE, STVAL, HTVAL, and HTINST because we might 848 * get an interrupt between __kvm_riscv_switch_to() and 849 * local_irq_enable() which can potentially change CSRs. 850 */ 851 trap.sepc = vcpu->arch.guest_context.sepc; 852 trap.scause = csr_read(CSR_SCAUSE); 853 trap.stval = csr_read(CSR_STVAL); 854 trap.htval = csr_read(CSR_HTVAL); 855 trap.htinst = csr_read(CSR_HTINST); 856 857 /* Syncup interrupts state with HW */ 858 kvm_riscv_vcpu_sync_interrupts(vcpu); 859 860 /* 861 * We must ensure that any pending interrupts are taken before 862 * we exit guest timing so that timer ticks are accounted as 863 * guest time. Transiently unmask interrupts so that any 864 * pending interrupts are taken. 865 * 866 * There's no barrier which ensures that pending interrupts are 867 * recognised, so we just hope that the CPU takes any pending 868 * interrupts between the enable and disable. 869 */ 870 local_irq_enable(); 871 local_irq_disable(); 872 873 guest_timing_exit_irqoff(); 874 875 local_irq_enable(); 876 877 trace_kvm_exit(&trap); 878 879 preempt_enable(); 880 881 kvm_vcpu_srcu_read_lock(vcpu); 882 883 ret = kvm_riscv_vcpu_exit(vcpu, run, &trap); 884 } 885 886 kvm_sigset_deactivate(vcpu); 887 888 vcpu_put(vcpu); 889 890 kvm_vcpu_srcu_read_unlock(vcpu); 891 892 return ret; 893 } 894