1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (C) 2019 Western Digital Corporation or its affiliates. 4 * 5 * Authors: 6 * Anup Patel <anup.patel@wdc.com> 7 */ 8 9 #include <linux/bitops.h> 10 #include <linux/entry-kvm.h> 11 #include <linux/errno.h> 12 #include <linux/err.h> 13 #include <linux/kdebug.h> 14 #include <linux/module.h> 15 #include <linux/percpu.h> 16 #include <linux/uaccess.h> 17 #include <linux/vmalloc.h> 18 #include <linux/sched/signal.h> 19 #include <linux/fs.h> 20 #include <linux/kvm_host.h> 21 #include <asm/csr.h> 22 #include <asm/cacheflush.h> 23 #include <asm/hwcap.h> 24 #include <asm/sbi.h> 25 #include <asm/vector.h> 26 #include <asm/kvm_vcpu_vector.h> 27 28 const struct _kvm_stats_desc kvm_vcpu_stats_desc[] = { 29 KVM_GENERIC_VCPU_STATS(), 30 STATS_DESC_COUNTER(VCPU, ecall_exit_stat), 31 STATS_DESC_COUNTER(VCPU, wfi_exit_stat), 32 STATS_DESC_COUNTER(VCPU, mmio_exit_user), 33 STATS_DESC_COUNTER(VCPU, mmio_exit_kernel), 34 STATS_DESC_COUNTER(VCPU, csr_exit_user), 35 STATS_DESC_COUNTER(VCPU, csr_exit_kernel), 36 STATS_DESC_COUNTER(VCPU, signal_exits), 37 STATS_DESC_COUNTER(VCPU, exits) 38 }; 39 40 const struct kvm_stats_header kvm_vcpu_stats_header = { 41 .name_size = KVM_STATS_NAME_SIZE, 42 .num_desc = ARRAY_SIZE(kvm_vcpu_stats_desc), 43 .id_offset = sizeof(struct kvm_stats_header), 44 .desc_offset = sizeof(struct kvm_stats_header) + KVM_STATS_NAME_SIZE, 45 .data_offset = sizeof(struct kvm_stats_header) + KVM_STATS_NAME_SIZE + 46 sizeof(kvm_vcpu_stats_desc), 47 }; 48 49 #define KVM_RISCV_BASE_ISA_MASK GENMASK(25, 0) 50 51 #define KVM_ISA_EXT_ARR(ext) [KVM_RISCV_ISA_EXT_##ext] = RISCV_ISA_EXT_##ext 52 53 /* Mapping between KVM ISA Extension ID & Host ISA extension ID */ 54 static const unsigned long kvm_isa_ext_arr[] = { 55 [KVM_RISCV_ISA_EXT_A] = RISCV_ISA_EXT_a, 56 [KVM_RISCV_ISA_EXT_C] = RISCV_ISA_EXT_c, 57 [KVM_RISCV_ISA_EXT_D] = RISCV_ISA_EXT_d, 58 [KVM_RISCV_ISA_EXT_F] = RISCV_ISA_EXT_f, 59 [KVM_RISCV_ISA_EXT_H] = RISCV_ISA_EXT_h, 60 [KVM_RISCV_ISA_EXT_I] = RISCV_ISA_EXT_i, 61 [KVM_RISCV_ISA_EXT_M] = RISCV_ISA_EXT_m, 62 [KVM_RISCV_ISA_EXT_V] = RISCV_ISA_EXT_v, 63 64 KVM_ISA_EXT_ARR(SSAIA), 65 KVM_ISA_EXT_ARR(SSTC), 66 KVM_ISA_EXT_ARR(SVINVAL), 67 KVM_ISA_EXT_ARR(SVPBMT), 68 KVM_ISA_EXT_ARR(ZBB), 69 KVM_ISA_EXT_ARR(ZIHINTPAUSE), 70 KVM_ISA_EXT_ARR(ZICBOM), 71 KVM_ISA_EXT_ARR(ZICBOZ), 72 }; 73 74 static unsigned long kvm_riscv_vcpu_base2isa_ext(unsigned long base_ext) 75 { 76 unsigned long i; 77 78 for (i = 0; i < KVM_RISCV_ISA_EXT_MAX; i++) { 79 if (kvm_isa_ext_arr[i] == base_ext) 80 return i; 81 } 82 83 return KVM_RISCV_ISA_EXT_MAX; 84 } 85 86 static bool kvm_riscv_vcpu_isa_enable_allowed(unsigned long ext) 87 { 88 switch (ext) { 89 case KVM_RISCV_ISA_EXT_H: 90 return false; 91 case KVM_RISCV_ISA_EXT_V: 92 return riscv_v_vstate_ctrl_user_allowed(); 93 default: 94 break; 95 } 96 97 return true; 98 } 99 100 static bool kvm_riscv_vcpu_isa_disable_allowed(unsigned long ext) 101 { 102 switch (ext) { 103 case KVM_RISCV_ISA_EXT_A: 104 case KVM_RISCV_ISA_EXT_C: 105 case KVM_RISCV_ISA_EXT_I: 106 case KVM_RISCV_ISA_EXT_M: 107 case KVM_RISCV_ISA_EXT_SSAIA: 108 case KVM_RISCV_ISA_EXT_SSTC: 109 case KVM_RISCV_ISA_EXT_SVINVAL: 110 case KVM_RISCV_ISA_EXT_ZIHINTPAUSE: 111 case KVM_RISCV_ISA_EXT_ZBB: 112 return false; 113 default: 114 break; 115 } 116 117 return true; 118 } 119 120 static void kvm_riscv_reset_vcpu(struct kvm_vcpu *vcpu) 121 { 122 struct kvm_vcpu_csr *csr = &vcpu->arch.guest_csr; 123 struct kvm_vcpu_csr *reset_csr = &vcpu->arch.guest_reset_csr; 124 struct kvm_cpu_context *cntx = &vcpu->arch.guest_context; 125 struct kvm_cpu_context *reset_cntx = &vcpu->arch.guest_reset_context; 126 bool loaded; 127 128 /** 129 * The preemption should be disabled here because it races with 130 * kvm_sched_out/kvm_sched_in(called from preempt notifiers) which 131 * also calls vcpu_load/put. 132 */ 133 get_cpu(); 134 loaded = (vcpu->cpu != -1); 135 if (loaded) 136 kvm_arch_vcpu_put(vcpu); 137 138 vcpu->arch.last_exit_cpu = -1; 139 140 memcpy(csr, reset_csr, sizeof(*csr)); 141 142 memcpy(cntx, reset_cntx, sizeof(*cntx)); 143 144 kvm_riscv_vcpu_fp_reset(vcpu); 145 146 kvm_riscv_vcpu_vector_reset(vcpu); 147 148 kvm_riscv_vcpu_timer_reset(vcpu); 149 150 kvm_riscv_vcpu_aia_reset(vcpu); 151 152 bitmap_zero(vcpu->arch.irqs_pending, KVM_RISCV_VCPU_NR_IRQS); 153 bitmap_zero(vcpu->arch.irqs_pending_mask, KVM_RISCV_VCPU_NR_IRQS); 154 155 kvm_riscv_vcpu_pmu_reset(vcpu); 156 157 vcpu->arch.hfence_head = 0; 158 vcpu->arch.hfence_tail = 0; 159 memset(vcpu->arch.hfence_queue, 0, sizeof(vcpu->arch.hfence_queue)); 160 161 /* Reset the guest CSRs for hotplug usecase */ 162 if (loaded) 163 kvm_arch_vcpu_load(vcpu, smp_processor_id()); 164 put_cpu(); 165 } 166 167 int kvm_arch_vcpu_precreate(struct kvm *kvm, unsigned int id) 168 { 169 return 0; 170 } 171 172 int kvm_arch_vcpu_create(struct kvm_vcpu *vcpu) 173 { 174 int rc; 175 struct kvm_cpu_context *cntx; 176 struct kvm_vcpu_csr *reset_csr = &vcpu->arch.guest_reset_csr; 177 unsigned long host_isa, i; 178 179 /* Mark this VCPU never ran */ 180 vcpu->arch.ran_atleast_once = false; 181 vcpu->arch.mmu_page_cache.gfp_zero = __GFP_ZERO; 182 bitmap_zero(vcpu->arch.isa, RISCV_ISA_EXT_MAX); 183 184 /* Setup ISA features available to VCPU */ 185 for (i = 0; i < ARRAY_SIZE(kvm_isa_ext_arr); i++) { 186 host_isa = kvm_isa_ext_arr[i]; 187 if (__riscv_isa_extension_available(NULL, host_isa) && 188 kvm_riscv_vcpu_isa_enable_allowed(i)) 189 set_bit(host_isa, vcpu->arch.isa); 190 } 191 192 /* Setup vendor, arch, and implementation details */ 193 vcpu->arch.mvendorid = sbi_get_mvendorid(); 194 vcpu->arch.marchid = sbi_get_marchid(); 195 vcpu->arch.mimpid = sbi_get_mimpid(); 196 197 /* Setup VCPU hfence queue */ 198 spin_lock_init(&vcpu->arch.hfence_lock); 199 200 /* Setup reset state of shadow SSTATUS and HSTATUS CSRs */ 201 cntx = &vcpu->arch.guest_reset_context; 202 cntx->sstatus = SR_SPP | SR_SPIE; 203 cntx->hstatus = 0; 204 cntx->hstatus |= HSTATUS_VTW; 205 cntx->hstatus |= HSTATUS_SPVP; 206 cntx->hstatus |= HSTATUS_SPV; 207 208 if (kvm_riscv_vcpu_alloc_vector_context(vcpu, cntx)) 209 return -ENOMEM; 210 211 /* By default, make CY, TM, and IR counters accessible in VU mode */ 212 reset_csr->scounteren = 0x7; 213 214 /* Setup VCPU timer */ 215 kvm_riscv_vcpu_timer_init(vcpu); 216 217 /* setup performance monitoring */ 218 kvm_riscv_vcpu_pmu_init(vcpu); 219 220 /* Setup VCPU AIA */ 221 rc = kvm_riscv_vcpu_aia_init(vcpu); 222 if (rc) 223 return rc; 224 225 /* Reset VCPU */ 226 kvm_riscv_reset_vcpu(vcpu); 227 228 return 0; 229 } 230 231 void kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu) 232 { 233 /** 234 * vcpu with id 0 is the designated boot cpu. 235 * Keep all vcpus with non-zero id in power-off state so that 236 * they can be brought up using SBI HSM extension. 237 */ 238 if (vcpu->vcpu_idx != 0) 239 kvm_riscv_vcpu_power_off(vcpu); 240 } 241 242 void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu) 243 { 244 /* Cleanup VCPU AIA context */ 245 kvm_riscv_vcpu_aia_deinit(vcpu); 246 247 /* Cleanup VCPU timer */ 248 kvm_riscv_vcpu_timer_deinit(vcpu); 249 250 kvm_riscv_vcpu_pmu_deinit(vcpu); 251 252 /* Free unused pages pre-allocated for G-stage page table mappings */ 253 kvm_mmu_free_memory_cache(&vcpu->arch.mmu_page_cache); 254 255 /* Free vector context space for host and guest kernel */ 256 kvm_riscv_vcpu_free_vector_context(vcpu); 257 } 258 259 int kvm_cpu_has_pending_timer(struct kvm_vcpu *vcpu) 260 { 261 return kvm_riscv_vcpu_timer_pending(vcpu); 262 } 263 264 void kvm_arch_vcpu_blocking(struct kvm_vcpu *vcpu) 265 { 266 } 267 268 void kvm_arch_vcpu_unblocking(struct kvm_vcpu *vcpu) 269 { 270 } 271 272 int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu) 273 { 274 return (kvm_riscv_vcpu_has_interrupts(vcpu, -1UL) && 275 !vcpu->arch.power_off && !vcpu->arch.pause); 276 } 277 278 int kvm_arch_vcpu_should_kick(struct kvm_vcpu *vcpu) 279 { 280 return kvm_vcpu_exiting_guest_mode(vcpu) == IN_GUEST_MODE; 281 } 282 283 bool kvm_arch_vcpu_in_kernel(struct kvm_vcpu *vcpu) 284 { 285 return (vcpu->arch.guest_context.sstatus & SR_SPP) ? true : false; 286 } 287 288 vm_fault_t kvm_arch_vcpu_fault(struct kvm_vcpu *vcpu, struct vm_fault *vmf) 289 { 290 return VM_FAULT_SIGBUS; 291 } 292 293 static int kvm_riscv_vcpu_get_reg_config(struct kvm_vcpu *vcpu, 294 const struct kvm_one_reg *reg) 295 { 296 unsigned long __user *uaddr = 297 (unsigned long __user *)(unsigned long)reg->addr; 298 unsigned long reg_num = reg->id & ~(KVM_REG_ARCH_MASK | 299 KVM_REG_SIZE_MASK | 300 KVM_REG_RISCV_CONFIG); 301 unsigned long reg_val; 302 303 if (KVM_REG_SIZE(reg->id) != sizeof(unsigned long)) 304 return -EINVAL; 305 306 switch (reg_num) { 307 case KVM_REG_RISCV_CONFIG_REG(isa): 308 reg_val = vcpu->arch.isa[0] & KVM_RISCV_BASE_ISA_MASK; 309 break; 310 case KVM_REG_RISCV_CONFIG_REG(zicbom_block_size): 311 if (!riscv_isa_extension_available(vcpu->arch.isa, ZICBOM)) 312 return -EINVAL; 313 reg_val = riscv_cbom_block_size; 314 break; 315 case KVM_REG_RISCV_CONFIG_REG(zicboz_block_size): 316 if (!riscv_isa_extension_available(vcpu->arch.isa, ZICBOZ)) 317 return -EINVAL; 318 reg_val = riscv_cboz_block_size; 319 break; 320 case KVM_REG_RISCV_CONFIG_REG(mvendorid): 321 reg_val = vcpu->arch.mvendorid; 322 break; 323 case KVM_REG_RISCV_CONFIG_REG(marchid): 324 reg_val = vcpu->arch.marchid; 325 break; 326 case KVM_REG_RISCV_CONFIG_REG(mimpid): 327 reg_val = vcpu->arch.mimpid; 328 break; 329 default: 330 return -EINVAL; 331 } 332 333 if (copy_to_user(uaddr, ®_val, KVM_REG_SIZE(reg->id))) 334 return -EFAULT; 335 336 return 0; 337 } 338 339 static int kvm_riscv_vcpu_set_reg_config(struct kvm_vcpu *vcpu, 340 const struct kvm_one_reg *reg) 341 { 342 unsigned long __user *uaddr = 343 (unsigned long __user *)(unsigned long)reg->addr; 344 unsigned long reg_num = reg->id & ~(KVM_REG_ARCH_MASK | 345 KVM_REG_SIZE_MASK | 346 KVM_REG_RISCV_CONFIG); 347 unsigned long i, isa_ext, reg_val; 348 349 if (KVM_REG_SIZE(reg->id) != sizeof(unsigned long)) 350 return -EINVAL; 351 352 if (copy_from_user(®_val, uaddr, KVM_REG_SIZE(reg->id))) 353 return -EFAULT; 354 355 switch (reg_num) { 356 case KVM_REG_RISCV_CONFIG_REG(isa): 357 /* 358 * This ONE REG interface is only defined for 359 * single letter extensions. 360 */ 361 if (fls(reg_val) >= RISCV_ISA_EXT_BASE) 362 return -EINVAL; 363 364 if (!vcpu->arch.ran_atleast_once) { 365 /* Ignore the enable/disable request for certain extensions */ 366 for (i = 0; i < RISCV_ISA_EXT_BASE; i++) { 367 isa_ext = kvm_riscv_vcpu_base2isa_ext(i); 368 if (isa_ext >= KVM_RISCV_ISA_EXT_MAX) { 369 reg_val &= ~BIT(i); 370 continue; 371 } 372 if (!kvm_riscv_vcpu_isa_enable_allowed(isa_ext)) 373 if (reg_val & BIT(i)) 374 reg_val &= ~BIT(i); 375 if (!kvm_riscv_vcpu_isa_disable_allowed(isa_ext)) 376 if (!(reg_val & BIT(i))) 377 reg_val |= BIT(i); 378 } 379 reg_val &= riscv_isa_extension_base(NULL); 380 /* Do not modify anything beyond single letter extensions */ 381 reg_val = (vcpu->arch.isa[0] & ~KVM_RISCV_BASE_ISA_MASK) | 382 (reg_val & KVM_RISCV_BASE_ISA_MASK); 383 vcpu->arch.isa[0] = reg_val; 384 kvm_riscv_vcpu_fp_reset(vcpu); 385 } else { 386 return -EOPNOTSUPP; 387 } 388 break; 389 case KVM_REG_RISCV_CONFIG_REG(zicbom_block_size): 390 return -EOPNOTSUPP; 391 case KVM_REG_RISCV_CONFIG_REG(zicboz_block_size): 392 return -EOPNOTSUPP; 393 case KVM_REG_RISCV_CONFIG_REG(mvendorid): 394 if (!vcpu->arch.ran_atleast_once) 395 vcpu->arch.mvendorid = reg_val; 396 else 397 return -EBUSY; 398 break; 399 case KVM_REG_RISCV_CONFIG_REG(marchid): 400 if (!vcpu->arch.ran_atleast_once) 401 vcpu->arch.marchid = reg_val; 402 else 403 return -EBUSY; 404 break; 405 case KVM_REG_RISCV_CONFIG_REG(mimpid): 406 if (!vcpu->arch.ran_atleast_once) 407 vcpu->arch.mimpid = reg_val; 408 else 409 return -EBUSY; 410 break; 411 default: 412 return -EINVAL; 413 } 414 415 return 0; 416 } 417 418 static int kvm_riscv_vcpu_get_reg_core(struct kvm_vcpu *vcpu, 419 const struct kvm_one_reg *reg) 420 { 421 struct kvm_cpu_context *cntx = &vcpu->arch.guest_context; 422 unsigned long __user *uaddr = 423 (unsigned long __user *)(unsigned long)reg->addr; 424 unsigned long reg_num = reg->id & ~(KVM_REG_ARCH_MASK | 425 KVM_REG_SIZE_MASK | 426 KVM_REG_RISCV_CORE); 427 unsigned long reg_val; 428 429 if (KVM_REG_SIZE(reg->id) != sizeof(unsigned long)) 430 return -EINVAL; 431 if (reg_num >= sizeof(struct kvm_riscv_core) / sizeof(unsigned long)) 432 return -EINVAL; 433 434 if (reg_num == KVM_REG_RISCV_CORE_REG(regs.pc)) 435 reg_val = cntx->sepc; 436 else if (KVM_REG_RISCV_CORE_REG(regs.pc) < reg_num && 437 reg_num <= KVM_REG_RISCV_CORE_REG(regs.t6)) 438 reg_val = ((unsigned long *)cntx)[reg_num]; 439 else if (reg_num == KVM_REG_RISCV_CORE_REG(mode)) 440 reg_val = (cntx->sstatus & SR_SPP) ? 441 KVM_RISCV_MODE_S : KVM_RISCV_MODE_U; 442 else 443 return -EINVAL; 444 445 if (copy_to_user(uaddr, ®_val, KVM_REG_SIZE(reg->id))) 446 return -EFAULT; 447 448 return 0; 449 } 450 451 static int kvm_riscv_vcpu_set_reg_core(struct kvm_vcpu *vcpu, 452 const struct kvm_one_reg *reg) 453 { 454 struct kvm_cpu_context *cntx = &vcpu->arch.guest_context; 455 unsigned long __user *uaddr = 456 (unsigned long __user *)(unsigned long)reg->addr; 457 unsigned long reg_num = reg->id & ~(KVM_REG_ARCH_MASK | 458 KVM_REG_SIZE_MASK | 459 KVM_REG_RISCV_CORE); 460 unsigned long reg_val; 461 462 if (KVM_REG_SIZE(reg->id) != sizeof(unsigned long)) 463 return -EINVAL; 464 if (reg_num >= sizeof(struct kvm_riscv_core) / sizeof(unsigned long)) 465 return -EINVAL; 466 467 if (copy_from_user(®_val, uaddr, KVM_REG_SIZE(reg->id))) 468 return -EFAULT; 469 470 if (reg_num == KVM_REG_RISCV_CORE_REG(regs.pc)) 471 cntx->sepc = reg_val; 472 else if (KVM_REG_RISCV_CORE_REG(regs.pc) < reg_num && 473 reg_num <= KVM_REG_RISCV_CORE_REG(regs.t6)) 474 ((unsigned long *)cntx)[reg_num] = reg_val; 475 else if (reg_num == KVM_REG_RISCV_CORE_REG(mode)) { 476 if (reg_val == KVM_RISCV_MODE_S) 477 cntx->sstatus |= SR_SPP; 478 else 479 cntx->sstatus &= ~SR_SPP; 480 } else 481 return -EINVAL; 482 483 return 0; 484 } 485 486 static int kvm_riscv_vcpu_general_get_csr(struct kvm_vcpu *vcpu, 487 unsigned long reg_num, 488 unsigned long *out_val) 489 { 490 struct kvm_vcpu_csr *csr = &vcpu->arch.guest_csr; 491 492 if (reg_num >= sizeof(struct kvm_riscv_csr) / sizeof(unsigned long)) 493 return -EINVAL; 494 495 if (reg_num == KVM_REG_RISCV_CSR_REG(sip)) { 496 kvm_riscv_vcpu_flush_interrupts(vcpu); 497 *out_val = (csr->hvip >> VSIP_TO_HVIP_SHIFT) & VSIP_VALID_MASK; 498 *out_val |= csr->hvip & ~IRQ_LOCAL_MASK; 499 } else 500 *out_val = ((unsigned long *)csr)[reg_num]; 501 502 return 0; 503 } 504 505 static inline int kvm_riscv_vcpu_general_set_csr(struct kvm_vcpu *vcpu, 506 unsigned long reg_num, 507 unsigned long reg_val) 508 { 509 struct kvm_vcpu_csr *csr = &vcpu->arch.guest_csr; 510 511 if (reg_num >= sizeof(struct kvm_riscv_csr) / sizeof(unsigned long)) 512 return -EINVAL; 513 514 if (reg_num == KVM_REG_RISCV_CSR_REG(sip)) { 515 reg_val &= VSIP_VALID_MASK; 516 reg_val <<= VSIP_TO_HVIP_SHIFT; 517 } 518 519 ((unsigned long *)csr)[reg_num] = reg_val; 520 521 if (reg_num == KVM_REG_RISCV_CSR_REG(sip)) 522 WRITE_ONCE(vcpu->arch.irqs_pending_mask[0], 0); 523 524 return 0; 525 } 526 527 static int kvm_riscv_vcpu_get_reg_csr(struct kvm_vcpu *vcpu, 528 const struct kvm_one_reg *reg) 529 { 530 int rc; 531 unsigned long __user *uaddr = 532 (unsigned long __user *)(unsigned long)reg->addr; 533 unsigned long reg_num = reg->id & ~(KVM_REG_ARCH_MASK | 534 KVM_REG_SIZE_MASK | 535 KVM_REG_RISCV_CSR); 536 unsigned long reg_val, reg_subtype; 537 538 if (KVM_REG_SIZE(reg->id) != sizeof(unsigned long)) 539 return -EINVAL; 540 541 reg_subtype = reg_num & KVM_REG_RISCV_SUBTYPE_MASK; 542 reg_num &= ~KVM_REG_RISCV_SUBTYPE_MASK; 543 switch (reg_subtype) { 544 case KVM_REG_RISCV_CSR_GENERAL: 545 rc = kvm_riscv_vcpu_general_get_csr(vcpu, reg_num, ®_val); 546 break; 547 case KVM_REG_RISCV_CSR_AIA: 548 rc = kvm_riscv_vcpu_aia_get_csr(vcpu, reg_num, ®_val); 549 break; 550 default: 551 rc = -EINVAL; 552 break; 553 } 554 if (rc) 555 return rc; 556 557 if (copy_to_user(uaddr, ®_val, KVM_REG_SIZE(reg->id))) 558 return -EFAULT; 559 560 return 0; 561 } 562 563 static int kvm_riscv_vcpu_set_reg_csr(struct kvm_vcpu *vcpu, 564 const struct kvm_one_reg *reg) 565 { 566 int rc; 567 unsigned long __user *uaddr = 568 (unsigned long __user *)(unsigned long)reg->addr; 569 unsigned long reg_num = reg->id & ~(KVM_REG_ARCH_MASK | 570 KVM_REG_SIZE_MASK | 571 KVM_REG_RISCV_CSR); 572 unsigned long reg_val, reg_subtype; 573 574 if (KVM_REG_SIZE(reg->id) != sizeof(unsigned long)) 575 return -EINVAL; 576 577 if (copy_from_user(®_val, uaddr, KVM_REG_SIZE(reg->id))) 578 return -EFAULT; 579 580 reg_subtype = reg_num & KVM_REG_RISCV_SUBTYPE_MASK; 581 reg_num &= ~KVM_REG_RISCV_SUBTYPE_MASK; 582 switch (reg_subtype) { 583 case KVM_REG_RISCV_CSR_GENERAL: 584 rc = kvm_riscv_vcpu_general_set_csr(vcpu, reg_num, reg_val); 585 break; 586 case KVM_REG_RISCV_CSR_AIA: 587 rc = kvm_riscv_vcpu_aia_set_csr(vcpu, reg_num, reg_val); 588 break; 589 default: 590 rc = -EINVAL; 591 break; 592 } 593 if (rc) 594 return rc; 595 596 return 0; 597 } 598 599 static int kvm_riscv_vcpu_get_reg_isa_ext(struct kvm_vcpu *vcpu, 600 const struct kvm_one_reg *reg) 601 { 602 unsigned long __user *uaddr = 603 (unsigned long __user *)(unsigned long)reg->addr; 604 unsigned long reg_num = reg->id & ~(KVM_REG_ARCH_MASK | 605 KVM_REG_SIZE_MASK | 606 KVM_REG_RISCV_ISA_EXT); 607 unsigned long reg_val = 0; 608 unsigned long host_isa_ext; 609 610 if (KVM_REG_SIZE(reg->id) != sizeof(unsigned long)) 611 return -EINVAL; 612 613 if (reg_num >= KVM_RISCV_ISA_EXT_MAX || 614 reg_num >= ARRAY_SIZE(kvm_isa_ext_arr)) 615 return -EINVAL; 616 617 host_isa_ext = kvm_isa_ext_arr[reg_num]; 618 if (__riscv_isa_extension_available(vcpu->arch.isa, host_isa_ext)) 619 reg_val = 1; /* Mark the given extension as available */ 620 621 if (copy_to_user(uaddr, ®_val, KVM_REG_SIZE(reg->id))) 622 return -EFAULT; 623 624 return 0; 625 } 626 627 static int kvm_riscv_vcpu_set_reg_isa_ext(struct kvm_vcpu *vcpu, 628 const struct kvm_one_reg *reg) 629 { 630 unsigned long __user *uaddr = 631 (unsigned long __user *)(unsigned long)reg->addr; 632 unsigned long reg_num = reg->id & ~(KVM_REG_ARCH_MASK | 633 KVM_REG_SIZE_MASK | 634 KVM_REG_RISCV_ISA_EXT); 635 unsigned long reg_val; 636 unsigned long host_isa_ext; 637 638 if (KVM_REG_SIZE(reg->id) != sizeof(unsigned long)) 639 return -EINVAL; 640 641 if (reg_num >= KVM_RISCV_ISA_EXT_MAX || 642 reg_num >= ARRAY_SIZE(kvm_isa_ext_arr)) 643 return -EINVAL; 644 645 if (copy_from_user(®_val, uaddr, KVM_REG_SIZE(reg->id))) 646 return -EFAULT; 647 648 host_isa_ext = kvm_isa_ext_arr[reg_num]; 649 if (!__riscv_isa_extension_available(NULL, host_isa_ext)) 650 return -EOPNOTSUPP; 651 652 if (!vcpu->arch.ran_atleast_once) { 653 /* 654 * All multi-letter extension and a few single letter 655 * extension can be disabled 656 */ 657 if (reg_val == 1 && 658 kvm_riscv_vcpu_isa_enable_allowed(reg_num)) 659 set_bit(host_isa_ext, vcpu->arch.isa); 660 else if (!reg_val && 661 kvm_riscv_vcpu_isa_disable_allowed(reg_num)) 662 clear_bit(host_isa_ext, vcpu->arch.isa); 663 else 664 return -EINVAL; 665 kvm_riscv_vcpu_fp_reset(vcpu); 666 } else { 667 return -EOPNOTSUPP; 668 } 669 670 return 0; 671 } 672 673 static int kvm_riscv_vcpu_set_reg(struct kvm_vcpu *vcpu, 674 const struct kvm_one_reg *reg) 675 { 676 switch (reg->id & KVM_REG_RISCV_TYPE_MASK) { 677 case KVM_REG_RISCV_CONFIG: 678 return kvm_riscv_vcpu_set_reg_config(vcpu, reg); 679 case KVM_REG_RISCV_CORE: 680 return kvm_riscv_vcpu_set_reg_core(vcpu, reg); 681 case KVM_REG_RISCV_CSR: 682 return kvm_riscv_vcpu_set_reg_csr(vcpu, reg); 683 case KVM_REG_RISCV_TIMER: 684 return kvm_riscv_vcpu_set_reg_timer(vcpu, reg); 685 case KVM_REG_RISCV_FP_F: 686 return kvm_riscv_vcpu_set_reg_fp(vcpu, reg, 687 KVM_REG_RISCV_FP_F); 688 case KVM_REG_RISCV_FP_D: 689 return kvm_riscv_vcpu_set_reg_fp(vcpu, reg, 690 KVM_REG_RISCV_FP_D); 691 case KVM_REG_RISCV_ISA_EXT: 692 return kvm_riscv_vcpu_set_reg_isa_ext(vcpu, reg); 693 case KVM_REG_RISCV_SBI_EXT: 694 return kvm_riscv_vcpu_set_reg_sbi_ext(vcpu, reg); 695 case KVM_REG_RISCV_VECTOR: 696 return kvm_riscv_vcpu_set_reg_vector(vcpu, reg, 697 KVM_REG_RISCV_VECTOR); 698 default: 699 break; 700 } 701 702 return -EINVAL; 703 } 704 705 static int kvm_riscv_vcpu_get_reg(struct kvm_vcpu *vcpu, 706 const struct kvm_one_reg *reg) 707 { 708 switch (reg->id & KVM_REG_RISCV_TYPE_MASK) { 709 case KVM_REG_RISCV_CONFIG: 710 return kvm_riscv_vcpu_get_reg_config(vcpu, reg); 711 case KVM_REG_RISCV_CORE: 712 return kvm_riscv_vcpu_get_reg_core(vcpu, reg); 713 case KVM_REG_RISCV_CSR: 714 return kvm_riscv_vcpu_get_reg_csr(vcpu, reg); 715 case KVM_REG_RISCV_TIMER: 716 return kvm_riscv_vcpu_get_reg_timer(vcpu, reg); 717 case KVM_REG_RISCV_FP_F: 718 return kvm_riscv_vcpu_get_reg_fp(vcpu, reg, 719 KVM_REG_RISCV_FP_F); 720 case KVM_REG_RISCV_FP_D: 721 return kvm_riscv_vcpu_get_reg_fp(vcpu, reg, 722 KVM_REG_RISCV_FP_D); 723 case KVM_REG_RISCV_ISA_EXT: 724 return kvm_riscv_vcpu_get_reg_isa_ext(vcpu, reg); 725 case KVM_REG_RISCV_SBI_EXT: 726 return kvm_riscv_vcpu_get_reg_sbi_ext(vcpu, reg); 727 case KVM_REG_RISCV_VECTOR: 728 return kvm_riscv_vcpu_get_reg_vector(vcpu, reg, 729 KVM_REG_RISCV_VECTOR); 730 default: 731 break; 732 } 733 734 return -EINVAL; 735 } 736 737 long kvm_arch_vcpu_async_ioctl(struct file *filp, 738 unsigned int ioctl, unsigned long arg) 739 { 740 struct kvm_vcpu *vcpu = filp->private_data; 741 void __user *argp = (void __user *)arg; 742 743 if (ioctl == KVM_INTERRUPT) { 744 struct kvm_interrupt irq; 745 746 if (copy_from_user(&irq, argp, sizeof(irq))) 747 return -EFAULT; 748 749 if (irq.irq == KVM_INTERRUPT_SET) 750 return kvm_riscv_vcpu_set_interrupt(vcpu, IRQ_VS_EXT); 751 else 752 return kvm_riscv_vcpu_unset_interrupt(vcpu, IRQ_VS_EXT); 753 } 754 755 return -ENOIOCTLCMD; 756 } 757 758 long kvm_arch_vcpu_ioctl(struct file *filp, 759 unsigned int ioctl, unsigned long arg) 760 { 761 struct kvm_vcpu *vcpu = filp->private_data; 762 void __user *argp = (void __user *)arg; 763 long r = -EINVAL; 764 765 switch (ioctl) { 766 case KVM_SET_ONE_REG: 767 case KVM_GET_ONE_REG: { 768 struct kvm_one_reg reg; 769 770 r = -EFAULT; 771 if (copy_from_user(®, argp, sizeof(reg))) 772 break; 773 774 if (ioctl == KVM_SET_ONE_REG) 775 r = kvm_riscv_vcpu_set_reg(vcpu, ®); 776 else 777 r = kvm_riscv_vcpu_get_reg(vcpu, ®); 778 break; 779 } 780 default: 781 break; 782 } 783 784 return r; 785 } 786 787 int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu, 788 struct kvm_sregs *sregs) 789 { 790 return -EINVAL; 791 } 792 793 int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu, 794 struct kvm_sregs *sregs) 795 { 796 return -EINVAL; 797 } 798 799 int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu) 800 { 801 return -EINVAL; 802 } 803 804 int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu) 805 { 806 return -EINVAL; 807 } 808 809 int kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu *vcpu, 810 struct kvm_translation *tr) 811 { 812 return -EINVAL; 813 } 814 815 int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs) 816 { 817 return -EINVAL; 818 } 819 820 int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs) 821 { 822 return -EINVAL; 823 } 824 825 void kvm_riscv_vcpu_flush_interrupts(struct kvm_vcpu *vcpu) 826 { 827 struct kvm_vcpu_csr *csr = &vcpu->arch.guest_csr; 828 unsigned long mask, val; 829 830 if (READ_ONCE(vcpu->arch.irqs_pending_mask[0])) { 831 mask = xchg_acquire(&vcpu->arch.irqs_pending_mask[0], 0); 832 val = READ_ONCE(vcpu->arch.irqs_pending[0]) & mask; 833 834 csr->hvip &= ~mask; 835 csr->hvip |= val; 836 } 837 838 /* Flush AIA high interrupts */ 839 kvm_riscv_vcpu_aia_flush_interrupts(vcpu); 840 } 841 842 void kvm_riscv_vcpu_sync_interrupts(struct kvm_vcpu *vcpu) 843 { 844 unsigned long hvip; 845 struct kvm_vcpu_arch *v = &vcpu->arch; 846 struct kvm_vcpu_csr *csr = &vcpu->arch.guest_csr; 847 848 /* Read current HVIP and VSIE CSRs */ 849 csr->vsie = csr_read(CSR_VSIE); 850 851 /* Sync-up HVIP.VSSIP bit changes does by Guest */ 852 hvip = csr_read(CSR_HVIP); 853 if ((csr->hvip ^ hvip) & (1UL << IRQ_VS_SOFT)) { 854 if (hvip & (1UL << IRQ_VS_SOFT)) { 855 if (!test_and_set_bit(IRQ_VS_SOFT, 856 v->irqs_pending_mask)) 857 set_bit(IRQ_VS_SOFT, v->irqs_pending); 858 } else { 859 if (!test_and_set_bit(IRQ_VS_SOFT, 860 v->irqs_pending_mask)) 861 clear_bit(IRQ_VS_SOFT, v->irqs_pending); 862 } 863 } 864 865 /* Sync-up AIA high interrupts */ 866 kvm_riscv_vcpu_aia_sync_interrupts(vcpu); 867 868 /* Sync-up timer CSRs */ 869 kvm_riscv_vcpu_timer_sync(vcpu); 870 } 871 872 int kvm_riscv_vcpu_set_interrupt(struct kvm_vcpu *vcpu, unsigned int irq) 873 { 874 /* 875 * We only allow VS-mode software, timer, and external 876 * interrupts when irq is one of the local interrupts 877 * defined by RISC-V privilege specification. 878 */ 879 if (irq < IRQ_LOCAL_MAX && 880 irq != IRQ_VS_SOFT && 881 irq != IRQ_VS_TIMER && 882 irq != IRQ_VS_EXT) 883 return -EINVAL; 884 885 set_bit(irq, vcpu->arch.irqs_pending); 886 smp_mb__before_atomic(); 887 set_bit(irq, vcpu->arch.irqs_pending_mask); 888 889 kvm_vcpu_kick(vcpu); 890 891 return 0; 892 } 893 894 int kvm_riscv_vcpu_unset_interrupt(struct kvm_vcpu *vcpu, unsigned int irq) 895 { 896 /* 897 * We only allow VS-mode software, timer, and external 898 * interrupts when irq is one of the local interrupts 899 * defined by RISC-V privilege specification. 900 */ 901 if (irq < IRQ_LOCAL_MAX && 902 irq != IRQ_VS_SOFT && 903 irq != IRQ_VS_TIMER && 904 irq != IRQ_VS_EXT) 905 return -EINVAL; 906 907 clear_bit(irq, vcpu->arch.irqs_pending); 908 smp_mb__before_atomic(); 909 set_bit(irq, vcpu->arch.irqs_pending_mask); 910 911 return 0; 912 } 913 914 bool kvm_riscv_vcpu_has_interrupts(struct kvm_vcpu *vcpu, u64 mask) 915 { 916 unsigned long ie; 917 918 ie = ((vcpu->arch.guest_csr.vsie & VSIP_VALID_MASK) 919 << VSIP_TO_HVIP_SHIFT) & (unsigned long)mask; 920 ie |= vcpu->arch.guest_csr.vsie & ~IRQ_LOCAL_MASK & 921 (unsigned long)mask; 922 if (READ_ONCE(vcpu->arch.irqs_pending[0]) & ie) 923 return true; 924 925 /* Check AIA high interrupts */ 926 return kvm_riscv_vcpu_aia_has_interrupts(vcpu, mask); 927 } 928 929 void kvm_riscv_vcpu_power_off(struct kvm_vcpu *vcpu) 930 { 931 vcpu->arch.power_off = true; 932 kvm_make_request(KVM_REQ_SLEEP, vcpu); 933 kvm_vcpu_kick(vcpu); 934 } 935 936 void kvm_riscv_vcpu_power_on(struct kvm_vcpu *vcpu) 937 { 938 vcpu->arch.power_off = false; 939 kvm_vcpu_wake_up(vcpu); 940 } 941 942 int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu, 943 struct kvm_mp_state *mp_state) 944 { 945 if (vcpu->arch.power_off) 946 mp_state->mp_state = KVM_MP_STATE_STOPPED; 947 else 948 mp_state->mp_state = KVM_MP_STATE_RUNNABLE; 949 950 return 0; 951 } 952 953 int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu, 954 struct kvm_mp_state *mp_state) 955 { 956 int ret = 0; 957 958 switch (mp_state->mp_state) { 959 case KVM_MP_STATE_RUNNABLE: 960 vcpu->arch.power_off = false; 961 break; 962 case KVM_MP_STATE_STOPPED: 963 kvm_riscv_vcpu_power_off(vcpu); 964 break; 965 default: 966 ret = -EINVAL; 967 } 968 969 return ret; 970 } 971 972 int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu, 973 struct kvm_guest_debug *dbg) 974 { 975 /* TODO; To be implemented later. */ 976 return -EINVAL; 977 } 978 979 static void kvm_riscv_vcpu_update_config(const unsigned long *isa) 980 { 981 u64 henvcfg = 0; 982 983 if (riscv_isa_extension_available(isa, SVPBMT)) 984 henvcfg |= ENVCFG_PBMTE; 985 986 if (riscv_isa_extension_available(isa, SSTC)) 987 henvcfg |= ENVCFG_STCE; 988 989 if (riscv_isa_extension_available(isa, ZICBOM)) 990 henvcfg |= (ENVCFG_CBIE | ENVCFG_CBCFE); 991 992 if (riscv_isa_extension_available(isa, ZICBOZ)) 993 henvcfg |= ENVCFG_CBZE; 994 995 csr_write(CSR_HENVCFG, henvcfg); 996 #ifdef CONFIG_32BIT 997 csr_write(CSR_HENVCFGH, henvcfg >> 32); 998 #endif 999 } 1000 1001 void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu) 1002 { 1003 struct kvm_vcpu_csr *csr = &vcpu->arch.guest_csr; 1004 1005 csr_write(CSR_VSSTATUS, csr->vsstatus); 1006 csr_write(CSR_VSIE, csr->vsie); 1007 csr_write(CSR_VSTVEC, csr->vstvec); 1008 csr_write(CSR_VSSCRATCH, csr->vsscratch); 1009 csr_write(CSR_VSEPC, csr->vsepc); 1010 csr_write(CSR_VSCAUSE, csr->vscause); 1011 csr_write(CSR_VSTVAL, csr->vstval); 1012 csr_write(CSR_HVIP, csr->hvip); 1013 csr_write(CSR_VSATP, csr->vsatp); 1014 1015 kvm_riscv_vcpu_update_config(vcpu->arch.isa); 1016 1017 kvm_riscv_gstage_update_hgatp(vcpu); 1018 1019 kvm_riscv_vcpu_timer_restore(vcpu); 1020 1021 kvm_riscv_vcpu_host_fp_save(&vcpu->arch.host_context); 1022 kvm_riscv_vcpu_guest_fp_restore(&vcpu->arch.guest_context, 1023 vcpu->arch.isa); 1024 kvm_riscv_vcpu_host_vector_save(&vcpu->arch.host_context); 1025 kvm_riscv_vcpu_guest_vector_restore(&vcpu->arch.guest_context, 1026 vcpu->arch.isa); 1027 1028 kvm_riscv_vcpu_aia_load(vcpu, cpu); 1029 1030 vcpu->cpu = cpu; 1031 } 1032 1033 void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu) 1034 { 1035 struct kvm_vcpu_csr *csr = &vcpu->arch.guest_csr; 1036 1037 vcpu->cpu = -1; 1038 1039 kvm_riscv_vcpu_aia_put(vcpu); 1040 1041 kvm_riscv_vcpu_guest_fp_save(&vcpu->arch.guest_context, 1042 vcpu->arch.isa); 1043 kvm_riscv_vcpu_host_fp_restore(&vcpu->arch.host_context); 1044 1045 kvm_riscv_vcpu_timer_save(vcpu); 1046 kvm_riscv_vcpu_guest_vector_save(&vcpu->arch.guest_context, 1047 vcpu->arch.isa); 1048 kvm_riscv_vcpu_host_vector_restore(&vcpu->arch.host_context); 1049 1050 csr->vsstatus = csr_read(CSR_VSSTATUS); 1051 csr->vsie = csr_read(CSR_VSIE); 1052 csr->vstvec = csr_read(CSR_VSTVEC); 1053 csr->vsscratch = csr_read(CSR_VSSCRATCH); 1054 csr->vsepc = csr_read(CSR_VSEPC); 1055 csr->vscause = csr_read(CSR_VSCAUSE); 1056 csr->vstval = csr_read(CSR_VSTVAL); 1057 csr->hvip = csr_read(CSR_HVIP); 1058 csr->vsatp = csr_read(CSR_VSATP); 1059 } 1060 1061 static void kvm_riscv_check_vcpu_requests(struct kvm_vcpu *vcpu) 1062 { 1063 struct rcuwait *wait = kvm_arch_vcpu_get_wait(vcpu); 1064 1065 if (kvm_request_pending(vcpu)) { 1066 if (kvm_check_request(KVM_REQ_SLEEP, vcpu)) { 1067 kvm_vcpu_srcu_read_unlock(vcpu); 1068 rcuwait_wait_event(wait, 1069 (!vcpu->arch.power_off) && (!vcpu->arch.pause), 1070 TASK_INTERRUPTIBLE); 1071 kvm_vcpu_srcu_read_lock(vcpu); 1072 1073 if (vcpu->arch.power_off || vcpu->arch.pause) { 1074 /* 1075 * Awaken to handle a signal, request to 1076 * sleep again later. 1077 */ 1078 kvm_make_request(KVM_REQ_SLEEP, vcpu); 1079 } 1080 } 1081 1082 if (kvm_check_request(KVM_REQ_VCPU_RESET, vcpu)) 1083 kvm_riscv_reset_vcpu(vcpu); 1084 1085 if (kvm_check_request(KVM_REQ_UPDATE_HGATP, vcpu)) 1086 kvm_riscv_gstage_update_hgatp(vcpu); 1087 1088 if (kvm_check_request(KVM_REQ_FENCE_I, vcpu)) 1089 kvm_riscv_fence_i_process(vcpu); 1090 1091 /* 1092 * The generic KVM_REQ_TLB_FLUSH is same as 1093 * KVM_REQ_HFENCE_GVMA_VMID_ALL 1094 */ 1095 if (kvm_check_request(KVM_REQ_HFENCE_GVMA_VMID_ALL, vcpu)) 1096 kvm_riscv_hfence_gvma_vmid_all_process(vcpu); 1097 1098 if (kvm_check_request(KVM_REQ_HFENCE_VVMA_ALL, vcpu)) 1099 kvm_riscv_hfence_vvma_all_process(vcpu); 1100 1101 if (kvm_check_request(KVM_REQ_HFENCE, vcpu)) 1102 kvm_riscv_hfence_process(vcpu); 1103 } 1104 } 1105 1106 static void kvm_riscv_update_hvip(struct kvm_vcpu *vcpu) 1107 { 1108 struct kvm_vcpu_csr *csr = &vcpu->arch.guest_csr; 1109 1110 csr_write(CSR_HVIP, csr->hvip); 1111 kvm_riscv_vcpu_aia_update_hvip(vcpu); 1112 } 1113 1114 /* 1115 * Actually run the vCPU, entering an RCU extended quiescent state (EQS) while 1116 * the vCPU is running. 1117 * 1118 * This must be noinstr as instrumentation may make use of RCU, and this is not 1119 * safe during the EQS. 1120 */ 1121 static void noinstr kvm_riscv_vcpu_enter_exit(struct kvm_vcpu *vcpu) 1122 { 1123 guest_state_enter_irqoff(); 1124 __kvm_riscv_switch_to(&vcpu->arch); 1125 vcpu->arch.last_exit_cpu = vcpu->cpu; 1126 guest_state_exit_irqoff(); 1127 } 1128 1129 int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu) 1130 { 1131 int ret; 1132 struct kvm_cpu_trap trap; 1133 struct kvm_run *run = vcpu->run; 1134 1135 /* Mark this VCPU ran at least once */ 1136 vcpu->arch.ran_atleast_once = true; 1137 1138 kvm_vcpu_srcu_read_lock(vcpu); 1139 1140 switch (run->exit_reason) { 1141 case KVM_EXIT_MMIO: 1142 /* Process MMIO value returned from user-space */ 1143 ret = kvm_riscv_vcpu_mmio_return(vcpu, vcpu->run); 1144 break; 1145 case KVM_EXIT_RISCV_SBI: 1146 /* Process SBI value returned from user-space */ 1147 ret = kvm_riscv_vcpu_sbi_return(vcpu, vcpu->run); 1148 break; 1149 case KVM_EXIT_RISCV_CSR: 1150 /* Process CSR value returned from user-space */ 1151 ret = kvm_riscv_vcpu_csr_return(vcpu, vcpu->run); 1152 break; 1153 default: 1154 ret = 0; 1155 break; 1156 } 1157 if (ret) { 1158 kvm_vcpu_srcu_read_unlock(vcpu); 1159 return ret; 1160 } 1161 1162 if (run->immediate_exit) { 1163 kvm_vcpu_srcu_read_unlock(vcpu); 1164 return -EINTR; 1165 } 1166 1167 vcpu_load(vcpu); 1168 1169 kvm_sigset_activate(vcpu); 1170 1171 ret = 1; 1172 run->exit_reason = KVM_EXIT_UNKNOWN; 1173 while (ret > 0) { 1174 /* Check conditions before entering the guest */ 1175 ret = xfer_to_guest_mode_handle_work(vcpu); 1176 if (ret) 1177 continue; 1178 ret = 1; 1179 1180 kvm_riscv_gstage_vmid_update(vcpu); 1181 1182 kvm_riscv_check_vcpu_requests(vcpu); 1183 1184 preempt_disable(); 1185 1186 /* Update AIA HW state before entering guest */ 1187 ret = kvm_riscv_vcpu_aia_update(vcpu); 1188 if (ret <= 0) { 1189 preempt_enable(); 1190 continue; 1191 } 1192 1193 local_irq_disable(); 1194 1195 /* 1196 * Ensure we set mode to IN_GUEST_MODE after we disable 1197 * interrupts and before the final VCPU requests check. 1198 * See the comment in kvm_vcpu_exiting_guest_mode() and 1199 * Documentation/virt/kvm/vcpu-requests.rst 1200 */ 1201 vcpu->mode = IN_GUEST_MODE; 1202 1203 kvm_vcpu_srcu_read_unlock(vcpu); 1204 smp_mb__after_srcu_read_unlock(); 1205 1206 /* 1207 * We might have got VCPU interrupts updated asynchronously 1208 * so update it in HW. 1209 */ 1210 kvm_riscv_vcpu_flush_interrupts(vcpu); 1211 1212 /* Update HVIP CSR for current CPU */ 1213 kvm_riscv_update_hvip(vcpu); 1214 1215 if (ret <= 0 || 1216 kvm_riscv_gstage_vmid_ver_changed(&vcpu->kvm->arch.vmid) || 1217 kvm_request_pending(vcpu) || 1218 xfer_to_guest_mode_work_pending()) { 1219 vcpu->mode = OUTSIDE_GUEST_MODE; 1220 local_irq_enable(); 1221 preempt_enable(); 1222 kvm_vcpu_srcu_read_lock(vcpu); 1223 continue; 1224 } 1225 1226 /* 1227 * Cleanup stale TLB enteries 1228 * 1229 * Note: This should be done after G-stage VMID has been 1230 * updated using kvm_riscv_gstage_vmid_ver_changed() 1231 */ 1232 kvm_riscv_local_tlb_sanitize(vcpu); 1233 1234 guest_timing_enter_irqoff(); 1235 1236 kvm_riscv_vcpu_enter_exit(vcpu); 1237 1238 vcpu->mode = OUTSIDE_GUEST_MODE; 1239 vcpu->stat.exits++; 1240 1241 /* 1242 * Save SCAUSE, STVAL, HTVAL, and HTINST because we might 1243 * get an interrupt between __kvm_riscv_switch_to() and 1244 * local_irq_enable() which can potentially change CSRs. 1245 */ 1246 trap.sepc = vcpu->arch.guest_context.sepc; 1247 trap.scause = csr_read(CSR_SCAUSE); 1248 trap.stval = csr_read(CSR_STVAL); 1249 trap.htval = csr_read(CSR_HTVAL); 1250 trap.htinst = csr_read(CSR_HTINST); 1251 1252 /* Syncup interrupts state with HW */ 1253 kvm_riscv_vcpu_sync_interrupts(vcpu); 1254 1255 /* 1256 * We must ensure that any pending interrupts are taken before 1257 * we exit guest timing so that timer ticks are accounted as 1258 * guest time. Transiently unmask interrupts so that any 1259 * pending interrupts are taken. 1260 * 1261 * There's no barrier which ensures that pending interrupts are 1262 * recognised, so we just hope that the CPU takes any pending 1263 * interrupts between the enable and disable. 1264 */ 1265 local_irq_enable(); 1266 local_irq_disable(); 1267 1268 guest_timing_exit_irqoff(); 1269 1270 local_irq_enable(); 1271 1272 preempt_enable(); 1273 1274 kvm_vcpu_srcu_read_lock(vcpu); 1275 1276 ret = kvm_riscv_vcpu_exit(vcpu, run, &trap); 1277 } 1278 1279 kvm_sigset_deactivate(vcpu); 1280 1281 vcpu_put(vcpu); 1282 1283 kvm_vcpu_srcu_read_unlock(vcpu); 1284 1285 return ret; 1286 } 1287