1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (C) 2020-2023 Loongson Technology Corporation Limited 4 */ 5 6 #include <linux/kvm_host.h> 7 #include <linux/entry-kvm.h> 8 #include <asm/fpu.h> 9 #include <asm/loongarch.h> 10 #include <asm/setup.h> 11 #include <asm/time.h> 12 13 #define CREATE_TRACE_POINTS 14 #include "trace.h" 15 16 const struct _kvm_stats_desc kvm_vcpu_stats_desc[] = { 17 KVM_GENERIC_VCPU_STATS(), 18 STATS_DESC_COUNTER(VCPU, int_exits), 19 STATS_DESC_COUNTER(VCPU, idle_exits), 20 STATS_DESC_COUNTER(VCPU, cpucfg_exits), 21 STATS_DESC_COUNTER(VCPU, signal_exits), 22 }; 23 24 const struct kvm_stats_header kvm_vcpu_stats_header = { 25 .name_size = KVM_STATS_NAME_SIZE, 26 .num_desc = ARRAY_SIZE(kvm_vcpu_stats_desc), 27 .id_offset = sizeof(struct kvm_stats_header), 28 .desc_offset = sizeof(struct kvm_stats_header) + KVM_STATS_NAME_SIZE, 29 .data_offset = sizeof(struct kvm_stats_header) + KVM_STATS_NAME_SIZE + 30 sizeof(kvm_vcpu_stats_desc), 31 }; 32 33 /* 34 * kvm_check_requests - check and handle pending vCPU requests 35 * 36 * Return: RESUME_GUEST if we should enter the guest 37 * RESUME_HOST if we should exit to userspace 38 */ 39 static int kvm_check_requests(struct kvm_vcpu *vcpu) 40 { 41 if (!kvm_request_pending(vcpu)) 42 return RESUME_GUEST; 43 44 if (kvm_check_request(KVM_REQ_TLB_FLUSH, vcpu)) 45 vcpu->arch.vpid = 0; /* Drop vpid for this vCPU */ 46 47 if (kvm_dirty_ring_check_request(vcpu)) 48 return RESUME_HOST; 49 50 return RESUME_GUEST; 51 } 52 53 /* 54 * Check and handle pending signal and vCPU requests etc 55 * Run with irq enabled and preempt enabled 56 * 57 * Return: RESUME_GUEST if we should enter the guest 58 * RESUME_HOST if we should exit to userspace 59 * < 0 if we should exit to userspace, where the return value 60 * indicates an error 61 */ 62 static int kvm_enter_guest_check(struct kvm_vcpu *vcpu) 63 { 64 int ret; 65 66 /* 67 * Check conditions before entering the guest 68 */ 69 ret = xfer_to_guest_mode_handle_work(vcpu); 70 if (ret < 0) 71 return ret; 72 73 ret = kvm_check_requests(vcpu); 74 75 return ret; 76 } 77 78 /* 79 * Called with irq enabled 80 * 81 * Return: RESUME_GUEST if we should enter the guest, and irq disabled 82 * Others if we should exit to userspace 83 */ 84 static int kvm_pre_enter_guest(struct kvm_vcpu *vcpu) 85 { 86 int ret; 87 88 do { 89 ret = kvm_enter_guest_check(vcpu); 90 if (ret != RESUME_GUEST) 91 break; 92 93 /* 94 * Handle vcpu timer, interrupts, check requests and 95 * check vmid before vcpu enter guest 96 */ 97 local_irq_disable(); 98 kvm_acquire_timer(vcpu); 99 kvm_deliver_intr(vcpu); 100 kvm_deliver_exception(vcpu); 101 /* Make sure the vcpu mode has been written */ 102 smp_store_mb(vcpu->mode, IN_GUEST_MODE); 103 kvm_check_vpid(vcpu); 104 vcpu->arch.host_eentry = csr_read64(LOONGARCH_CSR_EENTRY); 105 /* Clear KVM_LARCH_SWCSR_LATEST as CSR will change when enter guest */ 106 vcpu->arch.aux_inuse &= ~KVM_LARCH_SWCSR_LATEST; 107 108 if (kvm_request_pending(vcpu) || xfer_to_guest_mode_work_pending()) { 109 /* make sure the vcpu mode has been written */ 110 smp_store_mb(vcpu->mode, OUTSIDE_GUEST_MODE); 111 local_irq_enable(); 112 ret = -EAGAIN; 113 } 114 } while (ret != RESUME_GUEST); 115 116 return ret; 117 } 118 119 /* 120 * Return 1 for resume guest and "<= 0" for resume host. 121 */ 122 static int kvm_handle_exit(struct kvm_run *run, struct kvm_vcpu *vcpu) 123 { 124 int ret = RESUME_GUEST; 125 unsigned long estat = vcpu->arch.host_estat; 126 u32 intr = estat & 0x1fff; /* Ignore NMI */ 127 u32 ecode = (estat & CSR_ESTAT_EXC) >> CSR_ESTAT_EXC_SHIFT; 128 129 vcpu->mode = OUTSIDE_GUEST_MODE; 130 131 /* Set a default exit reason */ 132 run->exit_reason = KVM_EXIT_UNKNOWN; 133 134 guest_timing_exit_irqoff(); 135 guest_state_exit_irqoff(); 136 local_irq_enable(); 137 138 trace_kvm_exit(vcpu, ecode); 139 if (ecode) { 140 ret = kvm_handle_fault(vcpu, ecode); 141 } else { 142 WARN(!intr, "vm exiting with suspicious irq\n"); 143 ++vcpu->stat.int_exits; 144 } 145 146 if (ret == RESUME_GUEST) 147 ret = kvm_pre_enter_guest(vcpu); 148 149 if (ret != RESUME_GUEST) { 150 local_irq_disable(); 151 return ret; 152 } 153 154 guest_timing_enter_irqoff(); 155 guest_state_enter_irqoff(); 156 trace_kvm_reenter(vcpu); 157 158 return RESUME_GUEST; 159 } 160 161 int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu) 162 { 163 return !!(vcpu->arch.irq_pending) && 164 vcpu->arch.mp_state.mp_state == KVM_MP_STATE_RUNNABLE; 165 } 166 167 int kvm_arch_vcpu_should_kick(struct kvm_vcpu *vcpu) 168 { 169 return kvm_vcpu_exiting_guest_mode(vcpu) == IN_GUEST_MODE; 170 } 171 172 bool kvm_arch_vcpu_in_kernel(struct kvm_vcpu *vcpu) 173 { 174 return false; 175 } 176 177 vm_fault_t kvm_arch_vcpu_fault(struct kvm_vcpu *vcpu, struct vm_fault *vmf) 178 { 179 return VM_FAULT_SIGBUS; 180 } 181 182 int kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu *vcpu, 183 struct kvm_translation *tr) 184 { 185 return -EINVAL; 186 } 187 188 int kvm_cpu_has_pending_timer(struct kvm_vcpu *vcpu) 189 { 190 return kvm_pending_timer(vcpu) || 191 kvm_read_hw_gcsr(LOONGARCH_CSR_ESTAT) & (1 << INT_TI); 192 } 193 194 int kvm_arch_vcpu_dump_regs(struct kvm_vcpu *vcpu) 195 { 196 int i; 197 198 kvm_debug("vCPU Register Dump:\n"); 199 kvm_debug("\tPC = 0x%08lx\n", vcpu->arch.pc); 200 kvm_debug("\tExceptions: %08lx\n", vcpu->arch.irq_pending); 201 202 for (i = 0; i < 32; i += 4) { 203 kvm_debug("\tGPR%02d: %08lx %08lx %08lx %08lx\n", i, 204 vcpu->arch.gprs[i], vcpu->arch.gprs[i + 1], 205 vcpu->arch.gprs[i + 2], vcpu->arch.gprs[i + 3]); 206 } 207 208 kvm_debug("\tCRMD: 0x%08lx, ESTAT: 0x%08lx\n", 209 kvm_read_hw_gcsr(LOONGARCH_CSR_CRMD), 210 kvm_read_hw_gcsr(LOONGARCH_CSR_ESTAT)); 211 212 kvm_debug("\tERA: 0x%08lx\n", kvm_read_hw_gcsr(LOONGARCH_CSR_ERA)); 213 214 return 0; 215 } 216 217 int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu, 218 struct kvm_mp_state *mp_state) 219 { 220 *mp_state = vcpu->arch.mp_state; 221 222 return 0; 223 } 224 225 int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu, 226 struct kvm_mp_state *mp_state) 227 { 228 int ret = 0; 229 230 switch (mp_state->mp_state) { 231 case KVM_MP_STATE_RUNNABLE: 232 vcpu->arch.mp_state = *mp_state; 233 break; 234 default: 235 ret = -EINVAL; 236 } 237 238 return ret; 239 } 240 241 int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu, 242 struct kvm_guest_debug *dbg) 243 { 244 return -EINVAL; 245 } 246 247 /** 248 * kvm_migrate_count() - Migrate timer. 249 * @vcpu: Virtual CPU. 250 * 251 * Migrate hrtimer to the current CPU by cancelling and restarting it 252 * if the hrtimer is active. 253 * 254 * Must be called when the vCPU is migrated to a different CPU, so that 255 * the timer can interrupt the guest at the new CPU, and the timer irq can 256 * be delivered to the vCPU. 257 */ 258 static void kvm_migrate_count(struct kvm_vcpu *vcpu) 259 { 260 if (hrtimer_cancel(&vcpu->arch.swtimer)) 261 hrtimer_restart(&vcpu->arch.swtimer); 262 } 263 264 static int _kvm_getcsr(struct kvm_vcpu *vcpu, unsigned int id, u64 *val) 265 { 266 unsigned long gintc; 267 struct loongarch_csrs *csr = vcpu->arch.csr; 268 269 if (get_gcsr_flag(id) & INVALID_GCSR) 270 return -EINVAL; 271 272 if (id == LOONGARCH_CSR_ESTAT) { 273 /* ESTAT IP0~IP7 get from GINTC */ 274 gintc = kvm_read_sw_gcsr(csr, LOONGARCH_CSR_GINTC) & 0xff; 275 *val = kvm_read_sw_gcsr(csr, LOONGARCH_CSR_ESTAT) | (gintc << 2); 276 return 0; 277 } 278 279 /* 280 * Get software CSR state since software state is consistent 281 * with hardware for synchronous ioctl 282 */ 283 *val = kvm_read_sw_gcsr(csr, id); 284 285 return 0; 286 } 287 288 static int _kvm_setcsr(struct kvm_vcpu *vcpu, unsigned int id, u64 val) 289 { 290 int ret = 0, gintc; 291 struct loongarch_csrs *csr = vcpu->arch.csr; 292 293 if (get_gcsr_flag(id) & INVALID_GCSR) 294 return -EINVAL; 295 296 if (id == LOONGARCH_CSR_ESTAT) { 297 /* ESTAT IP0~IP7 inject through GINTC */ 298 gintc = (val >> 2) & 0xff; 299 kvm_set_sw_gcsr(csr, LOONGARCH_CSR_GINTC, gintc); 300 301 gintc = val & ~(0xffUL << 2); 302 kvm_set_sw_gcsr(csr, LOONGARCH_CSR_ESTAT, gintc); 303 304 return ret; 305 } 306 307 kvm_write_sw_gcsr(csr, id, val); 308 309 return ret; 310 } 311 312 static int kvm_get_one_reg(struct kvm_vcpu *vcpu, 313 const struct kvm_one_reg *reg, u64 *v) 314 { 315 int id, ret = 0; 316 u64 type = reg->id & KVM_REG_LOONGARCH_MASK; 317 318 switch (type) { 319 case KVM_REG_LOONGARCH_CSR: 320 id = KVM_GET_IOC_CSR_IDX(reg->id); 321 ret = _kvm_getcsr(vcpu, id, v); 322 break; 323 case KVM_REG_LOONGARCH_CPUCFG: 324 id = KVM_GET_IOC_CPUCFG_IDX(reg->id); 325 if (id >= 0 && id < KVM_MAX_CPUCFG_REGS) 326 *v = vcpu->arch.cpucfg[id]; 327 else 328 ret = -EINVAL; 329 break; 330 case KVM_REG_LOONGARCH_KVM: 331 switch (reg->id) { 332 case KVM_REG_LOONGARCH_COUNTER: 333 *v = drdtime() + vcpu->kvm->arch.time_offset; 334 break; 335 default: 336 ret = -EINVAL; 337 break; 338 } 339 break; 340 default: 341 ret = -EINVAL; 342 break; 343 } 344 345 return ret; 346 } 347 348 static int kvm_get_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg) 349 { 350 int ret = 0; 351 u64 v, size = reg->id & KVM_REG_SIZE_MASK; 352 353 switch (size) { 354 case KVM_REG_SIZE_U64: 355 ret = kvm_get_one_reg(vcpu, reg, &v); 356 if (ret) 357 return ret; 358 ret = put_user(v, (u64 __user *)(long)reg->addr); 359 break; 360 default: 361 ret = -EINVAL; 362 break; 363 } 364 365 return ret; 366 } 367 368 static int kvm_set_one_reg(struct kvm_vcpu *vcpu, 369 const struct kvm_one_reg *reg, u64 v) 370 { 371 int id, ret = 0; 372 u64 type = reg->id & KVM_REG_LOONGARCH_MASK; 373 374 switch (type) { 375 case KVM_REG_LOONGARCH_CSR: 376 id = KVM_GET_IOC_CSR_IDX(reg->id); 377 ret = _kvm_setcsr(vcpu, id, v); 378 break; 379 case KVM_REG_LOONGARCH_CPUCFG: 380 id = KVM_GET_IOC_CPUCFG_IDX(reg->id); 381 if (id >= 0 && id < KVM_MAX_CPUCFG_REGS) 382 vcpu->arch.cpucfg[id] = (u32)v; 383 else 384 ret = -EINVAL; 385 break; 386 case KVM_REG_LOONGARCH_KVM: 387 switch (reg->id) { 388 case KVM_REG_LOONGARCH_COUNTER: 389 /* 390 * gftoffset is relative with board, not vcpu 391 * only set for the first time for smp system 392 */ 393 if (vcpu->vcpu_id == 0) 394 vcpu->kvm->arch.time_offset = (signed long)(v - drdtime()); 395 break; 396 case KVM_REG_LOONGARCH_VCPU_RESET: 397 kvm_reset_timer(vcpu); 398 memset(&vcpu->arch.irq_pending, 0, sizeof(vcpu->arch.irq_pending)); 399 memset(&vcpu->arch.irq_clear, 0, sizeof(vcpu->arch.irq_clear)); 400 break; 401 default: 402 ret = -EINVAL; 403 break; 404 } 405 break; 406 default: 407 ret = -EINVAL; 408 break; 409 } 410 411 return ret; 412 } 413 414 static int kvm_set_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg) 415 { 416 int ret = 0; 417 u64 v, size = reg->id & KVM_REG_SIZE_MASK; 418 419 switch (size) { 420 case KVM_REG_SIZE_U64: 421 ret = get_user(v, (u64 __user *)(long)reg->addr); 422 if (ret) 423 return ret; 424 break; 425 default: 426 return -EINVAL; 427 } 428 429 return kvm_set_one_reg(vcpu, reg, v); 430 } 431 432 int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs) 433 { 434 return -ENOIOCTLCMD; 435 } 436 437 int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs) 438 { 439 return -ENOIOCTLCMD; 440 } 441 442 int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs) 443 { 444 int i; 445 446 for (i = 0; i < ARRAY_SIZE(vcpu->arch.gprs); i++) 447 regs->gpr[i] = vcpu->arch.gprs[i]; 448 449 regs->pc = vcpu->arch.pc; 450 451 return 0; 452 } 453 454 int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs) 455 { 456 int i; 457 458 for (i = 1; i < ARRAY_SIZE(vcpu->arch.gprs); i++) 459 vcpu->arch.gprs[i] = regs->gpr[i]; 460 461 vcpu->arch.gprs[0] = 0; /* zero is special, and cannot be set. */ 462 vcpu->arch.pc = regs->pc; 463 464 return 0; 465 } 466 467 static int kvm_vcpu_ioctl_enable_cap(struct kvm_vcpu *vcpu, 468 struct kvm_enable_cap *cap) 469 { 470 /* FPU is enabled by default, will support LSX/LASX later. */ 471 return -EINVAL; 472 } 473 474 long kvm_arch_vcpu_ioctl(struct file *filp, 475 unsigned int ioctl, unsigned long arg) 476 { 477 long r; 478 void __user *argp = (void __user *)arg; 479 struct kvm_vcpu *vcpu = filp->private_data; 480 481 /* 482 * Only software CSR should be modified 483 * 484 * If any hardware CSR register is modified, vcpu_load/vcpu_put pair 485 * should be used. Since CSR registers owns by this vcpu, if switch 486 * to other vcpus, other vcpus need reload CSR registers. 487 * 488 * If software CSR is modified, bit KVM_LARCH_HWCSR_USABLE should 489 * be clear in vcpu->arch.aux_inuse, and vcpu_load will check 490 * aux_inuse flag and reload CSR registers form software. 491 */ 492 493 switch (ioctl) { 494 case KVM_SET_ONE_REG: 495 case KVM_GET_ONE_REG: { 496 struct kvm_one_reg reg; 497 498 r = -EFAULT; 499 if (copy_from_user(®, argp, sizeof(reg))) 500 break; 501 if (ioctl == KVM_SET_ONE_REG) { 502 r = kvm_set_reg(vcpu, ®); 503 vcpu->arch.aux_inuse &= ~KVM_LARCH_HWCSR_USABLE; 504 } else 505 r = kvm_get_reg(vcpu, ®); 506 break; 507 } 508 case KVM_ENABLE_CAP: { 509 struct kvm_enable_cap cap; 510 511 r = -EFAULT; 512 if (copy_from_user(&cap, argp, sizeof(cap))) 513 break; 514 r = kvm_vcpu_ioctl_enable_cap(vcpu, &cap); 515 break; 516 } 517 default: 518 r = -ENOIOCTLCMD; 519 break; 520 } 521 522 return r; 523 } 524 525 int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu) 526 { 527 int i = 0; 528 529 fpu->fcc = vcpu->arch.fpu.fcc; 530 fpu->fcsr = vcpu->arch.fpu.fcsr; 531 for (i = 0; i < NUM_FPU_REGS; i++) 532 memcpy(&fpu->fpr[i], &vcpu->arch.fpu.fpr[i], FPU_REG_WIDTH / 64); 533 534 return 0; 535 } 536 537 int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu) 538 { 539 int i = 0; 540 541 vcpu->arch.fpu.fcc = fpu->fcc; 542 vcpu->arch.fpu.fcsr = fpu->fcsr; 543 for (i = 0; i < NUM_FPU_REGS; i++) 544 memcpy(&vcpu->arch.fpu.fpr[i], &fpu->fpr[i], FPU_REG_WIDTH / 64); 545 546 return 0; 547 } 548 549 /* Enable FPU and restore context */ 550 void kvm_own_fpu(struct kvm_vcpu *vcpu) 551 { 552 preempt_disable(); 553 554 /* Enable FPU */ 555 set_csr_euen(CSR_EUEN_FPEN); 556 557 kvm_restore_fpu(&vcpu->arch.fpu); 558 vcpu->arch.aux_inuse |= KVM_LARCH_FPU; 559 trace_kvm_aux(vcpu, KVM_TRACE_AUX_RESTORE, KVM_TRACE_AUX_FPU); 560 561 preempt_enable(); 562 } 563 564 /* Save context and disable FPU */ 565 void kvm_lose_fpu(struct kvm_vcpu *vcpu) 566 { 567 preempt_disable(); 568 569 if (vcpu->arch.aux_inuse & KVM_LARCH_FPU) { 570 kvm_save_fpu(&vcpu->arch.fpu); 571 vcpu->arch.aux_inuse &= ~KVM_LARCH_FPU; 572 trace_kvm_aux(vcpu, KVM_TRACE_AUX_SAVE, KVM_TRACE_AUX_FPU); 573 574 /* Disable FPU */ 575 clear_csr_euen(CSR_EUEN_FPEN); 576 } 577 578 preempt_enable(); 579 } 580 581 int kvm_vcpu_ioctl_interrupt(struct kvm_vcpu *vcpu, struct kvm_interrupt *irq) 582 { 583 int intr = (int)irq->irq; 584 585 if (intr > 0) 586 kvm_queue_irq(vcpu, intr); 587 else if (intr < 0) 588 kvm_dequeue_irq(vcpu, -intr); 589 else { 590 kvm_err("%s: invalid interrupt ioctl %d\n", __func__, irq->irq); 591 return -EINVAL; 592 } 593 594 kvm_vcpu_kick(vcpu); 595 596 return 0; 597 } 598 599 long kvm_arch_vcpu_async_ioctl(struct file *filp, 600 unsigned int ioctl, unsigned long arg) 601 { 602 void __user *argp = (void __user *)arg; 603 struct kvm_vcpu *vcpu = filp->private_data; 604 605 if (ioctl == KVM_INTERRUPT) { 606 struct kvm_interrupt irq; 607 608 if (copy_from_user(&irq, argp, sizeof(irq))) 609 return -EFAULT; 610 611 kvm_debug("[%d] %s: irq: %d\n", vcpu->vcpu_id, __func__, irq.irq); 612 613 return kvm_vcpu_ioctl_interrupt(vcpu, &irq); 614 } 615 616 return -ENOIOCTLCMD; 617 } 618 619 int kvm_arch_vcpu_precreate(struct kvm *kvm, unsigned int id) 620 { 621 return 0; 622 } 623 624 int kvm_arch_vcpu_create(struct kvm_vcpu *vcpu) 625 { 626 unsigned long timer_hz; 627 struct loongarch_csrs *csr; 628 629 vcpu->arch.vpid = 0; 630 631 hrtimer_init(&vcpu->arch.swtimer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS_PINNED); 632 vcpu->arch.swtimer.function = kvm_swtimer_wakeup; 633 634 vcpu->arch.handle_exit = kvm_handle_exit; 635 vcpu->arch.guest_eentry = (unsigned long)kvm_loongarch_ops->exc_entry; 636 vcpu->arch.csr = kzalloc(sizeof(struct loongarch_csrs), GFP_KERNEL); 637 if (!vcpu->arch.csr) 638 return -ENOMEM; 639 640 /* 641 * All kvm exceptions share one exception entry, and host <-> guest 642 * switch also switch ECFG.VS field, keep host ECFG.VS info here. 643 */ 644 vcpu->arch.host_ecfg = (read_csr_ecfg() & CSR_ECFG_VS); 645 646 /* Init */ 647 vcpu->arch.last_sched_cpu = -1; 648 649 /* 650 * Initialize guest register state to valid architectural reset state. 651 */ 652 timer_hz = calc_const_freq(); 653 kvm_init_timer(vcpu, timer_hz); 654 655 /* Set Initialize mode for guest */ 656 csr = vcpu->arch.csr; 657 kvm_write_sw_gcsr(csr, LOONGARCH_CSR_CRMD, CSR_CRMD_DA); 658 659 /* Set cpuid */ 660 kvm_write_sw_gcsr(csr, LOONGARCH_CSR_TMID, vcpu->vcpu_id); 661 662 /* Start with no pending virtual guest interrupts */ 663 csr->csrs[LOONGARCH_CSR_GINTC] = 0; 664 665 return 0; 666 } 667 668 void kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu) 669 { 670 } 671 672 void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu) 673 { 674 int cpu; 675 struct kvm_context *context; 676 677 hrtimer_cancel(&vcpu->arch.swtimer); 678 kvm_mmu_free_memory_cache(&vcpu->arch.mmu_page_cache); 679 kfree(vcpu->arch.csr); 680 681 /* 682 * If the vCPU is freed and reused as another vCPU, we don't want the 683 * matching pointer wrongly hanging around in last_vcpu. 684 */ 685 for_each_possible_cpu(cpu) { 686 context = per_cpu_ptr(vcpu->kvm->arch.vmcs, cpu); 687 if (context->last_vcpu == vcpu) 688 context->last_vcpu = NULL; 689 } 690 } 691 692 static int _kvm_vcpu_load(struct kvm_vcpu *vcpu, int cpu) 693 { 694 bool migrated; 695 struct kvm_context *context; 696 struct loongarch_csrs *csr = vcpu->arch.csr; 697 698 /* 699 * Have we migrated to a different CPU? 700 * If so, any old guest TLB state may be stale. 701 */ 702 migrated = (vcpu->arch.last_sched_cpu != cpu); 703 704 /* 705 * Was this the last vCPU to run on this CPU? 706 * If not, any old guest state from this vCPU will have been clobbered. 707 */ 708 context = per_cpu_ptr(vcpu->kvm->arch.vmcs, cpu); 709 if (migrated || (context->last_vcpu != vcpu)) 710 vcpu->arch.aux_inuse &= ~KVM_LARCH_HWCSR_USABLE; 711 context->last_vcpu = vcpu; 712 713 /* Restore timer state regardless */ 714 kvm_restore_timer(vcpu); 715 716 /* Control guest page CCA attribute */ 717 change_csr_gcfg(CSR_GCFG_MATC_MASK, CSR_GCFG_MATC_ROOT); 718 719 /* Don't bother restoring registers multiple times unless necessary */ 720 if (vcpu->arch.aux_inuse & KVM_LARCH_HWCSR_USABLE) 721 return 0; 722 723 write_csr_gcntc((ulong)vcpu->kvm->arch.time_offset); 724 725 /* Restore guest CSR registers */ 726 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_CRMD); 727 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_PRMD); 728 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_EUEN); 729 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_MISC); 730 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_ECFG); 731 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_ERA); 732 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_BADV); 733 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_BADI); 734 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_EENTRY); 735 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_TLBIDX); 736 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_TLBEHI); 737 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_TLBELO0); 738 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_TLBELO1); 739 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_ASID); 740 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_PGDL); 741 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_PGDH); 742 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_PWCTL0); 743 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_PWCTL1); 744 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_STLBPGSIZE); 745 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_RVACFG); 746 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_CPUID); 747 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_KS0); 748 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_KS1); 749 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_KS2); 750 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_KS3); 751 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_KS4); 752 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_KS5); 753 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_KS6); 754 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_KS7); 755 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_TMID); 756 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_CNTC); 757 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_TLBRENTRY); 758 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_TLBRBADV); 759 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_TLBRERA); 760 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_TLBRSAVE); 761 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_TLBRELO0); 762 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_TLBRELO1); 763 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_TLBREHI); 764 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_TLBRPRMD); 765 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_DMWIN0); 766 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_DMWIN1); 767 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_DMWIN2); 768 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_DMWIN3); 769 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_LLBCTL); 770 771 /* Restore Root.GINTC from unused Guest.GINTC register */ 772 write_csr_gintc(csr->csrs[LOONGARCH_CSR_GINTC]); 773 774 /* 775 * We should clear linked load bit to break interrupted atomics. This 776 * prevents a SC on the next vCPU from succeeding by matching a LL on 777 * the previous vCPU. 778 */ 779 if (vcpu->kvm->created_vcpus > 1) 780 set_gcsr_llbctl(CSR_LLBCTL_WCLLB); 781 782 vcpu->arch.aux_inuse |= KVM_LARCH_HWCSR_USABLE; 783 784 return 0; 785 } 786 787 void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu) 788 { 789 unsigned long flags; 790 791 local_irq_save(flags); 792 if (vcpu->arch.last_sched_cpu != cpu) { 793 kvm_debug("[%d->%d]KVM vCPU[%d] switch\n", 794 vcpu->arch.last_sched_cpu, cpu, vcpu->vcpu_id); 795 /* 796 * Migrate the timer interrupt to the current CPU so that it 797 * always interrupts the guest and synchronously triggers a 798 * guest timer interrupt. 799 */ 800 kvm_migrate_count(vcpu); 801 } 802 803 /* Restore guest state to registers */ 804 _kvm_vcpu_load(vcpu, cpu); 805 local_irq_restore(flags); 806 } 807 808 static int _kvm_vcpu_put(struct kvm_vcpu *vcpu, int cpu) 809 { 810 struct loongarch_csrs *csr = vcpu->arch.csr; 811 812 kvm_lose_fpu(vcpu); 813 814 /* 815 * Update CSR state from hardware if software CSR state is stale, 816 * most CSR registers are kept unchanged during process context 817 * switch except CSR registers like remaining timer tick value and 818 * injected interrupt state. 819 */ 820 if (vcpu->arch.aux_inuse & KVM_LARCH_SWCSR_LATEST) 821 goto out; 822 823 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_CRMD); 824 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_PRMD); 825 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_EUEN); 826 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_MISC); 827 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_ECFG); 828 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_ERA); 829 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_BADV); 830 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_BADI); 831 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_EENTRY); 832 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_TLBIDX); 833 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_TLBEHI); 834 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_TLBELO0); 835 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_TLBELO1); 836 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_ASID); 837 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_PGDL); 838 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_PGDH); 839 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_PWCTL0); 840 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_PWCTL1); 841 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_STLBPGSIZE); 842 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_RVACFG); 843 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_CPUID); 844 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_PRCFG1); 845 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_PRCFG2); 846 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_PRCFG3); 847 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_KS0); 848 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_KS1); 849 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_KS2); 850 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_KS3); 851 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_KS4); 852 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_KS5); 853 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_KS6); 854 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_KS7); 855 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_TMID); 856 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_CNTC); 857 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_LLBCTL); 858 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_TLBRENTRY); 859 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_TLBRBADV); 860 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_TLBRERA); 861 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_TLBRSAVE); 862 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_TLBRELO0); 863 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_TLBRELO1); 864 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_TLBREHI); 865 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_TLBRPRMD); 866 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_DMWIN0); 867 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_DMWIN1); 868 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_DMWIN2); 869 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_DMWIN3); 870 871 vcpu->arch.aux_inuse |= KVM_LARCH_SWCSR_LATEST; 872 873 out: 874 kvm_save_timer(vcpu); 875 /* Save Root.GINTC into unused Guest.GINTC register */ 876 csr->csrs[LOONGARCH_CSR_GINTC] = read_csr_gintc(); 877 878 return 0; 879 } 880 881 void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu) 882 { 883 int cpu; 884 unsigned long flags; 885 886 local_irq_save(flags); 887 cpu = smp_processor_id(); 888 vcpu->arch.last_sched_cpu = cpu; 889 890 /* Save guest state in registers */ 891 _kvm_vcpu_put(vcpu, cpu); 892 local_irq_restore(flags); 893 } 894 895 int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu) 896 { 897 int r = -EINTR; 898 struct kvm_run *run = vcpu->run; 899 900 if (vcpu->mmio_needed) { 901 if (!vcpu->mmio_is_write) 902 kvm_complete_mmio_read(vcpu, run); 903 vcpu->mmio_needed = 0; 904 } 905 906 if (run->exit_reason == KVM_EXIT_LOONGARCH_IOCSR) { 907 if (!run->iocsr_io.is_write) 908 kvm_complete_iocsr_read(vcpu, run); 909 } 910 911 if (run->immediate_exit) 912 return r; 913 914 /* Clear exit_reason */ 915 run->exit_reason = KVM_EXIT_UNKNOWN; 916 lose_fpu(1); 917 vcpu_load(vcpu); 918 kvm_sigset_activate(vcpu); 919 r = kvm_pre_enter_guest(vcpu); 920 if (r != RESUME_GUEST) 921 goto out; 922 923 guest_timing_enter_irqoff(); 924 guest_state_enter_irqoff(); 925 trace_kvm_enter(vcpu); 926 r = kvm_loongarch_ops->enter_guest(run, vcpu); 927 928 trace_kvm_out(vcpu); 929 /* 930 * Guest exit is already recorded at kvm_handle_exit() 931 * return value must not be RESUME_GUEST 932 */ 933 local_irq_enable(); 934 out: 935 kvm_sigset_deactivate(vcpu); 936 vcpu_put(vcpu); 937 938 return r; 939 } 940