1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (C) 2020-2023 Loongson Technology Corporation Limited 4 */ 5 6 #include <linux/kvm_host.h> 7 #include <asm/fpu.h> 8 #include <asm/lbt.h> 9 #include <asm/loongarch.h> 10 #include <asm/setup.h> 11 #include <asm/time.h> 12 #include <asm/timex.h> 13 14 #define CREATE_TRACE_POINTS 15 #include "trace.h" 16 17 const struct kvm_stats_desc kvm_vcpu_stats_desc[] = { 18 KVM_GENERIC_VCPU_STATS(), 19 STATS_DESC_COUNTER(VCPU, int_exits), 20 STATS_DESC_COUNTER(VCPU, idle_exits), 21 STATS_DESC_COUNTER(VCPU, cpucfg_exits), 22 STATS_DESC_COUNTER(VCPU, signal_exits), 23 STATS_DESC_COUNTER(VCPU, hypercall_exits), 24 STATS_DESC_COUNTER(VCPU, ipi_read_exits), 25 STATS_DESC_COUNTER(VCPU, ipi_write_exits), 26 STATS_DESC_COUNTER(VCPU, eiointc_read_exits), 27 STATS_DESC_COUNTER(VCPU, eiointc_write_exits), 28 STATS_DESC_COUNTER(VCPU, pch_pic_read_exits), 29 STATS_DESC_COUNTER(VCPU, pch_pic_write_exits) 30 }; 31 32 const struct kvm_stats_header kvm_vcpu_stats_header = { 33 .name_size = KVM_STATS_NAME_SIZE, 34 .num_desc = ARRAY_SIZE(kvm_vcpu_stats_desc), 35 .id_offset = sizeof(struct kvm_stats_header), 36 .desc_offset = sizeof(struct kvm_stats_header) + KVM_STATS_NAME_SIZE, 37 .data_offset = sizeof(struct kvm_stats_header) + KVM_STATS_NAME_SIZE + 38 sizeof(kvm_vcpu_stats_desc), 39 }; 40 41 static inline void kvm_save_host_pmu(struct kvm_vcpu *vcpu) 42 { 43 struct kvm_context *context; 44 45 context = this_cpu_ptr(vcpu->kvm->arch.vmcs); 46 context->perf_cntr[0] = read_csr_perfcntr0(); 47 context->perf_cntr[1] = read_csr_perfcntr1(); 48 context->perf_cntr[2] = read_csr_perfcntr2(); 49 context->perf_cntr[3] = read_csr_perfcntr3(); 50 context->perf_ctrl[0] = write_csr_perfctrl0(0); 51 context->perf_ctrl[1] = write_csr_perfctrl1(0); 52 context->perf_ctrl[2] = write_csr_perfctrl2(0); 53 context->perf_ctrl[3] = write_csr_perfctrl3(0); 54 } 55 56 static inline void kvm_restore_host_pmu(struct kvm_vcpu *vcpu) 57 { 58 struct kvm_context *context; 59 60 context = this_cpu_ptr(vcpu->kvm->arch.vmcs); 61 write_csr_perfcntr0(context->perf_cntr[0]); 62 write_csr_perfcntr1(context->perf_cntr[1]); 63 write_csr_perfcntr2(context->perf_cntr[2]); 64 write_csr_perfcntr3(context->perf_cntr[3]); 65 write_csr_perfctrl0(context->perf_ctrl[0]); 66 write_csr_perfctrl1(context->perf_ctrl[1]); 67 write_csr_perfctrl2(context->perf_ctrl[2]); 68 write_csr_perfctrl3(context->perf_ctrl[3]); 69 } 70 71 72 static inline void kvm_save_guest_pmu(struct kvm_vcpu *vcpu) 73 { 74 struct loongarch_csrs *csr = vcpu->arch.csr; 75 76 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_PERFCNTR0); 77 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_PERFCNTR1); 78 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_PERFCNTR2); 79 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_PERFCNTR3); 80 kvm_read_clear_hw_gcsr(csr, LOONGARCH_CSR_PERFCTRL0); 81 kvm_read_clear_hw_gcsr(csr, LOONGARCH_CSR_PERFCTRL1); 82 kvm_read_clear_hw_gcsr(csr, LOONGARCH_CSR_PERFCTRL2); 83 kvm_read_clear_hw_gcsr(csr, LOONGARCH_CSR_PERFCTRL3); 84 } 85 86 static inline void kvm_restore_guest_pmu(struct kvm_vcpu *vcpu) 87 { 88 struct loongarch_csrs *csr = vcpu->arch.csr; 89 90 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_PERFCNTR0); 91 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_PERFCNTR1); 92 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_PERFCNTR2); 93 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_PERFCNTR3); 94 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_PERFCTRL0); 95 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_PERFCTRL1); 96 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_PERFCTRL2); 97 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_PERFCTRL3); 98 } 99 100 static int kvm_own_pmu(struct kvm_vcpu *vcpu) 101 { 102 unsigned long val; 103 104 if (!kvm_guest_has_pmu(&vcpu->arch)) 105 return -EINVAL; 106 107 kvm_save_host_pmu(vcpu); 108 109 /* Set PM0-PM(num) to guest */ 110 val = read_csr_gcfg() & ~CSR_GCFG_GPERF; 111 val |= (kvm_get_pmu_num(&vcpu->arch) + 1) << CSR_GCFG_GPERF_SHIFT; 112 write_csr_gcfg(val); 113 114 kvm_restore_guest_pmu(vcpu); 115 116 return 0; 117 } 118 119 static void kvm_lose_pmu(struct kvm_vcpu *vcpu) 120 { 121 unsigned long val; 122 struct loongarch_csrs *csr = vcpu->arch.csr; 123 124 if (!(vcpu->arch.aux_inuse & KVM_LARCH_PMU)) 125 return; 126 127 kvm_save_guest_pmu(vcpu); 128 129 /* Disable pmu access from guest */ 130 write_csr_gcfg(read_csr_gcfg() & ~CSR_GCFG_GPERF); 131 132 /* 133 * Clear KVM_LARCH_PMU if the guest is not using PMU CSRs when 134 * exiting the guest, so that the next time trap into the guest. 135 * We don't need to deal with PMU CSRs contexts. 136 * 137 * Otherwise set the request bit KVM_REQ_PMU to restore guest PMU 138 * before entering guest VM 139 */ 140 val = kvm_read_sw_gcsr(csr, LOONGARCH_CSR_PERFCTRL0); 141 val |= kvm_read_sw_gcsr(csr, LOONGARCH_CSR_PERFCTRL1); 142 val |= kvm_read_sw_gcsr(csr, LOONGARCH_CSR_PERFCTRL2); 143 val |= kvm_read_sw_gcsr(csr, LOONGARCH_CSR_PERFCTRL3); 144 if (!(val & KVM_PMU_EVENT_ENABLED)) 145 vcpu->arch.aux_inuse &= ~KVM_LARCH_PMU; 146 else 147 kvm_make_request(KVM_REQ_PMU, vcpu); 148 149 kvm_restore_host_pmu(vcpu); 150 } 151 152 static void kvm_update_stolen_time(struct kvm_vcpu *vcpu) 153 { 154 u32 version; 155 u64 steal; 156 gpa_t gpa; 157 struct kvm_memslots *slots; 158 struct kvm_steal_time __user *st; 159 struct gfn_to_hva_cache *ghc; 160 161 ghc = &vcpu->arch.st.cache; 162 gpa = vcpu->arch.st.guest_addr; 163 if (!(gpa & KVM_STEAL_PHYS_VALID)) 164 return; 165 166 gpa &= KVM_STEAL_PHYS_MASK; 167 slots = kvm_memslots(vcpu->kvm); 168 if (slots->generation != ghc->generation || gpa != ghc->gpa) { 169 if (kvm_gfn_to_hva_cache_init(vcpu->kvm, ghc, gpa, sizeof(*st))) { 170 ghc->gpa = INVALID_GPA; 171 return; 172 } 173 } 174 175 st = (struct kvm_steal_time __user *)ghc->hva; 176 if (kvm_guest_has_pv_feature(vcpu, KVM_FEATURE_PREEMPT)) { 177 unsafe_put_user(0, &st->preempted, out); 178 vcpu->arch.st.preempted = 0; 179 } 180 181 unsafe_get_user(version, &st->version, out); 182 if (version & 1) 183 version += 1; /* first time write, random junk */ 184 185 version += 1; 186 unsafe_put_user(version, &st->version, out); 187 smp_wmb(); 188 189 unsafe_get_user(steal, &st->steal, out); 190 steal += current->sched_info.run_delay - vcpu->arch.st.last_steal; 191 vcpu->arch.st.last_steal = current->sched_info.run_delay; 192 unsafe_put_user(steal, &st->steal, out); 193 194 smp_wmb(); 195 version += 1; 196 unsafe_put_user(version, &st->version, out); 197 out: 198 mark_page_dirty_in_slot(vcpu->kvm, ghc->memslot, gpa_to_gfn(ghc->gpa)); 199 } 200 201 /* 202 * kvm_check_requests - check and handle pending vCPU requests 203 * 204 * Return: RESUME_GUEST if we should enter the guest 205 * RESUME_HOST if we should exit to userspace 206 */ 207 static int kvm_check_requests(struct kvm_vcpu *vcpu) 208 { 209 if (!kvm_request_pending(vcpu)) 210 return RESUME_GUEST; 211 212 if (kvm_check_request(KVM_REQ_TLB_FLUSH, vcpu)) 213 vcpu->arch.vpid = 0; /* Drop vpid for this vCPU */ 214 215 if (kvm_dirty_ring_check_request(vcpu)) 216 return RESUME_HOST; 217 218 if (kvm_check_request(KVM_REQ_STEAL_UPDATE, vcpu)) 219 kvm_update_stolen_time(vcpu); 220 221 return RESUME_GUEST; 222 } 223 224 static void kvm_late_check_requests(struct kvm_vcpu *vcpu) 225 { 226 lockdep_assert_irqs_disabled(); 227 228 if (!kvm_request_pending(vcpu)) 229 return; 230 231 if (kvm_check_request(KVM_REQ_PMU, vcpu)) { 232 kvm_own_pmu(vcpu); 233 vcpu->arch.aux_inuse |= KVM_LARCH_PMU; 234 } 235 236 if (kvm_check_request(KVM_REQ_TLB_FLUSH_GPA, vcpu)) 237 if (vcpu->arch.flush_gpa != INVALID_GPA) { 238 kvm_flush_tlb_gpa(vcpu, vcpu->arch.flush_gpa); 239 vcpu->arch.flush_gpa = INVALID_GPA; 240 } 241 242 if (kvm_check_request(KVM_REQ_FPU_LOAD, vcpu)) { 243 if (kvm_guest_has_lasx(&vcpu->arch)) 244 kvm_own_lasx(vcpu); 245 else if (kvm_guest_has_lsx(&vcpu->arch)) 246 kvm_own_lsx(vcpu); 247 else if (kvm_guest_has_fpu(&vcpu->arch)) 248 kvm_own_fpu(vcpu); 249 } 250 251 if (kvm_check_request(KVM_REQ_LBT_LOAD, vcpu)) 252 kvm_own_lbt(vcpu); 253 } 254 255 /* 256 * Check and handle pending signal and vCPU requests etc 257 * Run with irq enabled and preempt enabled 258 * 259 * Return: RESUME_GUEST if we should enter the guest 260 * RESUME_HOST if we should exit to userspace 261 * < 0 if we should exit to userspace, where the return value 262 * indicates an error 263 */ 264 static int kvm_enter_guest_check(struct kvm_vcpu *vcpu) 265 { 266 int idx, ret; 267 268 /* 269 * Check conditions before entering the guest 270 */ 271 ret = kvm_xfer_to_guest_mode_handle_work(vcpu); 272 if (ret < 0) 273 return ret; 274 275 idx = srcu_read_lock(&vcpu->kvm->srcu); 276 ret = kvm_check_requests(vcpu); 277 srcu_read_unlock(&vcpu->kvm->srcu, idx); 278 279 return ret; 280 } 281 282 /* 283 * Called with irq enabled 284 * 285 * Return: RESUME_GUEST if we should enter the guest, and irq disabled 286 * Others if we should exit to userspace 287 */ 288 static int kvm_pre_enter_guest(struct kvm_vcpu *vcpu) 289 { 290 int ret; 291 292 do { 293 ret = kvm_enter_guest_check(vcpu); 294 if (ret != RESUME_GUEST) 295 break; 296 297 /* 298 * Handle vcpu timer, interrupts, check requests and 299 * check vmid before vcpu enter guest 300 */ 301 local_irq_disable(); 302 kvm_deliver_exception(vcpu); 303 /* Make sure the vcpu mode has been written */ 304 smp_store_mb(vcpu->mode, IN_GUEST_MODE); 305 kvm_deliver_intr(vcpu); 306 kvm_check_vpid(vcpu); 307 308 /* 309 * Called after function kvm_check_vpid() 310 * Since it updates CSR.GSTAT used by kvm_flush_tlb_gpa(), 311 * and it may also clear KVM_REQ_TLB_FLUSH_GPA pending bit 312 */ 313 kvm_late_check_requests(vcpu); 314 /* Clear KVM_LARCH_SWCSR_LATEST as CSR will change when enter guest */ 315 vcpu->arch.aux_inuse &= ~KVM_LARCH_SWCSR_LATEST; 316 317 if (kvm_request_pending(vcpu) || xfer_to_guest_mode_work_pending()) { 318 if (vcpu->arch.aux_inuse & KVM_LARCH_PMU) { 319 kvm_lose_pmu(vcpu); 320 kvm_make_request(KVM_REQ_PMU, vcpu); 321 } 322 /* make sure the vcpu mode has been written */ 323 smp_store_mb(vcpu->mode, OUTSIDE_GUEST_MODE); 324 local_irq_enable(); 325 ret = -EAGAIN; 326 } 327 } while (ret != RESUME_GUEST); 328 329 return ret; 330 } 331 332 /* 333 * Return 1 for resume guest and "<= 0" for resume host. 334 */ 335 static int kvm_handle_exit(struct kvm_run *run, struct kvm_vcpu *vcpu) 336 { 337 int ret = RESUME_GUEST; 338 unsigned long estat = vcpu->arch.host_estat; 339 u32 intr = estat & CSR_ESTAT_IS; 340 u32 ecode = (estat & CSR_ESTAT_EXC) >> CSR_ESTAT_EXC_SHIFT; 341 342 smp_store_mb(vcpu->mode, OUTSIDE_GUEST_MODE); 343 344 /* Set a default exit reason */ 345 run->exit_reason = KVM_EXIT_UNKNOWN; 346 347 kvm_lose_pmu(vcpu); 348 349 guest_timing_exit_irqoff(); 350 guest_state_exit_irqoff(); 351 local_irq_enable(); 352 353 trace_kvm_exit(vcpu, ecode); 354 if (ecode) { 355 ret = kvm_handle_fault(vcpu, ecode); 356 } else { 357 WARN(!intr, "vm exiting with suspicious irq\n"); 358 ++vcpu->stat.int_exits; 359 } 360 361 if (ret == RESUME_GUEST) 362 ret = kvm_pre_enter_guest(vcpu); 363 364 if (ret != RESUME_GUEST) { 365 local_irq_disable(); 366 return ret; 367 } 368 369 guest_timing_enter_irqoff(); 370 guest_state_enter_irqoff(); 371 trace_kvm_reenter(vcpu); 372 373 return RESUME_GUEST; 374 } 375 376 int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu) 377 { 378 return !!(vcpu->arch.irq_pending) && 379 vcpu->arch.mp_state.mp_state == KVM_MP_STATE_RUNNABLE; 380 } 381 382 int kvm_arch_vcpu_should_kick(struct kvm_vcpu *vcpu) 383 { 384 return kvm_vcpu_exiting_guest_mode(vcpu) == IN_GUEST_MODE; 385 } 386 387 bool kvm_arch_vcpu_in_kernel(struct kvm_vcpu *vcpu) 388 { 389 unsigned long val; 390 391 preempt_disable(); 392 val = gcsr_read(LOONGARCH_CSR_CRMD); 393 preempt_enable(); 394 395 return (val & CSR_CRMD_PLV) == PLV_KERN; 396 } 397 398 #ifdef CONFIG_GUEST_PERF_EVENTS 399 unsigned long kvm_arch_vcpu_get_ip(struct kvm_vcpu *vcpu) 400 { 401 return vcpu->arch.pc; 402 } 403 404 /* 405 * Returns true if a Performance Monitoring Interrupt (PMI), a.k.a. perf event, 406 * arrived in guest context. For LoongArch64, if PMU is not passthrough to VM, 407 * any event that arrives while a vCPU is loaded is considered to be "in guest". 408 */ 409 bool kvm_arch_pmi_in_guest(struct kvm_vcpu *vcpu) 410 { 411 return (vcpu && !(vcpu->arch.aux_inuse & KVM_LARCH_PMU)); 412 } 413 #endif 414 415 bool kvm_arch_vcpu_preempted_in_kernel(struct kvm_vcpu *vcpu) 416 { 417 return false; 418 } 419 420 vm_fault_t kvm_arch_vcpu_fault(struct kvm_vcpu *vcpu, struct vm_fault *vmf) 421 { 422 return VM_FAULT_SIGBUS; 423 } 424 425 int kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu *vcpu, 426 struct kvm_translation *tr) 427 { 428 return -EINVAL; 429 } 430 431 int kvm_cpu_has_pending_timer(struct kvm_vcpu *vcpu) 432 { 433 int ret; 434 435 /* Protect from TOD sync and vcpu_load/put() */ 436 preempt_disable(); 437 ret = kvm_pending_timer(vcpu) || 438 kvm_read_hw_gcsr(LOONGARCH_CSR_ESTAT) & (1 << INT_TI); 439 preempt_enable(); 440 441 return ret; 442 } 443 444 int kvm_arch_vcpu_dump_regs(struct kvm_vcpu *vcpu) 445 { 446 int i; 447 448 kvm_debug("vCPU Register Dump:\n"); 449 kvm_debug("\tPC = 0x%08lx\n", vcpu->arch.pc); 450 kvm_debug("\tExceptions: %08lx\n", vcpu->arch.irq_pending); 451 452 for (i = 0; i < 32; i += 4) { 453 kvm_debug("\tGPR%02d: %08lx %08lx %08lx %08lx\n", i, 454 vcpu->arch.gprs[i], vcpu->arch.gprs[i + 1], 455 vcpu->arch.gprs[i + 2], vcpu->arch.gprs[i + 3]); 456 } 457 458 kvm_debug("\tCRMD: 0x%08lx, ESTAT: 0x%08lx\n", 459 kvm_read_hw_gcsr(LOONGARCH_CSR_CRMD), 460 kvm_read_hw_gcsr(LOONGARCH_CSR_ESTAT)); 461 462 kvm_debug("\tERA: 0x%08lx\n", kvm_read_hw_gcsr(LOONGARCH_CSR_ERA)); 463 464 return 0; 465 } 466 467 int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu, 468 struct kvm_mp_state *mp_state) 469 { 470 *mp_state = vcpu->arch.mp_state; 471 472 return 0; 473 } 474 475 int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu, 476 struct kvm_mp_state *mp_state) 477 { 478 int ret = 0; 479 480 switch (mp_state->mp_state) { 481 case KVM_MP_STATE_RUNNABLE: 482 vcpu->arch.mp_state = *mp_state; 483 break; 484 default: 485 ret = -EINVAL; 486 } 487 488 return ret; 489 } 490 491 int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu, 492 struct kvm_guest_debug *dbg) 493 { 494 if (dbg->control & ~KVM_GUESTDBG_VALID_MASK) 495 return -EINVAL; 496 497 if (dbg->control & KVM_GUESTDBG_ENABLE) 498 vcpu->guest_debug = dbg->control; 499 else 500 vcpu->guest_debug = 0; 501 502 return 0; 503 } 504 505 static inline int kvm_set_cpuid(struct kvm_vcpu *vcpu, u64 val) 506 { 507 int cpuid; 508 struct kvm_phyid_map *map; 509 struct loongarch_csrs *csr = vcpu->arch.csr; 510 511 if (val >= KVM_MAX_PHYID) 512 return -EINVAL; 513 514 map = vcpu->kvm->arch.phyid_map; 515 cpuid = kvm_read_sw_gcsr(csr, LOONGARCH_CSR_CPUID); 516 517 spin_lock(&vcpu->kvm->arch.phyid_map_lock); 518 if ((cpuid < KVM_MAX_PHYID) && map->phys_map[cpuid].enabled) { 519 /* Discard duplicated CPUID set operation */ 520 if (cpuid == val) { 521 spin_unlock(&vcpu->kvm->arch.phyid_map_lock); 522 return 0; 523 } 524 525 /* 526 * CPUID is already set before 527 * Forbid changing to a different CPUID at runtime 528 */ 529 spin_unlock(&vcpu->kvm->arch.phyid_map_lock); 530 return -EINVAL; 531 } 532 533 if (map->phys_map[val].enabled) { 534 /* Discard duplicated CPUID set operation */ 535 if (vcpu == map->phys_map[val].vcpu) { 536 spin_unlock(&vcpu->kvm->arch.phyid_map_lock); 537 return 0; 538 } 539 540 /* 541 * New CPUID is already set with other vcpu 542 * Forbid sharing the same CPUID between different vcpus 543 */ 544 spin_unlock(&vcpu->kvm->arch.phyid_map_lock); 545 return -EINVAL; 546 } 547 548 kvm_write_sw_gcsr(csr, LOONGARCH_CSR_CPUID, val); 549 map->phys_map[val].enabled = true; 550 map->phys_map[val].vcpu = vcpu; 551 spin_unlock(&vcpu->kvm->arch.phyid_map_lock); 552 553 return 0; 554 } 555 556 static inline void kvm_drop_cpuid(struct kvm_vcpu *vcpu) 557 { 558 int cpuid; 559 struct kvm_phyid_map *map; 560 struct loongarch_csrs *csr = vcpu->arch.csr; 561 562 map = vcpu->kvm->arch.phyid_map; 563 cpuid = kvm_read_sw_gcsr(csr, LOONGARCH_CSR_CPUID); 564 565 if (cpuid >= KVM_MAX_PHYID) 566 return; 567 568 spin_lock(&vcpu->kvm->arch.phyid_map_lock); 569 if (map->phys_map[cpuid].enabled) { 570 map->phys_map[cpuid].vcpu = NULL; 571 map->phys_map[cpuid].enabled = false; 572 kvm_write_sw_gcsr(csr, LOONGARCH_CSR_CPUID, KVM_MAX_PHYID); 573 } 574 spin_unlock(&vcpu->kvm->arch.phyid_map_lock); 575 } 576 577 struct kvm_vcpu *kvm_get_vcpu_by_cpuid(struct kvm *kvm, int cpuid) 578 { 579 struct kvm_phyid_map *map; 580 581 if (cpuid < 0) 582 return NULL; 583 584 if (cpuid >= KVM_MAX_PHYID) 585 return NULL; 586 587 map = kvm->arch.phyid_map; 588 if (!map->phys_map[cpuid].enabled) 589 return NULL; 590 591 return map->phys_map[cpuid].vcpu; 592 } 593 594 static int _kvm_getcsr(struct kvm_vcpu *vcpu, unsigned int id, u64 *val) 595 { 596 unsigned long estat, gintc; 597 struct loongarch_csrs *csr = vcpu->arch.csr; 598 599 if (get_gcsr_flag(id) & INVALID_GCSR) 600 return -EINVAL; 601 602 if (id == LOONGARCH_CSR_ESTAT) { 603 preempt_disable(); 604 vcpu_load(vcpu); 605 /* 606 * Sync pending interrupts into ESTAT so that interrupt 607 * remains during VM migration stage 608 */ 609 kvm_deliver_intr(vcpu); 610 vcpu->arch.aux_inuse &= ~KVM_LARCH_SWCSR_LATEST; 611 vcpu_put(vcpu); 612 preempt_enable(); 613 614 /* ESTAT IP0~IP7 get from GINTC */ 615 gintc = kvm_read_sw_gcsr(csr, LOONGARCH_CSR_GINTC) & KVM_GINTC_IRQ_MASK; 616 estat = kvm_read_sw_gcsr(csr, LOONGARCH_CSR_ESTAT) & ~KVM_ESTAT_EXTI_MASK; 617 *val = estat | (gintc << VIP_DELTA); 618 return 0; 619 } 620 621 /* 622 * Get software CSR state since software state is consistent 623 * with hardware for synchronous ioctl 624 */ 625 *val = kvm_read_sw_gcsr(csr, id); 626 627 return 0; 628 } 629 630 static int _kvm_setcsr(struct kvm_vcpu *vcpu, unsigned int id, u64 val) 631 { 632 int ret = 0; 633 unsigned long estat, gintc; 634 struct loongarch_csrs *csr = vcpu->arch.csr; 635 636 if (get_gcsr_flag(id) & INVALID_GCSR) 637 return -EINVAL; 638 639 if (id == LOONGARCH_CSR_CPUID) 640 return kvm_set_cpuid(vcpu, val); 641 642 if (id == LOONGARCH_CSR_ESTAT) { 643 /* ESTAT IP0~IP7 inject through GINTC */ 644 gintc = (val >> VIP_DELTA) & KVM_GINTC_IRQ_MASK; 645 gintc |= kvm_read_sw_gcsr(csr, LOONGARCH_CSR_GINTC) & ~KVM_GINTC_IRQ_MASK; 646 kvm_write_sw_gcsr(csr, LOONGARCH_CSR_GINTC, gintc); 647 648 /* Only set valid ESTAT bits */ 649 estat = val & ~KVM_ESTAT_EXTI_MASK; 650 estat &= CSR_ESTAT_IS | CSR_ESTAT_EXC | CSR_ESTAT_ESUBCODE; 651 if (!kvm_guest_has_msgint(&vcpu->arch)) 652 estat &= ~CPU_AVEC; 653 kvm_write_sw_gcsr(csr, LOONGARCH_CSR_ESTAT, estat); 654 655 return ret; 656 } 657 658 kvm_write_sw_gcsr(csr, id, val); 659 660 /* 661 * After modifying the PMU CSR register value of the vcpu. 662 * If the PMU CSRs are used, we need to set KVM_REQ_PMU. 663 */ 664 if (id >= LOONGARCH_CSR_PERFCTRL0 && id <= LOONGARCH_CSR_PERFCNTR3) { 665 unsigned long val; 666 667 val = kvm_read_sw_gcsr(csr, LOONGARCH_CSR_PERFCTRL0) | 668 kvm_read_sw_gcsr(csr, LOONGARCH_CSR_PERFCTRL1) | 669 kvm_read_sw_gcsr(csr, LOONGARCH_CSR_PERFCTRL2) | 670 kvm_read_sw_gcsr(csr, LOONGARCH_CSR_PERFCTRL3); 671 672 if (val & KVM_PMU_EVENT_ENABLED) 673 kvm_make_request(KVM_REQ_PMU, vcpu); 674 } 675 676 return ret; 677 } 678 679 static int _kvm_get_cpucfg_mask(int id, u64 *v) 680 { 681 unsigned int config; 682 683 if (id < 0 || id >= KVM_MAX_CPUCFG_REGS) 684 return -EINVAL; 685 686 switch (id) { 687 case LOONGARCH_CPUCFG0: 688 *v = GENMASK(31, 0); 689 return 0; 690 case LOONGARCH_CPUCFG1: 691 *v = GENMASK(26, 0); 692 return 0; 693 case LOONGARCH_CPUCFG2: 694 /* CPUCFG2 features unconditionally supported by KVM */ 695 *v = CPUCFG2_FP | CPUCFG2_FPSP | CPUCFG2_FPDP | 696 CPUCFG2_FPVERS | CPUCFG2_LLFTP | CPUCFG2_LLFTPREV | 697 CPUCFG2_LSPW | CPUCFG2_LAM; 698 /* 699 * For the ISA extensions listed below, if one is supported 700 * by the host, then it is also supported by KVM. 701 */ 702 if (cpu_has_lsx) 703 *v |= CPUCFG2_LSX; 704 if (cpu_has_lasx) 705 *v |= CPUCFG2_LASX; 706 if (cpu_has_lbt_x86) 707 *v |= CPUCFG2_X86BT; 708 if (cpu_has_lbt_arm) 709 *v |= CPUCFG2_ARMBT; 710 if (cpu_has_lbt_mips) 711 *v |= CPUCFG2_MIPSBT; 712 if (cpu_has_ptw) 713 *v |= CPUCFG2_PTW; 714 715 config = read_cpucfg(LOONGARCH_CPUCFG2); 716 *v |= config & (CPUCFG2_FRECIPE | CPUCFG2_DIV32 | CPUCFG2_LAM_BH); 717 *v |= config & (CPUCFG2_LAMCAS | CPUCFG2_LLACQ_SCREL | CPUCFG2_SCQ); 718 return 0; 719 case LOONGARCH_CPUCFG3: 720 *v = GENMASK(23, 0); 721 722 /* VM does not support memory order and SFB setting */ 723 config = read_cpucfg(LOONGARCH_CPUCFG3); 724 *v &= config & ~(CPUCFG3_SFB); 725 *v &= config & ~(CPUCFG3_ALDORDER_CAP | CPUCFG3_ASTORDER_CAP | CPUCFG3_SLDORDER_CAP); 726 return 0; 727 case LOONGARCH_CPUCFG4: 728 case LOONGARCH_CPUCFG5: 729 *v = GENMASK(31, 0); 730 return 0; 731 case LOONGARCH_CPUCFG6: 732 if (cpu_has_pmp) 733 *v = GENMASK(14, 0); 734 else 735 *v = 0; 736 return 0; 737 case LOONGARCH_CPUCFG16: 738 *v = GENMASK(16, 0); 739 return 0; 740 case LOONGARCH_CPUCFG17 ... LOONGARCH_CPUCFG20: 741 *v = GENMASK(30, 0); 742 return 0; 743 default: 744 /* 745 * CPUCFG bits should be zero if reserved by HW or not 746 * supported by KVM. 747 */ 748 *v = 0; 749 return 0; 750 } 751 } 752 753 static int kvm_check_cpucfg(int id, u64 val) 754 { 755 int ret; 756 u32 host; 757 u64 mask = 0; 758 759 ret = _kvm_get_cpucfg_mask(id, &mask); 760 if (ret) 761 return ret; 762 763 if (val & ~mask) 764 /* Unsupported features and/or the higher 32 bits should not be set */ 765 return -EINVAL; 766 767 switch (id) { 768 case LOONGARCH_CPUCFG1: 769 if ((val & CPUCFG1_MSGINT) && !cpu_has_msgint) 770 return -EINVAL; 771 return 0; 772 case LOONGARCH_CPUCFG2: 773 if (!(val & CPUCFG2_LLFTP)) 774 /* Guests must have a constant timer */ 775 return -EINVAL; 776 if ((val & CPUCFG2_FP) && (!(val & CPUCFG2_FPSP) || !(val & CPUCFG2_FPDP))) 777 /* Single and double float point must both be set when FP is enabled */ 778 return -EINVAL; 779 if ((val & CPUCFG2_LSX) && !(val & CPUCFG2_FP)) 780 /* LSX architecturally implies FP but val does not satisfy that */ 781 return -EINVAL; 782 if ((val & CPUCFG2_LASX) && !(val & CPUCFG2_LSX)) 783 /* LASX architecturally implies LSX and FP but val does not satisfy that */ 784 return -EINVAL; 785 return 0; 786 case LOONGARCH_CPUCFG3: 787 host = read_cpucfg(LOONGARCH_CPUCFG3); 788 if ((val & CPUCFG3_RVAMAX) > (host & CPUCFG3_RVAMAX)) 789 return -EINVAL; 790 if ((val & CPUCFG3_SPW_LVL) > (host & CPUCFG3_SPW_LVL)) 791 return -EINVAL; 792 return 0; 793 case LOONGARCH_CPUCFG6: 794 if (val & CPUCFG6_PMP) { 795 host = read_cpucfg(LOONGARCH_CPUCFG6); 796 if ((val & CPUCFG6_PMBITS) != (host & CPUCFG6_PMBITS)) 797 return -EINVAL; 798 if ((val & CPUCFG6_PMNUM) > (host & CPUCFG6_PMNUM)) 799 return -EINVAL; 800 if ((val & CPUCFG6_UPM) && !(host & CPUCFG6_UPM)) 801 return -EINVAL; 802 } 803 return 0; 804 default: 805 /* 806 * Values for the other CPUCFG IDs are not being further validated 807 * besides the mask check above. 808 */ 809 return 0; 810 } 811 } 812 813 static int kvm_get_one_reg(struct kvm_vcpu *vcpu, 814 const struct kvm_one_reg *reg, u64 *v) 815 { 816 int id, ret = 0; 817 u64 type = reg->id & KVM_REG_LOONGARCH_MASK; 818 819 switch (type) { 820 case KVM_REG_LOONGARCH_CSR: 821 id = KVM_GET_IOC_CSR_IDX(reg->id); 822 ret = _kvm_getcsr(vcpu, id, v); 823 break; 824 case KVM_REG_LOONGARCH_CPUCFG: 825 id = KVM_GET_IOC_CPUCFG_IDX(reg->id); 826 if (id >= 0 && id < KVM_MAX_CPUCFG_REGS) 827 *v = vcpu->arch.cpucfg[id]; 828 else 829 ret = -EINVAL; 830 break; 831 case KVM_REG_LOONGARCH_LBT: 832 if (!kvm_guest_has_lbt(&vcpu->arch)) 833 return -ENXIO; 834 835 switch (reg->id) { 836 case KVM_REG_LOONGARCH_LBT_SCR0: 837 *v = vcpu->arch.lbt.scr0; 838 break; 839 case KVM_REG_LOONGARCH_LBT_SCR1: 840 *v = vcpu->arch.lbt.scr1; 841 break; 842 case KVM_REG_LOONGARCH_LBT_SCR2: 843 *v = vcpu->arch.lbt.scr2; 844 break; 845 case KVM_REG_LOONGARCH_LBT_SCR3: 846 *v = vcpu->arch.lbt.scr3; 847 break; 848 case KVM_REG_LOONGARCH_LBT_EFLAGS: 849 *v = vcpu->arch.lbt.eflags; 850 break; 851 case KVM_REG_LOONGARCH_LBT_FTOP: 852 *v = vcpu->arch.fpu.ftop; 853 break; 854 default: 855 ret = -EINVAL; 856 break; 857 } 858 break; 859 case KVM_REG_LOONGARCH_KVM: 860 switch (reg->id) { 861 case KVM_REG_LOONGARCH_COUNTER: 862 *v = get_cycles() + vcpu->kvm->arch.time_offset; 863 break; 864 case KVM_REG_LOONGARCH_DEBUG_INST: 865 *v = INSN_HVCL | KVM_HCALL_SWDBG; 866 break; 867 default: 868 ret = -EINVAL; 869 break; 870 } 871 break; 872 default: 873 ret = -EINVAL; 874 break; 875 } 876 877 return ret; 878 } 879 880 static int kvm_get_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg) 881 { 882 int ret = 0; 883 u64 v, size = reg->id & KVM_REG_SIZE_MASK; 884 885 switch (size) { 886 case KVM_REG_SIZE_U64: 887 ret = kvm_get_one_reg(vcpu, reg, &v); 888 if (ret) 889 return ret; 890 ret = put_user(v, (u64 __user *)(long)reg->addr); 891 break; 892 default: 893 ret = -EINVAL; 894 break; 895 } 896 897 return ret; 898 } 899 900 static int kvm_set_one_reg(struct kvm_vcpu *vcpu, 901 const struct kvm_one_reg *reg, u64 v) 902 { 903 int id, ret = 0; 904 u64 type = reg->id & KVM_REG_LOONGARCH_MASK; 905 906 switch (type) { 907 case KVM_REG_LOONGARCH_CSR: 908 id = KVM_GET_IOC_CSR_IDX(reg->id); 909 ret = _kvm_setcsr(vcpu, id, v); 910 break; 911 case KVM_REG_LOONGARCH_CPUCFG: 912 id = KVM_GET_IOC_CPUCFG_IDX(reg->id); 913 ret = kvm_check_cpucfg(id, v); 914 if (ret) 915 break; 916 vcpu->arch.cpucfg[id] = (u32)v; 917 if (id == LOONGARCH_CPUCFG6) 918 vcpu->arch.max_pmu_csrid = 919 LOONGARCH_CSR_PERFCTRL0 + 2 * kvm_get_pmu_num(&vcpu->arch) + 1; 920 break; 921 case KVM_REG_LOONGARCH_LBT: 922 if (!kvm_guest_has_lbt(&vcpu->arch)) 923 return -ENXIO; 924 925 switch (reg->id) { 926 case KVM_REG_LOONGARCH_LBT_SCR0: 927 vcpu->arch.lbt.scr0 = v; 928 break; 929 case KVM_REG_LOONGARCH_LBT_SCR1: 930 vcpu->arch.lbt.scr1 = v; 931 break; 932 case KVM_REG_LOONGARCH_LBT_SCR2: 933 vcpu->arch.lbt.scr2 = v; 934 break; 935 case KVM_REG_LOONGARCH_LBT_SCR3: 936 vcpu->arch.lbt.scr3 = v; 937 break; 938 case KVM_REG_LOONGARCH_LBT_EFLAGS: 939 vcpu->arch.lbt.eflags = v; 940 break; 941 case KVM_REG_LOONGARCH_LBT_FTOP: 942 vcpu->arch.fpu.ftop = v; 943 break; 944 default: 945 ret = -EINVAL; 946 break; 947 } 948 break; 949 case KVM_REG_LOONGARCH_KVM: 950 switch (reg->id) { 951 case KVM_REG_LOONGARCH_COUNTER: 952 /* 953 * gftoffset is relative with board, not vcpu 954 * only set for the first time for smp system 955 */ 956 if (vcpu->vcpu_id == 0) 957 vcpu->kvm->arch.time_offset = (signed long)(v - get_cycles()); 958 break; 959 case KVM_REG_LOONGARCH_VCPU_RESET: 960 vcpu->arch.st.guest_addr = 0; 961 memset(&vcpu->arch.irq_pending, 0, sizeof(vcpu->arch.irq_pending)); 962 memset(&vcpu->arch.irq_clear, 0, sizeof(vcpu->arch.irq_clear)); 963 964 /* 965 * When vCPU reset, clear the ESTAT and GINTC registers 966 * Other CSR registers are cleared with function _kvm_setcsr(). 967 */ 968 kvm_write_sw_gcsr(vcpu->arch.csr, LOONGARCH_CSR_GINTC, 0); 969 kvm_write_sw_gcsr(vcpu->arch.csr, LOONGARCH_CSR_ESTAT, 0); 970 break; 971 default: 972 ret = -EINVAL; 973 break; 974 } 975 break; 976 default: 977 ret = -EINVAL; 978 break; 979 } 980 981 return ret; 982 } 983 984 static int kvm_set_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg) 985 { 986 int ret = 0; 987 u64 v, size = reg->id & KVM_REG_SIZE_MASK; 988 989 switch (size) { 990 case KVM_REG_SIZE_U64: 991 ret = get_user(v, (u64 __user *)(long)reg->addr); 992 if (ret) 993 return ret; 994 break; 995 default: 996 return -EINVAL; 997 } 998 999 return kvm_set_one_reg(vcpu, reg, v); 1000 } 1001 1002 int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs) 1003 { 1004 return -ENOIOCTLCMD; 1005 } 1006 1007 int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs) 1008 { 1009 return -ENOIOCTLCMD; 1010 } 1011 1012 int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs) 1013 { 1014 int i; 1015 1016 for (i = 0; i < ARRAY_SIZE(vcpu->arch.gprs); i++) 1017 regs->gpr[i] = vcpu->arch.gprs[i]; 1018 1019 regs->pc = vcpu->arch.pc; 1020 1021 return 0; 1022 } 1023 1024 int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs) 1025 { 1026 int i; 1027 1028 for (i = 1; i < ARRAY_SIZE(vcpu->arch.gprs); i++) 1029 vcpu->arch.gprs[i] = regs->gpr[i]; 1030 1031 vcpu->arch.gprs[0] = 0; /* zero is special, and cannot be set. */ 1032 vcpu->arch.pc = regs->pc; 1033 1034 return 0; 1035 } 1036 1037 static int kvm_vcpu_ioctl_enable_cap(struct kvm_vcpu *vcpu, 1038 struct kvm_enable_cap *cap) 1039 { 1040 /* FPU is enabled by default, will support LSX/LASX later. */ 1041 return -EINVAL; 1042 } 1043 1044 static int kvm_loongarch_cpucfg_has_attr(struct kvm_vcpu *vcpu, 1045 struct kvm_device_attr *attr) 1046 { 1047 switch (attr->attr) { 1048 case LOONGARCH_CPUCFG2: 1049 case LOONGARCH_CPUCFG6: 1050 return 0; 1051 case CPUCFG_KVM_FEATURE: 1052 return 0; 1053 default: 1054 return -ENXIO; 1055 } 1056 1057 return -ENXIO; 1058 } 1059 1060 static int kvm_loongarch_pvtime_has_attr(struct kvm_vcpu *vcpu, 1061 struct kvm_device_attr *attr) 1062 { 1063 if (!kvm_guest_has_pv_feature(vcpu, KVM_FEATURE_STEAL_TIME) 1064 || attr->attr != KVM_LOONGARCH_VCPU_PVTIME_GPA) 1065 return -ENXIO; 1066 1067 return 0; 1068 } 1069 1070 static int kvm_loongarch_vcpu_has_attr(struct kvm_vcpu *vcpu, 1071 struct kvm_device_attr *attr) 1072 { 1073 int ret = -ENXIO; 1074 1075 switch (attr->group) { 1076 case KVM_LOONGARCH_VCPU_CPUCFG: 1077 ret = kvm_loongarch_cpucfg_has_attr(vcpu, attr); 1078 break; 1079 case KVM_LOONGARCH_VCPU_PVTIME_CTRL: 1080 ret = kvm_loongarch_pvtime_has_attr(vcpu, attr); 1081 break; 1082 default: 1083 break; 1084 } 1085 1086 return ret; 1087 } 1088 1089 static int kvm_loongarch_cpucfg_get_attr(struct kvm_vcpu *vcpu, 1090 struct kvm_device_attr *attr) 1091 { 1092 int ret = 0; 1093 uint64_t val; 1094 uint64_t __user *uaddr = (uint64_t __user *)attr->addr; 1095 1096 switch (attr->attr) { 1097 case 0 ... (KVM_MAX_CPUCFG_REGS - 1): 1098 ret = _kvm_get_cpucfg_mask(attr->attr, &val); 1099 if (ret) 1100 return ret; 1101 break; 1102 case CPUCFG_KVM_FEATURE: 1103 val = vcpu->kvm->arch.pv_features & LOONGARCH_PV_FEAT_MASK; 1104 break; 1105 default: 1106 return -ENXIO; 1107 } 1108 1109 if (put_user(val, uaddr)) 1110 return -EFAULT; 1111 1112 return ret; 1113 } 1114 1115 static int kvm_loongarch_pvtime_get_attr(struct kvm_vcpu *vcpu, 1116 struct kvm_device_attr *attr) 1117 { 1118 u64 gpa; 1119 u64 __user *user = (u64 __user *)attr->addr; 1120 1121 if (!kvm_guest_has_pv_feature(vcpu, KVM_FEATURE_STEAL_TIME) 1122 || attr->attr != KVM_LOONGARCH_VCPU_PVTIME_GPA) 1123 return -ENXIO; 1124 1125 gpa = vcpu->arch.st.guest_addr; 1126 if (put_user(gpa, user)) 1127 return -EFAULT; 1128 1129 return 0; 1130 } 1131 1132 static int kvm_loongarch_vcpu_get_attr(struct kvm_vcpu *vcpu, 1133 struct kvm_device_attr *attr) 1134 { 1135 int ret = -ENXIO; 1136 1137 switch (attr->group) { 1138 case KVM_LOONGARCH_VCPU_CPUCFG: 1139 ret = kvm_loongarch_cpucfg_get_attr(vcpu, attr); 1140 break; 1141 case KVM_LOONGARCH_VCPU_PVTIME_CTRL: 1142 ret = kvm_loongarch_pvtime_get_attr(vcpu, attr); 1143 break; 1144 default: 1145 break; 1146 } 1147 1148 return ret; 1149 } 1150 1151 static int kvm_loongarch_cpucfg_set_attr(struct kvm_vcpu *vcpu, 1152 struct kvm_device_attr *attr) 1153 { 1154 u64 val, valid; 1155 u64 __user *user = (u64 __user *)attr->addr; 1156 struct kvm *kvm = vcpu->kvm; 1157 1158 switch (attr->attr) { 1159 case CPUCFG_KVM_FEATURE: 1160 if (get_user(val, user)) 1161 return -EFAULT; 1162 1163 valid = LOONGARCH_PV_FEAT_MASK; 1164 if (val & ~valid) 1165 return -EINVAL; 1166 1167 /* All vCPUs need set the same PV features */ 1168 if ((kvm->arch.pv_features & LOONGARCH_PV_FEAT_UPDATED) 1169 && ((kvm->arch.pv_features & valid) != val)) 1170 return -EINVAL; 1171 kvm->arch.pv_features = val | LOONGARCH_PV_FEAT_UPDATED; 1172 return 0; 1173 default: 1174 return -ENXIO; 1175 } 1176 } 1177 1178 static int kvm_loongarch_pvtime_set_attr(struct kvm_vcpu *vcpu, 1179 struct kvm_device_attr *attr) 1180 { 1181 int idx, ret = 0; 1182 u64 gpa, __user *user = (u64 __user *)attr->addr; 1183 struct kvm *kvm = vcpu->kvm; 1184 1185 if (!kvm_guest_has_pv_feature(vcpu, KVM_FEATURE_STEAL_TIME) 1186 || attr->attr != KVM_LOONGARCH_VCPU_PVTIME_GPA) 1187 return -ENXIO; 1188 1189 if (get_user(gpa, user)) 1190 return -EFAULT; 1191 1192 if (gpa & ~(KVM_STEAL_PHYS_MASK | KVM_STEAL_PHYS_VALID)) 1193 return -EINVAL; 1194 1195 if (!(gpa & KVM_STEAL_PHYS_VALID)) { 1196 vcpu->arch.st.guest_addr = gpa; 1197 return 0; 1198 } 1199 1200 /* Check the address is in a valid memslot */ 1201 idx = srcu_read_lock(&kvm->srcu); 1202 if (kvm_is_error_hva(gfn_to_hva(kvm, gpa >> PAGE_SHIFT))) 1203 ret = -EINVAL; 1204 srcu_read_unlock(&kvm->srcu, idx); 1205 1206 if (!ret) { 1207 vcpu->arch.st.guest_addr = gpa; 1208 vcpu->arch.st.last_steal = current->sched_info.run_delay; 1209 kvm_make_request(KVM_REQ_STEAL_UPDATE, vcpu); 1210 } 1211 1212 return ret; 1213 } 1214 1215 static int kvm_loongarch_vcpu_set_attr(struct kvm_vcpu *vcpu, 1216 struct kvm_device_attr *attr) 1217 { 1218 int ret = -ENXIO; 1219 1220 switch (attr->group) { 1221 case KVM_LOONGARCH_VCPU_CPUCFG: 1222 ret = kvm_loongarch_cpucfg_set_attr(vcpu, attr); 1223 break; 1224 case KVM_LOONGARCH_VCPU_PVTIME_CTRL: 1225 ret = kvm_loongarch_pvtime_set_attr(vcpu, attr); 1226 break; 1227 default: 1228 break; 1229 } 1230 1231 return ret; 1232 } 1233 1234 long kvm_arch_vcpu_ioctl(struct file *filp, 1235 unsigned int ioctl, unsigned long arg) 1236 { 1237 long r; 1238 struct kvm_device_attr attr; 1239 void __user *argp = (void __user *)arg; 1240 struct kvm_vcpu *vcpu = filp->private_data; 1241 1242 /* 1243 * Only software CSR should be modified 1244 * 1245 * If any hardware CSR register is modified, vcpu_load/vcpu_put pair 1246 * should be used. Since CSR registers owns by this vcpu, if switch 1247 * to other vcpus, other vcpus need reload CSR registers. 1248 * 1249 * If software CSR is modified, bit KVM_LARCH_HWCSR_USABLE should 1250 * be clear in vcpu->arch.aux_inuse, and vcpu_load will check 1251 * aux_inuse flag and reload CSR registers form software. 1252 */ 1253 1254 switch (ioctl) { 1255 case KVM_SET_ONE_REG: 1256 case KVM_GET_ONE_REG: { 1257 struct kvm_one_reg reg; 1258 1259 r = -EFAULT; 1260 if (copy_from_user(®, argp, sizeof(reg))) 1261 break; 1262 if (ioctl == KVM_SET_ONE_REG) { 1263 r = kvm_set_reg(vcpu, ®); 1264 vcpu->arch.aux_inuse &= ~KVM_LARCH_HWCSR_USABLE; 1265 } else 1266 r = kvm_get_reg(vcpu, ®); 1267 break; 1268 } 1269 case KVM_ENABLE_CAP: { 1270 struct kvm_enable_cap cap; 1271 1272 r = -EFAULT; 1273 if (copy_from_user(&cap, argp, sizeof(cap))) 1274 break; 1275 r = kvm_vcpu_ioctl_enable_cap(vcpu, &cap); 1276 break; 1277 } 1278 case KVM_HAS_DEVICE_ATTR: { 1279 r = -EFAULT; 1280 if (copy_from_user(&attr, argp, sizeof(attr))) 1281 break; 1282 r = kvm_loongarch_vcpu_has_attr(vcpu, &attr); 1283 break; 1284 } 1285 case KVM_GET_DEVICE_ATTR: { 1286 r = -EFAULT; 1287 if (copy_from_user(&attr, argp, sizeof(attr))) 1288 break; 1289 r = kvm_loongarch_vcpu_get_attr(vcpu, &attr); 1290 break; 1291 } 1292 case KVM_SET_DEVICE_ATTR: { 1293 r = -EFAULT; 1294 if (copy_from_user(&attr, argp, sizeof(attr))) 1295 break; 1296 r = kvm_loongarch_vcpu_set_attr(vcpu, &attr); 1297 break; 1298 } 1299 default: 1300 r = -ENOIOCTLCMD; 1301 break; 1302 } 1303 1304 return r; 1305 } 1306 1307 int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu) 1308 { 1309 int i = 0; 1310 1311 fpu->fcc = vcpu->arch.fpu.fcc; 1312 fpu->fcsr = vcpu->arch.fpu.fcsr; 1313 for (i = 0; i < NUM_FPU_REGS; i++) 1314 memcpy(&fpu->fpr[i], &vcpu->arch.fpu.fpr[i], sizeof(union fpureg)); 1315 1316 return 0; 1317 } 1318 1319 int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu) 1320 { 1321 int i = 0; 1322 1323 vcpu->arch.fpu.fcc = fpu->fcc; 1324 vcpu->arch.fpu.fcsr = fpu->fcsr; 1325 for (i = 0; i < NUM_FPU_REGS; i++) 1326 memcpy(&vcpu->arch.fpu.fpr[i], &fpu->fpr[i], sizeof(union fpureg)); 1327 1328 return 0; 1329 } 1330 1331 #ifdef CONFIG_CPU_HAS_LBT 1332 int kvm_own_lbt(struct kvm_vcpu *vcpu) 1333 { 1334 if (!(vcpu->arch.aux_inuse & KVM_LARCH_LBT)) { 1335 set_csr_euen(CSR_EUEN_LBTEN); 1336 _restore_lbt(&vcpu->arch.lbt); 1337 vcpu->arch.aux_inuse |= KVM_LARCH_LBT; 1338 } 1339 1340 return 0; 1341 } 1342 1343 static void kvm_lose_lbt(struct kvm_vcpu *vcpu) 1344 { 1345 preempt_disable(); 1346 if (vcpu->arch.aux_inuse & KVM_LARCH_LBT) { 1347 _save_lbt(&vcpu->arch.lbt); 1348 clear_csr_euen(CSR_EUEN_LBTEN); 1349 vcpu->arch.aux_inuse &= ~KVM_LARCH_LBT; 1350 } 1351 preempt_enable(); 1352 } 1353 1354 static void kvm_check_fcsr(struct kvm_vcpu *vcpu, unsigned long fcsr) 1355 { 1356 /* 1357 * If TM is enabled, top register save/restore will 1358 * cause lbt exception, here enable lbt in advance 1359 */ 1360 if (fcsr & FPU_CSR_TM) 1361 kvm_own_lbt(vcpu); 1362 } 1363 1364 static void kvm_check_fcsr_alive(struct kvm_vcpu *vcpu) 1365 { 1366 if (vcpu->arch.aux_inuse & KVM_LARCH_FPU) { 1367 if (vcpu->arch.aux_inuse & KVM_LARCH_LBT) 1368 return; 1369 kvm_check_fcsr(vcpu, read_fcsr(LOONGARCH_FCSR0)); 1370 } 1371 } 1372 #else 1373 static inline void kvm_lose_lbt(struct kvm_vcpu *vcpu) { } 1374 static inline void kvm_check_fcsr(struct kvm_vcpu *vcpu, unsigned long fcsr) { } 1375 static inline void kvm_check_fcsr_alive(struct kvm_vcpu *vcpu) { } 1376 #endif 1377 1378 /* Enable FPU and restore context */ 1379 void kvm_own_fpu(struct kvm_vcpu *vcpu) 1380 { 1381 /* 1382 * Enable FPU for guest 1383 * Set FR and FRE according to guest context 1384 */ 1385 kvm_check_fcsr(vcpu, vcpu->arch.fpu.fcsr); 1386 set_csr_euen(CSR_EUEN_FPEN); 1387 1388 kvm_restore_fpu(&vcpu->arch.fpu); 1389 vcpu->arch.aux_inuse |= KVM_LARCH_FPU; 1390 trace_kvm_aux(vcpu, KVM_TRACE_AUX_RESTORE, KVM_TRACE_AUX_FPU); 1391 } 1392 1393 #ifdef CONFIG_CPU_HAS_LSX 1394 /* Enable LSX and restore context */ 1395 int kvm_own_lsx(struct kvm_vcpu *vcpu) 1396 { 1397 /* Enable LSX for guest */ 1398 kvm_check_fcsr(vcpu, vcpu->arch.fpu.fcsr); 1399 set_csr_euen(CSR_EUEN_LSXEN | CSR_EUEN_FPEN); 1400 1401 kvm_restore_lsx(&vcpu->arch.fpu); 1402 vcpu->arch.aux_inuse |= KVM_LARCH_FPU; 1403 trace_kvm_aux(vcpu, KVM_TRACE_AUX_RESTORE, KVM_TRACE_AUX_LSX); 1404 1405 return 0; 1406 } 1407 #endif 1408 1409 #ifdef CONFIG_CPU_HAS_LASX 1410 /* Enable LASX and restore context */ 1411 int kvm_own_lasx(struct kvm_vcpu *vcpu) 1412 { 1413 kvm_check_fcsr(vcpu, vcpu->arch.fpu.fcsr); 1414 set_csr_euen(CSR_EUEN_FPEN | CSR_EUEN_LSXEN | CSR_EUEN_LASXEN); 1415 1416 kvm_restore_lasx(&vcpu->arch.fpu); 1417 vcpu->arch.aux_inuse |= KVM_LARCH_FPU; 1418 trace_kvm_aux(vcpu, KVM_TRACE_AUX_RESTORE, KVM_TRACE_AUX_LASX); 1419 1420 return 0; 1421 } 1422 #endif 1423 1424 /* Save context and disable FPU */ 1425 void kvm_lose_fpu(struct kvm_vcpu *vcpu) 1426 { 1427 preempt_disable(); 1428 1429 if (!(vcpu->arch.aux_inuse & KVM_LARCH_FPU)) 1430 goto done; 1431 1432 kvm_check_fcsr_alive(vcpu); 1433 if (kvm_guest_has_lasx(&vcpu->arch)) { 1434 kvm_save_lasx(&vcpu->arch.fpu); 1435 trace_kvm_aux(vcpu, KVM_TRACE_AUX_SAVE, KVM_TRACE_AUX_LASX); 1436 1437 /* Disable LASX & LSX & FPU */ 1438 clear_csr_euen(CSR_EUEN_FPEN | CSR_EUEN_LSXEN | CSR_EUEN_LASXEN); 1439 } else if (kvm_guest_has_lsx(&vcpu->arch)) { 1440 kvm_save_lsx(&vcpu->arch.fpu); 1441 trace_kvm_aux(vcpu, KVM_TRACE_AUX_SAVE, KVM_TRACE_AUX_LSX); 1442 1443 /* Disable LSX & FPU */ 1444 clear_csr_euen(CSR_EUEN_FPEN | CSR_EUEN_LSXEN); 1445 } else if (vcpu->arch.aux_inuse & KVM_LARCH_FPU) { 1446 kvm_save_fpu(&vcpu->arch.fpu); 1447 trace_kvm_aux(vcpu, KVM_TRACE_AUX_SAVE, KVM_TRACE_AUX_FPU); 1448 1449 /* Disable FPU */ 1450 clear_csr_euen(CSR_EUEN_FPEN); 1451 } 1452 vcpu->arch.aux_inuse &= ~KVM_LARCH_FPU; 1453 1454 done: 1455 kvm_lose_lbt(vcpu); 1456 1457 preempt_enable(); 1458 } 1459 1460 int kvm_vcpu_ioctl_interrupt(struct kvm_vcpu *vcpu, struct kvm_interrupt *irq) 1461 { 1462 int intr = (int)irq->irq; 1463 unsigned int vector = abs(intr); 1464 1465 if (vector >= EXCCODE_INT_NUM) 1466 return -EINVAL; 1467 1468 if (!kvm_guest_has_msgint(&vcpu->arch) && (vector == INT_AVEC)) 1469 return -EINVAL; 1470 1471 if (intr > 0) 1472 kvm_queue_irq(vcpu, intr); 1473 else if (intr < 0) 1474 kvm_dequeue_irq(vcpu, -intr); 1475 else { 1476 kvm_err("%s: invalid interrupt ioctl %d\n", __func__, irq->irq); 1477 return -EINVAL; 1478 } 1479 1480 kvm_vcpu_kick(vcpu); 1481 1482 return 0; 1483 } 1484 1485 long kvm_arch_vcpu_unlocked_ioctl(struct file *filp, unsigned int ioctl, 1486 unsigned long arg) 1487 { 1488 void __user *argp = (void __user *)arg; 1489 struct kvm_vcpu *vcpu = filp->private_data; 1490 1491 if (ioctl == KVM_INTERRUPT) { 1492 struct kvm_interrupt irq; 1493 1494 if (copy_from_user(&irq, argp, sizeof(irq))) 1495 return -EFAULT; 1496 1497 kvm_debug("[%d] %s: irq: %d\n", vcpu->vcpu_id, __func__, irq.irq); 1498 1499 return kvm_vcpu_ioctl_interrupt(vcpu, &irq); 1500 } 1501 1502 return -ENOIOCTLCMD; 1503 } 1504 1505 int kvm_arch_vcpu_precreate(struct kvm *kvm, unsigned int id) 1506 { 1507 return 0; 1508 } 1509 1510 int kvm_arch_vcpu_create(struct kvm_vcpu *vcpu) 1511 { 1512 unsigned long timer_hz; 1513 struct loongarch_csrs *csr; 1514 1515 vcpu->arch.vpid = 0; 1516 vcpu->arch.flush_gpa = INVALID_GPA; 1517 1518 hrtimer_setup(&vcpu->arch.swtimer, kvm_swtimer_wakeup, CLOCK_MONOTONIC, 1519 HRTIMER_MODE_ABS_PINNED_HARD); 1520 1521 /* Get GPA (=HVA) of PGD for kvm hypervisor */ 1522 vcpu->arch.kvm_pgd = __pa(vcpu->kvm->arch.pgd); 1523 1524 /* 1525 * Get PGD for primary mmu, virtual address is used since there is 1526 * memory access after loading from CSR_PGD in tlb exception fast path. 1527 */ 1528 vcpu->arch.host_pgd = (unsigned long)vcpu->kvm->mm->pgd; 1529 1530 vcpu->arch.handle_exit = kvm_handle_exit; 1531 vcpu->arch.guest_eentry = (unsigned long)kvm_loongarch_ops->exc_entry; 1532 vcpu->arch.csr = kzalloc_obj(struct loongarch_csrs); 1533 if (!vcpu->arch.csr) 1534 return -ENOMEM; 1535 1536 /* 1537 * All kvm exceptions share one exception entry, and host <-> guest 1538 * switch also switch ECFG.VS field, keep host ECFG.VS info here. 1539 */ 1540 vcpu->arch.host_ecfg = (read_csr_ecfg() & CSR_ECFG_VS); 1541 1542 /* Init */ 1543 vcpu->arch.last_sched_cpu = -1; 1544 1545 /* Init ipi_state lock */ 1546 spin_lock_init(&vcpu->arch.ipi_state.lock); 1547 1548 /* 1549 * Initialize guest register state to valid architectural reset state. 1550 */ 1551 timer_hz = calc_const_freq(); 1552 kvm_init_timer(vcpu, timer_hz); 1553 1554 /* Set Initialize mode for guest */ 1555 csr = vcpu->arch.csr; 1556 kvm_write_sw_gcsr(csr, LOONGARCH_CSR_CRMD, CSR_CRMD_DA); 1557 1558 /* Set cpuid */ 1559 kvm_write_sw_gcsr(csr, LOONGARCH_CSR_TMID, vcpu->vcpu_id); 1560 kvm_write_sw_gcsr(csr, LOONGARCH_CSR_CPUID, KVM_MAX_PHYID); 1561 1562 /* Start with no pending virtual guest interrupts */ 1563 csr->csrs[LOONGARCH_CSR_GINTC] = 0; 1564 1565 return 0; 1566 } 1567 1568 void kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu) 1569 { 1570 } 1571 1572 void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu) 1573 { 1574 int cpu; 1575 struct kvm_context *context; 1576 1577 hrtimer_cancel(&vcpu->arch.swtimer); 1578 kvm_mmu_free_memory_cache(&vcpu->arch.mmu_page_cache); 1579 kvm_drop_cpuid(vcpu); 1580 kfree(vcpu->arch.csr); 1581 1582 /* 1583 * If the vCPU is freed and reused as another vCPU, we don't want the 1584 * matching pointer wrongly hanging around in last_vcpu. 1585 */ 1586 for_each_possible_cpu(cpu) { 1587 context = per_cpu_ptr(vcpu->kvm->arch.vmcs, cpu); 1588 if (context->last_vcpu == vcpu) 1589 context->last_vcpu = NULL; 1590 } 1591 } 1592 1593 static int _kvm_vcpu_load(struct kvm_vcpu *vcpu, int cpu) 1594 { 1595 bool migrated; 1596 struct kvm_context *context; 1597 struct loongarch_csrs *csr = vcpu->arch.csr; 1598 1599 /* 1600 * Have we migrated to a different CPU? 1601 * If so, any old guest TLB state may be stale. 1602 */ 1603 migrated = (vcpu->arch.last_sched_cpu != cpu); 1604 1605 /* 1606 * Was this the last vCPU to run on this CPU? 1607 * If not, any old guest state from this vCPU will have been clobbered. 1608 */ 1609 context = per_cpu_ptr(vcpu->kvm->arch.vmcs, cpu); 1610 if (migrated || (context->last_vcpu != vcpu)) { 1611 context->last_vcpu = vcpu; 1612 vcpu->arch.aux_inuse &= ~KVM_LARCH_HWCSR_USABLE; 1613 vcpu->arch.host_eentry = csr_read64(LOONGARCH_CSR_EENTRY); 1614 } 1615 1616 /* Restore timer state regardless */ 1617 kvm_restore_timer(vcpu); 1618 kvm_make_request(KVM_REQ_STEAL_UPDATE, vcpu); 1619 1620 /* Don't bother restoring registers multiple times unless necessary */ 1621 if (vcpu->arch.aux_inuse & KVM_LARCH_HWCSR_USABLE) 1622 return 0; 1623 1624 write_csr_gcntc((ulong)vcpu->kvm->arch.time_offset); 1625 1626 /* Restore guest CSR registers */ 1627 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_CRMD); 1628 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_PRMD); 1629 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_EUEN); 1630 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_MISC); 1631 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_ECFG); 1632 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_ERA); 1633 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_BADV); 1634 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_BADI); 1635 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_EENTRY); 1636 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_TLBIDX); 1637 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_TLBEHI); 1638 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_TLBELO0); 1639 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_TLBELO1); 1640 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_ASID); 1641 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_PGDL); 1642 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_PGDH); 1643 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_PWCTL0); 1644 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_PWCTL1); 1645 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_STLBPGSIZE); 1646 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_RVACFG); 1647 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_CPUID); 1648 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_KS0); 1649 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_KS1); 1650 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_KS2); 1651 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_KS3); 1652 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_KS4); 1653 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_KS5); 1654 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_KS6); 1655 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_KS7); 1656 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_TMID); 1657 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_CNTC); 1658 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_TLBRENTRY); 1659 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_TLBRBADV); 1660 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_TLBRERA); 1661 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_TLBRSAVE); 1662 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_TLBRELO0); 1663 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_TLBRELO1); 1664 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_TLBREHI); 1665 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_TLBRPRMD); 1666 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_DMWIN0); 1667 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_DMWIN1); 1668 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_DMWIN2); 1669 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_DMWIN3); 1670 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_LLBCTL); 1671 1672 if (kvm_guest_has_msgint(&vcpu->arch)) { 1673 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_IPR); 1674 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_ISR0); 1675 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_ISR1); 1676 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_ISR2); 1677 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_ISR3); 1678 } 1679 1680 /* Restore Root.GINTC from unused Guest.GINTC register */ 1681 write_csr_gintc(csr->csrs[LOONGARCH_CSR_GINTC]); 1682 write_csr_gstat(csr->csrs[LOONGARCH_CSR_GSTAT]); 1683 1684 /* 1685 * We should clear linked load bit to break interrupted atomics. This 1686 * prevents a SC on the next vCPU from succeeding by matching a LL on 1687 * the previous vCPU. 1688 */ 1689 if (vcpu->kvm->created_vcpus > 1) 1690 set_gcsr_llbctl(CSR_LLBCTL_WCLLB); 1691 1692 vcpu->arch.aux_inuse |= KVM_LARCH_HWCSR_USABLE; 1693 1694 return 0; 1695 } 1696 1697 void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu) 1698 { 1699 unsigned long flags; 1700 1701 local_irq_save(flags); 1702 /* Restore guest state to registers */ 1703 _kvm_vcpu_load(vcpu, cpu); 1704 local_irq_restore(flags); 1705 } 1706 1707 static int _kvm_vcpu_put(struct kvm_vcpu *vcpu, int cpu) 1708 { 1709 struct loongarch_csrs *csr = vcpu->arch.csr; 1710 1711 kvm_lose_fpu(vcpu); 1712 1713 /* 1714 * Update CSR state from hardware if software CSR state is stale, 1715 * most CSR registers are kept unchanged during process context 1716 * switch except CSR registers like remaining timer tick value and 1717 * injected interrupt state. 1718 */ 1719 if (vcpu->arch.aux_inuse & KVM_LARCH_SWCSR_LATEST) 1720 goto out; 1721 1722 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_CRMD); 1723 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_PRMD); 1724 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_EUEN); 1725 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_MISC); 1726 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_ECFG); 1727 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_ERA); 1728 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_BADV); 1729 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_BADI); 1730 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_EENTRY); 1731 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_TLBIDX); 1732 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_TLBEHI); 1733 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_TLBELO0); 1734 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_TLBELO1); 1735 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_ASID); 1736 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_PGDL); 1737 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_PGDH); 1738 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_PWCTL0); 1739 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_PWCTL1); 1740 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_STLBPGSIZE); 1741 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_RVACFG); 1742 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_CPUID); 1743 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_PRCFG1); 1744 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_PRCFG2); 1745 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_PRCFG3); 1746 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_KS0); 1747 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_KS1); 1748 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_KS2); 1749 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_KS3); 1750 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_KS4); 1751 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_KS5); 1752 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_KS6); 1753 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_KS7); 1754 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_TMID); 1755 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_CNTC); 1756 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_LLBCTL); 1757 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_TLBRENTRY); 1758 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_TLBRBADV); 1759 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_TLBRERA); 1760 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_TLBRSAVE); 1761 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_TLBRELO0); 1762 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_TLBRELO1); 1763 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_TLBREHI); 1764 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_TLBRPRMD); 1765 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_DMWIN0); 1766 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_DMWIN1); 1767 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_DMWIN2); 1768 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_DMWIN3); 1769 1770 if (kvm_guest_has_msgint(&vcpu->arch)) { 1771 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_IPR); 1772 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_ISR0); 1773 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_ISR1); 1774 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_ISR2); 1775 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_ISR3); 1776 } 1777 1778 csr->csrs[LOONGARCH_CSR_GSTAT] = read_csr_gstat(); 1779 vcpu->arch.aux_inuse |= KVM_LARCH_SWCSR_LATEST; 1780 1781 out: 1782 kvm_save_timer(vcpu); 1783 /* Save Root.GINTC into unused Guest.GINTC register */ 1784 csr->csrs[LOONGARCH_CSR_GINTC] = read_csr_gintc(); 1785 1786 return 0; 1787 } 1788 1789 static void kvm_vcpu_set_pv_preempted(struct kvm_vcpu *vcpu) 1790 { 1791 gpa_t gpa; 1792 struct gfn_to_hva_cache *ghc; 1793 struct kvm_memslots *slots; 1794 struct kvm_steal_time __user *st; 1795 1796 gpa = vcpu->arch.st.guest_addr; 1797 if (!(gpa & KVM_STEAL_PHYS_VALID)) 1798 return; 1799 1800 /* vCPU may be preempted for many times */ 1801 if (vcpu->arch.st.preempted) 1802 return; 1803 1804 /* This happens on process exit */ 1805 if (unlikely(current->mm != vcpu->kvm->mm)) 1806 return; 1807 1808 gpa &= KVM_STEAL_PHYS_MASK; 1809 ghc = &vcpu->arch.st.cache; 1810 slots = kvm_memslots(vcpu->kvm); 1811 if (slots->generation != ghc->generation || gpa != ghc->gpa) { 1812 if (kvm_gfn_to_hva_cache_init(vcpu->kvm, ghc, gpa, sizeof(*st))) { 1813 ghc->gpa = INVALID_GPA; 1814 return; 1815 } 1816 } 1817 1818 st = (struct kvm_steal_time __user *)ghc->hva; 1819 unsafe_put_user(KVM_VCPU_PREEMPTED, &st->preempted, out); 1820 vcpu->arch.st.preempted = KVM_VCPU_PREEMPTED; 1821 out: 1822 mark_page_dirty_in_slot(vcpu->kvm, ghc->memslot, gpa_to_gfn(ghc->gpa)); 1823 } 1824 1825 void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu) 1826 { 1827 int cpu, idx; 1828 unsigned long flags; 1829 1830 if (vcpu->preempted && kvm_guest_has_pv_feature(vcpu, KVM_FEATURE_PREEMPT)) { 1831 /* 1832 * Take the srcu lock as memslots will be accessed to check 1833 * the gfn cache generation against the memslots generation. 1834 */ 1835 idx = srcu_read_lock(&vcpu->kvm->srcu); 1836 kvm_vcpu_set_pv_preempted(vcpu); 1837 srcu_read_unlock(&vcpu->kvm->srcu, idx); 1838 } 1839 1840 local_irq_save(flags); 1841 cpu = smp_processor_id(); 1842 vcpu->arch.last_sched_cpu = cpu; 1843 1844 /* Save guest state in registers */ 1845 _kvm_vcpu_put(vcpu, cpu); 1846 local_irq_restore(flags); 1847 } 1848 1849 int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu) 1850 { 1851 int r = -EINTR; 1852 struct kvm_run *run = vcpu->run; 1853 1854 if (vcpu->mmio_needed) { 1855 if (!vcpu->mmio_is_write) 1856 kvm_complete_mmio_read(vcpu, run); 1857 vcpu->mmio_needed = 0; 1858 } 1859 1860 switch (run->exit_reason) { 1861 case KVM_EXIT_HYPERCALL: 1862 kvm_complete_user_service(vcpu, run); 1863 break; 1864 case KVM_EXIT_LOONGARCH_IOCSR: 1865 if (!run->iocsr_io.is_write) 1866 kvm_complete_iocsr_read(vcpu, run); 1867 break; 1868 } 1869 1870 if (!vcpu->wants_to_run) 1871 return r; 1872 1873 /* Clear exit_reason */ 1874 run->exit_reason = KVM_EXIT_UNKNOWN; 1875 lose_fpu(1); 1876 vcpu_load(vcpu); 1877 kvm_sigset_activate(vcpu); 1878 r = kvm_pre_enter_guest(vcpu); 1879 if (r != RESUME_GUEST) 1880 goto out; 1881 1882 guest_timing_enter_irqoff(); 1883 guest_state_enter_irqoff(); 1884 trace_kvm_enter(vcpu); 1885 r = kvm_loongarch_ops->enter_guest(run, vcpu); 1886 1887 trace_kvm_out(vcpu); 1888 /* 1889 * Guest exit is already recorded at kvm_handle_exit() 1890 * return value must not be RESUME_GUEST 1891 */ 1892 local_irq_enable(); 1893 out: 1894 kvm_sigset_deactivate(vcpu); 1895 vcpu_put(vcpu); 1896 1897 return r; 1898 } 1899