1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (C) 2020-2023 Loongson Technology Corporation Limited 4 */ 5 6 #include <linux/kvm_host.h> 7 #include <linux/entry-kvm.h> 8 #include <asm/fpu.h> 9 #include <asm/loongarch.h> 10 #include <asm/setup.h> 11 #include <asm/time.h> 12 13 #define CREATE_TRACE_POINTS 14 #include "trace.h" 15 16 const struct _kvm_stats_desc kvm_vcpu_stats_desc[] = { 17 KVM_GENERIC_VCPU_STATS(), 18 STATS_DESC_COUNTER(VCPU, int_exits), 19 STATS_DESC_COUNTER(VCPU, idle_exits), 20 STATS_DESC_COUNTER(VCPU, cpucfg_exits), 21 STATS_DESC_COUNTER(VCPU, signal_exits), 22 STATS_DESC_COUNTER(VCPU, hypercall_exits) 23 }; 24 25 const struct kvm_stats_header kvm_vcpu_stats_header = { 26 .name_size = KVM_STATS_NAME_SIZE, 27 .num_desc = ARRAY_SIZE(kvm_vcpu_stats_desc), 28 .id_offset = sizeof(struct kvm_stats_header), 29 .desc_offset = sizeof(struct kvm_stats_header) + KVM_STATS_NAME_SIZE, 30 .data_offset = sizeof(struct kvm_stats_header) + KVM_STATS_NAME_SIZE + 31 sizeof(kvm_vcpu_stats_desc), 32 }; 33 34 static void kvm_update_stolen_time(struct kvm_vcpu *vcpu) 35 { 36 u32 version; 37 u64 steal; 38 gpa_t gpa; 39 struct kvm_memslots *slots; 40 struct kvm_steal_time __user *st; 41 struct gfn_to_hva_cache *ghc; 42 43 ghc = &vcpu->arch.st.cache; 44 gpa = vcpu->arch.st.guest_addr; 45 if (!(gpa & KVM_STEAL_PHYS_VALID)) 46 return; 47 48 gpa &= KVM_STEAL_PHYS_MASK; 49 slots = kvm_memslots(vcpu->kvm); 50 if (slots->generation != ghc->generation || gpa != ghc->gpa) { 51 if (kvm_gfn_to_hva_cache_init(vcpu->kvm, ghc, gpa, sizeof(*st))) { 52 ghc->gpa = INVALID_GPA; 53 return; 54 } 55 } 56 57 st = (struct kvm_steal_time __user *)ghc->hva; 58 unsafe_get_user(version, &st->version, out); 59 if (version & 1) 60 version += 1; /* first time write, random junk */ 61 62 version += 1; 63 unsafe_put_user(version, &st->version, out); 64 smp_wmb(); 65 66 unsafe_get_user(steal, &st->steal, out); 67 steal += current->sched_info.run_delay - vcpu->arch.st.last_steal; 68 vcpu->arch.st.last_steal = current->sched_info.run_delay; 69 unsafe_put_user(steal, &st->steal, out); 70 71 smp_wmb(); 72 version += 1; 73 unsafe_put_user(version, &st->version, out); 74 out: 75 mark_page_dirty_in_slot(vcpu->kvm, ghc->memslot, gpa_to_gfn(ghc->gpa)); 76 } 77 78 /* 79 * kvm_check_requests - check and handle pending vCPU requests 80 * 81 * Return: RESUME_GUEST if we should enter the guest 82 * RESUME_HOST if we should exit to userspace 83 */ 84 static int kvm_check_requests(struct kvm_vcpu *vcpu) 85 { 86 if (!kvm_request_pending(vcpu)) 87 return RESUME_GUEST; 88 89 if (kvm_check_request(KVM_REQ_TLB_FLUSH, vcpu)) 90 vcpu->arch.vpid = 0; /* Drop vpid for this vCPU */ 91 92 if (kvm_dirty_ring_check_request(vcpu)) 93 return RESUME_HOST; 94 95 if (kvm_check_request(KVM_REQ_STEAL_UPDATE, vcpu)) 96 kvm_update_stolen_time(vcpu); 97 98 return RESUME_GUEST; 99 } 100 101 static void kvm_late_check_requests(struct kvm_vcpu *vcpu) 102 { 103 lockdep_assert_irqs_disabled(); 104 if (kvm_check_request(KVM_REQ_TLB_FLUSH_GPA, vcpu)) 105 if (vcpu->arch.flush_gpa != INVALID_GPA) { 106 kvm_flush_tlb_gpa(vcpu, vcpu->arch.flush_gpa); 107 vcpu->arch.flush_gpa = INVALID_GPA; 108 } 109 } 110 111 /* 112 * Check and handle pending signal and vCPU requests etc 113 * Run with irq enabled and preempt enabled 114 * 115 * Return: RESUME_GUEST if we should enter the guest 116 * RESUME_HOST if we should exit to userspace 117 * < 0 if we should exit to userspace, where the return value 118 * indicates an error 119 */ 120 static int kvm_enter_guest_check(struct kvm_vcpu *vcpu) 121 { 122 int ret; 123 124 /* 125 * Check conditions before entering the guest 126 */ 127 ret = xfer_to_guest_mode_handle_work(vcpu); 128 if (ret < 0) 129 return ret; 130 131 ret = kvm_check_requests(vcpu); 132 133 return ret; 134 } 135 136 /* 137 * Called with irq enabled 138 * 139 * Return: RESUME_GUEST if we should enter the guest, and irq disabled 140 * Others if we should exit to userspace 141 */ 142 static int kvm_pre_enter_guest(struct kvm_vcpu *vcpu) 143 { 144 int ret; 145 146 do { 147 ret = kvm_enter_guest_check(vcpu); 148 if (ret != RESUME_GUEST) 149 break; 150 151 /* 152 * Handle vcpu timer, interrupts, check requests and 153 * check vmid before vcpu enter guest 154 */ 155 local_irq_disable(); 156 kvm_deliver_intr(vcpu); 157 kvm_deliver_exception(vcpu); 158 /* Make sure the vcpu mode has been written */ 159 smp_store_mb(vcpu->mode, IN_GUEST_MODE); 160 kvm_check_vpid(vcpu); 161 162 /* 163 * Called after function kvm_check_vpid() 164 * Since it updates CSR.GSTAT used by kvm_flush_tlb_gpa(), 165 * and it may also clear KVM_REQ_TLB_FLUSH_GPA pending bit 166 */ 167 kvm_late_check_requests(vcpu); 168 vcpu->arch.host_eentry = csr_read64(LOONGARCH_CSR_EENTRY); 169 /* Clear KVM_LARCH_SWCSR_LATEST as CSR will change when enter guest */ 170 vcpu->arch.aux_inuse &= ~KVM_LARCH_SWCSR_LATEST; 171 172 if (kvm_request_pending(vcpu) || xfer_to_guest_mode_work_pending()) { 173 /* make sure the vcpu mode has been written */ 174 smp_store_mb(vcpu->mode, OUTSIDE_GUEST_MODE); 175 local_irq_enable(); 176 ret = -EAGAIN; 177 } 178 } while (ret != RESUME_GUEST); 179 180 return ret; 181 } 182 183 /* 184 * Return 1 for resume guest and "<= 0" for resume host. 185 */ 186 static int kvm_handle_exit(struct kvm_run *run, struct kvm_vcpu *vcpu) 187 { 188 int ret = RESUME_GUEST; 189 unsigned long estat = vcpu->arch.host_estat; 190 u32 intr = estat & 0x1fff; /* Ignore NMI */ 191 u32 ecode = (estat & CSR_ESTAT_EXC) >> CSR_ESTAT_EXC_SHIFT; 192 193 vcpu->mode = OUTSIDE_GUEST_MODE; 194 195 /* Set a default exit reason */ 196 run->exit_reason = KVM_EXIT_UNKNOWN; 197 198 guest_timing_exit_irqoff(); 199 guest_state_exit_irqoff(); 200 local_irq_enable(); 201 202 trace_kvm_exit(vcpu, ecode); 203 if (ecode) { 204 ret = kvm_handle_fault(vcpu, ecode); 205 } else { 206 WARN(!intr, "vm exiting with suspicious irq\n"); 207 ++vcpu->stat.int_exits; 208 } 209 210 if (ret == RESUME_GUEST) 211 ret = kvm_pre_enter_guest(vcpu); 212 213 if (ret != RESUME_GUEST) { 214 local_irq_disable(); 215 return ret; 216 } 217 218 guest_timing_enter_irqoff(); 219 guest_state_enter_irqoff(); 220 trace_kvm_reenter(vcpu); 221 222 return RESUME_GUEST; 223 } 224 225 int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu) 226 { 227 return !!(vcpu->arch.irq_pending) && 228 vcpu->arch.mp_state.mp_state == KVM_MP_STATE_RUNNABLE; 229 } 230 231 int kvm_arch_vcpu_should_kick(struct kvm_vcpu *vcpu) 232 { 233 return kvm_vcpu_exiting_guest_mode(vcpu) == IN_GUEST_MODE; 234 } 235 236 bool kvm_arch_vcpu_in_kernel(struct kvm_vcpu *vcpu) 237 { 238 return false; 239 } 240 241 vm_fault_t kvm_arch_vcpu_fault(struct kvm_vcpu *vcpu, struct vm_fault *vmf) 242 { 243 return VM_FAULT_SIGBUS; 244 } 245 246 int kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu *vcpu, 247 struct kvm_translation *tr) 248 { 249 return -EINVAL; 250 } 251 252 int kvm_cpu_has_pending_timer(struct kvm_vcpu *vcpu) 253 { 254 int ret; 255 256 /* Protect from TOD sync and vcpu_load/put() */ 257 preempt_disable(); 258 ret = kvm_pending_timer(vcpu) || 259 kvm_read_hw_gcsr(LOONGARCH_CSR_ESTAT) & (1 << INT_TI); 260 preempt_enable(); 261 262 return ret; 263 } 264 265 int kvm_arch_vcpu_dump_regs(struct kvm_vcpu *vcpu) 266 { 267 int i; 268 269 kvm_debug("vCPU Register Dump:\n"); 270 kvm_debug("\tPC = 0x%08lx\n", vcpu->arch.pc); 271 kvm_debug("\tExceptions: %08lx\n", vcpu->arch.irq_pending); 272 273 for (i = 0; i < 32; i += 4) { 274 kvm_debug("\tGPR%02d: %08lx %08lx %08lx %08lx\n", i, 275 vcpu->arch.gprs[i], vcpu->arch.gprs[i + 1], 276 vcpu->arch.gprs[i + 2], vcpu->arch.gprs[i + 3]); 277 } 278 279 kvm_debug("\tCRMD: 0x%08lx, ESTAT: 0x%08lx\n", 280 kvm_read_hw_gcsr(LOONGARCH_CSR_CRMD), 281 kvm_read_hw_gcsr(LOONGARCH_CSR_ESTAT)); 282 283 kvm_debug("\tERA: 0x%08lx\n", kvm_read_hw_gcsr(LOONGARCH_CSR_ERA)); 284 285 return 0; 286 } 287 288 int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu, 289 struct kvm_mp_state *mp_state) 290 { 291 *mp_state = vcpu->arch.mp_state; 292 293 return 0; 294 } 295 296 int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu, 297 struct kvm_mp_state *mp_state) 298 { 299 int ret = 0; 300 301 switch (mp_state->mp_state) { 302 case KVM_MP_STATE_RUNNABLE: 303 vcpu->arch.mp_state = *mp_state; 304 break; 305 default: 306 ret = -EINVAL; 307 } 308 309 return ret; 310 } 311 312 int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu, 313 struct kvm_guest_debug *dbg) 314 { 315 if (dbg->control & ~KVM_GUESTDBG_VALID_MASK) 316 return -EINVAL; 317 318 if (dbg->control & KVM_GUESTDBG_ENABLE) 319 vcpu->guest_debug = dbg->control; 320 else 321 vcpu->guest_debug = 0; 322 323 return 0; 324 } 325 326 static inline int kvm_set_cpuid(struct kvm_vcpu *vcpu, u64 val) 327 { 328 int cpuid; 329 struct kvm_phyid_map *map; 330 struct loongarch_csrs *csr = vcpu->arch.csr; 331 332 if (val >= KVM_MAX_PHYID) 333 return -EINVAL; 334 335 map = vcpu->kvm->arch.phyid_map; 336 cpuid = kvm_read_sw_gcsr(csr, LOONGARCH_CSR_CPUID); 337 338 spin_lock(&vcpu->kvm->arch.phyid_map_lock); 339 if ((cpuid < KVM_MAX_PHYID) && map->phys_map[cpuid].enabled) { 340 /* Discard duplicated CPUID set operation */ 341 if (cpuid == val) { 342 spin_unlock(&vcpu->kvm->arch.phyid_map_lock); 343 return 0; 344 } 345 346 /* 347 * CPUID is already set before 348 * Forbid changing to a different CPUID at runtime 349 */ 350 spin_unlock(&vcpu->kvm->arch.phyid_map_lock); 351 return -EINVAL; 352 } 353 354 if (map->phys_map[val].enabled) { 355 /* Discard duplicated CPUID set operation */ 356 if (vcpu == map->phys_map[val].vcpu) { 357 spin_unlock(&vcpu->kvm->arch.phyid_map_lock); 358 return 0; 359 } 360 361 /* 362 * New CPUID is already set with other vcpu 363 * Forbid sharing the same CPUID between different vcpus 364 */ 365 spin_unlock(&vcpu->kvm->arch.phyid_map_lock); 366 return -EINVAL; 367 } 368 369 kvm_write_sw_gcsr(csr, LOONGARCH_CSR_CPUID, val); 370 map->phys_map[val].enabled = true; 371 map->phys_map[val].vcpu = vcpu; 372 spin_unlock(&vcpu->kvm->arch.phyid_map_lock); 373 374 return 0; 375 } 376 377 static inline void kvm_drop_cpuid(struct kvm_vcpu *vcpu) 378 { 379 int cpuid; 380 struct kvm_phyid_map *map; 381 struct loongarch_csrs *csr = vcpu->arch.csr; 382 383 map = vcpu->kvm->arch.phyid_map; 384 cpuid = kvm_read_sw_gcsr(csr, LOONGARCH_CSR_CPUID); 385 386 if (cpuid >= KVM_MAX_PHYID) 387 return; 388 389 spin_lock(&vcpu->kvm->arch.phyid_map_lock); 390 if (map->phys_map[cpuid].enabled) { 391 map->phys_map[cpuid].vcpu = NULL; 392 map->phys_map[cpuid].enabled = false; 393 kvm_write_sw_gcsr(csr, LOONGARCH_CSR_CPUID, KVM_MAX_PHYID); 394 } 395 spin_unlock(&vcpu->kvm->arch.phyid_map_lock); 396 } 397 398 struct kvm_vcpu *kvm_get_vcpu_by_cpuid(struct kvm *kvm, int cpuid) 399 { 400 struct kvm_phyid_map *map; 401 402 if (cpuid >= KVM_MAX_PHYID) 403 return NULL; 404 405 map = kvm->arch.phyid_map; 406 if (!map->phys_map[cpuid].enabled) 407 return NULL; 408 409 return map->phys_map[cpuid].vcpu; 410 } 411 412 static int _kvm_getcsr(struct kvm_vcpu *vcpu, unsigned int id, u64 *val) 413 { 414 unsigned long gintc; 415 struct loongarch_csrs *csr = vcpu->arch.csr; 416 417 if (get_gcsr_flag(id) & INVALID_GCSR) 418 return -EINVAL; 419 420 if (id == LOONGARCH_CSR_ESTAT) { 421 preempt_disable(); 422 vcpu_load(vcpu); 423 /* 424 * Sync pending interrupts into ESTAT so that interrupt 425 * remains during VM migration stage 426 */ 427 kvm_deliver_intr(vcpu); 428 vcpu->arch.aux_inuse &= ~KVM_LARCH_SWCSR_LATEST; 429 vcpu_put(vcpu); 430 preempt_enable(); 431 432 /* ESTAT IP0~IP7 get from GINTC */ 433 gintc = kvm_read_sw_gcsr(csr, LOONGARCH_CSR_GINTC) & 0xff; 434 *val = kvm_read_sw_gcsr(csr, LOONGARCH_CSR_ESTAT) | (gintc << 2); 435 return 0; 436 } 437 438 /* 439 * Get software CSR state since software state is consistent 440 * with hardware for synchronous ioctl 441 */ 442 *val = kvm_read_sw_gcsr(csr, id); 443 444 return 0; 445 } 446 447 static int _kvm_setcsr(struct kvm_vcpu *vcpu, unsigned int id, u64 val) 448 { 449 int ret = 0, gintc; 450 struct loongarch_csrs *csr = vcpu->arch.csr; 451 452 if (get_gcsr_flag(id) & INVALID_GCSR) 453 return -EINVAL; 454 455 if (id == LOONGARCH_CSR_CPUID) 456 return kvm_set_cpuid(vcpu, val); 457 458 if (id == LOONGARCH_CSR_ESTAT) { 459 /* ESTAT IP0~IP7 inject through GINTC */ 460 gintc = (val >> 2) & 0xff; 461 kvm_set_sw_gcsr(csr, LOONGARCH_CSR_GINTC, gintc); 462 463 gintc = val & ~(0xffUL << 2); 464 kvm_set_sw_gcsr(csr, LOONGARCH_CSR_ESTAT, gintc); 465 466 return ret; 467 } 468 469 kvm_write_sw_gcsr(csr, id, val); 470 471 return ret; 472 } 473 474 static int _kvm_get_cpucfg_mask(int id, u64 *v) 475 { 476 if (id < 0 || id >= KVM_MAX_CPUCFG_REGS) 477 return -EINVAL; 478 479 switch (id) { 480 case LOONGARCH_CPUCFG0: 481 *v = GENMASK(31, 0); 482 return 0; 483 case LOONGARCH_CPUCFG1: 484 /* CPUCFG1_MSGINT is not supported by KVM */ 485 *v = GENMASK(25, 0); 486 return 0; 487 case LOONGARCH_CPUCFG2: 488 /* CPUCFG2 features unconditionally supported by KVM */ 489 *v = CPUCFG2_FP | CPUCFG2_FPSP | CPUCFG2_FPDP | 490 CPUCFG2_FPVERS | CPUCFG2_LLFTP | CPUCFG2_LLFTPREV | 491 CPUCFG2_LSPW | CPUCFG2_LAM; 492 /* 493 * For the ISA extensions listed below, if one is supported 494 * by the host, then it is also supported by KVM. 495 */ 496 if (cpu_has_lsx) 497 *v |= CPUCFG2_LSX; 498 if (cpu_has_lasx) 499 *v |= CPUCFG2_LASX; 500 501 return 0; 502 case LOONGARCH_CPUCFG3: 503 *v = GENMASK(16, 0); 504 return 0; 505 case LOONGARCH_CPUCFG4: 506 case LOONGARCH_CPUCFG5: 507 *v = GENMASK(31, 0); 508 return 0; 509 case LOONGARCH_CPUCFG16: 510 *v = GENMASK(16, 0); 511 return 0; 512 case LOONGARCH_CPUCFG17 ... LOONGARCH_CPUCFG20: 513 *v = GENMASK(30, 0); 514 return 0; 515 default: 516 /* 517 * CPUCFG bits should be zero if reserved by HW or not 518 * supported by KVM. 519 */ 520 *v = 0; 521 return 0; 522 } 523 } 524 525 static int kvm_check_cpucfg(int id, u64 val) 526 { 527 int ret; 528 u64 mask = 0; 529 530 ret = _kvm_get_cpucfg_mask(id, &mask); 531 if (ret) 532 return ret; 533 534 if (val & ~mask) 535 /* Unsupported features and/or the higher 32 bits should not be set */ 536 return -EINVAL; 537 538 switch (id) { 539 case LOONGARCH_CPUCFG2: 540 if (!(val & CPUCFG2_LLFTP)) 541 /* Guests must have a constant timer */ 542 return -EINVAL; 543 if ((val & CPUCFG2_FP) && (!(val & CPUCFG2_FPSP) || !(val & CPUCFG2_FPDP))) 544 /* Single and double float point must both be set when FP is enabled */ 545 return -EINVAL; 546 if ((val & CPUCFG2_LSX) && !(val & CPUCFG2_FP)) 547 /* LSX architecturally implies FP but val does not satisfy that */ 548 return -EINVAL; 549 if ((val & CPUCFG2_LASX) && !(val & CPUCFG2_LSX)) 550 /* LASX architecturally implies LSX and FP but val does not satisfy that */ 551 return -EINVAL; 552 return 0; 553 default: 554 /* 555 * Values for the other CPUCFG IDs are not being further validated 556 * besides the mask check above. 557 */ 558 return 0; 559 } 560 } 561 562 static int kvm_get_one_reg(struct kvm_vcpu *vcpu, 563 const struct kvm_one_reg *reg, u64 *v) 564 { 565 int id, ret = 0; 566 u64 type = reg->id & KVM_REG_LOONGARCH_MASK; 567 568 switch (type) { 569 case KVM_REG_LOONGARCH_CSR: 570 id = KVM_GET_IOC_CSR_IDX(reg->id); 571 ret = _kvm_getcsr(vcpu, id, v); 572 break; 573 case KVM_REG_LOONGARCH_CPUCFG: 574 id = KVM_GET_IOC_CPUCFG_IDX(reg->id); 575 if (id >= 0 && id < KVM_MAX_CPUCFG_REGS) 576 *v = vcpu->arch.cpucfg[id]; 577 else 578 ret = -EINVAL; 579 break; 580 case KVM_REG_LOONGARCH_KVM: 581 switch (reg->id) { 582 case KVM_REG_LOONGARCH_COUNTER: 583 *v = drdtime() + vcpu->kvm->arch.time_offset; 584 break; 585 case KVM_REG_LOONGARCH_DEBUG_INST: 586 *v = INSN_HVCL | KVM_HCALL_SWDBG; 587 break; 588 default: 589 ret = -EINVAL; 590 break; 591 } 592 break; 593 default: 594 ret = -EINVAL; 595 break; 596 } 597 598 return ret; 599 } 600 601 static int kvm_get_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg) 602 { 603 int ret = 0; 604 u64 v, size = reg->id & KVM_REG_SIZE_MASK; 605 606 switch (size) { 607 case KVM_REG_SIZE_U64: 608 ret = kvm_get_one_reg(vcpu, reg, &v); 609 if (ret) 610 return ret; 611 ret = put_user(v, (u64 __user *)(long)reg->addr); 612 break; 613 default: 614 ret = -EINVAL; 615 break; 616 } 617 618 return ret; 619 } 620 621 static int kvm_set_one_reg(struct kvm_vcpu *vcpu, 622 const struct kvm_one_reg *reg, u64 v) 623 { 624 int id, ret = 0; 625 u64 type = reg->id & KVM_REG_LOONGARCH_MASK; 626 627 switch (type) { 628 case KVM_REG_LOONGARCH_CSR: 629 id = KVM_GET_IOC_CSR_IDX(reg->id); 630 ret = _kvm_setcsr(vcpu, id, v); 631 break; 632 case KVM_REG_LOONGARCH_CPUCFG: 633 id = KVM_GET_IOC_CPUCFG_IDX(reg->id); 634 ret = kvm_check_cpucfg(id, v); 635 if (ret) 636 break; 637 vcpu->arch.cpucfg[id] = (u32)v; 638 break; 639 case KVM_REG_LOONGARCH_KVM: 640 switch (reg->id) { 641 case KVM_REG_LOONGARCH_COUNTER: 642 /* 643 * gftoffset is relative with board, not vcpu 644 * only set for the first time for smp system 645 */ 646 if (vcpu->vcpu_id == 0) 647 vcpu->kvm->arch.time_offset = (signed long)(v - drdtime()); 648 break; 649 case KVM_REG_LOONGARCH_VCPU_RESET: 650 vcpu->arch.st.guest_addr = 0; 651 memset(&vcpu->arch.irq_pending, 0, sizeof(vcpu->arch.irq_pending)); 652 memset(&vcpu->arch.irq_clear, 0, sizeof(vcpu->arch.irq_clear)); 653 break; 654 default: 655 ret = -EINVAL; 656 break; 657 } 658 break; 659 default: 660 ret = -EINVAL; 661 break; 662 } 663 664 return ret; 665 } 666 667 static int kvm_set_reg(struct kvm_vcpu *vcpu, const struct kvm_one_reg *reg) 668 { 669 int ret = 0; 670 u64 v, size = reg->id & KVM_REG_SIZE_MASK; 671 672 switch (size) { 673 case KVM_REG_SIZE_U64: 674 ret = get_user(v, (u64 __user *)(long)reg->addr); 675 if (ret) 676 return ret; 677 break; 678 default: 679 return -EINVAL; 680 } 681 682 return kvm_set_one_reg(vcpu, reg, v); 683 } 684 685 int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs) 686 { 687 return -ENOIOCTLCMD; 688 } 689 690 int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu, struct kvm_sregs *sregs) 691 { 692 return -ENOIOCTLCMD; 693 } 694 695 int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs) 696 { 697 int i; 698 699 for (i = 0; i < ARRAY_SIZE(vcpu->arch.gprs); i++) 700 regs->gpr[i] = vcpu->arch.gprs[i]; 701 702 regs->pc = vcpu->arch.pc; 703 704 return 0; 705 } 706 707 int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs) 708 { 709 int i; 710 711 for (i = 1; i < ARRAY_SIZE(vcpu->arch.gprs); i++) 712 vcpu->arch.gprs[i] = regs->gpr[i]; 713 714 vcpu->arch.gprs[0] = 0; /* zero is special, and cannot be set. */ 715 vcpu->arch.pc = regs->pc; 716 717 return 0; 718 } 719 720 static int kvm_vcpu_ioctl_enable_cap(struct kvm_vcpu *vcpu, 721 struct kvm_enable_cap *cap) 722 { 723 /* FPU is enabled by default, will support LSX/LASX later. */ 724 return -EINVAL; 725 } 726 727 static int kvm_loongarch_cpucfg_has_attr(struct kvm_vcpu *vcpu, 728 struct kvm_device_attr *attr) 729 { 730 switch (attr->attr) { 731 case 2: 732 return 0; 733 default: 734 return -ENXIO; 735 } 736 737 return -ENXIO; 738 } 739 740 static int kvm_loongarch_pvtime_has_attr(struct kvm_vcpu *vcpu, 741 struct kvm_device_attr *attr) 742 { 743 if (!kvm_pvtime_supported() || 744 attr->attr != KVM_LOONGARCH_VCPU_PVTIME_GPA) 745 return -ENXIO; 746 747 return 0; 748 } 749 750 static int kvm_loongarch_vcpu_has_attr(struct kvm_vcpu *vcpu, 751 struct kvm_device_attr *attr) 752 { 753 int ret = -ENXIO; 754 755 switch (attr->group) { 756 case KVM_LOONGARCH_VCPU_CPUCFG: 757 ret = kvm_loongarch_cpucfg_has_attr(vcpu, attr); 758 break; 759 case KVM_LOONGARCH_VCPU_PVTIME_CTRL: 760 ret = kvm_loongarch_pvtime_has_attr(vcpu, attr); 761 break; 762 default: 763 break; 764 } 765 766 return ret; 767 } 768 769 static int kvm_loongarch_cpucfg_get_attr(struct kvm_vcpu *vcpu, 770 struct kvm_device_attr *attr) 771 { 772 int ret = 0; 773 uint64_t val; 774 uint64_t __user *uaddr = (uint64_t __user *)attr->addr; 775 776 ret = _kvm_get_cpucfg_mask(attr->attr, &val); 777 if (ret) 778 return ret; 779 780 put_user(val, uaddr); 781 782 return ret; 783 } 784 785 static int kvm_loongarch_pvtime_get_attr(struct kvm_vcpu *vcpu, 786 struct kvm_device_attr *attr) 787 { 788 u64 gpa; 789 u64 __user *user = (u64 __user *)attr->addr; 790 791 if (!kvm_pvtime_supported() || 792 attr->attr != KVM_LOONGARCH_VCPU_PVTIME_GPA) 793 return -ENXIO; 794 795 gpa = vcpu->arch.st.guest_addr; 796 if (put_user(gpa, user)) 797 return -EFAULT; 798 799 return 0; 800 } 801 802 static int kvm_loongarch_vcpu_get_attr(struct kvm_vcpu *vcpu, 803 struct kvm_device_attr *attr) 804 { 805 int ret = -ENXIO; 806 807 switch (attr->group) { 808 case KVM_LOONGARCH_VCPU_CPUCFG: 809 ret = kvm_loongarch_cpucfg_get_attr(vcpu, attr); 810 break; 811 case KVM_LOONGARCH_VCPU_PVTIME_CTRL: 812 ret = kvm_loongarch_pvtime_get_attr(vcpu, attr); 813 break; 814 default: 815 break; 816 } 817 818 return ret; 819 } 820 821 static int kvm_loongarch_cpucfg_set_attr(struct kvm_vcpu *vcpu, 822 struct kvm_device_attr *attr) 823 { 824 return -ENXIO; 825 } 826 827 static int kvm_loongarch_pvtime_set_attr(struct kvm_vcpu *vcpu, 828 struct kvm_device_attr *attr) 829 { 830 int idx, ret = 0; 831 u64 gpa, __user *user = (u64 __user *)attr->addr; 832 struct kvm *kvm = vcpu->kvm; 833 834 if (!kvm_pvtime_supported() || 835 attr->attr != KVM_LOONGARCH_VCPU_PVTIME_GPA) 836 return -ENXIO; 837 838 if (get_user(gpa, user)) 839 return -EFAULT; 840 841 if (gpa & ~(KVM_STEAL_PHYS_MASK | KVM_STEAL_PHYS_VALID)) 842 return -EINVAL; 843 844 if (!(gpa & KVM_STEAL_PHYS_VALID)) { 845 vcpu->arch.st.guest_addr = gpa; 846 return 0; 847 } 848 849 /* Check the address is in a valid memslot */ 850 idx = srcu_read_lock(&kvm->srcu); 851 if (kvm_is_error_hva(gfn_to_hva(kvm, gpa >> PAGE_SHIFT))) 852 ret = -EINVAL; 853 srcu_read_unlock(&kvm->srcu, idx); 854 855 if (!ret) { 856 vcpu->arch.st.guest_addr = gpa; 857 vcpu->arch.st.last_steal = current->sched_info.run_delay; 858 kvm_make_request(KVM_REQ_STEAL_UPDATE, vcpu); 859 } 860 861 return ret; 862 } 863 864 static int kvm_loongarch_vcpu_set_attr(struct kvm_vcpu *vcpu, 865 struct kvm_device_attr *attr) 866 { 867 int ret = -ENXIO; 868 869 switch (attr->group) { 870 case KVM_LOONGARCH_VCPU_CPUCFG: 871 ret = kvm_loongarch_cpucfg_set_attr(vcpu, attr); 872 break; 873 case KVM_LOONGARCH_VCPU_PVTIME_CTRL: 874 ret = kvm_loongarch_pvtime_set_attr(vcpu, attr); 875 break; 876 default: 877 break; 878 } 879 880 return ret; 881 } 882 883 long kvm_arch_vcpu_ioctl(struct file *filp, 884 unsigned int ioctl, unsigned long arg) 885 { 886 long r; 887 struct kvm_device_attr attr; 888 void __user *argp = (void __user *)arg; 889 struct kvm_vcpu *vcpu = filp->private_data; 890 891 /* 892 * Only software CSR should be modified 893 * 894 * If any hardware CSR register is modified, vcpu_load/vcpu_put pair 895 * should be used. Since CSR registers owns by this vcpu, if switch 896 * to other vcpus, other vcpus need reload CSR registers. 897 * 898 * If software CSR is modified, bit KVM_LARCH_HWCSR_USABLE should 899 * be clear in vcpu->arch.aux_inuse, and vcpu_load will check 900 * aux_inuse flag and reload CSR registers form software. 901 */ 902 903 switch (ioctl) { 904 case KVM_SET_ONE_REG: 905 case KVM_GET_ONE_REG: { 906 struct kvm_one_reg reg; 907 908 r = -EFAULT; 909 if (copy_from_user(®, argp, sizeof(reg))) 910 break; 911 if (ioctl == KVM_SET_ONE_REG) { 912 r = kvm_set_reg(vcpu, ®); 913 vcpu->arch.aux_inuse &= ~KVM_LARCH_HWCSR_USABLE; 914 } else 915 r = kvm_get_reg(vcpu, ®); 916 break; 917 } 918 case KVM_ENABLE_CAP: { 919 struct kvm_enable_cap cap; 920 921 r = -EFAULT; 922 if (copy_from_user(&cap, argp, sizeof(cap))) 923 break; 924 r = kvm_vcpu_ioctl_enable_cap(vcpu, &cap); 925 break; 926 } 927 case KVM_HAS_DEVICE_ATTR: { 928 r = -EFAULT; 929 if (copy_from_user(&attr, argp, sizeof(attr))) 930 break; 931 r = kvm_loongarch_vcpu_has_attr(vcpu, &attr); 932 break; 933 } 934 case KVM_GET_DEVICE_ATTR: { 935 r = -EFAULT; 936 if (copy_from_user(&attr, argp, sizeof(attr))) 937 break; 938 r = kvm_loongarch_vcpu_get_attr(vcpu, &attr); 939 break; 940 } 941 case KVM_SET_DEVICE_ATTR: { 942 r = -EFAULT; 943 if (copy_from_user(&attr, argp, sizeof(attr))) 944 break; 945 r = kvm_loongarch_vcpu_set_attr(vcpu, &attr); 946 break; 947 } 948 default: 949 r = -ENOIOCTLCMD; 950 break; 951 } 952 953 return r; 954 } 955 956 int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu) 957 { 958 int i = 0; 959 960 fpu->fcc = vcpu->arch.fpu.fcc; 961 fpu->fcsr = vcpu->arch.fpu.fcsr; 962 for (i = 0; i < NUM_FPU_REGS; i++) 963 memcpy(&fpu->fpr[i], &vcpu->arch.fpu.fpr[i], FPU_REG_WIDTH / 64); 964 965 return 0; 966 } 967 968 int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu) 969 { 970 int i = 0; 971 972 vcpu->arch.fpu.fcc = fpu->fcc; 973 vcpu->arch.fpu.fcsr = fpu->fcsr; 974 for (i = 0; i < NUM_FPU_REGS; i++) 975 memcpy(&vcpu->arch.fpu.fpr[i], &fpu->fpr[i], FPU_REG_WIDTH / 64); 976 977 return 0; 978 } 979 980 /* Enable FPU and restore context */ 981 void kvm_own_fpu(struct kvm_vcpu *vcpu) 982 { 983 preempt_disable(); 984 985 /* Enable FPU */ 986 set_csr_euen(CSR_EUEN_FPEN); 987 988 kvm_restore_fpu(&vcpu->arch.fpu); 989 vcpu->arch.aux_inuse |= KVM_LARCH_FPU; 990 trace_kvm_aux(vcpu, KVM_TRACE_AUX_RESTORE, KVM_TRACE_AUX_FPU); 991 992 preempt_enable(); 993 } 994 995 #ifdef CONFIG_CPU_HAS_LSX 996 /* Enable LSX and restore context */ 997 int kvm_own_lsx(struct kvm_vcpu *vcpu) 998 { 999 if (!kvm_guest_has_fpu(&vcpu->arch) || !kvm_guest_has_lsx(&vcpu->arch)) 1000 return -EINVAL; 1001 1002 preempt_disable(); 1003 1004 /* Enable LSX for guest */ 1005 set_csr_euen(CSR_EUEN_LSXEN | CSR_EUEN_FPEN); 1006 switch (vcpu->arch.aux_inuse & KVM_LARCH_FPU) { 1007 case KVM_LARCH_FPU: 1008 /* 1009 * Guest FPU state already loaded, 1010 * only restore upper LSX state 1011 */ 1012 _restore_lsx_upper(&vcpu->arch.fpu); 1013 break; 1014 default: 1015 /* Neither FP or LSX already active, 1016 * restore full LSX state 1017 */ 1018 kvm_restore_lsx(&vcpu->arch.fpu); 1019 break; 1020 } 1021 1022 trace_kvm_aux(vcpu, KVM_TRACE_AUX_RESTORE, KVM_TRACE_AUX_LSX); 1023 vcpu->arch.aux_inuse |= KVM_LARCH_LSX | KVM_LARCH_FPU; 1024 preempt_enable(); 1025 1026 return 0; 1027 } 1028 #endif 1029 1030 #ifdef CONFIG_CPU_HAS_LASX 1031 /* Enable LASX and restore context */ 1032 int kvm_own_lasx(struct kvm_vcpu *vcpu) 1033 { 1034 if (!kvm_guest_has_fpu(&vcpu->arch) || !kvm_guest_has_lsx(&vcpu->arch) || !kvm_guest_has_lasx(&vcpu->arch)) 1035 return -EINVAL; 1036 1037 preempt_disable(); 1038 1039 set_csr_euen(CSR_EUEN_FPEN | CSR_EUEN_LSXEN | CSR_EUEN_LASXEN); 1040 switch (vcpu->arch.aux_inuse & (KVM_LARCH_FPU | KVM_LARCH_LSX)) { 1041 case KVM_LARCH_LSX: 1042 case KVM_LARCH_LSX | KVM_LARCH_FPU: 1043 /* Guest LSX state already loaded, only restore upper LASX state */ 1044 _restore_lasx_upper(&vcpu->arch.fpu); 1045 break; 1046 case KVM_LARCH_FPU: 1047 /* Guest FP state already loaded, only restore upper LSX & LASX state */ 1048 _restore_lsx_upper(&vcpu->arch.fpu); 1049 _restore_lasx_upper(&vcpu->arch.fpu); 1050 break; 1051 default: 1052 /* Neither FP or LSX already active, restore full LASX state */ 1053 kvm_restore_lasx(&vcpu->arch.fpu); 1054 break; 1055 } 1056 1057 trace_kvm_aux(vcpu, KVM_TRACE_AUX_RESTORE, KVM_TRACE_AUX_LASX); 1058 vcpu->arch.aux_inuse |= KVM_LARCH_LASX | KVM_LARCH_LSX | KVM_LARCH_FPU; 1059 preempt_enable(); 1060 1061 return 0; 1062 } 1063 #endif 1064 1065 /* Save context and disable FPU */ 1066 void kvm_lose_fpu(struct kvm_vcpu *vcpu) 1067 { 1068 preempt_disable(); 1069 1070 if (vcpu->arch.aux_inuse & KVM_LARCH_LASX) { 1071 kvm_save_lasx(&vcpu->arch.fpu); 1072 vcpu->arch.aux_inuse &= ~(KVM_LARCH_LSX | KVM_LARCH_FPU | KVM_LARCH_LASX); 1073 trace_kvm_aux(vcpu, KVM_TRACE_AUX_SAVE, KVM_TRACE_AUX_LASX); 1074 1075 /* Disable LASX & LSX & FPU */ 1076 clear_csr_euen(CSR_EUEN_FPEN | CSR_EUEN_LSXEN | CSR_EUEN_LASXEN); 1077 } else if (vcpu->arch.aux_inuse & KVM_LARCH_LSX) { 1078 kvm_save_lsx(&vcpu->arch.fpu); 1079 vcpu->arch.aux_inuse &= ~(KVM_LARCH_LSX | KVM_LARCH_FPU); 1080 trace_kvm_aux(vcpu, KVM_TRACE_AUX_SAVE, KVM_TRACE_AUX_LSX); 1081 1082 /* Disable LSX & FPU */ 1083 clear_csr_euen(CSR_EUEN_FPEN | CSR_EUEN_LSXEN); 1084 } else if (vcpu->arch.aux_inuse & KVM_LARCH_FPU) { 1085 kvm_save_fpu(&vcpu->arch.fpu); 1086 vcpu->arch.aux_inuse &= ~KVM_LARCH_FPU; 1087 trace_kvm_aux(vcpu, KVM_TRACE_AUX_SAVE, KVM_TRACE_AUX_FPU); 1088 1089 /* Disable FPU */ 1090 clear_csr_euen(CSR_EUEN_FPEN); 1091 } 1092 1093 preempt_enable(); 1094 } 1095 1096 int kvm_vcpu_ioctl_interrupt(struct kvm_vcpu *vcpu, struct kvm_interrupt *irq) 1097 { 1098 int intr = (int)irq->irq; 1099 1100 if (intr > 0) 1101 kvm_queue_irq(vcpu, intr); 1102 else if (intr < 0) 1103 kvm_dequeue_irq(vcpu, -intr); 1104 else { 1105 kvm_err("%s: invalid interrupt ioctl %d\n", __func__, irq->irq); 1106 return -EINVAL; 1107 } 1108 1109 kvm_vcpu_kick(vcpu); 1110 1111 return 0; 1112 } 1113 1114 long kvm_arch_vcpu_async_ioctl(struct file *filp, 1115 unsigned int ioctl, unsigned long arg) 1116 { 1117 void __user *argp = (void __user *)arg; 1118 struct kvm_vcpu *vcpu = filp->private_data; 1119 1120 if (ioctl == KVM_INTERRUPT) { 1121 struct kvm_interrupt irq; 1122 1123 if (copy_from_user(&irq, argp, sizeof(irq))) 1124 return -EFAULT; 1125 1126 kvm_debug("[%d] %s: irq: %d\n", vcpu->vcpu_id, __func__, irq.irq); 1127 1128 return kvm_vcpu_ioctl_interrupt(vcpu, &irq); 1129 } 1130 1131 return -ENOIOCTLCMD; 1132 } 1133 1134 int kvm_arch_vcpu_precreate(struct kvm *kvm, unsigned int id) 1135 { 1136 return 0; 1137 } 1138 1139 int kvm_arch_vcpu_create(struct kvm_vcpu *vcpu) 1140 { 1141 unsigned long timer_hz; 1142 struct loongarch_csrs *csr; 1143 1144 vcpu->arch.vpid = 0; 1145 vcpu->arch.flush_gpa = INVALID_GPA; 1146 1147 hrtimer_init(&vcpu->arch.swtimer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS_PINNED); 1148 vcpu->arch.swtimer.function = kvm_swtimer_wakeup; 1149 1150 vcpu->arch.handle_exit = kvm_handle_exit; 1151 vcpu->arch.guest_eentry = (unsigned long)kvm_loongarch_ops->exc_entry; 1152 vcpu->arch.csr = kzalloc(sizeof(struct loongarch_csrs), GFP_KERNEL); 1153 if (!vcpu->arch.csr) 1154 return -ENOMEM; 1155 1156 /* 1157 * All kvm exceptions share one exception entry, and host <-> guest 1158 * switch also switch ECFG.VS field, keep host ECFG.VS info here. 1159 */ 1160 vcpu->arch.host_ecfg = (read_csr_ecfg() & CSR_ECFG_VS); 1161 1162 /* Init */ 1163 vcpu->arch.last_sched_cpu = -1; 1164 1165 /* 1166 * Initialize guest register state to valid architectural reset state. 1167 */ 1168 timer_hz = calc_const_freq(); 1169 kvm_init_timer(vcpu, timer_hz); 1170 1171 /* Set Initialize mode for guest */ 1172 csr = vcpu->arch.csr; 1173 kvm_write_sw_gcsr(csr, LOONGARCH_CSR_CRMD, CSR_CRMD_DA); 1174 1175 /* Set cpuid */ 1176 kvm_write_sw_gcsr(csr, LOONGARCH_CSR_TMID, vcpu->vcpu_id); 1177 kvm_write_sw_gcsr(csr, LOONGARCH_CSR_CPUID, KVM_MAX_PHYID); 1178 1179 /* Start with no pending virtual guest interrupts */ 1180 csr->csrs[LOONGARCH_CSR_GINTC] = 0; 1181 1182 return 0; 1183 } 1184 1185 void kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu) 1186 { 1187 } 1188 1189 void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu) 1190 { 1191 int cpu; 1192 struct kvm_context *context; 1193 1194 hrtimer_cancel(&vcpu->arch.swtimer); 1195 kvm_mmu_free_memory_cache(&vcpu->arch.mmu_page_cache); 1196 kvm_drop_cpuid(vcpu); 1197 kfree(vcpu->arch.csr); 1198 1199 /* 1200 * If the vCPU is freed and reused as another vCPU, we don't want the 1201 * matching pointer wrongly hanging around in last_vcpu. 1202 */ 1203 for_each_possible_cpu(cpu) { 1204 context = per_cpu_ptr(vcpu->kvm->arch.vmcs, cpu); 1205 if (context->last_vcpu == vcpu) 1206 context->last_vcpu = NULL; 1207 } 1208 } 1209 1210 static int _kvm_vcpu_load(struct kvm_vcpu *vcpu, int cpu) 1211 { 1212 bool migrated; 1213 struct kvm_context *context; 1214 struct loongarch_csrs *csr = vcpu->arch.csr; 1215 1216 /* 1217 * Have we migrated to a different CPU? 1218 * If so, any old guest TLB state may be stale. 1219 */ 1220 migrated = (vcpu->arch.last_sched_cpu != cpu); 1221 1222 /* 1223 * Was this the last vCPU to run on this CPU? 1224 * If not, any old guest state from this vCPU will have been clobbered. 1225 */ 1226 context = per_cpu_ptr(vcpu->kvm->arch.vmcs, cpu); 1227 if (migrated || (context->last_vcpu != vcpu)) 1228 vcpu->arch.aux_inuse &= ~KVM_LARCH_HWCSR_USABLE; 1229 context->last_vcpu = vcpu; 1230 1231 /* Restore timer state regardless */ 1232 kvm_restore_timer(vcpu); 1233 1234 /* Control guest page CCA attribute */ 1235 change_csr_gcfg(CSR_GCFG_MATC_MASK, CSR_GCFG_MATC_ROOT); 1236 kvm_make_request(KVM_REQ_STEAL_UPDATE, vcpu); 1237 1238 /* Don't bother restoring registers multiple times unless necessary */ 1239 if (vcpu->arch.aux_inuse & KVM_LARCH_HWCSR_USABLE) 1240 return 0; 1241 1242 write_csr_gcntc((ulong)vcpu->kvm->arch.time_offset); 1243 1244 /* Restore guest CSR registers */ 1245 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_CRMD); 1246 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_PRMD); 1247 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_EUEN); 1248 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_MISC); 1249 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_ECFG); 1250 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_ERA); 1251 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_BADV); 1252 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_BADI); 1253 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_EENTRY); 1254 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_TLBIDX); 1255 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_TLBEHI); 1256 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_TLBELO0); 1257 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_TLBELO1); 1258 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_ASID); 1259 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_PGDL); 1260 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_PGDH); 1261 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_PWCTL0); 1262 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_PWCTL1); 1263 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_STLBPGSIZE); 1264 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_RVACFG); 1265 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_CPUID); 1266 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_KS0); 1267 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_KS1); 1268 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_KS2); 1269 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_KS3); 1270 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_KS4); 1271 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_KS5); 1272 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_KS6); 1273 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_KS7); 1274 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_TMID); 1275 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_CNTC); 1276 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_TLBRENTRY); 1277 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_TLBRBADV); 1278 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_TLBRERA); 1279 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_TLBRSAVE); 1280 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_TLBRELO0); 1281 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_TLBRELO1); 1282 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_TLBREHI); 1283 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_TLBRPRMD); 1284 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_DMWIN0); 1285 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_DMWIN1); 1286 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_DMWIN2); 1287 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_DMWIN3); 1288 kvm_restore_hw_gcsr(csr, LOONGARCH_CSR_LLBCTL); 1289 1290 /* Restore Root.GINTC from unused Guest.GINTC register */ 1291 write_csr_gintc(csr->csrs[LOONGARCH_CSR_GINTC]); 1292 1293 /* 1294 * We should clear linked load bit to break interrupted atomics. This 1295 * prevents a SC on the next vCPU from succeeding by matching a LL on 1296 * the previous vCPU. 1297 */ 1298 if (vcpu->kvm->created_vcpus > 1) 1299 set_gcsr_llbctl(CSR_LLBCTL_WCLLB); 1300 1301 vcpu->arch.aux_inuse |= KVM_LARCH_HWCSR_USABLE; 1302 1303 return 0; 1304 } 1305 1306 void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu) 1307 { 1308 unsigned long flags; 1309 1310 local_irq_save(flags); 1311 /* Restore guest state to registers */ 1312 _kvm_vcpu_load(vcpu, cpu); 1313 local_irq_restore(flags); 1314 } 1315 1316 static int _kvm_vcpu_put(struct kvm_vcpu *vcpu, int cpu) 1317 { 1318 struct loongarch_csrs *csr = vcpu->arch.csr; 1319 1320 kvm_lose_fpu(vcpu); 1321 1322 /* 1323 * Update CSR state from hardware if software CSR state is stale, 1324 * most CSR registers are kept unchanged during process context 1325 * switch except CSR registers like remaining timer tick value and 1326 * injected interrupt state. 1327 */ 1328 if (vcpu->arch.aux_inuse & KVM_LARCH_SWCSR_LATEST) 1329 goto out; 1330 1331 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_CRMD); 1332 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_PRMD); 1333 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_EUEN); 1334 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_MISC); 1335 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_ECFG); 1336 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_ERA); 1337 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_BADV); 1338 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_BADI); 1339 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_EENTRY); 1340 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_TLBIDX); 1341 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_TLBEHI); 1342 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_TLBELO0); 1343 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_TLBELO1); 1344 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_ASID); 1345 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_PGDL); 1346 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_PGDH); 1347 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_PWCTL0); 1348 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_PWCTL1); 1349 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_STLBPGSIZE); 1350 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_RVACFG); 1351 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_CPUID); 1352 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_PRCFG1); 1353 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_PRCFG2); 1354 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_PRCFG3); 1355 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_KS0); 1356 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_KS1); 1357 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_KS2); 1358 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_KS3); 1359 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_KS4); 1360 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_KS5); 1361 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_KS6); 1362 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_KS7); 1363 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_TMID); 1364 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_CNTC); 1365 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_LLBCTL); 1366 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_TLBRENTRY); 1367 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_TLBRBADV); 1368 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_TLBRERA); 1369 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_TLBRSAVE); 1370 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_TLBRELO0); 1371 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_TLBRELO1); 1372 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_TLBREHI); 1373 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_TLBRPRMD); 1374 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_DMWIN0); 1375 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_DMWIN1); 1376 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_DMWIN2); 1377 kvm_save_hw_gcsr(csr, LOONGARCH_CSR_DMWIN3); 1378 1379 vcpu->arch.aux_inuse |= KVM_LARCH_SWCSR_LATEST; 1380 1381 out: 1382 kvm_save_timer(vcpu); 1383 /* Save Root.GINTC into unused Guest.GINTC register */ 1384 csr->csrs[LOONGARCH_CSR_GINTC] = read_csr_gintc(); 1385 1386 return 0; 1387 } 1388 1389 void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu) 1390 { 1391 int cpu; 1392 unsigned long flags; 1393 1394 local_irq_save(flags); 1395 cpu = smp_processor_id(); 1396 vcpu->arch.last_sched_cpu = cpu; 1397 1398 /* Save guest state in registers */ 1399 _kvm_vcpu_put(vcpu, cpu); 1400 local_irq_restore(flags); 1401 } 1402 1403 int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu) 1404 { 1405 int r = -EINTR; 1406 struct kvm_run *run = vcpu->run; 1407 1408 if (vcpu->mmio_needed) { 1409 if (!vcpu->mmio_is_write) 1410 kvm_complete_mmio_read(vcpu, run); 1411 vcpu->mmio_needed = 0; 1412 } 1413 1414 if (run->exit_reason == KVM_EXIT_LOONGARCH_IOCSR) { 1415 if (!run->iocsr_io.is_write) 1416 kvm_complete_iocsr_read(vcpu, run); 1417 } 1418 1419 if (!vcpu->wants_to_run) 1420 return r; 1421 1422 /* Clear exit_reason */ 1423 run->exit_reason = KVM_EXIT_UNKNOWN; 1424 lose_fpu(1); 1425 vcpu_load(vcpu); 1426 kvm_sigset_activate(vcpu); 1427 r = kvm_pre_enter_guest(vcpu); 1428 if (r != RESUME_GUEST) 1429 goto out; 1430 1431 guest_timing_enter_irqoff(); 1432 guest_state_enter_irqoff(); 1433 trace_kvm_enter(vcpu); 1434 r = kvm_loongarch_ops->enter_guest(run, vcpu); 1435 1436 trace_kvm_out(vcpu); 1437 /* 1438 * Guest exit is already recorded at kvm_handle_exit() 1439 * return value must not be RESUME_GUEST 1440 */ 1441 local_irq_enable(); 1442 out: 1443 kvm_sigset_deactivate(vcpu); 1444 vcpu_put(vcpu); 1445 1446 return r; 1447 } 1448