1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (C) 2020-2023 Loongson Technology Corporation Limited 4 */ 5 6 #include <linux/err.h> 7 #include <linux/errno.h> 8 #include <linux/kvm_host.h> 9 #include <linux/module.h> 10 #include <linux/preempt.h> 11 #include <linux/vmalloc.h> 12 #include <trace/events/kvm.h> 13 #include <asm/fpu.h> 14 #include <asm/inst.h> 15 #include <asm/loongarch.h> 16 #include <asm/mmzone.h> 17 #include <asm/numa.h> 18 #include <asm/time.h> 19 #include <asm/tlb.h> 20 #include <asm/kvm_csr.h> 21 #include <asm/kvm_vcpu.h> 22 #include "trace.h" 23 24 static int kvm_emu_cpucfg(struct kvm_vcpu *vcpu, larch_inst inst) 25 { 26 int rd, rj; 27 unsigned int index, ret; 28 29 if (inst.reg2_format.opcode != cpucfg_op) 30 return EMULATE_FAIL; 31 32 rd = inst.reg2_format.rd; 33 rj = inst.reg2_format.rj; 34 ++vcpu->stat.cpucfg_exits; 35 index = vcpu->arch.gprs[rj]; 36 37 /* 38 * By LoongArch Reference Manual 2.2.10.5 39 * Return value is 0 for undefined CPUCFG index 40 * 41 * Disable preemption since hw gcsr is accessed 42 */ 43 preempt_disable(); 44 switch (index) { 45 case 0 ... (KVM_MAX_CPUCFG_REGS - 1): 46 vcpu->arch.gprs[rd] = vcpu->arch.cpucfg[index]; 47 break; 48 case CPUCFG_KVM_SIG: 49 /* CPUCFG emulation between 0x40000000 -- 0x400000ff */ 50 vcpu->arch.gprs[rd] = *(unsigned int *)KVM_SIGNATURE; 51 break; 52 case CPUCFG_KVM_FEATURE: 53 ret = KVM_FEATURE_IPI; 54 if (kvm_pvtime_supported()) 55 ret |= KVM_FEATURE_STEAL_TIME; 56 vcpu->arch.gprs[rd] = ret; 57 break; 58 default: 59 vcpu->arch.gprs[rd] = 0; 60 break; 61 } 62 preempt_enable(); 63 64 return EMULATE_DONE; 65 } 66 67 static unsigned long kvm_emu_read_csr(struct kvm_vcpu *vcpu, int csrid) 68 { 69 unsigned long val = 0; 70 struct loongarch_csrs *csr = vcpu->arch.csr; 71 72 /* 73 * From LoongArch Reference Manual Volume 1 Chapter 4.2.1 74 * For undefined CSR id, return value is 0 75 */ 76 if (get_gcsr_flag(csrid) & SW_GCSR) 77 val = kvm_read_sw_gcsr(csr, csrid); 78 else 79 pr_warn_once("Unsupported csrrd 0x%x with pc %lx\n", csrid, vcpu->arch.pc); 80 81 return val; 82 } 83 84 static unsigned long kvm_emu_write_csr(struct kvm_vcpu *vcpu, int csrid, unsigned long val) 85 { 86 unsigned long old = 0; 87 struct loongarch_csrs *csr = vcpu->arch.csr; 88 89 if (get_gcsr_flag(csrid) & SW_GCSR) { 90 old = kvm_read_sw_gcsr(csr, csrid); 91 kvm_write_sw_gcsr(csr, csrid, val); 92 } else 93 pr_warn_once("Unsupported csrwr 0x%x with pc %lx\n", csrid, vcpu->arch.pc); 94 95 return old; 96 } 97 98 static unsigned long kvm_emu_xchg_csr(struct kvm_vcpu *vcpu, int csrid, 99 unsigned long csr_mask, unsigned long val) 100 { 101 unsigned long old = 0; 102 struct loongarch_csrs *csr = vcpu->arch.csr; 103 104 if (get_gcsr_flag(csrid) & SW_GCSR) { 105 old = kvm_read_sw_gcsr(csr, csrid); 106 val = (old & ~csr_mask) | (val & csr_mask); 107 kvm_write_sw_gcsr(csr, csrid, val); 108 old = old & csr_mask; 109 } else 110 pr_warn_once("Unsupported csrxchg 0x%x with pc %lx\n", csrid, vcpu->arch.pc); 111 112 return old; 113 } 114 115 static int kvm_handle_csr(struct kvm_vcpu *vcpu, larch_inst inst) 116 { 117 unsigned int rd, rj, csrid; 118 unsigned long csr_mask, val = 0; 119 120 /* 121 * CSR value mask imm 122 * rj = 0 means csrrd 123 * rj = 1 means csrwr 124 * rj != 0,1 means csrxchg 125 */ 126 rd = inst.reg2csr_format.rd; 127 rj = inst.reg2csr_format.rj; 128 csrid = inst.reg2csr_format.csr; 129 130 /* Process CSR ops */ 131 switch (rj) { 132 case 0: /* process csrrd */ 133 val = kvm_emu_read_csr(vcpu, csrid); 134 vcpu->arch.gprs[rd] = val; 135 break; 136 case 1: /* process csrwr */ 137 val = vcpu->arch.gprs[rd]; 138 val = kvm_emu_write_csr(vcpu, csrid, val); 139 vcpu->arch.gprs[rd] = val; 140 break; 141 default: /* process csrxchg */ 142 val = vcpu->arch.gprs[rd]; 143 csr_mask = vcpu->arch.gprs[rj]; 144 val = kvm_emu_xchg_csr(vcpu, csrid, csr_mask, val); 145 vcpu->arch.gprs[rd] = val; 146 } 147 148 return EMULATE_DONE; 149 } 150 151 int kvm_emu_iocsr(larch_inst inst, struct kvm_run *run, struct kvm_vcpu *vcpu) 152 { 153 int ret; 154 unsigned long val; 155 u32 addr, rd, rj, opcode; 156 157 /* 158 * Each IOCSR with different opcode 159 */ 160 rd = inst.reg2_format.rd; 161 rj = inst.reg2_format.rj; 162 opcode = inst.reg2_format.opcode; 163 addr = vcpu->arch.gprs[rj]; 164 ret = EMULATE_DO_IOCSR; 165 run->iocsr_io.phys_addr = addr; 166 run->iocsr_io.is_write = 0; 167 168 /* LoongArch is Little endian */ 169 switch (opcode) { 170 case iocsrrdb_op: 171 run->iocsr_io.len = 1; 172 break; 173 case iocsrrdh_op: 174 run->iocsr_io.len = 2; 175 break; 176 case iocsrrdw_op: 177 run->iocsr_io.len = 4; 178 break; 179 case iocsrrdd_op: 180 run->iocsr_io.len = 8; 181 break; 182 case iocsrwrb_op: 183 run->iocsr_io.len = 1; 184 run->iocsr_io.is_write = 1; 185 break; 186 case iocsrwrh_op: 187 run->iocsr_io.len = 2; 188 run->iocsr_io.is_write = 1; 189 break; 190 case iocsrwrw_op: 191 run->iocsr_io.len = 4; 192 run->iocsr_io.is_write = 1; 193 break; 194 case iocsrwrd_op: 195 run->iocsr_io.len = 8; 196 run->iocsr_io.is_write = 1; 197 break; 198 default: 199 ret = EMULATE_FAIL; 200 break; 201 } 202 203 if (ret == EMULATE_DO_IOCSR) { 204 if (run->iocsr_io.is_write) { 205 val = vcpu->arch.gprs[rd]; 206 memcpy(run->iocsr_io.data, &val, run->iocsr_io.len); 207 } 208 vcpu->arch.io_gpr = rd; 209 } 210 211 return ret; 212 } 213 214 int kvm_complete_iocsr_read(struct kvm_vcpu *vcpu, struct kvm_run *run) 215 { 216 enum emulation_result er = EMULATE_DONE; 217 unsigned long *gpr = &vcpu->arch.gprs[vcpu->arch.io_gpr]; 218 219 switch (run->iocsr_io.len) { 220 case 1: 221 *gpr = *(s8 *)run->iocsr_io.data; 222 break; 223 case 2: 224 *gpr = *(s16 *)run->iocsr_io.data; 225 break; 226 case 4: 227 *gpr = *(s32 *)run->iocsr_io.data; 228 break; 229 case 8: 230 *gpr = *(s64 *)run->iocsr_io.data; 231 break; 232 default: 233 kvm_err("Bad IOCSR length: %d, addr is 0x%lx\n", 234 run->iocsr_io.len, vcpu->arch.badv); 235 er = EMULATE_FAIL; 236 break; 237 } 238 239 return er; 240 } 241 242 int kvm_emu_idle(struct kvm_vcpu *vcpu) 243 { 244 ++vcpu->stat.idle_exits; 245 trace_kvm_exit_idle(vcpu, KVM_TRACE_EXIT_IDLE); 246 247 if (!kvm_arch_vcpu_runnable(vcpu)) 248 kvm_vcpu_halt(vcpu); 249 250 return EMULATE_DONE; 251 } 252 253 static int kvm_trap_handle_gspr(struct kvm_vcpu *vcpu) 254 { 255 unsigned long curr_pc; 256 larch_inst inst; 257 enum emulation_result er = EMULATE_DONE; 258 struct kvm_run *run = vcpu->run; 259 260 /* Fetch the instruction */ 261 inst.word = vcpu->arch.badi; 262 curr_pc = vcpu->arch.pc; 263 update_pc(&vcpu->arch); 264 265 trace_kvm_exit_gspr(vcpu, inst.word); 266 er = EMULATE_FAIL; 267 switch (((inst.word >> 24) & 0xff)) { 268 case 0x0: /* CPUCFG GSPR */ 269 er = kvm_emu_cpucfg(vcpu, inst); 270 break; 271 case 0x4: /* CSR{RD,WR,XCHG} GSPR */ 272 er = kvm_handle_csr(vcpu, inst); 273 break; 274 case 0x6: /* Cache, Idle and IOCSR GSPR */ 275 switch (((inst.word >> 22) & 0x3ff)) { 276 case 0x18: /* Cache GSPR */ 277 er = EMULATE_DONE; 278 trace_kvm_exit_cache(vcpu, KVM_TRACE_EXIT_CACHE); 279 break; 280 case 0x19: /* Idle/IOCSR GSPR */ 281 switch (((inst.word >> 15) & 0x1ffff)) { 282 case 0xc90: /* IOCSR GSPR */ 283 er = kvm_emu_iocsr(inst, run, vcpu); 284 break; 285 case 0xc91: /* Idle GSPR */ 286 er = kvm_emu_idle(vcpu); 287 break; 288 default: 289 er = EMULATE_FAIL; 290 break; 291 } 292 break; 293 default: 294 er = EMULATE_FAIL; 295 break; 296 } 297 break; 298 default: 299 er = EMULATE_FAIL; 300 break; 301 } 302 303 /* Rollback PC only if emulation was unsuccessful */ 304 if (er == EMULATE_FAIL) { 305 kvm_err("[%#lx]%s: unsupported gspr instruction 0x%08x\n", 306 curr_pc, __func__, inst.word); 307 308 kvm_arch_vcpu_dump_regs(vcpu); 309 vcpu->arch.pc = curr_pc; 310 } 311 312 return er; 313 } 314 315 /* 316 * Trigger GSPR: 317 * 1) Execute CPUCFG instruction; 318 * 2) Execute CACOP/IDLE instructions; 319 * 3) Access to unimplemented CSRs/IOCSRs. 320 */ 321 static int kvm_handle_gspr(struct kvm_vcpu *vcpu) 322 { 323 int ret = RESUME_GUEST; 324 enum emulation_result er = EMULATE_DONE; 325 326 er = kvm_trap_handle_gspr(vcpu); 327 328 if (er == EMULATE_DONE) { 329 ret = RESUME_GUEST; 330 } else if (er == EMULATE_DO_MMIO) { 331 vcpu->run->exit_reason = KVM_EXIT_MMIO; 332 ret = RESUME_HOST; 333 } else if (er == EMULATE_DO_IOCSR) { 334 vcpu->run->exit_reason = KVM_EXIT_LOONGARCH_IOCSR; 335 ret = RESUME_HOST; 336 } else { 337 kvm_queue_exception(vcpu, EXCCODE_INE, 0); 338 ret = RESUME_GUEST; 339 } 340 341 return ret; 342 } 343 344 int kvm_emu_mmio_read(struct kvm_vcpu *vcpu, larch_inst inst) 345 { 346 int ret; 347 unsigned int op8, opcode, rd; 348 struct kvm_run *run = vcpu->run; 349 350 run->mmio.phys_addr = vcpu->arch.badv; 351 vcpu->mmio_needed = 2; /* signed */ 352 op8 = (inst.word >> 24) & 0xff; 353 ret = EMULATE_DO_MMIO; 354 355 switch (op8) { 356 case 0x24 ... 0x27: /* ldptr.w/d process */ 357 rd = inst.reg2i14_format.rd; 358 opcode = inst.reg2i14_format.opcode; 359 360 switch (opcode) { 361 case ldptrw_op: 362 run->mmio.len = 4; 363 break; 364 case ldptrd_op: 365 run->mmio.len = 8; 366 break; 367 default: 368 break; 369 } 370 break; 371 case 0x28 ... 0x2e: /* ld.b/h/w/d, ld.bu/hu/wu process */ 372 rd = inst.reg2i12_format.rd; 373 opcode = inst.reg2i12_format.opcode; 374 375 switch (opcode) { 376 case ldb_op: 377 run->mmio.len = 1; 378 break; 379 case ldbu_op: 380 vcpu->mmio_needed = 1; /* unsigned */ 381 run->mmio.len = 1; 382 break; 383 case ldh_op: 384 run->mmio.len = 2; 385 break; 386 case ldhu_op: 387 vcpu->mmio_needed = 1; /* unsigned */ 388 run->mmio.len = 2; 389 break; 390 case ldw_op: 391 run->mmio.len = 4; 392 break; 393 case ldwu_op: 394 vcpu->mmio_needed = 1; /* unsigned */ 395 run->mmio.len = 4; 396 break; 397 case ldd_op: 398 run->mmio.len = 8; 399 break; 400 default: 401 ret = EMULATE_FAIL; 402 break; 403 } 404 break; 405 case 0x38: /* ldx.b/h/w/d, ldx.bu/hu/wu process */ 406 rd = inst.reg3_format.rd; 407 opcode = inst.reg3_format.opcode; 408 409 switch (opcode) { 410 case ldxb_op: 411 run->mmio.len = 1; 412 break; 413 case ldxbu_op: 414 run->mmio.len = 1; 415 vcpu->mmio_needed = 1; /* unsigned */ 416 break; 417 case ldxh_op: 418 run->mmio.len = 2; 419 break; 420 case ldxhu_op: 421 run->mmio.len = 2; 422 vcpu->mmio_needed = 1; /* unsigned */ 423 break; 424 case ldxw_op: 425 run->mmio.len = 4; 426 break; 427 case ldxwu_op: 428 run->mmio.len = 4; 429 vcpu->mmio_needed = 1; /* unsigned */ 430 break; 431 case ldxd_op: 432 run->mmio.len = 8; 433 break; 434 default: 435 ret = EMULATE_FAIL; 436 break; 437 } 438 break; 439 default: 440 ret = EMULATE_FAIL; 441 } 442 443 if (ret == EMULATE_DO_MMIO) { 444 /* Set for kvm_complete_mmio_read() use */ 445 vcpu->arch.io_gpr = rd; 446 run->mmio.is_write = 0; 447 vcpu->mmio_is_write = 0; 448 trace_kvm_mmio(KVM_TRACE_MMIO_READ_UNSATISFIED, run->mmio.len, 449 run->mmio.phys_addr, NULL); 450 } else { 451 kvm_err("Read not supported Inst=0x%08x @%lx BadVaddr:%#lx\n", 452 inst.word, vcpu->arch.pc, vcpu->arch.badv); 453 kvm_arch_vcpu_dump_regs(vcpu); 454 vcpu->mmio_needed = 0; 455 } 456 457 return ret; 458 } 459 460 int kvm_complete_mmio_read(struct kvm_vcpu *vcpu, struct kvm_run *run) 461 { 462 enum emulation_result er = EMULATE_DONE; 463 unsigned long *gpr = &vcpu->arch.gprs[vcpu->arch.io_gpr]; 464 465 /* Update with new PC */ 466 update_pc(&vcpu->arch); 467 switch (run->mmio.len) { 468 case 1: 469 if (vcpu->mmio_needed == 2) 470 *gpr = *(s8 *)run->mmio.data; 471 else 472 *gpr = *(u8 *)run->mmio.data; 473 break; 474 case 2: 475 if (vcpu->mmio_needed == 2) 476 *gpr = *(s16 *)run->mmio.data; 477 else 478 *gpr = *(u16 *)run->mmio.data; 479 break; 480 case 4: 481 if (vcpu->mmio_needed == 2) 482 *gpr = *(s32 *)run->mmio.data; 483 else 484 *gpr = *(u32 *)run->mmio.data; 485 break; 486 case 8: 487 *gpr = *(s64 *)run->mmio.data; 488 break; 489 default: 490 kvm_err("Bad MMIO length: %d, addr is 0x%lx\n", 491 run->mmio.len, vcpu->arch.badv); 492 er = EMULATE_FAIL; 493 break; 494 } 495 496 trace_kvm_mmio(KVM_TRACE_MMIO_READ, run->mmio.len, 497 run->mmio.phys_addr, run->mmio.data); 498 499 return er; 500 } 501 502 int kvm_emu_mmio_write(struct kvm_vcpu *vcpu, larch_inst inst) 503 { 504 int ret; 505 unsigned int rd, op8, opcode; 506 unsigned long curr_pc, rd_val = 0; 507 struct kvm_run *run = vcpu->run; 508 void *data = run->mmio.data; 509 510 /* 511 * Update PC and hold onto current PC in case there is 512 * an error and we want to rollback the PC 513 */ 514 curr_pc = vcpu->arch.pc; 515 update_pc(&vcpu->arch); 516 517 op8 = (inst.word >> 24) & 0xff; 518 run->mmio.phys_addr = vcpu->arch.badv; 519 ret = EMULATE_DO_MMIO; 520 switch (op8) { 521 case 0x24 ... 0x27: /* stptr.w/d process */ 522 rd = inst.reg2i14_format.rd; 523 opcode = inst.reg2i14_format.opcode; 524 525 switch (opcode) { 526 case stptrw_op: 527 run->mmio.len = 4; 528 *(unsigned int *)data = vcpu->arch.gprs[rd]; 529 break; 530 case stptrd_op: 531 run->mmio.len = 8; 532 *(unsigned long *)data = vcpu->arch.gprs[rd]; 533 break; 534 default: 535 ret = EMULATE_FAIL; 536 break; 537 } 538 break; 539 case 0x28 ... 0x2e: /* st.b/h/w/d process */ 540 rd = inst.reg2i12_format.rd; 541 opcode = inst.reg2i12_format.opcode; 542 rd_val = vcpu->arch.gprs[rd]; 543 544 switch (opcode) { 545 case stb_op: 546 run->mmio.len = 1; 547 *(unsigned char *)data = rd_val; 548 break; 549 case sth_op: 550 run->mmio.len = 2; 551 *(unsigned short *)data = rd_val; 552 break; 553 case stw_op: 554 run->mmio.len = 4; 555 *(unsigned int *)data = rd_val; 556 break; 557 case std_op: 558 run->mmio.len = 8; 559 *(unsigned long *)data = rd_val; 560 break; 561 default: 562 ret = EMULATE_FAIL; 563 break; 564 } 565 break; 566 case 0x38: /* stx.b/h/w/d process */ 567 rd = inst.reg3_format.rd; 568 opcode = inst.reg3_format.opcode; 569 570 switch (opcode) { 571 case stxb_op: 572 run->mmio.len = 1; 573 *(unsigned char *)data = vcpu->arch.gprs[rd]; 574 break; 575 case stxh_op: 576 run->mmio.len = 2; 577 *(unsigned short *)data = vcpu->arch.gprs[rd]; 578 break; 579 case stxw_op: 580 run->mmio.len = 4; 581 *(unsigned int *)data = vcpu->arch.gprs[rd]; 582 break; 583 case stxd_op: 584 run->mmio.len = 8; 585 *(unsigned long *)data = vcpu->arch.gprs[rd]; 586 break; 587 default: 588 ret = EMULATE_FAIL; 589 break; 590 } 591 break; 592 default: 593 ret = EMULATE_FAIL; 594 } 595 596 if (ret == EMULATE_DO_MMIO) { 597 run->mmio.is_write = 1; 598 vcpu->mmio_needed = 1; 599 vcpu->mmio_is_write = 1; 600 trace_kvm_mmio(KVM_TRACE_MMIO_WRITE, run->mmio.len, 601 run->mmio.phys_addr, data); 602 } else { 603 vcpu->arch.pc = curr_pc; 604 kvm_err("Write not supported Inst=0x%08x @%lx BadVaddr:%#lx\n", 605 inst.word, vcpu->arch.pc, vcpu->arch.badv); 606 kvm_arch_vcpu_dump_regs(vcpu); 607 /* Rollback PC if emulation was unsuccessful */ 608 } 609 610 return ret; 611 } 612 613 static int kvm_handle_rdwr_fault(struct kvm_vcpu *vcpu, bool write) 614 { 615 int ret; 616 larch_inst inst; 617 enum emulation_result er = EMULATE_DONE; 618 struct kvm_run *run = vcpu->run; 619 unsigned long badv = vcpu->arch.badv; 620 621 ret = kvm_handle_mm_fault(vcpu, badv, write); 622 if (ret) { 623 /* Treat as MMIO */ 624 inst.word = vcpu->arch.badi; 625 if (write) { 626 er = kvm_emu_mmio_write(vcpu, inst); 627 } else { 628 /* A code fetch fault doesn't count as an MMIO */ 629 if (kvm_is_ifetch_fault(&vcpu->arch)) { 630 kvm_queue_exception(vcpu, EXCCODE_ADE, EXSUBCODE_ADEF); 631 return RESUME_GUEST; 632 } 633 634 er = kvm_emu_mmio_read(vcpu, inst); 635 } 636 } 637 638 if (er == EMULATE_DONE) { 639 ret = RESUME_GUEST; 640 } else if (er == EMULATE_DO_MMIO) { 641 run->exit_reason = KVM_EXIT_MMIO; 642 ret = RESUME_HOST; 643 } else { 644 kvm_queue_exception(vcpu, EXCCODE_ADE, EXSUBCODE_ADEM); 645 ret = RESUME_GUEST; 646 } 647 648 return ret; 649 } 650 651 static int kvm_handle_read_fault(struct kvm_vcpu *vcpu) 652 { 653 return kvm_handle_rdwr_fault(vcpu, false); 654 } 655 656 static int kvm_handle_write_fault(struct kvm_vcpu *vcpu) 657 { 658 return kvm_handle_rdwr_fault(vcpu, true); 659 } 660 661 /** 662 * kvm_handle_fpu_disabled() - Guest used fpu however it is disabled at host 663 * @vcpu: Virtual CPU context. 664 * 665 * Handle when the guest attempts to use fpu which hasn't been allowed 666 * by the root context. 667 */ 668 static int kvm_handle_fpu_disabled(struct kvm_vcpu *vcpu) 669 { 670 struct kvm_run *run = vcpu->run; 671 672 if (!kvm_guest_has_fpu(&vcpu->arch)) { 673 kvm_queue_exception(vcpu, EXCCODE_INE, 0); 674 return RESUME_GUEST; 675 } 676 677 /* 678 * If guest FPU not present, the FPU operation should have been 679 * treated as a reserved instruction! 680 * If FPU already in use, we shouldn't get this at all. 681 */ 682 if (WARN_ON(vcpu->arch.aux_inuse & KVM_LARCH_FPU)) { 683 kvm_err("%s internal error\n", __func__); 684 run->exit_reason = KVM_EXIT_INTERNAL_ERROR; 685 return RESUME_HOST; 686 } 687 688 kvm_own_fpu(vcpu); 689 690 return RESUME_GUEST; 691 } 692 693 static long kvm_save_notify(struct kvm_vcpu *vcpu) 694 { 695 unsigned long id, data; 696 697 id = kvm_read_reg(vcpu, LOONGARCH_GPR_A1); 698 data = kvm_read_reg(vcpu, LOONGARCH_GPR_A2); 699 switch (id) { 700 case KVM_FEATURE_STEAL_TIME: 701 if (!kvm_pvtime_supported()) 702 return KVM_HCALL_INVALID_CODE; 703 704 if (data & ~(KVM_STEAL_PHYS_MASK | KVM_STEAL_PHYS_VALID)) 705 return KVM_HCALL_INVALID_PARAMETER; 706 707 vcpu->arch.st.guest_addr = data; 708 if (!(data & KVM_STEAL_PHYS_VALID)) 709 break; 710 711 vcpu->arch.st.last_steal = current->sched_info.run_delay; 712 kvm_make_request(KVM_REQ_STEAL_UPDATE, vcpu); 713 break; 714 default: 715 break; 716 }; 717 718 return 0; 719 }; 720 721 /* 722 * kvm_handle_lsx_disabled() - Guest used LSX while disabled in root. 723 * @vcpu: Virtual CPU context. 724 * 725 * Handle when the guest attempts to use LSX when it is disabled in the root 726 * context. 727 */ 728 static int kvm_handle_lsx_disabled(struct kvm_vcpu *vcpu) 729 { 730 if (kvm_own_lsx(vcpu)) 731 kvm_queue_exception(vcpu, EXCCODE_INE, 0); 732 733 return RESUME_GUEST; 734 } 735 736 /* 737 * kvm_handle_lasx_disabled() - Guest used LASX while disabled in root. 738 * @vcpu: Virtual CPU context. 739 * 740 * Handle when the guest attempts to use LASX when it is disabled in the root 741 * context. 742 */ 743 static int kvm_handle_lasx_disabled(struct kvm_vcpu *vcpu) 744 { 745 if (kvm_own_lasx(vcpu)) 746 kvm_queue_exception(vcpu, EXCCODE_INE, 0); 747 748 return RESUME_GUEST; 749 } 750 751 static int kvm_send_pv_ipi(struct kvm_vcpu *vcpu) 752 { 753 unsigned int min, cpu, i; 754 unsigned long ipi_bitmap; 755 struct kvm_vcpu *dest; 756 757 min = kvm_read_reg(vcpu, LOONGARCH_GPR_A3); 758 for (i = 0; i < 2; i++, min += BITS_PER_LONG) { 759 ipi_bitmap = kvm_read_reg(vcpu, LOONGARCH_GPR_A1 + i); 760 if (!ipi_bitmap) 761 continue; 762 763 cpu = find_first_bit((void *)&ipi_bitmap, BITS_PER_LONG); 764 while (cpu < BITS_PER_LONG) { 765 dest = kvm_get_vcpu_by_cpuid(vcpu->kvm, cpu + min); 766 cpu = find_next_bit((void *)&ipi_bitmap, BITS_PER_LONG, cpu + 1); 767 if (!dest) 768 continue; 769 770 /* Send SWI0 to dest vcpu to emulate IPI interrupt */ 771 kvm_queue_irq(dest, INT_SWI0); 772 kvm_vcpu_kick(dest); 773 } 774 } 775 776 return 0; 777 } 778 779 /* 780 * Hypercall emulation always return to guest, Caller should check retval. 781 */ 782 static void kvm_handle_service(struct kvm_vcpu *vcpu) 783 { 784 unsigned long func = kvm_read_reg(vcpu, LOONGARCH_GPR_A0); 785 long ret; 786 787 switch (func) { 788 case KVM_HCALL_FUNC_IPI: 789 kvm_send_pv_ipi(vcpu); 790 ret = KVM_HCALL_SUCCESS; 791 break; 792 case KVM_HCALL_FUNC_NOTIFY: 793 ret = kvm_save_notify(vcpu); 794 break; 795 default: 796 ret = KVM_HCALL_INVALID_CODE; 797 break; 798 } 799 800 kvm_write_reg(vcpu, LOONGARCH_GPR_A0, ret); 801 } 802 803 static int kvm_handle_hypercall(struct kvm_vcpu *vcpu) 804 { 805 int ret; 806 larch_inst inst; 807 unsigned int code; 808 809 inst.word = vcpu->arch.badi; 810 code = inst.reg0i15_format.immediate; 811 ret = RESUME_GUEST; 812 813 switch (code) { 814 case KVM_HCALL_SERVICE: 815 vcpu->stat.hypercall_exits++; 816 kvm_handle_service(vcpu); 817 break; 818 case KVM_HCALL_SWDBG: 819 /* KVM_HCALL_SWDBG only in effective when SW_BP is enabled */ 820 if (vcpu->guest_debug & KVM_GUESTDBG_SW_BP_MASK) { 821 vcpu->run->exit_reason = KVM_EXIT_DEBUG; 822 ret = RESUME_HOST; 823 break; 824 } 825 fallthrough; 826 default: 827 /* Treat it as noop intruction, only set return value */ 828 kvm_write_reg(vcpu, LOONGARCH_GPR_A0, KVM_HCALL_INVALID_CODE); 829 break; 830 } 831 832 if (ret == RESUME_GUEST) 833 update_pc(&vcpu->arch); 834 835 return ret; 836 } 837 838 /* 839 * LoongArch KVM callback handling for unimplemented guest exiting 840 */ 841 static int kvm_fault_ni(struct kvm_vcpu *vcpu) 842 { 843 unsigned int ecode, inst; 844 unsigned long estat, badv; 845 846 /* Fetch the instruction */ 847 inst = vcpu->arch.badi; 848 badv = vcpu->arch.badv; 849 estat = vcpu->arch.host_estat; 850 ecode = (estat & CSR_ESTAT_EXC) >> CSR_ESTAT_EXC_SHIFT; 851 kvm_err("ECode: %d PC=%#lx Inst=0x%08x BadVaddr=%#lx ESTAT=%#lx\n", 852 ecode, vcpu->arch.pc, inst, badv, read_gcsr_estat()); 853 kvm_arch_vcpu_dump_regs(vcpu); 854 kvm_queue_exception(vcpu, EXCCODE_INE, 0); 855 856 return RESUME_GUEST; 857 } 858 859 static exit_handle_fn kvm_fault_tables[EXCCODE_INT_START] = { 860 [0 ... EXCCODE_INT_START - 1] = kvm_fault_ni, 861 [EXCCODE_TLBI] = kvm_handle_read_fault, 862 [EXCCODE_TLBL] = kvm_handle_read_fault, 863 [EXCCODE_TLBS] = kvm_handle_write_fault, 864 [EXCCODE_TLBM] = kvm_handle_write_fault, 865 [EXCCODE_FPDIS] = kvm_handle_fpu_disabled, 866 [EXCCODE_LSXDIS] = kvm_handle_lsx_disabled, 867 [EXCCODE_LASXDIS] = kvm_handle_lasx_disabled, 868 [EXCCODE_GSPR] = kvm_handle_gspr, 869 [EXCCODE_HVC] = kvm_handle_hypercall, 870 }; 871 872 int kvm_handle_fault(struct kvm_vcpu *vcpu, int fault) 873 { 874 return kvm_fault_tables[fault](vcpu); 875 } 876