1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (C) 2020-2023 Loongson Technology Corporation Limited 4 */ 5 6 #include <linux/err.h> 7 #include <linux/errno.h> 8 #include <linux/kvm_host.h> 9 #include <linux/module.h> 10 #include <linux/preempt.h> 11 #include <linux/vmalloc.h> 12 #include <trace/events/kvm.h> 13 #include <asm/fpu.h> 14 #include <asm/inst.h> 15 #include <asm/loongarch.h> 16 #include <asm/mmzone.h> 17 #include <asm/numa.h> 18 #include <asm/time.h> 19 #include <asm/tlb.h> 20 #include <asm/kvm_csr.h> 21 #include <asm/kvm_vcpu.h> 22 #include "trace.h" 23 24 static int kvm_emu_cpucfg(struct kvm_vcpu *vcpu, larch_inst inst) 25 { 26 int rd, rj; 27 unsigned int index, ret; 28 29 if (inst.reg2_format.opcode != cpucfg_op) 30 return EMULATE_FAIL; 31 32 rd = inst.reg2_format.rd; 33 rj = inst.reg2_format.rj; 34 ++vcpu->stat.cpucfg_exits; 35 index = vcpu->arch.gprs[rj]; 36 37 /* 38 * By LoongArch Reference Manual 2.2.10.5 39 * Return value is 0 for undefined CPUCFG index 40 * 41 * Disable preemption since hw gcsr is accessed 42 */ 43 preempt_disable(); 44 switch (index) { 45 case 0 ... (KVM_MAX_CPUCFG_REGS - 1): 46 vcpu->arch.gprs[rd] = vcpu->arch.cpucfg[index]; 47 break; 48 case CPUCFG_KVM_SIG: 49 /* CPUCFG emulation between 0x40000000 -- 0x400000ff */ 50 vcpu->arch.gprs[rd] = *(unsigned int *)KVM_SIGNATURE; 51 break; 52 case CPUCFG_KVM_FEATURE: 53 ret = vcpu->kvm->arch.pv_features & LOONGARCH_PV_FEAT_MASK; 54 vcpu->arch.gprs[rd] = ret; 55 break; 56 default: 57 vcpu->arch.gprs[rd] = 0; 58 break; 59 } 60 preempt_enable(); 61 62 return EMULATE_DONE; 63 } 64 65 static unsigned long kvm_emu_read_csr(struct kvm_vcpu *vcpu, int csrid) 66 { 67 unsigned long val = 0; 68 struct loongarch_csrs *csr = vcpu->arch.csr; 69 70 /* 71 * From LoongArch Reference Manual Volume 1 Chapter 4.2.1 72 * For undefined CSR id, return value is 0 73 */ 74 if (get_gcsr_flag(csrid) & SW_GCSR) 75 val = kvm_read_sw_gcsr(csr, csrid); 76 else 77 pr_warn_once("Unsupported csrrd 0x%x with pc %lx\n", csrid, vcpu->arch.pc); 78 79 return val; 80 } 81 82 static unsigned long kvm_emu_write_csr(struct kvm_vcpu *vcpu, int csrid, unsigned long val) 83 { 84 unsigned long old = 0; 85 struct loongarch_csrs *csr = vcpu->arch.csr; 86 87 if (get_gcsr_flag(csrid) & SW_GCSR) { 88 old = kvm_read_sw_gcsr(csr, csrid); 89 kvm_write_sw_gcsr(csr, csrid, val); 90 } else 91 pr_warn_once("Unsupported csrwr 0x%x with pc %lx\n", csrid, vcpu->arch.pc); 92 93 return old; 94 } 95 96 static unsigned long kvm_emu_xchg_csr(struct kvm_vcpu *vcpu, int csrid, 97 unsigned long csr_mask, unsigned long val) 98 { 99 unsigned long old = 0; 100 struct loongarch_csrs *csr = vcpu->arch.csr; 101 102 if (get_gcsr_flag(csrid) & SW_GCSR) { 103 old = kvm_read_sw_gcsr(csr, csrid); 104 val = (old & ~csr_mask) | (val & csr_mask); 105 kvm_write_sw_gcsr(csr, csrid, val); 106 old = old & csr_mask; 107 } else 108 pr_warn_once("Unsupported csrxchg 0x%x with pc %lx\n", csrid, vcpu->arch.pc); 109 110 return old; 111 } 112 113 static int kvm_handle_csr(struct kvm_vcpu *vcpu, larch_inst inst) 114 { 115 unsigned int rd, rj, csrid; 116 unsigned long csr_mask, val = 0; 117 118 /* 119 * CSR value mask imm 120 * rj = 0 means csrrd 121 * rj = 1 means csrwr 122 * rj != 0,1 means csrxchg 123 */ 124 rd = inst.reg2csr_format.rd; 125 rj = inst.reg2csr_format.rj; 126 csrid = inst.reg2csr_format.csr; 127 128 if (csrid >= LOONGARCH_CSR_PERFCTRL0 && csrid <= vcpu->arch.max_pmu_csrid) { 129 if (kvm_guest_has_pmu(&vcpu->arch)) { 130 vcpu->arch.pc -= 4; 131 kvm_make_request(KVM_REQ_PMU, vcpu); 132 return EMULATE_DONE; 133 } 134 } 135 136 /* Process CSR ops */ 137 switch (rj) { 138 case 0: /* process csrrd */ 139 val = kvm_emu_read_csr(vcpu, csrid); 140 vcpu->arch.gprs[rd] = val; 141 break; 142 case 1: /* process csrwr */ 143 val = vcpu->arch.gprs[rd]; 144 val = kvm_emu_write_csr(vcpu, csrid, val); 145 vcpu->arch.gprs[rd] = val; 146 break; 147 default: /* process csrxchg */ 148 val = vcpu->arch.gprs[rd]; 149 csr_mask = vcpu->arch.gprs[rj]; 150 val = kvm_emu_xchg_csr(vcpu, csrid, csr_mask, val); 151 vcpu->arch.gprs[rd] = val; 152 } 153 154 return EMULATE_DONE; 155 } 156 157 int kvm_emu_iocsr(larch_inst inst, struct kvm_run *run, struct kvm_vcpu *vcpu) 158 { 159 int idx, ret; 160 unsigned long *val; 161 u32 addr, rd, rj, opcode; 162 163 /* 164 * Each IOCSR with different opcode 165 */ 166 rd = inst.reg2_format.rd; 167 rj = inst.reg2_format.rj; 168 opcode = inst.reg2_format.opcode; 169 addr = vcpu->arch.gprs[rj]; 170 run->iocsr_io.phys_addr = addr; 171 run->iocsr_io.is_write = 0; 172 val = &vcpu->arch.gprs[rd]; 173 174 /* LoongArch is Little endian */ 175 switch (opcode) { 176 case iocsrrdb_op: 177 run->iocsr_io.len = 1; 178 break; 179 case iocsrrdh_op: 180 run->iocsr_io.len = 2; 181 break; 182 case iocsrrdw_op: 183 run->iocsr_io.len = 4; 184 break; 185 case iocsrrdd_op: 186 run->iocsr_io.len = 8; 187 break; 188 case iocsrwrb_op: 189 run->iocsr_io.len = 1; 190 run->iocsr_io.is_write = 1; 191 break; 192 case iocsrwrh_op: 193 run->iocsr_io.len = 2; 194 run->iocsr_io.is_write = 1; 195 break; 196 case iocsrwrw_op: 197 run->iocsr_io.len = 4; 198 run->iocsr_io.is_write = 1; 199 break; 200 case iocsrwrd_op: 201 run->iocsr_io.len = 8; 202 run->iocsr_io.is_write = 1; 203 break; 204 default: 205 return EMULATE_FAIL; 206 } 207 208 if (run->iocsr_io.is_write) { 209 idx = srcu_read_lock(&vcpu->kvm->srcu); 210 ret = kvm_io_bus_write(vcpu, KVM_IOCSR_BUS, addr, run->iocsr_io.len, val); 211 srcu_read_unlock(&vcpu->kvm->srcu, idx); 212 if (ret == 0) 213 ret = EMULATE_DONE; 214 else { 215 ret = EMULATE_DO_IOCSR; 216 /* Save data and let user space to write it */ 217 memcpy(run->iocsr_io.data, val, run->iocsr_io.len); 218 } 219 trace_kvm_iocsr(KVM_TRACE_IOCSR_WRITE, run->iocsr_io.len, addr, val); 220 } else { 221 vcpu->arch.io_gpr = rd; /* Set register id for iocsr read completion */ 222 idx = srcu_read_lock(&vcpu->kvm->srcu); 223 ret = kvm_io_bus_read(vcpu, KVM_IOCSR_BUS, addr, 224 run->iocsr_io.len, run->iocsr_io.data); 225 srcu_read_unlock(&vcpu->kvm->srcu, idx); 226 if (ret == 0) { 227 kvm_complete_iocsr_read(vcpu, run); 228 ret = EMULATE_DONE; 229 } else 230 ret = EMULATE_DO_IOCSR; 231 trace_kvm_iocsr(KVM_TRACE_IOCSR_READ, run->iocsr_io.len, addr, NULL); 232 } 233 234 return ret; 235 } 236 237 int kvm_complete_iocsr_read(struct kvm_vcpu *vcpu, struct kvm_run *run) 238 { 239 enum emulation_result er = EMULATE_DONE; 240 unsigned long *gpr = &vcpu->arch.gprs[vcpu->arch.io_gpr]; 241 242 switch (run->iocsr_io.len) { 243 case 1: 244 *gpr = *(s8 *)run->iocsr_io.data; 245 break; 246 case 2: 247 *gpr = *(s16 *)run->iocsr_io.data; 248 break; 249 case 4: 250 *gpr = *(s32 *)run->iocsr_io.data; 251 break; 252 case 8: 253 *gpr = *(s64 *)run->iocsr_io.data; 254 break; 255 default: 256 kvm_err("Bad IOCSR length: %d, addr is 0x%lx\n", 257 run->iocsr_io.len, vcpu->arch.badv); 258 er = EMULATE_FAIL; 259 break; 260 } 261 262 return er; 263 } 264 265 int kvm_emu_idle(struct kvm_vcpu *vcpu) 266 { 267 ++vcpu->stat.idle_exits; 268 trace_kvm_exit_idle(vcpu, KVM_TRACE_EXIT_IDLE); 269 270 if (!kvm_arch_vcpu_runnable(vcpu)) 271 kvm_vcpu_halt(vcpu); 272 273 return EMULATE_DONE; 274 } 275 276 static int kvm_trap_handle_gspr(struct kvm_vcpu *vcpu) 277 { 278 unsigned long curr_pc; 279 larch_inst inst; 280 enum emulation_result er = EMULATE_DONE; 281 struct kvm_run *run = vcpu->run; 282 283 /* Fetch the instruction */ 284 inst.word = vcpu->arch.badi; 285 curr_pc = vcpu->arch.pc; 286 update_pc(&vcpu->arch); 287 288 trace_kvm_exit_gspr(vcpu, inst.word); 289 er = EMULATE_FAIL; 290 switch (((inst.word >> 24) & 0xff)) { 291 case 0x0: /* CPUCFG GSPR */ 292 trace_kvm_exit_cpucfg(vcpu, KVM_TRACE_EXIT_CPUCFG); 293 er = kvm_emu_cpucfg(vcpu, inst); 294 break; 295 case 0x4: /* CSR{RD,WR,XCHG} GSPR */ 296 trace_kvm_exit_csr(vcpu, KVM_TRACE_EXIT_CSR); 297 er = kvm_handle_csr(vcpu, inst); 298 break; 299 case 0x6: /* Cache, Idle and IOCSR GSPR */ 300 switch (((inst.word >> 22) & 0x3ff)) { 301 case 0x18: /* Cache GSPR */ 302 er = EMULATE_DONE; 303 trace_kvm_exit_cache(vcpu, KVM_TRACE_EXIT_CACHE); 304 break; 305 case 0x19: /* Idle/IOCSR GSPR */ 306 switch (((inst.word >> 15) & 0x1ffff)) { 307 case 0xc90: /* IOCSR GSPR */ 308 er = kvm_emu_iocsr(inst, run, vcpu); 309 break; 310 case 0xc91: /* Idle GSPR */ 311 er = kvm_emu_idle(vcpu); 312 break; 313 default: 314 er = EMULATE_FAIL; 315 break; 316 } 317 break; 318 default: 319 er = EMULATE_FAIL; 320 break; 321 } 322 break; 323 default: 324 er = EMULATE_FAIL; 325 break; 326 } 327 328 /* Rollback PC only if emulation was unsuccessful */ 329 if (er == EMULATE_FAIL) { 330 kvm_err("[%#lx]%s: unsupported gspr instruction 0x%08x\n", 331 curr_pc, __func__, inst.word); 332 333 kvm_arch_vcpu_dump_regs(vcpu); 334 vcpu->arch.pc = curr_pc; 335 } 336 337 return er; 338 } 339 340 /* 341 * Trigger GSPR: 342 * 1) Execute CPUCFG instruction; 343 * 2) Execute CACOP/IDLE instructions; 344 * 3) Access to unimplemented CSRs/IOCSRs. 345 */ 346 static int kvm_handle_gspr(struct kvm_vcpu *vcpu, int ecode) 347 { 348 int ret = RESUME_GUEST; 349 enum emulation_result er = EMULATE_DONE; 350 351 er = kvm_trap_handle_gspr(vcpu); 352 353 if (er == EMULATE_DONE) { 354 ret = RESUME_GUEST; 355 } else if (er == EMULATE_DO_MMIO) { 356 vcpu->run->exit_reason = KVM_EXIT_MMIO; 357 ret = RESUME_HOST; 358 } else if (er == EMULATE_DO_IOCSR) { 359 vcpu->run->exit_reason = KVM_EXIT_LOONGARCH_IOCSR; 360 ret = RESUME_HOST; 361 } else { 362 kvm_queue_exception(vcpu, EXCCODE_INE, 0); 363 ret = RESUME_GUEST; 364 } 365 366 return ret; 367 } 368 369 int kvm_emu_mmio_read(struct kvm_vcpu *vcpu, larch_inst inst) 370 { 371 int idx, ret; 372 unsigned int op8, opcode, rd; 373 struct kvm_run *run = vcpu->run; 374 375 run->mmio.phys_addr = vcpu->arch.badv; 376 vcpu->mmio_needed = 2; /* signed */ 377 op8 = (inst.word >> 24) & 0xff; 378 ret = EMULATE_DO_MMIO; 379 380 switch (op8) { 381 case 0x24 ... 0x27: /* ldptr.w/d process */ 382 rd = inst.reg2i14_format.rd; 383 opcode = inst.reg2i14_format.opcode; 384 385 switch (opcode) { 386 case ldptrw_op: 387 run->mmio.len = 4; 388 break; 389 case ldptrd_op: 390 run->mmio.len = 8; 391 break; 392 default: 393 break; 394 } 395 break; 396 case 0x28 ... 0x2e: /* ld.b/h/w/d, ld.bu/hu/wu process */ 397 rd = inst.reg2i12_format.rd; 398 opcode = inst.reg2i12_format.opcode; 399 400 switch (opcode) { 401 case ldb_op: 402 run->mmio.len = 1; 403 break; 404 case ldbu_op: 405 vcpu->mmio_needed = 1; /* unsigned */ 406 run->mmio.len = 1; 407 break; 408 case ldh_op: 409 run->mmio.len = 2; 410 break; 411 case ldhu_op: 412 vcpu->mmio_needed = 1; /* unsigned */ 413 run->mmio.len = 2; 414 break; 415 case ldw_op: 416 run->mmio.len = 4; 417 break; 418 case ldwu_op: 419 vcpu->mmio_needed = 1; /* unsigned */ 420 run->mmio.len = 4; 421 break; 422 case ldd_op: 423 run->mmio.len = 8; 424 break; 425 default: 426 ret = EMULATE_FAIL; 427 break; 428 } 429 break; 430 case 0x38: /* ldx.b/h/w/d, ldx.bu/hu/wu process */ 431 rd = inst.reg3_format.rd; 432 opcode = inst.reg3_format.opcode; 433 434 switch (opcode) { 435 case ldxb_op: 436 run->mmio.len = 1; 437 break; 438 case ldxbu_op: 439 run->mmio.len = 1; 440 vcpu->mmio_needed = 1; /* unsigned */ 441 break; 442 case ldxh_op: 443 run->mmio.len = 2; 444 break; 445 case ldxhu_op: 446 run->mmio.len = 2; 447 vcpu->mmio_needed = 1; /* unsigned */ 448 break; 449 case ldxw_op: 450 run->mmio.len = 4; 451 break; 452 case ldxwu_op: 453 run->mmio.len = 4; 454 vcpu->mmio_needed = 1; /* unsigned */ 455 break; 456 case ldxd_op: 457 run->mmio.len = 8; 458 break; 459 default: 460 ret = EMULATE_FAIL; 461 break; 462 } 463 break; 464 default: 465 ret = EMULATE_FAIL; 466 } 467 468 if (ret == EMULATE_DO_MMIO) { 469 trace_kvm_mmio(KVM_TRACE_MMIO_READ, run->mmio.len, run->mmio.phys_addr, NULL); 470 471 vcpu->arch.io_gpr = rd; /* Set for kvm_complete_mmio_read() use */ 472 473 /* 474 * If mmio device such as PCH-PIC is emulated in KVM, 475 * it need not return to user space to handle the mmio 476 * exception. 477 */ 478 idx = srcu_read_lock(&vcpu->kvm->srcu); 479 ret = kvm_io_bus_read(vcpu, KVM_MMIO_BUS, vcpu->arch.badv, 480 run->mmio.len, run->mmio.data); 481 srcu_read_unlock(&vcpu->kvm->srcu, idx); 482 if (!ret) { 483 kvm_complete_mmio_read(vcpu, run); 484 update_pc(&vcpu->arch); 485 vcpu->mmio_needed = 0; 486 return EMULATE_DONE; 487 } 488 489 run->mmio.is_write = 0; 490 vcpu->mmio_is_write = 0; 491 return EMULATE_DO_MMIO; 492 } 493 494 kvm_err("Read not supported Inst=0x%08x @%lx BadVaddr:%#lx\n", 495 inst.word, vcpu->arch.pc, vcpu->arch.badv); 496 kvm_arch_vcpu_dump_regs(vcpu); 497 vcpu->mmio_needed = 0; 498 499 return ret; 500 } 501 502 int kvm_complete_mmio_read(struct kvm_vcpu *vcpu, struct kvm_run *run) 503 { 504 enum emulation_result er = EMULATE_DONE; 505 unsigned long *gpr = &vcpu->arch.gprs[vcpu->arch.io_gpr]; 506 507 /* Update with new PC */ 508 update_pc(&vcpu->arch); 509 switch (run->mmio.len) { 510 case 1: 511 if (vcpu->mmio_needed == 2) 512 *gpr = *(s8 *)run->mmio.data; 513 else 514 *gpr = *(u8 *)run->mmio.data; 515 break; 516 case 2: 517 if (vcpu->mmio_needed == 2) 518 *gpr = *(s16 *)run->mmio.data; 519 else 520 *gpr = *(u16 *)run->mmio.data; 521 break; 522 case 4: 523 if (vcpu->mmio_needed == 2) 524 *gpr = *(s32 *)run->mmio.data; 525 else 526 *gpr = *(u32 *)run->mmio.data; 527 break; 528 case 8: 529 *gpr = *(s64 *)run->mmio.data; 530 break; 531 default: 532 kvm_err("Bad MMIO length: %d, addr is 0x%lx\n", 533 run->mmio.len, vcpu->arch.badv); 534 er = EMULATE_FAIL; 535 break; 536 } 537 538 trace_kvm_mmio(KVM_TRACE_MMIO_READ, run->mmio.len, 539 run->mmio.phys_addr, run->mmio.data); 540 541 return er; 542 } 543 544 int kvm_emu_mmio_write(struct kvm_vcpu *vcpu, larch_inst inst) 545 { 546 int idx, ret; 547 unsigned int rd, op8, opcode; 548 unsigned long curr_pc, rd_val = 0; 549 struct kvm_run *run = vcpu->run; 550 void *data = run->mmio.data; 551 552 /* 553 * Update PC and hold onto current PC in case there is 554 * an error and we want to rollback the PC 555 */ 556 curr_pc = vcpu->arch.pc; 557 update_pc(&vcpu->arch); 558 559 op8 = (inst.word >> 24) & 0xff; 560 run->mmio.phys_addr = vcpu->arch.badv; 561 ret = EMULATE_DO_MMIO; 562 switch (op8) { 563 case 0x24 ... 0x27: /* stptr.w/d process */ 564 rd = inst.reg2i14_format.rd; 565 opcode = inst.reg2i14_format.opcode; 566 567 switch (opcode) { 568 case stptrw_op: 569 run->mmio.len = 4; 570 *(unsigned int *)data = vcpu->arch.gprs[rd]; 571 break; 572 case stptrd_op: 573 run->mmio.len = 8; 574 *(unsigned long *)data = vcpu->arch.gprs[rd]; 575 break; 576 default: 577 ret = EMULATE_FAIL; 578 break; 579 } 580 break; 581 case 0x28 ... 0x2e: /* st.b/h/w/d process */ 582 rd = inst.reg2i12_format.rd; 583 opcode = inst.reg2i12_format.opcode; 584 rd_val = vcpu->arch.gprs[rd]; 585 586 switch (opcode) { 587 case stb_op: 588 run->mmio.len = 1; 589 *(unsigned char *)data = rd_val; 590 break; 591 case sth_op: 592 run->mmio.len = 2; 593 *(unsigned short *)data = rd_val; 594 break; 595 case stw_op: 596 run->mmio.len = 4; 597 *(unsigned int *)data = rd_val; 598 break; 599 case std_op: 600 run->mmio.len = 8; 601 *(unsigned long *)data = rd_val; 602 break; 603 default: 604 ret = EMULATE_FAIL; 605 break; 606 } 607 break; 608 case 0x38: /* stx.b/h/w/d process */ 609 rd = inst.reg3_format.rd; 610 opcode = inst.reg3_format.opcode; 611 612 switch (opcode) { 613 case stxb_op: 614 run->mmio.len = 1; 615 *(unsigned char *)data = vcpu->arch.gprs[rd]; 616 break; 617 case stxh_op: 618 run->mmio.len = 2; 619 *(unsigned short *)data = vcpu->arch.gprs[rd]; 620 break; 621 case stxw_op: 622 run->mmio.len = 4; 623 *(unsigned int *)data = vcpu->arch.gprs[rd]; 624 break; 625 case stxd_op: 626 run->mmio.len = 8; 627 *(unsigned long *)data = vcpu->arch.gprs[rd]; 628 break; 629 default: 630 ret = EMULATE_FAIL; 631 break; 632 } 633 break; 634 default: 635 ret = EMULATE_FAIL; 636 } 637 638 if (ret == EMULATE_DO_MMIO) { 639 trace_kvm_mmio(KVM_TRACE_MMIO_WRITE, run->mmio.len, run->mmio.phys_addr, data); 640 641 /* 642 * If mmio device such as PCH-PIC is emulated in KVM, 643 * it need not return to user space to handle the mmio 644 * exception. 645 */ 646 idx = srcu_read_lock(&vcpu->kvm->srcu); 647 ret = kvm_io_bus_write(vcpu, KVM_MMIO_BUS, vcpu->arch.badv, run->mmio.len, data); 648 srcu_read_unlock(&vcpu->kvm->srcu, idx); 649 if (!ret) 650 return EMULATE_DONE; 651 652 run->mmio.is_write = 1; 653 vcpu->mmio_needed = 1; 654 vcpu->mmio_is_write = 1; 655 return EMULATE_DO_MMIO; 656 } 657 658 vcpu->arch.pc = curr_pc; 659 kvm_err("Write not supported Inst=0x%08x @%lx BadVaddr:%#lx\n", 660 inst.word, vcpu->arch.pc, vcpu->arch.badv); 661 kvm_arch_vcpu_dump_regs(vcpu); 662 /* Rollback PC if emulation was unsuccessful */ 663 664 return ret; 665 } 666 667 static int kvm_handle_rdwr_fault(struct kvm_vcpu *vcpu, bool write, int ecode) 668 { 669 int ret; 670 larch_inst inst; 671 enum emulation_result er = EMULATE_DONE; 672 struct kvm_run *run = vcpu->run; 673 unsigned long badv = vcpu->arch.badv; 674 675 /* Inject ADE exception if exceed max GPA size */ 676 if (unlikely(badv >= vcpu->kvm->arch.gpa_size)) { 677 kvm_queue_exception(vcpu, EXCCODE_ADE, EXSUBCODE_ADEM); 678 return RESUME_GUEST; 679 } 680 681 ret = kvm_handle_mm_fault(vcpu, badv, write, ecode); 682 if (ret) { 683 /* Treat as MMIO */ 684 inst.word = vcpu->arch.badi; 685 if (write) { 686 er = kvm_emu_mmio_write(vcpu, inst); 687 } else { 688 /* A code fetch fault doesn't count as an MMIO */ 689 if (kvm_is_ifetch_fault(&vcpu->arch)) { 690 kvm_queue_exception(vcpu, EXCCODE_ADE, EXSUBCODE_ADEF); 691 return RESUME_GUEST; 692 } 693 694 er = kvm_emu_mmio_read(vcpu, inst); 695 } 696 } 697 698 if (er == EMULATE_DONE) { 699 ret = RESUME_GUEST; 700 } else if (er == EMULATE_DO_MMIO) { 701 run->exit_reason = KVM_EXIT_MMIO; 702 ret = RESUME_HOST; 703 } else { 704 kvm_queue_exception(vcpu, EXCCODE_ADE, EXSUBCODE_ADEM); 705 ret = RESUME_GUEST; 706 } 707 708 return ret; 709 } 710 711 static int kvm_handle_read_fault(struct kvm_vcpu *vcpu, int ecode) 712 { 713 return kvm_handle_rdwr_fault(vcpu, false, ecode); 714 } 715 716 static int kvm_handle_write_fault(struct kvm_vcpu *vcpu, int ecode) 717 { 718 return kvm_handle_rdwr_fault(vcpu, true, ecode); 719 } 720 721 int kvm_complete_user_service(struct kvm_vcpu *vcpu, struct kvm_run *run) 722 { 723 update_pc(&vcpu->arch); 724 kvm_write_reg(vcpu, LOONGARCH_GPR_A0, run->hypercall.ret); 725 726 return 0; 727 } 728 729 /** 730 * kvm_handle_fpu_disabled() - Guest used fpu however it is disabled at host 731 * @vcpu: Virtual CPU context. 732 * @ecode: Exception code. 733 * 734 * Handle when the guest attempts to use fpu which hasn't been allowed 735 * by the root context. 736 */ 737 static int kvm_handle_fpu_disabled(struct kvm_vcpu *vcpu, int ecode) 738 { 739 struct kvm_run *run = vcpu->run; 740 741 if (!kvm_guest_has_fpu(&vcpu->arch)) { 742 kvm_queue_exception(vcpu, EXCCODE_INE, 0); 743 return RESUME_GUEST; 744 } 745 746 /* 747 * If guest FPU not present, the FPU operation should have been 748 * treated as a reserved instruction! 749 * If FPU already in use, we shouldn't get this at all. 750 */ 751 if (WARN_ON(vcpu->arch.aux_inuse & KVM_LARCH_FPU)) { 752 kvm_err("%s internal error\n", __func__); 753 run->exit_reason = KVM_EXIT_INTERNAL_ERROR; 754 return RESUME_HOST; 755 } 756 757 vcpu->arch.aux_ldtype = KVM_LARCH_FPU; 758 kvm_make_request(KVM_REQ_AUX_LOAD, vcpu); 759 760 return RESUME_GUEST; 761 } 762 763 static long kvm_save_notify(struct kvm_vcpu *vcpu) 764 { 765 unsigned long id, data; 766 767 id = kvm_read_reg(vcpu, LOONGARCH_GPR_A1); 768 data = kvm_read_reg(vcpu, LOONGARCH_GPR_A2); 769 switch (id) { 770 case BIT(KVM_FEATURE_STEAL_TIME): 771 if (data & ~(KVM_STEAL_PHYS_MASK | KVM_STEAL_PHYS_VALID)) 772 return KVM_HCALL_INVALID_PARAMETER; 773 774 vcpu->arch.st.guest_addr = data; 775 if (!(data & KVM_STEAL_PHYS_VALID)) 776 return 0; 777 778 vcpu->arch.st.last_steal = current->sched_info.run_delay; 779 kvm_make_request(KVM_REQ_STEAL_UPDATE, vcpu); 780 return 0; 781 default: 782 return KVM_HCALL_INVALID_CODE; 783 } 784 } 785 786 /* 787 * kvm_handle_lsx_disabled() - Guest used LSX while disabled in root. 788 * @vcpu: Virtual CPU context. 789 * @ecode: Exception code. 790 * 791 * Handle when the guest attempts to use LSX when it is disabled in the root 792 * context. 793 */ 794 static int kvm_handle_lsx_disabled(struct kvm_vcpu *vcpu, int ecode) 795 { 796 if (!kvm_guest_has_lsx(&vcpu->arch)) 797 kvm_queue_exception(vcpu, EXCCODE_INE, 0); 798 else { 799 vcpu->arch.aux_ldtype = KVM_LARCH_LSX; 800 kvm_make_request(KVM_REQ_AUX_LOAD, vcpu); 801 } 802 803 return RESUME_GUEST; 804 } 805 806 /* 807 * kvm_handle_lasx_disabled() - Guest used LASX while disabled in root. 808 * @vcpu: Virtual CPU context. 809 * @ecode: Exception code. 810 * 811 * Handle when the guest attempts to use LASX when it is disabled in the root 812 * context. 813 */ 814 static int kvm_handle_lasx_disabled(struct kvm_vcpu *vcpu, int ecode) 815 { 816 if (!kvm_guest_has_lasx(&vcpu->arch)) 817 kvm_queue_exception(vcpu, EXCCODE_INE, 0); 818 else { 819 vcpu->arch.aux_ldtype = KVM_LARCH_LASX; 820 kvm_make_request(KVM_REQ_AUX_LOAD, vcpu); 821 } 822 823 return RESUME_GUEST; 824 } 825 826 static int kvm_handle_lbt_disabled(struct kvm_vcpu *vcpu, int ecode) 827 { 828 if (!kvm_guest_has_lbt(&vcpu->arch)) 829 kvm_queue_exception(vcpu, EXCCODE_INE, 0); 830 else { 831 vcpu->arch.aux_ldtype = KVM_LARCH_LBT; 832 kvm_make_request(KVM_REQ_AUX_LOAD, vcpu); 833 } 834 835 return RESUME_GUEST; 836 } 837 838 static void kvm_send_pv_ipi(struct kvm_vcpu *vcpu) 839 { 840 unsigned int min, cpu; 841 struct kvm_vcpu *dest; 842 DECLARE_BITMAP(ipi_bitmap, BITS_PER_LONG * 2) = { 843 kvm_read_reg(vcpu, LOONGARCH_GPR_A1), 844 kvm_read_reg(vcpu, LOONGARCH_GPR_A2) 845 }; 846 847 min = kvm_read_reg(vcpu, LOONGARCH_GPR_A3); 848 for_each_set_bit(cpu, ipi_bitmap, BITS_PER_LONG * 2) { 849 dest = kvm_get_vcpu_by_cpuid(vcpu->kvm, cpu + min); 850 if (!dest) 851 continue; 852 853 /* Send SWI0 to dest vcpu to emulate IPI interrupt */ 854 kvm_queue_irq(dest, INT_SWI0); 855 kvm_vcpu_kick(dest); 856 } 857 } 858 859 /* 860 * Hypercall emulation always return to guest, Caller should check retval. 861 */ 862 static void kvm_handle_service(struct kvm_vcpu *vcpu) 863 { 864 long ret = KVM_HCALL_INVALID_CODE; 865 unsigned long func = kvm_read_reg(vcpu, LOONGARCH_GPR_A0); 866 867 switch (func) { 868 case KVM_HCALL_FUNC_IPI: 869 if (kvm_guest_has_pv_feature(vcpu, KVM_FEATURE_IPI)) { 870 kvm_send_pv_ipi(vcpu); 871 ret = KVM_HCALL_SUCCESS; 872 } 873 break; 874 case KVM_HCALL_FUNC_NOTIFY: 875 if (kvm_guest_has_pv_feature(vcpu, KVM_FEATURE_STEAL_TIME)) 876 ret = kvm_save_notify(vcpu); 877 break; 878 default: 879 break; 880 } 881 882 kvm_write_reg(vcpu, LOONGARCH_GPR_A0, ret); 883 } 884 885 static int kvm_handle_hypercall(struct kvm_vcpu *vcpu, int ecode) 886 { 887 int ret; 888 larch_inst inst; 889 unsigned int code; 890 891 inst.word = vcpu->arch.badi; 892 code = inst.reg0i15_format.immediate; 893 ret = RESUME_GUEST; 894 895 switch (code) { 896 case KVM_HCALL_SERVICE: 897 vcpu->stat.hypercall_exits++; 898 kvm_handle_service(vcpu); 899 break; 900 case KVM_HCALL_USER_SERVICE: 901 if (!kvm_guest_has_pv_feature(vcpu, KVM_FEATURE_USER_HCALL)) { 902 kvm_write_reg(vcpu, LOONGARCH_GPR_A0, KVM_HCALL_INVALID_CODE); 903 break; 904 } 905 906 vcpu->stat.hypercall_exits++; 907 vcpu->run->exit_reason = KVM_EXIT_HYPERCALL; 908 vcpu->run->hypercall.nr = KVM_HCALL_USER_SERVICE; 909 vcpu->run->hypercall.args[0] = kvm_read_reg(vcpu, LOONGARCH_GPR_A0); 910 vcpu->run->hypercall.args[1] = kvm_read_reg(vcpu, LOONGARCH_GPR_A1); 911 vcpu->run->hypercall.args[2] = kvm_read_reg(vcpu, LOONGARCH_GPR_A2); 912 vcpu->run->hypercall.args[3] = kvm_read_reg(vcpu, LOONGARCH_GPR_A3); 913 vcpu->run->hypercall.args[4] = kvm_read_reg(vcpu, LOONGARCH_GPR_A4); 914 vcpu->run->hypercall.args[5] = kvm_read_reg(vcpu, LOONGARCH_GPR_A5); 915 vcpu->run->hypercall.flags = 0; 916 /* 917 * Set invalid return value by default, let user-mode VMM modify it. 918 */ 919 vcpu->run->hypercall.ret = KVM_HCALL_INVALID_CODE; 920 ret = RESUME_HOST; 921 break; 922 case KVM_HCALL_SWDBG: 923 /* KVM_HCALL_SWDBG only in effective when SW_BP is enabled */ 924 if (vcpu->guest_debug & KVM_GUESTDBG_SW_BP_MASK) { 925 vcpu->run->exit_reason = KVM_EXIT_DEBUG; 926 ret = RESUME_HOST; 927 break; 928 } 929 fallthrough; 930 default: 931 /* Treat it as noop intruction, only set return value */ 932 kvm_write_reg(vcpu, LOONGARCH_GPR_A0, KVM_HCALL_INVALID_CODE); 933 break; 934 } 935 936 if (ret == RESUME_GUEST) 937 update_pc(&vcpu->arch); 938 939 return ret; 940 } 941 942 /* 943 * LoongArch KVM callback handling for unimplemented guest exiting 944 */ 945 static int kvm_fault_ni(struct kvm_vcpu *vcpu, int ecode) 946 { 947 unsigned int inst; 948 unsigned long badv; 949 950 /* Fetch the instruction */ 951 inst = vcpu->arch.badi; 952 badv = vcpu->arch.badv; 953 kvm_err("ECode: %d PC=%#lx Inst=0x%08x BadVaddr=%#lx ESTAT=%#lx\n", 954 ecode, vcpu->arch.pc, inst, badv, read_gcsr_estat()); 955 kvm_arch_vcpu_dump_regs(vcpu); 956 kvm_queue_exception(vcpu, EXCCODE_INE, 0); 957 958 return RESUME_GUEST; 959 } 960 961 static exit_handle_fn kvm_fault_tables[EXCCODE_INT_START] = { 962 [0 ... EXCCODE_INT_START - 1] = kvm_fault_ni, 963 [EXCCODE_TLBI] = kvm_handle_read_fault, 964 [EXCCODE_TLBL] = kvm_handle_read_fault, 965 [EXCCODE_TLBS] = kvm_handle_write_fault, 966 [EXCCODE_TLBM] = kvm_handle_write_fault, 967 [EXCCODE_FPDIS] = kvm_handle_fpu_disabled, 968 [EXCCODE_LSXDIS] = kvm_handle_lsx_disabled, 969 [EXCCODE_LASXDIS] = kvm_handle_lasx_disabled, 970 [EXCCODE_BTDIS] = kvm_handle_lbt_disabled, 971 [EXCCODE_GSPR] = kvm_handle_gspr, 972 [EXCCODE_HVC] = kvm_handle_hypercall, 973 }; 974 975 int kvm_handle_fault(struct kvm_vcpu *vcpu, int fault) 976 { 977 return kvm_fault_tables[fault](vcpu, fault); 978 } 979