1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * Copyright (C) 2020-2023 Loongson Technology Corporation Limited 4 */ 5 6 #include <linux/err.h> 7 #include <linux/errno.h> 8 #include <linux/kvm_host.h> 9 #include <linux/module.h> 10 #include <linux/preempt.h> 11 #include <linux/vmalloc.h> 12 #include <trace/events/kvm.h> 13 #include <asm/fpu.h> 14 #include <asm/inst.h> 15 #include <asm/loongarch.h> 16 #include <asm/mmzone.h> 17 #include <asm/numa.h> 18 #include <asm/time.h> 19 #include <asm/tlb.h> 20 #include <asm/kvm_csr.h> 21 #include <asm/kvm_vcpu.h> 22 #include "trace.h" 23 24 static int kvm_emu_cpucfg(struct kvm_vcpu *vcpu, larch_inst inst) 25 { 26 int rd, rj; 27 unsigned int index, ret; 28 29 if (inst.reg2_format.opcode != cpucfg_op) 30 return EMULATE_FAIL; 31 32 rd = inst.reg2_format.rd; 33 rj = inst.reg2_format.rj; 34 ++vcpu->stat.cpucfg_exits; 35 index = vcpu->arch.gprs[rj]; 36 37 /* 38 * By LoongArch Reference Manual 2.2.10.5 39 * Return value is 0 for undefined CPUCFG index 40 * 41 * Disable preemption since hw gcsr is accessed 42 */ 43 preempt_disable(); 44 switch (index) { 45 case 0 ... (KVM_MAX_CPUCFG_REGS - 1): 46 vcpu->arch.gprs[rd] = vcpu->arch.cpucfg[index]; 47 break; 48 case CPUCFG_KVM_SIG: 49 /* CPUCFG emulation between 0x40000000 -- 0x400000ff */ 50 vcpu->arch.gprs[rd] = *(unsigned int *)KVM_SIGNATURE; 51 break; 52 case CPUCFG_KVM_FEATURE: 53 ret = vcpu->kvm->arch.pv_features & LOONGARCH_PV_FEAT_MASK; 54 vcpu->arch.gprs[rd] = ret; 55 break; 56 default: 57 vcpu->arch.gprs[rd] = 0; 58 break; 59 } 60 preempt_enable(); 61 62 return EMULATE_DONE; 63 } 64 65 static unsigned long kvm_emu_read_csr(struct kvm_vcpu *vcpu, int csrid) 66 { 67 unsigned long val = 0; 68 struct loongarch_csrs *csr = vcpu->arch.csr; 69 70 /* 71 * From LoongArch Reference Manual Volume 1 Chapter 4.2.1 72 * For undefined CSR id, return value is 0 73 */ 74 if (get_gcsr_flag(csrid) & SW_GCSR) 75 val = kvm_read_sw_gcsr(csr, csrid); 76 else 77 pr_warn_once("Unsupported csrrd 0x%x with pc %lx\n", csrid, vcpu->arch.pc); 78 79 return val; 80 } 81 82 static unsigned long kvm_emu_write_csr(struct kvm_vcpu *vcpu, int csrid, unsigned long val) 83 { 84 unsigned long old = 0; 85 struct loongarch_csrs *csr = vcpu->arch.csr; 86 87 if (get_gcsr_flag(csrid) & SW_GCSR) { 88 old = kvm_read_sw_gcsr(csr, csrid); 89 kvm_write_sw_gcsr(csr, csrid, val); 90 } else 91 pr_warn_once("Unsupported csrwr 0x%x with pc %lx\n", csrid, vcpu->arch.pc); 92 93 return old; 94 } 95 96 static unsigned long kvm_emu_xchg_csr(struct kvm_vcpu *vcpu, int csrid, 97 unsigned long csr_mask, unsigned long val) 98 { 99 unsigned long old = 0; 100 struct loongarch_csrs *csr = vcpu->arch.csr; 101 102 if (get_gcsr_flag(csrid) & SW_GCSR) { 103 old = kvm_read_sw_gcsr(csr, csrid); 104 val = (old & ~csr_mask) | (val & csr_mask); 105 kvm_write_sw_gcsr(csr, csrid, val); 106 } else 107 pr_warn_once("Unsupported csrxchg 0x%x with pc %lx\n", csrid, vcpu->arch.pc); 108 109 return old; 110 } 111 112 static int kvm_handle_csr(struct kvm_vcpu *vcpu, larch_inst inst) 113 { 114 unsigned int rd, rj, csrid; 115 unsigned long csr_mask, val = 0; 116 117 /* 118 * CSR value mask imm 119 * rj = 0 means csrrd 120 * rj = 1 means csrwr 121 * rj != 0,1 means csrxchg 122 */ 123 rd = inst.reg2csr_format.rd; 124 rj = inst.reg2csr_format.rj; 125 csrid = inst.reg2csr_format.csr; 126 127 if (csrid >= LOONGARCH_CSR_PERFCTRL0 && csrid <= vcpu->arch.max_pmu_csrid) { 128 if (kvm_guest_has_pmu(&vcpu->arch)) { 129 vcpu->arch.pc -= 4; 130 kvm_make_request(KVM_REQ_PMU, vcpu); 131 return EMULATE_DONE; 132 } 133 } 134 135 /* Process CSR ops */ 136 switch (rj) { 137 case 0: /* process csrrd */ 138 val = kvm_emu_read_csr(vcpu, csrid); 139 vcpu->arch.gprs[rd] = val; 140 break; 141 case 1: /* process csrwr */ 142 val = vcpu->arch.gprs[rd]; 143 val = kvm_emu_write_csr(vcpu, csrid, val); 144 vcpu->arch.gprs[rd] = val; 145 break; 146 default: /* process csrxchg */ 147 val = vcpu->arch.gprs[rd]; 148 csr_mask = vcpu->arch.gprs[rj]; 149 val = kvm_emu_xchg_csr(vcpu, csrid, csr_mask, val); 150 vcpu->arch.gprs[rd] = val; 151 } 152 153 return EMULATE_DONE; 154 } 155 156 int kvm_emu_iocsr(larch_inst inst, struct kvm_run *run, struct kvm_vcpu *vcpu) 157 { 158 int idx, ret; 159 unsigned long *val; 160 u32 addr, rd, rj, opcode; 161 162 /* 163 * Each IOCSR with different opcode 164 */ 165 rd = inst.reg2_format.rd; 166 rj = inst.reg2_format.rj; 167 opcode = inst.reg2_format.opcode; 168 addr = vcpu->arch.gprs[rj]; 169 run->iocsr_io.phys_addr = addr; 170 run->iocsr_io.is_write = 0; 171 val = &vcpu->arch.gprs[rd]; 172 173 /* LoongArch is Little endian */ 174 switch (opcode) { 175 case iocsrrdb_op: 176 run->iocsr_io.len = 1; 177 break; 178 case iocsrrdh_op: 179 run->iocsr_io.len = 2; 180 break; 181 case iocsrrdw_op: 182 run->iocsr_io.len = 4; 183 break; 184 case iocsrrdd_op: 185 run->iocsr_io.len = 8; 186 break; 187 case iocsrwrb_op: 188 run->iocsr_io.len = 1; 189 run->iocsr_io.is_write = 1; 190 break; 191 case iocsrwrh_op: 192 run->iocsr_io.len = 2; 193 run->iocsr_io.is_write = 1; 194 break; 195 case iocsrwrw_op: 196 run->iocsr_io.len = 4; 197 run->iocsr_io.is_write = 1; 198 break; 199 case iocsrwrd_op: 200 run->iocsr_io.len = 8; 201 run->iocsr_io.is_write = 1; 202 break; 203 default: 204 return EMULATE_FAIL; 205 } 206 207 if (run->iocsr_io.is_write) { 208 idx = srcu_read_lock(&vcpu->kvm->srcu); 209 ret = kvm_io_bus_write(vcpu, KVM_IOCSR_BUS, addr, run->iocsr_io.len, val); 210 srcu_read_unlock(&vcpu->kvm->srcu, idx); 211 if (ret == 0) 212 ret = EMULATE_DONE; 213 else { 214 ret = EMULATE_DO_IOCSR; 215 /* Save data and let user space to write it */ 216 memcpy(run->iocsr_io.data, val, run->iocsr_io.len); 217 } 218 trace_kvm_iocsr(KVM_TRACE_IOCSR_WRITE, run->iocsr_io.len, addr, val); 219 } else { 220 vcpu->arch.io_gpr = rd; /* Set register id for iocsr read completion */ 221 idx = srcu_read_lock(&vcpu->kvm->srcu); 222 ret = kvm_io_bus_read(vcpu, KVM_IOCSR_BUS, addr, 223 run->iocsr_io.len, run->iocsr_io.data); 224 srcu_read_unlock(&vcpu->kvm->srcu, idx); 225 if (ret == 0) { 226 kvm_complete_iocsr_read(vcpu, run); 227 ret = EMULATE_DONE; 228 } else 229 ret = EMULATE_DO_IOCSR; 230 trace_kvm_iocsr(KVM_TRACE_IOCSR_READ, run->iocsr_io.len, addr, NULL); 231 } 232 233 return ret; 234 } 235 236 int kvm_complete_iocsr_read(struct kvm_vcpu *vcpu, struct kvm_run *run) 237 { 238 enum emulation_result er = EMULATE_DONE; 239 unsigned long *gpr = &vcpu->arch.gprs[vcpu->arch.io_gpr]; 240 241 switch (run->iocsr_io.len) { 242 case 1: 243 *gpr = *(s8 *)run->iocsr_io.data; 244 break; 245 case 2: 246 *gpr = *(s16 *)run->iocsr_io.data; 247 break; 248 case 4: 249 *gpr = *(s32 *)run->iocsr_io.data; 250 break; 251 case 8: 252 *gpr = *(s64 *)run->iocsr_io.data; 253 break; 254 default: 255 kvm_pr_unimpl("Bad IOCSR length: %d, addr is 0x%lx\n", 256 run->iocsr_io.len, vcpu->arch.badv); 257 er = EMULATE_FAIL; 258 break; 259 } 260 261 return er; 262 } 263 264 int kvm_emu_idle(struct kvm_vcpu *vcpu) 265 { 266 ++vcpu->stat.idle_exits; 267 trace_kvm_exit_idle(vcpu, KVM_TRACE_EXIT_IDLE); 268 269 if (!kvm_arch_vcpu_runnable(vcpu)) 270 kvm_vcpu_halt(vcpu); 271 272 return EMULATE_DONE; 273 } 274 275 static int kvm_trap_handle_gspr(struct kvm_vcpu *vcpu) 276 { 277 unsigned long curr_pc; 278 larch_inst inst; 279 enum emulation_result er = EMULATE_DONE; 280 struct kvm_run *run = vcpu->run; 281 282 /* Fetch the instruction */ 283 inst.word = vcpu->arch.badi; 284 curr_pc = vcpu->arch.pc; 285 update_pc(&vcpu->arch); 286 287 trace_kvm_exit_gspr(vcpu, inst.word); 288 er = EMULATE_FAIL; 289 switch (((inst.word >> 24) & 0xff)) { 290 case 0x0: /* CPUCFG GSPR */ 291 trace_kvm_exit_cpucfg(vcpu, KVM_TRACE_EXIT_CPUCFG); 292 er = kvm_emu_cpucfg(vcpu, inst); 293 break; 294 case 0x4: /* CSR{RD,WR,XCHG} GSPR */ 295 trace_kvm_exit_csr(vcpu, KVM_TRACE_EXIT_CSR); 296 er = kvm_handle_csr(vcpu, inst); 297 break; 298 case 0x6: /* Cache, Idle and IOCSR GSPR */ 299 switch (((inst.word >> 22) & 0x3ff)) { 300 case 0x18: /* Cache GSPR */ 301 er = EMULATE_DONE; 302 trace_kvm_exit_cache(vcpu, KVM_TRACE_EXIT_CACHE); 303 break; 304 case 0x19: /* Idle/IOCSR GSPR */ 305 switch (((inst.word >> 15) & 0x1ffff)) { 306 case 0xc90: /* IOCSR GSPR */ 307 er = kvm_emu_iocsr(inst, run, vcpu); 308 break; 309 case 0xc91: /* Idle GSPR */ 310 er = kvm_emu_idle(vcpu); 311 break; 312 default: 313 er = EMULATE_FAIL; 314 break; 315 } 316 break; 317 default: 318 er = EMULATE_FAIL; 319 break; 320 } 321 break; 322 default: 323 er = EMULATE_FAIL; 324 break; 325 } 326 327 /* Rollback PC only if emulation was unsuccessful */ 328 if (er == EMULATE_FAIL) { 329 kvm_pr_unimpl("[%#lx]%s: unsupported gspr instruction 0x%08x\n", 330 curr_pc, __func__, inst.word); 331 332 kvm_arch_vcpu_dump_regs(vcpu); 333 vcpu->arch.pc = curr_pc; 334 } 335 336 return er; 337 } 338 339 /* 340 * Trigger GSPR: 341 * 1) Execute CPUCFG instruction; 342 * 2) Execute CACOP/IDLE instructions; 343 * 3) Access to unimplemented CSRs/IOCSRs. 344 */ 345 static int kvm_handle_gspr(struct kvm_vcpu *vcpu, int ecode) 346 { 347 int ret = RESUME_GUEST; 348 enum emulation_result er = EMULATE_DONE; 349 350 er = kvm_trap_handle_gspr(vcpu); 351 352 if (er == EMULATE_DONE) { 353 ret = RESUME_GUEST; 354 } else if (er == EMULATE_DO_MMIO) { 355 vcpu->run->exit_reason = KVM_EXIT_MMIO; 356 ret = RESUME_HOST; 357 } else if (er == EMULATE_DO_IOCSR) { 358 vcpu->run->exit_reason = KVM_EXIT_LOONGARCH_IOCSR; 359 ret = RESUME_HOST; 360 } else { 361 kvm_queue_exception(vcpu, EXCCODE_INE, 0); 362 ret = RESUME_GUEST; 363 } 364 365 return ret; 366 } 367 368 int kvm_emu_mmio_read(struct kvm_vcpu *vcpu, larch_inst inst) 369 { 370 int idx, ret; 371 unsigned int op8, opcode, rd; 372 struct kvm_run *run = vcpu->run; 373 374 run->mmio.phys_addr = vcpu->arch.badv; 375 vcpu->mmio_needed = 2; /* signed */ 376 op8 = (inst.word >> 24) & 0xff; 377 ret = EMULATE_DO_MMIO; 378 379 switch (op8) { 380 case 0x24 ... 0x27: /* ldptr.w/d process */ 381 rd = inst.reg2i14_format.rd; 382 opcode = inst.reg2i14_format.opcode; 383 384 switch (opcode) { 385 case ldptrw_op: 386 run->mmio.len = 4; 387 break; 388 case ldptrd_op: 389 run->mmio.len = 8; 390 break; 391 default: 392 ret = EMULATE_FAIL; 393 break; 394 } 395 break; 396 case 0x28 ... 0x2e: /* ld.b/h/w/d, ld.bu/hu/wu process */ 397 rd = inst.reg2i12_format.rd; 398 opcode = inst.reg2i12_format.opcode; 399 400 switch (opcode) { 401 case ldb_op: 402 run->mmio.len = 1; 403 break; 404 case ldbu_op: 405 vcpu->mmio_needed = 1; /* unsigned */ 406 run->mmio.len = 1; 407 break; 408 case ldh_op: 409 run->mmio.len = 2; 410 break; 411 case ldhu_op: 412 vcpu->mmio_needed = 1; /* unsigned */ 413 run->mmio.len = 2; 414 break; 415 case ldw_op: 416 run->mmio.len = 4; 417 break; 418 case ldwu_op: 419 vcpu->mmio_needed = 1; /* unsigned */ 420 run->mmio.len = 4; 421 break; 422 case ldd_op: 423 run->mmio.len = 8; 424 break; 425 default: 426 ret = EMULATE_FAIL; 427 break; 428 } 429 break; 430 case 0x38: /* ldx.b/h/w/d, ldx.bu/hu/wu process */ 431 rd = inst.reg3_format.rd; 432 opcode = inst.reg3_format.opcode; 433 434 switch (opcode) { 435 case ldxb_op: 436 run->mmio.len = 1; 437 break; 438 case ldxbu_op: 439 run->mmio.len = 1; 440 vcpu->mmio_needed = 1; /* unsigned */ 441 break; 442 case ldxh_op: 443 run->mmio.len = 2; 444 break; 445 case ldxhu_op: 446 run->mmio.len = 2; 447 vcpu->mmio_needed = 1; /* unsigned */ 448 break; 449 case ldxw_op: 450 run->mmio.len = 4; 451 break; 452 case ldxwu_op: 453 run->mmio.len = 4; 454 vcpu->mmio_needed = 1; /* unsigned */ 455 break; 456 case ldxd_op: 457 run->mmio.len = 8; 458 break; 459 default: 460 ret = EMULATE_FAIL; 461 break; 462 } 463 break; 464 default: 465 ret = EMULATE_FAIL; 466 } 467 468 if (ret == EMULATE_DO_MMIO) { 469 trace_kvm_mmio(KVM_TRACE_MMIO_READ, run->mmio.len, run->mmio.phys_addr, NULL); 470 471 vcpu->arch.io_gpr = rd; /* Set for kvm_complete_mmio_read() use */ 472 473 /* 474 * If mmio device such as PCH-PIC is emulated in KVM, 475 * it need not return to user space to handle the mmio 476 * exception. 477 */ 478 idx = srcu_read_lock(&vcpu->kvm->srcu); 479 ret = kvm_io_bus_read(vcpu, KVM_MMIO_BUS, vcpu->arch.badv, 480 run->mmio.len, run->mmio.data); 481 srcu_read_unlock(&vcpu->kvm->srcu, idx); 482 if (!ret) { 483 kvm_complete_mmio_read(vcpu, run); 484 vcpu->mmio_needed = 0; 485 return EMULATE_DONE; 486 } 487 488 run->mmio.is_write = 0; 489 vcpu->mmio_is_write = 0; 490 return EMULATE_DO_MMIO; 491 } 492 493 kvm_pr_unimpl("Read not supported Inst=0x%08x @%lx BadVaddr:%#lx\n", 494 inst.word, vcpu->arch.pc, vcpu->arch.badv); 495 kvm_arch_vcpu_dump_regs(vcpu); 496 vcpu->mmio_needed = 0; 497 498 return ret; 499 } 500 501 int kvm_complete_mmio_read(struct kvm_vcpu *vcpu, struct kvm_run *run) 502 { 503 enum emulation_result er = EMULATE_DONE; 504 unsigned long *gpr = &vcpu->arch.gprs[vcpu->arch.io_gpr]; 505 506 /* Update with new PC */ 507 update_pc(&vcpu->arch); 508 switch (run->mmio.len) { 509 case 1: 510 if (vcpu->mmio_needed == 2) 511 *gpr = *(s8 *)run->mmio.data; 512 else 513 *gpr = *(u8 *)run->mmio.data; 514 break; 515 case 2: 516 if (vcpu->mmio_needed == 2) 517 *gpr = *(s16 *)run->mmio.data; 518 else 519 *gpr = *(u16 *)run->mmio.data; 520 break; 521 case 4: 522 if (vcpu->mmio_needed == 2) 523 *gpr = *(s32 *)run->mmio.data; 524 else 525 *gpr = *(u32 *)run->mmio.data; 526 break; 527 case 8: 528 *gpr = *(s64 *)run->mmio.data; 529 break; 530 default: 531 kvm_pr_unimpl("Bad MMIO length: %d, addr is 0x%lx\n", 532 run->mmio.len, vcpu->arch.badv); 533 er = EMULATE_FAIL; 534 break; 535 } 536 537 trace_kvm_mmio(KVM_TRACE_MMIO_READ, run->mmio.len, 538 run->mmio.phys_addr, run->mmio.data); 539 540 return er; 541 } 542 543 int kvm_emu_mmio_write(struct kvm_vcpu *vcpu, larch_inst inst) 544 { 545 int idx, ret; 546 unsigned int rd, op8, opcode; 547 unsigned long curr_pc, rd_val = 0; 548 struct kvm_run *run = vcpu->run; 549 void *data = run->mmio.data; 550 551 /* 552 * Update PC and hold onto current PC in case there is 553 * an error and we want to rollback the PC 554 */ 555 curr_pc = vcpu->arch.pc; 556 update_pc(&vcpu->arch); 557 558 op8 = (inst.word >> 24) & 0xff; 559 run->mmio.phys_addr = vcpu->arch.badv; 560 ret = EMULATE_DO_MMIO; 561 switch (op8) { 562 case 0x24 ... 0x27: /* stptr.w/d process */ 563 rd = inst.reg2i14_format.rd; 564 opcode = inst.reg2i14_format.opcode; 565 566 switch (opcode) { 567 case stptrw_op: 568 run->mmio.len = 4; 569 *(unsigned int *)data = vcpu->arch.gprs[rd]; 570 break; 571 case stptrd_op: 572 run->mmio.len = 8; 573 *(unsigned long *)data = vcpu->arch.gprs[rd]; 574 break; 575 default: 576 ret = EMULATE_FAIL; 577 break; 578 } 579 break; 580 case 0x28 ... 0x2e: /* st.b/h/w/d process */ 581 rd = inst.reg2i12_format.rd; 582 opcode = inst.reg2i12_format.opcode; 583 rd_val = vcpu->arch.gprs[rd]; 584 585 switch (opcode) { 586 case stb_op: 587 run->mmio.len = 1; 588 *(unsigned char *)data = rd_val; 589 break; 590 case sth_op: 591 run->mmio.len = 2; 592 *(unsigned short *)data = rd_val; 593 break; 594 case stw_op: 595 run->mmio.len = 4; 596 *(unsigned int *)data = rd_val; 597 break; 598 case std_op: 599 run->mmio.len = 8; 600 *(unsigned long *)data = rd_val; 601 break; 602 default: 603 ret = EMULATE_FAIL; 604 break; 605 } 606 break; 607 case 0x38: /* stx.b/h/w/d process */ 608 rd = inst.reg3_format.rd; 609 opcode = inst.reg3_format.opcode; 610 611 switch (opcode) { 612 case stxb_op: 613 run->mmio.len = 1; 614 *(unsigned char *)data = vcpu->arch.gprs[rd]; 615 break; 616 case stxh_op: 617 run->mmio.len = 2; 618 *(unsigned short *)data = vcpu->arch.gprs[rd]; 619 break; 620 case stxw_op: 621 run->mmio.len = 4; 622 *(unsigned int *)data = vcpu->arch.gprs[rd]; 623 break; 624 case stxd_op: 625 run->mmio.len = 8; 626 *(unsigned long *)data = vcpu->arch.gprs[rd]; 627 break; 628 default: 629 ret = EMULATE_FAIL; 630 break; 631 } 632 break; 633 default: 634 ret = EMULATE_FAIL; 635 } 636 637 if (ret == EMULATE_DO_MMIO) { 638 trace_kvm_mmio(KVM_TRACE_MMIO_WRITE, run->mmio.len, run->mmio.phys_addr, data); 639 640 /* 641 * If mmio device such as PCH-PIC is emulated in KVM, 642 * it need not return to user space to handle the mmio 643 * exception. 644 */ 645 idx = srcu_read_lock(&vcpu->kvm->srcu); 646 ret = kvm_io_bus_write(vcpu, KVM_MMIO_BUS, vcpu->arch.badv, run->mmio.len, data); 647 srcu_read_unlock(&vcpu->kvm->srcu, idx); 648 if (!ret) 649 return EMULATE_DONE; 650 651 run->mmio.is_write = 1; 652 vcpu->mmio_needed = 1; 653 vcpu->mmio_is_write = 1; 654 return EMULATE_DO_MMIO; 655 } 656 657 vcpu->arch.pc = curr_pc; 658 kvm_pr_unimpl("Write not supported Inst=0x%08x @%lx BadVaddr:%#lx\n", 659 inst.word, vcpu->arch.pc, vcpu->arch.badv); 660 kvm_arch_vcpu_dump_regs(vcpu); 661 /* Rollback PC if emulation was unsuccessful */ 662 663 return ret; 664 } 665 666 static int kvm_handle_rdwr_fault(struct kvm_vcpu *vcpu, bool write, int ecode) 667 { 668 int ret; 669 larch_inst inst; 670 enum emulation_result er = EMULATE_DONE; 671 struct kvm_run *run = vcpu->run; 672 unsigned long badv = vcpu->arch.badv; 673 674 /* Inject ADE exception if exceed max GPA size */ 675 if (unlikely(badv >= vcpu->kvm->arch.gpa_size)) { 676 kvm_queue_exception(vcpu, EXCCODE_ADE, EXSUBCODE_ADEM); 677 return RESUME_GUEST; 678 } 679 680 ret = kvm_handle_mm_fault(vcpu, badv, write, ecode); 681 if (ret) { 682 /* Treat as MMIO */ 683 inst.word = vcpu->arch.badi; 684 if (write) { 685 er = kvm_emu_mmio_write(vcpu, inst); 686 } else { 687 /* A code fetch fault doesn't count as an MMIO */ 688 if (kvm_is_ifetch_fault(&vcpu->arch)) { 689 kvm_queue_exception(vcpu, EXCCODE_ADE, EXSUBCODE_ADEF); 690 return RESUME_GUEST; 691 } 692 693 er = kvm_emu_mmio_read(vcpu, inst); 694 } 695 } 696 697 if (er == EMULATE_DONE) { 698 ret = RESUME_GUEST; 699 } else if (er == EMULATE_DO_MMIO) { 700 run->exit_reason = KVM_EXIT_MMIO; 701 ret = RESUME_HOST; 702 } else { 703 kvm_queue_exception(vcpu, EXCCODE_ADE, EXSUBCODE_ADEM); 704 ret = RESUME_GUEST; 705 } 706 707 return ret; 708 } 709 710 static int kvm_handle_read_fault(struct kvm_vcpu *vcpu, int ecode) 711 { 712 return kvm_handle_rdwr_fault(vcpu, false, ecode); 713 } 714 715 static int kvm_handle_write_fault(struct kvm_vcpu *vcpu, int ecode) 716 { 717 return kvm_handle_rdwr_fault(vcpu, true, ecode); 718 } 719 720 int kvm_complete_user_service(struct kvm_vcpu *vcpu, struct kvm_run *run) 721 { 722 update_pc(&vcpu->arch); 723 kvm_write_reg(vcpu, LOONGARCH_GPR_A0, run->hypercall.ret); 724 725 return 0; 726 } 727 728 /** 729 * kvm_handle_fpu_disabled() - Guest used fpu however it is disabled at host 730 * @vcpu: Virtual CPU context. 731 * @ecode: Exception code. 732 * 733 * Handle when the guest attempts to use fpu which hasn't been allowed 734 * by the root context. 735 */ 736 static int kvm_handle_fpu_disabled(struct kvm_vcpu *vcpu, int ecode) 737 { 738 struct kvm_run *run = vcpu->run; 739 740 if (!kvm_guest_has_fpu(&vcpu->arch)) { 741 kvm_queue_exception(vcpu, EXCCODE_INE, 0); 742 return RESUME_GUEST; 743 } 744 745 /* 746 * If guest FPU not present, the FPU operation should have been 747 * treated as a reserved instruction! 748 * If FPU already in use, we shouldn't get this at all. 749 */ 750 if (vcpu->arch.aux_inuse & KVM_LARCH_FPU) { 751 kvm_pr_unimpl("%s internal error\n", __func__); 752 run->exit_reason = KVM_EXIT_INTERNAL_ERROR; 753 return RESUME_HOST; 754 } 755 756 kvm_make_request(KVM_REQ_FPU_LOAD, vcpu); 757 758 return RESUME_GUEST; 759 } 760 761 static long kvm_save_notify(struct kvm_vcpu *vcpu) 762 { 763 unsigned long id, data; 764 765 id = kvm_read_reg(vcpu, LOONGARCH_GPR_A1); 766 data = kvm_read_reg(vcpu, LOONGARCH_GPR_A2); 767 switch (id) { 768 case BIT(KVM_FEATURE_STEAL_TIME): 769 if (data & ~(KVM_STEAL_PHYS_MASK | KVM_STEAL_PHYS_VALID)) 770 return KVM_HCALL_INVALID_PARAMETER; 771 772 vcpu->arch.st.guest_addr = data; 773 if (!(data & KVM_STEAL_PHYS_VALID)) 774 return 0; 775 776 vcpu->arch.st.last_steal = current->sched_info.run_delay; 777 kvm_make_request(KVM_REQ_STEAL_UPDATE, vcpu); 778 return 0; 779 default: 780 return KVM_HCALL_INVALID_CODE; 781 } 782 } 783 784 /* 785 * kvm_handle_lsx_disabled() - Guest used LSX while disabled in root. 786 * @vcpu: Virtual CPU context. 787 * @ecode: Exception code. 788 * 789 * Handle when the guest attempts to use LSX when it is disabled in the root 790 * context. 791 */ 792 static int kvm_handle_lsx_disabled(struct kvm_vcpu *vcpu, int ecode) 793 { 794 if (!kvm_guest_has_lsx(&vcpu->arch)) 795 kvm_queue_exception(vcpu, EXCCODE_INE, 0); 796 else 797 kvm_make_request(KVM_REQ_FPU_LOAD, vcpu); 798 799 return RESUME_GUEST; 800 } 801 802 /* 803 * kvm_handle_lasx_disabled() - Guest used LASX while disabled in root. 804 * @vcpu: Virtual CPU context. 805 * @ecode: Exception code. 806 * 807 * Handle when the guest attempts to use LASX when it is disabled in the root 808 * context. 809 */ 810 static int kvm_handle_lasx_disabled(struct kvm_vcpu *vcpu, int ecode) 811 { 812 if (!kvm_guest_has_lasx(&vcpu->arch)) 813 kvm_queue_exception(vcpu, EXCCODE_INE, 0); 814 else 815 kvm_make_request(KVM_REQ_FPU_LOAD, vcpu); 816 817 return RESUME_GUEST; 818 } 819 820 static int kvm_handle_lbt_disabled(struct kvm_vcpu *vcpu, int ecode) 821 { 822 if (!kvm_guest_has_lbt(&vcpu->arch)) 823 kvm_queue_exception(vcpu, EXCCODE_INE, 0); 824 else 825 kvm_make_request(KVM_REQ_LBT_LOAD, vcpu); 826 827 return RESUME_GUEST; 828 } 829 830 static void kvm_send_pv_ipi(struct kvm_vcpu *vcpu) 831 { 832 unsigned int min, cpu; 833 struct kvm_vcpu *dest; 834 DECLARE_BITMAP(ipi_bitmap, BITS_PER_LONG * 2) = { 835 kvm_read_reg(vcpu, LOONGARCH_GPR_A1), 836 kvm_read_reg(vcpu, LOONGARCH_GPR_A2) 837 }; 838 839 min = kvm_read_reg(vcpu, LOONGARCH_GPR_A3); 840 for_each_set_bit(cpu, ipi_bitmap, BITS_PER_LONG * 2) { 841 dest = kvm_get_vcpu_by_cpuid(vcpu->kvm, cpu + min); 842 if (!dest) 843 continue; 844 845 /* Send SWI0 to dest vcpu to emulate IPI interrupt */ 846 kvm_queue_irq(dest, INT_SWI0); 847 kvm_vcpu_kick(dest); 848 } 849 } 850 851 /* 852 * Hypercall emulation always return to guest, Caller should check retval. 853 */ 854 static void kvm_handle_service(struct kvm_vcpu *vcpu) 855 { 856 long ret = KVM_HCALL_INVALID_CODE; 857 unsigned long func = kvm_read_reg(vcpu, LOONGARCH_GPR_A0); 858 859 switch (func) { 860 case KVM_HCALL_FUNC_IPI: 861 if (kvm_guest_has_pv_feature(vcpu, KVM_FEATURE_IPI)) { 862 kvm_send_pv_ipi(vcpu); 863 ret = KVM_HCALL_SUCCESS; 864 } 865 break; 866 case KVM_HCALL_FUNC_NOTIFY: 867 if (kvm_guest_has_pv_feature(vcpu, KVM_FEATURE_STEAL_TIME)) 868 ret = kvm_save_notify(vcpu); 869 break; 870 default: 871 break; 872 } 873 874 kvm_write_reg(vcpu, LOONGARCH_GPR_A0, ret); 875 } 876 877 static int kvm_handle_hypercall(struct kvm_vcpu *vcpu, int ecode) 878 { 879 int ret; 880 larch_inst inst; 881 unsigned int code; 882 883 inst.word = vcpu->arch.badi; 884 code = inst.reg0i15_format.immediate; 885 ret = RESUME_GUEST; 886 887 switch (code) { 888 case KVM_HCALL_SERVICE: 889 vcpu->stat.hypercall_exits++; 890 kvm_handle_service(vcpu); 891 break; 892 case KVM_HCALL_USER_SERVICE: 893 if (!kvm_guest_has_pv_feature(vcpu, KVM_FEATURE_USER_HCALL)) { 894 kvm_write_reg(vcpu, LOONGARCH_GPR_A0, KVM_HCALL_INVALID_CODE); 895 break; 896 } 897 898 vcpu->stat.hypercall_exits++; 899 vcpu->run->exit_reason = KVM_EXIT_HYPERCALL; 900 vcpu->run->hypercall.nr = KVM_HCALL_USER_SERVICE; 901 vcpu->run->hypercall.args[0] = kvm_read_reg(vcpu, LOONGARCH_GPR_A0); 902 vcpu->run->hypercall.args[1] = kvm_read_reg(vcpu, LOONGARCH_GPR_A1); 903 vcpu->run->hypercall.args[2] = kvm_read_reg(vcpu, LOONGARCH_GPR_A2); 904 vcpu->run->hypercall.args[3] = kvm_read_reg(vcpu, LOONGARCH_GPR_A3); 905 vcpu->run->hypercall.args[4] = kvm_read_reg(vcpu, LOONGARCH_GPR_A4); 906 vcpu->run->hypercall.args[5] = kvm_read_reg(vcpu, LOONGARCH_GPR_A5); 907 vcpu->run->hypercall.flags = 0; 908 /* 909 * Set invalid return value by default, let user-mode VMM modify it. 910 */ 911 vcpu->run->hypercall.ret = KVM_HCALL_INVALID_CODE; 912 ret = RESUME_HOST; 913 break; 914 case KVM_HCALL_SWDBG: 915 /* KVM_HCALL_SWDBG only in effective when SW_BP is enabled */ 916 if (vcpu->guest_debug & KVM_GUESTDBG_SW_BP_MASK) { 917 vcpu->run->exit_reason = KVM_EXIT_DEBUG; 918 ret = RESUME_HOST; 919 break; 920 } 921 fallthrough; 922 default: 923 /* Treat it as noop intruction, only set return value */ 924 kvm_write_reg(vcpu, LOONGARCH_GPR_A0, KVM_HCALL_INVALID_CODE); 925 break; 926 } 927 928 if (ret == RESUME_GUEST) 929 update_pc(&vcpu->arch); 930 931 return ret; 932 } 933 934 /* 935 * LoongArch KVM callback handling for unimplemented guest exiting 936 */ 937 static int kvm_fault_ni(struct kvm_vcpu *vcpu, int ecode) 938 { 939 unsigned int inst; 940 unsigned long badv; 941 942 /* Fetch the instruction */ 943 inst = vcpu->arch.badi; 944 badv = vcpu->arch.badv; 945 kvm_pr_unimpl("ECode: %d PC=%#lx Inst=0x%08x BadVaddr=%#lx ESTAT=%#lx\n", 946 ecode, vcpu->arch.pc, inst, badv, read_gcsr_estat()); 947 kvm_arch_vcpu_dump_regs(vcpu); 948 kvm_queue_exception(vcpu, EXCCODE_INE, 0); 949 950 return RESUME_GUEST; 951 } 952 953 static exit_handle_fn kvm_fault_tables[EXCCODE_INT_START] = { 954 [0 ... EXCCODE_INT_START - 1] = kvm_fault_ni, 955 [EXCCODE_TLBI] = kvm_handle_read_fault, 956 [EXCCODE_TLBL] = kvm_handle_read_fault, 957 [EXCCODE_TLBS] = kvm_handle_write_fault, 958 [EXCCODE_TLBM] = kvm_handle_write_fault, 959 [EXCCODE_FPDIS] = kvm_handle_fpu_disabled, 960 [EXCCODE_LSXDIS] = kvm_handle_lsx_disabled, 961 [EXCCODE_LASXDIS] = kvm_handle_lasx_disabled, 962 [EXCCODE_BTDIS] = kvm_handle_lbt_disabled, 963 [EXCCODE_GSPR] = kvm_handle_gspr, 964 [EXCCODE_HVC] = kvm_handle_hypercall, 965 }; 966 967 int kvm_handle_fault(struct kvm_vcpu *vcpu, int fault) 968 { 969 return kvm_fault_tables[fault](vcpu, fault); 970 } 971