1 /* 2 * This file is subject to the terms and conditions of the GNU General Public 3 * License. See the file "COPYING" in the main directory of this archive 4 * for more details. 5 * 6 * KVM/MIPS: Instruction/Exception emulation 7 * 8 * Copyright (C) 2012 MIPS Technologies, Inc. All rights reserved. 9 * Authors: Sanjay Lal <sanjayl@kymasys.com> 10 */ 11 12 #include <linux/errno.h> 13 #include <linux/err.h> 14 #include <linux/ktime.h> 15 #include <linux/kvm_host.h> 16 #include <linux/vmalloc.h> 17 #include <linux/fs.h> 18 #include <linux/memblock.h> 19 #include <linux/random.h> 20 #include <asm/page.h> 21 #include <asm/cacheflush.h> 22 #include <asm/cacheops.h> 23 #include <asm/cpu-info.h> 24 #include <asm/mmu_context.h> 25 #include <asm/tlbflush.h> 26 #include <asm/inst.h> 27 28 #undef CONFIG_MIPS_MT 29 #include <asm/r4kcache.h> 30 #define CONFIG_MIPS_MT 31 32 #include "interrupt.h" 33 34 #include "trace.h" 35 36 /* 37 * Compute the return address and do emulate branch simulation, if required. 38 * This function should be called only in branch delay slot active. 39 */ 40 static int kvm_compute_return_epc(struct kvm_vcpu *vcpu, unsigned long instpc, 41 unsigned long *out) 42 { 43 unsigned int dspcontrol; 44 union mips_instruction insn; 45 struct kvm_vcpu_arch *arch = &vcpu->arch; 46 long epc = instpc; 47 long nextpc; 48 int err; 49 50 if (epc & 3) { 51 kvm_err("%s: unaligned epc\n", __func__); 52 return -EINVAL; 53 } 54 55 /* Read the instruction */ 56 err = kvm_get_badinstrp((u32 *)epc, vcpu, &insn.word); 57 if (err) 58 return err; 59 60 switch (insn.i_format.opcode) { 61 /* jr and jalr are in r_format format. */ 62 case spec_op: 63 switch (insn.r_format.func) { 64 case jalr_op: 65 arch->gprs[insn.r_format.rd] = epc + 8; 66 fallthrough; 67 case jr_op: 68 nextpc = arch->gprs[insn.r_format.rs]; 69 break; 70 default: 71 return -EINVAL; 72 } 73 break; 74 75 /* 76 * This group contains: 77 * bltz_op, bgez_op, bltzl_op, bgezl_op, 78 * bltzal_op, bgezal_op, bltzall_op, bgezall_op. 79 */ 80 case bcond_op: 81 switch (insn.i_format.rt) { 82 case bltz_op: 83 case bltzl_op: 84 if ((long)arch->gprs[insn.i_format.rs] < 0) 85 epc = epc + 4 + (insn.i_format.simmediate << 2); 86 else 87 epc += 8; 88 nextpc = epc; 89 break; 90 91 case bgez_op: 92 case bgezl_op: 93 if ((long)arch->gprs[insn.i_format.rs] >= 0) 94 epc = epc + 4 + (insn.i_format.simmediate << 2); 95 else 96 epc += 8; 97 nextpc = epc; 98 break; 99 100 case bltzal_op: 101 case bltzall_op: 102 arch->gprs[31] = epc + 8; 103 if ((long)arch->gprs[insn.i_format.rs] < 0) 104 epc = epc + 4 + (insn.i_format.simmediate << 2); 105 else 106 epc += 8; 107 nextpc = epc; 108 break; 109 110 case bgezal_op: 111 case bgezall_op: 112 arch->gprs[31] = epc + 8; 113 if ((long)arch->gprs[insn.i_format.rs] >= 0) 114 epc = epc + 4 + (insn.i_format.simmediate << 2); 115 else 116 epc += 8; 117 nextpc = epc; 118 break; 119 case bposge32_op: 120 if (!cpu_has_dsp) { 121 kvm_err("%s: DSP branch but not DSP ASE\n", 122 __func__); 123 return -EINVAL; 124 } 125 126 dspcontrol = rddsp(0x01); 127 128 if (dspcontrol >= 32) 129 epc = epc + 4 + (insn.i_format.simmediate << 2); 130 else 131 epc += 8; 132 nextpc = epc; 133 break; 134 default: 135 return -EINVAL; 136 } 137 break; 138 139 /* These are unconditional and in j_format. */ 140 case jal_op: 141 arch->gprs[31] = instpc + 8; 142 fallthrough; 143 case j_op: 144 epc += 4; 145 epc >>= 28; 146 epc <<= 28; 147 epc |= (insn.j_format.target << 2); 148 nextpc = epc; 149 break; 150 151 /* These are conditional and in i_format. */ 152 case beq_op: 153 case beql_op: 154 if (arch->gprs[insn.i_format.rs] == 155 arch->gprs[insn.i_format.rt]) 156 epc = epc + 4 + (insn.i_format.simmediate << 2); 157 else 158 epc += 8; 159 nextpc = epc; 160 break; 161 162 case bne_op: 163 case bnel_op: 164 if (arch->gprs[insn.i_format.rs] != 165 arch->gprs[insn.i_format.rt]) 166 epc = epc + 4 + (insn.i_format.simmediate << 2); 167 else 168 epc += 8; 169 nextpc = epc; 170 break; 171 172 case blez_op: /* POP06 */ 173 #ifndef CONFIG_CPU_MIPSR6 174 case blezl_op: /* removed in R6 */ 175 #endif 176 if (insn.i_format.rt != 0) 177 goto compact_branch; 178 if ((long)arch->gprs[insn.i_format.rs] <= 0) 179 epc = epc + 4 + (insn.i_format.simmediate << 2); 180 else 181 epc += 8; 182 nextpc = epc; 183 break; 184 185 case bgtz_op: /* POP07 */ 186 #ifndef CONFIG_CPU_MIPSR6 187 case bgtzl_op: /* removed in R6 */ 188 #endif 189 if (insn.i_format.rt != 0) 190 goto compact_branch; 191 if ((long)arch->gprs[insn.i_format.rs] > 0) 192 epc = epc + 4 + (insn.i_format.simmediate << 2); 193 else 194 epc += 8; 195 nextpc = epc; 196 break; 197 198 /* And now the FPA/cp1 branch instructions. */ 199 case cop1_op: 200 kvm_err("%s: unsupported cop1_op\n", __func__); 201 return -EINVAL; 202 203 #ifdef CONFIG_CPU_MIPSR6 204 /* R6 added the following compact branches with forbidden slots */ 205 case blezl_op: /* POP26 */ 206 case bgtzl_op: /* POP27 */ 207 /* only rt == 0 isn't compact branch */ 208 if (insn.i_format.rt != 0) 209 goto compact_branch; 210 return -EINVAL; 211 case pop10_op: 212 case pop30_op: 213 /* only rs == rt == 0 is reserved, rest are compact branches */ 214 if (insn.i_format.rs != 0 || insn.i_format.rt != 0) 215 goto compact_branch; 216 return -EINVAL; 217 case pop66_op: 218 case pop76_op: 219 /* only rs == 0 isn't compact branch */ 220 if (insn.i_format.rs != 0) 221 goto compact_branch; 222 return -EINVAL; 223 compact_branch: 224 /* 225 * If we've hit an exception on the forbidden slot, then 226 * the branch must not have been taken. 227 */ 228 epc += 8; 229 nextpc = epc; 230 break; 231 #else 232 compact_branch: 233 /* Fall through - Compact branches not supported before R6 */ 234 #endif 235 default: 236 return -EINVAL; 237 } 238 239 *out = nextpc; 240 return 0; 241 } 242 243 enum emulation_result update_pc(struct kvm_vcpu *vcpu, u32 cause) 244 { 245 int err; 246 247 if (cause & CAUSEF_BD) { 248 err = kvm_compute_return_epc(vcpu, vcpu->arch.pc, 249 &vcpu->arch.pc); 250 if (err) 251 return EMULATE_FAIL; 252 } else { 253 vcpu->arch.pc += 4; 254 } 255 256 kvm_debug("update_pc(): New PC: %#lx\n", vcpu->arch.pc); 257 258 return EMULATE_DONE; 259 } 260 261 /** 262 * kvm_get_badinstr() - Get bad instruction encoding. 263 * @opc: Guest pointer to faulting instruction. 264 * @vcpu: KVM VCPU information. 265 * 266 * Gets the instruction encoding of the faulting instruction, using the saved 267 * BadInstr register value if it exists, otherwise falling back to reading guest 268 * memory at @opc. 269 * 270 * Returns: The instruction encoding of the faulting instruction. 271 */ 272 int kvm_get_badinstr(u32 *opc, struct kvm_vcpu *vcpu, u32 *out) 273 { 274 if (cpu_has_badinstr) { 275 *out = vcpu->arch.host_cp0_badinstr; 276 return 0; 277 } else { 278 WARN_ONCE(1, "CPU doesn't have BadInstr register\n"); 279 return -EINVAL; 280 } 281 } 282 283 /** 284 * kvm_get_badinstrp() - Get bad prior instruction encoding. 285 * @opc: Guest pointer to prior faulting instruction. 286 * @vcpu: KVM VCPU information. 287 * 288 * Gets the instruction encoding of the prior faulting instruction (the branch 289 * containing the delay slot which faulted), using the saved BadInstrP register 290 * value if it exists, otherwise falling back to reading guest memory at @opc. 291 * 292 * Returns: The instruction encoding of the prior faulting instruction. 293 */ 294 int kvm_get_badinstrp(u32 *opc, struct kvm_vcpu *vcpu, u32 *out) 295 { 296 if (cpu_has_badinstrp) { 297 *out = vcpu->arch.host_cp0_badinstrp; 298 return 0; 299 } else { 300 WARN_ONCE(1, "CPU doesn't have BadInstrp register\n"); 301 return -EINVAL; 302 } 303 } 304 305 /** 306 * kvm_mips_count_disabled() - Find whether the CP0_Count timer is disabled. 307 * @vcpu: Virtual CPU. 308 * 309 * Returns: 1 if the CP0_Count timer is disabled by either the guest 310 * CP0_Cause.DC bit or the count_ctl.DC bit. 311 * 0 otherwise (in which case CP0_Count timer is running). 312 */ 313 int kvm_mips_count_disabled(struct kvm_vcpu *vcpu) 314 { 315 struct mips_coproc *cop0 = vcpu->arch.cop0; 316 317 return (vcpu->arch.count_ctl & KVM_REG_MIPS_COUNT_CTL_DC) || 318 (kvm_read_c0_guest_cause(cop0) & CAUSEF_DC); 319 } 320 321 /** 322 * kvm_mips_ktime_to_count() - Scale ktime_t to a 32-bit count. 323 * 324 * Caches the dynamic nanosecond bias in vcpu->arch.count_dyn_bias. 325 * 326 * Assumes !kvm_mips_count_disabled(@vcpu) (guest CP0_Count timer is running). 327 */ 328 static u32 kvm_mips_ktime_to_count(struct kvm_vcpu *vcpu, ktime_t now) 329 { 330 s64 now_ns, periods; 331 u64 delta; 332 333 now_ns = ktime_to_ns(now); 334 delta = now_ns + vcpu->arch.count_dyn_bias; 335 336 if (delta >= vcpu->arch.count_period) { 337 /* If delta is out of safe range the bias needs adjusting */ 338 periods = div64_s64(now_ns, vcpu->arch.count_period); 339 vcpu->arch.count_dyn_bias = -periods * vcpu->arch.count_period; 340 /* Recalculate delta with new bias */ 341 delta = now_ns + vcpu->arch.count_dyn_bias; 342 } 343 344 /* 345 * We've ensured that: 346 * delta < count_period 347 * 348 * Therefore the intermediate delta*count_hz will never overflow since 349 * at the boundary condition: 350 * delta = count_period 351 * delta = NSEC_PER_SEC * 2^32 / count_hz 352 * delta * count_hz = NSEC_PER_SEC * 2^32 353 */ 354 return div_u64(delta * vcpu->arch.count_hz, NSEC_PER_SEC); 355 } 356 357 /** 358 * kvm_mips_count_time() - Get effective current time. 359 * @vcpu: Virtual CPU. 360 * 361 * Get effective monotonic ktime. This is usually a straightforward ktime_get(), 362 * except when the master disable bit is set in count_ctl, in which case it is 363 * count_resume, i.e. the time that the count was disabled. 364 * 365 * Returns: Effective monotonic ktime for CP0_Count. 366 */ 367 static inline ktime_t kvm_mips_count_time(struct kvm_vcpu *vcpu) 368 { 369 if (unlikely(vcpu->arch.count_ctl & KVM_REG_MIPS_COUNT_CTL_DC)) 370 return vcpu->arch.count_resume; 371 372 return ktime_get(); 373 } 374 375 /** 376 * kvm_mips_read_count_running() - Read the current count value as if running. 377 * @vcpu: Virtual CPU. 378 * @now: Kernel time to read CP0_Count at. 379 * 380 * Returns the current guest CP0_Count register at time @now and handles if the 381 * timer interrupt is pending and hasn't been handled yet. 382 * 383 * Returns: The current value of the guest CP0_Count register. 384 */ 385 static u32 kvm_mips_read_count_running(struct kvm_vcpu *vcpu, ktime_t now) 386 { 387 struct mips_coproc *cop0 = vcpu->arch.cop0; 388 ktime_t expires, threshold; 389 u32 count, compare; 390 int running; 391 392 /* Calculate the biased and scaled guest CP0_Count */ 393 count = vcpu->arch.count_bias + kvm_mips_ktime_to_count(vcpu, now); 394 compare = kvm_read_c0_guest_compare(cop0); 395 396 /* 397 * Find whether CP0_Count has reached the closest timer interrupt. If 398 * not, we shouldn't inject it. 399 */ 400 if ((s32)(count - compare) < 0) 401 return count; 402 403 /* 404 * The CP0_Count we're going to return has already reached the closest 405 * timer interrupt. Quickly check if it really is a new interrupt by 406 * looking at whether the interval until the hrtimer expiry time is 407 * less than 1/4 of the timer period. 408 */ 409 expires = hrtimer_get_expires(&vcpu->arch.comparecount_timer); 410 threshold = ktime_add_ns(now, vcpu->arch.count_period / 4); 411 if (ktime_before(expires, threshold)) { 412 /* 413 * Cancel it while we handle it so there's no chance of 414 * interference with the timeout handler. 415 */ 416 running = hrtimer_cancel(&vcpu->arch.comparecount_timer); 417 418 /* Nothing should be waiting on the timeout */ 419 kvm_mips_callbacks->queue_timer_int(vcpu); 420 421 /* 422 * Restart the timer if it was running based on the expiry time 423 * we read, so that we don't push it back 2 periods. 424 */ 425 if (running) { 426 expires = ktime_add_ns(expires, 427 vcpu->arch.count_period); 428 hrtimer_start(&vcpu->arch.comparecount_timer, expires, 429 HRTIMER_MODE_ABS); 430 } 431 } 432 433 return count; 434 } 435 436 /** 437 * kvm_mips_read_count() - Read the current count value. 438 * @vcpu: Virtual CPU. 439 * 440 * Read the current guest CP0_Count value, taking into account whether the timer 441 * is stopped. 442 * 443 * Returns: The current guest CP0_Count value. 444 */ 445 u32 kvm_mips_read_count(struct kvm_vcpu *vcpu) 446 { 447 struct mips_coproc *cop0 = vcpu->arch.cop0; 448 449 /* If count disabled just read static copy of count */ 450 if (kvm_mips_count_disabled(vcpu)) 451 return kvm_read_c0_guest_count(cop0); 452 453 return kvm_mips_read_count_running(vcpu, ktime_get()); 454 } 455 456 /** 457 * kvm_mips_freeze_hrtimer() - Safely stop the hrtimer. 458 * @vcpu: Virtual CPU. 459 * @count: Output pointer for CP0_Count value at point of freeze. 460 * 461 * Freeze the hrtimer safely and return both the ktime and the CP0_Count value 462 * at the point it was frozen. It is guaranteed that any pending interrupts at 463 * the point it was frozen are handled, and none after that point. 464 * 465 * This is useful where the time/CP0_Count is needed in the calculation of the 466 * new parameters. 467 * 468 * Assumes !kvm_mips_count_disabled(@vcpu) (guest CP0_Count timer is running). 469 * 470 * Returns: The ktime at the point of freeze. 471 */ 472 ktime_t kvm_mips_freeze_hrtimer(struct kvm_vcpu *vcpu, u32 *count) 473 { 474 ktime_t now; 475 476 /* stop hrtimer before finding time */ 477 hrtimer_cancel(&vcpu->arch.comparecount_timer); 478 now = ktime_get(); 479 480 /* find count at this point and handle pending hrtimer */ 481 *count = kvm_mips_read_count_running(vcpu, now); 482 483 return now; 484 } 485 486 /** 487 * kvm_mips_resume_hrtimer() - Resume hrtimer, updating expiry. 488 * @vcpu: Virtual CPU. 489 * @now: ktime at point of resume. 490 * @count: CP0_Count at point of resume. 491 * 492 * Resumes the timer and updates the timer expiry based on @now and @count. 493 * This can be used in conjunction with kvm_mips_freeze_timer() when timer 494 * parameters need to be changed. 495 * 496 * It is guaranteed that a timer interrupt immediately after resume will be 497 * handled, but not if CP_Compare is exactly at @count. That case is already 498 * handled by kvm_mips_freeze_timer(). 499 * 500 * Assumes !kvm_mips_count_disabled(@vcpu) (guest CP0_Count timer is running). 501 */ 502 static void kvm_mips_resume_hrtimer(struct kvm_vcpu *vcpu, 503 ktime_t now, u32 count) 504 { 505 struct mips_coproc *cop0 = vcpu->arch.cop0; 506 u32 compare; 507 u64 delta; 508 ktime_t expire; 509 510 /* Calculate timeout (wrap 0 to 2^32) */ 511 compare = kvm_read_c0_guest_compare(cop0); 512 delta = (u64)(u32)(compare - count - 1) + 1; 513 delta = div_u64(delta * NSEC_PER_SEC, vcpu->arch.count_hz); 514 expire = ktime_add_ns(now, delta); 515 516 /* Update hrtimer to use new timeout */ 517 hrtimer_cancel(&vcpu->arch.comparecount_timer); 518 hrtimer_start(&vcpu->arch.comparecount_timer, expire, HRTIMER_MODE_ABS); 519 } 520 521 /** 522 * kvm_mips_restore_hrtimer() - Restore hrtimer after a gap, updating expiry. 523 * @vcpu: Virtual CPU. 524 * @before: Time before Count was saved, lower bound of drift calculation. 525 * @count: CP0_Count at point of restore. 526 * @min_drift: Minimum amount of drift permitted before correction. 527 * Must be <= 0. 528 * 529 * Restores the timer from a particular @count, accounting for drift. This can 530 * be used in conjunction with kvm_mips_freeze_timer() when a hardware timer is 531 * to be used for a period of time, but the exact ktime corresponding to the 532 * final Count that must be restored is not known. 533 * 534 * It is gauranteed that a timer interrupt immediately after restore will be 535 * handled, but not if CP0_Compare is exactly at @count. That case should 536 * already be handled when the hardware timer state is saved. 537 * 538 * Assumes !kvm_mips_count_disabled(@vcpu) (guest CP0_Count timer is not 539 * stopped). 540 * 541 * Returns: Amount of correction to count_bias due to drift. 542 */ 543 int kvm_mips_restore_hrtimer(struct kvm_vcpu *vcpu, ktime_t before, 544 u32 count, int min_drift) 545 { 546 ktime_t now, count_time; 547 u32 now_count, before_count; 548 u64 delta; 549 int drift, ret = 0; 550 551 /* Calculate expected count at before */ 552 before_count = vcpu->arch.count_bias + 553 kvm_mips_ktime_to_count(vcpu, before); 554 555 /* 556 * Detect significantly negative drift, where count is lower than 557 * expected. Some negative drift is expected when hardware counter is 558 * set after kvm_mips_freeze_timer(), and it is harmless to allow the 559 * time to jump forwards a little, within reason. If the drift is too 560 * significant, adjust the bias to avoid a big Guest.CP0_Count jump. 561 */ 562 drift = count - before_count; 563 if (drift < min_drift) { 564 count_time = before; 565 vcpu->arch.count_bias += drift; 566 ret = drift; 567 goto resume; 568 } 569 570 /* Calculate expected count right now */ 571 now = ktime_get(); 572 now_count = vcpu->arch.count_bias + kvm_mips_ktime_to_count(vcpu, now); 573 574 /* 575 * Detect positive drift, where count is higher than expected, and 576 * adjust the bias to avoid guest time going backwards. 577 */ 578 drift = count - now_count; 579 if (drift > 0) { 580 count_time = now; 581 vcpu->arch.count_bias += drift; 582 ret = drift; 583 goto resume; 584 } 585 586 /* Subtract nanosecond delta to find ktime when count was read */ 587 delta = (u64)(u32)(now_count - count); 588 delta = div_u64(delta * NSEC_PER_SEC, vcpu->arch.count_hz); 589 count_time = ktime_sub_ns(now, delta); 590 591 resume: 592 /* Resume using the calculated ktime */ 593 kvm_mips_resume_hrtimer(vcpu, count_time, count); 594 return ret; 595 } 596 597 /** 598 * kvm_mips_write_count() - Modify the count and update timer. 599 * @vcpu: Virtual CPU. 600 * @count: Guest CP0_Count value to set. 601 * 602 * Sets the CP0_Count value and updates the timer accordingly. 603 */ 604 void kvm_mips_write_count(struct kvm_vcpu *vcpu, u32 count) 605 { 606 struct mips_coproc *cop0 = vcpu->arch.cop0; 607 ktime_t now; 608 609 /* Calculate bias */ 610 now = kvm_mips_count_time(vcpu); 611 vcpu->arch.count_bias = count - kvm_mips_ktime_to_count(vcpu, now); 612 613 if (kvm_mips_count_disabled(vcpu)) 614 /* The timer's disabled, adjust the static count */ 615 kvm_write_c0_guest_count(cop0, count); 616 else 617 /* Update timeout */ 618 kvm_mips_resume_hrtimer(vcpu, now, count); 619 } 620 621 /** 622 * kvm_mips_init_count() - Initialise timer. 623 * @vcpu: Virtual CPU. 624 * @count_hz: Frequency of timer. 625 * 626 * Initialise the timer to the specified frequency, zero it, and set it going if 627 * it's enabled. 628 */ 629 void kvm_mips_init_count(struct kvm_vcpu *vcpu, unsigned long count_hz) 630 { 631 vcpu->arch.count_hz = count_hz; 632 vcpu->arch.count_period = div_u64((u64)NSEC_PER_SEC << 32, count_hz); 633 vcpu->arch.count_dyn_bias = 0; 634 635 /* Starting at 0 */ 636 kvm_mips_write_count(vcpu, 0); 637 } 638 639 /** 640 * kvm_mips_set_count_hz() - Update the frequency of the timer. 641 * @vcpu: Virtual CPU. 642 * @count_hz: Frequency of CP0_Count timer in Hz. 643 * 644 * Change the frequency of the CP0_Count timer. This is done atomically so that 645 * CP0_Count is continuous and no timer interrupt is lost. 646 * 647 * Returns: -EINVAL if @count_hz is out of range. 648 * 0 on success. 649 */ 650 int kvm_mips_set_count_hz(struct kvm_vcpu *vcpu, s64 count_hz) 651 { 652 struct mips_coproc *cop0 = vcpu->arch.cop0; 653 int dc; 654 ktime_t now; 655 u32 count; 656 657 /* ensure the frequency is in a sensible range... */ 658 if (count_hz <= 0 || count_hz > NSEC_PER_SEC) 659 return -EINVAL; 660 /* ... and has actually changed */ 661 if (vcpu->arch.count_hz == count_hz) 662 return 0; 663 664 /* Safely freeze timer so we can keep it continuous */ 665 dc = kvm_mips_count_disabled(vcpu); 666 if (dc) { 667 now = kvm_mips_count_time(vcpu); 668 count = kvm_read_c0_guest_count(cop0); 669 } else { 670 now = kvm_mips_freeze_hrtimer(vcpu, &count); 671 } 672 673 /* Update the frequency */ 674 vcpu->arch.count_hz = count_hz; 675 vcpu->arch.count_period = div_u64((u64)NSEC_PER_SEC << 32, count_hz); 676 vcpu->arch.count_dyn_bias = 0; 677 678 /* Calculate adjusted bias so dynamic count is unchanged */ 679 vcpu->arch.count_bias = count - kvm_mips_ktime_to_count(vcpu, now); 680 681 /* Update and resume hrtimer */ 682 if (!dc) 683 kvm_mips_resume_hrtimer(vcpu, now, count); 684 return 0; 685 } 686 687 /** 688 * kvm_mips_write_compare() - Modify compare and update timer. 689 * @vcpu: Virtual CPU. 690 * @compare: New CP0_Compare value. 691 * @ack: Whether to acknowledge timer interrupt. 692 * 693 * Update CP0_Compare to a new value and update the timeout. 694 * If @ack, atomically acknowledge any pending timer interrupt, otherwise ensure 695 * any pending timer interrupt is preserved. 696 */ 697 void kvm_mips_write_compare(struct kvm_vcpu *vcpu, u32 compare, bool ack) 698 { 699 struct mips_coproc *cop0 = vcpu->arch.cop0; 700 int dc; 701 u32 old_compare = kvm_read_c0_guest_compare(cop0); 702 s32 delta = compare - old_compare; 703 u32 cause; 704 ktime_t now = ktime_set(0, 0); /* silence bogus GCC warning */ 705 u32 count; 706 707 /* if unchanged, must just be an ack */ 708 if (old_compare == compare) { 709 if (!ack) 710 return; 711 kvm_mips_callbacks->dequeue_timer_int(vcpu); 712 kvm_write_c0_guest_compare(cop0, compare); 713 return; 714 } 715 716 /* 717 * If guest CP0_Compare moves forward, CP0_GTOffset should be adjusted 718 * too to prevent guest CP0_Count hitting guest CP0_Compare. 719 * 720 * The new GTOffset corresponds to the new value of CP0_Compare, and is 721 * set prior to it being written into the guest context. We disable 722 * preemption until the new value is written to prevent restore of a 723 * GTOffset corresponding to the old CP0_Compare value. 724 */ 725 if (delta > 0) { 726 preempt_disable(); 727 write_c0_gtoffset(compare - read_c0_count()); 728 back_to_back_c0_hazard(); 729 } 730 731 /* freeze_hrtimer() takes care of timer interrupts <= count */ 732 dc = kvm_mips_count_disabled(vcpu); 733 if (!dc) 734 now = kvm_mips_freeze_hrtimer(vcpu, &count); 735 736 if (ack) 737 kvm_mips_callbacks->dequeue_timer_int(vcpu); 738 else 739 /* 740 * With VZ, writing CP0_Compare acks (clears) CP0_Cause.TI, so 741 * preserve guest CP0_Cause.TI if we don't want to ack it. 742 */ 743 cause = kvm_read_c0_guest_cause(cop0); 744 745 kvm_write_c0_guest_compare(cop0, compare); 746 747 if (delta > 0) 748 preempt_enable(); 749 750 back_to_back_c0_hazard(); 751 752 if (!ack && cause & CAUSEF_TI) 753 kvm_write_c0_guest_cause(cop0, cause); 754 755 /* resume_hrtimer() takes care of timer interrupts > count */ 756 if (!dc) 757 kvm_mips_resume_hrtimer(vcpu, now, count); 758 759 /* 760 * If guest CP0_Compare is moving backward, we delay CP0_GTOffset change 761 * until after the new CP0_Compare is written, otherwise new guest 762 * CP0_Count could hit new guest CP0_Compare. 763 */ 764 if (delta <= 0) 765 write_c0_gtoffset(compare - read_c0_count()); 766 } 767 768 /** 769 * kvm_mips_count_disable() - Disable count. 770 * @vcpu: Virtual CPU. 771 * 772 * Disable the CP0_Count timer. A timer interrupt on or before the final stop 773 * time will be handled but not after. 774 * 775 * Assumes CP0_Count was previously enabled but now Guest.CP0_Cause.DC or 776 * count_ctl.DC has been set (count disabled). 777 * 778 * Returns: The time that the timer was stopped. 779 */ 780 static ktime_t kvm_mips_count_disable(struct kvm_vcpu *vcpu) 781 { 782 struct mips_coproc *cop0 = vcpu->arch.cop0; 783 u32 count; 784 ktime_t now; 785 786 /* Stop hrtimer */ 787 hrtimer_cancel(&vcpu->arch.comparecount_timer); 788 789 /* Set the static count from the dynamic count, handling pending TI */ 790 now = ktime_get(); 791 count = kvm_mips_read_count_running(vcpu, now); 792 kvm_write_c0_guest_count(cop0, count); 793 794 return now; 795 } 796 797 /** 798 * kvm_mips_count_disable_cause() - Disable count using CP0_Cause.DC. 799 * @vcpu: Virtual CPU. 800 * 801 * Disable the CP0_Count timer and set CP0_Cause.DC. A timer interrupt on or 802 * before the final stop time will be handled if the timer isn't disabled by 803 * count_ctl.DC, but not after. 804 * 805 * Assumes CP0_Cause.DC is clear (count enabled). 806 */ 807 void kvm_mips_count_disable_cause(struct kvm_vcpu *vcpu) 808 { 809 struct mips_coproc *cop0 = vcpu->arch.cop0; 810 811 kvm_set_c0_guest_cause(cop0, CAUSEF_DC); 812 if (!(vcpu->arch.count_ctl & KVM_REG_MIPS_COUNT_CTL_DC)) 813 kvm_mips_count_disable(vcpu); 814 } 815 816 /** 817 * kvm_mips_count_enable_cause() - Enable count using CP0_Cause.DC. 818 * @vcpu: Virtual CPU. 819 * 820 * Enable the CP0_Count timer and clear CP0_Cause.DC. A timer interrupt after 821 * the start time will be handled if the timer isn't disabled by count_ctl.DC, 822 * potentially before even returning, so the caller should be careful with 823 * ordering of CP0_Cause modifications so as not to lose it. 824 * 825 * Assumes CP0_Cause.DC is set (count disabled). 826 */ 827 void kvm_mips_count_enable_cause(struct kvm_vcpu *vcpu) 828 { 829 struct mips_coproc *cop0 = vcpu->arch.cop0; 830 u32 count; 831 832 kvm_clear_c0_guest_cause(cop0, CAUSEF_DC); 833 834 /* 835 * Set the dynamic count to match the static count. 836 * This starts the hrtimer if count_ctl.DC allows it. 837 * Otherwise it conveniently updates the biases. 838 */ 839 count = kvm_read_c0_guest_count(cop0); 840 kvm_mips_write_count(vcpu, count); 841 } 842 843 /** 844 * kvm_mips_set_count_ctl() - Update the count control KVM register. 845 * @vcpu: Virtual CPU. 846 * @count_ctl: Count control register new value. 847 * 848 * Set the count control KVM register. The timer is updated accordingly. 849 * 850 * Returns: -EINVAL if reserved bits are set. 851 * 0 on success. 852 */ 853 int kvm_mips_set_count_ctl(struct kvm_vcpu *vcpu, s64 count_ctl) 854 { 855 struct mips_coproc *cop0 = vcpu->arch.cop0; 856 s64 changed = count_ctl ^ vcpu->arch.count_ctl; 857 s64 delta; 858 ktime_t expire, now; 859 u32 count, compare; 860 861 /* Only allow defined bits to be changed */ 862 if (changed & ~(s64)(KVM_REG_MIPS_COUNT_CTL_DC)) 863 return -EINVAL; 864 865 /* Apply new value */ 866 vcpu->arch.count_ctl = count_ctl; 867 868 /* Master CP0_Count disable */ 869 if (changed & KVM_REG_MIPS_COUNT_CTL_DC) { 870 /* Is CP0_Cause.DC already disabling CP0_Count? */ 871 if (kvm_read_c0_guest_cause(cop0) & CAUSEF_DC) { 872 if (count_ctl & KVM_REG_MIPS_COUNT_CTL_DC) 873 /* Just record the current time */ 874 vcpu->arch.count_resume = ktime_get(); 875 } else if (count_ctl & KVM_REG_MIPS_COUNT_CTL_DC) { 876 /* disable timer and record current time */ 877 vcpu->arch.count_resume = kvm_mips_count_disable(vcpu); 878 } else { 879 /* 880 * Calculate timeout relative to static count at resume 881 * time (wrap 0 to 2^32). 882 */ 883 count = kvm_read_c0_guest_count(cop0); 884 compare = kvm_read_c0_guest_compare(cop0); 885 delta = (u64)(u32)(compare - count - 1) + 1; 886 delta = div_u64(delta * NSEC_PER_SEC, 887 vcpu->arch.count_hz); 888 expire = ktime_add_ns(vcpu->arch.count_resume, delta); 889 890 /* Handle pending interrupt */ 891 now = ktime_get(); 892 if (ktime_compare(now, expire) >= 0) 893 /* Nothing should be waiting on the timeout */ 894 kvm_mips_callbacks->queue_timer_int(vcpu); 895 896 /* Resume hrtimer without changing bias */ 897 count = kvm_mips_read_count_running(vcpu, now); 898 kvm_mips_resume_hrtimer(vcpu, now, count); 899 } 900 } 901 902 return 0; 903 } 904 905 /** 906 * kvm_mips_set_count_resume() - Update the count resume KVM register. 907 * @vcpu: Virtual CPU. 908 * @count_resume: Count resume register new value. 909 * 910 * Set the count resume KVM register. 911 * 912 * Returns: -EINVAL if out of valid range (0..now). 913 * 0 on success. 914 */ 915 int kvm_mips_set_count_resume(struct kvm_vcpu *vcpu, s64 count_resume) 916 { 917 /* 918 * It doesn't make sense for the resume time to be in the future, as it 919 * would be possible for the next interrupt to be more than a full 920 * period in the future. 921 */ 922 if (count_resume < 0 || count_resume > ktime_to_ns(ktime_get())) 923 return -EINVAL; 924 925 vcpu->arch.count_resume = ns_to_ktime(count_resume); 926 return 0; 927 } 928 929 /** 930 * kvm_mips_count_timeout() - Push timer forward on timeout. 931 * @vcpu: Virtual CPU. 932 * 933 * Handle an hrtimer event by push the hrtimer forward a period. 934 * 935 * Returns: The hrtimer_restart value to return to the hrtimer subsystem. 936 */ 937 enum hrtimer_restart kvm_mips_count_timeout(struct kvm_vcpu *vcpu) 938 { 939 /* Add the Count period to the current expiry time */ 940 hrtimer_add_expires_ns(&vcpu->arch.comparecount_timer, 941 vcpu->arch.count_period); 942 return HRTIMER_RESTART; 943 } 944 945 enum emulation_result kvm_mips_emul_wait(struct kvm_vcpu *vcpu) 946 { 947 kvm_debug("[%#lx] !!!WAIT!!! (%#lx)\n", vcpu->arch.pc, 948 vcpu->arch.pending_exceptions); 949 950 ++vcpu->stat.wait_exits; 951 trace_kvm_exit(vcpu, KVM_TRACE_EXIT_WAIT); 952 if (!vcpu->arch.pending_exceptions) { 953 kvm_vz_lose_htimer(vcpu); 954 vcpu->arch.wait = 1; 955 kvm_vcpu_block(vcpu); 956 957 /* 958 * We we are runnable, then definitely go off to user space to 959 * check if any I/O interrupts are pending. 960 */ 961 if (kvm_check_request(KVM_REQ_UNHALT, vcpu)) { 962 kvm_clear_request(KVM_REQ_UNHALT, vcpu); 963 vcpu->run->exit_reason = KVM_EXIT_IRQ_WINDOW_OPEN; 964 } 965 } 966 967 return EMULATE_DONE; 968 } 969 970 enum emulation_result kvm_mips_emulate_store(union mips_instruction inst, 971 u32 cause, 972 struct kvm_vcpu *vcpu) 973 { 974 int r; 975 enum emulation_result er; 976 u32 rt; 977 struct kvm_run *run = vcpu->run; 978 void *data = run->mmio.data; 979 unsigned int imme; 980 unsigned long curr_pc; 981 982 /* 983 * Update PC and hold onto current PC in case there is 984 * an error and we want to rollback the PC 985 */ 986 curr_pc = vcpu->arch.pc; 987 er = update_pc(vcpu, cause); 988 if (er == EMULATE_FAIL) 989 return er; 990 991 rt = inst.i_format.rt; 992 993 run->mmio.phys_addr = kvm_mips_callbacks->gva_to_gpa( 994 vcpu->arch.host_cp0_badvaddr); 995 if (run->mmio.phys_addr == KVM_INVALID_ADDR) 996 goto out_fail; 997 998 switch (inst.i_format.opcode) { 999 #if defined(CONFIG_64BIT) 1000 case sd_op: 1001 run->mmio.len = 8; 1002 *(u64 *)data = vcpu->arch.gprs[rt]; 1003 1004 kvm_debug("[%#lx] OP_SD: eaddr: %#lx, gpr: %#lx, data: %#llx\n", 1005 vcpu->arch.pc, vcpu->arch.host_cp0_badvaddr, 1006 vcpu->arch.gprs[rt], *(u64 *)data); 1007 break; 1008 #endif 1009 1010 case sw_op: 1011 run->mmio.len = 4; 1012 *(u32 *)data = vcpu->arch.gprs[rt]; 1013 1014 kvm_debug("[%#lx] OP_SW: eaddr: %#lx, gpr: %#lx, data: %#x\n", 1015 vcpu->arch.pc, vcpu->arch.host_cp0_badvaddr, 1016 vcpu->arch.gprs[rt], *(u32 *)data); 1017 break; 1018 1019 case sh_op: 1020 run->mmio.len = 2; 1021 *(u16 *)data = vcpu->arch.gprs[rt]; 1022 1023 kvm_debug("[%#lx] OP_SH: eaddr: %#lx, gpr: %#lx, data: %#x\n", 1024 vcpu->arch.pc, vcpu->arch.host_cp0_badvaddr, 1025 vcpu->arch.gprs[rt], *(u16 *)data); 1026 break; 1027 1028 case sb_op: 1029 run->mmio.len = 1; 1030 *(u8 *)data = vcpu->arch.gprs[rt]; 1031 1032 kvm_debug("[%#lx] OP_SB: eaddr: %#lx, gpr: %#lx, data: %#x\n", 1033 vcpu->arch.pc, vcpu->arch.host_cp0_badvaddr, 1034 vcpu->arch.gprs[rt], *(u8 *)data); 1035 break; 1036 1037 case swl_op: 1038 run->mmio.phys_addr = kvm_mips_callbacks->gva_to_gpa( 1039 vcpu->arch.host_cp0_badvaddr) & (~0x3); 1040 run->mmio.len = 4; 1041 imme = vcpu->arch.host_cp0_badvaddr & 0x3; 1042 switch (imme) { 1043 case 0: 1044 *(u32 *)data = ((*(u32 *)data) & 0xffffff00) | 1045 (vcpu->arch.gprs[rt] >> 24); 1046 break; 1047 case 1: 1048 *(u32 *)data = ((*(u32 *)data) & 0xffff0000) | 1049 (vcpu->arch.gprs[rt] >> 16); 1050 break; 1051 case 2: 1052 *(u32 *)data = ((*(u32 *)data) & 0xff000000) | 1053 (vcpu->arch.gprs[rt] >> 8); 1054 break; 1055 case 3: 1056 *(u32 *)data = vcpu->arch.gprs[rt]; 1057 break; 1058 default: 1059 break; 1060 } 1061 1062 kvm_debug("[%#lx] OP_SWL: eaddr: %#lx, gpr: %#lx, data: %#x\n", 1063 vcpu->arch.pc, vcpu->arch.host_cp0_badvaddr, 1064 vcpu->arch.gprs[rt], *(u32 *)data); 1065 break; 1066 1067 case swr_op: 1068 run->mmio.phys_addr = kvm_mips_callbacks->gva_to_gpa( 1069 vcpu->arch.host_cp0_badvaddr) & (~0x3); 1070 run->mmio.len = 4; 1071 imme = vcpu->arch.host_cp0_badvaddr & 0x3; 1072 switch (imme) { 1073 case 0: 1074 *(u32 *)data = vcpu->arch.gprs[rt]; 1075 break; 1076 case 1: 1077 *(u32 *)data = ((*(u32 *)data) & 0xff) | 1078 (vcpu->arch.gprs[rt] << 8); 1079 break; 1080 case 2: 1081 *(u32 *)data = ((*(u32 *)data) & 0xffff) | 1082 (vcpu->arch.gprs[rt] << 16); 1083 break; 1084 case 3: 1085 *(u32 *)data = ((*(u32 *)data) & 0xffffff) | 1086 (vcpu->arch.gprs[rt] << 24); 1087 break; 1088 default: 1089 break; 1090 } 1091 1092 kvm_debug("[%#lx] OP_SWR: eaddr: %#lx, gpr: %#lx, data: %#x\n", 1093 vcpu->arch.pc, vcpu->arch.host_cp0_badvaddr, 1094 vcpu->arch.gprs[rt], *(u32 *)data); 1095 break; 1096 1097 #if defined(CONFIG_64BIT) 1098 case sdl_op: 1099 run->mmio.phys_addr = kvm_mips_callbacks->gva_to_gpa( 1100 vcpu->arch.host_cp0_badvaddr) & (~0x7); 1101 1102 run->mmio.len = 8; 1103 imme = vcpu->arch.host_cp0_badvaddr & 0x7; 1104 switch (imme) { 1105 case 0: 1106 *(u64 *)data = ((*(u64 *)data) & 0xffffffffffffff00) | 1107 ((vcpu->arch.gprs[rt] >> 56) & 0xff); 1108 break; 1109 case 1: 1110 *(u64 *)data = ((*(u64 *)data) & 0xffffffffffff0000) | 1111 ((vcpu->arch.gprs[rt] >> 48) & 0xffff); 1112 break; 1113 case 2: 1114 *(u64 *)data = ((*(u64 *)data) & 0xffffffffff000000) | 1115 ((vcpu->arch.gprs[rt] >> 40) & 0xffffff); 1116 break; 1117 case 3: 1118 *(u64 *)data = ((*(u64 *)data) & 0xffffffff00000000) | 1119 ((vcpu->arch.gprs[rt] >> 32) & 0xffffffff); 1120 break; 1121 case 4: 1122 *(u64 *)data = ((*(u64 *)data) & 0xffffff0000000000) | 1123 ((vcpu->arch.gprs[rt] >> 24) & 0xffffffffff); 1124 break; 1125 case 5: 1126 *(u64 *)data = ((*(u64 *)data) & 0xffff000000000000) | 1127 ((vcpu->arch.gprs[rt] >> 16) & 0xffffffffffff); 1128 break; 1129 case 6: 1130 *(u64 *)data = ((*(u64 *)data) & 0xff00000000000000) | 1131 ((vcpu->arch.gprs[rt] >> 8) & 0xffffffffffffff); 1132 break; 1133 case 7: 1134 *(u64 *)data = vcpu->arch.gprs[rt]; 1135 break; 1136 default: 1137 break; 1138 } 1139 1140 kvm_debug("[%#lx] OP_SDL: eaddr: %#lx, gpr: %#lx, data: %llx\n", 1141 vcpu->arch.pc, vcpu->arch.host_cp0_badvaddr, 1142 vcpu->arch.gprs[rt], *(u64 *)data); 1143 break; 1144 1145 case sdr_op: 1146 run->mmio.phys_addr = kvm_mips_callbacks->gva_to_gpa( 1147 vcpu->arch.host_cp0_badvaddr) & (~0x7); 1148 1149 run->mmio.len = 8; 1150 imme = vcpu->arch.host_cp0_badvaddr & 0x7; 1151 switch (imme) { 1152 case 0: 1153 *(u64 *)data = vcpu->arch.gprs[rt]; 1154 break; 1155 case 1: 1156 *(u64 *)data = ((*(u64 *)data) & 0xff) | 1157 (vcpu->arch.gprs[rt] << 8); 1158 break; 1159 case 2: 1160 *(u64 *)data = ((*(u64 *)data) & 0xffff) | 1161 (vcpu->arch.gprs[rt] << 16); 1162 break; 1163 case 3: 1164 *(u64 *)data = ((*(u64 *)data) & 0xffffff) | 1165 (vcpu->arch.gprs[rt] << 24); 1166 break; 1167 case 4: 1168 *(u64 *)data = ((*(u64 *)data) & 0xffffffff) | 1169 (vcpu->arch.gprs[rt] << 32); 1170 break; 1171 case 5: 1172 *(u64 *)data = ((*(u64 *)data) & 0xffffffffff) | 1173 (vcpu->arch.gprs[rt] << 40); 1174 break; 1175 case 6: 1176 *(u64 *)data = ((*(u64 *)data) & 0xffffffffffff) | 1177 (vcpu->arch.gprs[rt] << 48); 1178 break; 1179 case 7: 1180 *(u64 *)data = ((*(u64 *)data) & 0xffffffffffffff) | 1181 (vcpu->arch.gprs[rt] << 56); 1182 break; 1183 default: 1184 break; 1185 } 1186 1187 kvm_debug("[%#lx] OP_SDR: eaddr: %#lx, gpr: %#lx, data: %llx\n", 1188 vcpu->arch.pc, vcpu->arch.host_cp0_badvaddr, 1189 vcpu->arch.gprs[rt], *(u64 *)data); 1190 break; 1191 #endif 1192 1193 #ifdef CONFIG_CPU_LOONGSON64 1194 case sdc2_op: 1195 rt = inst.loongson3_lsdc2_format.rt; 1196 switch (inst.loongson3_lsdc2_format.opcode1) { 1197 /* 1198 * Loongson-3 overridden sdc2 instructions. 1199 * opcode1 instruction 1200 * 0x0 gssbx: store 1 bytes from GPR 1201 * 0x1 gsshx: store 2 bytes from GPR 1202 * 0x2 gsswx: store 4 bytes from GPR 1203 * 0x3 gssdx: store 8 bytes from GPR 1204 */ 1205 case 0x0: 1206 run->mmio.len = 1; 1207 *(u8 *)data = vcpu->arch.gprs[rt]; 1208 1209 kvm_debug("[%#lx] OP_GSSBX: eaddr: %#lx, gpr: %#lx, data: %#x\n", 1210 vcpu->arch.pc, vcpu->arch.host_cp0_badvaddr, 1211 vcpu->arch.gprs[rt], *(u8 *)data); 1212 break; 1213 case 0x1: 1214 run->mmio.len = 2; 1215 *(u16 *)data = vcpu->arch.gprs[rt]; 1216 1217 kvm_debug("[%#lx] OP_GSSSHX: eaddr: %#lx, gpr: %#lx, data: %#x\n", 1218 vcpu->arch.pc, vcpu->arch.host_cp0_badvaddr, 1219 vcpu->arch.gprs[rt], *(u16 *)data); 1220 break; 1221 case 0x2: 1222 run->mmio.len = 4; 1223 *(u32 *)data = vcpu->arch.gprs[rt]; 1224 1225 kvm_debug("[%#lx] OP_GSSWX: eaddr: %#lx, gpr: %#lx, data: %#x\n", 1226 vcpu->arch.pc, vcpu->arch.host_cp0_badvaddr, 1227 vcpu->arch.gprs[rt], *(u32 *)data); 1228 break; 1229 case 0x3: 1230 run->mmio.len = 8; 1231 *(u64 *)data = vcpu->arch.gprs[rt]; 1232 1233 kvm_debug("[%#lx] OP_GSSDX: eaddr: %#lx, gpr: %#lx, data: %#llx\n", 1234 vcpu->arch.pc, vcpu->arch.host_cp0_badvaddr, 1235 vcpu->arch.gprs[rt], *(u64 *)data); 1236 break; 1237 default: 1238 kvm_err("Godson Extended GS-Store not yet supported (inst=0x%08x)\n", 1239 inst.word); 1240 break; 1241 } 1242 break; 1243 #endif 1244 default: 1245 kvm_err("Store not yet supported (inst=0x%08x)\n", 1246 inst.word); 1247 goto out_fail; 1248 } 1249 1250 vcpu->mmio_needed = 1; 1251 run->mmio.is_write = 1; 1252 vcpu->mmio_is_write = 1; 1253 1254 r = kvm_io_bus_write(vcpu, KVM_MMIO_BUS, 1255 run->mmio.phys_addr, run->mmio.len, data); 1256 1257 if (!r) { 1258 vcpu->mmio_needed = 0; 1259 return EMULATE_DONE; 1260 } 1261 1262 return EMULATE_DO_MMIO; 1263 1264 out_fail: 1265 /* Rollback PC if emulation was unsuccessful */ 1266 vcpu->arch.pc = curr_pc; 1267 return EMULATE_FAIL; 1268 } 1269 1270 enum emulation_result kvm_mips_emulate_load(union mips_instruction inst, 1271 u32 cause, struct kvm_vcpu *vcpu) 1272 { 1273 struct kvm_run *run = vcpu->run; 1274 int r; 1275 enum emulation_result er; 1276 unsigned long curr_pc; 1277 u32 op, rt; 1278 unsigned int imme; 1279 1280 rt = inst.i_format.rt; 1281 op = inst.i_format.opcode; 1282 1283 /* 1284 * Find the resume PC now while we have safe and easy access to the 1285 * prior branch instruction, and save it for 1286 * kvm_mips_complete_mmio_load() to restore later. 1287 */ 1288 curr_pc = vcpu->arch.pc; 1289 er = update_pc(vcpu, cause); 1290 if (er == EMULATE_FAIL) 1291 return er; 1292 vcpu->arch.io_pc = vcpu->arch.pc; 1293 vcpu->arch.pc = curr_pc; 1294 1295 vcpu->arch.io_gpr = rt; 1296 1297 run->mmio.phys_addr = kvm_mips_callbacks->gva_to_gpa( 1298 vcpu->arch.host_cp0_badvaddr); 1299 if (run->mmio.phys_addr == KVM_INVALID_ADDR) 1300 return EMULATE_FAIL; 1301 1302 vcpu->mmio_needed = 2; /* signed */ 1303 switch (op) { 1304 #if defined(CONFIG_64BIT) 1305 case ld_op: 1306 run->mmio.len = 8; 1307 break; 1308 1309 case lwu_op: 1310 vcpu->mmio_needed = 1; /* unsigned */ 1311 fallthrough; 1312 #endif 1313 case lw_op: 1314 run->mmio.len = 4; 1315 break; 1316 1317 case lhu_op: 1318 vcpu->mmio_needed = 1; /* unsigned */ 1319 fallthrough; 1320 case lh_op: 1321 run->mmio.len = 2; 1322 break; 1323 1324 case lbu_op: 1325 vcpu->mmio_needed = 1; /* unsigned */ 1326 fallthrough; 1327 case lb_op: 1328 run->mmio.len = 1; 1329 break; 1330 1331 case lwl_op: 1332 run->mmio.phys_addr = kvm_mips_callbacks->gva_to_gpa( 1333 vcpu->arch.host_cp0_badvaddr) & (~0x3); 1334 1335 run->mmio.len = 4; 1336 imme = vcpu->arch.host_cp0_badvaddr & 0x3; 1337 switch (imme) { 1338 case 0: 1339 vcpu->mmio_needed = 3; /* 1 byte */ 1340 break; 1341 case 1: 1342 vcpu->mmio_needed = 4; /* 2 bytes */ 1343 break; 1344 case 2: 1345 vcpu->mmio_needed = 5; /* 3 bytes */ 1346 break; 1347 case 3: 1348 vcpu->mmio_needed = 6; /* 4 bytes */ 1349 break; 1350 default: 1351 break; 1352 } 1353 break; 1354 1355 case lwr_op: 1356 run->mmio.phys_addr = kvm_mips_callbacks->gva_to_gpa( 1357 vcpu->arch.host_cp0_badvaddr) & (~0x3); 1358 1359 run->mmio.len = 4; 1360 imme = vcpu->arch.host_cp0_badvaddr & 0x3; 1361 switch (imme) { 1362 case 0: 1363 vcpu->mmio_needed = 7; /* 4 bytes */ 1364 break; 1365 case 1: 1366 vcpu->mmio_needed = 8; /* 3 bytes */ 1367 break; 1368 case 2: 1369 vcpu->mmio_needed = 9; /* 2 bytes */ 1370 break; 1371 case 3: 1372 vcpu->mmio_needed = 10; /* 1 byte */ 1373 break; 1374 default: 1375 break; 1376 } 1377 break; 1378 1379 #if defined(CONFIG_64BIT) 1380 case ldl_op: 1381 run->mmio.phys_addr = kvm_mips_callbacks->gva_to_gpa( 1382 vcpu->arch.host_cp0_badvaddr) & (~0x7); 1383 1384 run->mmio.len = 8; 1385 imme = vcpu->arch.host_cp0_badvaddr & 0x7; 1386 switch (imme) { 1387 case 0: 1388 vcpu->mmio_needed = 11; /* 1 byte */ 1389 break; 1390 case 1: 1391 vcpu->mmio_needed = 12; /* 2 bytes */ 1392 break; 1393 case 2: 1394 vcpu->mmio_needed = 13; /* 3 bytes */ 1395 break; 1396 case 3: 1397 vcpu->mmio_needed = 14; /* 4 bytes */ 1398 break; 1399 case 4: 1400 vcpu->mmio_needed = 15; /* 5 bytes */ 1401 break; 1402 case 5: 1403 vcpu->mmio_needed = 16; /* 6 bytes */ 1404 break; 1405 case 6: 1406 vcpu->mmio_needed = 17; /* 7 bytes */ 1407 break; 1408 case 7: 1409 vcpu->mmio_needed = 18; /* 8 bytes */ 1410 break; 1411 default: 1412 break; 1413 } 1414 break; 1415 1416 case ldr_op: 1417 run->mmio.phys_addr = kvm_mips_callbacks->gva_to_gpa( 1418 vcpu->arch.host_cp0_badvaddr) & (~0x7); 1419 1420 run->mmio.len = 8; 1421 imme = vcpu->arch.host_cp0_badvaddr & 0x7; 1422 switch (imme) { 1423 case 0: 1424 vcpu->mmio_needed = 19; /* 8 bytes */ 1425 break; 1426 case 1: 1427 vcpu->mmio_needed = 20; /* 7 bytes */ 1428 break; 1429 case 2: 1430 vcpu->mmio_needed = 21; /* 6 bytes */ 1431 break; 1432 case 3: 1433 vcpu->mmio_needed = 22; /* 5 bytes */ 1434 break; 1435 case 4: 1436 vcpu->mmio_needed = 23; /* 4 bytes */ 1437 break; 1438 case 5: 1439 vcpu->mmio_needed = 24; /* 3 bytes */ 1440 break; 1441 case 6: 1442 vcpu->mmio_needed = 25; /* 2 bytes */ 1443 break; 1444 case 7: 1445 vcpu->mmio_needed = 26; /* 1 byte */ 1446 break; 1447 default: 1448 break; 1449 } 1450 break; 1451 #endif 1452 1453 #ifdef CONFIG_CPU_LOONGSON64 1454 case ldc2_op: 1455 rt = inst.loongson3_lsdc2_format.rt; 1456 switch (inst.loongson3_lsdc2_format.opcode1) { 1457 /* 1458 * Loongson-3 overridden ldc2 instructions. 1459 * opcode1 instruction 1460 * 0x0 gslbx: store 1 bytes from GPR 1461 * 0x1 gslhx: store 2 bytes from GPR 1462 * 0x2 gslwx: store 4 bytes from GPR 1463 * 0x3 gsldx: store 8 bytes from GPR 1464 */ 1465 case 0x0: 1466 run->mmio.len = 1; 1467 vcpu->mmio_needed = 27; /* signed */ 1468 break; 1469 case 0x1: 1470 run->mmio.len = 2; 1471 vcpu->mmio_needed = 28; /* signed */ 1472 break; 1473 case 0x2: 1474 run->mmio.len = 4; 1475 vcpu->mmio_needed = 29; /* signed */ 1476 break; 1477 case 0x3: 1478 run->mmio.len = 8; 1479 vcpu->mmio_needed = 30; /* signed */ 1480 break; 1481 default: 1482 kvm_err("Godson Extended GS-Load for float not yet supported (inst=0x%08x)\n", 1483 inst.word); 1484 break; 1485 } 1486 break; 1487 #endif 1488 1489 default: 1490 kvm_err("Load not yet supported (inst=0x%08x)\n", 1491 inst.word); 1492 vcpu->mmio_needed = 0; 1493 return EMULATE_FAIL; 1494 } 1495 1496 run->mmio.is_write = 0; 1497 vcpu->mmio_is_write = 0; 1498 1499 r = kvm_io_bus_read(vcpu, KVM_MMIO_BUS, 1500 run->mmio.phys_addr, run->mmio.len, run->mmio.data); 1501 1502 if (!r) { 1503 kvm_mips_complete_mmio_load(vcpu); 1504 vcpu->mmio_needed = 0; 1505 return EMULATE_DONE; 1506 } 1507 1508 return EMULATE_DO_MMIO; 1509 } 1510 1511 enum emulation_result kvm_mips_complete_mmio_load(struct kvm_vcpu *vcpu) 1512 { 1513 struct kvm_run *run = vcpu->run; 1514 unsigned long *gpr = &vcpu->arch.gprs[vcpu->arch.io_gpr]; 1515 enum emulation_result er = EMULATE_DONE; 1516 1517 if (run->mmio.len > sizeof(*gpr)) { 1518 kvm_err("Bad MMIO length: %d", run->mmio.len); 1519 er = EMULATE_FAIL; 1520 goto done; 1521 } 1522 1523 /* Restore saved resume PC */ 1524 vcpu->arch.pc = vcpu->arch.io_pc; 1525 1526 switch (run->mmio.len) { 1527 case 8: 1528 switch (vcpu->mmio_needed) { 1529 case 11: 1530 *gpr = (vcpu->arch.gprs[vcpu->arch.io_gpr] & 0xffffffffffffff) | 1531 (((*(s64 *)run->mmio.data) & 0xff) << 56); 1532 break; 1533 case 12: 1534 *gpr = (vcpu->arch.gprs[vcpu->arch.io_gpr] & 0xffffffffffff) | 1535 (((*(s64 *)run->mmio.data) & 0xffff) << 48); 1536 break; 1537 case 13: 1538 *gpr = (vcpu->arch.gprs[vcpu->arch.io_gpr] & 0xffffffffff) | 1539 (((*(s64 *)run->mmio.data) & 0xffffff) << 40); 1540 break; 1541 case 14: 1542 *gpr = (vcpu->arch.gprs[vcpu->arch.io_gpr] & 0xffffffff) | 1543 (((*(s64 *)run->mmio.data) & 0xffffffff) << 32); 1544 break; 1545 case 15: 1546 *gpr = (vcpu->arch.gprs[vcpu->arch.io_gpr] & 0xffffff) | 1547 (((*(s64 *)run->mmio.data) & 0xffffffffff) << 24); 1548 break; 1549 case 16: 1550 *gpr = (vcpu->arch.gprs[vcpu->arch.io_gpr] & 0xffff) | 1551 (((*(s64 *)run->mmio.data) & 0xffffffffffff) << 16); 1552 break; 1553 case 17: 1554 *gpr = (vcpu->arch.gprs[vcpu->arch.io_gpr] & 0xff) | 1555 (((*(s64 *)run->mmio.data) & 0xffffffffffffff) << 8); 1556 break; 1557 case 18: 1558 case 19: 1559 *gpr = *(s64 *)run->mmio.data; 1560 break; 1561 case 20: 1562 *gpr = (vcpu->arch.gprs[vcpu->arch.io_gpr] & 0xff00000000000000) | 1563 ((((*(s64 *)run->mmio.data)) >> 8) & 0xffffffffffffff); 1564 break; 1565 case 21: 1566 *gpr = (vcpu->arch.gprs[vcpu->arch.io_gpr] & 0xffff000000000000) | 1567 ((((*(s64 *)run->mmio.data)) >> 16) & 0xffffffffffff); 1568 break; 1569 case 22: 1570 *gpr = (vcpu->arch.gprs[vcpu->arch.io_gpr] & 0xffffff0000000000) | 1571 ((((*(s64 *)run->mmio.data)) >> 24) & 0xffffffffff); 1572 break; 1573 case 23: 1574 *gpr = (vcpu->arch.gprs[vcpu->arch.io_gpr] & 0xffffffff00000000) | 1575 ((((*(s64 *)run->mmio.data)) >> 32) & 0xffffffff); 1576 break; 1577 case 24: 1578 *gpr = (vcpu->arch.gprs[vcpu->arch.io_gpr] & 0xffffffffff000000) | 1579 ((((*(s64 *)run->mmio.data)) >> 40) & 0xffffff); 1580 break; 1581 case 25: 1582 *gpr = (vcpu->arch.gprs[vcpu->arch.io_gpr] & 0xffffffffffff0000) | 1583 ((((*(s64 *)run->mmio.data)) >> 48) & 0xffff); 1584 break; 1585 case 26: 1586 *gpr = (vcpu->arch.gprs[vcpu->arch.io_gpr] & 0xffffffffffffff00) | 1587 ((((*(s64 *)run->mmio.data)) >> 56) & 0xff); 1588 break; 1589 default: 1590 *gpr = *(s64 *)run->mmio.data; 1591 } 1592 break; 1593 1594 case 4: 1595 switch (vcpu->mmio_needed) { 1596 case 1: 1597 *gpr = *(u32 *)run->mmio.data; 1598 break; 1599 case 2: 1600 *gpr = *(s32 *)run->mmio.data; 1601 break; 1602 case 3: 1603 *gpr = (vcpu->arch.gprs[vcpu->arch.io_gpr] & 0xffffff) | 1604 (((*(s32 *)run->mmio.data) & 0xff) << 24); 1605 break; 1606 case 4: 1607 *gpr = (vcpu->arch.gprs[vcpu->arch.io_gpr] & 0xffff) | 1608 (((*(s32 *)run->mmio.data) & 0xffff) << 16); 1609 break; 1610 case 5: 1611 *gpr = (vcpu->arch.gprs[vcpu->arch.io_gpr] & 0xff) | 1612 (((*(s32 *)run->mmio.data) & 0xffffff) << 8); 1613 break; 1614 case 6: 1615 case 7: 1616 *gpr = *(s32 *)run->mmio.data; 1617 break; 1618 case 8: 1619 *gpr = (vcpu->arch.gprs[vcpu->arch.io_gpr] & 0xff000000) | 1620 ((((*(s32 *)run->mmio.data)) >> 8) & 0xffffff); 1621 break; 1622 case 9: 1623 *gpr = (vcpu->arch.gprs[vcpu->arch.io_gpr] & 0xffff0000) | 1624 ((((*(s32 *)run->mmio.data)) >> 16) & 0xffff); 1625 break; 1626 case 10: 1627 *gpr = (vcpu->arch.gprs[vcpu->arch.io_gpr] & 0xffffff00) | 1628 ((((*(s32 *)run->mmio.data)) >> 24) & 0xff); 1629 break; 1630 default: 1631 *gpr = *(s32 *)run->mmio.data; 1632 } 1633 break; 1634 1635 case 2: 1636 if (vcpu->mmio_needed == 1) 1637 *gpr = *(u16 *)run->mmio.data; 1638 else 1639 *gpr = *(s16 *)run->mmio.data; 1640 1641 break; 1642 case 1: 1643 if (vcpu->mmio_needed == 1) 1644 *gpr = *(u8 *)run->mmio.data; 1645 else 1646 *gpr = *(s8 *)run->mmio.data; 1647 break; 1648 } 1649 1650 done: 1651 return er; 1652 } 1653