1 /* 2 * Copyright 2012 Michael Ellerman, IBM Corporation. 3 * Copyright 2012 Benjamin Herrenschmidt, IBM Corporation 4 * 5 * This program is free software; you can redistribute it and/or modify 6 * it under the terms of the GNU General Public License, version 2, as 7 * published by the Free Software Foundation. 8 */ 9 10 #include <linux/kernel.h> 11 #include <linux/kvm_host.h> 12 #include <linux/err.h> 13 #include <linux/kernel_stat.h> 14 15 #include <asm/kvm_book3s.h> 16 #include <asm/kvm_ppc.h> 17 #include <asm/hvcall.h> 18 #include <asm/xics.h> 19 #include <asm/synch.h> 20 #include <asm/cputhreads.h> 21 #include <asm/pgtable.h> 22 #include <asm/ppc-opcode.h> 23 #include <asm/pnv-pci.h> 24 #include <asm/opal.h> 25 #include <asm/smp.h> 26 27 #include "book3s_xics.h" 28 29 #define DEBUG_PASSUP 30 31 int h_ipi_redirect = 1; 32 EXPORT_SYMBOL(h_ipi_redirect); 33 int kvm_irq_bypass = 1; 34 EXPORT_SYMBOL(kvm_irq_bypass); 35 36 static void icp_rm_deliver_irq(struct kvmppc_xics *xics, struct kvmppc_icp *icp, 37 u32 new_irq, bool check_resend); 38 static int xics_opal_set_server(unsigned int hw_irq, int server_cpu); 39 40 /* -- ICS routines -- */ 41 static void ics_rm_check_resend(struct kvmppc_xics *xics, 42 struct kvmppc_ics *ics, struct kvmppc_icp *icp) 43 { 44 int i; 45 46 for (i = 0; i < KVMPPC_XICS_IRQ_PER_ICS; i++) { 47 struct ics_irq_state *state = &ics->irq_state[i]; 48 if (state->resend) 49 icp_rm_deliver_irq(xics, icp, state->number, true); 50 } 51 52 } 53 54 /* -- ICP routines -- */ 55 56 #ifdef CONFIG_SMP 57 static inline void icp_send_hcore_msg(int hcore, struct kvm_vcpu *vcpu) 58 { 59 int hcpu; 60 61 hcpu = hcore << threads_shift; 62 kvmppc_host_rm_ops_hv->rm_core[hcore].rm_data = vcpu; 63 smp_muxed_ipi_set_message(hcpu, PPC_MSG_RM_HOST_ACTION); 64 kvmppc_set_host_ipi(hcpu, 1); 65 smp_mb(); 66 kvmhv_rm_send_ipi(hcpu); 67 } 68 #else 69 static inline void icp_send_hcore_msg(int hcore, struct kvm_vcpu *vcpu) { } 70 #endif 71 72 /* 73 * We start the search from our current CPU Id in the core map 74 * and go in a circle until we get back to our ID looking for a 75 * core that is running in host context and that hasn't already 76 * been targeted for another rm_host_ops. 77 * 78 * In the future, could consider using a fairer algorithm (one 79 * that distributes the IPIs better) 80 * 81 * Returns -1, if no CPU could be found in the host 82 * Else, returns a CPU Id which has been reserved for use 83 */ 84 static inline int grab_next_hostcore(int start, 85 struct kvmppc_host_rm_core *rm_core, int max, int action) 86 { 87 bool success; 88 int core; 89 union kvmppc_rm_state old, new; 90 91 for (core = start + 1; core < max; core++) { 92 old = new = READ_ONCE(rm_core[core].rm_state); 93 94 if (!old.in_host || old.rm_action) 95 continue; 96 97 /* Try to grab this host core if not taken already. */ 98 new.rm_action = action; 99 100 success = cmpxchg64(&rm_core[core].rm_state.raw, 101 old.raw, new.raw) == old.raw; 102 if (success) { 103 /* 104 * Make sure that the store to the rm_action is made 105 * visible before we return to caller (and the 106 * subsequent store to rm_data) to synchronize with 107 * the IPI handler. 108 */ 109 smp_wmb(); 110 return core; 111 } 112 } 113 114 return -1; 115 } 116 117 static inline int find_available_hostcore(int action) 118 { 119 int core; 120 int my_core = smp_processor_id() >> threads_shift; 121 struct kvmppc_host_rm_core *rm_core = kvmppc_host_rm_ops_hv->rm_core; 122 123 core = grab_next_hostcore(my_core, rm_core, cpu_nr_cores(), action); 124 if (core == -1) 125 core = grab_next_hostcore(core, rm_core, my_core, action); 126 127 return core; 128 } 129 130 static void icp_rm_set_vcpu_irq(struct kvm_vcpu *vcpu, 131 struct kvm_vcpu *this_vcpu) 132 { 133 struct kvmppc_icp *this_icp = this_vcpu->arch.icp; 134 int cpu; 135 int hcore; 136 137 /* Mark the target VCPU as having an interrupt pending */ 138 vcpu->stat.queue_intr++; 139 set_bit(BOOK3S_IRQPRIO_EXTERNAL, &vcpu->arch.pending_exceptions); 140 141 /* Kick self ? Just set MER and return */ 142 if (vcpu == this_vcpu) { 143 mtspr(SPRN_LPCR, mfspr(SPRN_LPCR) | LPCR_MER); 144 return; 145 } 146 147 /* 148 * Check if the core is loaded, 149 * if not, find an available host core to post to wake the VCPU, 150 * if we can't find one, set up state to eventually return too hard. 151 */ 152 cpu = vcpu->arch.thread_cpu; 153 if (cpu < 0 || cpu >= nr_cpu_ids) { 154 hcore = -1; 155 if (kvmppc_host_rm_ops_hv && h_ipi_redirect) 156 hcore = find_available_hostcore(XICS_RM_KICK_VCPU); 157 if (hcore != -1) { 158 icp_send_hcore_msg(hcore, vcpu); 159 } else { 160 this_icp->rm_action |= XICS_RM_KICK_VCPU; 161 this_icp->rm_kick_target = vcpu; 162 } 163 return; 164 } 165 166 smp_mb(); 167 kvmhv_rm_send_ipi(cpu); 168 } 169 170 static void icp_rm_clr_vcpu_irq(struct kvm_vcpu *vcpu) 171 { 172 /* Note: Only called on self ! */ 173 clear_bit(BOOK3S_IRQPRIO_EXTERNAL, &vcpu->arch.pending_exceptions); 174 mtspr(SPRN_LPCR, mfspr(SPRN_LPCR) & ~LPCR_MER); 175 } 176 177 static inline bool icp_rm_try_update(struct kvmppc_icp *icp, 178 union kvmppc_icp_state old, 179 union kvmppc_icp_state new) 180 { 181 struct kvm_vcpu *this_vcpu = local_paca->kvm_hstate.kvm_vcpu; 182 bool success; 183 184 /* Calculate new output value */ 185 new.out_ee = (new.xisr && (new.pending_pri < new.cppr)); 186 187 /* Attempt atomic update */ 188 success = cmpxchg64(&icp->state.raw, old.raw, new.raw) == old.raw; 189 if (!success) 190 goto bail; 191 192 /* 193 * Check for output state update 194 * 195 * Note that this is racy since another processor could be updating 196 * the state already. This is why we never clear the interrupt output 197 * here, we only ever set it. The clear only happens prior to doing 198 * an update and only by the processor itself. Currently we do it 199 * in Accept (H_XIRR) and Up_Cppr (H_XPPR). 200 * 201 * We also do not try to figure out whether the EE state has changed, 202 * we unconditionally set it if the new state calls for it. The reason 203 * for that is that we opportunistically remove the pending interrupt 204 * flag when raising CPPR, so we need to set it back here if an 205 * interrupt is still pending. 206 */ 207 if (new.out_ee) 208 icp_rm_set_vcpu_irq(icp->vcpu, this_vcpu); 209 210 /* Expose the state change for debug purposes */ 211 this_vcpu->arch.icp->rm_dbgstate = new; 212 this_vcpu->arch.icp->rm_dbgtgt = icp->vcpu; 213 214 bail: 215 return success; 216 } 217 218 static inline int check_too_hard(struct kvmppc_xics *xics, 219 struct kvmppc_icp *icp) 220 { 221 return (xics->real_mode_dbg || icp->rm_action) ? H_TOO_HARD : H_SUCCESS; 222 } 223 224 static void icp_rm_check_resend(struct kvmppc_xics *xics, 225 struct kvmppc_icp *icp) 226 { 227 u32 icsid; 228 229 /* Order this load with the test for need_resend in the caller */ 230 smp_rmb(); 231 for_each_set_bit(icsid, icp->resend_map, xics->max_icsid + 1) { 232 struct kvmppc_ics *ics = xics->ics[icsid]; 233 234 if (!test_and_clear_bit(icsid, icp->resend_map)) 235 continue; 236 if (!ics) 237 continue; 238 ics_rm_check_resend(xics, ics, icp); 239 } 240 } 241 242 static bool icp_rm_try_to_deliver(struct kvmppc_icp *icp, u32 irq, u8 priority, 243 u32 *reject) 244 { 245 union kvmppc_icp_state old_state, new_state; 246 bool success; 247 248 do { 249 old_state = new_state = READ_ONCE(icp->state); 250 251 *reject = 0; 252 253 /* See if we can deliver */ 254 success = new_state.cppr > priority && 255 new_state.mfrr > priority && 256 new_state.pending_pri > priority; 257 258 /* 259 * If we can, check for a rejection and perform the 260 * delivery 261 */ 262 if (success) { 263 *reject = new_state.xisr; 264 new_state.xisr = irq; 265 new_state.pending_pri = priority; 266 } else { 267 /* 268 * If we failed to deliver we set need_resend 269 * so a subsequent CPPR state change causes us 270 * to try a new delivery. 271 */ 272 new_state.need_resend = true; 273 } 274 275 } while (!icp_rm_try_update(icp, old_state, new_state)); 276 277 return success; 278 } 279 280 static void icp_rm_deliver_irq(struct kvmppc_xics *xics, struct kvmppc_icp *icp, 281 u32 new_irq, bool check_resend) 282 { 283 struct ics_irq_state *state; 284 struct kvmppc_ics *ics; 285 u32 reject; 286 u16 src; 287 288 /* 289 * This is used both for initial delivery of an interrupt and 290 * for subsequent rejection. 291 * 292 * Rejection can be racy vs. resends. We have evaluated the 293 * rejection in an atomic ICP transaction which is now complete, 294 * so potentially the ICP can already accept the interrupt again. 295 * 296 * So we need to retry the delivery. Essentially the reject path 297 * boils down to a failed delivery. Always. 298 * 299 * Now the interrupt could also have moved to a different target, 300 * thus we may need to re-do the ICP lookup as well 301 */ 302 303 again: 304 /* Get the ICS state and lock it */ 305 ics = kvmppc_xics_find_ics(xics, new_irq, &src); 306 if (!ics) { 307 /* Unsafe increment, but this does not need to be accurate */ 308 xics->err_noics++; 309 return; 310 } 311 state = &ics->irq_state[src]; 312 313 /* Get a lock on the ICS */ 314 arch_spin_lock(&ics->lock); 315 316 /* Get our server */ 317 if (!icp || state->server != icp->server_num) { 318 icp = kvmppc_xics_find_server(xics->kvm, state->server); 319 if (!icp) { 320 /* Unsafe increment again*/ 321 xics->err_noicp++; 322 goto out; 323 } 324 } 325 326 if (check_resend) 327 if (!state->resend) 328 goto out; 329 330 /* Clear the resend bit of that interrupt */ 331 state->resend = 0; 332 333 /* 334 * If masked, bail out 335 * 336 * Note: PAPR doesn't mention anything about masked pending 337 * when doing a resend, only when doing a delivery. 338 * 339 * However that would have the effect of losing a masked 340 * interrupt that was rejected and isn't consistent with 341 * the whole masked_pending business which is about not 342 * losing interrupts that occur while masked. 343 * 344 * I don't differentiate normal deliveries and resends, this 345 * implementation will differ from PAPR and not lose such 346 * interrupts. 347 */ 348 if (state->priority == MASKED) { 349 state->masked_pending = 1; 350 goto out; 351 } 352 353 /* 354 * Try the delivery, this will set the need_resend flag 355 * in the ICP as part of the atomic transaction if the 356 * delivery is not possible. 357 * 358 * Note that if successful, the new delivery might have itself 359 * rejected an interrupt that was "delivered" before we took the 360 * ics spin lock. 361 * 362 * In this case we do the whole sequence all over again for the 363 * new guy. We cannot assume that the rejected interrupt is less 364 * favored than the new one, and thus doesn't need to be delivered, 365 * because by the time we exit icp_rm_try_to_deliver() the target 366 * processor may well have already consumed & completed it, and thus 367 * the rejected interrupt might actually be already acceptable. 368 */ 369 if (icp_rm_try_to_deliver(icp, new_irq, state->priority, &reject)) { 370 /* 371 * Delivery was successful, did we reject somebody else ? 372 */ 373 if (reject && reject != XICS_IPI) { 374 arch_spin_unlock(&ics->lock); 375 icp->n_reject++; 376 new_irq = reject; 377 check_resend = 0; 378 goto again; 379 } 380 } else { 381 /* 382 * We failed to deliver the interrupt we need to set the 383 * resend map bit and mark the ICS state as needing a resend 384 */ 385 state->resend = 1; 386 387 /* 388 * Make sure when checking resend, we don't miss the resend 389 * if resend_map bit is seen and cleared. 390 */ 391 smp_wmb(); 392 set_bit(ics->icsid, icp->resend_map); 393 394 /* 395 * If the need_resend flag got cleared in the ICP some time 396 * between icp_rm_try_to_deliver() atomic update and now, then 397 * we know it might have missed the resend_map bit. So we 398 * retry 399 */ 400 smp_mb(); 401 if (!icp->state.need_resend) { 402 state->resend = 0; 403 arch_spin_unlock(&ics->lock); 404 check_resend = 0; 405 goto again; 406 } 407 } 408 out: 409 arch_spin_unlock(&ics->lock); 410 } 411 412 static void icp_rm_down_cppr(struct kvmppc_xics *xics, struct kvmppc_icp *icp, 413 u8 new_cppr) 414 { 415 union kvmppc_icp_state old_state, new_state; 416 bool resend; 417 418 /* 419 * This handles several related states in one operation: 420 * 421 * ICP State: Down_CPPR 422 * 423 * Load CPPR with new value and if the XISR is 0 424 * then check for resends: 425 * 426 * ICP State: Resend 427 * 428 * If MFRR is more favored than CPPR, check for IPIs 429 * and notify ICS of a potential resend. This is done 430 * asynchronously (when used in real mode, we will have 431 * to exit here). 432 * 433 * We do not handle the complete Check_IPI as documented 434 * here. In the PAPR, this state will be used for both 435 * Set_MFRR and Down_CPPR. However, we know that we aren't 436 * changing the MFRR state here so we don't need to handle 437 * the case of an MFRR causing a reject of a pending irq, 438 * this will have been handled when the MFRR was set in the 439 * first place. 440 * 441 * Thus we don't have to handle rejects, only resends. 442 * 443 * When implementing real mode for HV KVM, resend will lead to 444 * a H_TOO_HARD return and the whole transaction will be handled 445 * in virtual mode. 446 */ 447 do { 448 old_state = new_state = READ_ONCE(icp->state); 449 450 /* Down_CPPR */ 451 new_state.cppr = new_cppr; 452 453 /* 454 * Cut down Resend / Check_IPI / IPI 455 * 456 * The logic is that we cannot have a pending interrupt 457 * trumped by an IPI at this point (see above), so we 458 * know that either the pending interrupt is already an 459 * IPI (in which case we don't care to override it) or 460 * it's either more favored than us or non existent 461 */ 462 if (new_state.mfrr < new_cppr && 463 new_state.mfrr <= new_state.pending_pri) { 464 new_state.pending_pri = new_state.mfrr; 465 new_state.xisr = XICS_IPI; 466 } 467 468 /* Latch/clear resend bit */ 469 resend = new_state.need_resend; 470 new_state.need_resend = 0; 471 472 } while (!icp_rm_try_update(icp, old_state, new_state)); 473 474 /* 475 * Now handle resend checks. Those are asynchronous to the ICP 476 * state update in HW (ie bus transactions) so we can handle them 477 * separately here as well. 478 */ 479 if (resend) { 480 icp->n_check_resend++; 481 icp_rm_check_resend(xics, icp); 482 } 483 } 484 485 486 unsigned long xics_rm_h_xirr(struct kvm_vcpu *vcpu) 487 { 488 union kvmppc_icp_state old_state, new_state; 489 struct kvmppc_xics *xics = vcpu->kvm->arch.xics; 490 struct kvmppc_icp *icp = vcpu->arch.icp; 491 u32 xirr; 492 493 if (!xics || !xics->real_mode) 494 return H_TOO_HARD; 495 496 /* First clear the interrupt */ 497 icp_rm_clr_vcpu_irq(icp->vcpu); 498 499 /* 500 * ICP State: Accept_Interrupt 501 * 502 * Return the pending interrupt (if any) along with the 503 * current CPPR, then clear the XISR & set CPPR to the 504 * pending priority 505 */ 506 do { 507 old_state = new_state = READ_ONCE(icp->state); 508 509 xirr = old_state.xisr | (((u32)old_state.cppr) << 24); 510 if (!old_state.xisr) 511 break; 512 new_state.cppr = new_state.pending_pri; 513 new_state.pending_pri = 0xff; 514 new_state.xisr = 0; 515 516 } while (!icp_rm_try_update(icp, old_state, new_state)); 517 518 /* Return the result in GPR4 */ 519 vcpu->arch.regs.gpr[4] = xirr; 520 521 return check_too_hard(xics, icp); 522 } 523 524 int xics_rm_h_ipi(struct kvm_vcpu *vcpu, unsigned long server, 525 unsigned long mfrr) 526 { 527 union kvmppc_icp_state old_state, new_state; 528 struct kvmppc_xics *xics = vcpu->kvm->arch.xics; 529 struct kvmppc_icp *icp, *this_icp = vcpu->arch.icp; 530 u32 reject; 531 bool resend; 532 bool local; 533 534 if (!xics || !xics->real_mode) 535 return H_TOO_HARD; 536 537 local = this_icp->server_num == server; 538 if (local) 539 icp = this_icp; 540 else 541 icp = kvmppc_xics_find_server(vcpu->kvm, server); 542 if (!icp) 543 return H_PARAMETER; 544 545 /* 546 * ICP state: Set_MFRR 547 * 548 * If the CPPR is more favored than the new MFRR, then 549 * nothing needs to be done as there can be no XISR to 550 * reject. 551 * 552 * ICP state: Check_IPI 553 * 554 * If the CPPR is less favored, then we might be replacing 555 * an interrupt, and thus need to possibly reject it. 556 * 557 * ICP State: IPI 558 * 559 * Besides rejecting any pending interrupts, we also 560 * update XISR and pending_pri to mark IPI as pending. 561 * 562 * PAPR does not describe this state, but if the MFRR is being 563 * made less favored than its earlier value, there might be 564 * a previously-rejected interrupt needing to be resent. 565 * Ideally, we would want to resend only if 566 * prio(pending_interrupt) < mfrr && 567 * prio(pending_interrupt) < cppr 568 * where pending interrupt is the one that was rejected. But 569 * we don't have that state, so we simply trigger a resend 570 * whenever the MFRR is made less favored. 571 */ 572 do { 573 old_state = new_state = READ_ONCE(icp->state); 574 575 /* Set_MFRR */ 576 new_state.mfrr = mfrr; 577 578 /* Check_IPI */ 579 reject = 0; 580 resend = false; 581 if (mfrr < new_state.cppr) { 582 /* Reject a pending interrupt if not an IPI */ 583 if (mfrr <= new_state.pending_pri) { 584 reject = new_state.xisr; 585 new_state.pending_pri = mfrr; 586 new_state.xisr = XICS_IPI; 587 } 588 } 589 590 if (mfrr > old_state.mfrr) { 591 resend = new_state.need_resend; 592 new_state.need_resend = 0; 593 } 594 } while (!icp_rm_try_update(icp, old_state, new_state)); 595 596 /* Handle reject in real mode */ 597 if (reject && reject != XICS_IPI) { 598 this_icp->n_reject++; 599 icp_rm_deliver_irq(xics, icp, reject, false); 600 } 601 602 /* Handle resends in real mode */ 603 if (resend) { 604 this_icp->n_check_resend++; 605 icp_rm_check_resend(xics, icp); 606 } 607 608 return check_too_hard(xics, this_icp); 609 } 610 611 int xics_rm_h_cppr(struct kvm_vcpu *vcpu, unsigned long cppr) 612 { 613 union kvmppc_icp_state old_state, new_state; 614 struct kvmppc_xics *xics = vcpu->kvm->arch.xics; 615 struct kvmppc_icp *icp = vcpu->arch.icp; 616 u32 reject; 617 618 if (!xics || !xics->real_mode) 619 return H_TOO_HARD; 620 621 /* 622 * ICP State: Set_CPPR 623 * 624 * We can safely compare the new value with the current 625 * value outside of the transaction as the CPPR is only 626 * ever changed by the processor on itself 627 */ 628 if (cppr > icp->state.cppr) { 629 icp_rm_down_cppr(xics, icp, cppr); 630 goto bail; 631 } else if (cppr == icp->state.cppr) 632 return H_SUCCESS; 633 634 /* 635 * ICP State: Up_CPPR 636 * 637 * The processor is raising its priority, this can result 638 * in a rejection of a pending interrupt: 639 * 640 * ICP State: Reject_Current 641 * 642 * We can remove EE from the current processor, the update 643 * transaction will set it again if needed 644 */ 645 icp_rm_clr_vcpu_irq(icp->vcpu); 646 647 do { 648 old_state = new_state = READ_ONCE(icp->state); 649 650 reject = 0; 651 new_state.cppr = cppr; 652 653 if (cppr <= new_state.pending_pri) { 654 reject = new_state.xisr; 655 new_state.xisr = 0; 656 new_state.pending_pri = 0xff; 657 } 658 659 } while (!icp_rm_try_update(icp, old_state, new_state)); 660 661 /* 662 * Check for rejects. They are handled by doing a new delivery 663 * attempt (see comments in icp_rm_deliver_irq). 664 */ 665 if (reject && reject != XICS_IPI) { 666 icp->n_reject++; 667 icp_rm_deliver_irq(xics, icp, reject, false); 668 } 669 bail: 670 return check_too_hard(xics, icp); 671 } 672 673 static int ics_rm_eoi(struct kvm_vcpu *vcpu, u32 irq) 674 { 675 struct kvmppc_xics *xics = vcpu->kvm->arch.xics; 676 struct kvmppc_icp *icp = vcpu->arch.icp; 677 struct kvmppc_ics *ics; 678 struct ics_irq_state *state; 679 u16 src; 680 u32 pq_old, pq_new; 681 682 /* 683 * ICS EOI handling: For LSI, if P bit is still set, we need to 684 * resend it. 685 * 686 * For MSI, we move Q bit into P (and clear Q). If it is set, 687 * resend it. 688 */ 689 690 ics = kvmppc_xics_find_ics(xics, irq, &src); 691 if (!ics) 692 goto bail; 693 694 state = &ics->irq_state[src]; 695 696 if (state->lsi) 697 pq_new = state->pq_state; 698 else 699 do { 700 pq_old = state->pq_state; 701 pq_new = pq_old >> 1; 702 } while (cmpxchg(&state->pq_state, pq_old, pq_new) != pq_old); 703 704 if (pq_new & PQ_PRESENTED) 705 icp_rm_deliver_irq(xics, NULL, irq, false); 706 707 if (!hlist_empty(&vcpu->kvm->irq_ack_notifier_list)) { 708 icp->rm_action |= XICS_RM_NOTIFY_EOI; 709 icp->rm_eoied_irq = irq; 710 } 711 712 if (state->host_irq) { 713 ++vcpu->stat.pthru_all; 714 if (state->intr_cpu != -1) { 715 int pcpu = raw_smp_processor_id(); 716 717 pcpu = cpu_first_thread_sibling(pcpu); 718 ++vcpu->stat.pthru_host; 719 if (state->intr_cpu != pcpu) { 720 ++vcpu->stat.pthru_bad_aff; 721 xics_opal_set_server(state->host_irq, pcpu); 722 } 723 state->intr_cpu = -1; 724 } 725 } 726 727 bail: 728 return check_too_hard(xics, icp); 729 } 730 731 int xics_rm_h_eoi(struct kvm_vcpu *vcpu, unsigned long xirr) 732 { 733 struct kvmppc_xics *xics = vcpu->kvm->arch.xics; 734 struct kvmppc_icp *icp = vcpu->arch.icp; 735 u32 irq = xirr & 0x00ffffff; 736 737 if (!xics || !xics->real_mode) 738 return H_TOO_HARD; 739 740 /* 741 * ICP State: EOI 742 * 743 * Note: If EOI is incorrectly used by SW to lower the CPPR 744 * value (ie more favored), we do not check for rejection of 745 * a pending interrupt, this is a SW error and PAPR specifies 746 * that we don't have to deal with it. 747 * 748 * The sending of an EOI to the ICS is handled after the 749 * CPPR update 750 * 751 * ICP State: Down_CPPR which we handle 752 * in a separate function as it's shared with H_CPPR. 753 */ 754 icp_rm_down_cppr(xics, icp, xirr >> 24); 755 756 /* IPIs have no EOI */ 757 if (irq == XICS_IPI) 758 return check_too_hard(xics, icp); 759 760 return ics_rm_eoi(vcpu, irq); 761 } 762 763 unsigned long eoi_rc; 764 765 static void icp_eoi(struct irq_chip *c, u32 hwirq, __be32 xirr, bool *again) 766 { 767 void __iomem *xics_phys; 768 int64_t rc; 769 770 if (kvmhv_on_pseries()) { 771 unsigned long retbuf[PLPAR_HCALL_BUFSIZE]; 772 773 iosync(); 774 plpar_hcall_raw(H_EOI, retbuf, hwirq); 775 return; 776 } 777 778 rc = pnv_opal_pci_msi_eoi(c, hwirq); 779 780 if (rc) 781 eoi_rc = rc; 782 783 iosync(); 784 785 /* EOI it */ 786 xics_phys = local_paca->kvm_hstate.xics_phys; 787 if (xics_phys) { 788 __raw_rm_writel(xirr, xics_phys + XICS_XIRR); 789 } else { 790 rc = opal_int_eoi(be32_to_cpu(xirr)); 791 *again = rc > 0; 792 } 793 } 794 795 static int xics_opal_set_server(unsigned int hw_irq, int server_cpu) 796 { 797 unsigned int mangle_cpu = get_hard_smp_processor_id(server_cpu) << 2; 798 799 return opal_set_xive(hw_irq, mangle_cpu, DEFAULT_PRIORITY); 800 } 801 802 /* 803 * Increment a per-CPU 32-bit unsigned integer variable. 804 * Safe to call in real-mode. Handles vmalloc'ed addresses 805 * 806 * ToDo: Make this work for any integral type 807 */ 808 809 static inline void this_cpu_inc_rm(unsigned int __percpu *addr) 810 { 811 unsigned long l; 812 unsigned int *raddr; 813 int cpu = smp_processor_id(); 814 815 raddr = per_cpu_ptr(addr, cpu); 816 l = (unsigned long)raddr; 817 818 if (REGION_ID(l) == VMALLOC_REGION_ID) { 819 l = vmalloc_to_phys(raddr); 820 raddr = (unsigned int *)l; 821 } 822 ++*raddr; 823 } 824 825 /* 826 * We don't try to update the flags in the irq_desc 'istate' field in 827 * here as would happen in the normal IRQ handling path for several reasons: 828 * - state flags represent internal IRQ state and are not expected to be 829 * updated outside the IRQ subsystem 830 * - more importantly, these are useful for edge triggered interrupts, 831 * IRQ probing, etc., but we are only handling MSI/MSIx interrupts here 832 * and these states shouldn't apply to us. 833 * 834 * However, we do update irq_stats - we somewhat duplicate the code in 835 * kstat_incr_irqs_this_cpu() for this since this function is defined 836 * in irq/internal.h which we don't want to include here. 837 * The only difference is that desc->kstat_irqs is an allocated per CPU 838 * variable and could have been vmalloc'ed, so we can't directly 839 * call __this_cpu_inc() on it. The kstat structure is a static 840 * per CPU variable and it should be accessible by real-mode KVM. 841 * 842 */ 843 static void kvmppc_rm_handle_irq_desc(struct irq_desc *desc) 844 { 845 this_cpu_inc_rm(desc->kstat_irqs); 846 __this_cpu_inc(kstat.irqs_sum); 847 } 848 849 long kvmppc_deliver_irq_passthru(struct kvm_vcpu *vcpu, 850 __be32 xirr, 851 struct kvmppc_irq_map *irq_map, 852 struct kvmppc_passthru_irqmap *pimap, 853 bool *again) 854 { 855 struct kvmppc_xics *xics; 856 struct kvmppc_icp *icp; 857 struct kvmppc_ics *ics; 858 struct ics_irq_state *state; 859 u32 irq; 860 u16 src; 861 u32 pq_old, pq_new; 862 863 irq = irq_map->v_hwirq; 864 xics = vcpu->kvm->arch.xics; 865 icp = vcpu->arch.icp; 866 867 kvmppc_rm_handle_irq_desc(irq_map->desc); 868 869 ics = kvmppc_xics_find_ics(xics, irq, &src); 870 if (!ics) 871 return 2; 872 873 state = &ics->irq_state[src]; 874 875 /* only MSIs register bypass producers, so it must be MSI here */ 876 do { 877 pq_old = state->pq_state; 878 pq_new = ((pq_old << 1) & 3) | PQ_PRESENTED; 879 } while (cmpxchg(&state->pq_state, pq_old, pq_new) != pq_old); 880 881 /* Test P=1, Q=0, this is the only case where we present */ 882 if (pq_new == PQ_PRESENTED) 883 icp_rm_deliver_irq(xics, icp, irq, false); 884 885 /* EOI the interrupt */ 886 icp_eoi(irq_desc_get_chip(irq_map->desc), irq_map->r_hwirq, xirr, 887 again); 888 889 if (check_too_hard(xics, icp) == H_TOO_HARD) 890 return 2; 891 else 892 return -2; 893 } 894 895 /* --- Non-real mode XICS-related built-in routines --- */ 896 897 /** 898 * Host Operations poked by RM KVM 899 */ 900 static void rm_host_ipi_action(int action, void *data) 901 { 902 switch (action) { 903 case XICS_RM_KICK_VCPU: 904 kvmppc_host_rm_ops_hv->vcpu_kick(data); 905 break; 906 default: 907 WARN(1, "Unexpected rm_action=%d data=%p\n", action, data); 908 break; 909 } 910 911 } 912 913 void kvmppc_xics_ipi_action(void) 914 { 915 int core; 916 unsigned int cpu = smp_processor_id(); 917 struct kvmppc_host_rm_core *rm_corep; 918 919 core = cpu >> threads_shift; 920 rm_corep = &kvmppc_host_rm_ops_hv->rm_core[core]; 921 922 if (rm_corep->rm_data) { 923 rm_host_ipi_action(rm_corep->rm_state.rm_action, 924 rm_corep->rm_data); 925 /* Order these stores against the real mode KVM */ 926 rm_corep->rm_data = NULL; 927 smp_wmb(); 928 rm_corep->rm_state.rm_action = 0; 929 } 930 } 931