1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * handling kvm guest interrupts 4 * 5 * Copyright IBM Corp. 2008, 2015 6 * 7 * Author(s): Carsten Otte <cotte@de.ibm.com> 8 */ 9 10 #include <linux/interrupt.h> 11 #include <linux/kvm_host.h> 12 #include <linux/hrtimer.h> 13 #include <linux/mmu_context.h> 14 #include <linux/signal.h> 15 #include <linux/slab.h> 16 #include <linux/bitmap.h> 17 #include <linux/vmalloc.h> 18 #include <asm/asm-offsets.h> 19 #include <asm/dis.h> 20 #include <linux/uaccess.h> 21 #include <asm/sclp.h> 22 #include <asm/isc.h> 23 #include <asm/gmap.h> 24 #include <asm/switch_to.h> 25 #include <asm/nmi.h> 26 #include "kvm-s390.h" 27 #include "gaccess.h" 28 #include "trace-s390.h" 29 30 #define PFAULT_INIT 0x0600 31 #define PFAULT_DONE 0x0680 32 #define VIRTIO_PARAM 0x0d00 33 34 /* handle external calls via sigp interpretation facility */ 35 static int sca_ext_call_pending(struct kvm_vcpu *vcpu, int *src_id) 36 { 37 int c, scn; 38 39 if (!(atomic_read(&vcpu->arch.sie_block->cpuflags) & CPUSTAT_ECALL_PEND)) 40 return 0; 41 42 BUG_ON(!kvm_s390_use_sca_entries()); 43 read_lock(&vcpu->kvm->arch.sca_lock); 44 if (vcpu->kvm->arch.use_esca) { 45 struct esca_block *sca = vcpu->kvm->arch.sca; 46 union esca_sigp_ctrl sigp_ctrl = 47 sca->cpu[vcpu->vcpu_id].sigp_ctrl; 48 49 c = sigp_ctrl.c; 50 scn = sigp_ctrl.scn; 51 } else { 52 struct bsca_block *sca = vcpu->kvm->arch.sca; 53 union bsca_sigp_ctrl sigp_ctrl = 54 sca->cpu[vcpu->vcpu_id].sigp_ctrl; 55 56 c = sigp_ctrl.c; 57 scn = sigp_ctrl.scn; 58 } 59 read_unlock(&vcpu->kvm->arch.sca_lock); 60 61 if (src_id) 62 *src_id = scn; 63 64 return c; 65 } 66 67 static int sca_inject_ext_call(struct kvm_vcpu *vcpu, int src_id) 68 { 69 int expect, rc; 70 71 BUG_ON(!kvm_s390_use_sca_entries()); 72 read_lock(&vcpu->kvm->arch.sca_lock); 73 if (vcpu->kvm->arch.use_esca) { 74 struct esca_block *sca = vcpu->kvm->arch.sca; 75 union esca_sigp_ctrl *sigp_ctrl = 76 &(sca->cpu[vcpu->vcpu_id].sigp_ctrl); 77 union esca_sigp_ctrl new_val = {0}, old_val = *sigp_ctrl; 78 79 new_val.scn = src_id; 80 new_val.c = 1; 81 old_val.c = 0; 82 83 expect = old_val.value; 84 rc = cmpxchg(&sigp_ctrl->value, old_val.value, new_val.value); 85 } else { 86 struct bsca_block *sca = vcpu->kvm->arch.sca; 87 union bsca_sigp_ctrl *sigp_ctrl = 88 &(sca->cpu[vcpu->vcpu_id].sigp_ctrl); 89 union bsca_sigp_ctrl new_val = {0}, old_val = *sigp_ctrl; 90 91 new_val.scn = src_id; 92 new_val.c = 1; 93 old_val.c = 0; 94 95 expect = old_val.value; 96 rc = cmpxchg(&sigp_ctrl->value, old_val.value, new_val.value); 97 } 98 read_unlock(&vcpu->kvm->arch.sca_lock); 99 100 if (rc != expect) { 101 /* another external call is pending */ 102 return -EBUSY; 103 } 104 atomic_or(CPUSTAT_ECALL_PEND, &vcpu->arch.sie_block->cpuflags); 105 return 0; 106 } 107 108 static void sca_clear_ext_call(struct kvm_vcpu *vcpu) 109 { 110 struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int; 111 int rc, expect; 112 113 if (!kvm_s390_use_sca_entries()) 114 return; 115 atomic_andnot(CPUSTAT_ECALL_PEND, li->cpuflags); 116 read_lock(&vcpu->kvm->arch.sca_lock); 117 if (vcpu->kvm->arch.use_esca) { 118 struct esca_block *sca = vcpu->kvm->arch.sca; 119 union esca_sigp_ctrl *sigp_ctrl = 120 &(sca->cpu[vcpu->vcpu_id].sigp_ctrl); 121 union esca_sigp_ctrl old = *sigp_ctrl; 122 123 expect = old.value; 124 rc = cmpxchg(&sigp_ctrl->value, old.value, 0); 125 } else { 126 struct bsca_block *sca = vcpu->kvm->arch.sca; 127 union bsca_sigp_ctrl *sigp_ctrl = 128 &(sca->cpu[vcpu->vcpu_id].sigp_ctrl); 129 union bsca_sigp_ctrl old = *sigp_ctrl; 130 131 expect = old.value; 132 rc = cmpxchg(&sigp_ctrl->value, old.value, 0); 133 } 134 read_unlock(&vcpu->kvm->arch.sca_lock); 135 WARN_ON(rc != expect); /* cannot clear? */ 136 } 137 138 int psw_extint_disabled(struct kvm_vcpu *vcpu) 139 { 140 return !(vcpu->arch.sie_block->gpsw.mask & PSW_MASK_EXT); 141 } 142 143 static int psw_ioint_disabled(struct kvm_vcpu *vcpu) 144 { 145 return !(vcpu->arch.sie_block->gpsw.mask & PSW_MASK_IO); 146 } 147 148 static int psw_mchk_disabled(struct kvm_vcpu *vcpu) 149 { 150 return !(vcpu->arch.sie_block->gpsw.mask & PSW_MASK_MCHECK); 151 } 152 153 static int psw_interrupts_disabled(struct kvm_vcpu *vcpu) 154 { 155 return psw_extint_disabled(vcpu) && 156 psw_ioint_disabled(vcpu) && 157 psw_mchk_disabled(vcpu); 158 } 159 160 static int ckc_interrupts_enabled(struct kvm_vcpu *vcpu) 161 { 162 if (psw_extint_disabled(vcpu) || 163 !(vcpu->arch.sie_block->gcr[0] & 0x800ul)) 164 return 0; 165 if (guestdbg_enabled(vcpu) && guestdbg_sstep_enabled(vcpu)) 166 /* No timer interrupts when single stepping */ 167 return 0; 168 return 1; 169 } 170 171 static int ckc_irq_pending(struct kvm_vcpu *vcpu) 172 { 173 if (vcpu->arch.sie_block->ckc >= kvm_s390_get_tod_clock_fast(vcpu->kvm)) 174 return 0; 175 return ckc_interrupts_enabled(vcpu); 176 } 177 178 static int cpu_timer_interrupts_enabled(struct kvm_vcpu *vcpu) 179 { 180 return !psw_extint_disabled(vcpu) && 181 (vcpu->arch.sie_block->gcr[0] & 0x400ul); 182 } 183 184 static int cpu_timer_irq_pending(struct kvm_vcpu *vcpu) 185 { 186 if (!cpu_timer_interrupts_enabled(vcpu)) 187 return 0; 188 return kvm_s390_get_cpu_timer(vcpu) >> 63; 189 } 190 191 static inline int is_ioirq(unsigned long irq_type) 192 { 193 return ((irq_type >= IRQ_PEND_IO_ISC_0) && 194 (irq_type <= IRQ_PEND_IO_ISC_7)); 195 } 196 197 static uint64_t isc_to_isc_bits(int isc) 198 { 199 return (0x80 >> isc) << 24; 200 } 201 202 static inline u8 int_word_to_isc(u32 int_word) 203 { 204 return (int_word & 0x38000000) >> 27; 205 } 206 207 static inline unsigned long pending_irqs(struct kvm_vcpu *vcpu) 208 { 209 return vcpu->kvm->arch.float_int.pending_irqs | 210 vcpu->arch.local_int.pending_irqs; 211 } 212 213 static inline int isc_to_irq_type(unsigned long isc) 214 { 215 return IRQ_PEND_IO_ISC_0 + isc; 216 } 217 218 static inline int irq_type_to_isc(unsigned long irq_type) 219 { 220 return irq_type - IRQ_PEND_IO_ISC_0; 221 } 222 223 static unsigned long disable_iscs(struct kvm_vcpu *vcpu, 224 unsigned long active_mask) 225 { 226 int i; 227 228 for (i = 0; i <= MAX_ISC; i++) 229 if (!(vcpu->arch.sie_block->gcr[6] & isc_to_isc_bits(i))) 230 active_mask &= ~(1UL << (isc_to_irq_type(i))); 231 232 return active_mask; 233 } 234 235 static unsigned long deliverable_irqs(struct kvm_vcpu *vcpu) 236 { 237 unsigned long active_mask; 238 239 active_mask = pending_irqs(vcpu); 240 if (!active_mask) 241 return 0; 242 243 if (psw_extint_disabled(vcpu)) 244 active_mask &= ~IRQ_PEND_EXT_MASK; 245 if (psw_ioint_disabled(vcpu)) 246 active_mask &= ~IRQ_PEND_IO_MASK; 247 else 248 active_mask = disable_iscs(vcpu, active_mask); 249 if (!(vcpu->arch.sie_block->gcr[0] & 0x2000ul)) 250 __clear_bit(IRQ_PEND_EXT_EXTERNAL, &active_mask); 251 if (!(vcpu->arch.sie_block->gcr[0] & 0x4000ul)) 252 __clear_bit(IRQ_PEND_EXT_EMERGENCY, &active_mask); 253 if (!(vcpu->arch.sie_block->gcr[0] & 0x800ul)) 254 __clear_bit(IRQ_PEND_EXT_CLOCK_COMP, &active_mask); 255 if (!(vcpu->arch.sie_block->gcr[0] & 0x400ul)) 256 __clear_bit(IRQ_PEND_EXT_CPU_TIMER, &active_mask); 257 if (!(vcpu->arch.sie_block->gcr[0] & 0x200ul)) 258 __clear_bit(IRQ_PEND_EXT_SERVICE, &active_mask); 259 if (psw_mchk_disabled(vcpu)) 260 active_mask &= ~IRQ_PEND_MCHK_MASK; 261 /* 262 * Check both floating and local interrupt's cr14 because 263 * bit IRQ_PEND_MCHK_REP could be set in both cases. 264 */ 265 if (!(vcpu->arch.sie_block->gcr[14] & 266 (vcpu->kvm->arch.float_int.mchk.cr14 | 267 vcpu->arch.local_int.irq.mchk.cr14))) 268 __clear_bit(IRQ_PEND_MCHK_REP, &active_mask); 269 270 /* 271 * STOP irqs will never be actively delivered. They are triggered via 272 * intercept requests and cleared when the stop intercept is performed. 273 */ 274 __clear_bit(IRQ_PEND_SIGP_STOP, &active_mask); 275 276 return active_mask; 277 } 278 279 static void __set_cpu_idle(struct kvm_vcpu *vcpu) 280 { 281 atomic_or(CPUSTAT_WAIT, &vcpu->arch.sie_block->cpuflags); 282 set_bit(vcpu->vcpu_id, vcpu->arch.local_int.float_int->idle_mask); 283 } 284 285 static void __unset_cpu_idle(struct kvm_vcpu *vcpu) 286 { 287 atomic_andnot(CPUSTAT_WAIT, &vcpu->arch.sie_block->cpuflags); 288 clear_bit(vcpu->vcpu_id, vcpu->arch.local_int.float_int->idle_mask); 289 } 290 291 static void __reset_intercept_indicators(struct kvm_vcpu *vcpu) 292 { 293 atomic_andnot(CPUSTAT_IO_INT | CPUSTAT_EXT_INT | CPUSTAT_STOP_INT, 294 &vcpu->arch.sie_block->cpuflags); 295 vcpu->arch.sie_block->lctl = 0x0000; 296 vcpu->arch.sie_block->ictl &= ~(ICTL_LPSW | ICTL_STCTL | ICTL_PINT); 297 298 if (guestdbg_enabled(vcpu)) { 299 vcpu->arch.sie_block->lctl |= (LCTL_CR0 | LCTL_CR9 | 300 LCTL_CR10 | LCTL_CR11); 301 vcpu->arch.sie_block->ictl |= (ICTL_STCTL | ICTL_PINT); 302 } 303 } 304 305 static void __set_cpuflag(struct kvm_vcpu *vcpu, u32 flag) 306 { 307 atomic_or(flag, &vcpu->arch.sie_block->cpuflags); 308 } 309 310 static void set_intercept_indicators_io(struct kvm_vcpu *vcpu) 311 { 312 if (!(pending_irqs(vcpu) & IRQ_PEND_IO_MASK)) 313 return; 314 else if (psw_ioint_disabled(vcpu)) 315 __set_cpuflag(vcpu, CPUSTAT_IO_INT); 316 else 317 vcpu->arch.sie_block->lctl |= LCTL_CR6; 318 } 319 320 static void set_intercept_indicators_ext(struct kvm_vcpu *vcpu) 321 { 322 if (!(pending_irqs(vcpu) & IRQ_PEND_EXT_MASK)) 323 return; 324 if (psw_extint_disabled(vcpu)) 325 __set_cpuflag(vcpu, CPUSTAT_EXT_INT); 326 else 327 vcpu->arch.sie_block->lctl |= LCTL_CR0; 328 } 329 330 static void set_intercept_indicators_mchk(struct kvm_vcpu *vcpu) 331 { 332 if (!(pending_irqs(vcpu) & IRQ_PEND_MCHK_MASK)) 333 return; 334 if (psw_mchk_disabled(vcpu)) 335 vcpu->arch.sie_block->ictl |= ICTL_LPSW; 336 else 337 vcpu->arch.sie_block->lctl |= LCTL_CR14; 338 } 339 340 static void set_intercept_indicators_stop(struct kvm_vcpu *vcpu) 341 { 342 if (kvm_s390_is_stop_irq_pending(vcpu)) 343 __set_cpuflag(vcpu, CPUSTAT_STOP_INT); 344 } 345 346 /* Set interception request for non-deliverable interrupts */ 347 static void set_intercept_indicators(struct kvm_vcpu *vcpu) 348 { 349 set_intercept_indicators_io(vcpu); 350 set_intercept_indicators_ext(vcpu); 351 set_intercept_indicators_mchk(vcpu); 352 set_intercept_indicators_stop(vcpu); 353 } 354 355 static int __must_check __deliver_cpu_timer(struct kvm_vcpu *vcpu) 356 { 357 struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int; 358 int rc; 359 360 trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, KVM_S390_INT_CPU_TIMER, 361 0, 0); 362 363 rc = put_guest_lc(vcpu, EXT_IRQ_CPU_TIMER, 364 (u16 *)__LC_EXT_INT_CODE); 365 rc |= put_guest_lc(vcpu, 0, (u16 *)__LC_EXT_CPU_ADDR); 366 rc |= write_guest_lc(vcpu, __LC_EXT_OLD_PSW, 367 &vcpu->arch.sie_block->gpsw, sizeof(psw_t)); 368 rc |= read_guest_lc(vcpu, __LC_EXT_NEW_PSW, 369 &vcpu->arch.sie_block->gpsw, sizeof(psw_t)); 370 clear_bit(IRQ_PEND_EXT_CPU_TIMER, &li->pending_irqs); 371 return rc ? -EFAULT : 0; 372 } 373 374 static int __must_check __deliver_ckc(struct kvm_vcpu *vcpu) 375 { 376 struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int; 377 int rc; 378 379 trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, KVM_S390_INT_CLOCK_COMP, 380 0, 0); 381 382 rc = put_guest_lc(vcpu, EXT_IRQ_CLK_COMP, 383 (u16 __user *)__LC_EXT_INT_CODE); 384 rc |= put_guest_lc(vcpu, 0, (u16 *)__LC_EXT_CPU_ADDR); 385 rc |= write_guest_lc(vcpu, __LC_EXT_OLD_PSW, 386 &vcpu->arch.sie_block->gpsw, sizeof(psw_t)); 387 rc |= read_guest_lc(vcpu, __LC_EXT_NEW_PSW, 388 &vcpu->arch.sie_block->gpsw, sizeof(psw_t)); 389 clear_bit(IRQ_PEND_EXT_CLOCK_COMP, &li->pending_irqs); 390 return rc ? -EFAULT : 0; 391 } 392 393 static int __must_check __deliver_pfault_init(struct kvm_vcpu *vcpu) 394 { 395 struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int; 396 struct kvm_s390_ext_info ext; 397 int rc; 398 399 spin_lock(&li->lock); 400 ext = li->irq.ext; 401 clear_bit(IRQ_PEND_PFAULT_INIT, &li->pending_irqs); 402 li->irq.ext.ext_params2 = 0; 403 spin_unlock(&li->lock); 404 405 VCPU_EVENT(vcpu, 4, "deliver: pfault init token 0x%llx", 406 ext.ext_params2); 407 trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, 408 KVM_S390_INT_PFAULT_INIT, 409 0, ext.ext_params2); 410 411 rc = put_guest_lc(vcpu, EXT_IRQ_CP_SERVICE, (u16 *) __LC_EXT_INT_CODE); 412 rc |= put_guest_lc(vcpu, PFAULT_INIT, (u16 *) __LC_EXT_CPU_ADDR); 413 rc |= write_guest_lc(vcpu, __LC_EXT_OLD_PSW, 414 &vcpu->arch.sie_block->gpsw, sizeof(psw_t)); 415 rc |= read_guest_lc(vcpu, __LC_EXT_NEW_PSW, 416 &vcpu->arch.sie_block->gpsw, sizeof(psw_t)); 417 rc |= put_guest_lc(vcpu, ext.ext_params2, (u64 *) __LC_EXT_PARAMS2); 418 return rc ? -EFAULT : 0; 419 } 420 421 static int __write_machine_check(struct kvm_vcpu *vcpu, 422 struct kvm_s390_mchk_info *mchk) 423 { 424 unsigned long ext_sa_addr; 425 unsigned long lc; 426 freg_t fprs[NUM_FPRS]; 427 union mci mci; 428 int rc; 429 430 mci.val = mchk->mcic; 431 /* take care of lazy register loading */ 432 save_fpu_regs(); 433 save_access_regs(vcpu->run->s.regs.acrs); 434 if (MACHINE_HAS_GS && vcpu->arch.gs_enabled) 435 save_gs_cb(current->thread.gs_cb); 436 437 /* Extended save area */ 438 rc = read_guest_lc(vcpu, __LC_MCESAD, &ext_sa_addr, 439 sizeof(unsigned long)); 440 /* Only bits 0 through 63-LC are used for address formation */ 441 lc = ext_sa_addr & MCESA_LC_MASK; 442 if (test_kvm_facility(vcpu->kvm, 133)) { 443 switch (lc) { 444 case 0: 445 case 10: 446 ext_sa_addr &= ~0x3ffUL; 447 break; 448 case 11: 449 ext_sa_addr &= ~0x7ffUL; 450 break; 451 case 12: 452 ext_sa_addr &= ~0xfffUL; 453 break; 454 default: 455 ext_sa_addr = 0; 456 break; 457 } 458 } else { 459 ext_sa_addr &= ~0x3ffUL; 460 } 461 462 if (!rc && mci.vr && ext_sa_addr && test_kvm_facility(vcpu->kvm, 129)) { 463 if (write_guest_abs(vcpu, ext_sa_addr, vcpu->run->s.regs.vrs, 464 512)) 465 mci.vr = 0; 466 } else { 467 mci.vr = 0; 468 } 469 if (!rc && mci.gs && ext_sa_addr && test_kvm_facility(vcpu->kvm, 133) 470 && (lc == 11 || lc == 12)) { 471 if (write_guest_abs(vcpu, ext_sa_addr + 1024, 472 &vcpu->run->s.regs.gscb, 32)) 473 mci.gs = 0; 474 } else { 475 mci.gs = 0; 476 } 477 478 /* General interruption information */ 479 rc |= put_guest_lc(vcpu, 1, (u8 __user *) __LC_AR_MODE_ID); 480 rc |= write_guest_lc(vcpu, __LC_MCK_OLD_PSW, 481 &vcpu->arch.sie_block->gpsw, sizeof(psw_t)); 482 rc |= read_guest_lc(vcpu, __LC_MCK_NEW_PSW, 483 &vcpu->arch.sie_block->gpsw, sizeof(psw_t)); 484 rc |= put_guest_lc(vcpu, mci.val, (u64 __user *) __LC_MCCK_CODE); 485 486 /* Register-save areas */ 487 if (MACHINE_HAS_VX) { 488 convert_vx_to_fp(fprs, (__vector128 *) vcpu->run->s.regs.vrs); 489 rc |= write_guest_lc(vcpu, __LC_FPREGS_SAVE_AREA, fprs, 128); 490 } else { 491 rc |= write_guest_lc(vcpu, __LC_FPREGS_SAVE_AREA, 492 vcpu->run->s.regs.fprs, 128); 493 } 494 rc |= write_guest_lc(vcpu, __LC_GPREGS_SAVE_AREA, 495 vcpu->run->s.regs.gprs, 128); 496 rc |= put_guest_lc(vcpu, current->thread.fpu.fpc, 497 (u32 __user *) __LC_FP_CREG_SAVE_AREA); 498 rc |= put_guest_lc(vcpu, vcpu->arch.sie_block->todpr, 499 (u32 __user *) __LC_TOD_PROGREG_SAVE_AREA); 500 rc |= put_guest_lc(vcpu, kvm_s390_get_cpu_timer(vcpu), 501 (u64 __user *) __LC_CPU_TIMER_SAVE_AREA); 502 rc |= put_guest_lc(vcpu, vcpu->arch.sie_block->ckc >> 8, 503 (u64 __user *) __LC_CLOCK_COMP_SAVE_AREA); 504 rc |= write_guest_lc(vcpu, __LC_AREGS_SAVE_AREA, 505 &vcpu->run->s.regs.acrs, 64); 506 rc |= write_guest_lc(vcpu, __LC_CREGS_SAVE_AREA, 507 &vcpu->arch.sie_block->gcr, 128); 508 509 /* Extended interruption information */ 510 rc |= put_guest_lc(vcpu, mchk->ext_damage_code, 511 (u32 __user *) __LC_EXT_DAMAGE_CODE); 512 rc |= put_guest_lc(vcpu, mchk->failing_storage_address, 513 (u64 __user *) __LC_MCCK_FAIL_STOR_ADDR); 514 rc |= write_guest_lc(vcpu, __LC_PSW_SAVE_AREA, &mchk->fixed_logout, 515 sizeof(mchk->fixed_logout)); 516 return rc ? -EFAULT : 0; 517 } 518 519 static int __must_check __deliver_machine_check(struct kvm_vcpu *vcpu) 520 { 521 struct kvm_s390_float_interrupt *fi = &vcpu->kvm->arch.float_int; 522 struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int; 523 struct kvm_s390_mchk_info mchk = {}; 524 int deliver = 0; 525 int rc = 0; 526 527 spin_lock(&fi->lock); 528 spin_lock(&li->lock); 529 if (test_bit(IRQ_PEND_MCHK_EX, &li->pending_irqs) || 530 test_bit(IRQ_PEND_MCHK_REP, &li->pending_irqs)) { 531 /* 532 * If there was an exigent machine check pending, then any 533 * repressible machine checks that might have been pending 534 * are indicated along with it, so always clear bits for 535 * repressible and exigent interrupts 536 */ 537 mchk = li->irq.mchk; 538 clear_bit(IRQ_PEND_MCHK_EX, &li->pending_irqs); 539 clear_bit(IRQ_PEND_MCHK_REP, &li->pending_irqs); 540 memset(&li->irq.mchk, 0, sizeof(mchk)); 541 deliver = 1; 542 } 543 /* 544 * We indicate floating repressible conditions along with 545 * other pending conditions. Channel Report Pending and Channel 546 * Subsystem damage are the only two and and are indicated by 547 * bits in mcic and masked in cr14. 548 */ 549 if (test_and_clear_bit(IRQ_PEND_MCHK_REP, &fi->pending_irqs)) { 550 mchk.mcic |= fi->mchk.mcic; 551 mchk.cr14 |= fi->mchk.cr14; 552 memset(&fi->mchk, 0, sizeof(mchk)); 553 deliver = 1; 554 } 555 spin_unlock(&li->lock); 556 spin_unlock(&fi->lock); 557 558 if (deliver) { 559 VCPU_EVENT(vcpu, 3, "deliver: machine check mcic 0x%llx", 560 mchk.mcic); 561 trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, 562 KVM_S390_MCHK, 563 mchk.cr14, mchk.mcic); 564 rc = __write_machine_check(vcpu, &mchk); 565 } 566 return rc; 567 } 568 569 static int __must_check __deliver_restart(struct kvm_vcpu *vcpu) 570 { 571 struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int; 572 int rc; 573 574 VCPU_EVENT(vcpu, 3, "%s", "deliver: cpu restart"); 575 vcpu->stat.deliver_restart_signal++; 576 trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, KVM_S390_RESTART, 0, 0); 577 578 rc = write_guest_lc(vcpu, 579 offsetof(struct lowcore, restart_old_psw), 580 &vcpu->arch.sie_block->gpsw, sizeof(psw_t)); 581 rc |= read_guest_lc(vcpu, offsetof(struct lowcore, restart_psw), 582 &vcpu->arch.sie_block->gpsw, sizeof(psw_t)); 583 clear_bit(IRQ_PEND_RESTART, &li->pending_irqs); 584 return rc ? -EFAULT : 0; 585 } 586 587 static int __must_check __deliver_set_prefix(struct kvm_vcpu *vcpu) 588 { 589 struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int; 590 struct kvm_s390_prefix_info prefix; 591 592 spin_lock(&li->lock); 593 prefix = li->irq.prefix; 594 li->irq.prefix.address = 0; 595 clear_bit(IRQ_PEND_SET_PREFIX, &li->pending_irqs); 596 spin_unlock(&li->lock); 597 598 vcpu->stat.deliver_prefix_signal++; 599 trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, 600 KVM_S390_SIGP_SET_PREFIX, 601 prefix.address, 0); 602 603 kvm_s390_set_prefix(vcpu, prefix.address); 604 return 0; 605 } 606 607 static int __must_check __deliver_emergency_signal(struct kvm_vcpu *vcpu) 608 { 609 struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int; 610 int rc; 611 int cpu_addr; 612 613 spin_lock(&li->lock); 614 cpu_addr = find_first_bit(li->sigp_emerg_pending, KVM_MAX_VCPUS); 615 clear_bit(cpu_addr, li->sigp_emerg_pending); 616 if (bitmap_empty(li->sigp_emerg_pending, KVM_MAX_VCPUS)) 617 clear_bit(IRQ_PEND_EXT_EMERGENCY, &li->pending_irqs); 618 spin_unlock(&li->lock); 619 620 VCPU_EVENT(vcpu, 4, "%s", "deliver: sigp emerg"); 621 vcpu->stat.deliver_emergency_signal++; 622 trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, KVM_S390_INT_EMERGENCY, 623 cpu_addr, 0); 624 625 rc = put_guest_lc(vcpu, EXT_IRQ_EMERGENCY_SIG, 626 (u16 *)__LC_EXT_INT_CODE); 627 rc |= put_guest_lc(vcpu, cpu_addr, (u16 *)__LC_EXT_CPU_ADDR); 628 rc |= write_guest_lc(vcpu, __LC_EXT_OLD_PSW, 629 &vcpu->arch.sie_block->gpsw, sizeof(psw_t)); 630 rc |= read_guest_lc(vcpu, __LC_EXT_NEW_PSW, 631 &vcpu->arch.sie_block->gpsw, sizeof(psw_t)); 632 return rc ? -EFAULT : 0; 633 } 634 635 static int __must_check __deliver_external_call(struct kvm_vcpu *vcpu) 636 { 637 struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int; 638 struct kvm_s390_extcall_info extcall; 639 int rc; 640 641 spin_lock(&li->lock); 642 extcall = li->irq.extcall; 643 li->irq.extcall.code = 0; 644 clear_bit(IRQ_PEND_EXT_EXTERNAL, &li->pending_irqs); 645 spin_unlock(&li->lock); 646 647 VCPU_EVENT(vcpu, 4, "%s", "deliver: sigp ext call"); 648 vcpu->stat.deliver_external_call++; 649 trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, 650 KVM_S390_INT_EXTERNAL_CALL, 651 extcall.code, 0); 652 653 rc = put_guest_lc(vcpu, EXT_IRQ_EXTERNAL_CALL, 654 (u16 *)__LC_EXT_INT_CODE); 655 rc |= put_guest_lc(vcpu, extcall.code, (u16 *)__LC_EXT_CPU_ADDR); 656 rc |= write_guest_lc(vcpu, __LC_EXT_OLD_PSW, 657 &vcpu->arch.sie_block->gpsw, sizeof(psw_t)); 658 rc |= read_guest_lc(vcpu, __LC_EXT_NEW_PSW, &vcpu->arch.sie_block->gpsw, 659 sizeof(psw_t)); 660 return rc ? -EFAULT : 0; 661 } 662 663 static int __must_check __deliver_prog(struct kvm_vcpu *vcpu) 664 { 665 struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int; 666 struct kvm_s390_pgm_info pgm_info; 667 int rc = 0, nullifying = false; 668 u16 ilen; 669 670 spin_lock(&li->lock); 671 pgm_info = li->irq.pgm; 672 clear_bit(IRQ_PEND_PROG, &li->pending_irqs); 673 memset(&li->irq.pgm, 0, sizeof(pgm_info)); 674 spin_unlock(&li->lock); 675 676 ilen = pgm_info.flags & KVM_S390_PGM_FLAGS_ILC_MASK; 677 VCPU_EVENT(vcpu, 3, "deliver: program irq code 0x%x, ilen:%d", 678 pgm_info.code, ilen); 679 vcpu->stat.deliver_program_int++; 680 trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, KVM_S390_PROGRAM_INT, 681 pgm_info.code, 0); 682 683 switch (pgm_info.code & ~PGM_PER) { 684 case PGM_AFX_TRANSLATION: 685 case PGM_ASX_TRANSLATION: 686 case PGM_EX_TRANSLATION: 687 case PGM_LFX_TRANSLATION: 688 case PGM_LSTE_SEQUENCE: 689 case PGM_LSX_TRANSLATION: 690 case PGM_LX_TRANSLATION: 691 case PGM_PRIMARY_AUTHORITY: 692 case PGM_SECONDARY_AUTHORITY: 693 nullifying = true; 694 /* fall through */ 695 case PGM_SPACE_SWITCH: 696 rc = put_guest_lc(vcpu, pgm_info.trans_exc_code, 697 (u64 *)__LC_TRANS_EXC_CODE); 698 break; 699 case PGM_ALEN_TRANSLATION: 700 case PGM_ALE_SEQUENCE: 701 case PGM_ASTE_INSTANCE: 702 case PGM_ASTE_SEQUENCE: 703 case PGM_ASTE_VALIDITY: 704 case PGM_EXTENDED_AUTHORITY: 705 rc = put_guest_lc(vcpu, pgm_info.exc_access_id, 706 (u8 *)__LC_EXC_ACCESS_ID); 707 nullifying = true; 708 break; 709 case PGM_ASCE_TYPE: 710 case PGM_PAGE_TRANSLATION: 711 case PGM_REGION_FIRST_TRANS: 712 case PGM_REGION_SECOND_TRANS: 713 case PGM_REGION_THIRD_TRANS: 714 case PGM_SEGMENT_TRANSLATION: 715 rc = put_guest_lc(vcpu, pgm_info.trans_exc_code, 716 (u64 *)__LC_TRANS_EXC_CODE); 717 rc |= put_guest_lc(vcpu, pgm_info.exc_access_id, 718 (u8 *)__LC_EXC_ACCESS_ID); 719 rc |= put_guest_lc(vcpu, pgm_info.op_access_id, 720 (u8 *)__LC_OP_ACCESS_ID); 721 nullifying = true; 722 break; 723 case PGM_MONITOR: 724 rc = put_guest_lc(vcpu, pgm_info.mon_class_nr, 725 (u16 *)__LC_MON_CLASS_NR); 726 rc |= put_guest_lc(vcpu, pgm_info.mon_code, 727 (u64 *)__LC_MON_CODE); 728 break; 729 case PGM_VECTOR_PROCESSING: 730 case PGM_DATA: 731 rc = put_guest_lc(vcpu, pgm_info.data_exc_code, 732 (u32 *)__LC_DATA_EXC_CODE); 733 break; 734 case PGM_PROTECTION: 735 rc = put_guest_lc(vcpu, pgm_info.trans_exc_code, 736 (u64 *)__LC_TRANS_EXC_CODE); 737 rc |= put_guest_lc(vcpu, pgm_info.exc_access_id, 738 (u8 *)__LC_EXC_ACCESS_ID); 739 break; 740 case PGM_STACK_FULL: 741 case PGM_STACK_EMPTY: 742 case PGM_STACK_SPECIFICATION: 743 case PGM_STACK_TYPE: 744 case PGM_STACK_OPERATION: 745 case PGM_TRACE_TABEL: 746 case PGM_CRYPTO_OPERATION: 747 nullifying = true; 748 break; 749 } 750 751 if (pgm_info.code & PGM_PER) { 752 rc |= put_guest_lc(vcpu, pgm_info.per_code, 753 (u8 *) __LC_PER_CODE); 754 rc |= put_guest_lc(vcpu, pgm_info.per_atmid, 755 (u8 *)__LC_PER_ATMID); 756 rc |= put_guest_lc(vcpu, pgm_info.per_address, 757 (u64 *) __LC_PER_ADDRESS); 758 rc |= put_guest_lc(vcpu, pgm_info.per_access_id, 759 (u8 *) __LC_PER_ACCESS_ID); 760 } 761 762 if (nullifying && !(pgm_info.flags & KVM_S390_PGM_FLAGS_NO_REWIND)) 763 kvm_s390_rewind_psw(vcpu, ilen); 764 765 /* bit 1+2 of the target are the ilc, so we can directly use ilen */ 766 rc |= put_guest_lc(vcpu, ilen, (u16 *) __LC_PGM_ILC); 767 rc |= put_guest_lc(vcpu, vcpu->arch.sie_block->gbea, 768 (u64 *) __LC_LAST_BREAK); 769 rc |= put_guest_lc(vcpu, pgm_info.code, 770 (u16 *)__LC_PGM_INT_CODE); 771 rc |= write_guest_lc(vcpu, __LC_PGM_OLD_PSW, 772 &vcpu->arch.sie_block->gpsw, sizeof(psw_t)); 773 rc |= read_guest_lc(vcpu, __LC_PGM_NEW_PSW, 774 &vcpu->arch.sie_block->gpsw, sizeof(psw_t)); 775 return rc ? -EFAULT : 0; 776 } 777 778 static int __must_check __deliver_service(struct kvm_vcpu *vcpu) 779 { 780 struct kvm_s390_float_interrupt *fi = &vcpu->kvm->arch.float_int; 781 struct kvm_s390_ext_info ext; 782 int rc = 0; 783 784 spin_lock(&fi->lock); 785 if (!(test_bit(IRQ_PEND_EXT_SERVICE, &fi->pending_irqs))) { 786 spin_unlock(&fi->lock); 787 return 0; 788 } 789 ext = fi->srv_signal; 790 memset(&fi->srv_signal, 0, sizeof(ext)); 791 clear_bit(IRQ_PEND_EXT_SERVICE, &fi->pending_irqs); 792 spin_unlock(&fi->lock); 793 794 VCPU_EVENT(vcpu, 4, "deliver: sclp parameter 0x%x", 795 ext.ext_params); 796 vcpu->stat.deliver_service_signal++; 797 trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, KVM_S390_INT_SERVICE, 798 ext.ext_params, 0); 799 800 rc = put_guest_lc(vcpu, EXT_IRQ_SERVICE_SIG, (u16 *)__LC_EXT_INT_CODE); 801 rc |= put_guest_lc(vcpu, 0, (u16 *)__LC_EXT_CPU_ADDR); 802 rc |= write_guest_lc(vcpu, __LC_EXT_OLD_PSW, 803 &vcpu->arch.sie_block->gpsw, sizeof(psw_t)); 804 rc |= read_guest_lc(vcpu, __LC_EXT_NEW_PSW, 805 &vcpu->arch.sie_block->gpsw, sizeof(psw_t)); 806 rc |= put_guest_lc(vcpu, ext.ext_params, 807 (u32 *)__LC_EXT_PARAMS); 808 809 return rc ? -EFAULT : 0; 810 } 811 812 static int __must_check __deliver_pfault_done(struct kvm_vcpu *vcpu) 813 { 814 struct kvm_s390_float_interrupt *fi = &vcpu->kvm->arch.float_int; 815 struct kvm_s390_interrupt_info *inti; 816 int rc = 0; 817 818 spin_lock(&fi->lock); 819 inti = list_first_entry_or_null(&fi->lists[FIRQ_LIST_PFAULT], 820 struct kvm_s390_interrupt_info, 821 list); 822 if (inti) { 823 list_del(&inti->list); 824 fi->counters[FIRQ_CNTR_PFAULT] -= 1; 825 } 826 if (list_empty(&fi->lists[FIRQ_LIST_PFAULT])) 827 clear_bit(IRQ_PEND_PFAULT_DONE, &fi->pending_irqs); 828 spin_unlock(&fi->lock); 829 830 if (inti) { 831 trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, 832 KVM_S390_INT_PFAULT_DONE, 0, 833 inti->ext.ext_params2); 834 VCPU_EVENT(vcpu, 4, "deliver: pfault done token 0x%llx", 835 inti->ext.ext_params2); 836 837 rc = put_guest_lc(vcpu, EXT_IRQ_CP_SERVICE, 838 (u16 *)__LC_EXT_INT_CODE); 839 rc |= put_guest_lc(vcpu, PFAULT_DONE, 840 (u16 *)__LC_EXT_CPU_ADDR); 841 rc |= write_guest_lc(vcpu, __LC_EXT_OLD_PSW, 842 &vcpu->arch.sie_block->gpsw, 843 sizeof(psw_t)); 844 rc |= read_guest_lc(vcpu, __LC_EXT_NEW_PSW, 845 &vcpu->arch.sie_block->gpsw, 846 sizeof(psw_t)); 847 rc |= put_guest_lc(vcpu, inti->ext.ext_params2, 848 (u64 *)__LC_EXT_PARAMS2); 849 kfree(inti); 850 } 851 return rc ? -EFAULT : 0; 852 } 853 854 static int __must_check __deliver_virtio(struct kvm_vcpu *vcpu) 855 { 856 struct kvm_s390_float_interrupt *fi = &vcpu->kvm->arch.float_int; 857 struct kvm_s390_interrupt_info *inti; 858 int rc = 0; 859 860 spin_lock(&fi->lock); 861 inti = list_first_entry_or_null(&fi->lists[FIRQ_LIST_VIRTIO], 862 struct kvm_s390_interrupt_info, 863 list); 864 if (inti) { 865 VCPU_EVENT(vcpu, 4, 866 "deliver: virtio parm: 0x%x,parm64: 0x%llx", 867 inti->ext.ext_params, inti->ext.ext_params2); 868 vcpu->stat.deliver_virtio_interrupt++; 869 trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, 870 inti->type, 871 inti->ext.ext_params, 872 inti->ext.ext_params2); 873 list_del(&inti->list); 874 fi->counters[FIRQ_CNTR_VIRTIO] -= 1; 875 } 876 if (list_empty(&fi->lists[FIRQ_LIST_VIRTIO])) 877 clear_bit(IRQ_PEND_VIRTIO, &fi->pending_irqs); 878 spin_unlock(&fi->lock); 879 880 if (inti) { 881 rc = put_guest_lc(vcpu, EXT_IRQ_CP_SERVICE, 882 (u16 *)__LC_EXT_INT_CODE); 883 rc |= put_guest_lc(vcpu, VIRTIO_PARAM, 884 (u16 *)__LC_EXT_CPU_ADDR); 885 rc |= write_guest_lc(vcpu, __LC_EXT_OLD_PSW, 886 &vcpu->arch.sie_block->gpsw, 887 sizeof(psw_t)); 888 rc |= read_guest_lc(vcpu, __LC_EXT_NEW_PSW, 889 &vcpu->arch.sie_block->gpsw, 890 sizeof(psw_t)); 891 rc |= put_guest_lc(vcpu, inti->ext.ext_params, 892 (u32 *)__LC_EXT_PARAMS); 893 rc |= put_guest_lc(vcpu, inti->ext.ext_params2, 894 (u64 *)__LC_EXT_PARAMS2); 895 kfree(inti); 896 } 897 return rc ? -EFAULT : 0; 898 } 899 900 static int __must_check __deliver_io(struct kvm_vcpu *vcpu, 901 unsigned long irq_type) 902 { 903 struct list_head *isc_list; 904 struct kvm_s390_float_interrupt *fi; 905 struct kvm_s390_interrupt_info *inti = NULL; 906 int rc = 0; 907 908 fi = &vcpu->kvm->arch.float_int; 909 910 spin_lock(&fi->lock); 911 isc_list = &fi->lists[irq_type_to_isc(irq_type)]; 912 inti = list_first_entry_or_null(isc_list, 913 struct kvm_s390_interrupt_info, 914 list); 915 if (inti) { 916 if (inti->type & KVM_S390_INT_IO_AI_MASK) 917 VCPU_EVENT(vcpu, 4, "%s", "deliver: I/O (AI)"); 918 else 919 VCPU_EVENT(vcpu, 4, "deliver: I/O %x ss %x schid %04x", 920 inti->io.subchannel_id >> 8, 921 inti->io.subchannel_id >> 1 & 0x3, 922 inti->io.subchannel_nr); 923 924 vcpu->stat.deliver_io_int++; 925 trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, 926 inti->type, 927 ((__u32)inti->io.subchannel_id << 16) | 928 inti->io.subchannel_nr, 929 ((__u64)inti->io.io_int_parm << 32) | 930 inti->io.io_int_word); 931 list_del(&inti->list); 932 fi->counters[FIRQ_CNTR_IO] -= 1; 933 } 934 if (list_empty(isc_list)) 935 clear_bit(irq_type, &fi->pending_irqs); 936 spin_unlock(&fi->lock); 937 938 if (inti) { 939 rc = put_guest_lc(vcpu, inti->io.subchannel_id, 940 (u16 *)__LC_SUBCHANNEL_ID); 941 rc |= put_guest_lc(vcpu, inti->io.subchannel_nr, 942 (u16 *)__LC_SUBCHANNEL_NR); 943 rc |= put_guest_lc(vcpu, inti->io.io_int_parm, 944 (u32 *)__LC_IO_INT_PARM); 945 rc |= put_guest_lc(vcpu, inti->io.io_int_word, 946 (u32 *)__LC_IO_INT_WORD); 947 rc |= write_guest_lc(vcpu, __LC_IO_OLD_PSW, 948 &vcpu->arch.sie_block->gpsw, 949 sizeof(psw_t)); 950 rc |= read_guest_lc(vcpu, __LC_IO_NEW_PSW, 951 &vcpu->arch.sie_block->gpsw, 952 sizeof(psw_t)); 953 kfree(inti); 954 } 955 956 return rc ? -EFAULT : 0; 957 } 958 959 typedef int (*deliver_irq_t)(struct kvm_vcpu *vcpu); 960 961 static const deliver_irq_t deliver_irq_funcs[] = { 962 [IRQ_PEND_MCHK_EX] = __deliver_machine_check, 963 [IRQ_PEND_MCHK_REP] = __deliver_machine_check, 964 [IRQ_PEND_PROG] = __deliver_prog, 965 [IRQ_PEND_EXT_EMERGENCY] = __deliver_emergency_signal, 966 [IRQ_PEND_EXT_EXTERNAL] = __deliver_external_call, 967 [IRQ_PEND_EXT_CLOCK_COMP] = __deliver_ckc, 968 [IRQ_PEND_EXT_CPU_TIMER] = __deliver_cpu_timer, 969 [IRQ_PEND_RESTART] = __deliver_restart, 970 [IRQ_PEND_SET_PREFIX] = __deliver_set_prefix, 971 [IRQ_PEND_PFAULT_INIT] = __deliver_pfault_init, 972 [IRQ_PEND_EXT_SERVICE] = __deliver_service, 973 [IRQ_PEND_PFAULT_DONE] = __deliver_pfault_done, 974 [IRQ_PEND_VIRTIO] = __deliver_virtio, 975 }; 976 977 /* Check whether an external call is pending (deliverable or not) */ 978 int kvm_s390_ext_call_pending(struct kvm_vcpu *vcpu) 979 { 980 struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int; 981 982 if (!sclp.has_sigpif) 983 return test_bit(IRQ_PEND_EXT_EXTERNAL, &li->pending_irqs); 984 985 return sca_ext_call_pending(vcpu, NULL); 986 } 987 988 int kvm_s390_vcpu_has_irq(struct kvm_vcpu *vcpu, int exclude_stop) 989 { 990 if (deliverable_irqs(vcpu)) 991 return 1; 992 993 if (kvm_cpu_has_pending_timer(vcpu)) 994 return 1; 995 996 /* external call pending and deliverable */ 997 if (kvm_s390_ext_call_pending(vcpu) && 998 !psw_extint_disabled(vcpu) && 999 (vcpu->arch.sie_block->gcr[0] & 0x2000ul)) 1000 return 1; 1001 1002 if (!exclude_stop && kvm_s390_is_stop_irq_pending(vcpu)) 1003 return 1; 1004 return 0; 1005 } 1006 1007 int kvm_cpu_has_pending_timer(struct kvm_vcpu *vcpu) 1008 { 1009 return ckc_irq_pending(vcpu) || cpu_timer_irq_pending(vcpu); 1010 } 1011 1012 static u64 __calculate_sltime(struct kvm_vcpu *vcpu) 1013 { 1014 u64 now, cputm, sltime = 0; 1015 1016 if (ckc_interrupts_enabled(vcpu)) { 1017 now = kvm_s390_get_tod_clock_fast(vcpu->kvm); 1018 sltime = tod_to_ns(vcpu->arch.sie_block->ckc - now); 1019 /* already expired or overflow? */ 1020 if (!sltime || vcpu->arch.sie_block->ckc <= now) 1021 return 0; 1022 if (cpu_timer_interrupts_enabled(vcpu)) { 1023 cputm = kvm_s390_get_cpu_timer(vcpu); 1024 /* already expired? */ 1025 if (cputm >> 63) 1026 return 0; 1027 return min(sltime, tod_to_ns(cputm)); 1028 } 1029 } else if (cpu_timer_interrupts_enabled(vcpu)) { 1030 sltime = kvm_s390_get_cpu_timer(vcpu); 1031 /* already expired? */ 1032 if (sltime >> 63) 1033 return 0; 1034 } 1035 return sltime; 1036 } 1037 1038 int kvm_s390_handle_wait(struct kvm_vcpu *vcpu) 1039 { 1040 u64 sltime; 1041 1042 vcpu->stat.exit_wait_state++; 1043 1044 /* fast path */ 1045 if (kvm_arch_vcpu_runnable(vcpu)) 1046 return 0; 1047 1048 if (psw_interrupts_disabled(vcpu)) { 1049 VCPU_EVENT(vcpu, 3, "%s", "disabled wait"); 1050 return -EOPNOTSUPP; /* disabled wait */ 1051 } 1052 1053 if (!ckc_interrupts_enabled(vcpu) && 1054 !cpu_timer_interrupts_enabled(vcpu)) { 1055 VCPU_EVENT(vcpu, 3, "%s", "enabled wait w/o timer"); 1056 __set_cpu_idle(vcpu); 1057 goto no_timer; 1058 } 1059 1060 sltime = __calculate_sltime(vcpu); 1061 if (!sltime) 1062 return 0; 1063 1064 __set_cpu_idle(vcpu); 1065 hrtimer_start(&vcpu->arch.ckc_timer, sltime, HRTIMER_MODE_REL); 1066 VCPU_EVENT(vcpu, 4, "enabled wait: %llu ns", sltime); 1067 no_timer: 1068 srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx); 1069 kvm_vcpu_block(vcpu); 1070 __unset_cpu_idle(vcpu); 1071 vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu); 1072 1073 hrtimer_cancel(&vcpu->arch.ckc_timer); 1074 return 0; 1075 } 1076 1077 void kvm_s390_vcpu_wakeup(struct kvm_vcpu *vcpu) 1078 { 1079 /* 1080 * We cannot move this into the if, as the CPU might be already 1081 * in kvm_vcpu_block without having the waitqueue set (polling) 1082 */ 1083 vcpu->valid_wakeup = true; 1084 /* 1085 * This is mostly to document, that the read in swait_active could 1086 * be moved before other stores, leading to subtle races. 1087 * All current users do not store or use an atomic like update 1088 */ 1089 smp_mb__after_atomic(); 1090 if (swait_active(&vcpu->wq)) { 1091 /* 1092 * The vcpu gave up the cpu voluntarily, mark it as a good 1093 * yield-candidate. 1094 */ 1095 vcpu->preempted = true; 1096 swake_up(&vcpu->wq); 1097 vcpu->stat.halt_wakeup++; 1098 } 1099 /* 1100 * The VCPU might not be sleeping but is executing the VSIE. Let's 1101 * kick it, so it leaves the SIE to process the request. 1102 */ 1103 kvm_s390_vsie_kick(vcpu); 1104 } 1105 1106 enum hrtimer_restart kvm_s390_idle_wakeup(struct hrtimer *timer) 1107 { 1108 struct kvm_vcpu *vcpu; 1109 u64 sltime; 1110 1111 vcpu = container_of(timer, struct kvm_vcpu, arch.ckc_timer); 1112 sltime = __calculate_sltime(vcpu); 1113 1114 /* 1115 * If the monotonic clock runs faster than the tod clock we might be 1116 * woken up too early and have to go back to sleep to avoid deadlocks. 1117 */ 1118 if (sltime && hrtimer_forward_now(timer, ns_to_ktime(sltime))) 1119 return HRTIMER_RESTART; 1120 kvm_s390_vcpu_wakeup(vcpu); 1121 return HRTIMER_NORESTART; 1122 } 1123 1124 void kvm_s390_clear_local_irqs(struct kvm_vcpu *vcpu) 1125 { 1126 struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int; 1127 1128 spin_lock(&li->lock); 1129 li->pending_irqs = 0; 1130 bitmap_zero(li->sigp_emerg_pending, KVM_MAX_VCPUS); 1131 memset(&li->irq, 0, sizeof(li->irq)); 1132 spin_unlock(&li->lock); 1133 1134 sca_clear_ext_call(vcpu); 1135 } 1136 1137 int __must_check kvm_s390_deliver_pending_interrupts(struct kvm_vcpu *vcpu) 1138 { 1139 struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int; 1140 deliver_irq_t func; 1141 int rc = 0; 1142 unsigned long irq_type; 1143 unsigned long irqs; 1144 1145 __reset_intercept_indicators(vcpu); 1146 1147 /* pending ckc conditions might have been invalidated */ 1148 clear_bit(IRQ_PEND_EXT_CLOCK_COMP, &li->pending_irqs); 1149 if (ckc_irq_pending(vcpu)) 1150 set_bit(IRQ_PEND_EXT_CLOCK_COMP, &li->pending_irqs); 1151 1152 /* pending cpu timer conditions might have been invalidated */ 1153 clear_bit(IRQ_PEND_EXT_CPU_TIMER, &li->pending_irqs); 1154 if (cpu_timer_irq_pending(vcpu)) 1155 set_bit(IRQ_PEND_EXT_CPU_TIMER, &li->pending_irqs); 1156 1157 while ((irqs = deliverable_irqs(vcpu)) && !rc) { 1158 /* bits are in the order of interrupt priority */ 1159 irq_type = find_first_bit(&irqs, IRQ_PEND_COUNT); 1160 if (is_ioirq(irq_type)) { 1161 rc = __deliver_io(vcpu, irq_type); 1162 } else { 1163 func = deliver_irq_funcs[irq_type]; 1164 if (!func) { 1165 WARN_ON_ONCE(func == NULL); 1166 clear_bit(irq_type, &li->pending_irqs); 1167 continue; 1168 } 1169 rc = func(vcpu); 1170 } 1171 } 1172 1173 set_intercept_indicators(vcpu); 1174 1175 return rc; 1176 } 1177 1178 static int __inject_prog(struct kvm_vcpu *vcpu, struct kvm_s390_irq *irq) 1179 { 1180 struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int; 1181 1182 VCPU_EVENT(vcpu, 3, "inject: program irq code 0x%x", irq->u.pgm.code); 1183 trace_kvm_s390_inject_vcpu(vcpu->vcpu_id, KVM_S390_PROGRAM_INT, 1184 irq->u.pgm.code, 0); 1185 1186 if (!(irq->u.pgm.flags & KVM_S390_PGM_FLAGS_ILC_VALID)) { 1187 /* auto detection if no valid ILC was given */ 1188 irq->u.pgm.flags &= ~KVM_S390_PGM_FLAGS_ILC_MASK; 1189 irq->u.pgm.flags |= kvm_s390_get_ilen(vcpu); 1190 irq->u.pgm.flags |= KVM_S390_PGM_FLAGS_ILC_VALID; 1191 } 1192 1193 if (irq->u.pgm.code == PGM_PER) { 1194 li->irq.pgm.code |= PGM_PER; 1195 li->irq.pgm.flags = irq->u.pgm.flags; 1196 /* only modify PER related information */ 1197 li->irq.pgm.per_address = irq->u.pgm.per_address; 1198 li->irq.pgm.per_code = irq->u.pgm.per_code; 1199 li->irq.pgm.per_atmid = irq->u.pgm.per_atmid; 1200 li->irq.pgm.per_access_id = irq->u.pgm.per_access_id; 1201 } else if (!(irq->u.pgm.code & PGM_PER)) { 1202 li->irq.pgm.code = (li->irq.pgm.code & PGM_PER) | 1203 irq->u.pgm.code; 1204 li->irq.pgm.flags = irq->u.pgm.flags; 1205 /* only modify non-PER information */ 1206 li->irq.pgm.trans_exc_code = irq->u.pgm.trans_exc_code; 1207 li->irq.pgm.mon_code = irq->u.pgm.mon_code; 1208 li->irq.pgm.data_exc_code = irq->u.pgm.data_exc_code; 1209 li->irq.pgm.mon_class_nr = irq->u.pgm.mon_class_nr; 1210 li->irq.pgm.exc_access_id = irq->u.pgm.exc_access_id; 1211 li->irq.pgm.op_access_id = irq->u.pgm.op_access_id; 1212 } else { 1213 li->irq.pgm = irq->u.pgm; 1214 } 1215 set_bit(IRQ_PEND_PROG, &li->pending_irqs); 1216 return 0; 1217 } 1218 1219 static int __inject_pfault_init(struct kvm_vcpu *vcpu, struct kvm_s390_irq *irq) 1220 { 1221 struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int; 1222 1223 VCPU_EVENT(vcpu, 4, "inject: pfault init parameter block at 0x%llx", 1224 irq->u.ext.ext_params2); 1225 trace_kvm_s390_inject_vcpu(vcpu->vcpu_id, KVM_S390_INT_PFAULT_INIT, 1226 irq->u.ext.ext_params, 1227 irq->u.ext.ext_params2); 1228 1229 li->irq.ext = irq->u.ext; 1230 set_bit(IRQ_PEND_PFAULT_INIT, &li->pending_irqs); 1231 atomic_or(CPUSTAT_EXT_INT, li->cpuflags); 1232 return 0; 1233 } 1234 1235 static int __inject_extcall(struct kvm_vcpu *vcpu, struct kvm_s390_irq *irq) 1236 { 1237 struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int; 1238 struct kvm_s390_extcall_info *extcall = &li->irq.extcall; 1239 uint16_t src_id = irq->u.extcall.code; 1240 1241 VCPU_EVENT(vcpu, 4, "inject: external call source-cpu:%u", 1242 src_id); 1243 trace_kvm_s390_inject_vcpu(vcpu->vcpu_id, KVM_S390_INT_EXTERNAL_CALL, 1244 src_id, 0); 1245 1246 /* sending vcpu invalid */ 1247 if (kvm_get_vcpu_by_id(vcpu->kvm, src_id) == NULL) 1248 return -EINVAL; 1249 1250 if (sclp.has_sigpif) 1251 return sca_inject_ext_call(vcpu, src_id); 1252 1253 if (test_and_set_bit(IRQ_PEND_EXT_EXTERNAL, &li->pending_irqs)) 1254 return -EBUSY; 1255 *extcall = irq->u.extcall; 1256 atomic_or(CPUSTAT_EXT_INT, li->cpuflags); 1257 return 0; 1258 } 1259 1260 static int __inject_set_prefix(struct kvm_vcpu *vcpu, struct kvm_s390_irq *irq) 1261 { 1262 struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int; 1263 struct kvm_s390_prefix_info *prefix = &li->irq.prefix; 1264 1265 VCPU_EVENT(vcpu, 3, "inject: set prefix to %x", 1266 irq->u.prefix.address); 1267 trace_kvm_s390_inject_vcpu(vcpu->vcpu_id, KVM_S390_SIGP_SET_PREFIX, 1268 irq->u.prefix.address, 0); 1269 1270 if (!is_vcpu_stopped(vcpu)) 1271 return -EBUSY; 1272 1273 *prefix = irq->u.prefix; 1274 set_bit(IRQ_PEND_SET_PREFIX, &li->pending_irqs); 1275 return 0; 1276 } 1277 1278 #define KVM_S390_STOP_SUPP_FLAGS (KVM_S390_STOP_FLAG_STORE_STATUS) 1279 static int __inject_sigp_stop(struct kvm_vcpu *vcpu, struct kvm_s390_irq *irq) 1280 { 1281 struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int; 1282 struct kvm_s390_stop_info *stop = &li->irq.stop; 1283 int rc = 0; 1284 1285 trace_kvm_s390_inject_vcpu(vcpu->vcpu_id, KVM_S390_SIGP_STOP, 0, 0); 1286 1287 if (irq->u.stop.flags & ~KVM_S390_STOP_SUPP_FLAGS) 1288 return -EINVAL; 1289 1290 if (is_vcpu_stopped(vcpu)) { 1291 if (irq->u.stop.flags & KVM_S390_STOP_FLAG_STORE_STATUS) 1292 rc = kvm_s390_store_status_unloaded(vcpu, 1293 KVM_S390_STORE_STATUS_NOADDR); 1294 return rc; 1295 } 1296 1297 if (test_and_set_bit(IRQ_PEND_SIGP_STOP, &li->pending_irqs)) 1298 return -EBUSY; 1299 stop->flags = irq->u.stop.flags; 1300 __set_cpuflag(vcpu, CPUSTAT_STOP_INT); 1301 return 0; 1302 } 1303 1304 static int __inject_sigp_restart(struct kvm_vcpu *vcpu, 1305 struct kvm_s390_irq *irq) 1306 { 1307 struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int; 1308 1309 VCPU_EVENT(vcpu, 3, "%s", "inject: restart int"); 1310 trace_kvm_s390_inject_vcpu(vcpu->vcpu_id, KVM_S390_RESTART, 0, 0); 1311 1312 set_bit(IRQ_PEND_RESTART, &li->pending_irqs); 1313 return 0; 1314 } 1315 1316 static int __inject_sigp_emergency(struct kvm_vcpu *vcpu, 1317 struct kvm_s390_irq *irq) 1318 { 1319 struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int; 1320 1321 VCPU_EVENT(vcpu, 4, "inject: emergency from cpu %u", 1322 irq->u.emerg.code); 1323 trace_kvm_s390_inject_vcpu(vcpu->vcpu_id, KVM_S390_INT_EMERGENCY, 1324 irq->u.emerg.code, 0); 1325 1326 /* sending vcpu invalid */ 1327 if (kvm_get_vcpu_by_id(vcpu->kvm, irq->u.emerg.code) == NULL) 1328 return -EINVAL; 1329 1330 set_bit(irq->u.emerg.code, li->sigp_emerg_pending); 1331 set_bit(IRQ_PEND_EXT_EMERGENCY, &li->pending_irqs); 1332 atomic_or(CPUSTAT_EXT_INT, li->cpuflags); 1333 return 0; 1334 } 1335 1336 static int __inject_mchk(struct kvm_vcpu *vcpu, struct kvm_s390_irq *irq) 1337 { 1338 struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int; 1339 struct kvm_s390_mchk_info *mchk = &li->irq.mchk; 1340 1341 VCPU_EVENT(vcpu, 3, "inject: machine check mcic 0x%llx", 1342 irq->u.mchk.mcic); 1343 trace_kvm_s390_inject_vcpu(vcpu->vcpu_id, KVM_S390_MCHK, 0, 1344 irq->u.mchk.mcic); 1345 1346 /* 1347 * Because repressible machine checks can be indicated along with 1348 * exigent machine checks (PoP, Chapter 11, Interruption action) 1349 * we need to combine cr14, mcic and external damage code. 1350 * Failing storage address and the logout area should not be or'ed 1351 * together, we just indicate the last occurrence of the corresponding 1352 * machine check 1353 */ 1354 mchk->cr14 |= irq->u.mchk.cr14; 1355 mchk->mcic |= irq->u.mchk.mcic; 1356 mchk->ext_damage_code |= irq->u.mchk.ext_damage_code; 1357 mchk->failing_storage_address = irq->u.mchk.failing_storage_address; 1358 memcpy(&mchk->fixed_logout, &irq->u.mchk.fixed_logout, 1359 sizeof(mchk->fixed_logout)); 1360 if (mchk->mcic & MCHK_EX_MASK) 1361 set_bit(IRQ_PEND_MCHK_EX, &li->pending_irqs); 1362 else if (mchk->mcic & MCHK_REP_MASK) 1363 set_bit(IRQ_PEND_MCHK_REP, &li->pending_irqs); 1364 return 0; 1365 } 1366 1367 static int __inject_ckc(struct kvm_vcpu *vcpu) 1368 { 1369 struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int; 1370 1371 VCPU_EVENT(vcpu, 3, "%s", "inject: clock comparator external"); 1372 trace_kvm_s390_inject_vcpu(vcpu->vcpu_id, KVM_S390_INT_CLOCK_COMP, 1373 0, 0); 1374 1375 set_bit(IRQ_PEND_EXT_CLOCK_COMP, &li->pending_irqs); 1376 atomic_or(CPUSTAT_EXT_INT, li->cpuflags); 1377 return 0; 1378 } 1379 1380 static int __inject_cpu_timer(struct kvm_vcpu *vcpu) 1381 { 1382 struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int; 1383 1384 VCPU_EVENT(vcpu, 3, "%s", "inject: cpu timer external"); 1385 trace_kvm_s390_inject_vcpu(vcpu->vcpu_id, KVM_S390_INT_CPU_TIMER, 1386 0, 0); 1387 1388 set_bit(IRQ_PEND_EXT_CPU_TIMER, &li->pending_irqs); 1389 atomic_or(CPUSTAT_EXT_INT, li->cpuflags); 1390 return 0; 1391 } 1392 1393 static struct kvm_s390_interrupt_info *get_io_int(struct kvm *kvm, 1394 int isc, u32 schid) 1395 { 1396 struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int; 1397 struct list_head *isc_list = &fi->lists[FIRQ_LIST_IO_ISC_0 + isc]; 1398 struct kvm_s390_interrupt_info *iter; 1399 u16 id = (schid & 0xffff0000U) >> 16; 1400 u16 nr = schid & 0x0000ffffU; 1401 1402 spin_lock(&fi->lock); 1403 list_for_each_entry(iter, isc_list, list) { 1404 if (schid && (id != iter->io.subchannel_id || 1405 nr != iter->io.subchannel_nr)) 1406 continue; 1407 /* found an appropriate entry */ 1408 list_del_init(&iter->list); 1409 fi->counters[FIRQ_CNTR_IO] -= 1; 1410 if (list_empty(isc_list)) 1411 clear_bit(isc_to_irq_type(isc), &fi->pending_irqs); 1412 spin_unlock(&fi->lock); 1413 return iter; 1414 } 1415 spin_unlock(&fi->lock); 1416 return NULL; 1417 } 1418 1419 /* 1420 * Dequeue and return an I/O interrupt matching any of the interruption 1421 * subclasses as designated by the isc mask in cr6 and the schid (if != 0). 1422 */ 1423 struct kvm_s390_interrupt_info *kvm_s390_get_io_int(struct kvm *kvm, 1424 u64 isc_mask, u32 schid) 1425 { 1426 struct kvm_s390_interrupt_info *inti = NULL; 1427 int isc; 1428 1429 for (isc = 0; isc <= MAX_ISC && !inti; isc++) { 1430 if (isc_mask & isc_to_isc_bits(isc)) 1431 inti = get_io_int(kvm, isc, schid); 1432 } 1433 return inti; 1434 } 1435 1436 #define SCCB_MASK 0xFFFFFFF8 1437 #define SCCB_EVENT_PENDING 0x3 1438 1439 static int __inject_service(struct kvm *kvm, 1440 struct kvm_s390_interrupt_info *inti) 1441 { 1442 struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int; 1443 1444 spin_lock(&fi->lock); 1445 fi->srv_signal.ext_params |= inti->ext.ext_params & SCCB_EVENT_PENDING; 1446 /* 1447 * Early versions of the QEMU s390 bios will inject several 1448 * service interrupts after another without handling a 1449 * condition code indicating busy. 1450 * We will silently ignore those superfluous sccb values. 1451 * A future version of QEMU will take care of serialization 1452 * of servc requests 1453 */ 1454 if (fi->srv_signal.ext_params & SCCB_MASK) 1455 goto out; 1456 fi->srv_signal.ext_params |= inti->ext.ext_params & SCCB_MASK; 1457 set_bit(IRQ_PEND_EXT_SERVICE, &fi->pending_irqs); 1458 out: 1459 spin_unlock(&fi->lock); 1460 kfree(inti); 1461 return 0; 1462 } 1463 1464 static int __inject_virtio(struct kvm *kvm, 1465 struct kvm_s390_interrupt_info *inti) 1466 { 1467 struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int; 1468 1469 spin_lock(&fi->lock); 1470 if (fi->counters[FIRQ_CNTR_VIRTIO] >= KVM_S390_MAX_VIRTIO_IRQS) { 1471 spin_unlock(&fi->lock); 1472 return -EBUSY; 1473 } 1474 fi->counters[FIRQ_CNTR_VIRTIO] += 1; 1475 list_add_tail(&inti->list, &fi->lists[FIRQ_LIST_VIRTIO]); 1476 set_bit(IRQ_PEND_VIRTIO, &fi->pending_irqs); 1477 spin_unlock(&fi->lock); 1478 return 0; 1479 } 1480 1481 static int __inject_pfault_done(struct kvm *kvm, 1482 struct kvm_s390_interrupt_info *inti) 1483 { 1484 struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int; 1485 1486 spin_lock(&fi->lock); 1487 if (fi->counters[FIRQ_CNTR_PFAULT] >= 1488 (ASYNC_PF_PER_VCPU * KVM_MAX_VCPUS)) { 1489 spin_unlock(&fi->lock); 1490 return -EBUSY; 1491 } 1492 fi->counters[FIRQ_CNTR_PFAULT] += 1; 1493 list_add_tail(&inti->list, &fi->lists[FIRQ_LIST_PFAULT]); 1494 set_bit(IRQ_PEND_PFAULT_DONE, &fi->pending_irqs); 1495 spin_unlock(&fi->lock); 1496 return 0; 1497 } 1498 1499 #define CR_PENDING_SUBCLASS 28 1500 static int __inject_float_mchk(struct kvm *kvm, 1501 struct kvm_s390_interrupt_info *inti) 1502 { 1503 struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int; 1504 1505 spin_lock(&fi->lock); 1506 fi->mchk.cr14 |= inti->mchk.cr14 & (1UL << CR_PENDING_SUBCLASS); 1507 fi->mchk.mcic |= inti->mchk.mcic; 1508 set_bit(IRQ_PEND_MCHK_REP, &fi->pending_irqs); 1509 spin_unlock(&fi->lock); 1510 kfree(inti); 1511 return 0; 1512 } 1513 1514 static int __inject_io(struct kvm *kvm, struct kvm_s390_interrupt_info *inti) 1515 { 1516 struct kvm_s390_float_interrupt *fi; 1517 struct list_head *list; 1518 int isc; 1519 1520 fi = &kvm->arch.float_int; 1521 spin_lock(&fi->lock); 1522 if (fi->counters[FIRQ_CNTR_IO] >= KVM_S390_MAX_FLOAT_IRQS) { 1523 spin_unlock(&fi->lock); 1524 return -EBUSY; 1525 } 1526 fi->counters[FIRQ_CNTR_IO] += 1; 1527 1528 if (inti->type & KVM_S390_INT_IO_AI_MASK) 1529 VM_EVENT(kvm, 4, "%s", "inject: I/O (AI)"); 1530 else 1531 VM_EVENT(kvm, 4, "inject: I/O %x ss %x schid %04x", 1532 inti->io.subchannel_id >> 8, 1533 inti->io.subchannel_id >> 1 & 0x3, 1534 inti->io.subchannel_nr); 1535 isc = int_word_to_isc(inti->io.io_int_word); 1536 list = &fi->lists[FIRQ_LIST_IO_ISC_0 + isc]; 1537 list_add_tail(&inti->list, list); 1538 set_bit(isc_to_irq_type(isc), &fi->pending_irqs); 1539 spin_unlock(&fi->lock); 1540 return 0; 1541 } 1542 1543 /* 1544 * Find a destination VCPU for a floating irq and kick it. 1545 */ 1546 static void __floating_irq_kick(struct kvm *kvm, u64 type) 1547 { 1548 struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int; 1549 struct kvm_s390_local_interrupt *li; 1550 struct kvm_vcpu *dst_vcpu; 1551 int sigcpu, online_vcpus, nr_tries = 0; 1552 1553 online_vcpus = atomic_read(&kvm->online_vcpus); 1554 if (!online_vcpus) 1555 return; 1556 1557 /* find idle VCPUs first, then round robin */ 1558 sigcpu = find_first_bit(fi->idle_mask, online_vcpus); 1559 if (sigcpu == online_vcpus) { 1560 do { 1561 sigcpu = fi->next_rr_cpu; 1562 fi->next_rr_cpu = (fi->next_rr_cpu + 1) % online_vcpus; 1563 /* avoid endless loops if all vcpus are stopped */ 1564 if (nr_tries++ >= online_vcpus) 1565 return; 1566 } while (is_vcpu_stopped(kvm_get_vcpu(kvm, sigcpu))); 1567 } 1568 dst_vcpu = kvm_get_vcpu(kvm, sigcpu); 1569 1570 /* make the VCPU drop out of the SIE, or wake it up if sleeping */ 1571 li = &dst_vcpu->arch.local_int; 1572 spin_lock(&li->lock); 1573 switch (type) { 1574 case KVM_S390_MCHK: 1575 atomic_or(CPUSTAT_STOP_INT, li->cpuflags); 1576 break; 1577 case KVM_S390_INT_IO_MIN...KVM_S390_INT_IO_MAX: 1578 atomic_or(CPUSTAT_IO_INT, li->cpuflags); 1579 break; 1580 default: 1581 atomic_or(CPUSTAT_EXT_INT, li->cpuflags); 1582 break; 1583 } 1584 spin_unlock(&li->lock); 1585 kvm_s390_vcpu_wakeup(dst_vcpu); 1586 } 1587 1588 static int __inject_vm(struct kvm *kvm, struct kvm_s390_interrupt_info *inti) 1589 { 1590 u64 type = READ_ONCE(inti->type); 1591 int rc; 1592 1593 switch (type) { 1594 case KVM_S390_MCHK: 1595 rc = __inject_float_mchk(kvm, inti); 1596 break; 1597 case KVM_S390_INT_VIRTIO: 1598 rc = __inject_virtio(kvm, inti); 1599 break; 1600 case KVM_S390_INT_SERVICE: 1601 rc = __inject_service(kvm, inti); 1602 break; 1603 case KVM_S390_INT_PFAULT_DONE: 1604 rc = __inject_pfault_done(kvm, inti); 1605 break; 1606 case KVM_S390_INT_IO_MIN...KVM_S390_INT_IO_MAX: 1607 rc = __inject_io(kvm, inti); 1608 break; 1609 default: 1610 rc = -EINVAL; 1611 } 1612 if (rc) 1613 return rc; 1614 1615 __floating_irq_kick(kvm, type); 1616 return 0; 1617 } 1618 1619 int kvm_s390_inject_vm(struct kvm *kvm, 1620 struct kvm_s390_interrupt *s390int) 1621 { 1622 struct kvm_s390_interrupt_info *inti; 1623 int rc; 1624 1625 inti = kzalloc(sizeof(*inti), GFP_KERNEL); 1626 if (!inti) 1627 return -ENOMEM; 1628 1629 inti->type = s390int->type; 1630 switch (inti->type) { 1631 case KVM_S390_INT_VIRTIO: 1632 VM_EVENT(kvm, 5, "inject: virtio parm:%x,parm64:%llx", 1633 s390int->parm, s390int->parm64); 1634 inti->ext.ext_params = s390int->parm; 1635 inti->ext.ext_params2 = s390int->parm64; 1636 break; 1637 case KVM_S390_INT_SERVICE: 1638 VM_EVENT(kvm, 4, "inject: sclp parm:%x", s390int->parm); 1639 inti->ext.ext_params = s390int->parm; 1640 break; 1641 case KVM_S390_INT_PFAULT_DONE: 1642 inti->ext.ext_params2 = s390int->parm64; 1643 break; 1644 case KVM_S390_MCHK: 1645 VM_EVENT(kvm, 3, "inject: machine check mcic 0x%llx", 1646 s390int->parm64); 1647 inti->mchk.cr14 = s390int->parm; /* upper bits are not used */ 1648 inti->mchk.mcic = s390int->parm64; 1649 break; 1650 case KVM_S390_INT_IO_MIN...KVM_S390_INT_IO_MAX: 1651 inti->io.subchannel_id = s390int->parm >> 16; 1652 inti->io.subchannel_nr = s390int->parm & 0x0000ffffu; 1653 inti->io.io_int_parm = s390int->parm64 >> 32; 1654 inti->io.io_int_word = s390int->parm64 & 0x00000000ffffffffull; 1655 break; 1656 default: 1657 kfree(inti); 1658 return -EINVAL; 1659 } 1660 trace_kvm_s390_inject_vm(s390int->type, s390int->parm, s390int->parm64, 1661 2); 1662 1663 rc = __inject_vm(kvm, inti); 1664 if (rc) 1665 kfree(inti); 1666 return rc; 1667 } 1668 1669 int kvm_s390_reinject_io_int(struct kvm *kvm, 1670 struct kvm_s390_interrupt_info *inti) 1671 { 1672 return __inject_vm(kvm, inti); 1673 } 1674 1675 int s390int_to_s390irq(struct kvm_s390_interrupt *s390int, 1676 struct kvm_s390_irq *irq) 1677 { 1678 irq->type = s390int->type; 1679 switch (irq->type) { 1680 case KVM_S390_PROGRAM_INT: 1681 if (s390int->parm & 0xffff0000) 1682 return -EINVAL; 1683 irq->u.pgm.code = s390int->parm; 1684 break; 1685 case KVM_S390_SIGP_SET_PREFIX: 1686 irq->u.prefix.address = s390int->parm; 1687 break; 1688 case KVM_S390_SIGP_STOP: 1689 irq->u.stop.flags = s390int->parm; 1690 break; 1691 case KVM_S390_INT_EXTERNAL_CALL: 1692 if (s390int->parm & 0xffff0000) 1693 return -EINVAL; 1694 irq->u.extcall.code = s390int->parm; 1695 break; 1696 case KVM_S390_INT_EMERGENCY: 1697 if (s390int->parm & 0xffff0000) 1698 return -EINVAL; 1699 irq->u.emerg.code = s390int->parm; 1700 break; 1701 case KVM_S390_MCHK: 1702 irq->u.mchk.mcic = s390int->parm64; 1703 break; 1704 } 1705 return 0; 1706 } 1707 1708 int kvm_s390_is_stop_irq_pending(struct kvm_vcpu *vcpu) 1709 { 1710 struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int; 1711 1712 return test_bit(IRQ_PEND_SIGP_STOP, &li->pending_irqs); 1713 } 1714 1715 void kvm_s390_clear_stop_irq(struct kvm_vcpu *vcpu) 1716 { 1717 struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int; 1718 1719 spin_lock(&li->lock); 1720 li->irq.stop.flags = 0; 1721 clear_bit(IRQ_PEND_SIGP_STOP, &li->pending_irqs); 1722 spin_unlock(&li->lock); 1723 } 1724 1725 static int do_inject_vcpu(struct kvm_vcpu *vcpu, struct kvm_s390_irq *irq) 1726 { 1727 int rc; 1728 1729 switch (irq->type) { 1730 case KVM_S390_PROGRAM_INT: 1731 rc = __inject_prog(vcpu, irq); 1732 break; 1733 case KVM_S390_SIGP_SET_PREFIX: 1734 rc = __inject_set_prefix(vcpu, irq); 1735 break; 1736 case KVM_S390_SIGP_STOP: 1737 rc = __inject_sigp_stop(vcpu, irq); 1738 break; 1739 case KVM_S390_RESTART: 1740 rc = __inject_sigp_restart(vcpu, irq); 1741 break; 1742 case KVM_S390_INT_CLOCK_COMP: 1743 rc = __inject_ckc(vcpu); 1744 break; 1745 case KVM_S390_INT_CPU_TIMER: 1746 rc = __inject_cpu_timer(vcpu); 1747 break; 1748 case KVM_S390_INT_EXTERNAL_CALL: 1749 rc = __inject_extcall(vcpu, irq); 1750 break; 1751 case KVM_S390_INT_EMERGENCY: 1752 rc = __inject_sigp_emergency(vcpu, irq); 1753 break; 1754 case KVM_S390_MCHK: 1755 rc = __inject_mchk(vcpu, irq); 1756 break; 1757 case KVM_S390_INT_PFAULT_INIT: 1758 rc = __inject_pfault_init(vcpu, irq); 1759 break; 1760 case KVM_S390_INT_VIRTIO: 1761 case KVM_S390_INT_SERVICE: 1762 case KVM_S390_INT_IO_MIN...KVM_S390_INT_IO_MAX: 1763 default: 1764 rc = -EINVAL; 1765 } 1766 1767 return rc; 1768 } 1769 1770 int kvm_s390_inject_vcpu(struct kvm_vcpu *vcpu, struct kvm_s390_irq *irq) 1771 { 1772 struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int; 1773 int rc; 1774 1775 spin_lock(&li->lock); 1776 rc = do_inject_vcpu(vcpu, irq); 1777 spin_unlock(&li->lock); 1778 if (!rc) 1779 kvm_s390_vcpu_wakeup(vcpu); 1780 return rc; 1781 } 1782 1783 static inline void clear_irq_list(struct list_head *_list) 1784 { 1785 struct kvm_s390_interrupt_info *inti, *n; 1786 1787 list_for_each_entry_safe(inti, n, _list, list) { 1788 list_del(&inti->list); 1789 kfree(inti); 1790 } 1791 } 1792 1793 static void inti_to_irq(struct kvm_s390_interrupt_info *inti, 1794 struct kvm_s390_irq *irq) 1795 { 1796 irq->type = inti->type; 1797 switch (inti->type) { 1798 case KVM_S390_INT_PFAULT_INIT: 1799 case KVM_S390_INT_PFAULT_DONE: 1800 case KVM_S390_INT_VIRTIO: 1801 irq->u.ext = inti->ext; 1802 break; 1803 case KVM_S390_INT_IO_MIN...KVM_S390_INT_IO_MAX: 1804 irq->u.io = inti->io; 1805 break; 1806 } 1807 } 1808 1809 void kvm_s390_clear_float_irqs(struct kvm *kvm) 1810 { 1811 struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int; 1812 int i; 1813 1814 spin_lock(&fi->lock); 1815 fi->pending_irqs = 0; 1816 memset(&fi->srv_signal, 0, sizeof(fi->srv_signal)); 1817 memset(&fi->mchk, 0, sizeof(fi->mchk)); 1818 for (i = 0; i < FIRQ_LIST_COUNT; i++) 1819 clear_irq_list(&fi->lists[i]); 1820 for (i = 0; i < FIRQ_MAX_COUNT; i++) 1821 fi->counters[i] = 0; 1822 spin_unlock(&fi->lock); 1823 }; 1824 1825 static int get_all_floating_irqs(struct kvm *kvm, u8 __user *usrbuf, u64 len) 1826 { 1827 struct kvm_s390_interrupt_info *inti; 1828 struct kvm_s390_float_interrupt *fi; 1829 struct kvm_s390_irq *buf; 1830 struct kvm_s390_irq *irq; 1831 int max_irqs; 1832 int ret = 0; 1833 int n = 0; 1834 int i; 1835 1836 if (len > KVM_S390_FLIC_MAX_BUFFER || len == 0) 1837 return -EINVAL; 1838 1839 /* 1840 * We are already using -ENOMEM to signal 1841 * userspace it may retry with a bigger buffer, 1842 * so we need to use something else for this case 1843 */ 1844 buf = vzalloc(len); 1845 if (!buf) 1846 return -ENOBUFS; 1847 1848 max_irqs = len / sizeof(struct kvm_s390_irq); 1849 1850 fi = &kvm->arch.float_int; 1851 spin_lock(&fi->lock); 1852 for (i = 0; i < FIRQ_LIST_COUNT; i++) { 1853 list_for_each_entry(inti, &fi->lists[i], list) { 1854 if (n == max_irqs) { 1855 /* signal userspace to try again */ 1856 ret = -ENOMEM; 1857 goto out; 1858 } 1859 inti_to_irq(inti, &buf[n]); 1860 n++; 1861 } 1862 } 1863 if (test_bit(IRQ_PEND_EXT_SERVICE, &fi->pending_irqs)) { 1864 if (n == max_irqs) { 1865 /* signal userspace to try again */ 1866 ret = -ENOMEM; 1867 goto out; 1868 } 1869 irq = (struct kvm_s390_irq *) &buf[n]; 1870 irq->type = KVM_S390_INT_SERVICE; 1871 irq->u.ext = fi->srv_signal; 1872 n++; 1873 } 1874 if (test_bit(IRQ_PEND_MCHK_REP, &fi->pending_irqs)) { 1875 if (n == max_irqs) { 1876 /* signal userspace to try again */ 1877 ret = -ENOMEM; 1878 goto out; 1879 } 1880 irq = (struct kvm_s390_irq *) &buf[n]; 1881 irq->type = KVM_S390_MCHK; 1882 irq->u.mchk = fi->mchk; 1883 n++; 1884 } 1885 1886 out: 1887 spin_unlock(&fi->lock); 1888 if (!ret && n > 0) { 1889 if (copy_to_user(usrbuf, buf, sizeof(struct kvm_s390_irq) * n)) 1890 ret = -EFAULT; 1891 } 1892 vfree(buf); 1893 1894 return ret < 0 ? ret : n; 1895 } 1896 1897 static int flic_ais_mode_get_all(struct kvm *kvm, struct kvm_device_attr *attr) 1898 { 1899 struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int; 1900 struct kvm_s390_ais_all ais; 1901 1902 if (attr->attr < sizeof(ais)) 1903 return -EINVAL; 1904 1905 if (!test_kvm_facility(kvm, 72)) 1906 return -ENOTSUPP; 1907 1908 mutex_lock(&fi->ais_lock); 1909 ais.simm = fi->simm; 1910 ais.nimm = fi->nimm; 1911 mutex_unlock(&fi->ais_lock); 1912 1913 if (copy_to_user((void __user *)attr->addr, &ais, sizeof(ais))) 1914 return -EFAULT; 1915 1916 return 0; 1917 } 1918 1919 static int flic_get_attr(struct kvm_device *dev, struct kvm_device_attr *attr) 1920 { 1921 int r; 1922 1923 switch (attr->group) { 1924 case KVM_DEV_FLIC_GET_ALL_IRQS: 1925 r = get_all_floating_irqs(dev->kvm, (u8 __user *) attr->addr, 1926 attr->attr); 1927 break; 1928 case KVM_DEV_FLIC_AISM_ALL: 1929 r = flic_ais_mode_get_all(dev->kvm, attr); 1930 break; 1931 default: 1932 r = -EINVAL; 1933 } 1934 1935 return r; 1936 } 1937 1938 static inline int copy_irq_from_user(struct kvm_s390_interrupt_info *inti, 1939 u64 addr) 1940 { 1941 struct kvm_s390_irq __user *uptr = (struct kvm_s390_irq __user *) addr; 1942 void *target = NULL; 1943 void __user *source; 1944 u64 size; 1945 1946 if (get_user(inti->type, (u64 __user *)addr)) 1947 return -EFAULT; 1948 1949 switch (inti->type) { 1950 case KVM_S390_INT_PFAULT_INIT: 1951 case KVM_S390_INT_PFAULT_DONE: 1952 case KVM_S390_INT_VIRTIO: 1953 case KVM_S390_INT_SERVICE: 1954 target = (void *) &inti->ext; 1955 source = &uptr->u.ext; 1956 size = sizeof(inti->ext); 1957 break; 1958 case KVM_S390_INT_IO_MIN...KVM_S390_INT_IO_MAX: 1959 target = (void *) &inti->io; 1960 source = &uptr->u.io; 1961 size = sizeof(inti->io); 1962 break; 1963 case KVM_S390_MCHK: 1964 target = (void *) &inti->mchk; 1965 source = &uptr->u.mchk; 1966 size = sizeof(inti->mchk); 1967 break; 1968 default: 1969 return -EINVAL; 1970 } 1971 1972 if (copy_from_user(target, source, size)) 1973 return -EFAULT; 1974 1975 return 0; 1976 } 1977 1978 static int enqueue_floating_irq(struct kvm_device *dev, 1979 struct kvm_device_attr *attr) 1980 { 1981 struct kvm_s390_interrupt_info *inti = NULL; 1982 int r = 0; 1983 int len = attr->attr; 1984 1985 if (len % sizeof(struct kvm_s390_irq) != 0) 1986 return -EINVAL; 1987 else if (len > KVM_S390_FLIC_MAX_BUFFER) 1988 return -EINVAL; 1989 1990 while (len >= sizeof(struct kvm_s390_irq)) { 1991 inti = kzalloc(sizeof(*inti), GFP_KERNEL); 1992 if (!inti) 1993 return -ENOMEM; 1994 1995 r = copy_irq_from_user(inti, attr->addr); 1996 if (r) { 1997 kfree(inti); 1998 return r; 1999 } 2000 r = __inject_vm(dev->kvm, inti); 2001 if (r) { 2002 kfree(inti); 2003 return r; 2004 } 2005 len -= sizeof(struct kvm_s390_irq); 2006 attr->addr += sizeof(struct kvm_s390_irq); 2007 } 2008 2009 return r; 2010 } 2011 2012 static struct s390_io_adapter *get_io_adapter(struct kvm *kvm, unsigned int id) 2013 { 2014 if (id >= MAX_S390_IO_ADAPTERS) 2015 return NULL; 2016 return kvm->arch.adapters[id]; 2017 } 2018 2019 static int register_io_adapter(struct kvm_device *dev, 2020 struct kvm_device_attr *attr) 2021 { 2022 struct s390_io_adapter *adapter; 2023 struct kvm_s390_io_adapter adapter_info; 2024 2025 if (copy_from_user(&adapter_info, 2026 (void __user *)attr->addr, sizeof(adapter_info))) 2027 return -EFAULT; 2028 2029 if ((adapter_info.id >= MAX_S390_IO_ADAPTERS) || 2030 (dev->kvm->arch.adapters[adapter_info.id] != NULL)) 2031 return -EINVAL; 2032 2033 adapter = kzalloc(sizeof(*adapter), GFP_KERNEL); 2034 if (!adapter) 2035 return -ENOMEM; 2036 2037 INIT_LIST_HEAD(&adapter->maps); 2038 init_rwsem(&adapter->maps_lock); 2039 atomic_set(&adapter->nr_maps, 0); 2040 adapter->id = adapter_info.id; 2041 adapter->isc = adapter_info.isc; 2042 adapter->maskable = adapter_info.maskable; 2043 adapter->masked = false; 2044 adapter->swap = adapter_info.swap; 2045 adapter->suppressible = (adapter_info.flags) & 2046 KVM_S390_ADAPTER_SUPPRESSIBLE; 2047 dev->kvm->arch.adapters[adapter->id] = adapter; 2048 2049 return 0; 2050 } 2051 2052 int kvm_s390_mask_adapter(struct kvm *kvm, unsigned int id, bool masked) 2053 { 2054 int ret; 2055 struct s390_io_adapter *adapter = get_io_adapter(kvm, id); 2056 2057 if (!adapter || !adapter->maskable) 2058 return -EINVAL; 2059 ret = adapter->masked; 2060 adapter->masked = masked; 2061 return ret; 2062 } 2063 2064 static int kvm_s390_adapter_map(struct kvm *kvm, unsigned int id, __u64 addr) 2065 { 2066 struct s390_io_adapter *adapter = get_io_adapter(kvm, id); 2067 struct s390_map_info *map; 2068 int ret; 2069 2070 if (!adapter || !addr) 2071 return -EINVAL; 2072 2073 map = kzalloc(sizeof(*map), GFP_KERNEL); 2074 if (!map) { 2075 ret = -ENOMEM; 2076 goto out; 2077 } 2078 INIT_LIST_HEAD(&map->list); 2079 map->guest_addr = addr; 2080 map->addr = gmap_translate(kvm->arch.gmap, addr); 2081 if (map->addr == -EFAULT) { 2082 ret = -EFAULT; 2083 goto out; 2084 } 2085 ret = get_user_pages_fast(map->addr, 1, 1, &map->page); 2086 if (ret < 0) 2087 goto out; 2088 BUG_ON(ret != 1); 2089 down_write(&adapter->maps_lock); 2090 if (atomic_inc_return(&adapter->nr_maps) < MAX_S390_ADAPTER_MAPS) { 2091 list_add_tail(&map->list, &adapter->maps); 2092 ret = 0; 2093 } else { 2094 put_page(map->page); 2095 ret = -EINVAL; 2096 } 2097 up_write(&adapter->maps_lock); 2098 out: 2099 if (ret) 2100 kfree(map); 2101 return ret; 2102 } 2103 2104 static int kvm_s390_adapter_unmap(struct kvm *kvm, unsigned int id, __u64 addr) 2105 { 2106 struct s390_io_adapter *adapter = get_io_adapter(kvm, id); 2107 struct s390_map_info *map, *tmp; 2108 int found = 0; 2109 2110 if (!adapter || !addr) 2111 return -EINVAL; 2112 2113 down_write(&adapter->maps_lock); 2114 list_for_each_entry_safe(map, tmp, &adapter->maps, list) { 2115 if (map->guest_addr == addr) { 2116 found = 1; 2117 atomic_dec(&adapter->nr_maps); 2118 list_del(&map->list); 2119 put_page(map->page); 2120 kfree(map); 2121 break; 2122 } 2123 } 2124 up_write(&adapter->maps_lock); 2125 2126 return found ? 0 : -EINVAL; 2127 } 2128 2129 void kvm_s390_destroy_adapters(struct kvm *kvm) 2130 { 2131 int i; 2132 struct s390_map_info *map, *tmp; 2133 2134 for (i = 0; i < MAX_S390_IO_ADAPTERS; i++) { 2135 if (!kvm->arch.adapters[i]) 2136 continue; 2137 list_for_each_entry_safe(map, tmp, 2138 &kvm->arch.adapters[i]->maps, list) { 2139 list_del(&map->list); 2140 put_page(map->page); 2141 kfree(map); 2142 } 2143 kfree(kvm->arch.adapters[i]); 2144 } 2145 } 2146 2147 static int modify_io_adapter(struct kvm_device *dev, 2148 struct kvm_device_attr *attr) 2149 { 2150 struct kvm_s390_io_adapter_req req; 2151 struct s390_io_adapter *adapter; 2152 int ret; 2153 2154 if (copy_from_user(&req, (void __user *)attr->addr, sizeof(req))) 2155 return -EFAULT; 2156 2157 adapter = get_io_adapter(dev->kvm, req.id); 2158 if (!adapter) 2159 return -EINVAL; 2160 switch (req.type) { 2161 case KVM_S390_IO_ADAPTER_MASK: 2162 ret = kvm_s390_mask_adapter(dev->kvm, req.id, req.mask); 2163 if (ret > 0) 2164 ret = 0; 2165 break; 2166 case KVM_S390_IO_ADAPTER_MAP: 2167 ret = kvm_s390_adapter_map(dev->kvm, req.id, req.addr); 2168 break; 2169 case KVM_S390_IO_ADAPTER_UNMAP: 2170 ret = kvm_s390_adapter_unmap(dev->kvm, req.id, req.addr); 2171 break; 2172 default: 2173 ret = -EINVAL; 2174 } 2175 2176 return ret; 2177 } 2178 2179 static int clear_io_irq(struct kvm *kvm, struct kvm_device_attr *attr) 2180 2181 { 2182 const u64 isc_mask = 0xffUL << 24; /* all iscs set */ 2183 u32 schid; 2184 2185 if (attr->flags) 2186 return -EINVAL; 2187 if (attr->attr != sizeof(schid)) 2188 return -EINVAL; 2189 if (copy_from_user(&schid, (void __user *) attr->addr, sizeof(schid))) 2190 return -EFAULT; 2191 if (!schid) 2192 return -EINVAL; 2193 kfree(kvm_s390_get_io_int(kvm, isc_mask, schid)); 2194 /* 2195 * If userspace is conforming to the architecture, we can have at most 2196 * one pending I/O interrupt per subchannel, so this is effectively a 2197 * clear all. 2198 */ 2199 return 0; 2200 } 2201 2202 static int modify_ais_mode(struct kvm *kvm, struct kvm_device_attr *attr) 2203 { 2204 struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int; 2205 struct kvm_s390_ais_req req; 2206 int ret = 0; 2207 2208 if (!test_kvm_facility(kvm, 72)) 2209 return -ENOTSUPP; 2210 2211 if (copy_from_user(&req, (void __user *)attr->addr, sizeof(req))) 2212 return -EFAULT; 2213 2214 if (req.isc > MAX_ISC) 2215 return -EINVAL; 2216 2217 trace_kvm_s390_modify_ais_mode(req.isc, 2218 (fi->simm & AIS_MODE_MASK(req.isc)) ? 2219 (fi->nimm & AIS_MODE_MASK(req.isc)) ? 2220 2 : KVM_S390_AIS_MODE_SINGLE : 2221 KVM_S390_AIS_MODE_ALL, req.mode); 2222 2223 mutex_lock(&fi->ais_lock); 2224 switch (req.mode) { 2225 case KVM_S390_AIS_MODE_ALL: 2226 fi->simm &= ~AIS_MODE_MASK(req.isc); 2227 fi->nimm &= ~AIS_MODE_MASK(req.isc); 2228 break; 2229 case KVM_S390_AIS_MODE_SINGLE: 2230 fi->simm |= AIS_MODE_MASK(req.isc); 2231 fi->nimm &= ~AIS_MODE_MASK(req.isc); 2232 break; 2233 default: 2234 ret = -EINVAL; 2235 } 2236 mutex_unlock(&fi->ais_lock); 2237 2238 return ret; 2239 } 2240 2241 static int kvm_s390_inject_airq(struct kvm *kvm, 2242 struct s390_io_adapter *adapter) 2243 { 2244 struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int; 2245 struct kvm_s390_interrupt s390int = { 2246 .type = KVM_S390_INT_IO(1, 0, 0, 0), 2247 .parm = 0, 2248 .parm64 = (adapter->isc << 27) | 0x80000000, 2249 }; 2250 int ret = 0; 2251 2252 if (!test_kvm_facility(kvm, 72) || !adapter->suppressible) 2253 return kvm_s390_inject_vm(kvm, &s390int); 2254 2255 mutex_lock(&fi->ais_lock); 2256 if (fi->nimm & AIS_MODE_MASK(adapter->isc)) { 2257 trace_kvm_s390_airq_suppressed(adapter->id, adapter->isc); 2258 goto out; 2259 } 2260 2261 ret = kvm_s390_inject_vm(kvm, &s390int); 2262 if (!ret && (fi->simm & AIS_MODE_MASK(adapter->isc))) { 2263 fi->nimm |= AIS_MODE_MASK(adapter->isc); 2264 trace_kvm_s390_modify_ais_mode(adapter->isc, 2265 KVM_S390_AIS_MODE_SINGLE, 2); 2266 } 2267 out: 2268 mutex_unlock(&fi->ais_lock); 2269 return ret; 2270 } 2271 2272 static int flic_inject_airq(struct kvm *kvm, struct kvm_device_attr *attr) 2273 { 2274 unsigned int id = attr->attr; 2275 struct s390_io_adapter *adapter = get_io_adapter(kvm, id); 2276 2277 if (!adapter) 2278 return -EINVAL; 2279 2280 return kvm_s390_inject_airq(kvm, adapter); 2281 } 2282 2283 static int flic_ais_mode_set_all(struct kvm *kvm, struct kvm_device_attr *attr) 2284 { 2285 struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int; 2286 struct kvm_s390_ais_all ais; 2287 2288 if (!test_kvm_facility(kvm, 72)) 2289 return -ENOTSUPP; 2290 2291 if (copy_from_user(&ais, (void __user *)attr->addr, sizeof(ais))) 2292 return -EFAULT; 2293 2294 mutex_lock(&fi->ais_lock); 2295 fi->simm = ais.simm; 2296 fi->nimm = ais.nimm; 2297 mutex_unlock(&fi->ais_lock); 2298 2299 return 0; 2300 } 2301 2302 static int flic_set_attr(struct kvm_device *dev, struct kvm_device_attr *attr) 2303 { 2304 int r = 0; 2305 unsigned int i; 2306 struct kvm_vcpu *vcpu; 2307 2308 switch (attr->group) { 2309 case KVM_DEV_FLIC_ENQUEUE: 2310 r = enqueue_floating_irq(dev, attr); 2311 break; 2312 case KVM_DEV_FLIC_CLEAR_IRQS: 2313 kvm_s390_clear_float_irqs(dev->kvm); 2314 break; 2315 case KVM_DEV_FLIC_APF_ENABLE: 2316 dev->kvm->arch.gmap->pfault_enabled = 1; 2317 break; 2318 case KVM_DEV_FLIC_APF_DISABLE_WAIT: 2319 dev->kvm->arch.gmap->pfault_enabled = 0; 2320 /* 2321 * Make sure no async faults are in transition when 2322 * clearing the queues. So we don't need to worry 2323 * about late coming workers. 2324 */ 2325 synchronize_srcu(&dev->kvm->srcu); 2326 kvm_for_each_vcpu(i, vcpu, dev->kvm) 2327 kvm_clear_async_pf_completion_queue(vcpu); 2328 break; 2329 case KVM_DEV_FLIC_ADAPTER_REGISTER: 2330 r = register_io_adapter(dev, attr); 2331 break; 2332 case KVM_DEV_FLIC_ADAPTER_MODIFY: 2333 r = modify_io_adapter(dev, attr); 2334 break; 2335 case KVM_DEV_FLIC_CLEAR_IO_IRQ: 2336 r = clear_io_irq(dev->kvm, attr); 2337 break; 2338 case KVM_DEV_FLIC_AISM: 2339 r = modify_ais_mode(dev->kvm, attr); 2340 break; 2341 case KVM_DEV_FLIC_AIRQ_INJECT: 2342 r = flic_inject_airq(dev->kvm, attr); 2343 break; 2344 case KVM_DEV_FLIC_AISM_ALL: 2345 r = flic_ais_mode_set_all(dev->kvm, attr); 2346 break; 2347 default: 2348 r = -EINVAL; 2349 } 2350 2351 return r; 2352 } 2353 2354 static int flic_has_attr(struct kvm_device *dev, 2355 struct kvm_device_attr *attr) 2356 { 2357 switch (attr->group) { 2358 case KVM_DEV_FLIC_GET_ALL_IRQS: 2359 case KVM_DEV_FLIC_ENQUEUE: 2360 case KVM_DEV_FLIC_CLEAR_IRQS: 2361 case KVM_DEV_FLIC_APF_ENABLE: 2362 case KVM_DEV_FLIC_APF_DISABLE_WAIT: 2363 case KVM_DEV_FLIC_ADAPTER_REGISTER: 2364 case KVM_DEV_FLIC_ADAPTER_MODIFY: 2365 case KVM_DEV_FLIC_CLEAR_IO_IRQ: 2366 case KVM_DEV_FLIC_AISM: 2367 case KVM_DEV_FLIC_AIRQ_INJECT: 2368 case KVM_DEV_FLIC_AISM_ALL: 2369 return 0; 2370 } 2371 return -ENXIO; 2372 } 2373 2374 static int flic_create(struct kvm_device *dev, u32 type) 2375 { 2376 if (!dev) 2377 return -EINVAL; 2378 if (dev->kvm->arch.flic) 2379 return -EINVAL; 2380 dev->kvm->arch.flic = dev; 2381 return 0; 2382 } 2383 2384 static void flic_destroy(struct kvm_device *dev) 2385 { 2386 dev->kvm->arch.flic = NULL; 2387 kfree(dev); 2388 } 2389 2390 /* s390 floating irq controller (flic) */ 2391 struct kvm_device_ops kvm_flic_ops = { 2392 .name = "kvm-flic", 2393 .get_attr = flic_get_attr, 2394 .set_attr = flic_set_attr, 2395 .has_attr = flic_has_attr, 2396 .create = flic_create, 2397 .destroy = flic_destroy, 2398 }; 2399 2400 static unsigned long get_ind_bit(__u64 addr, unsigned long bit_nr, bool swap) 2401 { 2402 unsigned long bit; 2403 2404 bit = bit_nr + (addr % PAGE_SIZE) * 8; 2405 2406 return swap ? (bit ^ (BITS_PER_LONG - 1)) : bit; 2407 } 2408 2409 static struct s390_map_info *get_map_info(struct s390_io_adapter *adapter, 2410 u64 addr) 2411 { 2412 struct s390_map_info *map; 2413 2414 if (!adapter) 2415 return NULL; 2416 2417 list_for_each_entry(map, &adapter->maps, list) { 2418 if (map->guest_addr == addr) 2419 return map; 2420 } 2421 return NULL; 2422 } 2423 2424 static int adapter_indicators_set(struct kvm *kvm, 2425 struct s390_io_adapter *adapter, 2426 struct kvm_s390_adapter_int *adapter_int) 2427 { 2428 unsigned long bit; 2429 int summary_set, idx; 2430 struct s390_map_info *info; 2431 void *map; 2432 2433 info = get_map_info(adapter, adapter_int->ind_addr); 2434 if (!info) 2435 return -1; 2436 map = page_address(info->page); 2437 bit = get_ind_bit(info->addr, adapter_int->ind_offset, adapter->swap); 2438 set_bit(bit, map); 2439 idx = srcu_read_lock(&kvm->srcu); 2440 mark_page_dirty(kvm, info->guest_addr >> PAGE_SHIFT); 2441 set_page_dirty_lock(info->page); 2442 info = get_map_info(adapter, adapter_int->summary_addr); 2443 if (!info) { 2444 srcu_read_unlock(&kvm->srcu, idx); 2445 return -1; 2446 } 2447 map = page_address(info->page); 2448 bit = get_ind_bit(info->addr, adapter_int->summary_offset, 2449 adapter->swap); 2450 summary_set = test_and_set_bit(bit, map); 2451 mark_page_dirty(kvm, info->guest_addr >> PAGE_SHIFT); 2452 set_page_dirty_lock(info->page); 2453 srcu_read_unlock(&kvm->srcu, idx); 2454 return summary_set ? 0 : 1; 2455 } 2456 2457 /* 2458 * < 0 - not injected due to error 2459 * = 0 - coalesced, summary indicator already active 2460 * > 0 - injected interrupt 2461 */ 2462 static int set_adapter_int(struct kvm_kernel_irq_routing_entry *e, 2463 struct kvm *kvm, int irq_source_id, int level, 2464 bool line_status) 2465 { 2466 int ret; 2467 struct s390_io_adapter *adapter; 2468 2469 /* We're only interested in the 0->1 transition. */ 2470 if (!level) 2471 return 0; 2472 adapter = get_io_adapter(kvm, e->adapter.adapter_id); 2473 if (!adapter) 2474 return -1; 2475 down_read(&adapter->maps_lock); 2476 ret = adapter_indicators_set(kvm, adapter, &e->adapter); 2477 up_read(&adapter->maps_lock); 2478 if ((ret > 0) && !adapter->masked) { 2479 ret = kvm_s390_inject_airq(kvm, adapter); 2480 if (ret == 0) 2481 ret = 1; 2482 } 2483 return ret; 2484 } 2485 2486 /* 2487 * Inject the machine check to the guest. 2488 */ 2489 void kvm_s390_reinject_machine_check(struct kvm_vcpu *vcpu, 2490 struct mcck_volatile_info *mcck_info) 2491 { 2492 struct kvm_s390_interrupt_info inti; 2493 struct kvm_s390_irq irq; 2494 struct kvm_s390_mchk_info *mchk; 2495 union mci mci; 2496 __u64 cr14 = 0; /* upper bits are not used */ 2497 int rc; 2498 2499 mci.val = mcck_info->mcic; 2500 if (mci.sr) 2501 cr14 |= CR14_RECOVERY_SUBMASK; 2502 if (mci.dg) 2503 cr14 |= CR14_DEGRADATION_SUBMASK; 2504 if (mci.w) 2505 cr14 |= CR14_WARNING_SUBMASK; 2506 2507 mchk = mci.ck ? &inti.mchk : &irq.u.mchk; 2508 mchk->cr14 = cr14; 2509 mchk->mcic = mcck_info->mcic; 2510 mchk->ext_damage_code = mcck_info->ext_damage_code; 2511 mchk->failing_storage_address = mcck_info->failing_storage_address; 2512 if (mci.ck) { 2513 /* Inject the floating machine check */ 2514 inti.type = KVM_S390_MCHK; 2515 rc = __inject_vm(vcpu->kvm, &inti); 2516 } else { 2517 /* Inject the machine check to specified vcpu */ 2518 irq.type = KVM_S390_MCHK; 2519 rc = kvm_s390_inject_vcpu(vcpu, &irq); 2520 } 2521 WARN_ON_ONCE(rc); 2522 } 2523 2524 int kvm_set_routing_entry(struct kvm *kvm, 2525 struct kvm_kernel_irq_routing_entry *e, 2526 const struct kvm_irq_routing_entry *ue) 2527 { 2528 int ret; 2529 2530 switch (ue->type) { 2531 case KVM_IRQ_ROUTING_S390_ADAPTER: 2532 e->set = set_adapter_int; 2533 e->adapter.summary_addr = ue->u.adapter.summary_addr; 2534 e->adapter.ind_addr = ue->u.adapter.ind_addr; 2535 e->adapter.summary_offset = ue->u.adapter.summary_offset; 2536 e->adapter.ind_offset = ue->u.adapter.ind_offset; 2537 e->adapter.adapter_id = ue->u.adapter.adapter_id; 2538 ret = 0; 2539 break; 2540 default: 2541 ret = -EINVAL; 2542 } 2543 2544 return ret; 2545 } 2546 2547 int kvm_set_msi(struct kvm_kernel_irq_routing_entry *e, struct kvm *kvm, 2548 int irq_source_id, int level, bool line_status) 2549 { 2550 return -EINVAL; 2551 } 2552 2553 int kvm_s390_set_irq_state(struct kvm_vcpu *vcpu, void __user *irqstate, int len) 2554 { 2555 struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int; 2556 struct kvm_s390_irq *buf; 2557 int r = 0; 2558 int n; 2559 2560 buf = vmalloc(len); 2561 if (!buf) 2562 return -ENOMEM; 2563 2564 if (copy_from_user((void *) buf, irqstate, len)) { 2565 r = -EFAULT; 2566 goto out_free; 2567 } 2568 2569 /* 2570 * Don't allow setting the interrupt state 2571 * when there are already interrupts pending 2572 */ 2573 spin_lock(&li->lock); 2574 if (li->pending_irqs) { 2575 r = -EBUSY; 2576 goto out_unlock; 2577 } 2578 2579 for (n = 0; n < len / sizeof(*buf); n++) { 2580 r = do_inject_vcpu(vcpu, &buf[n]); 2581 if (r) 2582 break; 2583 } 2584 2585 out_unlock: 2586 spin_unlock(&li->lock); 2587 out_free: 2588 vfree(buf); 2589 2590 return r; 2591 } 2592 2593 static void store_local_irq(struct kvm_s390_local_interrupt *li, 2594 struct kvm_s390_irq *irq, 2595 unsigned long irq_type) 2596 { 2597 switch (irq_type) { 2598 case IRQ_PEND_MCHK_EX: 2599 case IRQ_PEND_MCHK_REP: 2600 irq->type = KVM_S390_MCHK; 2601 irq->u.mchk = li->irq.mchk; 2602 break; 2603 case IRQ_PEND_PROG: 2604 irq->type = KVM_S390_PROGRAM_INT; 2605 irq->u.pgm = li->irq.pgm; 2606 break; 2607 case IRQ_PEND_PFAULT_INIT: 2608 irq->type = KVM_S390_INT_PFAULT_INIT; 2609 irq->u.ext = li->irq.ext; 2610 break; 2611 case IRQ_PEND_EXT_EXTERNAL: 2612 irq->type = KVM_S390_INT_EXTERNAL_CALL; 2613 irq->u.extcall = li->irq.extcall; 2614 break; 2615 case IRQ_PEND_EXT_CLOCK_COMP: 2616 irq->type = KVM_S390_INT_CLOCK_COMP; 2617 break; 2618 case IRQ_PEND_EXT_CPU_TIMER: 2619 irq->type = KVM_S390_INT_CPU_TIMER; 2620 break; 2621 case IRQ_PEND_SIGP_STOP: 2622 irq->type = KVM_S390_SIGP_STOP; 2623 irq->u.stop = li->irq.stop; 2624 break; 2625 case IRQ_PEND_RESTART: 2626 irq->type = KVM_S390_RESTART; 2627 break; 2628 case IRQ_PEND_SET_PREFIX: 2629 irq->type = KVM_S390_SIGP_SET_PREFIX; 2630 irq->u.prefix = li->irq.prefix; 2631 break; 2632 } 2633 } 2634 2635 int kvm_s390_get_irq_state(struct kvm_vcpu *vcpu, __u8 __user *buf, int len) 2636 { 2637 int scn; 2638 unsigned long sigp_emerg_pending[BITS_TO_LONGS(KVM_MAX_VCPUS)]; 2639 struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int; 2640 unsigned long pending_irqs; 2641 struct kvm_s390_irq irq; 2642 unsigned long irq_type; 2643 int cpuaddr; 2644 int n = 0; 2645 2646 spin_lock(&li->lock); 2647 pending_irqs = li->pending_irqs; 2648 memcpy(&sigp_emerg_pending, &li->sigp_emerg_pending, 2649 sizeof(sigp_emerg_pending)); 2650 spin_unlock(&li->lock); 2651 2652 for_each_set_bit(irq_type, &pending_irqs, IRQ_PEND_COUNT) { 2653 memset(&irq, 0, sizeof(irq)); 2654 if (irq_type == IRQ_PEND_EXT_EMERGENCY) 2655 continue; 2656 if (n + sizeof(irq) > len) 2657 return -ENOBUFS; 2658 store_local_irq(&vcpu->arch.local_int, &irq, irq_type); 2659 if (copy_to_user(&buf[n], &irq, sizeof(irq))) 2660 return -EFAULT; 2661 n += sizeof(irq); 2662 } 2663 2664 if (test_bit(IRQ_PEND_EXT_EMERGENCY, &pending_irqs)) { 2665 for_each_set_bit(cpuaddr, sigp_emerg_pending, KVM_MAX_VCPUS) { 2666 memset(&irq, 0, sizeof(irq)); 2667 if (n + sizeof(irq) > len) 2668 return -ENOBUFS; 2669 irq.type = KVM_S390_INT_EMERGENCY; 2670 irq.u.emerg.code = cpuaddr; 2671 if (copy_to_user(&buf[n], &irq, sizeof(irq))) 2672 return -EFAULT; 2673 n += sizeof(irq); 2674 } 2675 } 2676 2677 if (sca_ext_call_pending(vcpu, &scn)) { 2678 if (n + sizeof(irq) > len) 2679 return -ENOBUFS; 2680 memset(&irq, 0, sizeof(irq)); 2681 irq.type = KVM_S390_INT_EXTERNAL_CALL; 2682 irq.u.extcall.code = scn; 2683 if (copy_to_user(&buf[n], &irq, sizeof(irq))) 2684 return -EFAULT; 2685 n += sizeof(irq); 2686 } 2687 2688 return n; 2689 } 2690