1 /* 2 * handling kvm guest interrupts 3 * 4 * Copyright IBM Corp. 2008, 2015 5 * 6 * This program is free software; you can redistribute it and/or modify 7 * it under the terms of the GNU General Public License (version 2 only) 8 * as published by the Free Software Foundation. 9 * 10 * Author(s): Carsten Otte <cotte@de.ibm.com> 11 */ 12 13 #include <linux/interrupt.h> 14 #include <linux/kvm_host.h> 15 #include <linux/hrtimer.h> 16 #include <linux/mmu_context.h> 17 #include <linux/signal.h> 18 #include <linux/slab.h> 19 #include <linux/bitmap.h> 20 #include <linux/vmalloc.h> 21 #include <asm/asm-offsets.h> 22 #include <asm/dis.h> 23 #include <linux/uaccess.h> 24 #include <asm/sclp.h> 25 #include <asm/isc.h> 26 #include <asm/gmap.h> 27 #include <asm/switch_to.h> 28 #include <asm/nmi.h> 29 #include "kvm-s390.h" 30 #include "gaccess.h" 31 #include "trace-s390.h" 32 33 #define PFAULT_INIT 0x0600 34 #define PFAULT_DONE 0x0680 35 #define VIRTIO_PARAM 0x0d00 36 37 /* handle external calls via sigp interpretation facility */ 38 static int sca_ext_call_pending(struct kvm_vcpu *vcpu, int *src_id) 39 { 40 int c, scn; 41 42 if (!(atomic_read(&vcpu->arch.sie_block->cpuflags) & CPUSTAT_ECALL_PEND)) 43 return 0; 44 45 BUG_ON(!kvm_s390_use_sca_entries()); 46 read_lock(&vcpu->kvm->arch.sca_lock); 47 if (vcpu->kvm->arch.use_esca) { 48 struct esca_block *sca = vcpu->kvm->arch.sca; 49 union esca_sigp_ctrl sigp_ctrl = 50 sca->cpu[vcpu->vcpu_id].sigp_ctrl; 51 52 c = sigp_ctrl.c; 53 scn = sigp_ctrl.scn; 54 } else { 55 struct bsca_block *sca = vcpu->kvm->arch.sca; 56 union bsca_sigp_ctrl sigp_ctrl = 57 sca->cpu[vcpu->vcpu_id].sigp_ctrl; 58 59 c = sigp_ctrl.c; 60 scn = sigp_ctrl.scn; 61 } 62 read_unlock(&vcpu->kvm->arch.sca_lock); 63 64 if (src_id) 65 *src_id = scn; 66 67 return c; 68 } 69 70 static int sca_inject_ext_call(struct kvm_vcpu *vcpu, int src_id) 71 { 72 int expect, rc; 73 74 BUG_ON(!kvm_s390_use_sca_entries()); 75 read_lock(&vcpu->kvm->arch.sca_lock); 76 if (vcpu->kvm->arch.use_esca) { 77 struct esca_block *sca = vcpu->kvm->arch.sca; 78 union esca_sigp_ctrl *sigp_ctrl = 79 &(sca->cpu[vcpu->vcpu_id].sigp_ctrl); 80 union esca_sigp_ctrl new_val = {0}, old_val = *sigp_ctrl; 81 82 new_val.scn = src_id; 83 new_val.c = 1; 84 old_val.c = 0; 85 86 expect = old_val.value; 87 rc = cmpxchg(&sigp_ctrl->value, old_val.value, new_val.value); 88 } else { 89 struct bsca_block *sca = vcpu->kvm->arch.sca; 90 union bsca_sigp_ctrl *sigp_ctrl = 91 &(sca->cpu[vcpu->vcpu_id].sigp_ctrl); 92 union bsca_sigp_ctrl new_val = {0}, old_val = *sigp_ctrl; 93 94 new_val.scn = src_id; 95 new_val.c = 1; 96 old_val.c = 0; 97 98 expect = old_val.value; 99 rc = cmpxchg(&sigp_ctrl->value, old_val.value, new_val.value); 100 } 101 read_unlock(&vcpu->kvm->arch.sca_lock); 102 103 if (rc != expect) { 104 /* another external call is pending */ 105 return -EBUSY; 106 } 107 atomic_or(CPUSTAT_ECALL_PEND, &vcpu->arch.sie_block->cpuflags); 108 return 0; 109 } 110 111 static void sca_clear_ext_call(struct kvm_vcpu *vcpu) 112 { 113 struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int; 114 int rc, expect; 115 116 if (!kvm_s390_use_sca_entries()) 117 return; 118 atomic_andnot(CPUSTAT_ECALL_PEND, li->cpuflags); 119 read_lock(&vcpu->kvm->arch.sca_lock); 120 if (vcpu->kvm->arch.use_esca) { 121 struct esca_block *sca = vcpu->kvm->arch.sca; 122 union esca_sigp_ctrl *sigp_ctrl = 123 &(sca->cpu[vcpu->vcpu_id].sigp_ctrl); 124 union esca_sigp_ctrl old = *sigp_ctrl; 125 126 expect = old.value; 127 rc = cmpxchg(&sigp_ctrl->value, old.value, 0); 128 } else { 129 struct bsca_block *sca = vcpu->kvm->arch.sca; 130 union bsca_sigp_ctrl *sigp_ctrl = 131 &(sca->cpu[vcpu->vcpu_id].sigp_ctrl); 132 union bsca_sigp_ctrl old = *sigp_ctrl; 133 134 expect = old.value; 135 rc = cmpxchg(&sigp_ctrl->value, old.value, 0); 136 } 137 read_unlock(&vcpu->kvm->arch.sca_lock); 138 WARN_ON(rc != expect); /* cannot clear? */ 139 } 140 141 int psw_extint_disabled(struct kvm_vcpu *vcpu) 142 { 143 return !(vcpu->arch.sie_block->gpsw.mask & PSW_MASK_EXT); 144 } 145 146 static int psw_ioint_disabled(struct kvm_vcpu *vcpu) 147 { 148 return !(vcpu->arch.sie_block->gpsw.mask & PSW_MASK_IO); 149 } 150 151 static int psw_mchk_disabled(struct kvm_vcpu *vcpu) 152 { 153 return !(vcpu->arch.sie_block->gpsw.mask & PSW_MASK_MCHECK); 154 } 155 156 static int psw_interrupts_disabled(struct kvm_vcpu *vcpu) 157 { 158 return psw_extint_disabled(vcpu) && 159 psw_ioint_disabled(vcpu) && 160 psw_mchk_disabled(vcpu); 161 } 162 163 static int ckc_interrupts_enabled(struct kvm_vcpu *vcpu) 164 { 165 if (psw_extint_disabled(vcpu) || 166 !(vcpu->arch.sie_block->gcr[0] & 0x800ul)) 167 return 0; 168 if (guestdbg_enabled(vcpu) && guestdbg_sstep_enabled(vcpu)) 169 /* No timer interrupts when single stepping */ 170 return 0; 171 return 1; 172 } 173 174 static int ckc_irq_pending(struct kvm_vcpu *vcpu) 175 { 176 if (vcpu->arch.sie_block->ckc >= kvm_s390_get_tod_clock_fast(vcpu->kvm)) 177 return 0; 178 return ckc_interrupts_enabled(vcpu); 179 } 180 181 static int cpu_timer_interrupts_enabled(struct kvm_vcpu *vcpu) 182 { 183 return !psw_extint_disabled(vcpu) && 184 (vcpu->arch.sie_block->gcr[0] & 0x400ul); 185 } 186 187 static int cpu_timer_irq_pending(struct kvm_vcpu *vcpu) 188 { 189 if (!cpu_timer_interrupts_enabled(vcpu)) 190 return 0; 191 return kvm_s390_get_cpu_timer(vcpu) >> 63; 192 } 193 194 static inline int is_ioirq(unsigned long irq_type) 195 { 196 return ((irq_type >= IRQ_PEND_IO_ISC_0) && 197 (irq_type <= IRQ_PEND_IO_ISC_7)); 198 } 199 200 static uint64_t isc_to_isc_bits(int isc) 201 { 202 return (0x80 >> isc) << 24; 203 } 204 205 static inline u8 int_word_to_isc(u32 int_word) 206 { 207 return (int_word & 0x38000000) >> 27; 208 } 209 210 static inline unsigned long pending_irqs(struct kvm_vcpu *vcpu) 211 { 212 return vcpu->kvm->arch.float_int.pending_irqs | 213 vcpu->arch.local_int.pending_irqs; 214 } 215 216 static inline int isc_to_irq_type(unsigned long isc) 217 { 218 return IRQ_PEND_IO_ISC_0 + isc; 219 } 220 221 static inline int irq_type_to_isc(unsigned long irq_type) 222 { 223 return irq_type - IRQ_PEND_IO_ISC_0; 224 } 225 226 static unsigned long disable_iscs(struct kvm_vcpu *vcpu, 227 unsigned long active_mask) 228 { 229 int i; 230 231 for (i = 0; i <= MAX_ISC; i++) 232 if (!(vcpu->arch.sie_block->gcr[6] & isc_to_isc_bits(i))) 233 active_mask &= ~(1UL << (isc_to_irq_type(i))); 234 235 return active_mask; 236 } 237 238 static unsigned long deliverable_irqs(struct kvm_vcpu *vcpu) 239 { 240 unsigned long active_mask; 241 242 active_mask = pending_irqs(vcpu); 243 if (!active_mask) 244 return 0; 245 246 if (psw_extint_disabled(vcpu)) 247 active_mask &= ~IRQ_PEND_EXT_MASK; 248 if (psw_ioint_disabled(vcpu)) 249 active_mask &= ~IRQ_PEND_IO_MASK; 250 else 251 active_mask = disable_iscs(vcpu, active_mask); 252 if (!(vcpu->arch.sie_block->gcr[0] & 0x2000ul)) 253 __clear_bit(IRQ_PEND_EXT_EXTERNAL, &active_mask); 254 if (!(vcpu->arch.sie_block->gcr[0] & 0x4000ul)) 255 __clear_bit(IRQ_PEND_EXT_EMERGENCY, &active_mask); 256 if (!(vcpu->arch.sie_block->gcr[0] & 0x800ul)) 257 __clear_bit(IRQ_PEND_EXT_CLOCK_COMP, &active_mask); 258 if (!(vcpu->arch.sie_block->gcr[0] & 0x400ul)) 259 __clear_bit(IRQ_PEND_EXT_CPU_TIMER, &active_mask); 260 if (!(vcpu->arch.sie_block->gcr[0] & 0x200ul)) 261 __clear_bit(IRQ_PEND_EXT_SERVICE, &active_mask); 262 if (psw_mchk_disabled(vcpu)) 263 active_mask &= ~IRQ_PEND_MCHK_MASK; 264 /* 265 * Check both floating and local interrupt's cr14 because 266 * bit IRQ_PEND_MCHK_REP could be set in both cases. 267 */ 268 if (!(vcpu->arch.sie_block->gcr[14] & 269 (vcpu->kvm->arch.float_int.mchk.cr14 | 270 vcpu->arch.local_int.irq.mchk.cr14))) 271 __clear_bit(IRQ_PEND_MCHK_REP, &active_mask); 272 273 /* 274 * STOP irqs will never be actively delivered. They are triggered via 275 * intercept requests and cleared when the stop intercept is performed. 276 */ 277 __clear_bit(IRQ_PEND_SIGP_STOP, &active_mask); 278 279 return active_mask; 280 } 281 282 static void __set_cpu_idle(struct kvm_vcpu *vcpu) 283 { 284 atomic_or(CPUSTAT_WAIT, &vcpu->arch.sie_block->cpuflags); 285 set_bit(vcpu->vcpu_id, vcpu->arch.local_int.float_int->idle_mask); 286 } 287 288 static void __unset_cpu_idle(struct kvm_vcpu *vcpu) 289 { 290 atomic_andnot(CPUSTAT_WAIT, &vcpu->arch.sie_block->cpuflags); 291 clear_bit(vcpu->vcpu_id, vcpu->arch.local_int.float_int->idle_mask); 292 } 293 294 static void __reset_intercept_indicators(struct kvm_vcpu *vcpu) 295 { 296 atomic_andnot(CPUSTAT_IO_INT | CPUSTAT_EXT_INT | CPUSTAT_STOP_INT, 297 &vcpu->arch.sie_block->cpuflags); 298 vcpu->arch.sie_block->lctl = 0x0000; 299 vcpu->arch.sie_block->ictl &= ~(ICTL_LPSW | ICTL_STCTL | ICTL_PINT); 300 301 if (guestdbg_enabled(vcpu)) { 302 vcpu->arch.sie_block->lctl |= (LCTL_CR0 | LCTL_CR9 | 303 LCTL_CR10 | LCTL_CR11); 304 vcpu->arch.sie_block->ictl |= (ICTL_STCTL | ICTL_PINT); 305 } 306 } 307 308 static void __set_cpuflag(struct kvm_vcpu *vcpu, u32 flag) 309 { 310 atomic_or(flag, &vcpu->arch.sie_block->cpuflags); 311 } 312 313 static void set_intercept_indicators_io(struct kvm_vcpu *vcpu) 314 { 315 if (!(pending_irqs(vcpu) & IRQ_PEND_IO_MASK)) 316 return; 317 else if (psw_ioint_disabled(vcpu)) 318 __set_cpuflag(vcpu, CPUSTAT_IO_INT); 319 else 320 vcpu->arch.sie_block->lctl |= LCTL_CR6; 321 } 322 323 static void set_intercept_indicators_ext(struct kvm_vcpu *vcpu) 324 { 325 if (!(pending_irqs(vcpu) & IRQ_PEND_EXT_MASK)) 326 return; 327 if (psw_extint_disabled(vcpu)) 328 __set_cpuflag(vcpu, CPUSTAT_EXT_INT); 329 else 330 vcpu->arch.sie_block->lctl |= LCTL_CR0; 331 } 332 333 static void set_intercept_indicators_mchk(struct kvm_vcpu *vcpu) 334 { 335 if (!(pending_irqs(vcpu) & IRQ_PEND_MCHK_MASK)) 336 return; 337 if (psw_mchk_disabled(vcpu)) 338 vcpu->arch.sie_block->ictl |= ICTL_LPSW; 339 else 340 vcpu->arch.sie_block->lctl |= LCTL_CR14; 341 } 342 343 static void set_intercept_indicators_stop(struct kvm_vcpu *vcpu) 344 { 345 if (kvm_s390_is_stop_irq_pending(vcpu)) 346 __set_cpuflag(vcpu, CPUSTAT_STOP_INT); 347 } 348 349 /* Set interception request for non-deliverable interrupts */ 350 static void set_intercept_indicators(struct kvm_vcpu *vcpu) 351 { 352 set_intercept_indicators_io(vcpu); 353 set_intercept_indicators_ext(vcpu); 354 set_intercept_indicators_mchk(vcpu); 355 set_intercept_indicators_stop(vcpu); 356 } 357 358 static int __must_check __deliver_cpu_timer(struct kvm_vcpu *vcpu) 359 { 360 struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int; 361 int rc; 362 363 trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, KVM_S390_INT_CPU_TIMER, 364 0, 0); 365 366 rc = put_guest_lc(vcpu, EXT_IRQ_CPU_TIMER, 367 (u16 *)__LC_EXT_INT_CODE); 368 rc |= put_guest_lc(vcpu, 0, (u16 *)__LC_EXT_CPU_ADDR); 369 rc |= write_guest_lc(vcpu, __LC_EXT_OLD_PSW, 370 &vcpu->arch.sie_block->gpsw, sizeof(psw_t)); 371 rc |= read_guest_lc(vcpu, __LC_EXT_NEW_PSW, 372 &vcpu->arch.sie_block->gpsw, sizeof(psw_t)); 373 clear_bit(IRQ_PEND_EXT_CPU_TIMER, &li->pending_irqs); 374 return rc ? -EFAULT : 0; 375 } 376 377 static int __must_check __deliver_ckc(struct kvm_vcpu *vcpu) 378 { 379 struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int; 380 int rc; 381 382 trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, KVM_S390_INT_CLOCK_COMP, 383 0, 0); 384 385 rc = put_guest_lc(vcpu, EXT_IRQ_CLK_COMP, 386 (u16 __user *)__LC_EXT_INT_CODE); 387 rc |= put_guest_lc(vcpu, 0, (u16 *)__LC_EXT_CPU_ADDR); 388 rc |= write_guest_lc(vcpu, __LC_EXT_OLD_PSW, 389 &vcpu->arch.sie_block->gpsw, sizeof(psw_t)); 390 rc |= read_guest_lc(vcpu, __LC_EXT_NEW_PSW, 391 &vcpu->arch.sie_block->gpsw, sizeof(psw_t)); 392 clear_bit(IRQ_PEND_EXT_CLOCK_COMP, &li->pending_irqs); 393 return rc ? -EFAULT : 0; 394 } 395 396 static int __must_check __deliver_pfault_init(struct kvm_vcpu *vcpu) 397 { 398 struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int; 399 struct kvm_s390_ext_info ext; 400 int rc; 401 402 spin_lock(&li->lock); 403 ext = li->irq.ext; 404 clear_bit(IRQ_PEND_PFAULT_INIT, &li->pending_irqs); 405 li->irq.ext.ext_params2 = 0; 406 spin_unlock(&li->lock); 407 408 VCPU_EVENT(vcpu, 4, "deliver: pfault init token 0x%llx", 409 ext.ext_params2); 410 trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, 411 KVM_S390_INT_PFAULT_INIT, 412 0, ext.ext_params2); 413 414 rc = put_guest_lc(vcpu, EXT_IRQ_CP_SERVICE, (u16 *) __LC_EXT_INT_CODE); 415 rc |= put_guest_lc(vcpu, PFAULT_INIT, (u16 *) __LC_EXT_CPU_ADDR); 416 rc |= write_guest_lc(vcpu, __LC_EXT_OLD_PSW, 417 &vcpu->arch.sie_block->gpsw, sizeof(psw_t)); 418 rc |= read_guest_lc(vcpu, __LC_EXT_NEW_PSW, 419 &vcpu->arch.sie_block->gpsw, sizeof(psw_t)); 420 rc |= put_guest_lc(vcpu, ext.ext_params2, (u64 *) __LC_EXT_PARAMS2); 421 return rc ? -EFAULT : 0; 422 } 423 424 static int __write_machine_check(struct kvm_vcpu *vcpu, 425 struct kvm_s390_mchk_info *mchk) 426 { 427 unsigned long ext_sa_addr; 428 unsigned long lc; 429 freg_t fprs[NUM_FPRS]; 430 union mci mci; 431 int rc; 432 433 mci.val = mchk->mcic; 434 /* take care of lazy register loading */ 435 save_fpu_regs(); 436 save_access_regs(vcpu->run->s.regs.acrs); 437 if (MACHINE_HAS_GS && vcpu->arch.gs_enabled) 438 save_gs_cb(current->thread.gs_cb); 439 440 /* Extended save area */ 441 rc = read_guest_lc(vcpu, __LC_MCESAD, &ext_sa_addr, 442 sizeof(unsigned long)); 443 /* Only bits 0 through 63-LC are used for address formation */ 444 lc = ext_sa_addr & MCESA_LC_MASK; 445 if (test_kvm_facility(vcpu->kvm, 133)) { 446 switch (lc) { 447 case 0: 448 case 10: 449 ext_sa_addr &= ~0x3ffUL; 450 break; 451 case 11: 452 ext_sa_addr &= ~0x7ffUL; 453 break; 454 case 12: 455 ext_sa_addr &= ~0xfffUL; 456 break; 457 default: 458 ext_sa_addr = 0; 459 break; 460 } 461 } else { 462 ext_sa_addr &= ~0x3ffUL; 463 } 464 465 if (!rc && mci.vr && ext_sa_addr && test_kvm_facility(vcpu->kvm, 129)) { 466 if (write_guest_abs(vcpu, ext_sa_addr, vcpu->run->s.regs.vrs, 467 512)) 468 mci.vr = 0; 469 } else { 470 mci.vr = 0; 471 } 472 if (!rc && mci.gs && ext_sa_addr && test_kvm_facility(vcpu->kvm, 133) 473 && (lc == 11 || lc == 12)) { 474 if (write_guest_abs(vcpu, ext_sa_addr + 1024, 475 &vcpu->run->s.regs.gscb, 32)) 476 mci.gs = 0; 477 } else { 478 mci.gs = 0; 479 } 480 481 /* General interruption information */ 482 rc |= put_guest_lc(vcpu, 1, (u8 __user *) __LC_AR_MODE_ID); 483 rc |= write_guest_lc(vcpu, __LC_MCK_OLD_PSW, 484 &vcpu->arch.sie_block->gpsw, sizeof(psw_t)); 485 rc |= read_guest_lc(vcpu, __LC_MCK_NEW_PSW, 486 &vcpu->arch.sie_block->gpsw, sizeof(psw_t)); 487 rc |= put_guest_lc(vcpu, mci.val, (u64 __user *) __LC_MCCK_CODE); 488 489 /* Register-save areas */ 490 if (MACHINE_HAS_VX) { 491 convert_vx_to_fp(fprs, (__vector128 *) vcpu->run->s.regs.vrs); 492 rc |= write_guest_lc(vcpu, __LC_FPREGS_SAVE_AREA, fprs, 128); 493 } else { 494 rc |= write_guest_lc(vcpu, __LC_FPREGS_SAVE_AREA, 495 vcpu->run->s.regs.fprs, 128); 496 } 497 rc |= write_guest_lc(vcpu, __LC_GPREGS_SAVE_AREA, 498 vcpu->run->s.regs.gprs, 128); 499 rc |= put_guest_lc(vcpu, current->thread.fpu.fpc, 500 (u32 __user *) __LC_FP_CREG_SAVE_AREA); 501 rc |= put_guest_lc(vcpu, vcpu->arch.sie_block->todpr, 502 (u32 __user *) __LC_TOD_PROGREG_SAVE_AREA); 503 rc |= put_guest_lc(vcpu, kvm_s390_get_cpu_timer(vcpu), 504 (u64 __user *) __LC_CPU_TIMER_SAVE_AREA); 505 rc |= put_guest_lc(vcpu, vcpu->arch.sie_block->ckc >> 8, 506 (u64 __user *) __LC_CLOCK_COMP_SAVE_AREA); 507 rc |= write_guest_lc(vcpu, __LC_AREGS_SAVE_AREA, 508 &vcpu->run->s.regs.acrs, 64); 509 rc |= write_guest_lc(vcpu, __LC_CREGS_SAVE_AREA, 510 &vcpu->arch.sie_block->gcr, 128); 511 512 /* Extended interruption information */ 513 rc |= put_guest_lc(vcpu, mchk->ext_damage_code, 514 (u32 __user *) __LC_EXT_DAMAGE_CODE); 515 rc |= put_guest_lc(vcpu, mchk->failing_storage_address, 516 (u64 __user *) __LC_MCCK_FAIL_STOR_ADDR); 517 rc |= write_guest_lc(vcpu, __LC_PSW_SAVE_AREA, &mchk->fixed_logout, 518 sizeof(mchk->fixed_logout)); 519 return rc ? -EFAULT : 0; 520 } 521 522 static int __must_check __deliver_machine_check(struct kvm_vcpu *vcpu) 523 { 524 struct kvm_s390_float_interrupt *fi = &vcpu->kvm->arch.float_int; 525 struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int; 526 struct kvm_s390_mchk_info mchk = {}; 527 int deliver = 0; 528 int rc = 0; 529 530 spin_lock(&fi->lock); 531 spin_lock(&li->lock); 532 if (test_bit(IRQ_PEND_MCHK_EX, &li->pending_irqs) || 533 test_bit(IRQ_PEND_MCHK_REP, &li->pending_irqs)) { 534 /* 535 * If there was an exigent machine check pending, then any 536 * repressible machine checks that might have been pending 537 * are indicated along with it, so always clear bits for 538 * repressible and exigent interrupts 539 */ 540 mchk = li->irq.mchk; 541 clear_bit(IRQ_PEND_MCHK_EX, &li->pending_irqs); 542 clear_bit(IRQ_PEND_MCHK_REP, &li->pending_irqs); 543 memset(&li->irq.mchk, 0, sizeof(mchk)); 544 deliver = 1; 545 } 546 /* 547 * We indicate floating repressible conditions along with 548 * other pending conditions. Channel Report Pending and Channel 549 * Subsystem damage are the only two and and are indicated by 550 * bits in mcic and masked in cr14. 551 */ 552 if (test_and_clear_bit(IRQ_PEND_MCHK_REP, &fi->pending_irqs)) { 553 mchk.mcic |= fi->mchk.mcic; 554 mchk.cr14 |= fi->mchk.cr14; 555 memset(&fi->mchk, 0, sizeof(mchk)); 556 deliver = 1; 557 } 558 spin_unlock(&li->lock); 559 spin_unlock(&fi->lock); 560 561 if (deliver) { 562 VCPU_EVENT(vcpu, 3, "deliver: machine check mcic 0x%llx", 563 mchk.mcic); 564 trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, 565 KVM_S390_MCHK, 566 mchk.cr14, mchk.mcic); 567 rc = __write_machine_check(vcpu, &mchk); 568 } 569 return rc; 570 } 571 572 static int __must_check __deliver_restart(struct kvm_vcpu *vcpu) 573 { 574 struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int; 575 int rc; 576 577 VCPU_EVENT(vcpu, 3, "%s", "deliver: cpu restart"); 578 vcpu->stat.deliver_restart_signal++; 579 trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, KVM_S390_RESTART, 0, 0); 580 581 rc = write_guest_lc(vcpu, 582 offsetof(struct lowcore, restart_old_psw), 583 &vcpu->arch.sie_block->gpsw, sizeof(psw_t)); 584 rc |= read_guest_lc(vcpu, offsetof(struct lowcore, restart_psw), 585 &vcpu->arch.sie_block->gpsw, sizeof(psw_t)); 586 clear_bit(IRQ_PEND_RESTART, &li->pending_irqs); 587 return rc ? -EFAULT : 0; 588 } 589 590 static int __must_check __deliver_set_prefix(struct kvm_vcpu *vcpu) 591 { 592 struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int; 593 struct kvm_s390_prefix_info prefix; 594 595 spin_lock(&li->lock); 596 prefix = li->irq.prefix; 597 li->irq.prefix.address = 0; 598 clear_bit(IRQ_PEND_SET_PREFIX, &li->pending_irqs); 599 spin_unlock(&li->lock); 600 601 vcpu->stat.deliver_prefix_signal++; 602 trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, 603 KVM_S390_SIGP_SET_PREFIX, 604 prefix.address, 0); 605 606 kvm_s390_set_prefix(vcpu, prefix.address); 607 return 0; 608 } 609 610 static int __must_check __deliver_emergency_signal(struct kvm_vcpu *vcpu) 611 { 612 struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int; 613 int rc; 614 int cpu_addr; 615 616 spin_lock(&li->lock); 617 cpu_addr = find_first_bit(li->sigp_emerg_pending, KVM_MAX_VCPUS); 618 clear_bit(cpu_addr, li->sigp_emerg_pending); 619 if (bitmap_empty(li->sigp_emerg_pending, KVM_MAX_VCPUS)) 620 clear_bit(IRQ_PEND_EXT_EMERGENCY, &li->pending_irqs); 621 spin_unlock(&li->lock); 622 623 VCPU_EVENT(vcpu, 4, "%s", "deliver: sigp emerg"); 624 vcpu->stat.deliver_emergency_signal++; 625 trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, KVM_S390_INT_EMERGENCY, 626 cpu_addr, 0); 627 628 rc = put_guest_lc(vcpu, EXT_IRQ_EMERGENCY_SIG, 629 (u16 *)__LC_EXT_INT_CODE); 630 rc |= put_guest_lc(vcpu, cpu_addr, (u16 *)__LC_EXT_CPU_ADDR); 631 rc |= write_guest_lc(vcpu, __LC_EXT_OLD_PSW, 632 &vcpu->arch.sie_block->gpsw, sizeof(psw_t)); 633 rc |= read_guest_lc(vcpu, __LC_EXT_NEW_PSW, 634 &vcpu->arch.sie_block->gpsw, sizeof(psw_t)); 635 return rc ? -EFAULT : 0; 636 } 637 638 static int __must_check __deliver_external_call(struct kvm_vcpu *vcpu) 639 { 640 struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int; 641 struct kvm_s390_extcall_info extcall; 642 int rc; 643 644 spin_lock(&li->lock); 645 extcall = li->irq.extcall; 646 li->irq.extcall.code = 0; 647 clear_bit(IRQ_PEND_EXT_EXTERNAL, &li->pending_irqs); 648 spin_unlock(&li->lock); 649 650 VCPU_EVENT(vcpu, 4, "%s", "deliver: sigp ext call"); 651 vcpu->stat.deliver_external_call++; 652 trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, 653 KVM_S390_INT_EXTERNAL_CALL, 654 extcall.code, 0); 655 656 rc = put_guest_lc(vcpu, EXT_IRQ_EXTERNAL_CALL, 657 (u16 *)__LC_EXT_INT_CODE); 658 rc |= put_guest_lc(vcpu, extcall.code, (u16 *)__LC_EXT_CPU_ADDR); 659 rc |= write_guest_lc(vcpu, __LC_EXT_OLD_PSW, 660 &vcpu->arch.sie_block->gpsw, sizeof(psw_t)); 661 rc |= read_guest_lc(vcpu, __LC_EXT_NEW_PSW, &vcpu->arch.sie_block->gpsw, 662 sizeof(psw_t)); 663 return rc ? -EFAULT : 0; 664 } 665 666 static int __must_check __deliver_prog(struct kvm_vcpu *vcpu) 667 { 668 struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int; 669 struct kvm_s390_pgm_info pgm_info; 670 int rc = 0, nullifying = false; 671 u16 ilen; 672 673 spin_lock(&li->lock); 674 pgm_info = li->irq.pgm; 675 clear_bit(IRQ_PEND_PROG, &li->pending_irqs); 676 memset(&li->irq.pgm, 0, sizeof(pgm_info)); 677 spin_unlock(&li->lock); 678 679 ilen = pgm_info.flags & KVM_S390_PGM_FLAGS_ILC_MASK; 680 VCPU_EVENT(vcpu, 3, "deliver: program irq code 0x%x, ilen:%d", 681 pgm_info.code, ilen); 682 vcpu->stat.deliver_program_int++; 683 trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, KVM_S390_PROGRAM_INT, 684 pgm_info.code, 0); 685 686 switch (pgm_info.code & ~PGM_PER) { 687 case PGM_AFX_TRANSLATION: 688 case PGM_ASX_TRANSLATION: 689 case PGM_EX_TRANSLATION: 690 case PGM_LFX_TRANSLATION: 691 case PGM_LSTE_SEQUENCE: 692 case PGM_LSX_TRANSLATION: 693 case PGM_LX_TRANSLATION: 694 case PGM_PRIMARY_AUTHORITY: 695 case PGM_SECONDARY_AUTHORITY: 696 nullifying = true; 697 /* fall through */ 698 case PGM_SPACE_SWITCH: 699 rc = put_guest_lc(vcpu, pgm_info.trans_exc_code, 700 (u64 *)__LC_TRANS_EXC_CODE); 701 break; 702 case PGM_ALEN_TRANSLATION: 703 case PGM_ALE_SEQUENCE: 704 case PGM_ASTE_INSTANCE: 705 case PGM_ASTE_SEQUENCE: 706 case PGM_ASTE_VALIDITY: 707 case PGM_EXTENDED_AUTHORITY: 708 rc = put_guest_lc(vcpu, pgm_info.exc_access_id, 709 (u8 *)__LC_EXC_ACCESS_ID); 710 nullifying = true; 711 break; 712 case PGM_ASCE_TYPE: 713 case PGM_PAGE_TRANSLATION: 714 case PGM_REGION_FIRST_TRANS: 715 case PGM_REGION_SECOND_TRANS: 716 case PGM_REGION_THIRD_TRANS: 717 case PGM_SEGMENT_TRANSLATION: 718 rc = put_guest_lc(vcpu, pgm_info.trans_exc_code, 719 (u64 *)__LC_TRANS_EXC_CODE); 720 rc |= put_guest_lc(vcpu, pgm_info.exc_access_id, 721 (u8 *)__LC_EXC_ACCESS_ID); 722 rc |= put_guest_lc(vcpu, pgm_info.op_access_id, 723 (u8 *)__LC_OP_ACCESS_ID); 724 nullifying = true; 725 break; 726 case PGM_MONITOR: 727 rc = put_guest_lc(vcpu, pgm_info.mon_class_nr, 728 (u16 *)__LC_MON_CLASS_NR); 729 rc |= put_guest_lc(vcpu, pgm_info.mon_code, 730 (u64 *)__LC_MON_CODE); 731 break; 732 case PGM_VECTOR_PROCESSING: 733 case PGM_DATA: 734 rc = put_guest_lc(vcpu, pgm_info.data_exc_code, 735 (u32 *)__LC_DATA_EXC_CODE); 736 break; 737 case PGM_PROTECTION: 738 rc = put_guest_lc(vcpu, pgm_info.trans_exc_code, 739 (u64 *)__LC_TRANS_EXC_CODE); 740 rc |= put_guest_lc(vcpu, pgm_info.exc_access_id, 741 (u8 *)__LC_EXC_ACCESS_ID); 742 break; 743 case PGM_STACK_FULL: 744 case PGM_STACK_EMPTY: 745 case PGM_STACK_SPECIFICATION: 746 case PGM_STACK_TYPE: 747 case PGM_STACK_OPERATION: 748 case PGM_TRACE_TABEL: 749 case PGM_CRYPTO_OPERATION: 750 nullifying = true; 751 break; 752 } 753 754 if (pgm_info.code & PGM_PER) { 755 rc |= put_guest_lc(vcpu, pgm_info.per_code, 756 (u8 *) __LC_PER_CODE); 757 rc |= put_guest_lc(vcpu, pgm_info.per_atmid, 758 (u8 *)__LC_PER_ATMID); 759 rc |= put_guest_lc(vcpu, pgm_info.per_address, 760 (u64 *) __LC_PER_ADDRESS); 761 rc |= put_guest_lc(vcpu, pgm_info.per_access_id, 762 (u8 *) __LC_PER_ACCESS_ID); 763 } 764 765 if (nullifying && !(pgm_info.flags & KVM_S390_PGM_FLAGS_NO_REWIND)) 766 kvm_s390_rewind_psw(vcpu, ilen); 767 768 /* bit 1+2 of the target are the ilc, so we can directly use ilen */ 769 rc |= put_guest_lc(vcpu, ilen, (u16 *) __LC_PGM_ILC); 770 rc |= put_guest_lc(vcpu, vcpu->arch.sie_block->gbea, 771 (u64 *) __LC_LAST_BREAK); 772 rc |= put_guest_lc(vcpu, pgm_info.code, 773 (u16 *)__LC_PGM_INT_CODE); 774 rc |= write_guest_lc(vcpu, __LC_PGM_OLD_PSW, 775 &vcpu->arch.sie_block->gpsw, sizeof(psw_t)); 776 rc |= read_guest_lc(vcpu, __LC_PGM_NEW_PSW, 777 &vcpu->arch.sie_block->gpsw, sizeof(psw_t)); 778 return rc ? -EFAULT : 0; 779 } 780 781 static int __must_check __deliver_service(struct kvm_vcpu *vcpu) 782 { 783 struct kvm_s390_float_interrupt *fi = &vcpu->kvm->arch.float_int; 784 struct kvm_s390_ext_info ext; 785 int rc = 0; 786 787 spin_lock(&fi->lock); 788 if (!(test_bit(IRQ_PEND_EXT_SERVICE, &fi->pending_irqs))) { 789 spin_unlock(&fi->lock); 790 return 0; 791 } 792 ext = fi->srv_signal; 793 memset(&fi->srv_signal, 0, sizeof(ext)); 794 clear_bit(IRQ_PEND_EXT_SERVICE, &fi->pending_irqs); 795 spin_unlock(&fi->lock); 796 797 VCPU_EVENT(vcpu, 4, "deliver: sclp parameter 0x%x", 798 ext.ext_params); 799 vcpu->stat.deliver_service_signal++; 800 trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, KVM_S390_INT_SERVICE, 801 ext.ext_params, 0); 802 803 rc = put_guest_lc(vcpu, EXT_IRQ_SERVICE_SIG, (u16 *)__LC_EXT_INT_CODE); 804 rc |= put_guest_lc(vcpu, 0, (u16 *)__LC_EXT_CPU_ADDR); 805 rc |= write_guest_lc(vcpu, __LC_EXT_OLD_PSW, 806 &vcpu->arch.sie_block->gpsw, sizeof(psw_t)); 807 rc |= read_guest_lc(vcpu, __LC_EXT_NEW_PSW, 808 &vcpu->arch.sie_block->gpsw, sizeof(psw_t)); 809 rc |= put_guest_lc(vcpu, ext.ext_params, 810 (u32 *)__LC_EXT_PARAMS); 811 812 return rc ? -EFAULT : 0; 813 } 814 815 static int __must_check __deliver_pfault_done(struct kvm_vcpu *vcpu) 816 { 817 struct kvm_s390_float_interrupt *fi = &vcpu->kvm->arch.float_int; 818 struct kvm_s390_interrupt_info *inti; 819 int rc = 0; 820 821 spin_lock(&fi->lock); 822 inti = list_first_entry_or_null(&fi->lists[FIRQ_LIST_PFAULT], 823 struct kvm_s390_interrupt_info, 824 list); 825 if (inti) { 826 list_del(&inti->list); 827 fi->counters[FIRQ_CNTR_PFAULT] -= 1; 828 } 829 if (list_empty(&fi->lists[FIRQ_LIST_PFAULT])) 830 clear_bit(IRQ_PEND_PFAULT_DONE, &fi->pending_irqs); 831 spin_unlock(&fi->lock); 832 833 if (inti) { 834 trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, 835 KVM_S390_INT_PFAULT_DONE, 0, 836 inti->ext.ext_params2); 837 VCPU_EVENT(vcpu, 4, "deliver: pfault done token 0x%llx", 838 inti->ext.ext_params2); 839 840 rc = put_guest_lc(vcpu, EXT_IRQ_CP_SERVICE, 841 (u16 *)__LC_EXT_INT_CODE); 842 rc |= put_guest_lc(vcpu, PFAULT_DONE, 843 (u16 *)__LC_EXT_CPU_ADDR); 844 rc |= write_guest_lc(vcpu, __LC_EXT_OLD_PSW, 845 &vcpu->arch.sie_block->gpsw, 846 sizeof(psw_t)); 847 rc |= read_guest_lc(vcpu, __LC_EXT_NEW_PSW, 848 &vcpu->arch.sie_block->gpsw, 849 sizeof(psw_t)); 850 rc |= put_guest_lc(vcpu, inti->ext.ext_params2, 851 (u64 *)__LC_EXT_PARAMS2); 852 kfree(inti); 853 } 854 return rc ? -EFAULT : 0; 855 } 856 857 static int __must_check __deliver_virtio(struct kvm_vcpu *vcpu) 858 { 859 struct kvm_s390_float_interrupt *fi = &vcpu->kvm->arch.float_int; 860 struct kvm_s390_interrupt_info *inti; 861 int rc = 0; 862 863 spin_lock(&fi->lock); 864 inti = list_first_entry_or_null(&fi->lists[FIRQ_LIST_VIRTIO], 865 struct kvm_s390_interrupt_info, 866 list); 867 if (inti) { 868 VCPU_EVENT(vcpu, 4, 869 "deliver: virtio parm: 0x%x,parm64: 0x%llx", 870 inti->ext.ext_params, inti->ext.ext_params2); 871 vcpu->stat.deliver_virtio_interrupt++; 872 trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, 873 inti->type, 874 inti->ext.ext_params, 875 inti->ext.ext_params2); 876 list_del(&inti->list); 877 fi->counters[FIRQ_CNTR_VIRTIO] -= 1; 878 } 879 if (list_empty(&fi->lists[FIRQ_LIST_VIRTIO])) 880 clear_bit(IRQ_PEND_VIRTIO, &fi->pending_irqs); 881 spin_unlock(&fi->lock); 882 883 if (inti) { 884 rc = put_guest_lc(vcpu, EXT_IRQ_CP_SERVICE, 885 (u16 *)__LC_EXT_INT_CODE); 886 rc |= put_guest_lc(vcpu, VIRTIO_PARAM, 887 (u16 *)__LC_EXT_CPU_ADDR); 888 rc |= write_guest_lc(vcpu, __LC_EXT_OLD_PSW, 889 &vcpu->arch.sie_block->gpsw, 890 sizeof(psw_t)); 891 rc |= read_guest_lc(vcpu, __LC_EXT_NEW_PSW, 892 &vcpu->arch.sie_block->gpsw, 893 sizeof(psw_t)); 894 rc |= put_guest_lc(vcpu, inti->ext.ext_params, 895 (u32 *)__LC_EXT_PARAMS); 896 rc |= put_guest_lc(vcpu, inti->ext.ext_params2, 897 (u64 *)__LC_EXT_PARAMS2); 898 kfree(inti); 899 } 900 return rc ? -EFAULT : 0; 901 } 902 903 static int __must_check __deliver_io(struct kvm_vcpu *vcpu, 904 unsigned long irq_type) 905 { 906 struct list_head *isc_list; 907 struct kvm_s390_float_interrupt *fi; 908 struct kvm_s390_interrupt_info *inti = NULL; 909 int rc = 0; 910 911 fi = &vcpu->kvm->arch.float_int; 912 913 spin_lock(&fi->lock); 914 isc_list = &fi->lists[irq_type_to_isc(irq_type)]; 915 inti = list_first_entry_or_null(isc_list, 916 struct kvm_s390_interrupt_info, 917 list); 918 if (inti) { 919 if (inti->type & KVM_S390_INT_IO_AI_MASK) 920 VCPU_EVENT(vcpu, 4, "%s", "deliver: I/O (AI)"); 921 else 922 VCPU_EVENT(vcpu, 4, "deliver: I/O %x ss %x schid %04x", 923 inti->io.subchannel_id >> 8, 924 inti->io.subchannel_id >> 1 & 0x3, 925 inti->io.subchannel_nr); 926 927 vcpu->stat.deliver_io_int++; 928 trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, 929 inti->type, 930 ((__u32)inti->io.subchannel_id << 16) | 931 inti->io.subchannel_nr, 932 ((__u64)inti->io.io_int_parm << 32) | 933 inti->io.io_int_word); 934 list_del(&inti->list); 935 fi->counters[FIRQ_CNTR_IO] -= 1; 936 } 937 if (list_empty(isc_list)) 938 clear_bit(irq_type, &fi->pending_irqs); 939 spin_unlock(&fi->lock); 940 941 if (inti) { 942 rc = put_guest_lc(vcpu, inti->io.subchannel_id, 943 (u16 *)__LC_SUBCHANNEL_ID); 944 rc |= put_guest_lc(vcpu, inti->io.subchannel_nr, 945 (u16 *)__LC_SUBCHANNEL_NR); 946 rc |= put_guest_lc(vcpu, inti->io.io_int_parm, 947 (u32 *)__LC_IO_INT_PARM); 948 rc |= put_guest_lc(vcpu, inti->io.io_int_word, 949 (u32 *)__LC_IO_INT_WORD); 950 rc |= write_guest_lc(vcpu, __LC_IO_OLD_PSW, 951 &vcpu->arch.sie_block->gpsw, 952 sizeof(psw_t)); 953 rc |= read_guest_lc(vcpu, __LC_IO_NEW_PSW, 954 &vcpu->arch.sie_block->gpsw, 955 sizeof(psw_t)); 956 kfree(inti); 957 } 958 959 return rc ? -EFAULT : 0; 960 } 961 962 typedef int (*deliver_irq_t)(struct kvm_vcpu *vcpu); 963 964 static const deliver_irq_t deliver_irq_funcs[] = { 965 [IRQ_PEND_MCHK_EX] = __deliver_machine_check, 966 [IRQ_PEND_MCHK_REP] = __deliver_machine_check, 967 [IRQ_PEND_PROG] = __deliver_prog, 968 [IRQ_PEND_EXT_EMERGENCY] = __deliver_emergency_signal, 969 [IRQ_PEND_EXT_EXTERNAL] = __deliver_external_call, 970 [IRQ_PEND_EXT_CLOCK_COMP] = __deliver_ckc, 971 [IRQ_PEND_EXT_CPU_TIMER] = __deliver_cpu_timer, 972 [IRQ_PEND_RESTART] = __deliver_restart, 973 [IRQ_PEND_SET_PREFIX] = __deliver_set_prefix, 974 [IRQ_PEND_PFAULT_INIT] = __deliver_pfault_init, 975 [IRQ_PEND_EXT_SERVICE] = __deliver_service, 976 [IRQ_PEND_PFAULT_DONE] = __deliver_pfault_done, 977 [IRQ_PEND_VIRTIO] = __deliver_virtio, 978 }; 979 980 /* Check whether an external call is pending (deliverable or not) */ 981 int kvm_s390_ext_call_pending(struct kvm_vcpu *vcpu) 982 { 983 struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int; 984 985 if (!sclp.has_sigpif) 986 return test_bit(IRQ_PEND_EXT_EXTERNAL, &li->pending_irqs); 987 988 return sca_ext_call_pending(vcpu, NULL); 989 } 990 991 int kvm_s390_vcpu_has_irq(struct kvm_vcpu *vcpu, int exclude_stop) 992 { 993 if (deliverable_irqs(vcpu)) 994 return 1; 995 996 if (kvm_cpu_has_pending_timer(vcpu)) 997 return 1; 998 999 /* external call pending and deliverable */ 1000 if (kvm_s390_ext_call_pending(vcpu) && 1001 !psw_extint_disabled(vcpu) && 1002 (vcpu->arch.sie_block->gcr[0] & 0x2000ul)) 1003 return 1; 1004 1005 if (!exclude_stop && kvm_s390_is_stop_irq_pending(vcpu)) 1006 return 1; 1007 return 0; 1008 } 1009 1010 int kvm_cpu_has_pending_timer(struct kvm_vcpu *vcpu) 1011 { 1012 return ckc_irq_pending(vcpu) || cpu_timer_irq_pending(vcpu); 1013 } 1014 1015 static u64 __calculate_sltime(struct kvm_vcpu *vcpu) 1016 { 1017 u64 now, cputm, sltime = 0; 1018 1019 if (ckc_interrupts_enabled(vcpu)) { 1020 now = kvm_s390_get_tod_clock_fast(vcpu->kvm); 1021 sltime = tod_to_ns(vcpu->arch.sie_block->ckc - now); 1022 /* already expired or overflow? */ 1023 if (!sltime || vcpu->arch.sie_block->ckc <= now) 1024 return 0; 1025 if (cpu_timer_interrupts_enabled(vcpu)) { 1026 cputm = kvm_s390_get_cpu_timer(vcpu); 1027 /* already expired? */ 1028 if (cputm >> 63) 1029 return 0; 1030 return min(sltime, tod_to_ns(cputm)); 1031 } 1032 } else if (cpu_timer_interrupts_enabled(vcpu)) { 1033 sltime = kvm_s390_get_cpu_timer(vcpu); 1034 /* already expired? */ 1035 if (sltime >> 63) 1036 return 0; 1037 } 1038 return sltime; 1039 } 1040 1041 int kvm_s390_handle_wait(struct kvm_vcpu *vcpu) 1042 { 1043 u64 sltime; 1044 1045 vcpu->stat.exit_wait_state++; 1046 1047 /* fast path */ 1048 if (kvm_arch_vcpu_runnable(vcpu)) 1049 return 0; 1050 1051 if (psw_interrupts_disabled(vcpu)) { 1052 VCPU_EVENT(vcpu, 3, "%s", "disabled wait"); 1053 return -EOPNOTSUPP; /* disabled wait */ 1054 } 1055 1056 if (!ckc_interrupts_enabled(vcpu) && 1057 !cpu_timer_interrupts_enabled(vcpu)) { 1058 VCPU_EVENT(vcpu, 3, "%s", "enabled wait w/o timer"); 1059 __set_cpu_idle(vcpu); 1060 goto no_timer; 1061 } 1062 1063 sltime = __calculate_sltime(vcpu); 1064 if (!sltime) 1065 return 0; 1066 1067 __set_cpu_idle(vcpu); 1068 hrtimer_start(&vcpu->arch.ckc_timer, sltime, HRTIMER_MODE_REL); 1069 VCPU_EVENT(vcpu, 4, "enabled wait: %llu ns", sltime); 1070 no_timer: 1071 srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx); 1072 kvm_vcpu_block(vcpu); 1073 __unset_cpu_idle(vcpu); 1074 vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu); 1075 1076 hrtimer_cancel(&vcpu->arch.ckc_timer); 1077 return 0; 1078 } 1079 1080 void kvm_s390_vcpu_wakeup(struct kvm_vcpu *vcpu) 1081 { 1082 /* 1083 * We cannot move this into the if, as the CPU might be already 1084 * in kvm_vcpu_block without having the waitqueue set (polling) 1085 */ 1086 vcpu->valid_wakeup = true; 1087 /* 1088 * This is mostly to document, that the read in swait_active could 1089 * be moved before other stores, leading to subtle races. 1090 * All current users do not store or use an atomic like update 1091 */ 1092 smp_mb__after_atomic(); 1093 if (swait_active(&vcpu->wq)) { 1094 /* 1095 * The vcpu gave up the cpu voluntarily, mark it as a good 1096 * yield-candidate. 1097 */ 1098 vcpu->preempted = true; 1099 swake_up(&vcpu->wq); 1100 vcpu->stat.halt_wakeup++; 1101 } 1102 /* 1103 * The VCPU might not be sleeping but is executing the VSIE. Let's 1104 * kick it, so it leaves the SIE to process the request. 1105 */ 1106 kvm_s390_vsie_kick(vcpu); 1107 } 1108 1109 enum hrtimer_restart kvm_s390_idle_wakeup(struct hrtimer *timer) 1110 { 1111 struct kvm_vcpu *vcpu; 1112 u64 sltime; 1113 1114 vcpu = container_of(timer, struct kvm_vcpu, arch.ckc_timer); 1115 sltime = __calculate_sltime(vcpu); 1116 1117 /* 1118 * If the monotonic clock runs faster than the tod clock we might be 1119 * woken up too early and have to go back to sleep to avoid deadlocks. 1120 */ 1121 if (sltime && hrtimer_forward_now(timer, ns_to_ktime(sltime))) 1122 return HRTIMER_RESTART; 1123 kvm_s390_vcpu_wakeup(vcpu); 1124 return HRTIMER_NORESTART; 1125 } 1126 1127 void kvm_s390_clear_local_irqs(struct kvm_vcpu *vcpu) 1128 { 1129 struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int; 1130 1131 spin_lock(&li->lock); 1132 li->pending_irqs = 0; 1133 bitmap_zero(li->sigp_emerg_pending, KVM_MAX_VCPUS); 1134 memset(&li->irq, 0, sizeof(li->irq)); 1135 spin_unlock(&li->lock); 1136 1137 sca_clear_ext_call(vcpu); 1138 } 1139 1140 int __must_check kvm_s390_deliver_pending_interrupts(struct kvm_vcpu *vcpu) 1141 { 1142 struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int; 1143 deliver_irq_t func; 1144 int rc = 0; 1145 unsigned long irq_type; 1146 unsigned long irqs; 1147 1148 __reset_intercept_indicators(vcpu); 1149 1150 /* pending ckc conditions might have been invalidated */ 1151 clear_bit(IRQ_PEND_EXT_CLOCK_COMP, &li->pending_irqs); 1152 if (ckc_irq_pending(vcpu)) 1153 set_bit(IRQ_PEND_EXT_CLOCK_COMP, &li->pending_irqs); 1154 1155 /* pending cpu timer conditions might have been invalidated */ 1156 clear_bit(IRQ_PEND_EXT_CPU_TIMER, &li->pending_irqs); 1157 if (cpu_timer_irq_pending(vcpu)) 1158 set_bit(IRQ_PEND_EXT_CPU_TIMER, &li->pending_irqs); 1159 1160 while ((irqs = deliverable_irqs(vcpu)) && !rc) { 1161 /* bits are in the order of interrupt priority */ 1162 irq_type = find_first_bit(&irqs, IRQ_PEND_COUNT); 1163 if (is_ioirq(irq_type)) { 1164 rc = __deliver_io(vcpu, irq_type); 1165 } else { 1166 func = deliver_irq_funcs[irq_type]; 1167 if (!func) { 1168 WARN_ON_ONCE(func == NULL); 1169 clear_bit(irq_type, &li->pending_irqs); 1170 continue; 1171 } 1172 rc = func(vcpu); 1173 } 1174 } 1175 1176 set_intercept_indicators(vcpu); 1177 1178 return rc; 1179 } 1180 1181 static int __inject_prog(struct kvm_vcpu *vcpu, struct kvm_s390_irq *irq) 1182 { 1183 struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int; 1184 1185 VCPU_EVENT(vcpu, 3, "inject: program irq code 0x%x", irq->u.pgm.code); 1186 trace_kvm_s390_inject_vcpu(vcpu->vcpu_id, KVM_S390_PROGRAM_INT, 1187 irq->u.pgm.code, 0); 1188 1189 if (!(irq->u.pgm.flags & KVM_S390_PGM_FLAGS_ILC_VALID)) { 1190 /* auto detection if no valid ILC was given */ 1191 irq->u.pgm.flags &= ~KVM_S390_PGM_FLAGS_ILC_MASK; 1192 irq->u.pgm.flags |= kvm_s390_get_ilen(vcpu); 1193 irq->u.pgm.flags |= KVM_S390_PGM_FLAGS_ILC_VALID; 1194 } 1195 1196 if (irq->u.pgm.code == PGM_PER) { 1197 li->irq.pgm.code |= PGM_PER; 1198 li->irq.pgm.flags = irq->u.pgm.flags; 1199 /* only modify PER related information */ 1200 li->irq.pgm.per_address = irq->u.pgm.per_address; 1201 li->irq.pgm.per_code = irq->u.pgm.per_code; 1202 li->irq.pgm.per_atmid = irq->u.pgm.per_atmid; 1203 li->irq.pgm.per_access_id = irq->u.pgm.per_access_id; 1204 } else if (!(irq->u.pgm.code & PGM_PER)) { 1205 li->irq.pgm.code = (li->irq.pgm.code & PGM_PER) | 1206 irq->u.pgm.code; 1207 li->irq.pgm.flags = irq->u.pgm.flags; 1208 /* only modify non-PER information */ 1209 li->irq.pgm.trans_exc_code = irq->u.pgm.trans_exc_code; 1210 li->irq.pgm.mon_code = irq->u.pgm.mon_code; 1211 li->irq.pgm.data_exc_code = irq->u.pgm.data_exc_code; 1212 li->irq.pgm.mon_class_nr = irq->u.pgm.mon_class_nr; 1213 li->irq.pgm.exc_access_id = irq->u.pgm.exc_access_id; 1214 li->irq.pgm.op_access_id = irq->u.pgm.op_access_id; 1215 } else { 1216 li->irq.pgm = irq->u.pgm; 1217 } 1218 set_bit(IRQ_PEND_PROG, &li->pending_irqs); 1219 return 0; 1220 } 1221 1222 static int __inject_pfault_init(struct kvm_vcpu *vcpu, struct kvm_s390_irq *irq) 1223 { 1224 struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int; 1225 1226 VCPU_EVENT(vcpu, 4, "inject: pfault init parameter block at 0x%llx", 1227 irq->u.ext.ext_params2); 1228 trace_kvm_s390_inject_vcpu(vcpu->vcpu_id, KVM_S390_INT_PFAULT_INIT, 1229 irq->u.ext.ext_params, 1230 irq->u.ext.ext_params2); 1231 1232 li->irq.ext = irq->u.ext; 1233 set_bit(IRQ_PEND_PFAULT_INIT, &li->pending_irqs); 1234 atomic_or(CPUSTAT_EXT_INT, li->cpuflags); 1235 return 0; 1236 } 1237 1238 static int __inject_extcall(struct kvm_vcpu *vcpu, struct kvm_s390_irq *irq) 1239 { 1240 struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int; 1241 struct kvm_s390_extcall_info *extcall = &li->irq.extcall; 1242 uint16_t src_id = irq->u.extcall.code; 1243 1244 VCPU_EVENT(vcpu, 4, "inject: external call source-cpu:%u", 1245 src_id); 1246 trace_kvm_s390_inject_vcpu(vcpu->vcpu_id, KVM_S390_INT_EXTERNAL_CALL, 1247 src_id, 0); 1248 1249 /* sending vcpu invalid */ 1250 if (kvm_get_vcpu_by_id(vcpu->kvm, src_id) == NULL) 1251 return -EINVAL; 1252 1253 if (sclp.has_sigpif) 1254 return sca_inject_ext_call(vcpu, src_id); 1255 1256 if (test_and_set_bit(IRQ_PEND_EXT_EXTERNAL, &li->pending_irqs)) 1257 return -EBUSY; 1258 *extcall = irq->u.extcall; 1259 atomic_or(CPUSTAT_EXT_INT, li->cpuflags); 1260 return 0; 1261 } 1262 1263 static int __inject_set_prefix(struct kvm_vcpu *vcpu, struct kvm_s390_irq *irq) 1264 { 1265 struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int; 1266 struct kvm_s390_prefix_info *prefix = &li->irq.prefix; 1267 1268 VCPU_EVENT(vcpu, 3, "inject: set prefix to %x", 1269 irq->u.prefix.address); 1270 trace_kvm_s390_inject_vcpu(vcpu->vcpu_id, KVM_S390_SIGP_SET_PREFIX, 1271 irq->u.prefix.address, 0); 1272 1273 if (!is_vcpu_stopped(vcpu)) 1274 return -EBUSY; 1275 1276 *prefix = irq->u.prefix; 1277 set_bit(IRQ_PEND_SET_PREFIX, &li->pending_irqs); 1278 return 0; 1279 } 1280 1281 #define KVM_S390_STOP_SUPP_FLAGS (KVM_S390_STOP_FLAG_STORE_STATUS) 1282 static int __inject_sigp_stop(struct kvm_vcpu *vcpu, struct kvm_s390_irq *irq) 1283 { 1284 struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int; 1285 struct kvm_s390_stop_info *stop = &li->irq.stop; 1286 int rc = 0; 1287 1288 trace_kvm_s390_inject_vcpu(vcpu->vcpu_id, KVM_S390_SIGP_STOP, 0, 0); 1289 1290 if (irq->u.stop.flags & ~KVM_S390_STOP_SUPP_FLAGS) 1291 return -EINVAL; 1292 1293 if (is_vcpu_stopped(vcpu)) { 1294 if (irq->u.stop.flags & KVM_S390_STOP_FLAG_STORE_STATUS) 1295 rc = kvm_s390_store_status_unloaded(vcpu, 1296 KVM_S390_STORE_STATUS_NOADDR); 1297 return rc; 1298 } 1299 1300 if (test_and_set_bit(IRQ_PEND_SIGP_STOP, &li->pending_irqs)) 1301 return -EBUSY; 1302 stop->flags = irq->u.stop.flags; 1303 __set_cpuflag(vcpu, CPUSTAT_STOP_INT); 1304 return 0; 1305 } 1306 1307 static int __inject_sigp_restart(struct kvm_vcpu *vcpu, 1308 struct kvm_s390_irq *irq) 1309 { 1310 struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int; 1311 1312 VCPU_EVENT(vcpu, 3, "%s", "inject: restart int"); 1313 trace_kvm_s390_inject_vcpu(vcpu->vcpu_id, KVM_S390_RESTART, 0, 0); 1314 1315 set_bit(IRQ_PEND_RESTART, &li->pending_irqs); 1316 return 0; 1317 } 1318 1319 static int __inject_sigp_emergency(struct kvm_vcpu *vcpu, 1320 struct kvm_s390_irq *irq) 1321 { 1322 struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int; 1323 1324 VCPU_EVENT(vcpu, 4, "inject: emergency from cpu %u", 1325 irq->u.emerg.code); 1326 trace_kvm_s390_inject_vcpu(vcpu->vcpu_id, KVM_S390_INT_EMERGENCY, 1327 irq->u.emerg.code, 0); 1328 1329 /* sending vcpu invalid */ 1330 if (kvm_get_vcpu_by_id(vcpu->kvm, irq->u.emerg.code) == NULL) 1331 return -EINVAL; 1332 1333 set_bit(irq->u.emerg.code, li->sigp_emerg_pending); 1334 set_bit(IRQ_PEND_EXT_EMERGENCY, &li->pending_irqs); 1335 atomic_or(CPUSTAT_EXT_INT, li->cpuflags); 1336 return 0; 1337 } 1338 1339 static int __inject_mchk(struct kvm_vcpu *vcpu, struct kvm_s390_irq *irq) 1340 { 1341 struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int; 1342 struct kvm_s390_mchk_info *mchk = &li->irq.mchk; 1343 1344 VCPU_EVENT(vcpu, 3, "inject: machine check mcic 0x%llx", 1345 irq->u.mchk.mcic); 1346 trace_kvm_s390_inject_vcpu(vcpu->vcpu_id, KVM_S390_MCHK, 0, 1347 irq->u.mchk.mcic); 1348 1349 /* 1350 * Because repressible machine checks can be indicated along with 1351 * exigent machine checks (PoP, Chapter 11, Interruption action) 1352 * we need to combine cr14, mcic and external damage code. 1353 * Failing storage address and the logout area should not be or'ed 1354 * together, we just indicate the last occurrence of the corresponding 1355 * machine check 1356 */ 1357 mchk->cr14 |= irq->u.mchk.cr14; 1358 mchk->mcic |= irq->u.mchk.mcic; 1359 mchk->ext_damage_code |= irq->u.mchk.ext_damage_code; 1360 mchk->failing_storage_address = irq->u.mchk.failing_storage_address; 1361 memcpy(&mchk->fixed_logout, &irq->u.mchk.fixed_logout, 1362 sizeof(mchk->fixed_logout)); 1363 if (mchk->mcic & MCHK_EX_MASK) 1364 set_bit(IRQ_PEND_MCHK_EX, &li->pending_irqs); 1365 else if (mchk->mcic & MCHK_REP_MASK) 1366 set_bit(IRQ_PEND_MCHK_REP, &li->pending_irqs); 1367 return 0; 1368 } 1369 1370 static int __inject_ckc(struct kvm_vcpu *vcpu) 1371 { 1372 struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int; 1373 1374 VCPU_EVENT(vcpu, 3, "%s", "inject: clock comparator external"); 1375 trace_kvm_s390_inject_vcpu(vcpu->vcpu_id, KVM_S390_INT_CLOCK_COMP, 1376 0, 0); 1377 1378 set_bit(IRQ_PEND_EXT_CLOCK_COMP, &li->pending_irqs); 1379 atomic_or(CPUSTAT_EXT_INT, li->cpuflags); 1380 return 0; 1381 } 1382 1383 static int __inject_cpu_timer(struct kvm_vcpu *vcpu) 1384 { 1385 struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int; 1386 1387 VCPU_EVENT(vcpu, 3, "%s", "inject: cpu timer external"); 1388 trace_kvm_s390_inject_vcpu(vcpu->vcpu_id, KVM_S390_INT_CPU_TIMER, 1389 0, 0); 1390 1391 set_bit(IRQ_PEND_EXT_CPU_TIMER, &li->pending_irqs); 1392 atomic_or(CPUSTAT_EXT_INT, li->cpuflags); 1393 return 0; 1394 } 1395 1396 static struct kvm_s390_interrupt_info *get_io_int(struct kvm *kvm, 1397 int isc, u32 schid) 1398 { 1399 struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int; 1400 struct list_head *isc_list = &fi->lists[FIRQ_LIST_IO_ISC_0 + isc]; 1401 struct kvm_s390_interrupt_info *iter; 1402 u16 id = (schid & 0xffff0000U) >> 16; 1403 u16 nr = schid & 0x0000ffffU; 1404 1405 spin_lock(&fi->lock); 1406 list_for_each_entry(iter, isc_list, list) { 1407 if (schid && (id != iter->io.subchannel_id || 1408 nr != iter->io.subchannel_nr)) 1409 continue; 1410 /* found an appropriate entry */ 1411 list_del_init(&iter->list); 1412 fi->counters[FIRQ_CNTR_IO] -= 1; 1413 if (list_empty(isc_list)) 1414 clear_bit(isc_to_irq_type(isc), &fi->pending_irqs); 1415 spin_unlock(&fi->lock); 1416 return iter; 1417 } 1418 spin_unlock(&fi->lock); 1419 return NULL; 1420 } 1421 1422 /* 1423 * Dequeue and return an I/O interrupt matching any of the interruption 1424 * subclasses as designated by the isc mask in cr6 and the schid (if != 0). 1425 */ 1426 struct kvm_s390_interrupt_info *kvm_s390_get_io_int(struct kvm *kvm, 1427 u64 isc_mask, u32 schid) 1428 { 1429 struct kvm_s390_interrupt_info *inti = NULL; 1430 int isc; 1431 1432 for (isc = 0; isc <= MAX_ISC && !inti; isc++) { 1433 if (isc_mask & isc_to_isc_bits(isc)) 1434 inti = get_io_int(kvm, isc, schid); 1435 } 1436 return inti; 1437 } 1438 1439 #define SCCB_MASK 0xFFFFFFF8 1440 #define SCCB_EVENT_PENDING 0x3 1441 1442 static int __inject_service(struct kvm *kvm, 1443 struct kvm_s390_interrupt_info *inti) 1444 { 1445 struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int; 1446 1447 spin_lock(&fi->lock); 1448 fi->srv_signal.ext_params |= inti->ext.ext_params & SCCB_EVENT_PENDING; 1449 /* 1450 * Early versions of the QEMU s390 bios will inject several 1451 * service interrupts after another without handling a 1452 * condition code indicating busy. 1453 * We will silently ignore those superfluous sccb values. 1454 * A future version of QEMU will take care of serialization 1455 * of servc requests 1456 */ 1457 if (fi->srv_signal.ext_params & SCCB_MASK) 1458 goto out; 1459 fi->srv_signal.ext_params |= inti->ext.ext_params & SCCB_MASK; 1460 set_bit(IRQ_PEND_EXT_SERVICE, &fi->pending_irqs); 1461 out: 1462 spin_unlock(&fi->lock); 1463 kfree(inti); 1464 return 0; 1465 } 1466 1467 static int __inject_virtio(struct kvm *kvm, 1468 struct kvm_s390_interrupt_info *inti) 1469 { 1470 struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int; 1471 1472 spin_lock(&fi->lock); 1473 if (fi->counters[FIRQ_CNTR_VIRTIO] >= KVM_S390_MAX_VIRTIO_IRQS) { 1474 spin_unlock(&fi->lock); 1475 return -EBUSY; 1476 } 1477 fi->counters[FIRQ_CNTR_VIRTIO] += 1; 1478 list_add_tail(&inti->list, &fi->lists[FIRQ_LIST_VIRTIO]); 1479 set_bit(IRQ_PEND_VIRTIO, &fi->pending_irqs); 1480 spin_unlock(&fi->lock); 1481 return 0; 1482 } 1483 1484 static int __inject_pfault_done(struct kvm *kvm, 1485 struct kvm_s390_interrupt_info *inti) 1486 { 1487 struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int; 1488 1489 spin_lock(&fi->lock); 1490 if (fi->counters[FIRQ_CNTR_PFAULT] >= 1491 (ASYNC_PF_PER_VCPU * KVM_MAX_VCPUS)) { 1492 spin_unlock(&fi->lock); 1493 return -EBUSY; 1494 } 1495 fi->counters[FIRQ_CNTR_PFAULT] += 1; 1496 list_add_tail(&inti->list, &fi->lists[FIRQ_LIST_PFAULT]); 1497 set_bit(IRQ_PEND_PFAULT_DONE, &fi->pending_irqs); 1498 spin_unlock(&fi->lock); 1499 return 0; 1500 } 1501 1502 #define CR_PENDING_SUBCLASS 28 1503 static int __inject_float_mchk(struct kvm *kvm, 1504 struct kvm_s390_interrupt_info *inti) 1505 { 1506 struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int; 1507 1508 spin_lock(&fi->lock); 1509 fi->mchk.cr14 |= inti->mchk.cr14 & (1UL << CR_PENDING_SUBCLASS); 1510 fi->mchk.mcic |= inti->mchk.mcic; 1511 set_bit(IRQ_PEND_MCHK_REP, &fi->pending_irqs); 1512 spin_unlock(&fi->lock); 1513 kfree(inti); 1514 return 0; 1515 } 1516 1517 static int __inject_io(struct kvm *kvm, struct kvm_s390_interrupt_info *inti) 1518 { 1519 struct kvm_s390_float_interrupt *fi; 1520 struct list_head *list; 1521 int isc; 1522 1523 fi = &kvm->arch.float_int; 1524 spin_lock(&fi->lock); 1525 if (fi->counters[FIRQ_CNTR_IO] >= KVM_S390_MAX_FLOAT_IRQS) { 1526 spin_unlock(&fi->lock); 1527 return -EBUSY; 1528 } 1529 fi->counters[FIRQ_CNTR_IO] += 1; 1530 1531 if (inti->type & KVM_S390_INT_IO_AI_MASK) 1532 VM_EVENT(kvm, 4, "%s", "inject: I/O (AI)"); 1533 else 1534 VM_EVENT(kvm, 4, "inject: I/O %x ss %x schid %04x", 1535 inti->io.subchannel_id >> 8, 1536 inti->io.subchannel_id >> 1 & 0x3, 1537 inti->io.subchannel_nr); 1538 isc = int_word_to_isc(inti->io.io_int_word); 1539 list = &fi->lists[FIRQ_LIST_IO_ISC_0 + isc]; 1540 list_add_tail(&inti->list, list); 1541 set_bit(isc_to_irq_type(isc), &fi->pending_irqs); 1542 spin_unlock(&fi->lock); 1543 return 0; 1544 } 1545 1546 /* 1547 * Find a destination VCPU for a floating irq and kick it. 1548 */ 1549 static void __floating_irq_kick(struct kvm *kvm, u64 type) 1550 { 1551 struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int; 1552 struct kvm_s390_local_interrupt *li; 1553 struct kvm_vcpu *dst_vcpu; 1554 int sigcpu, online_vcpus, nr_tries = 0; 1555 1556 online_vcpus = atomic_read(&kvm->online_vcpus); 1557 if (!online_vcpus) 1558 return; 1559 1560 /* find idle VCPUs first, then round robin */ 1561 sigcpu = find_first_bit(fi->idle_mask, online_vcpus); 1562 if (sigcpu == online_vcpus) { 1563 do { 1564 sigcpu = fi->next_rr_cpu; 1565 fi->next_rr_cpu = (fi->next_rr_cpu + 1) % online_vcpus; 1566 /* avoid endless loops if all vcpus are stopped */ 1567 if (nr_tries++ >= online_vcpus) 1568 return; 1569 } while (is_vcpu_stopped(kvm_get_vcpu(kvm, sigcpu))); 1570 } 1571 dst_vcpu = kvm_get_vcpu(kvm, sigcpu); 1572 1573 /* make the VCPU drop out of the SIE, or wake it up if sleeping */ 1574 li = &dst_vcpu->arch.local_int; 1575 spin_lock(&li->lock); 1576 switch (type) { 1577 case KVM_S390_MCHK: 1578 atomic_or(CPUSTAT_STOP_INT, li->cpuflags); 1579 break; 1580 case KVM_S390_INT_IO_MIN...KVM_S390_INT_IO_MAX: 1581 atomic_or(CPUSTAT_IO_INT, li->cpuflags); 1582 break; 1583 default: 1584 atomic_or(CPUSTAT_EXT_INT, li->cpuflags); 1585 break; 1586 } 1587 spin_unlock(&li->lock); 1588 kvm_s390_vcpu_wakeup(dst_vcpu); 1589 } 1590 1591 static int __inject_vm(struct kvm *kvm, struct kvm_s390_interrupt_info *inti) 1592 { 1593 u64 type = READ_ONCE(inti->type); 1594 int rc; 1595 1596 switch (type) { 1597 case KVM_S390_MCHK: 1598 rc = __inject_float_mchk(kvm, inti); 1599 break; 1600 case KVM_S390_INT_VIRTIO: 1601 rc = __inject_virtio(kvm, inti); 1602 break; 1603 case KVM_S390_INT_SERVICE: 1604 rc = __inject_service(kvm, inti); 1605 break; 1606 case KVM_S390_INT_PFAULT_DONE: 1607 rc = __inject_pfault_done(kvm, inti); 1608 break; 1609 case KVM_S390_INT_IO_MIN...KVM_S390_INT_IO_MAX: 1610 rc = __inject_io(kvm, inti); 1611 break; 1612 default: 1613 rc = -EINVAL; 1614 } 1615 if (rc) 1616 return rc; 1617 1618 __floating_irq_kick(kvm, type); 1619 return 0; 1620 } 1621 1622 int kvm_s390_inject_vm(struct kvm *kvm, 1623 struct kvm_s390_interrupt *s390int) 1624 { 1625 struct kvm_s390_interrupt_info *inti; 1626 int rc; 1627 1628 inti = kzalloc(sizeof(*inti), GFP_KERNEL); 1629 if (!inti) 1630 return -ENOMEM; 1631 1632 inti->type = s390int->type; 1633 switch (inti->type) { 1634 case KVM_S390_INT_VIRTIO: 1635 VM_EVENT(kvm, 5, "inject: virtio parm:%x,parm64:%llx", 1636 s390int->parm, s390int->parm64); 1637 inti->ext.ext_params = s390int->parm; 1638 inti->ext.ext_params2 = s390int->parm64; 1639 break; 1640 case KVM_S390_INT_SERVICE: 1641 VM_EVENT(kvm, 4, "inject: sclp parm:%x", s390int->parm); 1642 inti->ext.ext_params = s390int->parm; 1643 break; 1644 case KVM_S390_INT_PFAULT_DONE: 1645 inti->ext.ext_params2 = s390int->parm64; 1646 break; 1647 case KVM_S390_MCHK: 1648 VM_EVENT(kvm, 3, "inject: machine check mcic 0x%llx", 1649 s390int->parm64); 1650 inti->mchk.cr14 = s390int->parm; /* upper bits are not used */ 1651 inti->mchk.mcic = s390int->parm64; 1652 break; 1653 case KVM_S390_INT_IO_MIN...KVM_S390_INT_IO_MAX: 1654 inti->io.subchannel_id = s390int->parm >> 16; 1655 inti->io.subchannel_nr = s390int->parm & 0x0000ffffu; 1656 inti->io.io_int_parm = s390int->parm64 >> 32; 1657 inti->io.io_int_word = s390int->parm64 & 0x00000000ffffffffull; 1658 break; 1659 default: 1660 kfree(inti); 1661 return -EINVAL; 1662 } 1663 trace_kvm_s390_inject_vm(s390int->type, s390int->parm, s390int->parm64, 1664 2); 1665 1666 rc = __inject_vm(kvm, inti); 1667 if (rc) 1668 kfree(inti); 1669 return rc; 1670 } 1671 1672 int kvm_s390_reinject_io_int(struct kvm *kvm, 1673 struct kvm_s390_interrupt_info *inti) 1674 { 1675 return __inject_vm(kvm, inti); 1676 } 1677 1678 int s390int_to_s390irq(struct kvm_s390_interrupt *s390int, 1679 struct kvm_s390_irq *irq) 1680 { 1681 irq->type = s390int->type; 1682 switch (irq->type) { 1683 case KVM_S390_PROGRAM_INT: 1684 if (s390int->parm & 0xffff0000) 1685 return -EINVAL; 1686 irq->u.pgm.code = s390int->parm; 1687 break; 1688 case KVM_S390_SIGP_SET_PREFIX: 1689 irq->u.prefix.address = s390int->parm; 1690 break; 1691 case KVM_S390_SIGP_STOP: 1692 irq->u.stop.flags = s390int->parm; 1693 break; 1694 case KVM_S390_INT_EXTERNAL_CALL: 1695 if (s390int->parm & 0xffff0000) 1696 return -EINVAL; 1697 irq->u.extcall.code = s390int->parm; 1698 break; 1699 case KVM_S390_INT_EMERGENCY: 1700 if (s390int->parm & 0xffff0000) 1701 return -EINVAL; 1702 irq->u.emerg.code = s390int->parm; 1703 break; 1704 case KVM_S390_MCHK: 1705 irq->u.mchk.mcic = s390int->parm64; 1706 break; 1707 } 1708 return 0; 1709 } 1710 1711 int kvm_s390_is_stop_irq_pending(struct kvm_vcpu *vcpu) 1712 { 1713 struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int; 1714 1715 return test_bit(IRQ_PEND_SIGP_STOP, &li->pending_irqs); 1716 } 1717 1718 void kvm_s390_clear_stop_irq(struct kvm_vcpu *vcpu) 1719 { 1720 struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int; 1721 1722 spin_lock(&li->lock); 1723 li->irq.stop.flags = 0; 1724 clear_bit(IRQ_PEND_SIGP_STOP, &li->pending_irqs); 1725 spin_unlock(&li->lock); 1726 } 1727 1728 static int do_inject_vcpu(struct kvm_vcpu *vcpu, struct kvm_s390_irq *irq) 1729 { 1730 int rc; 1731 1732 switch (irq->type) { 1733 case KVM_S390_PROGRAM_INT: 1734 rc = __inject_prog(vcpu, irq); 1735 break; 1736 case KVM_S390_SIGP_SET_PREFIX: 1737 rc = __inject_set_prefix(vcpu, irq); 1738 break; 1739 case KVM_S390_SIGP_STOP: 1740 rc = __inject_sigp_stop(vcpu, irq); 1741 break; 1742 case KVM_S390_RESTART: 1743 rc = __inject_sigp_restart(vcpu, irq); 1744 break; 1745 case KVM_S390_INT_CLOCK_COMP: 1746 rc = __inject_ckc(vcpu); 1747 break; 1748 case KVM_S390_INT_CPU_TIMER: 1749 rc = __inject_cpu_timer(vcpu); 1750 break; 1751 case KVM_S390_INT_EXTERNAL_CALL: 1752 rc = __inject_extcall(vcpu, irq); 1753 break; 1754 case KVM_S390_INT_EMERGENCY: 1755 rc = __inject_sigp_emergency(vcpu, irq); 1756 break; 1757 case KVM_S390_MCHK: 1758 rc = __inject_mchk(vcpu, irq); 1759 break; 1760 case KVM_S390_INT_PFAULT_INIT: 1761 rc = __inject_pfault_init(vcpu, irq); 1762 break; 1763 case KVM_S390_INT_VIRTIO: 1764 case KVM_S390_INT_SERVICE: 1765 case KVM_S390_INT_IO_MIN...KVM_S390_INT_IO_MAX: 1766 default: 1767 rc = -EINVAL; 1768 } 1769 1770 return rc; 1771 } 1772 1773 int kvm_s390_inject_vcpu(struct kvm_vcpu *vcpu, struct kvm_s390_irq *irq) 1774 { 1775 struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int; 1776 int rc; 1777 1778 spin_lock(&li->lock); 1779 rc = do_inject_vcpu(vcpu, irq); 1780 spin_unlock(&li->lock); 1781 if (!rc) 1782 kvm_s390_vcpu_wakeup(vcpu); 1783 return rc; 1784 } 1785 1786 static inline void clear_irq_list(struct list_head *_list) 1787 { 1788 struct kvm_s390_interrupt_info *inti, *n; 1789 1790 list_for_each_entry_safe(inti, n, _list, list) { 1791 list_del(&inti->list); 1792 kfree(inti); 1793 } 1794 } 1795 1796 static void inti_to_irq(struct kvm_s390_interrupt_info *inti, 1797 struct kvm_s390_irq *irq) 1798 { 1799 irq->type = inti->type; 1800 switch (inti->type) { 1801 case KVM_S390_INT_PFAULT_INIT: 1802 case KVM_S390_INT_PFAULT_DONE: 1803 case KVM_S390_INT_VIRTIO: 1804 irq->u.ext = inti->ext; 1805 break; 1806 case KVM_S390_INT_IO_MIN...KVM_S390_INT_IO_MAX: 1807 irq->u.io = inti->io; 1808 break; 1809 } 1810 } 1811 1812 void kvm_s390_clear_float_irqs(struct kvm *kvm) 1813 { 1814 struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int; 1815 int i; 1816 1817 spin_lock(&fi->lock); 1818 fi->pending_irqs = 0; 1819 memset(&fi->srv_signal, 0, sizeof(fi->srv_signal)); 1820 memset(&fi->mchk, 0, sizeof(fi->mchk)); 1821 for (i = 0; i < FIRQ_LIST_COUNT; i++) 1822 clear_irq_list(&fi->lists[i]); 1823 for (i = 0; i < FIRQ_MAX_COUNT; i++) 1824 fi->counters[i] = 0; 1825 spin_unlock(&fi->lock); 1826 }; 1827 1828 static int get_all_floating_irqs(struct kvm *kvm, u8 __user *usrbuf, u64 len) 1829 { 1830 struct kvm_s390_interrupt_info *inti; 1831 struct kvm_s390_float_interrupt *fi; 1832 struct kvm_s390_irq *buf; 1833 struct kvm_s390_irq *irq; 1834 int max_irqs; 1835 int ret = 0; 1836 int n = 0; 1837 int i; 1838 1839 if (len > KVM_S390_FLIC_MAX_BUFFER || len == 0) 1840 return -EINVAL; 1841 1842 /* 1843 * We are already using -ENOMEM to signal 1844 * userspace it may retry with a bigger buffer, 1845 * so we need to use something else for this case 1846 */ 1847 buf = vzalloc(len); 1848 if (!buf) 1849 return -ENOBUFS; 1850 1851 max_irqs = len / sizeof(struct kvm_s390_irq); 1852 1853 fi = &kvm->arch.float_int; 1854 spin_lock(&fi->lock); 1855 for (i = 0; i < FIRQ_LIST_COUNT; i++) { 1856 list_for_each_entry(inti, &fi->lists[i], list) { 1857 if (n == max_irqs) { 1858 /* signal userspace to try again */ 1859 ret = -ENOMEM; 1860 goto out; 1861 } 1862 inti_to_irq(inti, &buf[n]); 1863 n++; 1864 } 1865 } 1866 if (test_bit(IRQ_PEND_EXT_SERVICE, &fi->pending_irqs)) { 1867 if (n == max_irqs) { 1868 /* signal userspace to try again */ 1869 ret = -ENOMEM; 1870 goto out; 1871 } 1872 irq = (struct kvm_s390_irq *) &buf[n]; 1873 irq->type = KVM_S390_INT_SERVICE; 1874 irq->u.ext = fi->srv_signal; 1875 n++; 1876 } 1877 if (test_bit(IRQ_PEND_MCHK_REP, &fi->pending_irqs)) { 1878 if (n == max_irqs) { 1879 /* signal userspace to try again */ 1880 ret = -ENOMEM; 1881 goto out; 1882 } 1883 irq = (struct kvm_s390_irq *) &buf[n]; 1884 irq->type = KVM_S390_MCHK; 1885 irq->u.mchk = fi->mchk; 1886 n++; 1887 } 1888 1889 out: 1890 spin_unlock(&fi->lock); 1891 if (!ret && n > 0) { 1892 if (copy_to_user(usrbuf, buf, sizeof(struct kvm_s390_irq) * n)) 1893 ret = -EFAULT; 1894 } 1895 vfree(buf); 1896 1897 return ret < 0 ? ret : n; 1898 } 1899 1900 static int flic_ais_mode_get_all(struct kvm *kvm, struct kvm_device_attr *attr) 1901 { 1902 struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int; 1903 struct kvm_s390_ais_all ais; 1904 1905 if (attr->attr < sizeof(ais)) 1906 return -EINVAL; 1907 1908 if (!test_kvm_facility(kvm, 72)) 1909 return -ENOTSUPP; 1910 1911 mutex_lock(&fi->ais_lock); 1912 ais.simm = fi->simm; 1913 ais.nimm = fi->nimm; 1914 mutex_unlock(&fi->ais_lock); 1915 1916 if (copy_to_user((void __user *)attr->addr, &ais, sizeof(ais))) 1917 return -EFAULT; 1918 1919 return 0; 1920 } 1921 1922 static int flic_get_attr(struct kvm_device *dev, struct kvm_device_attr *attr) 1923 { 1924 int r; 1925 1926 switch (attr->group) { 1927 case KVM_DEV_FLIC_GET_ALL_IRQS: 1928 r = get_all_floating_irqs(dev->kvm, (u8 __user *) attr->addr, 1929 attr->attr); 1930 break; 1931 case KVM_DEV_FLIC_AISM_ALL: 1932 r = flic_ais_mode_get_all(dev->kvm, attr); 1933 break; 1934 default: 1935 r = -EINVAL; 1936 } 1937 1938 return r; 1939 } 1940 1941 static inline int copy_irq_from_user(struct kvm_s390_interrupt_info *inti, 1942 u64 addr) 1943 { 1944 struct kvm_s390_irq __user *uptr = (struct kvm_s390_irq __user *) addr; 1945 void *target = NULL; 1946 void __user *source; 1947 u64 size; 1948 1949 if (get_user(inti->type, (u64 __user *)addr)) 1950 return -EFAULT; 1951 1952 switch (inti->type) { 1953 case KVM_S390_INT_PFAULT_INIT: 1954 case KVM_S390_INT_PFAULT_DONE: 1955 case KVM_S390_INT_VIRTIO: 1956 case KVM_S390_INT_SERVICE: 1957 target = (void *) &inti->ext; 1958 source = &uptr->u.ext; 1959 size = sizeof(inti->ext); 1960 break; 1961 case KVM_S390_INT_IO_MIN...KVM_S390_INT_IO_MAX: 1962 target = (void *) &inti->io; 1963 source = &uptr->u.io; 1964 size = sizeof(inti->io); 1965 break; 1966 case KVM_S390_MCHK: 1967 target = (void *) &inti->mchk; 1968 source = &uptr->u.mchk; 1969 size = sizeof(inti->mchk); 1970 break; 1971 default: 1972 return -EINVAL; 1973 } 1974 1975 if (copy_from_user(target, source, size)) 1976 return -EFAULT; 1977 1978 return 0; 1979 } 1980 1981 static int enqueue_floating_irq(struct kvm_device *dev, 1982 struct kvm_device_attr *attr) 1983 { 1984 struct kvm_s390_interrupt_info *inti = NULL; 1985 int r = 0; 1986 int len = attr->attr; 1987 1988 if (len % sizeof(struct kvm_s390_irq) != 0) 1989 return -EINVAL; 1990 else if (len > KVM_S390_FLIC_MAX_BUFFER) 1991 return -EINVAL; 1992 1993 while (len >= sizeof(struct kvm_s390_irq)) { 1994 inti = kzalloc(sizeof(*inti), GFP_KERNEL); 1995 if (!inti) 1996 return -ENOMEM; 1997 1998 r = copy_irq_from_user(inti, attr->addr); 1999 if (r) { 2000 kfree(inti); 2001 return r; 2002 } 2003 r = __inject_vm(dev->kvm, inti); 2004 if (r) { 2005 kfree(inti); 2006 return r; 2007 } 2008 len -= sizeof(struct kvm_s390_irq); 2009 attr->addr += sizeof(struct kvm_s390_irq); 2010 } 2011 2012 return r; 2013 } 2014 2015 static struct s390_io_adapter *get_io_adapter(struct kvm *kvm, unsigned int id) 2016 { 2017 if (id >= MAX_S390_IO_ADAPTERS) 2018 return NULL; 2019 return kvm->arch.adapters[id]; 2020 } 2021 2022 static int register_io_adapter(struct kvm_device *dev, 2023 struct kvm_device_attr *attr) 2024 { 2025 struct s390_io_adapter *adapter; 2026 struct kvm_s390_io_adapter adapter_info; 2027 2028 if (copy_from_user(&adapter_info, 2029 (void __user *)attr->addr, sizeof(adapter_info))) 2030 return -EFAULT; 2031 2032 if ((adapter_info.id >= MAX_S390_IO_ADAPTERS) || 2033 (dev->kvm->arch.adapters[adapter_info.id] != NULL)) 2034 return -EINVAL; 2035 2036 adapter = kzalloc(sizeof(*adapter), GFP_KERNEL); 2037 if (!adapter) 2038 return -ENOMEM; 2039 2040 INIT_LIST_HEAD(&adapter->maps); 2041 init_rwsem(&adapter->maps_lock); 2042 atomic_set(&adapter->nr_maps, 0); 2043 adapter->id = adapter_info.id; 2044 adapter->isc = adapter_info.isc; 2045 adapter->maskable = adapter_info.maskable; 2046 adapter->masked = false; 2047 adapter->swap = adapter_info.swap; 2048 adapter->suppressible = (adapter_info.flags) & 2049 KVM_S390_ADAPTER_SUPPRESSIBLE; 2050 dev->kvm->arch.adapters[adapter->id] = adapter; 2051 2052 return 0; 2053 } 2054 2055 int kvm_s390_mask_adapter(struct kvm *kvm, unsigned int id, bool masked) 2056 { 2057 int ret; 2058 struct s390_io_adapter *adapter = get_io_adapter(kvm, id); 2059 2060 if (!adapter || !adapter->maskable) 2061 return -EINVAL; 2062 ret = adapter->masked; 2063 adapter->masked = masked; 2064 return ret; 2065 } 2066 2067 static int kvm_s390_adapter_map(struct kvm *kvm, unsigned int id, __u64 addr) 2068 { 2069 struct s390_io_adapter *adapter = get_io_adapter(kvm, id); 2070 struct s390_map_info *map; 2071 int ret; 2072 2073 if (!adapter || !addr) 2074 return -EINVAL; 2075 2076 map = kzalloc(sizeof(*map), GFP_KERNEL); 2077 if (!map) { 2078 ret = -ENOMEM; 2079 goto out; 2080 } 2081 INIT_LIST_HEAD(&map->list); 2082 map->guest_addr = addr; 2083 map->addr = gmap_translate(kvm->arch.gmap, addr); 2084 if (map->addr == -EFAULT) { 2085 ret = -EFAULT; 2086 goto out; 2087 } 2088 ret = get_user_pages_fast(map->addr, 1, 1, &map->page); 2089 if (ret < 0) 2090 goto out; 2091 BUG_ON(ret != 1); 2092 down_write(&adapter->maps_lock); 2093 if (atomic_inc_return(&adapter->nr_maps) < MAX_S390_ADAPTER_MAPS) { 2094 list_add_tail(&map->list, &adapter->maps); 2095 ret = 0; 2096 } else { 2097 put_page(map->page); 2098 ret = -EINVAL; 2099 } 2100 up_write(&adapter->maps_lock); 2101 out: 2102 if (ret) 2103 kfree(map); 2104 return ret; 2105 } 2106 2107 static int kvm_s390_adapter_unmap(struct kvm *kvm, unsigned int id, __u64 addr) 2108 { 2109 struct s390_io_adapter *adapter = get_io_adapter(kvm, id); 2110 struct s390_map_info *map, *tmp; 2111 int found = 0; 2112 2113 if (!adapter || !addr) 2114 return -EINVAL; 2115 2116 down_write(&adapter->maps_lock); 2117 list_for_each_entry_safe(map, tmp, &adapter->maps, list) { 2118 if (map->guest_addr == addr) { 2119 found = 1; 2120 atomic_dec(&adapter->nr_maps); 2121 list_del(&map->list); 2122 put_page(map->page); 2123 kfree(map); 2124 break; 2125 } 2126 } 2127 up_write(&adapter->maps_lock); 2128 2129 return found ? 0 : -EINVAL; 2130 } 2131 2132 void kvm_s390_destroy_adapters(struct kvm *kvm) 2133 { 2134 int i; 2135 struct s390_map_info *map, *tmp; 2136 2137 for (i = 0; i < MAX_S390_IO_ADAPTERS; i++) { 2138 if (!kvm->arch.adapters[i]) 2139 continue; 2140 list_for_each_entry_safe(map, tmp, 2141 &kvm->arch.adapters[i]->maps, list) { 2142 list_del(&map->list); 2143 put_page(map->page); 2144 kfree(map); 2145 } 2146 kfree(kvm->arch.adapters[i]); 2147 } 2148 } 2149 2150 static int modify_io_adapter(struct kvm_device *dev, 2151 struct kvm_device_attr *attr) 2152 { 2153 struct kvm_s390_io_adapter_req req; 2154 struct s390_io_adapter *adapter; 2155 int ret; 2156 2157 if (copy_from_user(&req, (void __user *)attr->addr, sizeof(req))) 2158 return -EFAULT; 2159 2160 adapter = get_io_adapter(dev->kvm, req.id); 2161 if (!adapter) 2162 return -EINVAL; 2163 switch (req.type) { 2164 case KVM_S390_IO_ADAPTER_MASK: 2165 ret = kvm_s390_mask_adapter(dev->kvm, req.id, req.mask); 2166 if (ret > 0) 2167 ret = 0; 2168 break; 2169 case KVM_S390_IO_ADAPTER_MAP: 2170 ret = kvm_s390_adapter_map(dev->kvm, req.id, req.addr); 2171 break; 2172 case KVM_S390_IO_ADAPTER_UNMAP: 2173 ret = kvm_s390_adapter_unmap(dev->kvm, req.id, req.addr); 2174 break; 2175 default: 2176 ret = -EINVAL; 2177 } 2178 2179 return ret; 2180 } 2181 2182 static int clear_io_irq(struct kvm *kvm, struct kvm_device_attr *attr) 2183 2184 { 2185 const u64 isc_mask = 0xffUL << 24; /* all iscs set */ 2186 u32 schid; 2187 2188 if (attr->flags) 2189 return -EINVAL; 2190 if (attr->attr != sizeof(schid)) 2191 return -EINVAL; 2192 if (copy_from_user(&schid, (void __user *) attr->addr, sizeof(schid))) 2193 return -EFAULT; 2194 if (!schid) 2195 return -EINVAL; 2196 kfree(kvm_s390_get_io_int(kvm, isc_mask, schid)); 2197 /* 2198 * If userspace is conforming to the architecture, we can have at most 2199 * one pending I/O interrupt per subchannel, so this is effectively a 2200 * clear all. 2201 */ 2202 return 0; 2203 } 2204 2205 static int modify_ais_mode(struct kvm *kvm, struct kvm_device_attr *attr) 2206 { 2207 struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int; 2208 struct kvm_s390_ais_req req; 2209 int ret = 0; 2210 2211 if (!test_kvm_facility(kvm, 72)) 2212 return -ENOTSUPP; 2213 2214 if (copy_from_user(&req, (void __user *)attr->addr, sizeof(req))) 2215 return -EFAULT; 2216 2217 if (req.isc > MAX_ISC) 2218 return -EINVAL; 2219 2220 trace_kvm_s390_modify_ais_mode(req.isc, 2221 (fi->simm & AIS_MODE_MASK(req.isc)) ? 2222 (fi->nimm & AIS_MODE_MASK(req.isc)) ? 2223 2 : KVM_S390_AIS_MODE_SINGLE : 2224 KVM_S390_AIS_MODE_ALL, req.mode); 2225 2226 mutex_lock(&fi->ais_lock); 2227 switch (req.mode) { 2228 case KVM_S390_AIS_MODE_ALL: 2229 fi->simm &= ~AIS_MODE_MASK(req.isc); 2230 fi->nimm &= ~AIS_MODE_MASK(req.isc); 2231 break; 2232 case KVM_S390_AIS_MODE_SINGLE: 2233 fi->simm |= AIS_MODE_MASK(req.isc); 2234 fi->nimm &= ~AIS_MODE_MASK(req.isc); 2235 break; 2236 default: 2237 ret = -EINVAL; 2238 } 2239 mutex_unlock(&fi->ais_lock); 2240 2241 return ret; 2242 } 2243 2244 static int kvm_s390_inject_airq(struct kvm *kvm, 2245 struct s390_io_adapter *adapter) 2246 { 2247 struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int; 2248 struct kvm_s390_interrupt s390int = { 2249 .type = KVM_S390_INT_IO(1, 0, 0, 0), 2250 .parm = 0, 2251 .parm64 = (adapter->isc << 27) | 0x80000000, 2252 }; 2253 int ret = 0; 2254 2255 if (!test_kvm_facility(kvm, 72) || !adapter->suppressible) 2256 return kvm_s390_inject_vm(kvm, &s390int); 2257 2258 mutex_lock(&fi->ais_lock); 2259 if (fi->nimm & AIS_MODE_MASK(adapter->isc)) { 2260 trace_kvm_s390_airq_suppressed(adapter->id, adapter->isc); 2261 goto out; 2262 } 2263 2264 ret = kvm_s390_inject_vm(kvm, &s390int); 2265 if (!ret && (fi->simm & AIS_MODE_MASK(adapter->isc))) { 2266 fi->nimm |= AIS_MODE_MASK(adapter->isc); 2267 trace_kvm_s390_modify_ais_mode(adapter->isc, 2268 KVM_S390_AIS_MODE_SINGLE, 2); 2269 } 2270 out: 2271 mutex_unlock(&fi->ais_lock); 2272 return ret; 2273 } 2274 2275 static int flic_inject_airq(struct kvm *kvm, struct kvm_device_attr *attr) 2276 { 2277 unsigned int id = attr->attr; 2278 struct s390_io_adapter *adapter = get_io_adapter(kvm, id); 2279 2280 if (!adapter) 2281 return -EINVAL; 2282 2283 return kvm_s390_inject_airq(kvm, adapter); 2284 } 2285 2286 static int flic_ais_mode_set_all(struct kvm *kvm, struct kvm_device_attr *attr) 2287 { 2288 struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int; 2289 struct kvm_s390_ais_all ais; 2290 2291 if (!test_kvm_facility(kvm, 72)) 2292 return -ENOTSUPP; 2293 2294 if (copy_from_user(&ais, (void __user *)attr->addr, sizeof(ais))) 2295 return -EFAULT; 2296 2297 mutex_lock(&fi->ais_lock); 2298 fi->simm = ais.simm; 2299 fi->nimm = ais.nimm; 2300 mutex_unlock(&fi->ais_lock); 2301 2302 return 0; 2303 } 2304 2305 static int flic_set_attr(struct kvm_device *dev, struct kvm_device_attr *attr) 2306 { 2307 int r = 0; 2308 unsigned int i; 2309 struct kvm_vcpu *vcpu; 2310 2311 switch (attr->group) { 2312 case KVM_DEV_FLIC_ENQUEUE: 2313 r = enqueue_floating_irq(dev, attr); 2314 break; 2315 case KVM_DEV_FLIC_CLEAR_IRQS: 2316 kvm_s390_clear_float_irqs(dev->kvm); 2317 break; 2318 case KVM_DEV_FLIC_APF_ENABLE: 2319 dev->kvm->arch.gmap->pfault_enabled = 1; 2320 break; 2321 case KVM_DEV_FLIC_APF_DISABLE_WAIT: 2322 dev->kvm->arch.gmap->pfault_enabled = 0; 2323 /* 2324 * Make sure no async faults are in transition when 2325 * clearing the queues. So we don't need to worry 2326 * about late coming workers. 2327 */ 2328 synchronize_srcu(&dev->kvm->srcu); 2329 kvm_for_each_vcpu(i, vcpu, dev->kvm) 2330 kvm_clear_async_pf_completion_queue(vcpu); 2331 break; 2332 case KVM_DEV_FLIC_ADAPTER_REGISTER: 2333 r = register_io_adapter(dev, attr); 2334 break; 2335 case KVM_DEV_FLIC_ADAPTER_MODIFY: 2336 r = modify_io_adapter(dev, attr); 2337 break; 2338 case KVM_DEV_FLIC_CLEAR_IO_IRQ: 2339 r = clear_io_irq(dev->kvm, attr); 2340 break; 2341 case KVM_DEV_FLIC_AISM: 2342 r = modify_ais_mode(dev->kvm, attr); 2343 break; 2344 case KVM_DEV_FLIC_AIRQ_INJECT: 2345 r = flic_inject_airq(dev->kvm, attr); 2346 break; 2347 case KVM_DEV_FLIC_AISM_ALL: 2348 r = flic_ais_mode_set_all(dev->kvm, attr); 2349 break; 2350 default: 2351 r = -EINVAL; 2352 } 2353 2354 return r; 2355 } 2356 2357 static int flic_has_attr(struct kvm_device *dev, 2358 struct kvm_device_attr *attr) 2359 { 2360 switch (attr->group) { 2361 case KVM_DEV_FLIC_GET_ALL_IRQS: 2362 case KVM_DEV_FLIC_ENQUEUE: 2363 case KVM_DEV_FLIC_CLEAR_IRQS: 2364 case KVM_DEV_FLIC_APF_ENABLE: 2365 case KVM_DEV_FLIC_APF_DISABLE_WAIT: 2366 case KVM_DEV_FLIC_ADAPTER_REGISTER: 2367 case KVM_DEV_FLIC_ADAPTER_MODIFY: 2368 case KVM_DEV_FLIC_CLEAR_IO_IRQ: 2369 case KVM_DEV_FLIC_AISM: 2370 case KVM_DEV_FLIC_AIRQ_INJECT: 2371 case KVM_DEV_FLIC_AISM_ALL: 2372 return 0; 2373 } 2374 return -ENXIO; 2375 } 2376 2377 static int flic_create(struct kvm_device *dev, u32 type) 2378 { 2379 if (!dev) 2380 return -EINVAL; 2381 if (dev->kvm->arch.flic) 2382 return -EINVAL; 2383 dev->kvm->arch.flic = dev; 2384 return 0; 2385 } 2386 2387 static void flic_destroy(struct kvm_device *dev) 2388 { 2389 dev->kvm->arch.flic = NULL; 2390 kfree(dev); 2391 } 2392 2393 /* s390 floating irq controller (flic) */ 2394 struct kvm_device_ops kvm_flic_ops = { 2395 .name = "kvm-flic", 2396 .get_attr = flic_get_attr, 2397 .set_attr = flic_set_attr, 2398 .has_attr = flic_has_attr, 2399 .create = flic_create, 2400 .destroy = flic_destroy, 2401 }; 2402 2403 static unsigned long get_ind_bit(__u64 addr, unsigned long bit_nr, bool swap) 2404 { 2405 unsigned long bit; 2406 2407 bit = bit_nr + (addr % PAGE_SIZE) * 8; 2408 2409 return swap ? (bit ^ (BITS_PER_LONG - 1)) : bit; 2410 } 2411 2412 static struct s390_map_info *get_map_info(struct s390_io_adapter *adapter, 2413 u64 addr) 2414 { 2415 struct s390_map_info *map; 2416 2417 if (!adapter) 2418 return NULL; 2419 2420 list_for_each_entry(map, &adapter->maps, list) { 2421 if (map->guest_addr == addr) 2422 return map; 2423 } 2424 return NULL; 2425 } 2426 2427 static int adapter_indicators_set(struct kvm *kvm, 2428 struct s390_io_adapter *adapter, 2429 struct kvm_s390_adapter_int *adapter_int) 2430 { 2431 unsigned long bit; 2432 int summary_set, idx; 2433 struct s390_map_info *info; 2434 void *map; 2435 2436 info = get_map_info(adapter, adapter_int->ind_addr); 2437 if (!info) 2438 return -1; 2439 map = page_address(info->page); 2440 bit = get_ind_bit(info->addr, adapter_int->ind_offset, adapter->swap); 2441 set_bit(bit, map); 2442 idx = srcu_read_lock(&kvm->srcu); 2443 mark_page_dirty(kvm, info->guest_addr >> PAGE_SHIFT); 2444 set_page_dirty_lock(info->page); 2445 info = get_map_info(adapter, adapter_int->summary_addr); 2446 if (!info) { 2447 srcu_read_unlock(&kvm->srcu, idx); 2448 return -1; 2449 } 2450 map = page_address(info->page); 2451 bit = get_ind_bit(info->addr, adapter_int->summary_offset, 2452 adapter->swap); 2453 summary_set = test_and_set_bit(bit, map); 2454 mark_page_dirty(kvm, info->guest_addr >> PAGE_SHIFT); 2455 set_page_dirty_lock(info->page); 2456 srcu_read_unlock(&kvm->srcu, idx); 2457 return summary_set ? 0 : 1; 2458 } 2459 2460 /* 2461 * < 0 - not injected due to error 2462 * = 0 - coalesced, summary indicator already active 2463 * > 0 - injected interrupt 2464 */ 2465 static int set_adapter_int(struct kvm_kernel_irq_routing_entry *e, 2466 struct kvm *kvm, int irq_source_id, int level, 2467 bool line_status) 2468 { 2469 int ret; 2470 struct s390_io_adapter *adapter; 2471 2472 /* We're only interested in the 0->1 transition. */ 2473 if (!level) 2474 return 0; 2475 adapter = get_io_adapter(kvm, e->adapter.adapter_id); 2476 if (!adapter) 2477 return -1; 2478 down_read(&adapter->maps_lock); 2479 ret = adapter_indicators_set(kvm, adapter, &e->adapter); 2480 up_read(&adapter->maps_lock); 2481 if ((ret > 0) && !adapter->masked) { 2482 ret = kvm_s390_inject_airq(kvm, adapter); 2483 if (ret == 0) 2484 ret = 1; 2485 } 2486 return ret; 2487 } 2488 2489 /* 2490 * Inject the machine check to the guest. 2491 */ 2492 void kvm_s390_reinject_machine_check(struct kvm_vcpu *vcpu, 2493 struct mcck_volatile_info *mcck_info) 2494 { 2495 struct kvm_s390_interrupt_info inti; 2496 struct kvm_s390_irq irq; 2497 struct kvm_s390_mchk_info *mchk; 2498 union mci mci; 2499 __u64 cr14 = 0; /* upper bits are not used */ 2500 int rc; 2501 2502 mci.val = mcck_info->mcic; 2503 if (mci.sr) 2504 cr14 |= CR14_RECOVERY_SUBMASK; 2505 if (mci.dg) 2506 cr14 |= CR14_DEGRADATION_SUBMASK; 2507 if (mci.w) 2508 cr14 |= CR14_WARNING_SUBMASK; 2509 2510 mchk = mci.ck ? &inti.mchk : &irq.u.mchk; 2511 mchk->cr14 = cr14; 2512 mchk->mcic = mcck_info->mcic; 2513 mchk->ext_damage_code = mcck_info->ext_damage_code; 2514 mchk->failing_storage_address = mcck_info->failing_storage_address; 2515 if (mci.ck) { 2516 /* Inject the floating machine check */ 2517 inti.type = KVM_S390_MCHK; 2518 rc = __inject_vm(vcpu->kvm, &inti); 2519 } else { 2520 /* Inject the machine check to specified vcpu */ 2521 irq.type = KVM_S390_MCHK; 2522 rc = kvm_s390_inject_vcpu(vcpu, &irq); 2523 } 2524 WARN_ON_ONCE(rc); 2525 } 2526 2527 int kvm_set_routing_entry(struct kvm *kvm, 2528 struct kvm_kernel_irq_routing_entry *e, 2529 const struct kvm_irq_routing_entry *ue) 2530 { 2531 int ret; 2532 2533 switch (ue->type) { 2534 case KVM_IRQ_ROUTING_S390_ADAPTER: 2535 e->set = set_adapter_int; 2536 e->adapter.summary_addr = ue->u.adapter.summary_addr; 2537 e->adapter.ind_addr = ue->u.adapter.ind_addr; 2538 e->adapter.summary_offset = ue->u.adapter.summary_offset; 2539 e->adapter.ind_offset = ue->u.adapter.ind_offset; 2540 e->adapter.adapter_id = ue->u.adapter.adapter_id; 2541 ret = 0; 2542 break; 2543 default: 2544 ret = -EINVAL; 2545 } 2546 2547 return ret; 2548 } 2549 2550 int kvm_set_msi(struct kvm_kernel_irq_routing_entry *e, struct kvm *kvm, 2551 int irq_source_id, int level, bool line_status) 2552 { 2553 return -EINVAL; 2554 } 2555 2556 int kvm_s390_set_irq_state(struct kvm_vcpu *vcpu, void __user *irqstate, int len) 2557 { 2558 struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int; 2559 struct kvm_s390_irq *buf; 2560 int r = 0; 2561 int n; 2562 2563 buf = vmalloc(len); 2564 if (!buf) 2565 return -ENOMEM; 2566 2567 if (copy_from_user((void *) buf, irqstate, len)) { 2568 r = -EFAULT; 2569 goto out_free; 2570 } 2571 2572 /* 2573 * Don't allow setting the interrupt state 2574 * when there are already interrupts pending 2575 */ 2576 spin_lock(&li->lock); 2577 if (li->pending_irqs) { 2578 r = -EBUSY; 2579 goto out_unlock; 2580 } 2581 2582 for (n = 0; n < len / sizeof(*buf); n++) { 2583 r = do_inject_vcpu(vcpu, &buf[n]); 2584 if (r) 2585 break; 2586 } 2587 2588 out_unlock: 2589 spin_unlock(&li->lock); 2590 out_free: 2591 vfree(buf); 2592 2593 return r; 2594 } 2595 2596 static void store_local_irq(struct kvm_s390_local_interrupt *li, 2597 struct kvm_s390_irq *irq, 2598 unsigned long irq_type) 2599 { 2600 switch (irq_type) { 2601 case IRQ_PEND_MCHK_EX: 2602 case IRQ_PEND_MCHK_REP: 2603 irq->type = KVM_S390_MCHK; 2604 irq->u.mchk = li->irq.mchk; 2605 break; 2606 case IRQ_PEND_PROG: 2607 irq->type = KVM_S390_PROGRAM_INT; 2608 irq->u.pgm = li->irq.pgm; 2609 break; 2610 case IRQ_PEND_PFAULT_INIT: 2611 irq->type = KVM_S390_INT_PFAULT_INIT; 2612 irq->u.ext = li->irq.ext; 2613 break; 2614 case IRQ_PEND_EXT_EXTERNAL: 2615 irq->type = KVM_S390_INT_EXTERNAL_CALL; 2616 irq->u.extcall = li->irq.extcall; 2617 break; 2618 case IRQ_PEND_EXT_CLOCK_COMP: 2619 irq->type = KVM_S390_INT_CLOCK_COMP; 2620 break; 2621 case IRQ_PEND_EXT_CPU_TIMER: 2622 irq->type = KVM_S390_INT_CPU_TIMER; 2623 break; 2624 case IRQ_PEND_SIGP_STOP: 2625 irq->type = KVM_S390_SIGP_STOP; 2626 irq->u.stop = li->irq.stop; 2627 break; 2628 case IRQ_PEND_RESTART: 2629 irq->type = KVM_S390_RESTART; 2630 break; 2631 case IRQ_PEND_SET_PREFIX: 2632 irq->type = KVM_S390_SIGP_SET_PREFIX; 2633 irq->u.prefix = li->irq.prefix; 2634 break; 2635 } 2636 } 2637 2638 int kvm_s390_get_irq_state(struct kvm_vcpu *vcpu, __u8 __user *buf, int len) 2639 { 2640 int scn; 2641 unsigned long sigp_emerg_pending[BITS_TO_LONGS(KVM_MAX_VCPUS)]; 2642 struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int; 2643 unsigned long pending_irqs; 2644 struct kvm_s390_irq irq; 2645 unsigned long irq_type; 2646 int cpuaddr; 2647 int n = 0; 2648 2649 spin_lock(&li->lock); 2650 pending_irqs = li->pending_irqs; 2651 memcpy(&sigp_emerg_pending, &li->sigp_emerg_pending, 2652 sizeof(sigp_emerg_pending)); 2653 spin_unlock(&li->lock); 2654 2655 for_each_set_bit(irq_type, &pending_irqs, IRQ_PEND_COUNT) { 2656 memset(&irq, 0, sizeof(irq)); 2657 if (irq_type == IRQ_PEND_EXT_EMERGENCY) 2658 continue; 2659 if (n + sizeof(irq) > len) 2660 return -ENOBUFS; 2661 store_local_irq(&vcpu->arch.local_int, &irq, irq_type); 2662 if (copy_to_user(&buf[n], &irq, sizeof(irq))) 2663 return -EFAULT; 2664 n += sizeof(irq); 2665 } 2666 2667 if (test_bit(IRQ_PEND_EXT_EMERGENCY, &pending_irqs)) { 2668 for_each_set_bit(cpuaddr, sigp_emerg_pending, KVM_MAX_VCPUS) { 2669 memset(&irq, 0, sizeof(irq)); 2670 if (n + sizeof(irq) > len) 2671 return -ENOBUFS; 2672 irq.type = KVM_S390_INT_EMERGENCY; 2673 irq.u.emerg.code = cpuaddr; 2674 if (copy_to_user(&buf[n], &irq, sizeof(irq))) 2675 return -EFAULT; 2676 n += sizeof(irq); 2677 } 2678 } 2679 2680 if (sca_ext_call_pending(vcpu, &scn)) { 2681 if (n + sizeof(irq) > len) 2682 return -ENOBUFS; 2683 memset(&irq, 0, sizeof(irq)); 2684 irq.type = KVM_S390_INT_EXTERNAL_CALL; 2685 irq.u.extcall.code = scn; 2686 if (copy_to_user(&buf[n], &irq, sizeof(irq))) 2687 return -EFAULT; 2688 n += sizeof(irq); 2689 } 2690 2691 return n; 2692 } 2693