1 /* 2 * handling kvm guest interrupts 3 * 4 * Copyright IBM Corp. 2008, 2015 5 * 6 * This program is free software; you can redistribute it and/or modify 7 * it under the terms of the GNU General Public License (version 2 only) 8 * as published by the Free Software Foundation. 9 * 10 * Author(s): Carsten Otte <cotte@de.ibm.com> 11 */ 12 13 #include <linux/interrupt.h> 14 #include <linux/kvm_host.h> 15 #include <linux/hrtimer.h> 16 #include <linux/mmu_context.h> 17 #include <linux/signal.h> 18 #include <linux/slab.h> 19 #include <linux/bitmap.h> 20 #include <linux/vmalloc.h> 21 #include <asm/asm-offsets.h> 22 #include <asm/dis.h> 23 #include <linux/uaccess.h> 24 #include <asm/sclp.h> 25 #include <asm/isc.h> 26 #include <asm/gmap.h> 27 #include <asm/switch_to.h> 28 #include <asm/nmi.h> 29 #include "kvm-s390.h" 30 #include "gaccess.h" 31 #include "trace-s390.h" 32 33 #define PFAULT_INIT 0x0600 34 #define PFAULT_DONE 0x0680 35 #define VIRTIO_PARAM 0x0d00 36 37 /* handle external calls via sigp interpretation facility */ 38 static int sca_ext_call_pending(struct kvm_vcpu *vcpu, int *src_id) 39 { 40 int c, scn; 41 42 if (!(atomic_read(&vcpu->arch.sie_block->cpuflags) & CPUSTAT_ECALL_PEND)) 43 return 0; 44 45 BUG_ON(!kvm_s390_use_sca_entries()); 46 read_lock(&vcpu->kvm->arch.sca_lock); 47 if (vcpu->kvm->arch.use_esca) { 48 struct esca_block *sca = vcpu->kvm->arch.sca; 49 union esca_sigp_ctrl sigp_ctrl = 50 sca->cpu[vcpu->vcpu_id].sigp_ctrl; 51 52 c = sigp_ctrl.c; 53 scn = sigp_ctrl.scn; 54 } else { 55 struct bsca_block *sca = vcpu->kvm->arch.sca; 56 union bsca_sigp_ctrl sigp_ctrl = 57 sca->cpu[vcpu->vcpu_id].sigp_ctrl; 58 59 c = sigp_ctrl.c; 60 scn = sigp_ctrl.scn; 61 } 62 read_unlock(&vcpu->kvm->arch.sca_lock); 63 64 if (src_id) 65 *src_id = scn; 66 67 return c; 68 } 69 70 static int sca_inject_ext_call(struct kvm_vcpu *vcpu, int src_id) 71 { 72 int expect, rc; 73 74 BUG_ON(!kvm_s390_use_sca_entries()); 75 read_lock(&vcpu->kvm->arch.sca_lock); 76 if (vcpu->kvm->arch.use_esca) { 77 struct esca_block *sca = vcpu->kvm->arch.sca; 78 union esca_sigp_ctrl *sigp_ctrl = 79 &(sca->cpu[vcpu->vcpu_id].sigp_ctrl); 80 union esca_sigp_ctrl new_val = {0}, old_val = *sigp_ctrl; 81 82 new_val.scn = src_id; 83 new_val.c = 1; 84 old_val.c = 0; 85 86 expect = old_val.value; 87 rc = cmpxchg(&sigp_ctrl->value, old_val.value, new_val.value); 88 } else { 89 struct bsca_block *sca = vcpu->kvm->arch.sca; 90 union bsca_sigp_ctrl *sigp_ctrl = 91 &(sca->cpu[vcpu->vcpu_id].sigp_ctrl); 92 union bsca_sigp_ctrl new_val = {0}, old_val = *sigp_ctrl; 93 94 new_val.scn = src_id; 95 new_val.c = 1; 96 old_val.c = 0; 97 98 expect = old_val.value; 99 rc = cmpxchg(&sigp_ctrl->value, old_val.value, new_val.value); 100 } 101 read_unlock(&vcpu->kvm->arch.sca_lock); 102 103 if (rc != expect) { 104 /* another external call is pending */ 105 return -EBUSY; 106 } 107 atomic_or(CPUSTAT_ECALL_PEND, &vcpu->arch.sie_block->cpuflags); 108 return 0; 109 } 110 111 static void sca_clear_ext_call(struct kvm_vcpu *vcpu) 112 { 113 struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int; 114 int rc, expect; 115 116 if (!kvm_s390_use_sca_entries()) 117 return; 118 atomic_andnot(CPUSTAT_ECALL_PEND, li->cpuflags); 119 read_lock(&vcpu->kvm->arch.sca_lock); 120 if (vcpu->kvm->arch.use_esca) { 121 struct esca_block *sca = vcpu->kvm->arch.sca; 122 union esca_sigp_ctrl *sigp_ctrl = 123 &(sca->cpu[vcpu->vcpu_id].sigp_ctrl); 124 union esca_sigp_ctrl old = *sigp_ctrl; 125 126 expect = old.value; 127 rc = cmpxchg(&sigp_ctrl->value, old.value, 0); 128 } else { 129 struct bsca_block *sca = vcpu->kvm->arch.sca; 130 union bsca_sigp_ctrl *sigp_ctrl = 131 &(sca->cpu[vcpu->vcpu_id].sigp_ctrl); 132 union bsca_sigp_ctrl old = *sigp_ctrl; 133 134 expect = old.value; 135 rc = cmpxchg(&sigp_ctrl->value, old.value, 0); 136 } 137 read_unlock(&vcpu->kvm->arch.sca_lock); 138 WARN_ON(rc != expect); /* cannot clear? */ 139 } 140 141 int psw_extint_disabled(struct kvm_vcpu *vcpu) 142 { 143 return !(vcpu->arch.sie_block->gpsw.mask & PSW_MASK_EXT); 144 } 145 146 static int psw_ioint_disabled(struct kvm_vcpu *vcpu) 147 { 148 return !(vcpu->arch.sie_block->gpsw.mask & PSW_MASK_IO); 149 } 150 151 static int psw_mchk_disabled(struct kvm_vcpu *vcpu) 152 { 153 return !(vcpu->arch.sie_block->gpsw.mask & PSW_MASK_MCHECK); 154 } 155 156 static int psw_interrupts_disabled(struct kvm_vcpu *vcpu) 157 { 158 return psw_extint_disabled(vcpu) && 159 psw_ioint_disabled(vcpu) && 160 psw_mchk_disabled(vcpu); 161 } 162 163 static int ckc_interrupts_enabled(struct kvm_vcpu *vcpu) 164 { 165 if (psw_extint_disabled(vcpu) || 166 !(vcpu->arch.sie_block->gcr[0] & 0x800ul)) 167 return 0; 168 if (guestdbg_enabled(vcpu) && guestdbg_sstep_enabled(vcpu)) 169 /* No timer interrupts when single stepping */ 170 return 0; 171 return 1; 172 } 173 174 static int ckc_irq_pending(struct kvm_vcpu *vcpu) 175 { 176 if (vcpu->arch.sie_block->ckc >= kvm_s390_get_tod_clock_fast(vcpu->kvm)) 177 return 0; 178 return ckc_interrupts_enabled(vcpu); 179 } 180 181 static int cpu_timer_interrupts_enabled(struct kvm_vcpu *vcpu) 182 { 183 return !psw_extint_disabled(vcpu) && 184 (vcpu->arch.sie_block->gcr[0] & 0x400ul); 185 } 186 187 static int cpu_timer_irq_pending(struct kvm_vcpu *vcpu) 188 { 189 if (!cpu_timer_interrupts_enabled(vcpu)) 190 return 0; 191 return kvm_s390_get_cpu_timer(vcpu) >> 63; 192 } 193 194 static inline int is_ioirq(unsigned long irq_type) 195 { 196 return ((irq_type >= IRQ_PEND_IO_ISC_0) && 197 (irq_type <= IRQ_PEND_IO_ISC_7)); 198 } 199 200 static uint64_t isc_to_isc_bits(int isc) 201 { 202 return (0x80 >> isc) << 24; 203 } 204 205 static inline u8 int_word_to_isc(u32 int_word) 206 { 207 return (int_word & 0x38000000) >> 27; 208 } 209 210 static inline unsigned long pending_irqs(struct kvm_vcpu *vcpu) 211 { 212 return vcpu->kvm->arch.float_int.pending_irqs | 213 vcpu->arch.local_int.pending_irqs; 214 } 215 216 static unsigned long disable_iscs(struct kvm_vcpu *vcpu, 217 unsigned long active_mask) 218 { 219 int i; 220 221 for (i = 0; i <= MAX_ISC; i++) 222 if (!(vcpu->arch.sie_block->gcr[6] & isc_to_isc_bits(i))) 223 active_mask &= ~(1UL << (IRQ_PEND_IO_ISC_0 + i)); 224 225 return active_mask; 226 } 227 228 static unsigned long deliverable_irqs(struct kvm_vcpu *vcpu) 229 { 230 unsigned long active_mask; 231 232 active_mask = pending_irqs(vcpu); 233 if (!active_mask) 234 return 0; 235 236 if (psw_extint_disabled(vcpu)) 237 active_mask &= ~IRQ_PEND_EXT_MASK; 238 if (psw_ioint_disabled(vcpu)) 239 active_mask &= ~IRQ_PEND_IO_MASK; 240 else 241 active_mask = disable_iscs(vcpu, active_mask); 242 if (!(vcpu->arch.sie_block->gcr[0] & 0x2000ul)) 243 __clear_bit(IRQ_PEND_EXT_EXTERNAL, &active_mask); 244 if (!(vcpu->arch.sie_block->gcr[0] & 0x4000ul)) 245 __clear_bit(IRQ_PEND_EXT_EMERGENCY, &active_mask); 246 if (!(vcpu->arch.sie_block->gcr[0] & 0x800ul)) 247 __clear_bit(IRQ_PEND_EXT_CLOCK_COMP, &active_mask); 248 if (!(vcpu->arch.sie_block->gcr[0] & 0x400ul)) 249 __clear_bit(IRQ_PEND_EXT_CPU_TIMER, &active_mask); 250 if (!(vcpu->arch.sie_block->gcr[0] & 0x200ul)) 251 __clear_bit(IRQ_PEND_EXT_SERVICE, &active_mask); 252 if (psw_mchk_disabled(vcpu)) 253 active_mask &= ~IRQ_PEND_MCHK_MASK; 254 /* 255 * Check both floating and local interrupt's cr14 because 256 * bit IRQ_PEND_MCHK_REP could be set in both cases. 257 */ 258 if (!(vcpu->arch.sie_block->gcr[14] & 259 (vcpu->kvm->arch.float_int.mchk.cr14 | 260 vcpu->arch.local_int.irq.mchk.cr14))) 261 __clear_bit(IRQ_PEND_MCHK_REP, &active_mask); 262 263 /* 264 * STOP irqs will never be actively delivered. They are triggered via 265 * intercept requests and cleared when the stop intercept is performed. 266 */ 267 __clear_bit(IRQ_PEND_SIGP_STOP, &active_mask); 268 269 return active_mask; 270 } 271 272 static void __set_cpu_idle(struct kvm_vcpu *vcpu) 273 { 274 atomic_or(CPUSTAT_WAIT, &vcpu->arch.sie_block->cpuflags); 275 set_bit(vcpu->vcpu_id, vcpu->arch.local_int.float_int->idle_mask); 276 } 277 278 static void __unset_cpu_idle(struct kvm_vcpu *vcpu) 279 { 280 atomic_andnot(CPUSTAT_WAIT, &vcpu->arch.sie_block->cpuflags); 281 clear_bit(vcpu->vcpu_id, vcpu->arch.local_int.float_int->idle_mask); 282 } 283 284 static void __reset_intercept_indicators(struct kvm_vcpu *vcpu) 285 { 286 atomic_andnot(CPUSTAT_IO_INT | CPUSTAT_EXT_INT | CPUSTAT_STOP_INT, 287 &vcpu->arch.sie_block->cpuflags); 288 vcpu->arch.sie_block->lctl = 0x0000; 289 vcpu->arch.sie_block->ictl &= ~(ICTL_LPSW | ICTL_STCTL | ICTL_PINT); 290 291 if (guestdbg_enabled(vcpu)) { 292 vcpu->arch.sie_block->lctl |= (LCTL_CR0 | LCTL_CR9 | 293 LCTL_CR10 | LCTL_CR11); 294 vcpu->arch.sie_block->ictl |= (ICTL_STCTL | ICTL_PINT); 295 } 296 } 297 298 static void __set_cpuflag(struct kvm_vcpu *vcpu, u32 flag) 299 { 300 atomic_or(flag, &vcpu->arch.sie_block->cpuflags); 301 } 302 303 static void set_intercept_indicators_io(struct kvm_vcpu *vcpu) 304 { 305 if (!(pending_irqs(vcpu) & IRQ_PEND_IO_MASK)) 306 return; 307 else if (psw_ioint_disabled(vcpu)) 308 __set_cpuflag(vcpu, CPUSTAT_IO_INT); 309 else 310 vcpu->arch.sie_block->lctl |= LCTL_CR6; 311 } 312 313 static void set_intercept_indicators_ext(struct kvm_vcpu *vcpu) 314 { 315 if (!(pending_irqs(vcpu) & IRQ_PEND_EXT_MASK)) 316 return; 317 if (psw_extint_disabled(vcpu)) 318 __set_cpuflag(vcpu, CPUSTAT_EXT_INT); 319 else 320 vcpu->arch.sie_block->lctl |= LCTL_CR0; 321 } 322 323 static void set_intercept_indicators_mchk(struct kvm_vcpu *vcpu) 324 { 325 if (!(pending_irqs(vcpu) & IRQ_PEND_MCHK_MASK)) 326 return; 327 if (psw_mchk_disabled(vcpu)) 328 vcpu->arch.sie_block->ictl |= ICTL_LPSW; 329 else 330 vcpu->arch.sie_block->lctl |= LCTL_CR14; 331 } 332 333 static void set_intercept_indicators_stop(struct kvm_vcpu *vcpu) 334 { 335 if (kvm_s390_is_stop_irq_pending(vcpu)) 336 __set_cpuflag(vcpu, CPUSTAT_STOP_INT); 337 } 338 339 /* Set interception request for non-deliverable interrupts */ 340 static void set_intercept_indicators(struct kvm_vcpu *vcpu) 341 { 342 set_intercept_indicators_io(vcpu); 343 set_intercept_indicators_ext(vcpu); 344 set_intercept_indicators_mchk(vcpu); 345 set_intercept_indicators_stop(vcpu); 346 } 347 348 static int __must_check __deliver_cpu_timer(struct kvm_vcpu *vcpu) 349 { 350 struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int; 351 int rc; 352 353 trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, KVM_S390_INT_CPU_TIMER, 354 0, 0); 355 356 rc = put_guest_lc(vcpu, EXT_IRQ_CPU_TIMER, 357 (u16 *)__LC_EXT_INT_CODE); 358 rc |= put_guest_lc(vcpu, 0, (u16 *)__LC_EXT_CPU_ADDR); 359 rc |= write_guest_lc(vcpu, __LC_EXT_OLD_PSW, 360 &vcpu->arch.sie_block->gpsw, sizeof(psw_t)); 361 rc |= read_guest_lc(vcpu, __LC_EXT_NEW_PSW, 362 &vcpu->arch.sie_block->gpsw, sizeof(psw_t)); 363 clear_bit(IRQ_PEND_EXT_CPU_TIMER, &li->pending_irqs); 364 return rc ? -EFAULT : 0; 365 } 366 367 static int __must_check __deliver_ckc(struct kvm_vcpu *vcpu) 368 { 369 struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int; 370 int rc; 371 372 trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, KVM_S390_INT_CLOCK_COMP, 373 0, 0); 374 375 rc = put_guest_lc(vcpu, EXT_IRQ_CLK_COMP, 376 (u16 __user *)__LC_EXT_INT_CODE); 377 rc |= put_guest_lc(vcpu, 0, (u16 *)__LC_EXT_CPU_ADDR); 378 rc |= write_guest_lc(vcpu, __LC_EXT_OLD_PSW, 379 &vcpu->arch.sie_block->gpsw, sizeof(psw_t)); 380 rc |= read_guest_lc(vcpu, __LC_EXT_NEW_PSW, 381 &vcpu->arch.sie_block->gpsw, sizeof(psw_t)); 382 clear_bit(IRQ_PEND_EXT_CLOCK_COMP, &li->pending_irqs); 383 return rc ? -EFAULT : 0; 384 } 385 386 static int __must_check __deliver_pfault_init(struct kvm_vcpu *vcpu) 387 { 388 struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int; 389 struct kvm_s390_ext_info ext; 390 int rc; 391 392 spin_lock(&li->lock); 393 ext = li->irq.ext; 394 clear_bit(IRQ_PEND_PFAULT_INIT, &li->pending_irqs); 395 li->irq.ext.ext_params2 = 0; 396 spin_unlock(&li->lock); 397 398 VCPU_EVENT(vcpu, 4, "deliver: pfault init token 0x%llx", 399 ext.ext_params2); 400 trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, 401 KVM_S390_INT_PFAULT_INIT, 402 0, ext.ext_params2); 403 404 rc = put_guest_lc(vcpu, EXT_IRQ_CP_SERVICE, (u16 *) __LC_EXT_INT_CODE); 405 rc |= put_guest_lc(vcpu, PFAULT_INIT, (u16 *) __LC_EXT_CPU_ADDR); 406 rc |= write_guest_lc(vcpu, __LC_EXT_OLD_PSW, 407 &vcpu->arch.sie_block->gpsw, sizeof(psw_t)); 408 rc |= read_guest_lc(vcpu, __LC_EXT_NEW_PSW, 409 &vcpu->arch.sie_block->gpsw, sizeof(psw_t)); 410 rc |= put_guest_lc(vcpu, ext.ext_params2, (u64 *) __LC_EXT_PARAMS2); 411 return rc ? -EFAULT : 0; 412 } 413 414 static int __write_machine_check(struct kvm_vcpu *vcpu, 415 struct kvm_s390_mchk_info *mchk) 416 { 417 unsigned long ext_sa_addr; 418 unsigned long lc; 419 freg_t fprs[NUM_FPRS]; 420 union mci mci; 421 int rc; 422 423 mci.val = mchk->mcic; 424 /* take care of lazy register loading */ 425 save_fpu_regs(); 426 save_access_regs(vcpu->run->s.regs.acrs); 427 if (MACHINE_HAS_GS && vcpu->arch.gs_enabled) 428 save_gs_cb(current->thread.gs_cb); 429 430 /* Extended save area */ 431 rc = read_guest_lc(vcpu, __LC_MCESAD, &ext_sa_addr, 432 sizeof(unsigned long)); 433 /* Only bits 0 through 63-LC are used for address formation */ 434 lc = ext_sa_addr & MCESA_LC_MASK; 435 if (test_kvm_facility(vcpu->kvm, 133)) { 436 switch (lc) { 437 case 0: 438 case 10: 439 ext_sa_addr &= ~0x3ffUL; 440 break; 441 case 11: 442 ext_sa_addr &= ~0x7ffUL; 443 break; 444 case 12: 445 ext_sa_addr &= ~0xfffUL; 446 break; 447 default: 448 ext_sa_addr = 0; 449 break; 450 } 451 } else { 452 ext_sa_addr &= ~0x3ffUL; 453 } 454 455 if (!rc && mci.vr && ext_sa_addr && test_kvm_facility(vcpu->kvm, 129)) { 456 if (write_guest_abs(vcpu, ext_sa_addr, vcpu->run->s.regs.vrs, 457 512)) 458 mci.vr = 0; 459 } else { 460 mci.vr = 0; 461 } 462 if (!rc && mci.gs && ext_sa_addr && test_kvm_facility(vcpu->kvm, 133) 463 && (lc == 11 || lc == 12)) { 464 if (write_guest_abs(vcpu, ext_sa_addr + 1024, 465 &vcpu->run->s.regs.gscb, 32)) 466 mci.gs = 0; 467 } else { 468 mci.gs = 0; 469 } 470 471 /* General interruption information */ 472 rc |= put_guest_lc(vcpu, 1, (u8 __user *) __LC_AR_MODE_ID); 473 rc |= write_guest_lc(vcpu, __LC_MCK_OLD_PSW, 474 &vcpu->arch.sie_block->gpsw, sizeof(psw_t)); 475 rc |= read_guest_lc(vcpu, __LC_MCK_NEW_PSW, 476 &vcpu->arch.sie_block->gpsw, sizeof(psw_t)); 477 rc |= put_guest_lc(vcpu, mci.val, (u64 __user *) __LC_MCCK_CODE); 478 479 /* Register-save areas */ 480 if (MACHINE_HAS_VX) { 481 convert_vx_to_fp(fprs, (__vector128 *) vcpu->run->s.regs.vrs); 482 rc |= write_guest_lc(vcpu, __LC_FPREGS_SAVE_AREA, fprs, 128); 483 } else { 484 rc |= write_guest_lc(vcpu, __LC_FPREGS_SAVE_AREA, 485 vcpu->run->s.regs.fprs, 128); 486 } 487 rc |= write_guest_lc(vcpu, __LC_GPREGS_SAVE_AREA, 488 vcpu->run->s.regs.gprs, 128); 489 rc |= put_guest_lc(vcpu, current->thread.fpu.fpc, 490 (u32 __user *) __LC_FP_CREG_SAVE_AREA); 491 rc |= put_guest_lc(vcpu, vcpu->arch.sie_block->todpr, 492 (u32 __user *) __LC_TOD_PROGREG_SAVE_AREA); 493 rc |= put_guest_lc(vcpu, kvm_s390_get_cpu_timer(vcpu), 494 (u64 __user *) __LC_CPU_TIMER_SAVE_AREA); 495 rc |= put_guest_lc(vcpu, vcpu->arch.sie_block->ckc >> 8, 496 (u64 __user *) __LC_CLOCK_COMP_SAVE_AREA); 497 rc |= write_guest_lc(vcpu, __LC_AREGS_SAVE_AREA, 498 &vcpu->run->s.regs.acrs, 64); 499 rc |= write_guest_lc(vcpu, __LC_CREGS_SAVE_AREA, 500 &vcpu->arch.sie_block->gcr, 128); 501 502 /* Extended interruption information */ 503 rc |= put_guest_lc(vcpu, mchk->ext_damage_code, 504 (u32 __user *) __LC_EXT_DAMAGE_CODE); 505 rc |= put_guest_lc(vcpu, mchk->failing_storage_address, 506 (u64 __user *) __LC_MCCK_FAIL_STOR_ADDR); 507 rc |= write_guest_lc(vcpu, __LC_PSW_SAVE_AREA, &mchk->fixed_logout, 508 sizeof(mchk->fixed_logout)); 509 return rc ? -EFAULT : 0; 510 } 511 512 static int __must_check __deliver_machine_check(struct kvm_vcpu *vcpu) 513 { 514 struct kvm_s390_float_interrupt *fi = &vcpu->kvm->arch.float_int; 515 struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int; 516 struct kvm_s390_mchk_info mchk = {}; 517 int deliver = 0; 518 int rc = 0; 519 520 spin_lock(&fi->lock); 521 spin_lock(&li->lock); 522 if (test_bit(IRQ_PEND_MCHK_EX, &li->pending_irqs) || 523 test_bit(IRQ_PEND_MCHK_REP, &li->pending_irqs)) { 524 /* 525 * If there was an exigent machine check pending, then any 526 * repressible machine checks that might have been pending 527 * are indicated along with it, so always clear bits for 528 * repressible and exigent interrupts 529 */ 530 mchk = li->irq.mchk; 531 clear_bit(IRQ_PEND_MCHK_EX, &li->pending_irqs); 532 clear_bit(IRQ_PEND_MCHK_REP, &li->pending_irqs); 533 memset(&li->irq.mchk, 0, sizeof(mchk)); 534 deliver = 1; 535 } 536 /* 537 * We indicate floating repressible conditions along with 538 * other pending conditions. Channel Report Pending and Channel 539 * Subsystem damage are the only two and and are indicated by 540 * bits in mcic and masked in cr14. 541 */ 542 if (test_and_clear_bit(IRQ_PEND_MCHK_REP, &fi->pending_irqs)) { 543 mchk.mcic |= fi->mchk.mcic; 544 mchk.cr14 |= fi->mchk.cr14; 545 memset(&fi->mchk, 0, sizeof(mchk)); 546 deliver = 1; 547 } 548 spin_unlock(&li->lock); 549 spin_unlock(&fi->lock); 550 551 if (deliver) { 552 VCPU_EVENT(vcpu, 3, "deliver: machine check mcic 0x%llx", 553 mchk.mcic); 554 trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, 555 KVM_S390_MCHK, 556 mchk.cr14, mchk.mcic); 557 rc = __write_machine_check(vcpu, &mchk); 558 } 559 return rc; 560 } 561 562 static int __must_check __deliver_restart(struct kvm_vcpu *vcpu) 563 { 564 struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int; 565 int rc; 566 567 VCPU_EVENT(vcpu, 3, "%s", "deliver: cpu restart"); 568 vcpu->stat.deliver_restart_signal++; 569 trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, KVM_S390_RESTART, 0, 0); 570 571 rc = write_guest_lc(vcpu, 572 offsetof(struct lowcore, restart_old_psw), 573 &vcpu->arch.sie_block->gpsw, sizeof(psw_t)); 574 rc |= read_guest_lc(vcpu, offsetof(struct lowcore, restart_psw), 575 &vcpu->arch.sie_block->gpsw, sizeof(psw_t)); 576 clear_bit(IRQ_PEND_RESTART, &li->pending_irqs); 577 return rc ? -EFAULT : 0; 578 } 579 580 static int __must_check __deliver_set_prefix(struct kvm_vcpu *vcpu) 581 { 582 struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int; 583 struct kvm_s390_prefix_info prefix; 584 585 spin_lock(&li->lock); 586 prefix = li->irq.prefix; 587 li->irq.prefix.address = 0; 588 clear_bit(IRQ_PEND_SET_PREFIX, &li->pending_irqs); 589 spin_unlock(&li->lock); 590 591 vcpu->stat.deliver_prefix_signal++; 592 trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, 593 KVM_S390_SIGP_SET_PREFIX, 594 prefix.address, 0); 595 596 kvm_s390_set_prefix(vcpu, prefix.address); 597 return 0; 598 } 599 600 static int __must_check __deliver_emergency_signal(struct kvm_vcpu *vcpu) 601 { 602 struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int; 603 int rc; 604 int cpu_addr; 605 606 spin_lock(&li->lock); 607 cpu_addr = find_first_bit(li->sigp_emerg_pending, KVM_MAX_VCPUS); 608 clear_bit(cpu_addr, li->sigp_emerg_pending); 609 if (bitmap_empty(li->sigp_emerg_pending, KVM_MAX_VCPUS)) 610 clear_bit(IRQ_PEND_EXT_EMERGENCY, &li->pending_irqs); 611 spin_unlock(&li->lock); 612 613 VCPU_EVENT(vcpu, 4, "%s", "deliver: sigp emerg"); 614 vcpu->stat.deliver_emergency_signal++; 615 trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, KVM_S390_INT_EMERGENCY, 616 cpu_addr, 0); 617 618 rc = put_guest_lc(vcpu, EXT_IRQ_EMERGENCY_SIG, 619 (u16 *)__LC_EXT_INT_CODE); 620 rc |= put_guest_lc(vcpu, cpu_addr, (u16 *)__LC_EXT_CPU_ADDR); 621 rc |= write_guest_lc(vcpu, __LC_EXT_OLD_PSW, 622 &vcpu->arch.sie_block->gpsw, sizeof(psw_t)); 623 rc |= read_guest_lc(vcpu, __LC_EXT_NEW_PSW, 624 &vcpu->arch.sie_block->gpsw, sizeof(psw_t)); 625 return rc ? -EFAULT : 0; 626 } 627 628 static int __must_check __deliver_external_call(struct kvm_vcpu *vcpu) 629 { 630 struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int; 631 struct kvm_s390_extcall_info extcall; 632 int rc; 633 634 spin_lock(&li->lock); 635 extcall = li->irq.extcall; 636 li->irq.extcall.code = 0; 637 clear_bit(IRQ_PEND_EXT_EXTERNAL, &li->pending_irqs); 638 spin_unlock(&li->lock); 639 640 VCPU_EVENT(vcpu, 4, "%s", "deliver: sigp ext call"); 641 vcpu->stat.deliver_external_call++; 642 trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, 643 KVM_S390_INT_EXTERNAL_CALL, 644 extcall.code, 0); 645 646 rc = put_guest_lc(vcpu, EXT_IRQ_EXTERNAL_CALL, 647 (u16 *)__LC_EXT_INT_CODE); 648 rc |= put_guest_lc(vcpu, extcall.code, (u16 *)__LC_EXT_CPU_ADDR); 649 rc |= write_guest_lc(vcpu, __LC_EXT_OLD_PSW, 650 &vcpu->arch.sie_block->gpsw, sizeof(psw_t)); 651 rc |= read_guest_lc(vcpu, __LC_EXT_NEW_PSW, &vcpu->arch.sie_block->gpsw, 652 sizeof(psw_t)); 653 return rc ? -EFAULT : 0; 654 } 655 656 static int __must_check __deliver_prog(struct kvm_vcpu *vcpu) 657 { 658 struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int; 659 struct kvm_s390_pgm_info pgm_info; 660 int rc = 0, nullifying = false; 661 u16 ilen; 662 663 spin_lock(&li->lock); 664 pgm_info = li->irq.pgm; 665 clear_bit(IRQ_PEND_PROG, &li->pending_irqs); 666 memset(&li->irq.pgm, 0, sizeof(pgm_info)); 667 spin_unlock(&li->lock); 668 669 ilen = pgm_info.flags & KVM_S390_PGM_FLAGS_ILC_MASK; 670 VCPU_EVENT(vcpu, 3, "deliver: program irq code 0x%x, ilen:%d", 671 pgm_info.code, ilen); 672 vcpu->stat.deliver_program_int++; 673 trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, KVM_S390_PROGRAM_INT, 674 pgm_info.code, 0); 675 676 switch (pgm_info.code & ~PGM_PER) { 677 case PGM_AFX_TRANSLATION: 678 case PGM_ASX_TRANSLATION: 679 case PGM_EX_TRANSLATION: 680 case PGM_LFX_TRANSLATION: 681 case PGM_LSTE_SEQUENCE: 682 case PGM_LSX_TRANSLATION: 683 case PGM_LX_TRANSLATION: 684 case PGM_PRIMARY_AUTHORITY: 685 case PGM_SECONDARY_AUTHORITY: 686 nullifying = true; 687 /* fall through */ 688 case PGM_SPACE_SWITCH: 689 rc = put_guest_lc(vcpu, pgm_info.trans_exc_code, 690 (u64 *)__LC_TRANS_EXC_CODE); 691 break; 692 case PGM_ALEN_TRANSLATION: 693 case PGM_ALE_SEQUENCE: 694 case PGM_ASTE_INSTANCE: 695 case PGM_ASTE_SEQUENCE: 696 case PGM_ASTE_VALIDITY: 697 case PGM_EXTENDED_AUTHORITY: 698 rc = put_guest_lc(vcpu, pgm_info.exc_access_id, 699 (u8 *)__LC_EXC_ACCESS_ID); 700 nullifying = true; 701 break; 702 case PGM_ASCE_TYPE: 703 case PGM_PAGE_TRANSLATION: 704 case PGM_REGION_FIRST_TRANS: 705 case PGM_REGION_SECOND_TRANS: 706 case PGM_REGION_THIRD_TRANS: 707 case PGM_SEGMENT_TRANSLATION: 708 rc = put_guest_lc(vcpu, pgm_info.trans_exc_code, 709 (u64 *)__LC_TRANS_EXC_CODE); 710 rc |= put_guest_lc(vcpu, pgm_info.exc_access_id, 711 (u8 *)__LC_EXC_ACCESS_ID); 712 rc |= put_guest_lc(vcpu, pgm_info.op_access_id, 713 (u8 *)__LC_OP_ACCESS_ID); 714 nullifying = true; 715 break; 716 case PGM_MONITOR: 717 rc = put_guest_lc(vcpu, pgm_info.mon_class_nr, 718 (u16 *)__LC_MON_CLASS_NR); 719 rc |= put_guest_lc(vcpu, pgm_info.mon_code, 720 (u64 *)__LC_MON_CODE); 721 break; 722 case PGM_VECTOR_PROCESSING: 723 case PGM_DATA: 724 rc = put_guest_lc(vcpu, pgm_info.data_exc_code, 725 (u32 *)__LC_DATA_EXC_CODE); 726 break; 727 case PGM_PROTECTION: 728 rc = put_guest_lc(vcpu, pgm_info.trans_exc_code, 729 (u64 *)__LC_TRANS_EXC_CODE); 730 rc |= put_guest_lc(vcpu, pgm_info.exc_access_id, 731 (u8 *)__LC_EXC_ACCESS_ID); 732 break; 733 case PGM_STACK_FULL: 734 case PGM_STACK_EMPTY: 735 case PGM_STACK_SPECIFICATION: 736 case PGM_STACK_TYPE: 737 case PGM_STACK_OPERATION: 738 case PGM_TRACE_TABEL: 739 case PGM_CRYPTO_OPERATION: 740 nullifying = true; 741 break; 742 } 743 744 if (pgm_info.code & PGM_PER) { 745 rc |= put_guest_lc(vcpu, pgm_info.per_code, 746 (u8 *) __LC_PER_CODE); 747 rc |= put_guest_lc(vcpu, pgm_info.per_atmid, 748 (u8 *)__LC_PER_ATMID); 749 rc |= put_guest_lc(vcpu, pgm_info.per_address, 750 (u64 *) __LC_PER_ADDRESS); 751 rc |= put_guest_lc(vcpu, pgm_info.per_access_id, 752 (u8 *) __LC_PER_ACCESS_ID); 753 } 754 755 if (nullifying && !(pgm_info.flags & KVM_S390_PGM_FLAGS_NO_REWIND)) 756 kvm_s390_rewind_psw(vcpu, ilen); 757 758 /* bit 1+2 of the target are the ilc, so we can directly use ilen */ 759 rc |= put_guest_lc(vcpu, ilen, (u16 *) __LC_PGM_ILC); 760 rc |= put_guest_lc(vcpu, vcpu->arch.sie_block->gbea, 761 (u64 *) __LC_LAST_BREAK); 762 rc |= put_guest_lc(vcpu, pgm_info.code, 763 (u16 *)__LC_PGM_INT_CODE); 764 rc |= write_guest_lc(vcpu, __LC_PGM_OLD_PSW, 765 &vcpu->arch.sie_block->gpsw, sizeof(psw_t)); 766 rc |= read_guest_lc(vcpu, __LC_PGM_NEW_PSW, 767 &vcpu->arch.sie_block->gpsw, sizeof(psw_t)); 768 return rc ? -EFAULT : 0; 769 } 770 771 static int __must_check __deliver_service(struct kvm_vcpu *vcpu) 772 { 773 struct kvm_s390_float_interrupt *fi = &vcpu->kvm->arch.float_int; 774 struct kvm_s390_ext_info ext; 775 int rc = 0; 776 777 spin_lock(&fi->lock); 778 if (!(test_bit(IRQ_PEND_EXT_SERVICE, &fi->pending_irqs))) { 779 spin_unlock(&fi->lock); 780 return 0; 781 } 782 ext = fi->srv_signal; 783 memset(&fi->srv_signal, 0, sizeof(ext)); 784 clear_bit(IRQ_PEND_EXT_SERVICE, &fi->pending_irqs); 785 spin_unlock(&fi->lock); 786 787 VCPU_EVENT(vcpu, 4, "deliver: sclp parameter 0x%x", 788 ext.ext_params); 789 vcpu->stat.deliver_service_signal++; 790 trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, KVM_S390_INT_SERVICE, 791 ext.ext_params, 0); 792 793 rc = put_guest_lc(vcpu, EXT_IRQ_SERVICE_SIG, (u16 *)__LC_EXT_INT_CODE); 794 rc |= put_guest_lc(vcpu, 0, (u16 *)__LC_EXT_CPU_ADDR); 795 rc |= write_guest_lc(vcpu, __LC_EXT_OLD_PSW, 796 &vcpu->arch.sie_block->gpsw, sizeof(psw_t)); 797 rc |= read_guest_lc(vcpu, __LC_EXT_NEW_PSW, 798 &vcpu->arch.sie_block->gpsw, sizeof(psw_t)); 799 rc |= put_guest_lc(vcpu, ext.ext_params, 800 (u32 *)__LC_EXT_PARAMS); 801 802 return rc ? -EFAULT : 0; 803 } 804 805 static int __must_check __deliver_pfault_done(struct kvm_vcpu *vcpu) 806 { 807 struct kvm_s390_float_interrupt *fi = &vcpu->kvm->arch.float_int; 808 struct kvm_s390_interrupt_info *inti; 809 int rc = 0; 810 811 spin_lock(&fi->lock); 812 inti = list_first_entry_or_null(&fi->lists[FIRQ_LIST_PFAULT], 813 struct kvm_s390_interrupt_info, 814 list); 815 if (inti) { 816 list_del(&inti->list); 817 fi->counters[FIRQ_CNTR_PFAULT] -= 1; 818 } 819 if (list_empty(&fi->lists[FIRQ_LIST_PFAULT])) 820 clear_bit(IRQ_PEND_PFAULT_DONE, &fi->pending_irqs); 821 spin_unlock(&fi->lock); 822 823 if (inti) { 824 trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, 825 KVM_S390_INT_PFAULT_DONE, 0, 826 inti->ext.ext_params2); 827 VCPU_EVENT(vcpu, 4, "deliver: pfault done token 0x%llx", 828 inti->ext.ext_params2); 829 830 rc = put_guest_lc(vcpu, EXT_IRQ_CP_SERVICE, 831 (u16 *)__LC_EXT_INT_CODE); 832 rc |= put_guest_lc(vcpu, PFAULT_DONE, 833 (u16 *)__LC_EXT_CPU_ADDR); 834 rc |= write_guest_lc(vcpu, __LC_EXT_OLD_PSW, 835 &vcpu->arch.sie_block->gpsw, 836 sizeof(psw_t)); 837 rc |= read_guest_lc(vcpu, __LC_EXT_NEW_PSW, 838 &vcpu->arch.sie_block->gpsw, 839 sizeof(psw_t)); 840 rc |= put_guest_lc(vcpu, inti->ext.ext_params2, 841 (u64 *)__LC_EXT_PARAMS2); 842 kfree(inti); 843 } 844 return rc ? -EFAULT : 0; 845 } 846 847 static int __must_check __deliver_virtio(struct kvm_vcpu *vcpu) 848 { 849 struct kvm_s390_float_interrupt *fi = &vcpu->kvm->arch.float_int; 850 struct kvm_s390_interrupt_info *inti; 851 int rc = 0; 852 853 spin_lock(&fi->lock); 854 inti = list_first_entry_or_null(&fi->lists[FIRQ_LIST_VIRTIO], 855 struct kvm_s390_interrupt_info, 856 list); 857 if (inti) { 858 VCPU_EVENT(vcpu, 4, 859 "deliver: virtio parm: 0x%x,parm64: 0x%llx", 860 inti->ext.ext_params, inti->ext.ext_params2); 861 vcpu->stat.deliver_virtio_interrupt++; 862 trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, 863 inti->type, 864 inti->ext.ext_params, 865 inti->ext.ext_params2); 866 list_del(&inti->list); 867 fi->counters[FIRQ_CNTR_VIRTIO] -= 1; 868 } 869 if (list_empty(&fi->lists[FIRQ_LIST_VIRTIO])) 870 clear_bit(IRQ_PEND_VIRTIO, &fi->pending_irqs); 871 spin_unlock(&fi->lock); 872 873 if (inti) { 874 rc = put_guest_lc(vcpu, EXT_IRQ_CP_SERVICE, 875 (u16 *)__LC_EXT_INT_CODE); 876 rc |= put_guest_lc(vcpu, VIRTIO_PARAM, 877 (u16 *)__LC_EXT_CPU_ADDR); 878 rc |= write_guest_lc(vcpu, __LC_EXT_OLD_PSW, 879 &vcpu->arch.sie_block->gpsw, 880 sizeof(psw_t)); 881 rc |= read_guest_lc(vcpu, __LC_EXT_NEW_PSW, 882 &vcpu->arch.sie_block->gpsw, 883 sizeof(psw_t)); 884 rc |= put_guest_lc(vcpu, inti->ext.ext_params, 885 (u32 *)__LC_EXT_PARAMS); 886 rc |= put_guest_lc(vcpu, inti->ext.ext_params2, 887 (u64 *)__LC_EXT_PARAMS2); 888 kfree(inti); 889 } 890 return rc ? -EFAULT : 0; 891 } 892 893 static int __must_check __deliver_io(struct kvm_vcpu *vcpu, 894 unsigned long irq_type) 895 { 896 struct list_head *isc_list; 897 struct kvm_s390_float_interrupt *fi; 898 struct kvm_s390_interrupt_info *inti = NULL; 899 int rc = 0; 900 901 fi = &vcpu->kvm->arch.float_int; 902 903 spin_lock(&fi->lock); 904 isc_list = &fi->lists[irq_type - IRQ_PEND_IO_ISC_0]; 905 inti = list_first_entry_or_null(isc_list, 906 struct kvm_s390_interrupt_info, 907 list); 908 if (inti) { 909 if (inti->type & KVM_S390_INT_IO_AI_MASK) 910 VCPU_EVENT(vcpu, 4, "%s", "deliver: I/O (AI)"); 911 else 912 VCPU_EVENT(vcpu, 4, "deliver: I/O %x ss %x schid %04x", 913 inti->io.subchannel_id >> 8, 914 inti->io.subchannel_id >> 1 & 0x3, 915 inti->io.subchannel_nr); 916 917 vcpu->stat.deliver_io_int++; 918 trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, 919 inti->type, 920 ((__u32)inti->io.subchannel_id << 16) | 921 inti->io.subchannel_nr, 922 ((__u64)inti->io.io_int_parm << 32) | 923 inti->io.io_int_word); 924 list_del(&inti->list); 925 fi->counters[FIRQ_CNTR_IO] -= 1; 926 } 927 if (list_empty(isc_list)) 928 clear_bit(irq_type, &fi->pending_irqs); 929 spin_unlock(&fi->lock); 930 931 if (inti) { 932 rc = put_guest_lc(vcpu, inti->io.subchannel_id, 933 (u16 *)__LC_SUBCHANNEL_ID); 934 rc |= put_guest_lc(vcpu, inti->io.subchannel_nr, 935 (u16 *)__LC_SUBCHANNEL_NR); 936 rc |= put_guest_lc(vcpu, inti->io.io_int_parm, 937 (u32 *)__LC_IO_INT_PARM); 938 rc |= put_guest_lc(vcpu, inti->io.io_int_word, 939 (u32 *)__LC_IO_INT_WORD); 940 rc |= write_guest_lc(vcpu, __LC_IO_OLD_PSW, 941 &vcpu->arch.sie_block->gpsw, 942 sizeof(psw_t)); 943 rc |= read_guest_lc(vcpu, __LC_IO_NEW_PSW, 944 &vcpu->arch.sie_block->gpsw, 945 sizeof(psw_t)); 946 kfree(inti); 947 } 948 949 return rc ? -EFAULT : 0; 950 } 951 952 typedef int (*deliver_irq_t)(struct kvm_vcpu *vcpu); 953 954 static const deliver_irq_t deliver_irq_funcs[] = { 955 [IRQ_PEND_MCHK_EX] = __deliver_machine_check, 956 [IRQ_PEND_MCHK_REP] = __deliver_machine_check, 957 [IRQ_PEND_PROG] = __deliver_prog, 958 [IRQ_PEND_EXT_EMERGENCY] = __deliver_emergency_signal, 959 [IRQ_PEND_EXT_EXTERNAL] = __deliver_external_call, 960 [IRQ_PEND_EXT_CLOCK_COMP] = __deliver_ckc, 961 [IRQ_PEND_EXT_CPU_TIMER] = __deliver_cpu_timer, 962 [IRQ_PEND_RESTART] = __deliver_restart, 963 [IRQ_PEND_SET_PREFIX] = __deliver_set_prefix, 964 [IRQ_PEND_PFAULT_INIT] = __deliver_pfault_init, 965 [IRQ_PEND_EXT_SERVICE] = __deliver_service, 966 [IRQ_PEND_PFAULT_DONE] = __deliver_pfault_done, 967 [IRQ_PEND_VIRTIO] = __deliver_virtio, 968 }; 969 970 /* Check whether an external call is pending (deliverable or not) */ 971 int kvm_s390_ext_call_pending(struct kvm_vcpu *vcpu) 972 { 973 struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int; 974 975 if (!sclp.has_sigpif) 976 return test_bit(IRQ_PEND_EXT_EXTERNAL, &li->pending_irqs); 977 978 return sca_ext_call_pending(vcpu, NULL); 979 } 980 981 int kvm_s390_vcpu_has_irq(struct kvm_vcpu *vcpu, int exclude_stop) 982 { 983 if (deliverable_irqs(vcpu)) 984 return 1; 985 986 if (kvm_cpu_has_pending_timer(vcpu)) 987 return 1; 988 989 /* external call pending and deliverable */ 990 if (kvm_s390_ext_call_pending(vcpu) && 991 !psw_extint_disabled(vcpu) && 992 (vcpu->arch.sie_block->gcr[0] & 0x2000ul)) 993 return 1; 994 995 if (!exclude_stop && kvm_s390_is_stop_irq_pending(vcpu)) 996 return 1; 997 return 0; 998 } 999 1000 int kvm_cpu_has_pending_timer(struct kvm_vcpu *vcpu) 1001 { 1002 return ckc_irq_pending(vcpu) || cpu_timer_irq_pending(vcpu); 1003 } 1004 1005 static u64 __calculate_sltime(struct kvm_vcpu *vcpu) 1006 { 1007 u64 now, cputm, sltime = 0; 1008 1009 if (ckc_interrupts_enabled(vcpu)) { 1010 now = kvm_s390_get_tod_clock_fast(vcpu->kvm); 1011 sltime = tod_to_ns(vcpu->arch.sie_block->ckc - now); 1012 /* already expired or overflow? */ 1013 if (!sltime || vcpu->arch.sie_block->ckc <= now) 1014 return 0; 1015 if (cpu_timer_interrupts_enabled(vcpu)) { 1016 cputm = kvm_s390_get_cpu_timer(vcpu); 1017 /* already expired? */ 1018 if (cputm >> 63) 1019 return 0; 1020 return min(sltime, tod_to_ns(cputm)); 1021 } 1022 } else if (cpu_timer_interrupts_enabled(vcpu)) { 1023 sltime = kvm_s390_get_cpu_timer(vcpu); 1024 /* already expired? */ 1025 if (sltime >> 63) 1026 return 0; 1027 } 1028 return sltime; 1029 } 1030 1031 int kvm_s390_handle_wait(struct kvm_vcpu *vcpu) 1032 { 1033 u64 sltime; 1034 1035 vcpu->stat.exit_wait_state++; 1036 1037 /* fast path */ 1038 if (kvm_arch_vcpu_runnable(vcpu)) 1039 return 0; 1040 1041 if (psw_interrupts_disabled(vcpu)) { 1042 VCPU_EVENT(vcpu, 3, "%s", "disabled wait"); 1043 return -EOPNOTSUPP; /* disabled wait */ 1044 } 1045 1046 if (!ckc_interrupts_enabled(vcpu) && 1047 !cpu_timer_interrupts_enabled(vcpu)) { 1048 VCPU_EVENT(vcpu, 3, "%s", "enabled wait w/o timer"); 1049 __set_cpu_idle(vcpu); 1050 goto no_timer; 1051 } 1052 1053 sltime = __calculate_sltime(vcpu); 1054 if (!sltime) 1055 return 0; 1056 1057 __set_cpu_idle(vcpu); 1058 hrtimer_start(&vcpu->arch.ckc_timer, sltime, HRTIMER_MODE_REL); 1059 VCPU_EVENT(vcpu, 4, "enabled wait: %llu ns", sltime); 1060 no_timer: 1061 srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx); 1062 kvm_vcpu_block(vcpu); 1063 __unset_cpu_idle(vcpu); 1064 vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu); 1065 1066 hrtimer_cancel(&vcpu->arch.ckc_timer); 1067 return 0; 1068 } 1069 1070 void kvm_s390_vcpu_wakeup(struct kvm_vcpu *vcpu) 1071 { 1072 /* 1073 * We cannot move this into the if, as the CPU might be already 1074 * in kvm_vcpu_block without having the waitqueue set (polling) 1075 */ 1076 vcpu->valid_wakeup = true; 1077 if (swait_active(&vcpu->wq)) { 1078 /* 1079 * The vcpu gave up the cpu voluntarily, mark it as a good 1080 * yield-candidate. 1081 */ 1082 vcpu->preempted = true; 1083 swake_up(&vcpu->wq); 1084 vcpu->stat.halt_wakeup++; 1085 } 1086 /* 1087 * The VCPU might not be sleeping but is executing the VSIE. Let's 1088 * kick it, so it leaves the SIE to process the request. 1089 */ 1090 kvm_s390_vsie_kick(vcpu); 1091 } 1092 1093 enum hrtimer_restart kvm_s390_idle_wakeup(struct hrtimer *timer) 1094 { 1095 struct kvm_vcpu *vcpu; 1096 u64 sltime; 1097 1098 vcpu = container_of(timer, struct kvm_vcpu, arch.ckc_timer); 1099 sltime = __calculate_sltime(vcpu); 1100 1101 /* 1102 * If the monotonic clock runs faster than the tod clock we might be 1103 * woken up too early and have to go back to sleep to avoid deadlocks. 1104 */ 1105 if (sltime && hrtimer_forward_now(timer, ns_to_ktime(sltime))) 1106 return HRTIMER_RESTART; 1107 kvm_s390_vcpu_wakeup(vcpu); 1108 return HRTIMER_NORESTART; 1109 } 1110 1111 void kvm_s390_clear_local_irqs(struct kvm_vcpu *vcpu) 1112 { 1113 struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int; 1114 1115 spin_lock(&li->lock); 1116 li->pending_irqs = 0; 1117 bitmap_zero(li->sigp_emerg_pending, KVM_MAX_VCPUS); 1118 memset(&li->irq, 0, sizeof(li->irq)); 1119 spin_unlock(&li->lock); 1120 1121 sca_clear_ext_call(vcpu); 1122 } 1123 1124 int __must_check kvm_s390_deliver_pending_interrupts(struct kvm_vcpu *vcpu) 1125 { 1126 struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int; 1127 deliver_irq_t func; 1128 int rc = 0; 1129 unsigned long irq_type; 1130 unsigned long irqs; 1131 1132 __reset_intercept_indicators(vcpu); 1133 1134 /* pending ckc conditions might have been invalidated */ 1135 clear_bit(IRQ_PEND_EXT_CLOCK_COMP, &li->pending_irqs); 1136 if (ckc_irq_pending(vcpu)) 1137 set_bit(IRQ_PEND_EXT_CLOCK_COMP, &li->pending_irqs); 1138 1139 /* pending cpu timer conditions might have been invalidated */ 1140 clear_bit(IRQ_PEND_EXT_CPU_TIMER, &li->pending_irqs); 1141 if (cpu_timer_irq_pending(vcpu)) 1142 set_bit(IRQ_PEND_EXT_CPU_TIMER, &li->pending_irqs); 1143 1144 while ((irqs = deliverable_irqs(vcpu)) && !rc) { 1145 /* bits are in the order of interrupt priority */ 1146 irq_type = find_first_bit(&irqs, IRQ_PEND_COUNT); 1147 if (is_ioirq(irq_type)) { 1148 rc = __deliver_io(vcpu, irq_type); 1149 } else { 1150 func = deliver_irq_funcs[irq_type]; 1151 if (!func) { 1152 WARN_ON_ONCE(func == NULL); 1153 clear_bit(irq_type, &li->pending_irqs); 1154 continue; 1155 } 1156 rc = func(vcpu); 1157 } 1158 } 1159 1160 set_intercept_indicators(vcpu); 1161 1162 return rc; 1163 } 1164 1165 static int __inject_prog(struct kvm_vcpu *vcpu, struct kvm_s390_irq *irq) 1166 { 1167 struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int; 1168 1169 VCPU_EVENT(vcpu, 3, "inject: program irq code 0x%x", irq->u.pgm.code); 1170 trace_kvm_s390_inject_vcpu(vcpu->vcpu_id, KVM_S390_PROGRAM_INT, 1171 irq->u.pgm.code, 0); 1172 1173 if (!(irq->u.pgm.flags & KVM_S390_PGM_FLAGS_ILC_VALID)) { 1174 /* auto detection if no valid ILC was given */ 1175 irq->u.pgm.flags &= ~KVM_S390_PGM_FLAGS_ILC_MASK; 1176 irq->u.pgm.flags |= kvm_s390_get_ilen(vcpu); 1177 irq->u.pgm.flags |= KVM_S390_PGM_FLAGS_ILC_VALID; 1178 } 1179 1180 if (irq->u.pgm.code == PGM_PER) { 1181 li->irq.pgm.code |= PGM_PER; 1182 li->irq.pgm.flags = irq->u.pgm.flags; 1183 /* only modify PER related information */ 1184 li->irq.pgm.per_address = irq->u.pgm.per_address; 1185 li->irq.pgm.per_code = irq->u.pgm.per_code; 1186 li->irq.pgm.per_atmid = irq->u.pgm.per_atmid; 1187 li->irq.pgm.per_access_id = irq->u.pgm.per_access_id; 1188 } else if (!(irq->u.pgm.code & PGM_PER)) { 1189 li->irq.pgm.code = (li->irq.pgm.code & PGM_PER) | 1190 irq->u.pgm.code; 1191 li->irq.pgm.flags = irq->u.pgm.flags; 1192 /* only modify non-PER information */ 1193 li->irq.pgm.trans_exc_code = irq->u.pgm.trans_exc_code; 1194 li->irq.pgm.mon_code = irq->u.pgm.mon_code; 1195 li->irq.pgm.data_exc_code = irq->u.pgm.data_exc_code; 1196 li->irq.pgm.mon_class_nr = irq->u.pgm.mon_class_nr; 1197 li->irq.pgm.exc_access_id = irq->u.pgm.exc_access_id; 1198 li->irq.pgm.op_access_id = irq->u.pgm.op_access_id; 1199 } else { 1200 li->irq.pgm = irq->u.pgm; 1201 } 1202 set_bit(IRQ_PEND_PROG, &li->pending_irqs); 1203 return 0; 1204 } 1205 1206 static int __inject_pfault_init(struct kvm_vcpu *vcpu, struct kvm_s390_irq *irq) 1207 { 1208 struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int; 1209 1210 VCPU_EVENT(vcpu, 4, "inject: pfault init parameter block at 0x%llx", 1211 irq->u.ext.ext_params2); 1212 trace_kvm_s390_inject_vcpu(vcpu->vcpu_id, KVM_S390_INT_PFAULT_INIT, 1213 irq->u.ext.ext_params, 1214 irq->u.ext.ext_params2); 1215 1216 li->irq.ext = irq->u.ext; 1217 set_bit(IRQ_PEND_PFAULT_INIT, &li->pending_irqs); 1218 atomic_or(CPUSTAT_EXT_INT, li->cpuflags); 1219 return 0; 1220 } 1221 1222 static int __inject_extcall(struct kvm_vcpu *vcpu, struct kvm_s390_irq *irq) 1223 { 1224 struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int; 1225 struct kvm_s390_extcall_info *extcall = &li->irq.extcall; 1226 uint16_t src_id = irq->u.extcall.code; 1227 1228 VCPU_EVENT(vcpu, 4, "inject: external call source-cpu:%u", 1229 src_id); 1230 trace_kvm_s390_inject_vcpu(vcpu->vcpu_id, KVM_S390_INT_EXTERNAL_CALL, 1231 src_id, 0); 1232 1233 /* sending vcpu invalid */ 1234 if (kvm_get_vcpu_by_id(vcpu->kvm, src_id) == NULL) 1235 return -EINVAL; 1236 1237 if (sclp.has_sigpif) 1238 return sca_inject_ext_call(vcpu, src_id); 1239 1240 if (test_and_set_bit(IRQ_PEND_EXT_EXTERNAL, &li->pending_irqs)) 1241 return -EBUSY; 1242 *extcall = irq->u.extcall; 1243 atomic_or(CPUSTAT_EXT_INT, li->cpuflags); 1244 return 0; 1245 } 1246 1247 static int __inject_set_prefix(struct kvm_vcpu *vcpu, struct kvm_s390_irq *irq) 1248 { 1249 struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int; 1250 struct kvm_s390_prefix_info *prefix = &li->irq.prefix; 1251 1252 VCPU_EVENT(vcpu, 3, "inject: set prefix to %x", 1253 irq->u.prefix.address); 1254 trace_kvm_s390_inject_vcpu(vcpu->vcpu_id, KVM_S390_SIGP_SET_PREFIX, 1255 irq->u.prefix.address, 0); 1256 1257 if (!is_vcpu_stopped(vcpu)) 1258 return -EBUSY; 1259 1260 *prefix = irq->u.prefix; 1261 set_bit(IRQ_PEND_SET_PREFIX, &li->pending_irqs); 1262 return 0; 1263 } 1264 1265 #define KVM_S390_STOP_SUPP_FLAGS (KVM_S390_STOP_FLAG_STORE_STATUS) 1266 static int __inject_sigp_stop(struct kvm_vcpu *vcpu, struct kvm_s390_irq *irq) 1267 { 1268 struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int; 1269 struct kvm_s390_stop_info *stop = &li->irq.stop; 1270 int rc = 0; 1271 1272 trace_kvm_s390_inject_vcpu(vcpu->vcpu_id, KVM_S390_SIGP_STOP, 0, 0); 1273 1274 if (irq->u.stop.flags & ~KVM_S390_STOP_SUPP_FLAGS) 1275 return -EINVAL; 1276 1277 if (is_vcpu_stopped(vcpu)) { 1278 if (irq->u.stop.flags & KVM_S390_STOP_FLAG_STORE_STATUS) 1279 rc = kvm_s390_store_status_unloaded(vcpu, 1280 KVM_S390_STORE_STATUS_NOADDR); 1281 return rc; 1282 } 1283 1284 if (test_and_set_bit(IRQ_PEND_SIGP_STOP, &li->pending_irqs)) 1285 return -EBUSY; 1286 stop->flags = irq->u.stop.flags; 1287 __set_cpuflag(vcpu, CPUSTAT_STOP_INT); 1288 return 0; 1289 } 1290 1291 static int __inject_sigp_restart(struct kvm_vcpu *vcpu, 1292 struct kvm_s390_irq *irq) 1293 { 1294 struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int; 1295 1296 VCPU_EVENT(vcpu, 3, "%s", "inject: restart int"); 1297 trace_kvm_s390_inject_vcpu(vcpu->vcpu_id, KVM_S390_RESTART, 0, 0); 1298 1299 set_bit(IRQ_PEND_RESTART, &li->pending_irqs); 1300 return 0; 1301 } 1302 1303 static int __inject_sigp_emergency(struct kvm_vcpu *vcpu, 1304 struct kvm_s390_irq *irq) 1305 { 1306 struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int; 1307 1308 VCPU_EVENT(vcpu, 4, "inject: emergency from cpu %u", 1309 irq->u.emerg.code); 1310 trace_kvm_s390_inject_vcpu(vcpu->vcpu_id, KVM_S390_INT_EMERGENCY, 1311 irq->u.emerg.code, 0); 1312 1313 /* sending vcpu invalid */ 1314 if (kvm_get_vcpu_by_id(vcpu->kvm, irq->u.emerg.code) == NULL) 1315 return -EINVAL; 1316 1317 set_bit(irq->u.emerg.code, li->sigp_emerg_pending); 1318 set_bit(IRQ_PEND_EXT_EMERGENCY, &li->pending_irqs); 1319 atomic_or(CPUSTAT_EXT_INT, li->cpuflags); 1320 return 0; 1321 } 1322 1323 static int __inject_mchk(struct kvm_vcpu *vcpu, struct kvm_s390_irq *irq) 1324 { 1325 struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int; 1326 struct kvm_s390_mchk_info *mchk = &li->irq.mchk; 1327 1328 VCPU_EVENT(vcpu, 3, "inject: machine check mcic 0x%llx", 1329 irq->u.mchk.mcic); 1330 trace_kvm_s390_inject_vcpu(vcpu->vcpu_id, KVM_S390_MCHK, 0, 1331 irq->u.mchk.mcic); 1332 1333 /* 1334 * Because repressible machine checks can be indicated along with 1335 * exigent machine checks (PoP, Chapter 11, Interruption action) 1336 * we need to combine cr14, mcic and external damage code. 1337 * Failing storage address and the logout area should not be or'ed 1338 * together, we just indicate the last occurrence of the corresponding 1339 * machine check 1340 */ 1341 mchk->cr14 |= irq->u.mchk.cr14; 1342 mchk->mcic |= irq->u.mchk.mcic; 1343 mchk->ext_damage_code |= irq->u.mchk.ext_damage_code; 1344 mchk->failing_storage_address = irq->u.mchk.failing_storage_address; 1345 memcpy(&mchk->fixed_logout, &irq->u.mchk.fixed_logout, 1346 sizeof(mchk->fixed_logout)); 1347 if (mchk->mcic & MCHK_EX_MASK) 1348 set_bit(IRQ_PEND_MCHK_EX, &li->pending_irqs); 1349 else if (mchk->mcic & MCHK_REP_MASK) 1350 set_bit(IRQ_PEND_MCHK_REP, &li->pending_irqs); 1351 return 0; 1352 } 1353 1354 static int __inject_ckc(struct kvm_vcpu *vcpu) 1355 { 1356 struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int; 1357 1358 VCPU_EVENT(vcpu, 3, "%s", "inject: clock comparator external"); 1359 trace_kvm_s390_inject_vcpu(vcpu->vcpu_id, KVM_S390_INT_CLOCK_COMP, 1360 0, 0); 1361 1362 set_bit(IRQ_PEND_EXT_CLOCK_COMP, &li->pending_irqs); 1363 atomic_or(CPUSTAT_EXT_INT, li->cpuflags); 1364 return 0; 1365 } 1366 1367 static int __inject_cpu_timer(struct kvm_vcpu *vcpu) 1368 { 1369 struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int; 1370 1371 VCPU_EVENT(vcpu, 3, "%s", "inject: cpu timer external"); 1372 trace_kvm_s390_inject_vcpu(vcpu->vcpu_id, KVM_S390_INT_CPU_TIMER, 1373 0, 0); 1374 1375 set_bit(IRQ_PEND_EXT_CPU_TIMER, &li->pending_irqs); 1376 atomic_or(CPUSTAT_EXT_INT, li->cpuflags); 1377 return 0; 1378 } 1379 1380 static struct kvm_s390_interrupt_info *get_io_int(struct kvm *kvm, 1381 int isc, u32 schid) 1382 { 1383 struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int; 1384 struct list_head *isc_list = &fi->lists[FIRQ_LIST_IO_ISC_0 + isc]; 1385 struct kvm_s390_interrupt_info *iter; 1386 u16 id = (schid & 0xffff0000U) >> 16; 1387 u16 nr = schid & 0x0000ffffU; 1388 1389 spin_lock(&fi->lock); 1390 list_for_each_entry(iter, isc_list, list) { 1391 if (schid && (id != iter->io.subchannel_id || 1392 nr != iter->io.subchannel_nr)) 1393 continue; 1394 /* found an appropriate entry */ 1395 list_del_init(&iter->list); 1396 fi->counters[FIRQ_CNTR_IO] -= 1; 1397 if (list_empty(isc_list)) 1398 clear_bit(IRQ_PEND_IO_ISC_0 + isc, &fi->pending_irqs); 1399 spin_unlock(&fi->lock); 1400 return iter; 1401 } 1402 spin_unlock(&fi->lock); 1403 return NULL; 1404 } 1405 1406 /* 1407 * Dequeue and return an I/O interrupt matching any of the interruption 1408 * subclasses as designated by the isc mask in cr6 and the schid (if != 0). 1409 */ 1410 struct kvm_s390_interrupt_info *kvm_s390_get_io_int(struct kvm *kvm, 1411 u64 isc_mask, u32 schid) 1412 { 1413 struct kvm_s390_interrupt_info *inti = NULL; 1414 int isc; 1415 1416 for (isc = 0; isc <= MAX_ISC && !inti; isc++) { 1417 if (isc_mask & isc_to_isc_bits(isc)) 1418 inti = get_io_int(kvm, isc, schid); 1419 } 1420 return inti; 1421 } 1422 1423 #define SCCB_MASK 0xFFFFFFF8 1424 #define SCCB_EVENT_PENDING 0x3 1425 1426 static int __inject_service(struct kvm *kvm, 1427 struct kvm_s390_interrupt_info *inti) 1428 { 1429 struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int; 1430 1431 spin_lock(&fi->lock); 1432 fi->srv_signal.ext_params |= inti->ext.ext_params & SCCB_EVENT_PENDING; 1433 /* 1434 * Early versions of the QEMU s390 bios will inject several 1435 * service interrupts after another without handling a 1436 * condition code indicating busy. 1437 * We will silently ignore those superfluous sccb values. 1438 * A future version of QEMU will take care of serialization 1439 * of servc requests 1440 */ 1441 if (fi->srv_signal.ext_params & SCCB_MASK) 1442 goto out; 1443 fi->srv_signal.ext_params |= inti->ext.ext_params & SCCB_MASK; 1444 set_bit(IRQ_PEND_EXT_SERVICE, &fi->pending_irqs); 1445 out: 1446 spin_unlock(&fi->lock); 1447 kfree(inti); 1448 return 0; 1449 } 1450 1451 static int __inject_virtio(struct kvm *kvm, 1452 struct kvm_s390_interrupt_info *inti) 1453 { 1454 struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int; 1455 1456 spin_lock(&fi->lock); 1457 if (fi->counters[FIRQ_CNTR_VIRTIO] >= KVM_S390_MAX_VIRTIO_IRQS) { 1458 spin_unlock(&fi->lock); 1459 return -EBUSY; 1460 } 1461 fi->counters[FIRQ_CNTR_VIRTIO] += 1; 1462 list_add_tail(&inti->list, &fi->lists[FIRQ_LIST_VIRTIO]); 1463 set_bit(IRQ_PEND_VIRTIO, &fi->pending_irqs); 1464 spin_unlock(&fi->lock); 1465 return 0; 1466 } 1467 1468 static int __inject_pfault_done(struct kvm *kvm, 1469 struct kvm_s390_interrupt_info *inti) 1470 { 1471 struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int; 1472 1473 spin_lock(&fi->lock); 1474 if (fi->counters[FIRQ_CNTR_PFAULT] >= 1475 (ASYNC_PF_PER_VCPU * KVM_MAX_VCPUS)) { 1476 spin_unlock(&fi->lock); 1477 return -EBUSY; 1478 } 1479 fi->counters[FIRQ_CNTR_PFAULT] += 1; 1480 list_add_tail(&inti->list, &fi->lists[FIRQ_LIST_PFAULT]); 1481 set_bit(IRQ_PEND_PFAULT_DONE, &fi->pending_irqs); 1482 spin_unlock(&fi->lock); 1483 return 0; 1484 } 1485 1486 #define CR_PENDING_SUBCLASS 28 1487 static int __inject_float_mchk(struct kvm *kvm, 1488 struct kvm_s390_interrupt_info *inti) 1489 { 1490 struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int; 1491 1492 spin_lock(&fi->lock); 1493 fi->mchk.cr14 |= inti->mchk.cr14 & (1UL << CR_PENDING_SUBCLASS); 1494 fi->mchk.mcic |= inti->mchk.mcic; 1495 set_bit(IRQ_PEND_MCHK_REP, &fi->pending_irqs); 1496 spin_unlock(&fi->lock); 1497 kfree(inti); 1498 return 0; 1499 } 1500 1501 static int __inject_io(struct kvm *kvm, struct kvm_s390_interrupt_info *inti) 1502 { 1503 struct kvm_s390_float_interrupt *fi; 1504 struct list_head *list; 1505 int isc; 1506 1507 fi = &kvm->arch.float_int; 1508 spin_lock(&fi->lock); 1509 if (fi->counters[FIRQ_CNTR_IO] >= KVM_S390_MAX_FLOAT_IRQS) { 1510 spin_unlock(&fi->lock); 1511 return -EBUSY; 1512 } 1513 fi->counters[FIRQ_CNTR_IO] += 1; 1514 1515 if (inti->type & KVM_S390_INT_IO_AI_MASK) 1516 VM_EVENT(kvm, 4, "%s", "inject: I/O (AI)"); 1517 else 1518 VM_EVENT(kvm, 4, "inject: I/O %x ss %x schid %04x", 1519 inti->io.subchannel_id >> 8, 1520 inti->io.subchannel_id >> 1 & 0x3, 1521 inti->io.subchannel_nr); 1522 isc = int_word_to_isc(inti->io.io_int_word); 1523 list = &fi->lists[FIRQ_LIST_IO_ISC_0 + isc]; 1524 list_add_tail(&inti->list, list); 1525 set_bit(IRQ_PEND_IO_ISC_0 + isc, &fi->pending_irqs); 1526 spin_unlock(&fi->lock); 1527 return 0; 1528 } 1529 1530 /* 1531 * Find a destination VCPU for a floating irq and kick it. 1532 */ 1533 static void __floating_irq_kick(struct kvm *kvm, u64 type) 1534 { 1535 struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int; 1536 struct kvm_s390_local_interrupt *li; 1537 struct kvm_vcpu *dst_vcpu; 1538 int sigcpu, online_vcpus, nr_tries = 0; 1539 1540 online_vcpus = atomic_read(&kvm->online_vcpus); 1541 if (!online_vcpus) 1542 return; 1543 1544 /* find idle VCPUs first, then round robin */ 1545 sigcpu = find_first_bit(fi->idle_mask, online_vcpus); 1546 if (sigcpu == online_vcpus) { 1547 do { 1548 sigcpu = fi->next_rr_cpu; 1549 fi->next_rr_cpu = (fi->next_rr_cpu + 1) % online_vcpus; 1550 /* avoid endless loops if all vcpus are stopped */ 1551 if (nr_tries++ >= online_vcpus) 1552 return; 1553 } while (is_vcpu_stopped(kvm_get_vcpu(kvm, sigcpu))); 1554 } 1555 dst_vcpu = kvm_get_vcpu(kvm, sigcpu); 1556 1557 /* make the VCPU drop out of the SIE, or wake it up if sleeping */ 1558 li = &dst_vcpu->arch.local_int; 1559 spin_lock(&li->lock); 1560 switch (type) { 1561 case KVM_S390_MCHK: 1562 atomic_or(CPUSTAT_STOP_INT, li->cpuflags); 1563 break; 1564 case KVM_S390_INT_IO_MIN...KVM_S390_INT_IO_MAX: 1565 atomic_or(CPUSTAT_IO_INT, li->cpuflags); 1566 break; 1567 default: 1568 atomic_or(CPUSTAT_EXT_INT, li->cpuflags); 1569 break; 1570 } 1571 spin_unlock(&li->lock); 1572 kvm_s390_vcpu_wakeup(dst_vcpu); 1573 } 1574 1575 static int __inject_vm(struct kvm *kvm, struct kvm_s390_interrupt_info *inti) 1576 { 1577 u64 type = READ_ONCE(inti->type); 1578 int rc; 1579 1580 switch (type) { 1581 case KVM_S390_MCHK: 1582 rc = __inject_float_mchk(kvm, inti); 1583 break; 1584 case KVM_S390_INT_VIRTIO: 1585 rc = __inject_virtio(kvm, inti); 1586 break; 1587 case KVM_S390_INT_SERVICE: 1588 rc = __inject_service(kvm, inti); 1589 break; 1590 case KVM_S390_INT_PFAULT_DONE: 1591 rc = __inject_pfault_done(kvm, inti); 1592 break; 1593 case KVM_S390_INT_IO_MIN...KVM_S390_INT_IO_MAX: 1594 rc = __inject_io(kvm, inti); 1595 break; 1596 default: 1597 rc = -EINVAL; 1598 } 1599 if (rc) 1600 return rc; 1601 1602 __floating_irq_kick(kvm, type); 1603 return 0; 1604 } 1605 1606 int kvm_s390_inject_vm(struct kvm *kvm, 1607 struct kvm_s390_interrupt *s390int) 1608 { 1609 struct kvm_s390_interrupt_info *inti; 1610 int rc; 1611 1612 inti = kzalloc(sizeof(*inti), GFP_KERNEL); 1613 if (!inti) 1614 return -ENOMEM; 1615 1616 inti->type = s390int->type; 1617 switch (inti->type) { 1618 case KVM_S390_INT_VIRTIO: 1619 VM_EVENT(kvm, 5, "inject: virtio parm:%x,parm64:%llx", 1620 s390int->parm, s390int->parm64); 1621 inti->ext.ext_params = s390int->parm; 1622 inti->ext.ext_params2 = s390int->parm64; 1623 break; 1624 case KVM_S390_INT_SERVICE: 1625 VM_EVENT(kvm, 4, "inject: sclp parm:%x", s390int->parm); 1626 inti->ext.ext_params = s390int->parm; 1627 break; 1628 case KVM_S390_INT_PFAULT_DONE: 1629 inti->ext.ext_params2 = s390int->parm64; 1630 break; 1631 case KVM_S390_MCHK: 1632 VM_EVENT(kvm, 3, "inject: machine check mcic 0x%llx", 1633 s390int->parm64); 1634 inti->mchk.cr14 = s390int->parm; /* upper bits are not used */ 1635 inti->mchk.mcic = s390int->parm64; 1636 break; 1637 case KVM_S390_INT_IO_MIN...KVM_S390_INT_IO_MAX: 1638 inti->io.subchannel_id = s390int->parm >> 16; 1639 inti->io.subchannel_nr = s390int->parm & 0x0000ffffu; 1640 inti->io.io_int_parm = s390int->parm64 >> 32; 1641 inti->io.io_int_word = s390int->parm64 & 0x00000000ffffffffull; 1642 break; 1643 default: 1644 kfree(inti); 1645 return -EINVAL; 1646 } 1647 trace_kvm_s390_inject_vm(s390int->type, s390int->parm, s390int->parm64, 1648 2); 1649 1650 rc = __inject_vm(kvm, inti); 1651 if (rc) 1652 kfree(inti); 1653 return rc; 1654 } 1655 1656 int kvm_s390_reinject_io_int(struct kvm *kvm, 1657 struct kvm_s390_interrupt_info *inti) 1658 { 1659 return __inject_vm(kvm, inti); 1660 } 1661 1662 int s390int_to_s390irq(struct kvm_s390_interrupt *s390int, 1663 struct kvm_s390_irq *irq) 1664 { 1665 irq->type = s390int->type; 1666 switch (irq->type) { 1667 case KVM_S390_PROGRAM_INT: 1668 if (s390int->parm & 0xffff0000) 1669 return -EINVAL; 1670 irq->u.pgm.code = s390int->parm; 1671 break; 1672 case KVM_S390_SIGP_SET_PREFIX: 1673 irq->u.prefix.address = s390int->parm; 1674 break; 1675 case KVM_S390_SIGP_STOP: 1676 irq->u.stop.flags = s390int->parm; 1677 break; 1678 case KVM_S390_INT_EXTERNAL_CALL: 1679 if (s390int->parm & 0xffff0000) 1680 return -EINVAL; 1681 irq->u.extcall.code = s390int->parm; 1682 break; 1683 case KVM_S390_INT_EMERGENCY: 1684 if (s390int->parm & 0xffff0000) 1685 return -EINVAL; 1686 irq->u.emerg.code = s390int->parm; 1687 break; 1688 case KVM_S390_MCHK: 1689 irq->u.mchk.mcic = s390int->parm64; 1690 break; 1691 } 1692 return 0; 1693 } 1694 1695 int kvm_s390_is_stop_irq_pending(struct kvm_vcpu *vcpu) 1696 { 1697 struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int; 1698 1699 return test_bit(IRQ_PEND_SIGP_STOP, &li->pending_irqs); 1700 } 1701 1702 void kvm_s390_clear_stop_irq(struct kvm_vcpu *vcpu) 1703 { 1704 struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int; 1705 1706 spin_lock(&li->lock); 1707 li->irq.stop.flags = 0; 1708 clear_bit(IRQ_PEND_SIGP_STOP, &li->pending_irqs); 1709 spin_unlock(&li->lock); 1710 } 1711 1712 static int do_inject_vcpu(struct kvm_vcpu *vcpu, struct kvm_s390_irq *irq) 1713 { 1714 int rc; 1715 1716 switch (irq->type) { 1717 case KVM_S390_PROGRAM_INT: 1718 rc = __inject_prog(vcpu, irq); 1719 break; 1720 case KVM_S390_SIGP_SET_PREFIX: 1721 rc = __inject_set_prefix(vcpu, irq); 1722 break; 1723 case KVM_S390_SIGP_STOP: 1724 rc = __inject_sigp_stop(vcpu, irq); 1725 break; 1726 case KVM_S390_RESTART: 1727 rc = __inject_sigp_restart(vcpu, irq); 1728 break; 1729 case KVM_S390_INT_CLOCK_COMP: 1730 rc = __inject_ckc(vcpu); 1731 break; 1732 case KVM_S390_INT_CPU_TIMER: 1733 rc = __inject_cpu_timer(vcpu); 1734 break; 1735 case KVM_S390_INT_EXTERNAL_CALL: 1736 rc = __inject_extcall(vcpu, irq); 1737 break; 1738 case KVM_S390_INT_EMERGENCY: 1739 rc = __inject_sigp_emergency(vcpu, irq); 1740 break; 1741 case KVM_S390_MCHK: 1742 rc = __inject_mchk(vcpu, irq); 1743 break; 1744 case KVM_S390_INT_PFAULT_INIT: 1745 rc = __inject_pfault_init(vcpu, irq); 1746 break; 1747 case KVM_S390_INT_VIRTIO: 1748 case KVM_S390_INT_SERVICE: 1749 case KVM_S390_INT_IO_MIN...KVM_S390_INT_IO_MAX: 1750 default: 1751 rc = -EINVAL; 1752 } 1753 1754 return rc; 1755 } 1756 1757 int kvm_s390_inject_vcpu(struct kvm_vcpu *vcpu, struct kvm_s390_irq *irq) 1758 { 1759 struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int; 1760 int rc; 1761 1762 spin_lock(&li->lock); 1763 rc = do_inject_vcpu(vcpu, irq); 1764 spin_unlock(&li->lock); 1765 if (!rc) 1766 kvm_s390_vcpu_wakeup(vcpu); 1767 return rc; 1768 } 1769 1770 static inline void clear_irq_list(struct list_head *_list) 1771 { 1772 struct kvm_s390_interrupt_info *inti, *n; 1773 1774 list_for_each_entry_safe(inti, n, _list, list) { 1775 list_del(&inti->list); 1776 kfree(inti); 1777 } 1778 } 1779 1780 static void inti_to_irq(struct kvm_s390_interrupt_info *inti, 1781 struct kvm_s390_irq *irq) 1782 { 1783 irq->type = inti->type; 1784 switch (inti->type) { 1785 case KVM_S390_INT_PFAULT_INIT: 1786 case KVM_S390_INT_PFAULT_DONE: 1787 case KVM_S390_INT_VIRTIO: 1788 irq->u.ext = inti->ext; 1789 break; 1790 case KVM_S390_INT_IO_MIN...KVM_S390_INT_IO_MAX: 1791 irq->u.io = inti->io; 1792 break; 1793 } 1794 } 1795 1796 void kvm_s390_clear_float_irqs(struct kvm *kvm) 1797 { 1798 struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int; 1799 int i; 1800 1801 spin_lock(&fi->lock); 1802 fi->pending_irqs = 0; 1803 memset(&fi->srv_signal, 0, sizeof(fi->srv_signal)); 1804 memset(&fi->mchk, 0, sizeof(fi->mchk)); 1805 for (i = 0; i < FIRQ_LIST_COUNT; i++) 1806 clear_irq_list(&fi->lists[i]); 1807 for (i = 0; i < FIRQ_MAX_COUNT; i++) 1808 fi->counters[i] = 0; 1809 spin_unlock(&fi->lock); 1810 }; 1811 1812 static int get_all_floating_irqs(struct kvm *kvm, u8 __user *usrbuf, u64 len) 1813 { 1814 struct kvm_s390_interrupt_info *inti; 1815 struct kvm_s390_float_interrupt *fi; 1816 struct kvm_s390_irq *buf; 1817 struct kvm_s390_irq *irq; 1818 int max_irqs; 1819 int ret = 0; 1820 int n = 0; 1821 int i; 1822 1823 if (len > KVM_S390_FLIC_MAX_BUFFER || len == 0) 1824 return -EINVAL; 1825 1826 /* 1827 * We are already using -ENOMEM to signal 1828 * userspace it may retry with a bigger buffer, 1829 * so we need to use something else for this case 1830 */ 1831 buf = vzalloc(len); 1832 if (!buf) 1833 return -ENOBUFS; 1834 1835 max_irqs = len / sizeof(struct kvm_s390_irq); 1836 1837 fi = &kvm->arch.float_int; 1838 spin_lock(&fi->lock); 1839 for (i = 0; i < FIRQ_LIST_COUNT; i++) { 1840 list_for_each_entry(inti, &fi->lists[i], list) { 1841 if (n == max_irqs) { 1842 /* signal userspace to try again */ 1843 ret = -ENOMEM; 1844 goto out; 1845 } 1846 inti_to_irq(inti, &buf[n]); 1847 n++; 1848 } 1849 } 1850 if (test_bit(IRQ_PEND_EXT_SERVICE, &fi->pending_irqs)) { 1851 if (n == max_irqs) { 1852 /* signal userspace to try again */ 1853 ret = -ENOMEM; 1854 goto out; 1855 } 1856 irq = (struct kvm_s390_irq *) &buf[n]; 1857 irq->type = KVM_S390_INT_SERVICE; 1858 irq->u.ext = fi->srv_signal; 1859 n++; 1860 } 1861 if (test_bit(IRQ_PEND_MCHK_REP, &fi->pending_irqs)) { 1862 if (n == max_irqs) { 1863 /* signal userspace to try again */ 1864 ret = -ENOMEM; 1865 goto out; 1866 } 1867 irq = (struct kvm_s390_irq *) &buf[n]; 1868 irq->type = KVM_S390_MCHK; 1869 irq->u.mchk = fi->mchk; 1870 n++; 1871 } 1872 1873 out: 1874 spin_unlock(&fi->lock); 1875 if (!ret && n > 0) { 1876 if (copy_to_user(usrbuf, buf, sizeof(struct kvm_s390_irq) * n)) 1877 ret = -EFAULT; 1878 } 1879 vfree(buf); 1880 1881 return ret < 0 ? ret : n; 1882 } 1883 1884 static int flic_ais_mode_get_all(struct kvm *kvm, struct kvm_device_attr *attr) 1885 { 1886 struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int; 1887 struct kvm_s390_ais_all ais; 1888 1889 if (attr->attr < sizeof(ais)) 1890 return -EINVAL; 1891 1892 if (!test_kvm_facility(kvm, 72)) 1893 return -ENOTSUPP; 1894 1895 mutex_lock(&fi->ais_lock); 1896 ais.simm = fi->simm; 1897 ais.nimm = fi->nimm; 1898 mutex_unlock(&fi->ais_lock); 1899 1900 if (copy_to_user((void __user *)attr->addr, &ais, sizeof(ais))) 1901 return -EFAULT; 1902 1903 return 0; 1904 } 1905 1906 static int flic_get_attr(struct kvm_device *dev, struct kvm_device_attr *attr) 1907 { 1908 int r; 1909 1910 switch (attr->group) { 1911 case KVM_DEV_FLIC_GET_ALL_IRQS: 1912 r = get_all_floating_irqs(dev->kvm, (u8 __user *) attr->addr, 1913 attr->attr); 1914 break; 1915 case KVM_DEV_FLIC_AISM_ALL: 1916 r = flic_ais_mode_get_all(dev->kvm, attr); 1917 break; 1918 default: 1919 r = -EINVAL; 1920 } 1921 1922 return r; 1923 } 1924 1925 static inline int copy_irq_from_user(struct kvm_s390_interrupt_info *inti, 1926 u64 addr) 1927 { 1928 struct kvm_s390_irq __user *uptr = (struct kvm_s390_irq __user *) addr; 1929 void *target = NULL; 1930 void __user *source; 1931 u64 size; 1932 1933 if (get_user(inti->type, (u64 __user *)addr)) 1934 return -EFAULT; 1935 1936 switch (inti->type) { 1937 case KVM_S390_INT_PFAULT_INIT: 1938 case KVM_S390_INT_PFAULT_DONE: 1939 case KVM_S390_INT_VIRTIO: 1940 case KVM_S390_INT_SERVICE: 1941 target = (void *) &inti->ext; 1942 source = &uptr->u.ext; 1943 size = sizeof(inti->ext); 1944 break; 1945 case KVM_S390_INT_IO_MIN...KVM_S390_INT_IO_MAX: 1946 target = (void *) &inti->io; 1947 source = &uptr->u.io; 1948 size = sizeof(inti->io); 1949 break; 1950 case KVM_S390_MCHK: 1951 target = (void *) &inti->mchk; 1952 source = &uptr->u.mchk; 1953 size = sizeof(inti->mchk); 1954 break; 1955 default: 1956 return -EINVAL; 1957 } 1958 1959 if (copy_from_user(target, source, size)) 1960 return -EFAULT; 1961 1962 return 0; 1963 } 1964 1965 static int enqueue_floating_irq(struct kvm_device *dev, 1966 struct kvm_device_attr *attr) 1967 { 1968 struct kvm_s390_interrupt_info *inti = NULL; 1969 int r = 0; 1970 int len = attr->attr; 1971 1972 if (len % sizeof(struct kvm_s390_irq) != 0) 1973 return -EINVAL; 1974 else if (len > KVM_S390_FLIC_MAX_BUFFER) 1975 return -EINVAL; 1976 1977 while (len >= sizeof(struct kvm_s390_irq)) { 1978 inti = kzalloc(sizeof(*inti), GFP_KERNEL); 1979 if (!inti) 1980 return -ENOMEM; 1981 1982 r = copy_irq_from_user(inti, attr->addr); 1983 if (r) { 1984 kfree(inti); 1985 return r; 1986 } 1987 r = __inject_vm(dev->kvm, inti); 1988 if (r) { 1989 kfree(inti); 1990 return r; 1991 } 1992 len -= sizeof(struct kvm_s390_irq); 1993 attr->addr += sizeof(struct kvm_s390_irq); 1994 } 1995 1996 return r; 1997 } 1998 1999 static struct s390_io_adapter *get_io_adapter(struct kvm *kvm, unsigned int id) 2000 { 2001 if (id >= MAX_S390_IO_ADAPTERS) 2002 return NULL; 2003 return kvm->arch.adapters[id]; 2004 } 2005 2006 static int register_io_adapter(struct kvm_device *dev, 2007 struct kvm_device_attr *attr) 2008 { 2009 struct s390_io_adapter *adapter; 2010 struct kvm_s390_io_adapter adapter_info; 2011 2012 if (copy_from_user(&adapter_info, 2013 (void __user *)attr->addr, sizeof(adapter_info))) 2014 return -EFAULT; 2015 2016 if ((adapter_info.id >= MAX_S390_IO_ADAPTERS) || 2017 (dev->kvm->arch.adapters[adapter_info.id] != NULL)) 2018 return -EINVAL; 2019 2020 adapter = kzalloc(sizeof(*adapter), GFP_KERNEL); 2021 if (!adapter) 2022 return -ENOMEM; 2023 2024 INIT_LIST_HEAD(&adapter->maps); 2025 init_rwsem(&adapter->maps_lock); 2026 atomic_set(&adapter->nr_maps, 0); 2027 adapter->id = adapter_info.id; 2028 adapter->isc = adapter_info.isc; 2029 adapter->maskable = adapter_info.maskable; 2030 adapter->masked = false; 2031 adapter->swap = adapter_info.swap; 2032 adapter->suppressible = (adapter_info.flags) & 2033 KVM_S390_ADAPTER_SUPPRESSIBLE; 2034 dev->kvm->arch.adapters[adapter->id] = adapter; 2035 2036 return 0; 2037 } 2038 2039 int kvm_s390_mask_adapter(struct kvm *kvm, unsigned int id, bool masked) 2040 { 2041 int ret; 2042 struct s390_io_adapter *adapter = get_io_adapter(kvm, id); 2043 2044 if (!adapter || !adapter->maskable) 2045 return -EINVAL; 2046 ret = adapter->masked; 2047 adapter->masked = masked; 2048 return ret; 2049 } 2050 2051 static int kvm_s390_adapter_map(struct kvm *kvm, unsigned int id, __u64 addr) 2052 { 2053 struct s390_io_adapter *adapter = get_io_adapter(kvm, id); 2054 struct s390_map_info *map; 2055 int ret; 2056 2057 if (!adapter || !addr) 2058 return -EINVAL; 2059 2060 map = kzalloc(sizeof(*map), GFP_KERNEL); 2061 if (!map) { 2062 ret = -ENOMEM; 2063 goto out; 2064 } 2065 INIT_LIST_HEAD(&map->list); 2066 map->guest_addr = addr; 2067 map->addr = gmap_translate(kvm->arch.gmap, addr); 2068 if (map->addr == -EFAULT) { 2069 ret = -EFAULT; 2070 goto out; 2071 } 2072 ret = get_user_pages_fast(map->addr, 1, 1, &map->page); 2073 if (ret < 0) 2074 goto out; 2075 BUG_ON(ret != 1); 2076 down_write(&adapter->maps_lock); 2077 if (atomic_inc_return(&adapter->nr_maps) < MAX_S390_ADAPTER_MAPS) { 2078 list_add_tail(&map->list, &adapter->maps); 2079 ret = 0; 2080 } else { 2081 put_page(map->page); 2082 ret = -EINVAL; 2083 } 2084 up_write(&adapter->maps_lock); 2085 out: 2086 if (ret) 2087 kfree(map); 2088 return ret; 2089 } 2090 2091 static int kvm_s390_adapter_unmap(struct kvm *kvm, unsigned int id, __u64 addr) 2092 { 2093 struct s390_io_adapter *adapter = get_io_adapter(kvm, id); 2094 struct s390_map_info *map, *tmp; 2095 int found = 0; 2096 2097 if (!adapter || !addr) 2098 return -EINVAL; 2099 2100 down_write(&adapter->maps_lock); 2101 list_for_each_entry_safe(map, tmp, &adapter->maps, list) { 2102 if (map->guest_addr == addr) { 2103 found = 1; 2104 atomic_dec(&adapter->nr_maps); 2105 list_del(&map->list); 2106 put_page(map->page); 2107 kfree(map); 2108 break; 2109 } 2110 } 2111 up_write(&adapter->maps_lock); 2112 2113 return found ? 0 : -EINVAL; 2114 } 2115 2116 void kvm_s390_destroy_adapters(struct kvm *kvm) 2117 { 2118 int i; 2119 struct s390_map_info *map, *tmp; 2120 2121 for (i = 0; i < MAX_S390_IO_ADAPTERS; i++) { 2122 if (!kvm->arch.adapters[i]) 2123 continue; 2124 list_for_each_entry_safe(map, tmp, 2125 &kvm->arch.adapters[i]->maps, list) { 2126 list_del(&map->list); 2127 put_page(map->page); 2128 kfree(map); 2129 } 2130 kfree(kvm->arch.adapters[i]); 2131 } 2132 } 2133 2134 static int modify_io_adapter(struct kvm_device *dev, 2135 struct kvm_device_attr *attr) 2136 { 2137 struct kvm_s390_io_adapter_req req; 2138 struct s390_io_adapter *adapter; 2139 int ret; 2140 2141 if (copy_from_user(&req, (void __user *)attr->addr, sizeof(req))) 2142 return -EFAULT; 2143 2144 adapter = get_io_adapter(dev->kvm, req.id); 2145 if (!adapter) 2146 return -EINVAL; 2147 switch (req.type) { 2148 case KVM_S390_IO_ADAPTER_MASK: 2149 ret = kvm_s390_mask_adapter(dev->kvm, req.id, req.mask); 2150 if (ret > 0) 2151 ret = 0; 2152 break; 2153 case KVM_S390_IO_ADAPTER_MAP: 2154 ret = kvm_s390_adapter_map(dev->kvm, req.id, req.addr); 2155 break; 2156 case KVM_S390_IO_ADAPTER_UNMAP: 2157 ret = kvm_s390_adapter_unmap(dev->kvm, req.id, req.addr); 2158 break; 2159 default: 2160 ret = -EINVAL; 2161 } 2162 2163 return ret; 2164 } 2165 2166 static int clear_io_irq(struct kvm *kvm, struct kvm_device_attr *attr) 2167 2168 { 2169 const u64 isc_mask = 0xffUL << 24; /* all iscs set */ 2170 u32 schid; 2171 2172 if (attr->flags) 2173 return -EINVAL; 2174 if (attr->attr != sizeof(schid)) 2175 return -EINVAL; 2176 if (copy_from_user(&schid, (void __user *) attr->addr, sizeof(schid))) 2177 return -EFAULT; 2178 kfree(kvm_s390_get_io_int(kvm, isc_mask, schid)); 2179 /* 2180 * If userspace is conforming to the architecture, we can have at most 2181 * one pending I/O interrupt per subchannel, so this is effectively a 2182 * clear all. 2183 */ 2184 return 0; 2185 } 2186 2187 static int modify_ais_mode(struct kvm *kvm, struct kvm_device_attr *attr) 2188 { 2189 struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int; 2190 struct kvm_s390_ais_req req; 2191 int ret = 0; 2192 2193 if (!test_kvm_facility(kvm, 72)) 2194 return -ENOTSUPP; 2195 2196 if (copy_from_user(&req, (void __user *)attr->addr, sizeof(req))) 2197 return -EFAULT; 2198 2199 if (req.isc > MAX_ISC) 2200 return -EINVAL; 2201 2202 trace_kvm_s390_modify_ais_mode(req.isc, 2203 (fi->simm & AIS_MODE_MASK(req.isc)) ? 2204 (fi->nimm & AIS_MODE_MASK(req.isc)) ? 2205 2 : KVM_S390_AIS_MODE_SINGLE : 2206 KVM_S390_AIS_MODE_ALL, req.mode); 2207 2208 mutex_lock(&fi->ais_lock); 2209 switch (req.mode) { 2210 case KVM_S390_AIS_MODE_ALL: 2211 fi->simm &= ~AIS_MODE_MASK(req.isc); 2212 fi->nimm &= ~AIS_MODE_MASK(req.isc); 2213 break; 2214 case KVM_S390_AIS_MODE_SINGLE: 2215 fi->simm |= AIS_MODE_MASK(req.isc); 2216 fi->nimm &= ~AIS_MODE_MASK(req.isc); 2217 break; 2218 default: 2219 ret = -EINVAL; 2220 } 2221 mutex_unlock(&fi->ais_lock); 2222 2223 return ret; 2224 } 2225 2226 static int kvm_s390_inject_airq(struct kvm *kvm, 2227 struct s390_io_adapter *adapter) 2228 { 2229 struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int; 2230 struct kvm_s390_interrupt s390int = { 2231 .type = KVM_S390_INT_IO(1, 0, 0, 0), 2232 .parm = 0, 2233 .parm64 = (adapter->isc << 27) | 0x80000000, 2234 }; 2235 int ret = 0; 2236 2237 if (!test_kvm_facility(kvm, 72) || !adapter->suppressible) 2238 return kvm_s390_inject_vm(kvm, &s390int); 2239 2240 mutex_lock(&fi->ais_lock); 2241 if (fi->nimm & AIS_MODE_MASK(adapter->isc)) { 2242 trace_kvm_s390_airq_suppressed(adapter->id, adapter->isc); 2243 goto out; 2244 } 2245 2246 ret = kvm_s390_inject_vm(kvm, &s390int); 2247 if (!ret && (fi->simm & AIS_MODE_MASK(adapter->isc))) { 2248 fi->nimm |= AIS_MODE_MASK(adapter->isc); 2249 trace_kvm_s390_modify_ais_mode(adapter->isc, 2250 KVM_S390_AIS_MODE_SINGLE, 2); 2251 } 2252 out: 2253 mutex_unlock(&fi->ais_lock); 2254 return ret; 2255 } 2256 2257 static int flic_inject_airq(struct kvm *kvm, struct kvm_device_attr *attr) 2258 { 2259 unsigned int id = attr->attr; 2260 struct s390_io_adapter *adapter = get_io_adapter(kvm, id); 2261 2262 if (!adapter) 2263 return -EINVAL; 2264 2265 return kvm_s390_inject_airq(kvm, adapter); 2266 } 2267 2268 static int flic_ais_mode_set_all(struct kvm *kvm, struct kvm_device_attr *attr) 2269 { 2270 struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int; 2271 struct kvm_s390_ais_all ais; 2272 2273 if (!test_kvm_facility(kvm, 72)) 2274 return -ENOTSUPP; 2275 2276 if (copy_from_user(&ais, (void __user *)attr->addr, sizeof(ais))) 2277 return -EFAULT; 2278 2279 mutex_lock(&fi->ais_lock); 2280 fi->simm = ais.simm; 2281 fi->nimm = ais.nimm; 2282 mutex_unlock(&fi->ais_lock); 2283 2284 return 0; 2285 } 2286 2287 static int flic_set_attr(struct kvm_device *dev, struct kvm_device_attr *attr) 2288 { 2289 int r = 0; 2290 unsigned int i; 2291 struct kvm_vcpu *vcpu; 2292 2293 switch (attr->group) { 2294 case KVM_DEV_FLIC_ENQUEUE: 2295 r = enqueue_floating_irq(dev, attr); 2296 break; 2297 case KVM_DEV_FLIC_CLEAR_IRQS: 2298 kvm_s390_clear_float_irqs(dev->kvm); 2299 break; 2300 case KVM_DEV_FLIC_APF_ENABLE: 2301 dev->kvm->arch.gmap->pfault_enabled = 1; 2302 break; 2303 case KVM_DEV_FLIC_APF_DISABLE_WAIT: 2304 dev->kvm->arch.gmap->pfault_enabled = 0; 2305 /* 2306 * Make sure no async faults are in transition when 2307 * clearing the queues. So we don't need to worry 2308 * about late coming workers. 2309 */ 2310 synchronize_srcu(&dev->kvm->srcu); 2311 kvm_for_each_vcpu(i, vcpu, dev->kvm) 2312 kvm_clear_async_pf_completion_queue(vcpu); 2313 break; 2314 case KVM_DEV_FLIC_ADAPTER_REGISTER: 2315 r = register_io_adapter(dev, attr); 2316 break; 2317 case KVM_DEV_FLIC_ADAPTER_MODIFY: 2318 r = modify_io_adapter(dev, attr); 2319 break; 2320 case KVM_DEV_FLIC_CLEAR_IO_IRQ: 2321 r = clear_io_irq(dev->kvm, attr); 2322 break; 2323 case KVM_DEV_FLIC_AISM: 2324 r = modify_ais_mode(dev->kvm, attr); 2325 break; 2326 case KVM_DEV_FLIC_AIRQ_INJECT: 2327 r = flic_inject_airq(dev->kvm, attr); 2328 break; 2329 case KVM_DEV_FLIC_AISM_ALL: 2330 r = flic_ais_mode_set_all(dev->kvm, attr); 2331 break; 2332 default: 2333 r = -EINVAL; 2334 } 2335 2336 return r; 2337 } 2338 2339 static int flic_has_attr(struct kvm_device *dev, 2340 struct kvm_device_attr *attr) 2341 { 2342 switch (attr->group) { 2343 case KVM_DEV_FLIC_GET_ALL_IRQS: 2344 case KVM_DEV_FLIC_ENQUEUE: 2345 case KVM_DEV_FLIC_CLEAR_IRQS: 2346 case KVM_DEV_FLIC_APF_ENABLE: 2347 case KVM_DEV_FLIC_APF_DISABLE_WAIT: 2348 case KVM_DEV_FLIC_ADAPTER_REGISTER: 2349 case KVM_DEV_FLIC_ADAPTER_MODIFY: 2350 case KVM_DEV_FLIC_CLEAR_IO_IRQ: 2351 case KVM_DEV_FLIC_AISM: 2352 case KVM_DEV_FLIC_AIRQ_INJECT: 2353 case KVM_DEV_FLIC_AISM_ALL: 2354 return 0; 2355 } 2356 return -ENXIO; 2357 } 2358 2359 static int flic_create(struct kvm_device *dev, u32 type) 2360 { 2361 if (!dev) 2362 return -EINVAL; 2363 if (dev->kvm->arch.flic) 2364 return -EINVAL; 2365 dev->kvm->arch.flic = dev; 2366 return 0; 2367 } 2368 2369 static void flic_destroy(struct kvm_device *dev) 2370 { 2371 dev->kvm->arch.flic = NULL; 2372 kfree(dev); 2373 } 2374 2375 /* s390 floating irq controller (flic) */ 2376 struct kvm_device_ops kvm_flic_ops = { 2377 .name = "kvm-flic", 2378 .get_attr = flic_get_attr, 2379 .set_attr = flic_set_attr, 2380 .has_attr = flic_has_attr, 2381 .create = flic_create, 2382 .destroy = flic_destroy, 2383 }; 2384 2385 static unsigned long get_ind_bit(__u64 addr, unsigned long bit_nr, bool swap) 2386 { 2387 unsigned long bit; 2388 2389 bit = bit_nr + (addr % PAGE_SIZE) * 8; 2390 2391 return swap ? (bit ^ (BITS_PER_LONG - 1)) : bit; 2392 } 2393 2394 static struct s390_map_info *get_map_info(struct s390_io_adapter *adapter, 2395 u64 addr) 2396 { 2397 struct s390_map_info *map; 2398 2399 if (!adapter) 2400 return NULL; 2401 2402 list_for_each_entry(map, &adapter->maps, list) { 2403 if (map->guest_addr == addr) 2404 return map; 2405 } 2406 return NULL; 2407 } 2408 2409 static int adapter_indicators_set(struct kvm *kvm, 2410 struct s390_io_adapter *adapter, 2411 struct kvm_s390_adapter_int *adapter_int) 2412 { 2413 unsigned long bit; 2414 int summary_set, idx; 2415 struct s390_map_info *info; 2416 void *map; 2417 2418 info = get_map_info(adapter, adapter_int->ind_addr); 2419 if (!info) 2420 return -1; 2421 map = page_address(info->page); 2422 bit = get_ind_bit(info->addr, adapter_int->ind_offset, adapter->swap); 2423 set_bit(bit, map); 2424 idx = srcu_read_lock(&kvm->srcu); 2425 mark_page_dirty(kvm, info->guest_addr >> PAGE_SHIFT); 2426 set_page_dirty_lock(info->page); 2427 info = get_map_info(adapter, adapter_int->summary_addr); 2428 if (!info) { 2429 srcu_read_unlock(&kvm->srcu, idx); 2430 return -1; 2431 } 2432 map = page_address(info->page); 2433 bit = get_ind_bit(info->addr, adapter_int->summary_offset, 2434 adapter->swap); 2435 summary_set = test_and_set_bit(bit, map); 2436 mark_page_dirty(kvm, info->guest_addr >> PAGE_SHIFT); 2437 set_page_dirty_lock(info->page); 2438 srcu_read_unlock(&kvm->srcu, idx); 2439 return summary_set ? 0 : 1; 2440 } 2441 2442 /* 2443 * < 0 - not injected due to error 2444 * = 0 - coalesced, summary indicator already active 2445 * > 0 - injected interrupt 2446 */ 2447 static int set_adapter_int(struct kvm_kernel_irq_routing_entry *e, 2448 struct kvm *kvm, int irq_source_id, int level, 2449 bool line_status) 2450 { 2451 int ret; 2452 struct s390_io_adapter *adapter; 2453 2454 /* We're only interested in the 0->1 transition. */ 2455 if (!level) 2456 return 0; 2457 adapter = get_io_adapter(kvm, e->adapter.adapter_id); 2458 if (!adapter) 2459 return -1; 2460 down_read(&adapter->maps_lock); 2461 ret = adapter_indicators_set(kvm, adapter, &e->adapter); 2462 up_read(&adapter->maps_lock); 2463 if ((ret > 0) && !adapter->masked) { 2464 ret = kvm_s390_inject_airq(kvm, adapter); 2465 if (ret == 0) 2466 ret = 1; 2467 } 2468 return ret; 2469 } 2470 2471 /* 2472 * Inject the machine check to the guest. 2473 */ 2474 void kvm_s390_reinject_machine_check(struct kvm_vcpu *vcpu, 2475 struct mcck_volatile_info *mcck_info) 2476 { 2477 struct kvm_s390_interrupt_info inti; 2478 struct kvm_s390_irq irq; 2479 struct kvm_s390_mchk_info *mchk; 2480 union mci mci; 2481 __u64 cr14 = 0; /* upper bits are not used */ 2482 2483 mci.val = mcck_info->mcic; 2484 if (mci.sr) 2485 cr14 |= MCCK_CR14_RECOVERY_SUB_MASK; 2486 if (mci.dg) 2487 cr14 |= MCCK_CR14_DEGRAD_SUB_MASK; 2488 if (mci.w) 2489 cr14 |= MCCK_CR14_WARN_SUB_MASK; 2490 2491 mchk = mci.ck ? &inti.mchk : &irq.u.mchk; 2492 mchk->cr14 = cr14; 2493 mchk->mcic = mcck_info->mcic; 2494 mchk->ext_damage_code = mcck_info->ext_damage_code; 2495 mchk->failing_storage_address = mcck_info->failing_storage_address; 2496 if (mci.ck) { 2497 /* Inject the floating machine check */ 2498 inti.type = KVM_S390_MCHK; 2499 WARN_ON_ONCE(__inject_vm(vcpu->kvm, &inti)); 2500 } else { 2501 /* Inject the machine check to specified vcpu */ 2502 irq.type = KVM_S390_MCHK; 2503 WARN_ON_ONCE(kvm_s390_inject_vcpu(vcpu, &irq)); 2504 } 2505 } 2506 2507 int kvm_set_routing_entry(struct kvm *kvm, 2508 struct kvm_kernel_irq_routing_entry *e, 2509 const struct kvm_irq_routing_entry *ue) 2510 { 2511 int ret; 2512 2513 switch (ue->type) { 2514 case KVM_IRQ_ROUTING_S390_ADAPTER: 2515 e->set = set_adapter_int; 2516 e->adapter.summary_addr = ue->u.adapter.summary_addr; 2517 e->adapter.ind_addr = ue->u.adapter.ind_addr; 2518 e->adapter.summary_offset = ue->u.adapter.summary_offset; 2519 e->adapter.ind_offset = ue->u.adapter.ind_offset; 2520 e->adapter.adapter_id = ue->u.adapter.adapter_id; 2521 ret = 0; 2522 break; 2523 default: 2524 ret = -EINVAL; 2525 } 2526 2527 return ret; 2528 } 2529 2530 int kvm_set_msi(struct kvm_kernel_irq_routing_entry *e, struct kvm *kvm, 2531 int irq_source_id, int level, bool line_status) 2532 { 2533 return -EINVAL; 2534 } 2535 2536 int kvm_s390_set_irq_state(struct kvm_vcpu *vcpu, void __user *irqstate, int len) 2537 { 2538 struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int; 2539 struct kvm_s390_irq *buf; 2540 int r = 0; 2541 int n; 2542 2543 buf = vmalloc(len); 2544 if (!buf) 2545 return -ENOMEM; 2546 2547 if (copy_from_user((void *) buf, irqstate, len)) { 2548 r = -EFAULT; 2549 goto out_free; 2550 } 2551 2552 /* 2553 * Don't allow setting the interrupt state 2554 * when there are already interrupts pending 2555 */ 2556 spin_lock(&li->lock); 2557 if (li->pending_irqs) { 2558 r = -EBUSY; 2559 goto out_unlock; 2560 } 2561 2562 for (n = 0; n < len / sizeof(*buf); n++) { 2563 r = do_inject_vcpu(vcpu, &buf[n]); 2564 if (r) 2565 break; 2566 } 2567 2568 out_unlock: 2569 spin_unlock(&li->lock); 2570 out_free: 2571 vfree(buf); 2572 2573 return r; 2574 } 2575 2576 static void store_local_irq(struct kvm_s390_local_interrupt *li, 2577 struct kvm_s390_irq *irq, 2578 unsigned long irq_type) 2579 { 2580 switch (irq_type) { 2581 case IRQ_PEND_MCHK_EX: 2582 case IRQ_PEND_MCHK_REP: 2583 irq->type = KVM_S390_MCHK; 2584 irq->u.mchk = li->irq.mchk; 2585 break; 2586 case IRQ_PEND_PROG: 2587 irq->type = KVM_S390_PROGRAM_INT; 2588 irq->u.pgm = li->irq.pgm; 2589 break; 2590 case IRQ_PEND_PFAULT_INIT: 2591 irq->type = KVM_S390_INT_PFAULT_INIT; 2592 irq->u.ext = li->irq.ext; 2593 break; 2594 case IRQ_PEND_EXT_EXTERNAL: 2595 irq->type = KVM_S390_INT_EXTERNAL_CALL; 2596 irq->u.extcall = li->irq.extcall; 2597 break; 2598 case IRQ_PEND_EXT_CLOCK_COMP: 2599 irq->type = KVM_S390_INT_CLOCK_COMP; 2600 break; 2601 case IRQ_PEND_EXT_CPU_TIMER: 2602 irq->type = KVM_S390_INT_CPU_TIMER; 2603 break; 2604 case IRQ_PEND_SIGP_STOP: 2605 irq->type = KVM_S390_SIGP_STOP; 2606 irq->u.stop = li->irq.stop; 2607 break; 2608 case IRQ_PEND_RESTART: 2609 irq->type = KVM_S390_RESTART; 2610 break; 2611 case IRQ_PEND_SET_PREFIX: 2612 irq->type = KVM_S390_SIGP_SET_PREFIX; 2613 irq->u.prefix = li->irq.prefix; 2614 break; 2615 } 2616 } 2617 2618 int kvm_s390_get_irq_state(struct kvm_vcpu *vcpu, __u8 __user *buf, int len) 2619 { 2620 int scn; 2621 unsigned long sigp_emerg_pending[BITS_TO_LONGS(KVM_MAX_VCPUS)]; 2622 struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int; 2623 unsigned long pending_irqs; 2624 struct kvm_s390_irq irq; 2625 unsigned long irq_type; 2626 int cpuaddr; 2627 int n = 0; 2628 2629 spin_lock(&li->lock); 2630 pending_irqs = li->pending_irqs; 2631 memcpy(&sigp_emerg_pending, &li->sigp_emerg_pending, 2632 sizeof(sigp_emerg_pending)); 2633 spin_unlock(&li->lock); 2634 2635 for_each_set_bit(irq_type, &pending_irqs, IRQ_PEND_COUNT) { 2636 memset(&irq, 0, sizeof(irq)); 2637 if (irq_type == IRQ_PEND_EXT_EMERGENCY) 2638 continue; 2639 if (n + sizeof(irq) > len) 2640 return -ENOBUFS; 2641 store_local_irq(&vcpu->arch.local_int, &irq, irq_type); 2642 if (copy_to_user(&buf[n], &irq, sizeof(irq))) 2643 return -EFAULT; 2644 n += sizeof(irq); 2645 } 2646 2647 if (test_bit(IRQ_PEND_EXT_EMERGENCY, &pending_irqs)) { 2648 for_each_set_bit(cpuaddr, sigp_emerg_pending, KVM_MAX_VCPUS) { 2649 memset(&irq, 0, sizeof(irq)); 2650 if (n + sizeof(irq) > len) 2651 return -ENOBUFS; 2652 irq.type = KVM_S390_INT_EMERGENCY; 2653 irq.u.emerg.code = cpuaddr; 2654 if (copy_to_user(&buf[n], &irq, sizeof(irq))) 2655 return -EFAULT; 2656 n += sizeof(irq); 2657 } 2658 } 2659 2660 if (sca_ext_call_pending(vcpu, &scn)) { 2661 if (n + sizeof(irq) > len) 2662 return -ENOBUFS; 2663 memset(&irq, 0, sizeof(irq)); 2664 irq.type = KVM_S390_INT_EXTERNAL_CALL; 2665 irq.u.extcall.code = scn; 2666 if (copy_to_user(&buf[n], &irq, sizeof(irq))) 2667 return -EFAULT; 2668 n += sizeof(irq); 2669 } 2670 2671 return n; 2672 } 2673