1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * handling kvm guest interrupts 4 * 5 * Copyright IBM Corp. 2008, 2020 6 * 7 * Author(s): Carsten Otte <cotte@de.ibm.com> 8 */ 9 10 #define pr_fmt(fmt) "kvm-s390: " fmt 11 12 #include <linux/cpufeature.h> 13 #include <linux/interrupt.h> 14 #include <linux/kvm_host.h> 15 #include <linux/hrtimer.h> 16 #include <linux/export.h> 17 #include <linux/mmu_context.h> 18 #include <linux/nospec.h> 19 #include <linux/signal.h> 20 #include <linux/slab.h> 21 #include <linux/bitmap.h> 22 #include <linux/vmalloc.h> 23 #include <asm/access-regs.h> 24 #include <asm/asm-offsets.h> 25 #include <asm/dis.h> 26 #include <linux/uaccess.h> 27 #include <asm/sclp.h> 28 #include <asm/isc.h> 29 #include <asm/nmi.h> 30 #include <asm/airq.h> 31 #include <asm/tpi.h> 32 #include "kvm-s390.h" 33 #include "gaccess.h" 34 #include "trace-s390.h" 35 #include "pci.h" 36 #include "gmap.h" 37 38 #define PFAULT_INIT 0x0600 39 #define PFAULT_DONE 0x0680 40 #define VIRTIO_PARAM 0x0d00 41 42 static struct kvm_s390_gib *gib; 43 44 /* handle external calls via sigp interpretation facility */ 45 static int sca_ext_call_pending(struct kvm_vcpu *vcpu, int *src_id) 46 { 47 struct esca_block *sca = vcpu->kvm->arch.sca; 48 union esca_sigp_ctrl sigp_ctrl = sca->cpu[vcpu->vcpu_id].sigp_ctrl; 49 50 if (!kvm_s390_test_cpuflags(vcpu, CPUSTAT_ECALL_PEND)) 51 return 0; 52 53 BUG_ON(!kvm_s390_use_sca_entries()); 54 55 if (src_id) 56 *src_id = sigp_ctrl.scn; 57 58 return sigp_ctrl.c; 59 } 60 61 static int sca_inject_ext_call(struct kvm_vcpu *vcpu, int src_id) 62 { 63 struct esca_block *sca = vcpu->kvm->arch.sca; 64 union esca_sigp_ctrl *sigp_ctrl = &sca->cpu[vcpu->vcpu_id].sigp_ctrl; 65 union esca_sigp_ctrl old_val, new_val = {.scn = src_id, .c = 1}; 66 int expect, rc; 67 68 BUG_ON(!kvm_s390_use_sca_entries()); 69 70 old_val = READ_ONCE(*sigp_ctrl); 71 old_val.c = 0; 72 73 expect = old_val.value; 74 rc = cmpxchg(&sigp_ctrl->value, old_val.value, new_val.value); 75 76 if (rc != expect) { 77 /* another external call is pending */ 78 return -EBUSY; 79 } 80 kvm_s390_set_cpuflags(vcpu, CPUSTAT_ECALL_PEND); 81 return 0; 82 } 83 84 static void sca_clear_ext_call(struct kvm_vcpu *vcpu) 85 { 86 struct esca_block *sca = vcpu->kvm->arch.sca; 87 union esca_sigp_ctrl *sigp_ctrl = &sca->cpu[vcpu->vcpu_id].sigp_ctrl; 88 89 if (!kvm_s390_use_sca_entries()) 90 return; 91 kvm_s390_clear_cpuflags(vcpu, CPUSTAT_ECALL_PEND); 92 93 WRITE_ONCE(sigp_ctrl->value, 0); 94 } 95 96 int psw_extint_disabled(struct kvm_vcpu *vcpu) 97 { 98 return !(vcpu->arch.sie_block->gpsw.mask & PSW_MASK_EXT); 99 } 100 101 static int psw_ioint_disabled(struct kvm_vcpu *vcpu) 102 { 103 return !(vcpu->arch.sie_block->gpsw.mask & PSW_MASK_IO); 104 } 105 106 static int psw_mchk_disabled(struct kvm_vcpu *vcpu) 107 { 108 return !(vcpu->arch.sie_block->gpsw.mask & PSW_MASK_MCHECK); 109 } 110 111 static int psw_interrupts_disabled(struct kvm_vcpu *vcpu) 112 { 113 return psw_extint_disabled(vcpu) && 114 psw_ioint_disabled(vcpu) && 115 psw_mchk_disabled(vcpu); 116 } 117 118 static int ckc_interrupts_enabled(struct kvm_vcpu *vcpu) 119 { 120 if (psw_extint_disabled(vcpu) || 121 !(vcpu->arch.sie_block->gcr[0] & CR0_CLOCK_COMPARATOR_SUBMASK)) 122 return 0; 123 if (guestdbg_enabled(vcpu) && guestdbg_sstep_enabled(vcpu)) 124 /* No timer interrupts when single stepping */ 125 return 0; 126 return 1; 127 } 128 129 static int ckc_irq_pending(struct kvm_vcpu *vcpu) 130 { 131 const u64 now = kvm_s390_get_tod_clock_fast(vcpu->kvm); 132 const u64 ckc = vcpu->arch.sie_block->ckc; 133 134 if (vcpu->arch.sie_block->gcr[0] & CR0_CLOCK_COMPARATOR_SIGN) { 135 if ((s64)ckc >= (s64)now) 136 return 0; 137 } else if (ckc >= now) { 138 return 0; 139 } 140 return ckc_interrupts_enabled(vcpu); 141 } 142 143 static int cpu_timer_interrupts_enabled(struct kvm_vcpu *vcpu) 144 { 145 return !psw_extint_disabled(vcpu) && 146 (vcpu->arch.sie_block->gcr[0] & CR0_CPU_TIMER_SUBMASK); 147 } 148 149 static int cpu_timer_irq_pending(struct kvm_vcpu *vcpu) 150 { 151 if (!cpu_timer_interrupts_enabled(vcpu)) 152 return 0; 153 return kvm_s390_get_cpu_timer(vcpu) >> 63; 154 } 155 156 static uint64_t isc_to_isc_bits(int isc) 157 { 158 return (0x80 >> isc) << 24; 159 } 160 161 static inline u32 isc_to_int_word(u8 isc) 162 { 163 return ((u32)isc << 27) | 0x80000000; 164 } 165 166 static inline u8 int_word_to_isc(u32 int_word) 167 { 168 return (int_word & 0x38000000) >> 27; 169 } 170 171 /* 172 * To use atomic bitmap functions, we have to provide a bitmap address 173 * that is u64 aligned. However, the ipm might be u32 aligned. 174 * Therefore, we logically start the bitmap at the very beginning of the 175 * struct and fixup the bit number. 176 */ 177 #define IPM_BIT_OFFSET (offsetof(struct kvm_s390_gisa, ipm) * BITS_PER_BYTE) 178 179 /** 180 * gisa_set_iam - change the GISA interruption alert mask 181 * 182 * @gisa: gisa to operate on 183 * @iam: new IAM value to use 184 * 185 * Change the IAM atomically with the next alert address and the IPM 186 * of the GISA if the GISA is not part of the GIB alert list. All three 187 * fields are located in the first long word of the GISA. 188 * 189 * Returns: 0 on success 190 * -EBUSY in case the gisa is part of the alert list 191 */ 192 static inline int gisa_set_iam(struct kvm_s390_gisa *gisa, u8 iam) 193 { 194 u64 word, _word; 195 196 word = READ_ONCE(gisa->u64.word[0]); 197 do { 198 if ((u64)gisa != word >> 32) 199 return -EBUSY; 200 _word = (word & ~0xffUL) | iam; 201 } while (!try_cmpxchg(&gisa->u64.word[0], &word, _word)); 202 203 return 0; 204 } 205 206 /** 207 * gisa_clear_ipm - clear the GISA interruption pending mask 208 * 209 * @gisa: gisa to operate on 210 * 211 * Clear the IPM atomically with the next alert address and the IAM 212 * of the GISA unconditionally. All three fields are located in the 213 * first long word of the GISA. 214 */ 215 static inline void gisa_clear_ipm(struct kvm_s390_gisa *gisa) 216 { 217 u64 word, _word; 218 219 word = READ_ONCE(gisa->u64.word[0]); 220 do { 221 _word = word & ~(0xffUL << 24); 222 } while (!try_cmpxchg(&gisa->u64.word[0], &word, _word)); 223 } 224 225 /** 226 * gisa_get_ipm_or_restore_iam - return IPM or restore GISA IAM 227 * 228 * @gi: gisa interrupt struct to work on 229 * 230 * Atomically restores the interruption alert mask if none of the 231 * relevant ISCs are pending and return the IPM. 232 * 233 * Returns: the relevant pending ISCs 234 */ 235 static inline u8 gisa_get_ipm_or_restore_iam(struct kvm_s390_gisa_interrupt *gi) 236 { 237 u8 pending_mask, alert_mask; 238 u64 word, _word; 239 240 word = READ_ONCE(gi->origin->u64.word[0]); 241 do { 242 alert_mask = READ_ONCE(gi->alert.mask); 243 pending_mask = (u8)(word >> 24) & alert_mask; 244 if (pending_mask) 245 return pending_mask; 246 _word = (word & ~0xffUL) | alert_mask; 247 } while (!try_cmpxchg(&gi->origin->u64.word[0], &word, _word)); 248 249 return 0; 250 } 251 252 static inline void gisa_set_ipm_gisc(struct kvm_s390_gisa *gisa, u32 gisc) 253 { 254 set_bit_inv(IPM_BIT_OFFSET + gisc, (unsigned long *) gisa); 255 } 256 257 static inline u8 gisa_get_ipm(struct kvm_s390_gisa *gisa) 258 { 259 return READ_ONCE(gisa->ipm); 260 } 261 262 static inline int gisa_tac_ipm_gisc(struct kvm_s390_gisa *gisa, u32 gisc) 263 { 264 return test_and_clear_bit_inv(IPM_BIT_OFFSET + gisc, (unsigned long *) gisa); 265 } 266 267 static inline unsigned long pending_irqs_no_gisa(struct kvm_vcpu *vcpu) 268 { 269 unsigned long pending = vcpu->kvm->arch.float_int.pending_irqs | 270 vcpu->arch.local_int.pending_irqs; 271 272 pending &= ~vcpu->kvm->arch.float_int.masked_irqs; 273 return pending; 274 } 275 276 static inline unsigned long pending_irqs(struct kvm_vcpu *vcpu) 277 { 278 struct kvm_s390_gisa_interrupt *gi = &vcpu->kvm->arch.gisa_int; 279 unsigned long pending_mask; 280 281 pending_mask = pending_irqs_no_gisa(vcpu); 282 if (gi->origin) 283 pending_mask |= gisa_get_ipm(gi->origin) << IRQ_PEND_IO_ISC_7; 284 return pending_mask; 285 } 286 287 static inline int isc_to_irq_type(unsigned long isc) 288 { 289 return IRQ_PEND_IO_ISC_0 - isc; 290 } 291 292 static inline int irq_type_to_isc(unsigned long irq_type) 293 { 294 return IRQ_PEND_IO_ISC_0 - irq_type; 295 } 296 297 static unsigned long disable_iscs(struct kvm_vcpu *vcpu, 298 unsigned long active_mask) 299 { 300 int i; 301 302 for (i = 0; i <= MAX_ISC; i++) 303 if (!(vcpu->arch.sie_block->gcr[6] & isc_to_isc_bits(i))) 304 active_mask &= ~(1UL << (isc_to_irq_type(i))); 305 306 return active_mask; 307 } 308 309 static unsigned long deliverable_irqs(struct kvm_vcpu *vcpu) 310 { 311 unsigned long active_mask; 312 313 active_mask = pending_irqs(vcpu); 314 if (!active_mask) 315 return 0; 316 317 if (psw_extint_disabled(vcpu)) 318 active_mask &= ~IRQ_PEND_EXT_MASK; 319 if (psw_ioint_disabled(vcpu)) 320 active_mask &= ~IRQ_PEND_IO_MASK; 321 else 322 active_mask = disable_iscs(vcpu, active_mask); 323 if (!(vcpu->arch.sie_block->gcr[0] & CR0_EXTERNAL_CALL_SUBMASK)) 324 __clear_bit(IRQ_PEND_EXT_EXTERNAL, &active_mask); 325 if (!(vcpu->arch.sie_block->gcr[0] & CR0_EMERGENCY_SIGNAL_SUBMASK)) 326 __clear_bit(IRQ_PEND_EXT_EMERGENCY, &active_mask); 327 if (!(vcpu->arch.sie_block->gcr[0] & CR0_CLOCK_COMPARATOR_SUBMASK)) 328 __clear_bit(IRQ_PEND_EXT_CLOCK_COMP, &active_mask); 329 if (!(vcpu->arch.sie_block->gcr[0] & CR0_CPU_TIMER_SUBMASK)) 330 __clear_bit(IRQ_PEND_EXT_CPU_TIMER, &active_mask); 331 if (!(vcpu->arch.sie_block->gcr[0] & CR0_SERVICE_SIGNAL_SUBMASK)) { 332 __clear_bit(IRQ_PEND_EXT_SERVICE, &active_mask); 333 __clear_bit(IRQ_PEND_EXT_SERVICE_EV, &active_mask); 334 } 335 if (psw_mchk_disabled(vcpu)) 336 active_mask &= ~IRQ_PEND_MCHK_MASK; 337 /* PV guest cpus can have a single interruption injected at a time. */ 338 if (kvm_s390_pv_cpu_get_handle(vcpu) && 339 vcpu->arch.sie_block->iictl != IICTL_CODE_NONE) 340 active_mask &= ~(IRQ_PEND_EXT_II_MASK | 341 IRQ_PEND_IO_MASK | 342 IRQ_PEND_MCHK_MASK); 343 /* 344 * Check both floating and local interrupt's cr14 because 345 * bit IRQ_PEND_MCHK_REP could be set in both cases. 346 */ 347 if (!(vcpu->arch.sie_block->gcr[14] & 348 (vcpu->kvm->arch.float_int.mchk.cr14 | 349 vcpu->arch.local_int.irq.mchk.cr14))) 350 __clear_bit(IRQ_PEND_MCHK_REP, &active_mask); 351 352 /* 353 * STOP irqs will never be actively delivered. They are triggered via 354 * intercept requests and cleared when the stop intercept is performed. 355 */ 356 __clear_bit(IRQ_PEND_SIGP_STOP, &active_mask); 357 358 return active_mask; 359 } 360 361 static void __set_cpu_idle(struct kvm_vcpu *vcpu) 362 { 363 kvm_s390_set_cpuflags(vcpu, CPUSTAT_WAIT); 364 set_bit(vcpu->vcpu_idx, vcpu->kvm->arch.idle_mask); 365 } 366 367 static void __unset_cpu_idle(struct kvm_vcpu *vcpu) 368 { 369 kvm_s390_clear_cpuflags(vcpu, CPUSTAT_WAIT); 370 clear_bit(vcpu->vcpu_idx, vcpu->kvm->arch.idle_mask); 371 } 372 373 static void __reset_intercept_indicators(struct kvm_vcpu *vcpu) 374 { 375 kvm_s390_clear_cpuflags(vcpu, CPUSTAT_IO_INT | CPUSTAT_EXT_INT | 376 CPUSTAT_STOP_INT); 377 vcpu->arch.sie_block->lctl = 0x0000; 378 vcpu->arch.sie_block->ictl &= ~(ICTL_LPSW | ICTL_STCTL | ICTL_PINT); 379 380 if (guestdbg_enabled(vcpu)) { 381 vcpu->arch.sie_block->lctl |= (LCTL_CR0 | LCTL_CR9 | 382 LCTL_CR10 | LCTL_CR11); 383 vcpu->arch.sie_block->ictl |= (ICTL_STCTL | ICTL_PINT); 384 } 385 } 386 387 static void set_intercept_indicators_io(struct kvm_vcpu *vcpu) 388 { 389 if (!(pending_irqs_no_gisa(vcpu) & IRQ_PEND_IO_MASK)) 390 return; 391 if (psw_ioint_disabled(vcpu)) 392 kvm_s390_set_cpuflags(vcpu, CPUSTAT_IO_INT); 393 else 394 vcpu->arch.sie_block->lctl |= LCTL_CR6; 395 } 396 397 static void set_intercept_indicators_ext(struct kvm_vcpu *vcpu) 398 { 399 if (!(pending_irqs_no_gisa(vcpu) & IRQ_PEND_EXT_MASK)) 400 return; 401 if (psw_extint_disabled(vcpu)) 402 kvm_s390_set_cpuflags(vcpu, CPUSTAT_EXT_INT); 403 else 404 vcpu->arch.sie_block->lctl |= LCTL_CR0; 405 } 406 407 static void set_intercept_indicators_mchk(struct kvm_vcpu *vcpu) 408 { 409 if (!(pending_irqs_no_gisa(vcpu) & IRQ_PEND_MCHK_MASK)) 410 return; 411 if (psw_mchk_disabled(vcpu)) 412 vcpu->arch.sie_block->ictl |= ICTL_LPSW; 413 else 414 vcpu->arch.sie_block->lctl |= LCTL_CR14; 415 } 416 417 static void set_intercept_indicators_stop(struct kvm_vcpu *vcpu) 418 { 419 if (kvm_s390_is_stop_irq_pending(vcpu)) 420 kvm_s390_set_cpuflags(vcpu, CPUSTAT_STOP_INT); 421 } 422 423 /* Set interception request for non-deliverable interrupts */ 424 static void set_intercept_indicators(struct kvm_vcpu *vcpu) 425 { 426 set_intercept_indicators_io(vcpu); 427 set_intercept_indicators_ext(vcpu); 428 set_intercept_indicators_mchk(vcpu); 429 set_intercept_indicators_stop(vcpu); 430 } 431 432 static int __must_check __deliver_cpu_timer(struct kvm_vcpu *vcpu) 433 { 434 struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int; 435 int rc = 0; 436 437 vcpu->stat.deliver_cputm++; 438 trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, KVM_S390_INT_CPU_TIMER, 439 0, 0); 440 if (kvm_s390_pv_cpu_is_protected(vcpu)) { 441 vcpu->arch.sie_block->iictl = IICTL_CODE_EXT; 442 vcpu->arch.sie_block->eic = EXT_IRQ_CPU_TIMER; 443 } else { 444 rc = put_guest_lc(vcpu, EXT_IRQ_CPU_TIMER, 445 (u16 *)__LC_EXT_INT_CODE); 446 rc |= put_guest_lc(vcpu, 0, (u16 *)__LC_EXT_CPU_ADDR); 447 rc |= write_guest_lc(vcpu, __LC_EXT_OLD_PSW, 448 &vcpu->arch.sie_block->gpsw, sizeof(psw_t)); 449 rc |= read_guest_lc(vcpu, __LC_EXT_NEW_PSW, 450 &vcpu->arch.sie_block->gpsw, sizeof(psw_t)); 451 } 452 clear_bit(IRQ_PEND_EXT_CPU_TIMER, &li->pending_irqs); 453 return rc ? -EFAULT : 0; 454 } 455 456 static int __must_check __deliver_ckc(struct kvm_vcpu *vcpu) 457 { 458 struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int; 459 int rc = 0; 460 461 vcpu->stat.deliver_ckc++; 462 trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, KVM_S390_INT_CLOCK_COMP, 463 0, 0); 464 if (kvm_s390_pv_cpu_is_protected(vcpu)) { 465 vcpu->arch.sie_block->iictl = IICTL_CODE_EXT; 466 vcpu->arch.sie_block->eic = EXT_IRQ_CLK_COMP; 467 } else { 468 rc = put_guest_lc(vcpu, EXT_IRQ_CLK_COMP, 469 (u16 __user *)__LC_EXT_INT_CODE); 470 rc |= put_guest_lc(vcpu, 0, (u16 *)__LC_EXT_CPU_ADDR); 471 rc |= write_guest_lc(vcpu, __LC_EXT_OLD_PSW, 472 &vcpu->arch.sie_block->gpsw, sizeof(psw_t)); 473 rc |= read_guest_lc(vcpu, __LC_EXT_NEW_PSW, 474 &vcpu->arch.sie_block->gpsw, sizeof(psw_t)); 475 } 476 clear_bit(IRQ_PEND_EXT_CLOCK_COMP, &li->pending_irqs); 477 return rc ? -EFAULT : 0; 478 } 479 480 static int __must_check __deliver_pfault_init(struct kvm_vcpu *vcpu) 481 { 482 struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int; 483 struct kvm_s390_ext_info ext; 484 int rc; 485 486 spin_lock(&li->lock); 487 ext = li->irq.ext; 488 clear_bit(IRQ_PEND_PFAULT_INIT, &li->pending_irqs); 489 li->irq.ext.ext_params2 = 0; 490 spin_unlock(&li->lock); 491 492 VCPU_EVENT(vcpu, 4, "deliver: pfault init token 0x%llx", 493 ext.ext_params2); 494 trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, 495 KVM_S390_INT_PFAULT_INIT, 496 0, ext.ext_params2); 497 498 rc = put_guest_lc(vcpu, EXT_IRQ_CP_SERVICE, (u16 *) __LC_EXT_INT_CODE); 499 rc |= put_guest_lc(vcpu, PFAULT_INIT, (u16 *) __LC_EXT_CPU_ADDR); 500 rc |= write_guest_lc(vcpu, __LC_EXT_OLD_PSW, 501 &vcpu->arch.sie_block->gpsw, sizeof(psw_t)); 502 rc |= read_guest_lc(vcpu, __LC_EXT_NEW_PSW, 503 &vcpu->arch.sie_block->gpsw, sizeof(psw_t)); 504 rc |= put_guest_lc(vcpu, ext.ext_params2, (u64 *) __LC_EXT_PARAMS2); 505 return rc ? -EFAULT : 0; 506 } 507 508 static int __write_machine_check(struct kvm_vcpu *vcpu, 509 struct kvm_s390_mchk_info *mchk) 510 { 511 unsigned long ext_sa_addr; 512 unsigned long lc; 513 freg_t fprs[NUM_FPRS]; 514 union mci mci; 515 int rc; 516 517 /* 518 * All other possible payload for a machine check (e.g. the register 519 * contents in the save area) will be handled by the ultravisor, as 520 * the hypervisor does not not have the needed information for 521 * protected guests. 522 */ 523 if (kvm_s390_pv_cpu_is_protected(vcpu)) { 524 vcpu->arch.sie_block->iictl = IICTL_CODE_MCHK; 525 vcpu->arch.sie_block->mcic = mchk->mcic; 526 vcpu->arch.sie_block->faddr = mchk->failing_storage_address; 527 vcpu->arch.sie_block->edc = mchk->ext_damage_code; 528 return 0; 529 } 530 531 mci.val = mchk->mcic; 532 /* take care of lazy register loading */ 533 kvm_s390_fpu_store(vcpu->run); 534 save_access_regs(vcpu->run->s.regs.acrs); 535 if (cpu_has_gs() && vcpu->arch.gs_enabled) 536 save_gs_cb(current->thread.gs_cb); 537 538 /* Extended save area */ 539 rc = read_guest_lc(vcpu, __LC_MCESAD, &ext_sa_addr, 540 sizeof(unsigned long)); 541 /* Only bits 0 through 63-LC are used for address formation */ 542 lc = ext_sa_addr & MCESA_LC_MASK; 543 if (test_kvm_facility(vcpu->kvm, 133)) { 544 switch (lc) { 545 case 0: 546 case 10: 547 ext_sa_addr &= ~0x3ffUL; 548 break; 549 case 11: 550 ext_sa_addr &= ~0x7ffUL; 551 break; 552 case 12: 553 ext_sa_addr &= ~0xfffUL; 554 break; 555 default: 556 ext_sa_addr = 0; 557 break; 558 } 559 } else { 560 ext_sa_addr &= ~0x3ffUL; 561 } 562 563 if (!rc && mci.vr && ext_sa_addr && test_kvm_facility(vcpu->kvm, 129)) { 564 if (write_guest_abs(vcpu, ext_sa_addr, vcpu->run->s.regs.vrs, 565 512)) 566 mci.vr = 0; 567 } else { 568 mci.vr = 0; 569 } 570 if (!rc && mci.gs && ext_sa_addr && test_kvm_facility(vcpu->kvm, 133) 571 && (lc == 11 || lc == 12)) { 572 if (write_guest_abs(vcpu, ext_sa_addr + 1024, 573 &vcpu->run->s.regs.gscb, 32)) 574 mci.gs = 0; 575 } else { 576 mci.gs = 0; 577 } 578 579 /* General interruption information */ 580 rc |= put_guest_lc(vcpu, 1, (u8 __user *) __LC_AR_MODE_ID); 581 rc |= write_guest_lc(vcpu, __LC_MCK_OLD_PSW, 582 &vcpu->arch.sie_block->gpsw, sizeof(psw_t)); 583 rc |= read_guest_lc(vcpu, __LC_MCK_NEW_PSW, 584 &vcpu->arch.sie_block->gpsw, sizeof(psw_t)); 585 rc |= put_guest_lc(vcpu, mci.val, (u64 __user *) __LC_MCCK_CODE); 586 587 /* Register-save areas */ 588 if (cpu_has_vx()) { 589 convert_vx_to_fp(fprs, (__vector128 *) vcpu->run->s.regs.vrs); 590 rc |= write_guest_lc(vcpu, __LC_FPREGS_SAVE_AREA, fprs, 128); 591 } else { 592 rc |= write_guest_lc(vcpu, __LC_FPREGS_SAVE_AREA, 593 vcpu->run->s.regs.fprs, 128); 594 } 595 rc |= write_guest_lc(vcpu, __LC_GPREGS_SAVE_AREA, 596 vcpu->run->s.regs.gprs, 128); 597 rc |= put_guest_lc(vcpu, vcpu->run->s.regs.fpc, 598 (u32 __user *) __LC_FP_CREG_SAVE_AREA); 599 rc |= put_guest_lc(vcpu, vcpu->arch.sie_block->todpr, 600 (u32 __user *) __LC_TOD_PROGREG_SAVE_AREA); 601 rc |= put_guest_lc(vcpu, kvm_s390_get_cpu_timer(vcpu), 602 (u64 __user *) __LC_CPU_TIMER_SAVE_AREA); 603 rc |= put_guest_lc(vcpu, vcpu->arch.sie_block->ckc >> 8, 604 (u64 __user *) __LC_CLOCK_COMP_SAVE_AREA); 605 rc |= write_guest_lc(vcpu, __LC_AREGS_SAVE_AREA, 606 &vcpu->run->s.regs.acrs, 64); 607 rc |= write_guest_lc(vcpu, __LC_CREGS_SAVE_AREA, 608 &vcpu->arch.sie_block->gcr, 128); 609 610 /* Extended interruption information */ 611 rc |= put_guest_lc(vcpu, mchk->ext_damage_code, 612 (u32 __user *) __LC_EXT_DAMAGE_CODE); 613 rc |= put_guest_lc(vcpu, mchk->failing_storage_address, 614 (u64 __user *) __LC_MCCK_FAIL_STOR_ADDR); 615 rc |= write_guest_lc(vcpu, __LC_PSW_SAVE_AREA, &mchk->fixed_logout, 616 sizeof(mchk->fixed_logout)); 617 return rc ? -EFAULT : 0; 618 } 619 620 static int __must_check __deliver_machine_check(struct kvm_vcpu *vcpu) 621 { 622 struct kvm_s390_float_interrupt *fi = &vcpu->kvm->arch.float_int; 623 struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int; 624 struct kvm_s390_mchk_info mchk = {}; 625 int deliver = 0; 626 int rc = 0; 627 unsigned long flags; 628 629 spin_lock_irqsave(&fi->lock, flags); 630 spin_lock(&li->lock); 631 if (test_bit(IRQ_PEND_MCHK_EX, &li->pending_irqs) || 632 test_bit(IRQ_PEND_MCHK_REP, &li->pending_irqs)) { 633 /* 634 * If there was an exigent machine check pending, then any 635 * repressible machine checks that might have been pending 636 * are indicated along with it, so always clear bits for 637 * repressible and exigent interrupts 638 */ 639 mchk = li->irq.mchk; 640 clear_bit(IRQ_PEND_MCHK_EX, &li->pending_irqs); 641 clear_bit(IRQ_PEND_MCHK_REP, &li->pending_irqs); 642 memset(&li->irq.mchk, 0, sizeof(mchk)); 643 deliver = 1; 644 } 645 /* 646 * We indicate floating repressible conditions along with 647 * other pending conditions. Channel Report Pending and Channel 648 * Subsystem damage are the only two and are indicated by 649 * bits in mcic and masked in cr14. 650 */ 651 if (test_and_clear_bit(IRQ_PEND_MCHK_REP, &fi->pending_irqs)) { 652 mchk.mcic |= fi->mchk.mcic; 653 mchk.cr14 |= fi->mchk.cr14; 654 memset(&fi->mchk, 0, sizeof(mchk)); 655 deliver = 1; 656 } 657 spin_unlock(&li->lock); 658 spin_unlock_irqrestore(&fi->lock, flags); 659 660 if (deliver) { 661 VCPU_EVENT(vcpu, 3, "deliver: machine check mcic 0x%llx", 662 mchk.mcic); 663 trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, 664 KVM_S390_MCHK, 665 mchk.cr14, mchk.mcic); 666 vcpu->stat.deliver_machine_check++; 667 rc = __write_machine_check(vcpu, &mchk); 668 } 669 return rc; 670 } 671 672 static int __must_check __deliver_restart(struct kvm_vcpu *vcpu) 673 { 674 struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int; 675 int rc = 0; 676 677 VCPU_EVENT(vcpu, 3, "%s", "deliver: cpu restart"); 678 vcpu->stat.deliver_restart_signal++; 679 trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, KVM_S390_RESTART, 0, 0); 680 681 if (kvm_s390_pv_cpu_is_protected(vcpu)) { 682 vcpu->arch.sie_block->iictl = IICTL_CODE_RESTART; 683 } else { 684 rc = write_guest_lc(vcpu, 685 offsetof(struct lowcore, restart_old_psw), 686 &vcpu->arch.sie_block->gpsw, sizeof(psw_t)); 687 rc |= read_guest_lc(vcpu, offsetof(struct lowcore, restart_psw), 688 &vcpu->arch.sie_block->gpsw, sizeof(psw_t)); 689 } 690 clear_bit(IRQ_PEND_RESTART, &li->pending_irqs); 691 return rc ? -EFAULT : 0; 692 } 693 694 static int __must_check __deliver_set_prefix(struct kvm_vcpu *vcpu) 695 { 696 struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int; 697 struct kvm_s390_prefix_info prefix; 698 699 spin_lock(&li->lock); 700 prefix = li->irq.prefix; 701 li->irq.prefix.address = 0; 702 clear_bit(IRQ_PEND_SET_PREFIX, &li->pending_irqs); 703 spin_unlock(&li->lock); 704 705 vcpu->stat.deliver_prefix_signal++; 706 trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, 707 KVM_S390_SIGP_SET_PREFIX, 708 prefix.address, 0); 709 710 kvm_s390_set_prefix(vcpu, prefix.address); 711 return 0; 712 } 713 714 static int __must_check __deliver_emergency_signal(struct kvm_vcpu *vcpu) 715 { 716 struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int; 717 int rc; 718 int cpu_addr; 719 720 spin_lock(&li->lock); 721 cpu_addr = find_first_bit(li->sigp_emerg_pending, KVM_MAX_VCPUS); 722 clear_bit(cpu_addr, li->sigp_emerg_pending); 723 if (bitmap_empty(li->sigp_emerg_pending, KVM_MAX_VCPUS)) 724 clear_bit(IRQ_PEND_EXT_EMERGENCY, &li->pending_irqs); 725 spin_unlock(&li->lock); 726 727 VCPU_EVENT(vcpu, 4, "%s", "deliver: sigp emerg"); 728 vcpu->stat.deliver_emergency_signal++; 729 trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, KVM_S390_INT_EMERGENCY, 730 cpu_addr, 0); 731 if (kvm_s390_pv_cpu_is_protected(vcpu)) { 732 vcpu->arch.sie_block->iictl = IICTL_CODE_EXT; 733 vcpu->arch.sie_block->eic = EXT_IRQ_EMERGENCY_SIG; 734 vcpu->arch.sie_block->extcpuaddr = cpu_addr; 735 return 0; 736 } 737 738 rc = put_guest_lc(vcpu, EXT_IRQ_EMERGENCY_SIG, 739 (u16 *)__LC_EXT_INT_CODE); 740 rc |= put_guest_lc(vcpu, cpu_addr, (u16 *)__LC_EXT_CPU_ADDR); 741 rc |= write_guest_lc(vcpu, __LC_EXT_OLD_PSW, 742 &vcpu->arch.sie_block->gpsw, sizeof(psw_t)); 743 rc |= read_guest_lc(vcpu, __LC_EXT_NEW_PSW, 744 &vcpu->arch.sie_block->gpsw, sizeof(psw_t)); 745 return rc ? -EFAULT : 0; 746 } 747 748 static int __must_check __deliver_external_call(struct kvm_vcpu *vcpu) 749 { 750 struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int; 751 struct kvm_s390_extcall_info extcall; 752 int rc; 753 754 spin_lock(&li->lock); 755 extcall = li->irq.extcall; 756 li->irq.extcall.code = 0; 757 clear_bit(IRQ_PEND_EXT_EXTERNAL, &li->pending_irqs); 758 spin_unlock(&li->lock); 759 760 VCPU_EVENT(vcpu, 4, "%s", "deliver: sigp ext call"); 761 vcpu->stat.deliver_external_call++; 762 trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, 763 KVM_S390_INT_EXTERNAL_CALL, 764 extcall.code, 0); 765 if (kvm_s390_pv_cpu_is_protected(vcpu)) { 766 vcpu->arch.sie_block->iictl = IICTL_CODE_EXT; 767 vcpu->arch.sie_block->eic = EXT_IRQ_EXTERNAL_CALL; 768 vcpu->arch.sie_block->extcpuaddr = extcall.code; 769 return 0; 770 } 771 772 rc = put_guest_lc(vcpu, EXT_IRQ_EXTERNAL_CALL, 773 (u16 *)__LC_EXT_INT_CODE); 774 rc |= put_guest_lc(vcpu, extcall.code, (u16 *)__LC_EXT_CPU_ADDR); 775 rc |= write_guest_lc(vcpu, __LC_EXT_OLD_PSW, 776 &vcpu->arch.sie_block->gpsw, sizeof(psw_t)); 777 rc |= read_guest_lc(vcpu, __LC_EXT_NEW_PSW, &vcpu->arch.sie_block->gpsw, 778 sizeof(psw_t)); 779 return rc ? -EFAULT : 0; 780 } 781 782 static int __deliver_prog_pv(struct kvm_vcpu *vcpu, u16 code) 783 { 784 switch (code) { 785 case PGM_SPECIFICATION: 786 vcpu->arch.sie_block->iictl = IICTL_CODE_SPECIFICATION; 787 break; 788 case PGM_OPERAND: 789 vcpu->arch.sie_block->iictl = IICTL_CODE_OPERAND; 790 break; 791 default: 792 return -EINVAL; 793 } 794 return 0; 795 } 796 797 static int __must_check __deliver_prog(struct kvm_vcpu *vcpu) 798 { 799 struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int; 800 struct kvm_s390_pgm_info pgm_info; 801 int rc = 0, nullifying = false; 802 u16 ilen; 803 804 spin_lock(&li->lock); 805 pgm_info = li->irq.pgm; 806 clear_bit(IRQ_PEND_PROG, &li->pending_irqs); 807 memset(&li->irq.pgm, 0, sizeof(pgm_info)); 808 spin_unlock(&li->lock); 809 810 ilen = pgm_info.flags & KVM_S390_PGM_FLAGS_ILC_MASK; 811 VCPU_EVENT(vcpu, 3, "deliver: program irq code 0x%x, ilen:%d", 812 pgm_info.code, ilen); 813 vcpu->stat.deliver_program++; 814 trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, KVM_S390_PROGRAM_INT, 815 pgm_info.code, 0); 816 817 /* PER is handled by the ultravisor */ 818 if (kvm_s390_pv_cpu_is_protected(vcpu)) 819 return __deliver_prog_pv(vcpu, pgm_info.code & ~PGM_PER); 820 821 switch (pgm_info.code & ~PGM_PER) { 822 case PGM_AFX_TRANSLATION: 823 case PGM_ASX_TRANSLATION: 824 case PGM_EX_TRANSLATION: 825 case PGM_LFX_TRANSLATION: 826 case PGM_LSTE_SEQUENCE: 827 case PGM_LSX_TRANSLATION: 828 case PGM_LX_TRANSLATION: 829 case PGM_PRIMARY_AUTHORITY: 830 case PGM_SECONDARY_AUTHORITY: 831 nullifying = true; 832 fallthrough; 833 case PGM_SPACE_SWITCH: 834 rc = put_guest_lc(vcpu, pgm_info.trans_exc_code, 835 (u64 *)__LC_TRANS_EXC_CODE); 836 break; 837 case PGM_ALEN_TRANSLATION: 838 case PGM_ALE_SEQUENCE: 839 case PGM_ASTE_INSTANCE: 840 case PGM_ASTE_SEQUENCE: 841 case PGM_ASTE_VALIDITY: 842 case PGM_EXTENDED_AUTHORITY: 843 rc = put_guest_lc(vcpu, pgm_info.exc_access_id, 844 (u8 *)__LC_EXC_ACCESS_ID); 845 nullifying = true; 846 break; 847 case PGM_ASCE_TYPE: 848 case PGM_PAGE_TRANSLATION: 849 case PGM_REGION_FIRST_TRANS: 850 case PGM_REGION_SECOND_TRANS: 851 case PGM_REGION_THIRD_TRANS: 852 case PGM_SEGMENT_TRANSLATION: 853 rc = put_guest_lc(vcpu, pgm_info.trans_exc_code, 854 (u64 *)__LC_TRANS_EXC_CODE); 855 rc |= put_guest_lc(vcpu, pgm_info.exc_access_id, 856 (u8 *)__LC_EXC_ACCESS_ID); 857 rc |= put_guest_lc(vcpu, pgm_info.op_access_id, 858 (u8 *)__LC_OP_ACCESS_ID); 859 nullifying = true; 860 break; 861 case PGM_MONITOR: 862 rc = put_guest_lc(vcpu, pgm_info.mon_class_nr, 863 (u16 *)__LC_MON_CLASS_NR); 864 rc |= put_guest_lc(vcpu, pgm_info.mon_code, 865 (u64 *)__LC_MON_CODE); 866 break; 867 case PGM_VECTOR_PROCESSING: 868 case PGM_DATA: 869 rc = put_guest_lc(vcpu, pgm_info.data_exc_code, 870 (u32 *)__LC_DATA_EXC_CODE); 871 break; 872 case PGM_PROTECTION: 873 rc = put_guest_lc(vcpu, pgm_info.trans_exc_code, 874 (u64 *)__LC_TRANS_EXC_CODE); 875 rc |= put_guest_lc(vcpu, pgm_info.exc_access_id, 876 (u8 *)__LC_EXC_ACCESS_ID); 877 break; 878 case PGM_STACK_FULL: 879 case PGM_STACK_EMPTY: 880 case PGM_STACK_SPECIFICATION: 881 case PGM_STACK_TYPE: 882 case PGM_STACK_OPERATION: 883 case PGM_TRACE_TABEL: 884 case PGM_CRYPTO_OPERATION: 885 nullifying = true; 886 break; 887 } 888 889 if (pgm_info.code & PGM_PER) { 890 rc |= put_guest_lc(vcpu, pgm_info.per_code, 891 (u8 *) __LC_PER_CODE); 892 rc |= put_guest_lc(vcpu, pgm_info.per_atmid, 893 (u8 *)__LC_PER_ATMID); 894 rc |= put_guest_lc(vcpu, pgm_info.per_address, 895 (u64 *) __LC_PER_ADDRESS); 896 rc |= put_guest_lc(vcpu, pgm_info.per_access_id, 897 (u8 *) __LC_PER_ACCESS_ID); 898 } 899 900 if (nullifying && !(pgm_info.flags & KVM_S390_PGM_FLAGS_NO_REWIND)) 901 kvm_s390_rewind_psw(vcpu, ilen); 902 903 /* bit 1+2 of the target are the ilc, so we can directly use ilen */ 904 rc |= put_guest_lc(vcpu, ilen, (u16 *) __LC_PGM_ILC); 905 rc |= put_guest_lc(vcpu, vcpu->arch.sie_block->gbea, 906 (u64 *) __LC_PGM_LAST_BREAK); 907 rc |= put_guest_lc(vcpu, pgm_info.code, (u16 *)__LC_PGM_CODE); 908 rc |= write_guest_lc(vcpu, __LC_PGM_OLD_PSW, 909 &vcpu->arch.sie_block->gpsw, sizeof(psw_t)); 910 rc |= read_guest_lc(vcpu, __LC_PGM_NEW_PSW, 911 &vcpu->arch.sie_block->gpsw, sizeof(psw_t)); 912 return rc ? -EFAULT : 0; 913 } 914 915 #define SCCB_MASK 0xFFFFFFF8 916 #define SCCB_EVENT_PENDING 0x3 917 918 static int write_sclp(struct kvm_vcpu *vcpu, u32 parm) 919 { 920 int rc; 921 922 if (kvm_s390_pv_cpu_get_handle(vcpu)) { 923 vcpu->arch.sie_block->iictl = IICTL_CODE_EXT; 924 vcpu->arch.sie_block->eic = EXT_IRQ_SERVICE_SIG; 925 vcpu->arch.sie_block->eiparams = parm; 926 return 0; 927 } 928 929 rc = put_guest_lc(vcpu, EXT_IRQ_SERVICE_SIG, (u16 *)__LC_EXT_INT_CODE); 930 rc |= put_guest_lc(vcpu, 0, (u16 *)__LC_EXT_CPU_ADDR); 931 rc |= write_guest_lc(vcpu, __LC_EXT_OLD_PSW, 932 &vcpu->arch.sie_block->gpsw, sizeof(psw_t)); 933 rc |= read_guest_lc(vcpu, __LC_EXT_NEW_PSW, 934 &vcpu->arch.sie_block->gpsw, sizeof(psw_t)); 935 rc |= put_guest_lc(vcpu, parm, 936 (u32 *)__LC_EXT_PARAMS); 937 938 return rc ? -EFAULT : 0; 939 } 940 941 static int __must_check __deliver_service(struct kvm_vcpu *vcpu) 942 { 943 struct kvm_s390_float_interrupt *fi = &vcpu->kvm->arch.float_int; 944 struct kvm_s390_ext_info ext; 945 unsigned long flags; 946 947 spin_lock_irqsave(&fi->lock, flags); 948 if (test_bit(IRQ_PEND_EXT_SERVICE, &fi->masked_irqs) || 949 !(test_bit(IRQ_PEND_EXT_SERVICE, &fi->pending_irqs))) { 950 spin_unlock_irqrestore(&fi->lock, flags); 951 return 0; 952 } 953 ext = fi->srv_signal; 954 memset(&fi->srv_signal, 0, sizeof(ext)); 955 clear_bit(IRQ_PEND_EXT_SERVICE, &fi->pending_irqs); 956 clear_bit(IRQ_PEND_EXT_SERVICE_EV, &fi->pending_irqs); 957 if (kvm_s390_pv_cpu_is_protected(vcpu)) 958 set_bit(IRQ_PEND_EXT_SERVICE, &fi->masked_irqs); 959 spin_unlock_irqrestore(&fi->lock, flags); 960 961 if (!ext.ext_params) 962 return 0; 963 964 VCPU_EVENT(vcpu, 4, "deliver: sclp parameter 0x%x", 965 ext.ext_params); 966 vcpu->stat.deliver_service_signal++; 967 trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, KVM_S390_INT_SERVICE, 968 ext.ext_params, 0); 969 970 return write_sclp(vcpu, ext.ext_params); 971 } 972 973 static int __must_check __deliver_service_ev(struct kvm_vcpu *vcpu) 974 { 975 struct kvm_s390_float_interrupt *fi = &vcpu->kvm->arch.float_int; 976 struct kvm_s390_ext_info ext; 977 unsigned long flags; 978 979 spin_lock_irqsave(&fi->lock, flags); 980 if (!(test_bit(IRQ_PEND_EXT_SERVICE_EV, &fi->pending_irqs))) { 981 spin_unlock_irqrestore(&fi->lock, flags); 982 return 0; 983 } 984 ext = fi->srv_signal; 985 /* only clear the event bits */ 986 fi->srv_signal.ext_params &= ~SCCB_EVENT_PENDING; 987 clear_bit(IRQ_PEND_EXT_SERVICE_EV, &fi->pending_irqs); 988 spin_unlock_irqrestore(&fi->lock, flags); 989 990 VCPU_EVENT(vcpu, 4, "%s", "deliver: sclp parameter event"); 991 vcpu->stat.deliver_service_signal++; 992 trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, KVM_S390_INT_SERVICE, 993 ext.ext_params, 0); 994 995 return write_sclp(vcpu, ext.ext_params & SCCB_EVENT_PENDING); 996 } 997 998 static int __must_check __deliver_pfault_done(struct kvm_vcpu *vcpu) 999 { 1000 struct kvm_s390_float_interrupt *fi = &vcpu->kvm->arch.float_int; 1001 struct kvm_s390_interrupt_info *inti; 1002 int rc = 0; 1003 unsigned long flags; 1004 1005 spin_lock_irqsave(&fi->lock, flags); 1006 inti = list_first_entry_or_null(&fi->lists[FIRQ_LIST_PFAULT], 1007 struct kvm_s390_interrupt_info, 1008 list); 1009 if (inti) { 1010 list_del(&inti->list); 1011 fi->counters[FIRQ_CNTR_PFAULT] -= 1; 1012 } 1013 if (list_empty(&fi->lists[FIRQ_LIST_PFAULT])) 1014 clear_bit(IRQ_PEND_PFAULT_DONE, &fi->pending_irqs); 1015 spin_unlock_irqrestore(&fi->lock, flags); 1016 1017 if (inti) { 1018 trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, 1019 KVM_S390_INT_PFAULT_DONE, 0, 1020 inti->ext.ext_params2); 1021 VCPU_EVENT(vcpu, 4, "deliver: pfault done token 0x%llx", 1022 inti->ext.ext_params2); 1023 1024 rc = put_guest_lc(vcpu, EXT_IRQ_CP_SERVICE, 1025 (u16 *)__LC_EXT_INT_CODE); 1026 rc |= put_guest_lc(vcpu, PFAULT_DONE, 1027 (u16 *)__LC_EXT_CPU_ADDR); 1028 rc |= write_guest_lc(vcpu, __LC_EXT_OLD_PSW, 1029 &vcpu->arch.sie_block->gpsw, 1030 sizeof(psw_t)); 1031 rc |= read_guest_lc(vcpu, __LC_EXT_NEW_PSW, 1032 &vcpu->arch.sie_block->gpsw, 1033 sizeof(psw_t)); 1034 rc |= put_guest_lc(vcpu, inti->ext.ext_params2, 1035 (u64 *)__LC_EXT_PARAMS2); 1036 kfree(inti); 1037 } 1038 return rc ? -EFAULT : 0; 1039 } 1040 1041 static int __must_check __deliver_virtio(struct kvm_vcpu *vcpu) 1042 { 1043 struct kvm_s390_float_interrupt *fi = &vcpu->kvm->arch.float_int; 1044 struct kvm_s390_interrupt_info *inti; 1045 int rc = 0; 1046 unsigned long flags; 1047 1048 spin_lock_irqsave(&fi->lock, flags); 1049 inti = list_first_entry_or_null(&fi->lists[FIRQ_LIST_VIRTIO], 1050 struct kvm_s390_interrupt_info, 1051 list); 1052 if (inti) { 1053 VCPU_EVENT(vcpu, 4, 1054 "deliver: virtio parm: 0x%x,parm64: 0x%llx", 1055 inti->ext.ext_params, inti->ext.ext_params2); 1056 vcpu->stat.deliver_virtio++; 1057 trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, 1058 inti->type, 1059 inti->ext.ext_params, 1060 inti->ext.ext_params2); 1061 list_del(&inti->list); 1062 fi->counters[FIRQ_CNTR_VIRTIO] -= 1; 1063 } 1064 if (list_empty(&fi->lists[FIRQ_LIST_VIRTIO])) 1065 clear_bit(IRQ_PEND_VIRTIO, &fi->pending_irqs); 1066 spin_unlock_irqrestore(&fi->lock, flags); 1067 1068 if (inti) { 1069 rc = put_guest_lc(vcpu, EXT_IRQ_CP_SERVICE, 1070 (u16 *)__LC_EXT_INT_CODE); 1071 rc |= put_guest_lc(vcpu, VIRTIO_PARAM, 1072 (u16 *)__LC_EXT_CPU_ADDR); 1073 rc |= write_guest_lc(vcpu, __LC_EXT_OLD_PSW, 1074 &vcpu->arch.sie_block->gpsw, 1075 sizeof(psw_t)); 1076 rc |= read_guest_lc(vcpu, __LC_EXT_NEW_PSW, 1077 &vcpu->arch.sie_block->gpsw, 1078 sizeof(psw_t)); 1079 rc |= put_guest_lc(vcpu, inti->ext.ext_params, 1080 (u32 *)__LC_EXT_PARAMS); 1081 rc |= put_guest_lc(vcpu, inti->ext.ext_params2, 1082 (u64 *)__LC_EXT_PARAMS2); 1083 kfree(inti); 1084 } 1085 return rc ? -EFAULT : 0; 1086 } 1087 1088 static int __do_deliver_io(struct kvm_vcpu *vcpu, struct kvm_s390_io_info *io) 1089 { 1090 int rc; 1091 1092 if (kvm_s390_pv_cpu_is_protected(vcpu)) { 1093 vcpu->arch.sie_block->iictl = IICTL_CODE_IO; 1094 vcpu->arch.sie_block->subchannel_id = io->subchannel_id; 1095 vcpu->arch.sie_block->subchannel_nr = io->subchannel_nr; 1096 vcpu->arch.sie_block->io_int_parm = io->io_int_parm; 1097 vcpu->arch.sie_block->io_int_word = io->io_int_word; 1098 return 0; 1099 } 1100 1101 rc = put_guest_lc(vcpu, io->subchannel_id, (u16 *)__LC_SUBCHANNEL_ID); 1102 rc |= put_guest_lc(vcpu, io->subchannel_nr, (u16 *)__LC_SUBCHANNEL_NR); 1103 rc |= put_guest_lc(vcpu, io->io_int_parm, (u32 *)__LC_IO_INT_PARM); 1104 rc |= put_guest_lc(vcpu, io->io_int_word, (u32 *)__LC_IO_INT_WORD); 1105 rc |= write_guest_lc(vcpu, __LC_IO_OLD_PSW, 1106 &vcpu->arch.sie_block->gpsw, 1107 sizeof(psw_t)); 1108 rc |= read_guest_lc(vcpu, __LC_IO_NEW_PSW, 1109 &vcpu->arch.sie_block->gpsw, 1110 sizeof(psw_t)); 1111 return rc ? -EFAULT : 0; 1112 } 1113 1114 static int __must_check __deliver_io(struct kvm_vcpu *vcpu, 1115 unsigned long irq_type) 1116 { 1117 struct list_head *isc_list; 1118 struct kvm_s390_float_interrupt *fi; 1119 struct kvm_s390_gisa_interrupt *gi = &vcpu->kvm->arch.gisa_int; 1120 struct kvm_s390_interrupt_info *inti = NULL; 1121 struct kvm_s390_io_info io; 1122 u32 isc; 1123 int rc = 0; 1124 unsigned long flags; 1125 1126 fi = &vcpu->kvm->arch.float_int; 1127 1128 spin_lock_irqsave(&fi->lock, flags); 1129 isc = irq_type_to_isc(irq_type); 1130 isc_list = &fi->lists[isc]; 1131 inti = list_first_entry_or_null(isc_list, 1132 struct kvm_s390_interrupt_info, 1133 list); 1134 if (inti) { 1135 if (inti->type & KVM_S390_INT_IO_AI_MASK) 1136 VCPU_EVENT(vcpu, 4, "%s", "deliver: I/O (AI)"); 1137 else 1138 VCPU_EVENT(vcpu, 4, "deliver: I/O %x ss %x schid %04x", 1139 inti->io.subchannel_id >> 8, 1140 inti->io.subchannel_id >> 1 & 0x3, 1141 inti->io.subchannel_nr); 1142 1143 vcpu->stat.deliver_io++; 1144 trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, 1145 inti->type, 1146 ((__u32)inti->io.subchannel_id << 16) | 1147 inti->io.subchannel_nr, 1148 ((__u64)inti->io.io_int_parm << 32) | 1149 inti->io.io_int_word); 1150 list_del(&inti->list); 1151 fi->counters[FIRQ_CNTR_IO] -= 1; 1152 } 1153 if (list_empty(isc_list)) 1154 clear_bit(irq_type, &fi->pending_irqs); 1155 spin_unlock_irqrestore(&fi->lock, flags); 1156 1157 if (inti) { 1158 rc = __do_deliver_io(vcpu, &(inti->io)); 1159 kfree(inti); 1160 goto out; 1161 } 1162 1163 if (gi->origin && gisa_tac_ipm_gisc(gi->origin, isc)) { 1164 /* 1165 * in case an adapter interrupt was not delivered 1166 * in SIE context KVM will handle the delivery 1167 */ 1168 VCPU_EVENT(vcpu, 4, "%s isc %u", "deliver: I/O (AI/gisa)", isc); 1169 memset(&io, 0, sizeof(io)); 1170 io.io_int_word = isc_to_int_word(isc); 1171 vcpu->stat.deliver_io++; 1172 trace_kvm_s390_deliver_interrupt(vcpu->vcpu_id, 1173 KVM_S390_INT_IO(1, 0, 0, 0), 1174 ((__u32)io.subchannel_id << 16) | 1175 io.subchannel_nr, 1176 ((__u64)io.io_int_parm << 32) | 1177 io.io_int_word); 1178 rc = __do_deliver_io(vcpu, &io); 1179 } 1180 out: 1181 return rc; 1182 } 1183 1184 /* Check whether an external call is pending (deliverable or not) */ 1185 int kvm_s390_ext_call_pending(struct kvm_vcpu *vcpu) 1186 { 1187 struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int; 1188 1189 if (!kvm_s390_use_sca_entries()) 1190 return test_bit(IRQ_PEND_EXT_EXTERNAL, &li->pending_irqs); 1191 1192 return sca_ext_call_pending(vcpu, NULL); 1193 } 1194 1195 int kvm_s390_vcpu_has_irq(struct kvm_vcpu *vcpu, int exclude_stop) 1196 { 1197 if (deliverable_irqs(vcpu)) 1198 return 1; 1199 1200 if (kvm_cpu_has_pending_timer(vcpu)) 1201 return 1; 1202 1203 /* external call pending and deliverable */ 1204 if (kvm_s390_ext_call_pending(vcpu) && 1205 !psw_extint_disabled(vcpu) && 1206 (vcpu->arch.sie_block->gcr[0] & CR0_EXTERNAL_CALL_SUBMASK)) 1207 return 1; 1208 1209 if (!exclude_stop && kvm_s390_is_stop_irq_pending(vcpu)) 1210 return 1; 1211 return 0; 1212 } 1213 1214 int kvm_cpu_has_pending_timer(struct kvm_vcpu *vcpu) 1215 { 1216 return ckc_irq_pending(vcpu) || cpu_timer_irq_pending(vcpu); 1217 } 1218 1219 static u64 __calculate_sltime(struct kvm_vcpu *vcpu) 1220 { 1221 const u64 now = kvm_s390_get_tod_clock_fast(vcpu->kvm); 1222 const u64 ckc = vcpu->arch.sie_block->ckc; 1223 u64 cputm, sltime = 0; 1224 1225 if (ckc_interrupts_enabled(vcpu)) { 1226 if (vcpu->arch.sie_block->gcr[0] & CR0_CLOCK_COMPARATOR_SIGN) { 1227 if ((s64)now < (s64)ckc) 1228 sltime = tod_to_ns((s64)ckc - (s64)now); 1229 } else if (now < ckc) { 1230 sltime = tod_to_ns(ckc - now); 1231 } 1232 /* already expired */ 1233 if (!sltime) 1234 return 0; 1235 if (cpu_timer_interrupts_enabled(vcpu)) { 1236 cputm = kvm_s390_get_cpu_timer(vcpu); 1237 /* already expired? */ 1238 if (cputm >> 63) 1239 return 0; 1240 return min_t(u64, sltime, tod_to_ns(cputm)); 1241 } 1242 } else if (cpu_timer_interrupts_enabled(vcpu)) { 1243 sltime = kvm_s390_get_cpu_timer(vcpu); 1244 /* already expired? */ 1245 if (sltime >> 63) 1246 return 0; 1247 } 1248 return sltime; 1249 } 1250 1251 int kvm_s390_handle_wait(struct kvm_vcpu *vcpu) 1252 { 1253 struct kvm_s390_gisa_interrupt *gi = &vcpu->kvm->arch.gisa_int; 1254 u64 sltime; 1255 1256 vcpu->stat.exit_wait_state++; 1257 1258 /* fast path */ 1259 if (kvm_arch_vcpu_runnable(vcpu)) 1260 return 0; 1261 1262 if (psw_interrupts_disabled(vcpu)) { 1263 VCPU_EVENT(vcpu, 3, "%s", "disabled wait"); 1264 return -EOPNOTSUPP; /* disabled wait */ 1265 } 1266 1267 if (gi->origin && 1268 (gisa_get_ipm_or_restore_iam(gi) & 1269 vcpu->arch.sie_block->gcr[6] >> 24)) 1270 return 0; 1271 1272 if (!ckc_interrupts_enabled(vcpu) && 1273 !cpu_timer_interrupts_enabled(vcpu)) { 1274 VCPU_EVENT(vcpu, 3, "%s", "enabled wait w/o timer"); 1275 __set_cpu_idle(vcpu); 1276 goto no_timer; 1277 } 1278 1279 sltime = __calculate_sltime(vcpu); 1280 if (!sltime) 1281 return 0; 1282 1283 __set_cpu_idle(vcpu); 1284 hrtimer_start(&vcpu->arch.ckc_timer, sltime, HRTIMER_MODE_REL); 1285 VCPU_EVENT(vcpu, 4, "enabled wait: %llu ns", sltime); 1286 no_timer: 1287 kvm_vcpu_srcu_read_unlock(vcpu); 1288 vcpu->kvm->arch.float_int.last_sleep_cpu = vcpu->vcpu_idx; 1289 kvm_vcpu_halt(vcpu); 1290 vcpu->valid_wakeup = false; 1291 __unset_cpu_idle(vcpu); 1292 kvm_vcpu_srcu_read_lock(vcpu); 1293 1294 hrtimer_cancel(&vcpu->arch.ckc_timer); 1295 return 0; 1296 } 1297 1298 void kvm_s390_vcpu_wakeup(struct kvm_vcpu *vcpu) 1299 { 1300 vcpu->valid_wakeup = true; 1301 kvm_vcpu_wake_up(vcpu); 1302 1303 /* 1304 * The VCPU might not be sleeping but rather executing VSIE. Let's 1305 * kick it, so it leaves the SIE to process the request. 1306 */ 1307 kvm_s390_vsie_kick(vcpu); 1308 } 1309 1310 enum hrtimer_restart kvm_s390_idle_wakeup(struct hrtimer *timer) 1311 { 1312 struct kvm_vcpu *vcpu; 1313 u64 sltime; 1314 1315 vcpu = container_of(timer, struct kvm_vcpu, arch.ckc_timer); 1316 sltime = __calculate_sltime(vcpu); 1317 1318 /* 1319 * If the monotonic clock runs faster than the tod clock we might be 1320 * woken up too early and have to go back to sleep to avoid deadlocks. 1321 */ 1322 if (sltime && hrtimer_forward_now(timer, ns_to_ktime(sltime))) 1323 return HRTIMER_RESTART; 1324 kvm_s390_vcpu_wakeup(vcpu); 1325 return HRTIMER_NORESTART; 1326 } 1327 1328 void kvm_s390_clear_local_irqs(struct kvm_vcpu *vcpu) 1329 { 1330 struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int; 1331 1332 spin_lock(&li->lock); 1333 li->pending_irqs = 0; 1334 bitmap_zero(li->sigp_emerg_pending, KVM_MAX_VCPUS); 1335 memset(&li->irq, 0, sizeof(li->irq)); 1336 spin_unlock(&li->lock); 1337 1338 sca_clear_ext_call(vcpu); 1339 } 1340 1341 int __must_check kvm_s390_deliver_pending_interrupts(struct kvm_vcpu *vcpu) 1342 { 1343 struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int; 1344 int rc = 0; 1345 bool delivered = false; 1346 unsigned long irq_type; 1347 unsigned long irqs; 1348 1349 __reset_intercept_indicators(vcpu); 1350 1351 /* pending ckc conditions might have been invalidated */ 1352 clear_bit(IRQ_PEND_EXT_CLOCK_COMP, &li->pending_irqs); 1353 if (ckc_irq_pending(vcpu)) 1354 set_bit(IRQ_PEND_EXT_CLOCK_COMP, &li->pending_irqs); 1355 1356 /* pending cpu timer conditions might have been invalidated */ 1357 clear_bit(IRQ_PEND_EXT_CPU_TIMER, &li->pending_irqs); 1358 if (cpu_timer_irq_pending(vcpu)) 1359 set_bit(IRQ_PEND_EXT_CPU_TIMER, &li->pending_irqs); 1360 1361 while ((irqs = deliverable_irqs(vcpu)) && !rc) { 1362 /* bits are in the reverse order of interrupt priority */ 1363 irq_type = find_last_bit(&irqs, IRQ_PEND_COUNT); 1364 switch (irq_type) { 1365 case IRQ_PEND_IO_ISC_0: 1366 case IRQ_PEND_IO_ISC_1: 1367 case IRQ_PEND_IO_ISC_2: 1368 case IRQ_PEND_IO_ISC_3: 1369 case IRQ_PEND_IO_ISC_4: 1370 case IRQ_PEND_IO_ISC_5: 1371 case IRQ_PEND_IO_ISC_6: 1372 case IRQ_PEND_IO_ISC_7: 1373 rc = __deliver_io(vcpu, irq_type); 1374 break; 1375 case IRQ_PEND_MCHK_EX: 1376 case IRQ_PEND_MCHK_REP: 1377 rc = __deliver_machine_check(vcpu); 1378 break; 1379 case IRQ_PEND_PROG: 1380 rc = __deliver_prog(vcpu); 1381 break; 1382 case IRQ_PEND_EXT_EMERGENCY: 1383 rc = __deliver_emergency_signal(vcpu); 1384 break; 1385 case IRQ_PEND_EXT_EXTERNAL: 1386 rc = __deliver_external_call(vcpu); 1387 break; 1388 case IRQ_PEND_EXT_CLOCK_COMP: 1389 rc = __deliver_ckc(vcpu); 1390 break; 1391 case IRQ_PEND_EXT_CPU_TIMER: 1392 rc = __deliver_cpu_timer(vcpu); 1393 break; 1394 case IRQ_PEND_RESTART: 1395 rc = __deliver_restart(vcpu); 1396 break; 1397 case IRQ_PEND_SET_PREFIX: 1398 rc = __deliver_set_prefix(vcpu); 1399 break; 1400 case IRQ_PEND_PFAULT_INIT: 1401 rc = __deliver_pfault_init(vcpu); 1402 break; 1403 case IRQ_PEND_EXT_SERVICE: 1404 rc = __deliver_service(vcpu); 1405 break; 1406 case IRQ_PEND_EXT_SERVICE_EV: 1407 rc = __deliver_service_ev(vcpu); 1408 break; 1409 case IRQ_PEND_PFAULT_DONE: 1410 rc = __deliver_pfault_done(vcpu); 1411 break; 1412 case IRQ_PEND_VIRTIO: 1413 rc = __deliver_virtio(vcpu); 1414 break; 1415 default: 1416 WARN_ONCE(1, "Unknown pending irq type %ld", irq_type); 1417 clear_bit(irq_type, &li->pending_irqs); 1418 } 1419 delivered |= !rc; 1420 } 1421 1422 /* 1423 * We delivered at least one interrupt and modified the PC. Force a 1424 * singlestep event now. 1425 */ 1426 if (delivered && guestdbg_sstep_enabled(vcpu)) { 1427 struct kvm_debug_exit_arch *debug_exit = &vcpu->run->debug.arch; 1428 1429 debug_exit->addr = vcpu->arch.sie_block->gpsw.addr; 1430 debug_exit->type = KVM_SINGLESTEP; 1431 vcpu->guest_debug |= KVM_GUESTDBG_EXIT_PENDING; 1432 } 1433 1434 set_intercept_indicators(vcpu); 1435 1436 return rc; 1437 } 1438 1439 static int __inject_prog(struct kvm_vcpu *vcpu, struct kvm_s390_irq *irq) 1440 { 1441 struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int; 1442 1443 vcpu->stat.inject_program++; 1444 VCPU_EVENT(vcpu, 3, "inject: program irq code 0x%x", irq->u.pgm.code); 1445 trace_kvm_s390_inject_vcpu(vcpu->vcpu_id, KVM_S390_PROGRAM_INT, 1446 irq->u.pgm.code, 0); 1447 1448 if (!(irq->u.pgm.flags & KVM_S390_PGM_FLAGS_ILC_VALID)) { 1449 /* auto detection if no valid ILC was given */ 1450 irq->u.pgm.flags &= ~KVM_S390_PGM_FLAGS_ILC_MASK; 1451 irq->u.pgm.flags |= kvm_s390_get_ilen(vcpu); 1452 irq->u.pgm.flags |= KVM_S390_PGM_FLAGS_ILC_VALID; 1453 } 1454 1455 if (irq->u.pgm.code == PGM_PER) { 1456 li->irq.pgm.code |= PGM_PER; 1457 li->irq.pgm.flags = irq->u.pgm.flags; 1458 /* only modify PER related information */ 1459 li->irq.pgm.per_address = irq->u.pgm.per_address; 1460 li->irq.pgm.per_code = irq->u.pgm.per_code; 1461 li->irq.pgm.per_atmid = irq->u.pgm.per_atmid; 1462 li->irq.pgm.per_access_id = irq->u.pgm.per_access_id; 1463 } else if (!(irq->u.pgm.code & PGM_PER)) { 1464 li->irq.pgm.code = (li->irq.pgm.code & PGM_PER) | 1465 irq->u.pgm.code; 1466 li->irq.pgm.flags = irq->u.pgm.flags; 1467 /* only modify non-PER information */ 1468 li->irq.pgm.trans_exc_code = irq->u.pgm.trans_exc_code; 1469 li->irq.pgm.mon_code = irq->u.pgm.mon_code; 1470 li->irq.pgm.data_exc_code = irq->u.pgm.data_exc_code; 1471 li->irq.pgm.mon_class_nr = irq->u.pgm.mon_class_nr; 1472 li->irq.pgm.exc_access_id = irq->u.pgm.exc_access_id; 1473 li->irq.pgm.op_access_id = irq->u.pgm.op_access_id; 1474 } else { 1475 li->irq.pgm = irq->u.pgm; 1476 } 1477 set_bit(IRQ_PEND_PROG, &li->pending_irqs); 1478 return 0; 1479 } 1480 1481 static int __inject_pfault_init(struct kvm_vcpu *vcpu, struct kvm_s390_irq *irq) 1482 { 1483 struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int; 1484 1485 vcpu->stat.inject_pfault_init++; 1486 VCPU_EVENT(vcpu, 4, "inject: pfault init parameter block at 0x%llx", 1487 irq->u.ext.ext_params2); 1488 trace_kvm_s390_inject_vcpu(vcpu->vcpu_id, KVM_S390_INT_PFAULT_INIT, 1489 irq->u.ext.ext_params, 1490 irq->u.ext.ext_params2); 1491 1492 li->irq.ext = irq->u.ext; 1493 set_bit(IRQ_PEND_PFAULT_INIT, &li->pending_irqs); 1494 kvm_s390_set_cpuflags(vcpu, CPUSTAT_EXT_INT); 1495 return 0; 1496 } 1497 1498 static int __inject_extcall(struct kvm_vcpu *vcpu, struct kvm_s390_irq *irq) 1499 { 1500 struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int; 1501 struct kvm_s390_extcall_info *extcall = &li->irq.extcall; 1502 uint16_t src_id = irq->u.extcall.code; 1503 1504 vcpu->stat.inject_external_call++; 1505 VCPU_EVENT(vcpu, 4, "inject: external call source-cpu:%u", 1506 src_id); 1507 trace_kvm_s390_inject_vcpu(vcpu->vcpu_id, KVM_S390_INT_EXTERNAL_CALL, 1508 src_id, 0); 1509 1510 /* sending vcpu invalid */ 1511 if (kvm_get_vcpu_by_id(vcpu->kvm, src_id) == NULL) 1512 return -EINVAL; 1513 1514 if (kvm_s390_use_sca_entries() && !kvm_s390_pv_cpu_get_handle(vcpu)) 1515 return sca_inject_ext_call(vcpu, src_id); 1516 1517 if (test_and_set_bit(IRQ_PEND_EXT_EXTERNAL, &li->pending_irqs)) 1518 return -EBUSY; 1519 *extcall = irq->u.extcall; 1520 kvm_s390_set_cpuflags(vcpu, CPUSTAT_EXT_INT); 1521 return 0; 1522 } 1523 1524 static int __inject_set_prefix(struct kvm_vcpu *vcpu, struct kvm_s390_irq *irq) 1525 { 1526 struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int; 1527 struct kvm_s390_prefix_info *prefix = &li->irq.prefix; 1528 1529 vcpu->stat.inject_set_prefix++; 1530 VCPU_EVENT(vcpu, 3, "inject: set prefix to %x", 1531 irq->u.prefix.address); 1532 trace_kvm_s390_inject_vcpu(vcpu->vcpu_id, KVM_S390_SIGP_SET_PREFIX, 1533 irq->u.prefix.address, 0); 1534 1535 if (!is_vcpu_stopped(vcpu)) 1536 return -EBUSY; 1537 1538 *prefix = irq->u.prefix; 1539 set_bit(IRQ_PEND_SET_PREFIX, &li->pending_irqs); 1540 return 0; 1541 } 1542 1543 #define KVM_S390_STOP_SUPP_FLAGS (KVM_S390_STOP_FLAG_STORE_STATUS) 1544 static int __inject_sigp_stop(struct kvm_vcpu *vcpu, struct kvm_s390_irq *irq) 1545 { 1546 struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int; 1547 struct kvm_s390_stop_info *stop = &li->irq.stop; 1548 int rc = 0; 1549 1550 vcpu->stat.inject_stop_signal++; 1551 trace_kvm_s390_inject_vcpu(vcpu->vcpu_id, KVM_S390_SIGP_STOP, 0, 0); 1552 1553 if (irq->u.stop.flags & ~KVM_S390_STOP_SUPP_FLAGS) 1554 return -EINVAL; 1555 1556 if (is_vcpu_stopped(vcpu)) { 1557 if (irq->u.stop.flags & KVM_S390_STOP_FLAG_STORE_STATUS) 1558 rc = kvm_s390_store_status_unloaded(vcpu, 1559 KVM_S390_STORE_STATUS_NOADDR); 1560 return rc; 1561 } 1562 1563 if (test_and_set_bit(IRQ_PEND_SIGP_STOP, &li->pending_irqs)) 1564 return -EBUSY; 1565 stop->flags = irq->u.stop.flags; 1566 kvm_s390_set_cpuflags(vcpu, CPUSTAT_STOP_INT); 1567 return 0; 1568 } 1569 1570 static int __inject_sigp_restart(struct kvm_vcpu *vcpu) 1571 { 1572 struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int; 1573 1574 vcpu->stat.inject_restart++; 1575 VCPU_EVENT(vcpu, 3, "%s", "inject: restart int"); 1576 trace_kvm_s390_inject_vcpu(vcpu->vcpu_id, KVM_S390_RESTART, 0, 0); 1577 1578 set_bit(IRQ_PEND_RESTART, &li->pending_irqs); 1579 return 0; 1580 } 1581 1582 static int __inject_sigp_emergency(struct kvm_vcpu *vcpu, 1583 struct kvm_s390_irq *irq) 1584 { 1585 struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int; 1586 1587 vcpu->stat.inject_emergency_signal++; 1588 VCPU_EVENT(vcpu, 4, "inject: emergency from cpu %u", 1589 irq->u.emerg.code); 1590 trace_kvm_s390_inject_vcpu(vcpu->vcpu_id, KVM_S390_INT_EMERGENCY, 1591 irq->u.emerg.code, 0); 1592 1593 /* sending vcpu invalid */ 1594 if (kvm_get_vcpu_by_id(vcpu->kvm, irq->u.emerg.code) == NULL) 1595 return -EINVAL; 1596 1597 set_bit(irq->u.emerg.code, li->sigp_emerg_pending); 1598 set_bit(IRQ_PEND_EXT_EMERGENCY, &li->pending_irqs); 1599 kvm_s390_set_cpuflags(vcpu, CPUSTAT_EXT_INT); 1600 return 0; 1601 } 1602 1603 static int __inject_mchk(struct kvm_vcpu *vcpu, struct kvm_s390_irq *irq) 1604 { 1605 struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int; 1606 struct kvm_s390_mchk_info *mchk = &li->irq.mchk; 1607 1608 vcpu->stat.inject_mchk++; 1609 VCPU_EVENT(vcpu, 3, "inject: machine check mcic 0x%llx", 1610 irq->u.mchk.mcic); 1611 trace_kvm_s390_inject_vcpu(vcpu->vcpu_id, KVM_S390_MCHK, 0, 1612 irq->u.mchk.mcic); 1613 1614 /* 1615 * Because repressible machine checks can be indicated along with 1616 * exigent machine checks (PoP, Chapter 11, Interruption action) 1617 * we need to combine cr14, mcic and external damage code. 1618 * Failing storage address and the logout area should not be or'ed 1619 * together, we just indicate the last occurrence of the corresponding 1620 * machine check 1621 */ 1622 mchk->cr14 |= irq->u.mchk.cr14; 1623 mchk->mcic |= irq->u.mchk.mcic; 1624 mchk->ext_damage_code |= irq->u.mchk.ext_damage_code; 1625 mchk->failing_storage_address = irq->u.mchk.failing_storage_address; 1626 memcpy(&mchk->fixed_logout, &irq->u.mchk.fixed_logout, 1627 sizeof(mchk->fixed_logout)); 1628 if (mchk->mcic & MCHK_EX_MASK) 1629 set_bit(IRQ_PEND_MCHK_EX, &li->pending_irqs); 1630 else if (mchk->mcic & MCHK_REP_MASK) 1631 set_bit(IRQ_PEND_MCHK_REP, &li->pending_irqs); 1632 return 0; 1633 } 1634 1635 static int __inject_ckc(struct kvm_vcpu *vcpu) 1636 { 1637 struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int; 1638 1639 vcpu->stat.inject_ckc++; 1640 VCPU_EVENT(vcpu, 3, "%s", "inject: clock comparator external"); 1641 trace_kvm_s390_inject_vcpu(vcpu->vcpu_id, KVM_S390_INT_CLOCK_COMP, 1642 0, 0); 1643 1644 set_bit(IRQ_PEND_EXT_CLOCK_COMP, &li->pending_irqs); 1645 kvm_s390_set_cpuflags(vcpu, CPUSTAT_EXT_INT); 1646 return 0; 1647 } 1648 1649 static int __inject_cpu_timer(struct kvm_vcpu *vcpu) 1650 { 1651 struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int; 1652 1653 vcpu->stat.inject_cputm++; 1654 VCPU_EVENT(vcpu, 3, "%s", "inject: cpu timer external"); 1655 trace_kvm_s390_inject_vcpu(vcpu->vcpu_id, KVM_S390_INT_CPU_TIMER, 1656 0, 0); 1657 1658 set_bit(IRQ_PEND_EXT_CPU_TIMER, &li->pending_irqs); 1659 kvm_s390_set_cpuflags(vcpu, CPUSTAT_EXT_INT); 1660 return 0; 1661 } 1662 1663 static struct kvm_s390_interrupt_info *get_io_int(struct kvm *kvm, 1664 int isc, u32 schid) 1665 { 1666 struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int; 1667 struct list_head *isc_list = &fi->lists[FIRQ_LIST_IO_ISC_0 + isc]; 1668 struct kvm_s390_interrupt_info *iter; 1669 u16 id = (schid & 0xffff0000U) >> 16; 1670 u16 nr = schid & 0x0000ffffU; 1671 unsigned long flags; 1672 1673 spin_lock_irqsave(&fi->lock, flags); 1674 list_for_each_entry(iter, isc_list, list) { 1675 if (schid && (id != iter->io.subchannel_id || 1676 nr != iter->io.subchannel_nr)) 1677 continue; 1678 /* found an appropriate entry */ 1679 list_del_init(&iter->list); 1680 fi->counters[FIRQ_CNTR_IO] -= 1; 1681 if (list_empty(isc_list)) 1682 clear_bit(isc_to_irq_type(isc), &fi->pending_irqs); 1683 spin_unlock_irqrestore(&fi->lock, flags); 1684 return iter; 1685 } 1686 spin_unlock_irqrestore(&fi->lock, flags); 1687 return NULL; 1688 } 1689 1690 static struct kvm_s390_interrupt_info *get_top_io_int(struct kvm *kvm, 1691 u64 isc_mask, u32 schid) 1692 { 1693 struct kvm_s390_interrupt_info *inti = NULL; 1694 int isc; 1695 1696 for (isc = 0; isc <= MAX_ISC && !inti; isc++) { 1697 if (isc_mask & isc_to_isc_bits(isc)) 1698 inti = get_io_int(kvm, isc, schid); 1699 } 1700 return inti; 1701 } 1702 1703 static int get_top_gisa_isc(struct kvm *kvm, u64 isc_mask, u32 schid) 1704 { 1705 struct kvm_s390_gisa_interrupt *gi = &kvm->arch.gisa_int; 1706 unsigned long active_mask; 1707 int isc; 1708 1709 if (schid) 1710 goto out; 1711 if (!gi->origin) 1712 goto out; 1713 1714 active_mask = (isc_mask & gisa_get_ipm(gi->origin) << 24) << 32; 1715 while (active_mask) { 1716 isc = __fls(active_mask) ^ (BITS_PER_LONG - 1); 1717 if (gisa_tac_ipm_gisc(gi->origin, isc)) 1718 return isc; 1719 clear_bit_inv(isc, &active_mask); 1720 } 1721 out: 1722 return -EINVAL; 1723 } 1724 1725 /* 1726 * Dequeue and return an I/O interrupt matching any of the interruption 1727 * subclasses as designated by the isc mask in cr6 and the schid (if != 0). 1728 * Take into account the interrupts pending in the interrupt list and in GISA. 1729 * 1730 * Note that for a guest that does not enable I/O interrupts 1731 * but relies on TPI, a flood of classic interrupts may starve 1732 * out adapter interrupts on the same isc. Linux does not do 1733 * that, and it is possible to work around the issue by configuring 1734 * different iscs for classic and adapter interrupts in the guest, 1735 * but we may want to revisit this in the future. 1736 */ 1737 struct kvm_s390_interrupt_info *kvm_s390_get_io_int(struct kvm *kvm, 1738 u64 isc_mask, u32 schid) 1739 { 1740 struct kvm_s390_gisa_interrupt *gi = &kvm->arch.gisa_int; 1741 struct kvm_s390_interrupt_info *inti, *tmp_inti; 1742 int isc; 1743 1744 inti = get_top_io_int(kvm, isc_mask, schid); 1745 1746 isc = get_top_gisa_isc(kvm, isc_mask, schid); 1747 if (isc < 0) 1748 /* no AI in GISA */ 1749 goto out; 1750 1751 if (!inti) 1752 /* AI in GISA but no classical IO int */ 1753 goto gisa_out; 1754 1755 /* both types of interrupts present */ 1756 if (int_word_to_isc(inti->io.io_int_word) <= isc) { 1757 /* classical IO int with higher priority */ 1758 gisa_set_ipm_gisc(gi->origin, isc); 1759 goto out; 1760 } 1761 gisa_out: 1762 tmp_inti = kzalloc_obj(*inti, GFP_KERNEL_ACCOUNT); 1763 if (tmp_inti) { 1764 tmp_inti->type = KVM_S390_INT_IO(1, 0, 0, 0); 1765 tmp_inti->io.io_int_word = isc_to_int_word(isc); 1766 if (inti) 1767 kvm_s390_reinject_io_int(kvm, inti); 1768 inti = tmp_inti; 1769 } else 1770 gisa_set_ipm_gisc(gi->origin, isc); 1771 out: 1772 return inti; 1773 } 1774 1775 static int __inject_service(struct kvm *kvm, 1776 struct kvm_s390_interrupt_info *inti) 1777 { 1778 struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int; 1779 unsigned long flags; 1780 1781 kvm->stat.inject_service_signal++; 1782 spin_lock_irqsave(&fi->lock, flags); 1783 fi->srv_signal.ext_params |= inti->ext.ext_params & SCCB_EVENT_PENDING; 1784 1785 /* We always allow events, track them separately from the sccb ints */ 1786 if (fi->srv_signal.ext_params & SCCB_EVENT_PENDING) 1787 set_bit(IRQ_PEND_EXT_SERVICE_EV, &fi->pending_irqs); 1788 1789 /* 1790 * Early versions of the QEMU s390 bios will inject several 1791 * service interrupts after another without handling a 1792 * condition code indicating busy. 1793 * We will silently ignore those superfluous sccb values. 1794 * A future version of QEMU will take care of serialization 1795 * of servc requests 1796 */ 1797 if (fi->srv_signal.ext_params & SCCB_MASK) 1798 goto out; 1799 fi->srv_signal.ext_params |= inti->ext.ext_params & SCCB_MASK; 1800 set_bit(IRQ_PEND_EXT_SERVICE, &fi->pending_irqs); 1801 out: 1802 spin_unlock_irqrestore(&fi->lock, flags); 1803 kfree(inti); 1804 return 0; 1805 } 1806 1807 static int __inject_virtio(struct kvm *kvm, 1808 struct kvm_s390_interrupt_info *inti) 1809 { 1810 struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int; 1811 unsigned long flags; 1812 1813 kvm->stat.inject_virtio++; 1814 spin_lock_irqsave(&fi->lock, flags); 1815 if (fi->counters[FIRQ_CNTR_VIRTIO] >= KVM_S390_MAX_VIRTIO_IRQS) { 1816 spin_unlock_irqrestore(&fi->lock, flags); 1817 return -EBUSY; 1818 } 1819 fi->counters[FIRQ_CNTR_VIRTIO] += 1; 1820 list_add_tail(&inti->list, &fi->lists[FIRQ_LIST_VIRTIO]); 1821 set_bit(IRQ_PEND_VIRTIO, &fi->pending_irqs); 1822 spin_unlock_irqrestore(&fi->lock, flags); 1823 return 0; 1824 } 1825 1826 static int __inject_pfault_done(struct kvm *kvm, 1827 struct kvm_s390_interrupt_info *inti) 1828 { 1829 struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int; 1830 unsigned long flags; 1831 1832 kvm->stat.inject_pfault_done++; 1833 spin_lock_irqsave(&fi->lock, flags); 1834 if (fi->counters[FIRQ_CNTR_PFAULT] >= 1835 (ASYNC_PF_PER_VCPU * KVM_MAX_VCPUS)) { 1836 spin_unlock_irqrestore(&fi->lock, flags); 1837 return -EBUSY; 1838 } 1839 fi->counters[FIRQ_CNTR_PFAULT] += 1; 1840 list_add_tail(&inti->list, &fi->lists[FIRQ_LIST_PFAULT]); 1841 set_bit(IRQ_PEND_PFAULT_DONE, &fi->pending_irqs); 1842 spin_unlock_irqrestore(&fi->lock, flags); 1843 return 0; 1844 } 1845 1846 #define CR_PENDING_SUBCLASS 28 1847 static int __inject_float_mchk(struct kvm *kvm, 1848 struct kvm_s390_interrupt_info *inti) 1849 { 1850 struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int; 1851 unsigned long flags; 1852 1853 kvm->stat.inject_float_mchk++; 1854 spin_lock_irqsave(&fi->lock, flags); 1855 fi->mchk.cr14 |= inti->mchk.cr14 & (1UL << CR_PENDING_SUBCLASS); 1856 fi->mchk.mcic |= inti->mchk.mcic; 1857 set_bit(IRQ_PEND_MCHK_REP, &fi->pending_irqs); 1858 spin_unlock_irqrestore(&fi->lock, flags); 1859 kfree(inti); 1860 return 0; 1861 } 1862 1863 static int __inject_io(struct kvm *kvm, struct kvm_s390_interrupt_info *inti) 1864 { 1865 struct kvm_s390_gisa_interrupt *gi = &kvm->arch.gisa_int; 1866 struct kvm_s390_float_interrupt *fi; 1867 struct list_head *list; 1868 int isc; 1869 unsigned long flags; 1870 1871 kvm->stat.inject_io++; 1872 isc = int_word_to_isc(inti->io.io_int_word); 1873 1874 /* 1875 * We do not use the lock checking variant as this is just a 1876 * performance optimization and we do not hold the lock here. 1877 * This is ok as the code will pick interrupts from both "lists" 1878 * for delivery. 1879 */ 1880 if (gi->origin && inti->type & KVM_S390_INT_IO_AI_MASK) { 1881 VM_EVENT(kvm, 4, "%s isc %1u", "inject: I/O (AI/gisa)", isc); 1882 gisa_set_ipm_gisc(gi->origin, isc); 1883 kfree(inti); 1884 return 0; 1885 } 1886 1887 fi = &kvm->arch.float_int; 1888 spin_lock_irqsave(&fi->lock, flags); 1889 if (fi->counters[FIRQ_CNTR_IO] >= KVM_S390_MAX_FLOAT_IRQS) { 1890 spin_unlock_irqrestore(&fi->lock, flags); 1891 return -EBUSY; 1892 } 1893 fi->counters[FIRQ_CNTR_IO] += 1; 1894 1895 if (inti->type & KVM_S390_INT_IO_AI_MASK) 1896 VM_EVENT(kvm, 4, "%s", "inject: I/O (AI)"); 1897 else 1898 VM_EVENT(kvm, 4, "inject: I/O %x ss %x schid %04x", 1899 inti->io.subchannel_id >> 8, 1900 inti->io.subchannel_id >> 1 & 0x3, 1901 inti->io.subchannel_nr); 1902 list = &fi->lists[FIRQ_LIST_IO_ISC_0 + isc]; 1903 list_add_tail(&inti->list, list); 1904 set_bit(isc_to_irq_type(isc), &fi->pending_irqs); 1905 spin_unlock_irqrestore(&fi->lock, flags); 1906 return 0; 1907 } 1908 1909 /* 1910 * Find a destination VCPU for a floating irq and kick it. 1911 */ 1912 static void __floating_irq_kick(struct kvm *kvm, u64 type) 1913 { 1914 struct kvm_vcpu *dst_vcpu; 1915 int sigcpu, online_vcpus, nr_tries = 0; 1916 1917 online_vcpus = atomic_read(&kvm->online_vcpus); 1918 if (!online_vcpus) 1919 return; 1920 1921 for (sigcpu = kvm->arch.float_int.last_sleep_cpu; ; sigcpu++) { 1922 sigcpu %= online_vcpus; 1923 dst_vcpu = kvm_get_vcpu(kvm, sigcpu); 1924 if (!is_vcpu_stopped(dst_vcpu)) 1925 break; 1926 /* avoid endless loops if all vcpus are stopped */ 1927 if (nr_tries++ >= online_vcpus) 1928 return; 1929 } 1930 1931 /* make the VCPU drop out of the SIE, or wake it up if sleeping */ 1932 switch (type) { 1933 case KVM_S390_MCHK: 1934 kvm_s390_set_cpuflags(dst_vcpu, CPUSTAT_STOP_INT); 1935 break; 1936 case KVM_S390_INT_IO_MIN...KVM_S390_INT_IO_MAX: 1937 if (!(type & KVM_S390_INT_IO_AI_MASK && 1938 kvm->arch.gisa_int.origin) || 1939 kvm_s390_pv_cpu_get_handle(dst_vcpu)) 1940 kvm_s390_set_cpuflags(dst_vcpu, CPUSTAT_IO_INT); 1941 break; 1942 default: 1943 kvm_s390_set_cpuflags(dst_vcpu, CPUSTAT_EXT_INT); 1944 break; 1945 } 1946 kvm_s390_vcpu_wakeup(dst_vcpu); 1947 } 1948 1949 static int __inject_vm(struct kvm *kvm, struct kvm_s390_interrupt_info *inti) 1950 { 1951 u64 type = READ_ONCE(inti->type); 1952 int rc; 1953 1954 switch (type) { 1955 case KVM_S390_MCHK: 1956 rc = __inject_float_mchk(kvm, inti); 1957 break; 1958 case KVM_S390_INT_VIRTIO: 1959 rc = __inject_virtio(kvm, inti); 1960 break; 1961 case KVM_S390_INT_SERVICE: 1962 rc = __inject_service(kvm, inti); 1963 break; 1964 case KVM_S390_INT_PFAULT_DONE: 1965 rc = __inject_pfault_done(kvm, inti); 1966 break; 1967 case KVM_S390_INT_IO_MIN...KVM_S390_INT_IO_MAX: 1968 rc = __inject_io(kvm, inti); 1969 break; 1970 default: 1971 rc = -EINVAL; 1972 } 1973 if (rc) 1974 return rc; 1975 1976 __floating_irq_kick(kvm, type); 1977 return 0; 1978 } 1979 1980 int kvm_s390_inject_vm(struct kvm *kvm, 1981 struct kvm_s390_interrupt *s390int, struct kvm_s390_interrupt_info *inti) 1982 { 1983 int rc; 1984 1985 inti->type = s390int->type; 1986 switch (inti->type) { 1987 case KVM_S390_INT_VIRTIO: 1988 VM_EVENT(kvm, 5, "inject: virtio parm:%x,parm64:%llx", 1989 s390int->parm, s390int->parm64); 1990 inti->ext.ext_params = s390int->parm; 1991 inti->ext.ext_params2 = s390int->parm64; 1992 break; 1993 case KVM_S390_INT_SERVICE: 1994 VM_EVENT(kvm, 4, "inject: sclp parm:%x", s390int->parm); 1995 inti->ext.ext_params = s390int->parm; 1996 break; 1997 case KVM_S390_INT_PFAULT_DONE: 1998 inti->ext.ext_params2 = s390int->parm64; 1999 break; 2000 case KVM_S390_MCHK: 2001 VM_EVENT(kvm, 3, "inject: machine check mcic 0x%llx", 2002 s390int->parm64); 2003 inti->mchk.cr14 = s390int->parm; /* upper bits are not used */ 2004 inti->mchk.mcic = s390int->parm64; 2005 break; 2006 case KVM_S390_INT_IO_MIN...KVM_S390_INT_IO_MAX: 2007 inti->io.subchannel_id = s390int->parm >> 16; 2008 inti->io.subchannel_nr = s390int->parm & 0x0000ffffu; 2009 inti->io.io_int_parm = s390int->parm64 >> 32; 2010 inti->io.io_int_word = s390int->parm64 & 0x00000000ffffffffull; 2011 break; 2012 default: 2013 return -EINVAL; 2014 } 2015 trace_kvm_s390_inject_vm(s390int->type, s390int->parm, s390int->parm64, 2016 2); 2017 2018 rc = __inject_vm(kvm, inti); 2019 2020 return rc; 2021 } 2022 2023 int kvm_s390_reinject_io_int(struct kvm *kvm, 2024 struct kvm_s390_interrupt_info *inti) 2025 { 2026 return __inject_vm(kvm, inti); 2027 } 2028 2029 int s390int_to_s390irq(struct kvm_s390_interrupt *s390int, 2030 struct kvm_s390_irq *irq) 2031 { 2032 irq->type = s390int->type; 2033 switch (irq->type) { 2034 case KVM_S390_PROGRAM_INT: 2035 if (s390int->parm & 0xffff0000) 2036 return -EINVAL; 2037 irq->u.pgm.code = s390int->parm; 2038 break; 2039 case KVM_S390_SIGP_SET_PREFIX: 2040 irq->u.prefix.address = s390int->parm; 2041 break; 2042 case KVM_S390_SIGP_STOP: 2043 irq->u.stop.flags = s390int->parm; 2044 break; 2045 case KVM_S390_INT_EXTERNAL_CALL: 2046 if (s390int->parm & 0xffff0000) 2047 return -EINVAL; 2048 irq->u.extcall.code = s390int->parm; 2049 break; 2050 case KVM_S390_INT_EMERGENCY: 2051 if (s390int->parm & 0xffff0000) 2052 return -EINVAL; 2053 irq->u.emerg.code = s390int->parm; 2054 break; 2055 case KVM_S390_MCHK: 2056 irq->u.mchk.mcic = s390int->parm64; 2057 break; 2058 case KVM_S390_INT_PFAULT_INIT: 2059 irq->u.ext.ext_params = s390int->parm; 2060 irq->u.ext.ext_params2 = s390int->parm64; 2061 break; 2062 case KVM_S390_RESTART: 2063 case KVM_S390_INT_CLOCK_COMP: 2064 case KVM_S390_INT_CPU_TIMER: 2065 break; 2066 default: 2067 return -EINVAL; 2068 } 2069 return 0; 2070 } 2071 2072 int kvm_s390_is_stop_irq_pending(struct kvm_vcpu *vcpu) 2073 { 2074 struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int; 2075 2076 return test_bit(IRQ_PEND_SIGP_STOP, &li->pending_irqs); 2077 } 2078 2079 int kvm_s390_is_restart_irq_pending(struct kvm_vcpu *vcpu) 2080 { 2081 struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int; 2082 2083 return test_bit(IRQ_PEND_RESTART, &li->pending_irqs); 2084 } 2085 2086 void kvm_s390_clear_stop_irq(struct kvm_vcpu *vcpu) 2087 { 2088 struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int; 2089 2090 spin_lock(&li->lock); 2091 li->irq.stop.flags = 0; 2092 clear_bit(IRQ_PEND_SIGP_STOP, &li->pending_irqs); 2093 spin_unlock(&li->lock); 2094 } 2095 2096 static int do_inject_vcpu(struct kvm_vcpu *vcpu, struct kvm_s390_irq *irq) 2097 { 2098 int rc; 2099 2100 switch (irq->type) { 2101 case KVM_S390_PROGRAM_INT: 2102 rc = __inject_prog(vcpu, irq); 2103 break; 2104 case KVM_S390_SIGP_SET_PREFIX: 2105 rc = __inject_set_prefix(vcpu, irq); 2106 break; 2107 case KVM_S390_SIGP_STOP: 2108 rc = __inject_sigp_stop(vcpu, irq); 2109 break; 2110 case KVM_S390_RESTART: 2111 rc = __inject_sigp_restart(vcpu); 2112 break; 2113 case KVM_S390_INT_CLOCK_COMP: 2114 rc = __inject_ckc(vcpu); 2115 break; 2116 case KVM_S390_INT_CPU_TIMER: 2117 rc = __inject_cpu_timer(vcpu); 2118 break; 2119 case KVM_S390_INT_EXTERNAL_CALL: 2120 rc = __inject_extcall(vcpu, irq); 2121 break; 2122 case KVM_S390_INT_EMERGENCY: 2123 rc = __inject_sigp_emergency(vcpu, irq); 2124 break; 2125 case KVM_S390_MCHK: 2126 rc = __inject_mchk(vcpu, irq); 2127 break; 2128 case KVM_S390_INT_PFAULT_INIT: 2129 rc = __inject_pfault_init(vcpu, irq); 2130 break; 2131 case KVM_S390_INT_VIRTIO: 2132 case KVM_S390_INT_SERVICE: 2133 case KVM_S390_INT_IO_MIN...KVM_S390_INT_IO_MAX: 2134 default: 2135 rc = -EINVAL; 2136 } 2137 2138 return rc; 2139 } 2140 2141 int kvm_s390_inject_vcpu(struct kvm_vcpu *vcpu, struct kvm_s390_irq *irq) 2142 { 2143 struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int; 2144 int rc; 2145 2146 spin_lock(&li->lock); 2147 rc = do_inject_vcpu(vcpu, irq); 2148 spin_unlock(&li->lock); 2149 if (!rc) 2150 kvm_s390_vcpu_wakeup(vcpu); 2151 return rc; 2152 } 2153 2154 static inline void clear_irq_list(struct list_head *_list) 2155 { 2156 struct kvm_s390_interrupt_info *inti, *n; 2157 2158 list_for_each_entry_safe(inti, n, _list, list) { 2159 list_del(&inti->list); 2160 kfree(inti); 2161 } 2162 } 2163 2164 static void inti_to_irq(struct kvm_s390_interrupt_info *inti, 2165 struct kvm_s390_irq *irq) 2166 { 2167 irq->type = inti->type; 2168 switch (inti->type) { 2169 case KVM_S390_INT_PFAULT_INIT: 2170 case KVM_S390_INT_PFAULT_DONE: 2171 case KVM_S390_INT_VIRTIO: 2172 irq->u.ext = inti->ext; 2173 break; 2174 case KVM_S390_INT_IO_MIN...KVM_S390_INT_IO_MAX: 2175 irq->u.io = inti->io; 2176 break; 2177 } 2178 } 2179 2180 void kvm_s390_clear_float_irqs(struct kvm *kvm) 2181 { 2182 struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int; 2183 int i; 2184 unsigned long flags; 2185 2186 mutex_lock(&kvm->lock); 2187 if (!kvm_s390_pv_is_protected(kvm)) 2188 fi->masked_irqs = 0; 2189 mutex_unlock(&kvm->lock); 2190 spin_lock_irqsave(&fi->lock, flags); 2191 fi->pending_irqs = 0; 2192 memset(&fi->srv_signal, 0, sizeof(fi->srv_signal)); 2193 memset(&fi->mchk, 0, sizeof(fi->mchk)); 2194 for (i = 0; i < FIRQ_LIST_COUNT; i++) 2195 clear_irq_list(&fi->lists[i]); 2196 for (i = 0; i < FIRQ_MAX_COUNT; i++) 2197 fi->counters[i] = 0; 2198 spin_unlock_irqrestore(&fi->lock, flags); 2199 kvm_s390_gisa_clear(kvm); 2200 }; 2201 2202 static int get_all_floating_irqs(struct kvm *kvm, u8 __user *usrbuf, u64 len) 2203 { 2204 struct kvm_s390_gisa_interrupt *gi = &kvm->arch.gisa_int; 2205 struct kvm_s390_interrupt_info *inti; 2206 struct kvm_s390_float_interrupt *fi; 2207 struct kvm_s390_irq *buf; 2208 struct kvm_s390_irq *irq; 2209 int max_irqs; 2210 int ret = 0; 2211 int n = 0; 2212 int i; 2213 unsigned long flags; 2214 2215 if (len > KVM_S390_FLIC_MAX_BUFFER || len == 0) 2216 return -EINVAL; 2217 2218 /* 2219 * We are already using -ENOMEM to signal 2220 * userspace it may retry with a bigger buffer, 2221 * so we need to use something else for this case 2222 */ 2223 buf = vzalloc(len); 2224 if (!buf) 2225 return -ENOBUFS; 2226 2227 max_irqs = len / sizeof(struct kvm_s390_irq); 2228 2229 if (gi->origin && gisa_get_ipm(gi->origin)) { 2230 for (i = 0; i <= MAX_ISC; i++) { 2231 if (n == max_irqs) { 2232 /* signal userspace to try again */ 2233 ret = -ENOMEM; 2234 goto out_nolock; 2235 } 2236 if (gisa_tac_ipm_gisc(gi->origin, i)) { 2237 irq = (struct kvm_s390_irq *) &buf[n]; 2238 irq->type = KVM_S390_INT_IO(1, 0, 0, 0); 2239 irq->u.io.io_int_word = isc_to_int_word(i); 2240 n++; 2241 } 2242 } 2243 } 2244 fi = &kvm->arch.float_int; 2245 spin_lock_irqsave(&fi->lock, flags); 2246 for (i = 0; i < FIRQ_LIST_COUNT; i++) { 2247 list_for_each_entry(inti, &fi->lists[i], list) { 2248 if (n == max_irqs) { 2249 /* signal userspace to try again */ 2250 ret = -ENOMEM; 2251 goto out; 2252 } 2253 inti_to_irq(inti, &buf[n]); 2254 n++; 2255 } 2256 } 2257 if (test_bit(IRQ_PEND_EXT_SERVICE, &fi->pending_irqs) || 2258 test_bit(IRQ_PEND_EXT_SERVICE_EV, &fi->pending_irqs)) { 2259 if (n == max_irqs) { 2260 /* signal userspace to try again */ 2261 ret = -ENOMEM; 2262 goto out; 2263 } 2264 irq = (struct kvm_s390_irq *) &buf[n]; 2265 irq->type = KVM_S390_INT_SERVICE; 2266 irq->u.ext = fi->srv_signal; 2267 n++; 2268 } 2269 if (test_bit(IRQ_PEND_MCHK_REP, &fi->pending_irqs)) { 2270 if (n == max_irqs) { 2271 /* signal userspace to try again */ 2272 ret = -ENOMEM; 2273 goto out; 2274 } 2275 irq = (struct kvm_s390_irq *) &buf[n]; 2276 irq->type = KVM_S390_MCHK; 2277 irq->u.mchk = fi->mchk; 2278 n++; 2279 } 2280 2281 out: 2282 spin_unlock_irqrestore(&fi->lock, flags); 2283 out_nolock: 2284 if (!ret && n > 0) { 2285 if (copy_to_user(usrbuf, buf, sizeof(struct kvm_s390_irq) * n)) 2286 ret = -EFAULT; 2287 } 2288 vfree(buf); 2289 2290 return ret < 0 ? ret : n; 2291 } 2292 2293 static int flic_ais_mode_get_all(struct kvm *kvm, struct kvm_device_attr *attr) 2294 { 2295 struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int; 2296 struct kvm_s390_ais_all ais; 2297 unsigned long flags; 2298 2299 if (attr->attr < sizeof(ais)) 2300 return -EINVAL; 2301 2302 if (!test_kvm_facility(kvm, 72)) 2303 return -EOPNOTSUPP; 2304 2305 spin_lock_irqsave(&fi->ais_lock, flags); 2306 ais.simm = fi->simm; 2307 ais.nimm = fi->nimm; 2308 spin_unlock_irqrestore(&fi->ais_lock, flags); 2309 2310 if (copy_to_user((void __user *)attr->addr, &ais, sizeof(ais))) 2311 return -EFAULT; 2312 2313 return 0; 2314 } 2315 2316 static int flic_get_attr(struct kvm_device *dev, struct kvm_device_attr *attr) 2317 { 2318 int r; 2319 2320 switch (attr->group) { 2321 case KVM_DEV_FLIC_GET_ALL_IRQS: 2322 r = get_all_floating_irqs(dev->kvm, (u8 __user *) attr->addr, 2323 attr->attr); 2324 break; 2325 case KVM_DEV_FLIC_AISM_ALL: 2326 r = flic_ais_mode_get_all(dev->kvm, attr); 2327 break; 2328 default: 2329 r = -EINVAL; 2330 } 2331 2332 return r; 2333 } 2334 2335 static inline int copy_irq_from_user(struct kvm_s390_interrupt_info *inti, 2336 u64 addr) 2337 { 2338 struct kvm_s390_irq __user *uptr = (struct kvm_s390_irq __user *) addr; 2339 void *target = NULL; 2340 void __user *source; 2341 u64 size; 2342 2343 if (get_user(inti->type, (u64 __user *)addr)) 2344 return -EFAULT; 2345 2346 switch (inti->type) { 2347 case KVM_S390_INT_PFAULT_INIT: 2348 case KVM_S390_INT_PFAULT_DONE: 2349 case KVM_S390_INT_VIRTIO: 2350 case KVM_S390_INT_SERVICE: 2351 target = (void *) &inti->ext; 2352 source = &uptr->u.ext; 2353 size = sizeof(inti->ext); 2354 break; 2355 case KVM_S390_INT_IO_MIN...KVM_S390_INT_IO_MAX: 2356 target = (void *) &inti->io; 2357 source = &uptr->u.io; 2358 size = sizeof(inti->io); 2359 break; 2360 case KVM_S390_MCHK: 2361 target = (void *) &inti->mchk; 2362 source = &uptr->u.mchk; 2363 size = sizeof(inti->mchk); 2364 break; 2365 default: 2366 return -EINVAL; 2367 } 2368 2369 if (copy_from_user(target, source, size)) 2370 return -EFAULT; 2371 2372 return 0; 2373 } 2374 2375 static int enqueue_floating_irq(struct kvm_device *dev, 2376 struct kvm_device_attr *attr) 2377 { 2378 struct kvm_s390_interrupt_info *inti = NULL; 2379 int r = 0; 2380 int len = attr->attr; 2381 2382 if (len % sizeof(struct kvm_s390_irq) != 0) 2383 return -EINVAL; 2384 else if (len > KVM_S390_FLIC_MAX_BUFFER) 2385 return -EINVAL; 2386 2387 while (len >= sizeof(struct kvm_s390_irq)) { 2388 inti = kzalloc_obj(*inti, GFP_KERNEL_ACCOUNT); 2389 if (!inti) 2390 return -ENOMEM; 2391 2392 r = copy_irq_from_user(inti, attr->addr); 2393 if (r) { 2394 kfree(inti); 2395 return r; 2396 } 2397 r = __inject_vm(dev->kvm, inti); 2398 if (r) { 2399 kfree(inti); 2400 return r; 2401 } 2402 len -= sizeof(struct kvm_s390_irq); 2403 attr->addr += sizeof(struct kvm_s390_irq); 2404 } 2405 2406 return r; 2407 } 2408 2409 static struct s390_io_adapter *get_io_adapter(struct kvm *kvm, unsigned int id) 2410 { 2411 if (id >= MAX_S390_IO_ADAPTERS) 2412 return NULL; 2413 id = array_index_nospec(id, MAX_S390_IO_ADAPTERS); 2414 return kvm->arch.adapters[id]; 2415 } 2416 2417 static int register_io_adapter(struct kvm_device *dev, 2418 struct kvm_device_attr *attr) 2419 { 2420 struct s390_io_adapter *adapter; 2421 struct kvm_s390_io_adapter adapter_info; 2422 int rc = 0; 2423 2424 mutex_lock(&dev->kvm->lock); 2425 if (copy_from_user(&adapter_info, 2426 (void __user *)attr->addr, sizeof(adapter_info))) { 2427 rc = -EFAULT; 2428 goto out; 2429 } 2430 if (adapter_info.id >= MAX_S390_IO_ADAPTERS) { 2431 rc = -EINVAL; 2432 goto out; 2433 } 2434 adapter_info.id = array_index_nospec(adapter_info.id, 2435 MAX_S390_IO_ADAPTERS); 2436 2437 if (dev->kvm->arch.adapters[adapter_info.id] != NULL) { 2438 rc = -EINVAL; 2439 goto out; 2440 } 2441 adapter = kzalloc_obj(*adapter, GFP_KERNEL_ACCOUNT); 2442 if (!adapter) { 2443 rc = -ENOMEM; 2444 goto out; 2445 } 2446 2447 INIT_LIST_HEAD(&adapter->maps); 2448 spin_lock_init(&adapter->maps_lock); 2449 adapter->nr_maps = 0; 2450 adapter->id = adapter_info.id; 2451 adapter->isc = adapter_info.isc; 2452 adapter->maskable = adapter_info.maskable; 2453 adapter->masked = false; 2454 adapter->swap = adapter_info.swap; 2455 adapter->suppressible = adapter_info.flags & 2456 KVM_S390_ADAPTER_SUPPRESSIBLE; 2457 dev->kvm->arch.adapters[adapter->id] = adapter; 2458 2459 out: 2460 mutex_unlock(&dev->kvm->lock); 2461 return rc; 2462 } 2463 2464 int kvm_s390_mask_adapter(struct kvm *kvm, unsigned int id, bool masked) 2465 { 2466 int ret; 2467 struct s390_io_adapter *adapter = get_io_adapter(kvm, id); 2468 2469 if (!adapter || !adapter->maskable) 2470 return -EINVAL; 2471 ret = adapter->masked; 2472 adapter->masked = masked; 2473 return ret; 2474 } 2475 2476 static struct page *pin_map_page(struct kvm *kvm, u64 uaddr, 2477 unsigned int gup_flags) 2478 { 2479 struct mm_struct *mm = kvm->mm; 2480 struct page *page = NULL; 2481 int locked = 1; 2482 2483 if (mmget_not_zero(mm)) { 2484 mmap_read_lock(mm); 2485 pin_user_pages_remote(mm, uaddr, 1, FOLL_WRITE | gup_flags, 2486 &page, &locked); 2487 if (locked) 2488 mmap_read_unlock(mm); 2489 mmput(mm); 2490 } 2491 2492 return page; 2493 } 2494 2495 static int kvm_s390_adapter_map(struct kvm *kvm, unsigned int id, __u64 addr) 2496 { 2497 struct s390_io_adapter *adapter = get_io_adapter(kvm, id); 2498 struct s390_map_info *map; 2499 unsigned long flags; 2500 __u64 host_addr; 2501 int ret, idx; 2502 2503 if (!adapter || !addr) 2504 return -EINVAL; 2505 2506 map = kzalloc_obj(*map, GFP_KERNEL_ACCOUNT); 2507 if (!map) 2508 return -ENOMEM; 2509 2510 INIT_LIST_HEAD(&map->list); 2511 idx = srcu_read_lock(&kvm->srcu); 2512 host_addr = gpa_to_hva(kvm, addr); 2513 if (kvm_is_error_hva(host_addr)) { 2514 srcu_read_unlock(&kvm->srcu, idx); 2515 ret = -EFAULT; 2516 goto out; 2517 } 2518 srcu_read_unlock(&kvm->srcu, idx); 2519 map->guest_addr = addr; 2520 map->addr = host_addr; 2521 map->page = pin_map_page(kvm, host_addr, FOLL_LONGTERM); 2522 if (!map->page) { 2523 ret = -EINVAL; 2524 goto out; 2525 } 2526 spin_lock_irqsave(&adapter->maps_lock, flags); 2527 if (adapter->nr_maps < MAX_S390_ADAPTER_MAPS) { 2528 list_add_tail(&map->list, &adapter->maps); 2529 adapter->nr_maps++; 2530 ret = 0; 2531 } else { 2532 ret = -EINVAL; 2533 } 2534 spin_unlock_irqrestore(&adapter->maps_lock, flags); 2535 if (ret) 2536 unpin_user_page(map->page); 2537 out: 2538 if (ret) 2539 kfree(map); 2540 return ret; 2541 } 2542 2543 static int kvm_s390_adapter_unmap(struct kvm *kvm, unsigned int id, __u64 addr) 2544 { 2545 struct s390_io_adapter *adapter = get_io_adapter(kvm, id); 2546 struct s390_map_info *map, *tmp, *map_to_free; 2547 struct page *map_page_to_put = NULL; 2548 u64 map_addr_to_mark = 0; 2549 unsigned long flags; 2550 int found = 0, idx; 2551 2552 if (!adapter || !addr) 2553 return -EINVAL; 2554 2555 spin_lock_irqsave(&adapter->maps_lock, flags); 2556 list_for_each_entry_safe(map, tmp, &adapter->maps, list) { 2557 if (map->guest_addr == addr) { 2558 found = 1; 2559 adapter->nr_maps--; 2560 list_del(&map->list); 2561 map_page_to_put = map->page; 2562 map_addr_to_mark = map->guest_addr; 2563 map_to_free = map; 2564 break; 2565 } 2566 } 2567 spin_unlock_irqrestore(&adapter->maps_lock, flags); 2568 2569 if (found) { 2570 kfree(map_to_free); 2571 idx = srcu_read_lock(&kvm->srcu); 2572 mark_page_dirty(kvm, map_addr_to_mark >> PAGE_SHIFT); 2573 set_page_dirty_lock(map_page_to_put); 2574 srcu_read_unlock(&kvm->srcu, idx); 2575 unpin_user_page(map_page_to_put); 2576 } 2577 2578 return found ? 0 : -ENOENT; 2579 } 2580 2581 void kvm_s390_unmap_all_adapters(struct kvm *kvm) 2582 { 2583 struct s390_map_info *map, *tmp; 2584 unsigned long flags; 2585 int i, idx; 2586 2587 for (i = 0; i < MAX_S390_IO_ADAPTERS; i++) { 2588 struct s390_io_adapter *adapter = kvm->arch.adapters[i]; 2589 LIST_HEAD(local_list); 2590 2591 if (!adapter) 2592 continue; 2593 2594 spin_lock_irqsave(&adapter->maps_lock, flags); 2595 list_splice_init(&adapter->maps, &local_list); 2596 adapter->nr_maps = 0; 2597 spin_unlock_irqrestore(&adapter->maps_lock, flags); 2598 2599 list_for_each_entry_safe(map, tmp, &local_list, list) { 2600 list_del(&map->list); 2601 idx = srcu_read_lock(&kvm->srcu); 2602 mark_page_dirty(kvm, map->guest_addr >> PAGE_SHIFT); 2603 set_page_dirty_lock(map->page); 2604 srcu_read_unlock(&kvm->srcu, idx); 2605 unpin_user_page(map->page); 2606 kfree(map); 2607 } 2608 } 2609 } 2610 2611 void kvm_s390_destroy_adapters(struct kvm *kvm) 2612 { 2613 int i; 2614 2615 kvm_s390_unmap_all_adapters(kvm); 2616 2617 for (i = 0; i < MAX_S390_IO_ADAPTERS; i++) { 2618 kfree(kvm->arch.adapters[i]); 2619 kvm->arch.adapters[i] = NULL; 2620 } 2621 } 2622 2623 static int modify_io_adapter(struct kvm_device *dev, 2624 struct kvm_device_attr *attr) 2625 { 2626 struct kvm_s390_io_adapter_req req; 2627 struct s390_io_adapter *adapter; 2628 int ret; 2629 2630 if (copy_from_user(&req, (void __user *)attr->addr, sizeof(req))) 2631 return -EFAULT; 2632 2633 adapter = get_io_adapter(dev->kvm, req.id); 2634 if (!adapter) 2635 return -EINVAL; 2636 switch (req.type) { 2637 case KVM_S390_IO_ADAPTER_MASK: 2638 ret = kvm_s390_mask_adapter(dev->kvm, req.id, req.mask); 2639 if (ret > 0) 2640 ret = 0; 2641 break; 2642 case KVM_S390_IO_ADAPTER_MAP: 2643 case KVM_S390_IO_ADAPTER_UNMAP: 2644 /* If in Secure Execution mode do not long term pin. */ 2645 mutex_lock(&dev->kvm->lock); 2646 if (kvm_s390_pv_is_protected(dev->kvm)) { 2647 mutex_unlock(&dev->kvm->lock); 2648 return 0; 2649 } 2650 if (req.type == KVM_S390_IO_ADAPTER_MAP) { 2651 dev->kvm->stat.io_390_adapter_map++; 2652 ret = kvm_s390_adapter_map(dev->kvm, req.id, req.addr); 2653 } else { 2654 dev->kvm->stat.io_390_adapter_unmap++; 2655 ret = kvm_s390_adapter_unmap(dev->kvm, req.id, req.addr); 2656 } 2657 mutex_unlock(&dev->kvm->lock); 2658 break; 2659 default: 2660 ret = -EINVAL; 2661 } 2662 2663 return ret; 2664 } 2665 2666 static int clear_io_irq(struct kvm *kvm, struct kvm_device_attr *attr) 2667 2668 { 2669 const u64 isc_mask = 0xffUL << 24; /* all iscs set */ 2670 u32 schid; 2671 2672 if (attr->flags) 2673 return -EINVAL; 2674 if (attr->attr != sizeof(schid)) 2675 return -EINVAL; 2676 if (copy_from_user(&schid, (void __user *) attr->addr, sizeof(schid))) 2677 return -EFAULT; 2678 if (!schid) 2679 return -EINVAL; 2680 kfree(kvm_s390_get_io_int(kvm, isc_mask, schid)); 2681 /* 2682 * If userspace is conforming to the architecture, we can have at most 2683 * one pending I/O interrupt per subchannel, so this is effectively a 2684 * clear all. 2685 */ 2686 return 0; 2687 } 2688 2689 static int modify_ais_mode(struct kvm *kvm, struct kvm_device_attr *attr) 2690 { 2691 struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int; 2692 struct kvm_s390_ais_req req; 2693 int ret = 0; 2694 unsigned long flags; 2695 2696 if (!test_kvm_facility(kvm, 72)) 2697 return -EOPNOTSUPP; 2698 2699 if (copy_from_user(&req, (void __user *)attr->addr, sizeof(req))) 2700 return -EFAULT; 2701 2702 if (req.isc > MAX_ISC) 2703 return -EINVAL; 2704 2705 trace_kvm_s390_modify_ais_mode(req.isc, 2706 (fi->simm & AIS_MODE_MASK(req.isc)) ? 2707 (fi->nimm & AIS_MODE_MASK(req.isc)) ? 2708 2 : KVM_S390_AIS_MODE_SINGLE : 2709 KVM_S390_AIS_MODE_ALL, req.mode); 2710 2711 spin_lock_irqsave(&fi->ais_lock, flags); 2712 switch (req.mode) { 2713 case KVM_S390_AIS_MODE_ALL: 2714 fi->simm &= ~AIS_MODE_MASK(req.isc); 2715 fi->nimm &= ~AIS_MODE_MASK(req.isc); 2716 break; 2717 case KVM_S390_AIS_MODE_SINGLE: 2718 fi->simm |= AIS_MODE_MASK(req.isc); 2719 fi->nimm &= ~AIS_MODE_MASK(req.isc); 2720 break; 2721 default: 2722 ret = -EINVAL; 2723 } 2724 spin_unlock_irqrestore(&fi->ais_lock, flags); 2725 2726 return ret; 2727 } 2728 2729 static int kvm_s390_inject_airq(struct kvm *kvm, 2730 struct s390_io_adapter *adapter) 2731 { 2732 struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int; 2733 struct kvm_s390_interrupt s390int = { 2734 .type = KVM_S390_INT_IO(1, 0, 0, 0), 2735 .parm = 0, 2736 .parm64 = isc_to_int_word(adapter->isc), 2737 }; 2738 struct kvm_s390_interrupt_info *inti; 2739 unsigned long flags; 2740 2741 int ret = 0; 2742 2743 inti = kzalloc_obj(*inti, GFP_KERNEL_ACCOUNT); 2744 if (!inti) 2745 return -ENOMEM; 2746 2747 if (!test_kvm_facility(kvm, 72) || !adapter->suppressible) { 2748 ret = kvm_s390_inject_vm(kvm, &s390int, inti); 2749 if (ret) 2750 kfree(inti); 2751 return ret; 2752 } 2753 2754 spin_lock_irqsave(&fi->ais_lock, flags); 2755 if (fi->nimm & AIS_MODE_MASK(adapter->isc)) { 2756 trace_kvm_s390_airq_suppressed(adapter->id, adapter->isc); 2757 spin_unlock_irqrestore(&fi->ais_lock, flags); 2758 kfree(inti); 2759 return ret; 2760 } 2761 2762 ret = kvm_s390_inject_vm(kvm, &s390int, inti); 2763 2764 if (!ret && (fi->simm & AIS_MODE_MASK(adapter->isc))) { 2765 fi->nimm |= AIS_MODE_MASK(adapter->isc); 2766 trace_kvm_s390_modify_ais_mode(adapter->isc, 2767 KVM_S390_AIS_MODE_SINGLE, 2); 2768 } 2769 2770 spin_unlock_irqrestore(&fi->ais_lock, flags); 2771 if (ret) 2772 kfree(inti); 2773 return ret; 2774 } 2775 2776 static int flic_inject_airq(struct kvm *kvm, struct kvm_device_attr *attr) 2777 { 2778 unsigned int id = attr->attr; 2779 struct s390_io_adapter *adapter = get_io_adapter(kvm, id); 2780 2781 kvm->stat.io_flic_inject_airq++; 2782 2783 if (!adapter) 2784 return -EINVAL; 2785 2786 return kvm_s390_inject_airq(kvm, adapter); 2787 } 2788 2789 static int flic_ais_mode_set_all(struct kvm *kvm, struct kvm_device_attr *attr) 2790 { 2791 struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int; 2792 struct kvm_s390_ais_all ais; 2793 unsigned long flags; 2794 2795 if (!test_kvm_facility(kvm, 72)) 2796 return -EOPNOTSUPP; 2797 2798 if (copy_from_user(&ais, (void __user *)attr->addr, sizeof(ais))) 2799 return -EFAULT; 2800 2801 spin_lock_irqsave(&fi->ais_lock, flags); 2802 fi->simm = ais.simm; 2803 fi->nimm = ais.nimm; 2804 spin_unlock_irqrestore(&fi->ais_lock, flags); 2805 2806 return 0; 2807 } 2808 2809 static int flic_set_attr(struct kvm_device *dev, struct kvm_device_attr *attr) 2810 { 2811 int r = 0; 2812 unsigned long i; 2813 struct kvm_vcpu *vcpu; 2814 2815 switch (attr->group) { 2816 case KVM_DEV_FLIC_ENQUEUE: 2817 r = enqueue_floating_irq(dev, attr); 2818 break; 2819 case KVM_DEV_FLIC_CLEAR_IRQS: 2820 kvm_s390_clear_float_irqs(dev->kvm); 2821 break; 2822 case KVM_DEV_FLIC_APF_ENABLE: 2823 if (kvm_is_ucontrol(dev->kvm)) 2824 return -EINVAL; 2825 set_bit(GMAP_FLAG_PFAULT_ENABLED, &dev->kvm->arch.gmap->flags); 2826 break; 2827 case KVM_DEV_FLIC_APF_DISABLE_WAIT: 2828 if (kvm_is_ucontrol(dev->kvm)) 2829 return -EINVAL; 2830 clear_bit(GMAP_FLAG_PFAULT_ENABLED, &dev->kvm->arch.gmap->flags); 2831 /* 2832 * Make sure no async faults are in transition when 2833 * clearing the queues. So we don't need to worry 2834 * about late coming workers. 2835 */ 2836 synchronize_srcu(&dev->kvm->srcu); 2837 kvm_for_each_vcpu(i, vcpu, dev->kvm) 2838 kvm_clear_async_pf_completion_queue(vcpu); 2839 break; 2840 case KVM_DEV_FLIC_ADAPTER_REGISTER: 2841 r = register_io_adapter(dev, attr); 2842 break; 2843 case KVM_DEV_FLIC_ADAPTER_MODIFY: 2844 r = modify_io_adapter(dev, attr); 2845 break; 2846 case KVM_DEV_FLIC_CLEAR_IO_IRQ: 2847 r = clear_io_irq(dev->kvm, attr); 2848 break; 2849 case KVM_DEV_FLIC_AISM: 2850 r = modify_ais_mode(dev->kvm, attr); 2851 break; 2852 case KVM_DEV_FLIC_AIRQ_INJECT: 2853 r = flic_inject_airq(dev->kvm, attr); 2854 break; 2855 case KVM_DEV_FLIC_AISM_ALL: 2856 r = flic_ais_mode_set_all(dev->kvm, attr); 2857 break; 2858 default: 2859 r = -EINVAL; 2860 } 2861 2862 return r; 2863 } 2864 2865 static int flic_has_attr(struct kvm_device *dev, 2866 struct kvm_device_attr *attr) 2867 { 2868 switch (attr->group) { 2869 case KVM_DEV_FLIC_GET_ALL_IRQS: 2870 case KVM_DEV_FLIC_ENQUEUE: 2871 case KVM_DEV_FLIC_CLEAR_IRQS: 2872 case KVM_DEV_FLIC_APF_ENABLE: 2873 case KVM_DEV_FLIC_APF_DISABLE_WAIT: 2874 case KVM_DEV_FLIC_ADAPTER_REGISTER: 2875 case KVM_DEV_FLIC_ADAPTER_MODIFY: 2876 case KVM_DEV_FLIC_CLEAR_IO_IRQ: 2877 case KVM_DEV_FLIC_AISM: 2878 case KVM_DEV_FLIC_AIRQ_INJECT: 2879 case KVM_DEV_FLIC_AISM_ALL: 2880 return 0; 2881 } 2882 return -ENXIO; 2883 } 2884 2885 static int flic_create(struct kvm_device *dev, u32 type) 2886 { 2887 if (!dev) 2888 return -EINVAL; 2889 if (dev->kvm->arch.flic) 2890 return -EINVAL; 2891 dev->kvm->arch.flic = dev; 2892 return 0; 2893 } 2894 2895 static void flic_destroy(struct kvm_device *dev) 2896 { 2897 dev->kvm->arch.flic = NULL; 2898 kfree(dev); 2899 } 2900 2901 /* s390 floating irq controller (flic) */ 2902 struct kvm_device_ops kvm_flic_ops = { 2903 .name = "kvm-flic", 2904 .get_attr = flic_get_attr, 2905 .set_attr = flic_set_attr, 2906 .has_attr = flic_has_attr, 2907 .create = flic_create, 2908 .destroy = flic_destroy, 2909 }; 2910 2911 static unsigned long get_ind_bit(__u64 addr, unsigned long bit_nr, bool swap) 2912 { 2913 unsigned long bit; 2914 2915 bit = bit_nr + (addr % PAGE_SIZE) * 8; 2916 2917 /* kvm_set_routing_entry() should never allow this to happen */ 2918 WARN_ON_ONCE(bit > (PAGE_SIZE * BITS_PER_BYTE - 1)); 2919 2920 return swap ? (bit ^ (BITS_PER_LONG - 1)) : bit; 2921 } 2922 2923 static struct s390_map_info *get_map_info(struct s390_io_adapter *adapter, 2924 u64 addr) 2925 { 2926 struct s390_map_info *map; 2927 2928 if (!adapter) 2929 return NULL; 2930 2931 list_for_each_entry(map, &adapter->maps, list) { 2932 if (map->addr == addr) 2933 return map; 2934 } 2935 return NULL; 2936 } 2937 2938 static int adapter_indicators_set(struct kvm *kvm, 2939 struct s390_io_adapter *adapter, 2940 struct kvm_s390_adapter_int *adapter_int) 2941 { 2942 unsigned long bit; 2943 int summary_set, idx; 2944 struct s390_map_info *ind_info, *summary_info; 2945 void *map; 2946 struct page *ind_page, *summary_page; 2947 unsigned long flags; 2948 2949 ind_page = NULL; 2950 2951 spin_lock_irqsave(&adapter->maps_lock, flags); 2952 ind_info = get_map_info(adapter, adapter_int->ind_addr); 2953 if (!ind_info) { 2954 spin_unlock_irqrestore(&adapter->maps_lock, flags); 2955 ind_page = pin_map_page(kvm, adapter_int->ind_addr, 0); 2956 if (!ind_page) 2957 return -1; 2958 idx = srcu_read_lock(&kvm->srcu); 2959 map = page_address(ind_page); 2960 bit = get_ind_bit(adapter_int->ind_addr, 2961 adapter_int->ind_offset, adapter->swap); 2962 set_bit(bit, map); 2963 mark_page_dirty(kvm, adapter_int->ind_gaddr >> PAGE_SHIFT); 2964 set_page_dirty_lock(ind_page); 2965 srcu_read_unlock(&kvm->srcu, idx); 2966 unpin_user_page(ind_page); 2967 } else { 2968 map = page_address(ind_info->page); 2969 bit = get_ind_bit(ind_info->addr, adapter_int->ind_offset, adapter->swap); 2970 set_bit(bit, map); 2971 spin_unlock_irqrestore(&adapter->maps_lock, flags); 2972 } 2973 2974 spin_lock_irqsave(&adapter->maps_lock, flags); 2975 summary_info = get_map_info(adapter, adapter_int->summary_addr); 2976 if (!summary_info) { 2977 spin_unlock_irqrestore(&adapter->maps_lock, flags); 2978 summary_page = pin_map_page(kvm, adapter_int->summary_addr, 0); 2979 if (WARN_ON_ONCE(!summary_page)) 2980 return -1; 2981 idx = srcu_read_lock(&kvm->srcu); 2982 map = page_address(summary_page); 2983 bit = get_ind_bit(adapter_int->summary_addr, 2984 adapter_int->summary_offset, adapter->swap); 2985 summary_set = test_and_set_bit(bit, map); 2986 mark_page_dirty(kvm, adapter_int->summary_gaddr >> PAGE_SHIFT); 2987 set_page_dirty_lock(summary_page); 2988 srcu_read_unlock(&kvm->srcu, idx); 2989 unpin_user_page(summary_page); 2990 } else { 2991 map = page_address(summary_info->page); 2992 bit = get_ind_bit(summary_info->addr, adapter_int->summary_offset, 2993 adapter->swap); 2994 summary_set = test_and_set_bit(bit, map); 2995 spin_unlock_irqrestore(&adapter->maps_lock, flags); 2996 } 2997 2998 return summary_set ? 0 : 1; 2999 } 3000 3001 static int adapter_indicators_set_fast(struct kvm *kvm, 3002 struct s390_io_adapter *adapter, 3003 struct kvm_s390_adapter_int *adapter_int, 3004 int setbit) 3005 { 3006 unsigned long bit; 3007 int summary_set; 3008 struct s390_map_info *ind_info, *summary_info; 3009 void *map; 3010 3011 spin_lock(&adapter->maps_lock); 3012 ind_info = get_map_info(adapter, adapter_int->ind_addr); 3013 if (!ind_info) { 3014 spin_unlock(&adapter->maps_lock); 3015 return -EWOULDBLOCK; 3016 } 3017 map = page_address(ind_info->page); 3018 bit = get_ind_bit(ind_info->addr, adapter_int->ind_offset, adapter->swap); 3019 if (setbit) 3020 set_bit(bit, map); 3021 summary_info = get_map_info(adapter, adapter_int->summary_addr); 3022 if (!summary_info) { 3023 spin_unlock(&adapter->maps_lock); 3024 return -EWOULDBLOCK; 3025 } 3026 map = page_address(summary_info->page); 3027 bit = get_ind_bit(summary_info->addr, adapter_int->summary_offset, 3028 adapter->swap); 3029 /* If setbit then set summary bit. Else if falling back to the slow path */ 3030 /* with setbit==0 then clear the summary bit so the slow path re-injects */ 3031 if (setbit) 3032 summary_set = test_and_set_bit(bit, map); 3033 else 3034 summary_set = test_and_clear_bit(bit, map); 3035 spin_unlock(&adapter->maps_lock); 3036 return summary_set ? 0 : 1; 3037 } 3038 3039 /* 3040 * < 0 - not injected due to error 3041 * = 0 - coalesced, summary indicator already active 3042 * > 0 - injected interrupt 3043 */ 3044 static int set_adapter_int(struct kvm_kernel_irq_routing_entry *e, 3045 struct kvm *kvm, int irq_source_id, int level, 3046 bool line_status) 3047 { 3048 int ret; 3049 struct s390_io_adapter *adapter; 3050 3051 kvm->stat.io_set_adapter_int++; 3052 3053 /* We're only interested in the 0->1 transition. */ 3054 if (!level) 3055 return 0; 3056 adapter = get_io_adapter(kvm, e->adapter.adapter_id); 3057 if (!adapter) 3058 return -1; 3059 ret = adapter_indicators_set(kvm, adapter, &e->adapter); 3060 if ((ret > 0) && !adapter->masked) { 3061 ret = kvm_s390_inject_airq(kvm, adapter); 3062 if (ret == 0) 3063 ret = 1; 3064 } 3065 return ret; 3066 } 3067 3068 /* 3069 * Inject the machine check to the guest. 3070 */ 3071 void kvm_s390_reinject_machine_check(struct kvm_vcpu *vcpu, 3072 struct mcck_volatile_info *mcck_info) 3073 { 3074 struct kvm_s390_interrupt_info inti; 3075 struct kvm_s390_irq irq; 3076 struct kvm_s390_mchk_info *mchk; 3077 union mci mci; 3078 __u64 cr14 = 0; /* upper bits are not used */ 3079 int rc; 3080 3081 mci.val = mcck_info->mcic; 3082 3083 /* log machine checks being reinjected on all debugs */ 3084 VCPU_EVENT(vcpu, 2, "guest machine check %lx", mci.val); 3085 KVM_EVENT(2, "guest machine check %lx", mci.val); 3086 pr_info("guest machine check pid %d: %lx", current->pid, mci.val); 3087 3088 if (mci.sr) 3089 cr14 |= CR14_RECOVERY_SUBMASK; 3090 if (mci.dg) 3091 cr14 |= CR14_DEGRADATION_SUBMASK; 3092 if (mci.w) 3093 cr14 |= CR14_WARNING_SUBMASK; 3094 3095 mchk = mci.ck ? &inti.mchk : &irq.u.mchk; 3096 mchk->cr14 = cr14; 3097 mchk->mcic = mcck_info->mcic; 3098 mchk->ext_damage_code = mcck_info->ext_damage_code; 3099 mchk->failing_storage_address = mcck_info->failing_storage_address; 3100 if (mci.ck) { 3101 /* Inject the floating machine check */ 3102 inti.type = KVM_S390_MCHK; 3103 rc = __inject_vm(vcpu->kvm, &inti); 3104 } else { 3105 /* Inject the machine check to specified vcpu */ 3106 irq.type = KVM_S390_MCHK; 3107 rc = kvm_s390_inject_vcpu(vcpu, &irq); 3108 } 3109 WARN_ON_ONCE(rc); 3110 } 3111 3112 int kvm_set_routing_entry(struct kvm *kvm, 3113 struct kvm_kernel_irq_routing_entry *e, 3114 const struct kvm_irq_routing_entry *ue) 3115 { 3116 const struct kvm_irq_routing_s390_adapter *adapter; 3117 u64 uaddr_s, uaddr_i; 3118 int idx; 3119 3120 switch (ue->type) { 3121 case KVM_IRQ_ROUTING_S390_ADAPTER: 3122 if (kvm_is_ucontrol(kvm)) 3123 return -EINVAL; 3124 e->set = set_adapter_int; 3125 3126 adapter = &ue->u.adapter; 3127 if (adapter->summary_addr + (adapter->summary_offset / 8) >= 3128 (adapter->summary_addr & PAGE_MASK) + PAGE_SIZE) 3129 return -EINVAL; 3130 if (adapter->ind_addr + (adapter->ind_offset / 8) >= 3131 (adapter->ind_addr & PAGE_MASK) + PAGE_SIZE) 3132 return -EINVAL; 3133 3134 idx = srcu_read_lock(&kvm->srcu); 3135 uaddr_s = gpa_to_hva(kvm, ue->u.adapter.summary_addr); 3136 uaddr_i = gpa_to_hva(kvm, ue->u.adapter.ind_addr); 3137 srcu_read_unlock(&kvm->srcu, idx); 3138 3139 if (kvm_is_error_hva(uaddr_s) || kvm_is_error_hva(uaddr_i)) 3140 return -EFAULT; 3141 e->adapter.summary_addr = uaddr_s; 3142 e->adapter.summary_gaddr = ue->u.adapter.summary_addr; 3143 e->adapter.ind_addr = uaddr_i; 3144 e->adapter.ind_gaddr = ue->u.adapter.ind_addr; 3145 e->adapter.summary_offset = ue->u.adapter.summary_offset; 3146 e->adapter.ind_offset = ue->u.adapter.ind_offset; 3147 e->adapter.adapter_id = ue->u.adapter.adapter_id; 3148 return 0; 3149 default: 3150 return -EINVAL; 3151 } 3152 } 3153 3154 int kvm_set_msi(struct kvm_kernel_irq_routing_entry *e, struct kvm *kvm, 3155 int irq_source_id, int level, bool line_status) 3156 { 3157 return -EINVAL; 3158 } 3159 3160 int kvm_s390_set_irq_state(struct kvm_vcpu *vcpu, void __user *irqstate, int len) 3161 { 3162 struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int; 3163 struct kvm_s390_irq *buf; 3164 int r = 0; 3165 int n; 3166 3167 buf = vmalloc(len); 3168 if (!buf) 3169 return -ENOMEM; 3170 3171 if (copy_from_user((void *) buf, irqstate, len)) { 3172 r = -EFAULT; 3173 goto out_free; 3174 } 3175 3176 /* 3177 * Don't allow setting the interrupt state 3178 * when there are already interrupts pending 3179 */ 3180 spin_lock(&li->lock); 3181 if (li->pending_irqs) { 3182 r = -EBUSY; 3183 goto out_unlock; 3184 } 3185 3186 for (n = 0; n < len / sizeof(*buf); n++) { 3187 r = do_inject_vcpu(vcpu, &buf[n]); 3188 if (r) 3189 break; 3190 } 3191 3192 out_unlock: 3193 spin_unlock(&li->lock); 3194 out_free: 3195 vfree(buf); 3196 3197 return r; 3198 } 3199 3200 static void store_local_irq(struct kvm_s390_local_interrupt *li, 3201 struct kvm_s390_irq *irq, 3202 unsigned long irq_type) 3203 { 3204 switch (irq_type) { 3205 case IRQ_PEND_MCHK_EX: 3206 case IRQ_PEND_MCHK_REP: 3207 irq->type = KVM_S390_MCHK; 3208 irq->u.mchk = li->irq.mchk; 3209 break; 3210 case IRQ_PEND_PROG: 3211 irq->type = KVM_S390_PROGRAM_INT; 3212 irq->u.pgm = li->irq.pgm; 3213 break; 3214 case IRQ_PEND_PFAULT_INIT: 3215 irq->type = KVM_S390_INT_PFAULT_INIT; 3216 irq->u.ext = li->irq.ext; 3217 break; 3218 case IRQ_PEND_EXT_EXTERNAL: 3219 irq->type = KVM_S390_INT_EXTERNAL_CALL; 3220 irq->u.extcall = li->irq.extcall; 3221 break; 3222 case IRQ_PEND_EXT_CLOCK_COMP: 3223 irq->type = KVM_S390_INT_CLOCK_COMP; 3224 break; 3225 case IRQ_PEND_EXT_CPU_TIMER: 3226 irq->type = KVM_S390_INT_CPU_TIMER; 3227 break; 3228 case IRQ_PEND_SIGP_STOP: 3229 irq->type = KVM_S390_SIGP_STOP; 3230 irq->u.stop = li->irq.stop; 3231 break; 3232 case IRQ_PEND_RESTART: 3233 irq->type = KVM_S390_RESTART; 3234 break; 3235 case IRQ_PEND_SET_PREFIX: 3236 irq->type = KVM_S390_SIGP_SET_PREFIX; 3237 irq->u.prefix = li->irq.prefix; 3238 break; 3239 } 3240 } 3241 3242 int kvm_s390_get_irq_state(struct kvm_vcpu *vcpu, __u8 __user *buf, int len) 3243 { 3244 int scn; 3245 DECLARE_BITMAP(sigp_emerg_pending, KVM_MAX_VCPUS); 3246 struct kvm_s390_local_interrupt *li = &vcpu->arch.local_int; 3247 unsigned long pending_irqs; 3248 struct kvm_s390_irq irq; 3249 unsigned long irq_type; 3250 int cpuaddr; 3251 int n = 0; 3252 3253 spin_lock(&li->lock); 3254 pending_irqs = li->pending_irqs; 3255 memcpy(&sigp_emerg_pending, &li->sigp_emerg_pending, 3256 sizeof(sigp_emerg_pending)); 3257 spin_unlock(&li->lock); 3258 3259 for_each_set_bit(irq_type, &pending_irqs, IRQ_PEND_COUNT) { 3260 memset(&irq, 0, sizeof(irq)); 3261 if (irq_type == IRQ_PEND_EXT_EMERGENCY) 3262 continue; 3263 if (n + sizeof(irq) > len) 3264 return -ENOBUFS; 3265 store_local_irq(&vcpu->arch.local_int, &irq, irq_type); 3266 if (copy_to_user(&buf[n], &irq, sizeof(irq))) 3267 return -EFAULT; 3268 n += sizeof(irq); 3269 } 3270 3271 if (test_bit(IRQ_PEND_EXT_EMERGENCY, &pending_irqs)) { 3272 for_each_set_bit(cpuaddr, sigp_emerg_pending, KVM_MAX_VCPUS) { 3273 memset(&irq, 0, sizeof(irq)); 3274 if (n + sizeof(irq) > len) 3275 return -ENOBUFS; 3276 irq.type = KVM_S390_INT_EMERGENCY; 3277 irq.u.emerg.code = cpuaddr; 3278 if (copy_to_user(&buf[n], &irq, sizeof(irq))) 3279 return -EFAULT; 3280 n += sizeof(irq); 3281 } 3282 } 3283 3284 if (sca_ext_call_pending(vcpu, &scn)) { 3285 if (n + sizeof(irq) > len) 3286 return -ENOBUFS; 3287 memset(&irq, 0, sizeof(irq)); 3288 irq.type = KVM_S390_INT_EXTERNAL_CALL; 3289 irq.u.extcall.code = scn; 3290 if (copy_to_user(&buf[n], &irq, sizeof(irq))) 3291 return -EFAULT; 3292 n += sizeof(irq); 3293 } 3294 3295 return n; 3296 } 3297 3298 static void __airqs_kick_single_vcpu(struct kvm *kvm, u8 deliverable_mask) 3299 { 3300 int vcpu_idx, online_vcpus = atomic_read(&kvm->online_vcpus); 3301 struct kvm_s390_gisa_interrupt *gi = &kvm->arch.gisa_int; 3302 struct kvm_vcpu *vcpu; 3303 u8 vcpu_isc_mask; 3304 3305 for_each_set_bit(vcpu_idx, kvm->arch.idle_mask, online_vcpus) { 3306 vcpu = kvm_get_vcpu(kvm, vcpu_idx); 3307 if (psw_ioint_disabled(vcpu)) 3308 continue; 3309 vcpu_isc_mask = (u8)(vcpu->arch.sie_block->gcr[6] >> 24); 3310 if (deliverable_mask & vcpu_isc_mask) { 3311 /* lately kicked but not yet running */ 3312 if (test_and_set_bit(vcpu_idx, gi->kicked_mask)) 3313 return; 3314 kvm_s390_vcpu_wakeup(vcpu); 3315 return; 3316 } 3317 } 3318 } 3319 3320 static enum hrtimer_restart gisa_vcpu_kicker(struct hrtimer *timer) 3321 { 3322 struct kvm_s390_gisa_interrupt *gi = 3323 container_of(timer, struct kvm_s390_gisa_interrupt, timer); 3324 struct kvm *kvm = 3325 container_of(gi->origin, struct sie_page2, gisa)->kvm; 3326 u8 pending_mask; 3327 3328 pending_mask = gisa_get_ipm_or_restore_iam(gi); 3329 if (pending_mask) { 3330 __airqs_kick_single_vcpu(kvm, pending_mask); 3331 hrtimer_forward_now(timer, ns_to_ktime(gi->expires)); 3332 return HRTIMER_RESTART; 3333 } 3334 3335 return HRTIMER_NORESTART; 3336 } 3337 3338 #define NULL_GISA_ADDR 0x00000000UL 3339 #define NONE_GISA_ADDR 0x00000001UL 3340 #define GISA_ADDR_MASK 0xfffff000UL 3341 3342 static void process_gib_alert_list(void) 3343 { 3344 struct kvm_s390_gisa_interrupt *gi; 3345 u32 final, gisa_phys, origin = 0UL; 3346 struct kvm_s390_gisa *gisa; 3347 struct kvm *kvm; 3348 3349 do { 3350 /* 3351 * If the NONE_GISA_ADDR is still stored in the alert list 3352 * origin, we will leave the outer loop. No further GISA has 3353 * been added to the alert list by millicode while processing 3354 * the current alert list. 3355 */ 3356 final = (origin & NONE_GISA_ADDR); 3357 /* 3358 * Cut off the alert list and store the NONE_GISA_ADDR in the 3359 * alert list origin to avoid further GAL interruptions. 3360 * A new alert list can be build up by millicode in parallel 3361 * for guests not in the yet cut-off alert list. When in the 3362 * final loop, store the NULL_GISA_ADDR instead. This will re- 3363 * enable GAL interruptions on the host again. 3364 */ 3365 origin = xchg(&gib->alert_list_origin, 3366 (!final) ? NONE_GISA_ADDR : NULL_GISA_ADDR); 3367 /* 3368 * Loop through the just cut-off alert list and start the 3369 * gisa timers to kick idle vcpus to consume the pending 3370 * interruptions asap. 3371 */ 3372 while (origin & GISA_ADDR_MASK) { 3373 gisa_phys = origin; 3374 gisa = phys_to_virt(gisa_phys); 3375 origin = gisa->next_alert; 3376 gisa->next_alert = gisa_phys; 3377 kvm = container_of(gisa, struct sie_page2, gisa)->kvm; 3378 gi = &kvm->arch.gisa_int; 3379 if (hrtimer_active(&gi->timer)) 3380 hrtimer_cancel(&gi->timer); 3381 hrtimer_start(&gi->timer, 0, HRTIMER_MODE_REL); 3382 } 3383 } while (!final); 3384 3385 } 3386 3387 void kvm_s390_gisa_clear(struct kvm *kvm) 3388 { 3389 struct kvm_s390_gisa_interrupt *gi = &kvm->arch.gisa_int; 3390 3391 if (!gi->origin) 3392 return; 3393 gisa_clear_ipm(gi->origin); 3394 VM_EVENT(kvm, 3, "gisa 0x%p cleared", gi->origin); 3395 } 3396 3397 void kvm_s390_gisa_init(struct kvm *kvm) 3398 { 3399 struct kvm_s390_gisa_interrupt *gi = &kvm->arch.gisa_int; 3400 3401 if (!css_general_characteristics.aiv) 3402 return; 3403 gi->origin = &kvm->arch.sie_page2->gisa; 3404 gi->alert.mask = 0; 3405 spin_lock_init(&gi->alert.ref_lock); 3406 gi->expires = 50 * 1000; /* 50 usec */ 3407 hrtimer_setup(&gi->timer, gisa_vcpu_kicker, CLOCK_MONOTONIC, HRTIMER_MODE_REL); 3408 memset(gi->origin, 0, sizeof(struct kvm_s390_gisa)); 3409 gi->origin->next_alert = (u32)virt_to_phys(gi->origin); 3410 VM_EVENT(kvm, 3, "gisa 0x%p initialized", gi->origin); 3411 } 3412 3413 void kvm_s390_gisa_enable(struct kvm *kvm) 3414 { 3415 struct kvm_s390_gisa_interrupt *gi = &kvm->arch.gisa_int; 3416 struct kvm_vcpu *vcpu; 3417 unsigned long i; 3418 u32 gisa_desc; 3419 3420 if (gi->origin) 3421 return; 3422 kvm_s390_gisa_init(kvm); 3423 gisa_desc = kvm_s390_get_gisa_desc(kvm); 3424 if (!gisa_desc) 3425 return; 3426 kvm_for_each_vcpu(i, vcpu, kvm) { 3427 mutex_lock(&vcpu->mutex); 3428 vcpu->arch.sie_block->gd = gisa_desc; 3429 vcpu->arch.sie_block->eca |= ECA_AIV; 3430 VCPU_EVENT(vcpu, 3, "AIV gisa format-%u enabled for cpu %03u", 3431 vcpu->arch.sie_block->gd & 0x3, vcpu->vcpu_id); 3432 mutex_unlock(&vcpu->mutex); 3433 } 3434 } 3435 3436 void kvm_s390_gisa_destroy(struct kvm *kvm) 3437 { 3438 struct kvm_s390_gisa_interrupt *gi = &kvm->arch.gisa_int; 3439 struct kvm_s390_gisa *gisa = gi->origin; 3440 3441 if (!gi->origin) 3442 return; 3443 WARN(gi->alert.mask != 0x00, 3444 "unexpected non zero alert.mask 0x%02x", 3445 gi->alert.mask); 3446 gi->alert.mask = 0x00; 3447 if (gisa_set_iam(gi->origin, gi->alert.mask)) 3448 process_gib_alert_list(); 3449 hrtimer_cancel(&gi->timer); 3450 gi->origin = NULL; 3451 VM_EVENT(kvm, 3, "gisa 0x%p destroyed", gisa); 3452 } 3453 3454 void kvm_s390_gisa_disable(struct kvm *kvm) 3455 { 3456 struct kvm_s390_gisa_interrupt *gi = &kvm->arch.gisa_int; 3457 struct kvm_vcpu *vcpu; 3458 unsigned long i; 3459 3460 if (!gi->origin) 3461 return; 3462 kvm_for_each_vcpu(i, vcpu, kvm) { 3463 mutex_lock(&vcpu->mutex); 3464 vcpu->arch.sie_block->eca &= ~ECA_AIV; 3465 vcpu->arch.sie_block->gd = 0U; 3466 mutex_unlock(&vcpu->mutex); 3467 VCPU_EVENT(vcpu, 3, "AIV disabled for cpu %03u", vcpu->vcpu_id); 3468 } 3469 kvm_s390_gisa_destroy(kvm); 3470 } 3471 3472 /** 3473 * kvm_s390_gisc_register - register a guest ISC 3474 * 3475 * @kvm: the kernel vm to work with 3476 * @gisc: the guest interruption sub class to register 3477 * 3478 * The function extends the vm specific alert mask to use. 3479 * The effective IAM mask in the GISA is updated as well 3480 * in case the GISA is not part of the GIB alert list. 3481 * It will be updated latest when the IAM gets restored 3482 * by gisa_get_ipm_or_restore_iam(). 3483 * 3484 * Returns: the nonspecific ISC (NISC) the gib alert mechanism 3485 * has registered with the channel subsystem. 3486 * -ENODEV in case the vm uses no GISA 3487 * -ERANGE in case the guest ISC is invalid 3488 */ 3489 int kvm_s390_gisc_register(struct kvm *kvm, u32 gisc) 3490 { 3491 struct kvm_s390_gisa_interrupt *gi = &kvm->arch.gisa_int; 3492 3493 if (!gi->origin) 3494 return -ENODEV; 3495 if (gisc > MAX_ISC) 3496 return -ERANGE; 3497 3498 spin_lock(&gi->alert.ref_lock); 3499 gi->alert.ref_count[gisc]++; 3500 if (gi->alert.ref_count[gisc] == 1) { 3501 gi->alert.mask |= 0x80 >> gisc; 3502 gisa_set_iam(gi->origin, gi->alert.mask); 3503 } 3504 spin_unlock(&gi->alert.ref_lock); 3505 3506 return gib->nisc; 3507 } 3508 EXPORT_SYMBOL_GPL(kvm_s390_gisc_register); 3509 3510 /** 3511 * kvm_s390_gisc_unregister - unregister a guest ISC 3512 * 3513 * @kvm: the kernel vm to work with 3514 * @gisc: the guest interruption sub class to register 3515 * 3516 * The function reduces the vm specific alert mask to use. 3517 * The effective IAM mask in the GISA is updated as well 3518 * in case the GISA is not part of the GIB alert list. 3519 * It will be updated latest when the IAM gets restored 3520 * by gisa_get_ipm_or_restore_iam(). 3521 * 3522 * Returns: the nonspecific ISC (NISC) the gib alert mechanism 3523 * has registered with the channel subsystem. 3524 * -ENODEV in case the vm uses no GISA 3525 * -ERANGE in case the guest ISC is invalid 3526 * -EINVAL in case the guest ISC is not registered 3527 */ 3528 int kvm_s390_gisc_unregister(struct kvm *kvm, u32 gisc) 3529 { 3530 struct kvm_s390_gisa_interrupt *gi = &kvm->arch.gisa_int; 3531 int rc = 0; 3532 3533 if (!gi->origin) 3534 return -ENODEV; 3535 if (gisc > MAX_ISC) 3536 return -ERANGE; 3537 3538 spin_lock(&gi->alert.ref_lock); 3539 if (gi->alert.ref_count[gisc] == 0) { 3540 rc = -EINVAL; 3541 goto out; 3542 } 3543 gi->alert.ref_count[gisc]--; 3544 if (gi->alert.ref_count[gisc] == 0) { 3545 gi->alert.mask &= ~(0x80 >> gisc); 3546 gisa_set_iam(gi->origin, gi->alert.mask); 3547 } 3548 out: 3549 spin_unlock(&gi->alert.ref_lock); 3550 3551 return rc; 3552 } 3553 EXPORT_SYMBOL_GPL(kvm_s390_gisc_unregister); 3554 3555 static void aen_host_forward(unsigned long si) 3556 { 3557 struct kvm_s390_gisa_interrupt *gi; 3558 struct zpci_gaite *gaite; 3559 struct kvm *kvm; 3560 3561 gaite = aift->gait + si; 3562 if (gaite->count == 0) 3563 return; 3564 if (gaite->aisb != 0) 3565 set_bit_inv(gaite->aisbo, phys_to_virt(gaite->aisb)); 3566 3567 kvm = kvm_s390_pci_si_to_kvm(aift, si); 3568 if (!kvm) 3569 return; 3570 gi = &kvm->arch.gisa_int; 3571 3572 if (!(gi->origin->g1.simm & AIS_MODE_MASK(gaite->gisc)) || 3573 !(gi->origin->g1.nimm & AIS_MODE_MASK(gaite->gisc))) { 3574 gisa_set_ipm_gisc(gi->origin, gaite->gisc); 3575 if (hrtimer_active(&gi->timer)) 3576 hrtimer_cancel(&gi->timer); 3577 hrtimer_start(&gi->timer, 0, HRTIMER_MODE_REL); 3578 kvm->stat.aen_forward++; 3579 } 3580 } 3581 3582 static void aen_process_gait(u8 isc) 3583 { 3584 bool found = false, first = true; 3585 union zpci_sic_iib iib = {{0}}; 3586 unsigned long si, flags; 3587 3588 spin_lock_irqsave(&aift->gait_lock, flags); 3589 3590 if (!aift->gait) { 3591 spin_unlock_irqrestore(&aift->gait_lock, flags); 3592 return; 3593 } 3594 3595 for (si = 0;;) { 3596 /* Scan adapter summary indicator bit vector */ 3597 si = airq_iv_scan(aift->sbv, si, airq_iv_end(aift->sbv)); 3598 if (si == -1UL) { 3599 if (first || found) { 3600 /* Re-enable interrupts. */ 3601 zpci_set_irq_ctrl(SIC_IRQ_MODE_SINGLE, isc, 3602 &iib); 3603 first = found = false; 3604 } else { 3605 /* Interrupts on and all bits processed */ 3606 break; 3607 } 3608 found = false; 3609 si = 0; 3610 /* Scan again after re-enabling interrupts */ 3611 continue; 3612 } 3613 found = true; 3614 aen_host_forward(si); 3615 } 3616 3617 spin_unlock_irqrestore(&aift->gait_lock, flags); 3618 } 3619 3620 static void gib_alert_irq_handler(struct airq_struct *airq, 3621 struct tpi_info *tpi_info) 3622 { 3623 struct tpi_adapter_info *info = (struct tpi_adapter_info *)tpi_info; 3624 3625 inc_irq_stat(IRQIO_GAL); 3626 3627 if ((info->forward || info->error) && 3628 IS_ENABLED(CONFIG_VFIO_PCI_ZDEV_KVM)) { 3629 aen_process_gait(info->isc); 3630 if (info->aism != 0) 3631 process_gib_alert_list(); 3632 } else { 3633 process_gib_alert_list(); 3634 } 3635 } 3636 3637 static struct airq_struct gib_alert_irq = { 3638 .handler = gib_alert_irq_handler, 3639 }; 3640 3641 void kvm_s390_gib_destroy(void) 3642 { 3643 if (!gib) 3644 return; 3645 if (kvm_s390_pci_interp_allowed() && aift) { 3646 mutex_lock(&aift->aift_lock); 3647 kvm_s390_pci_aen_exit(); 3648 mutex_unlock(&aift->aift_lock); 3649 } 3650 chsc_sgib(0); 3651 unregister_adapter_interrupt(&gib_alert_irq); 3652 free_page((unsigned long)gib); 3653 gib = NULL; 3654 } 3655 3656 int __init kvm_s390_gib_init(u8 nisc) 3657 { 3658 u32 gib_origin; 3659 int rc = 0; 3660 3661 if (!css_general_characteristics.aiv) { 3662 KVM_EVENT(3, "%s", "gib not initialized, no AIV facility"); 3663 goto out; 3664 } 3665 3666 gib = (struct kvm_s390_gib *)get_zeroed_page(GFP_KERNEL_ACCOUNT | GFP_DMA); 3667 if (!gib) { 3668 rc = -ENOMEM; 3669 goto out; 3670 } 3671 3672 gib_alert_irq.isc = nisc; 3673 if (register_adapter_interrupt(&gib_alert_irq)) { 3674 pr_err("Registering the GIB alert interruption handler failed\n"); 3675 rc = -EIO; 3676 goto out_free_gib; 3677 } 3678 /* adapter interrupts used for AP (applicable here) don't use the LSI */ 3679 *gib_alert_irq.lsi_ptr = 0xff; 3680 3681 gib->nisc = nisc; 3682 gib_origin = virt_to_phys(gib); 3683 if (chsc_sgib(gib_origin)) { 3684 pr_err("Associating the GIB with the AIV facility failed\n"); 3685 free_page((unsigned long)gib); 3686 gib = NULL; 3687 rc = -EIO; 3688 goto out_unreg_gal; 3689 } 3690 3691 if (kvm_s390_pci_interp_allowed()) { 3692 if (kvm_s390_pci_aen_init(nisc)) { 3693 pr_err("Initializing AEN for PCI failed\n"); 3694 rc = -EIO; 3695 goto out_unreg_gal; 3696 } 3697 } 3698 3699 KVM_EVENT(3, "gib 0x%p (nisc=%d) initialized", gib, gib->nisc); 3700 goto out; 3701 3702 out_unreg_gal: 3703 unregister_adapter_interrupt(&gib_alert_irq); 3704 out_free_gib: 3705 free_page((unsigned long)gib); 3706 gib = NULL; 3707 out: 3708 return rc; 3709 } 3710 3711 /* 3712 * kvm_arch_set_irq_inatomic: fast-path for irqfd injection 3713 */ 3714 int kvm_arch_set_irq_inatomic(struct kvm_kernel_irq_routing_entry *e, 3715 struct kvm *kvm, int irq_source_id, int level, 3716 bool line_status) 3717 { 3718 int ret, setbit; 3719 struct s390_io_adapter *adapter; 3720 struct kvm_s390_float_interrupt *fi = &kvm->arch.float_int; 3721 struct kvm_s390_interrupt_info *inti; 3722 struct kvm_s390_interrupt s390int = { 3723 .type = KVM_S390_INT_IO(1, 0, 0, 0), 3724 .parm = 0, 3725 }; 3726 3727 kvm->stat.io_390_inatomic++; 3728 3729 /* We're only interested in the 0->1 transition. */ 3730 if (!level) 3731 return 0; 3732 if (e->type != KVM_IRQ_ROUTING_S390_ADAPTER) 3733 return -EWOULDBLOCK; 3734 3735 adapter = get_io_adapter(kvm, e->adapter.adapter_id); 3736 if (!adapter) 3737 return -EWOULDBLOCK; 3738 3739 s390int.parm64 = isc_to_int_word(adapter->isc); 3740 setbit = 1; 3741 ret = adapter_indicators_set_fast(kvm, adapter, &e->adapter, setbit); 3742 if (ret < 0) 3743 return -EWOULDBLOCK; 3744 if (!ret || adapter->masked) { 3745 kvm->stat.io_390_inatomic_no_inject++; 3746 return 0; 3747 } 3748 3749 inti = kzalloc_obj(*inti, GFP_ATOMIC); 3750 if (!inti) { 3751 setbit = 0; 3752 adapter_indicators_set_fast(kvm, adapter, &e->adapter, setbit); 3753 return -EWOULDBLOCK; 3754 } 3755 3756 if (!test_kvm_facility(kvm, 72) || !adapter->suppressible) { 3757 ret = kvm_s390_inject_vm(kvm, &s390int, inti); 3758 if (ret == 0) { 3759 return ret; 3760 } else { 3761 setbit = 0; 3762 adapter_indicators_set_fast(kvm, adapter, &e->adapter, setbit); 3763 kfree(inti); 3764 return -EWOULDBLOCK; 3765 } 3766 } 3767 3768 spin_lock(&fi->ais_lock); 3769 if (fi->nimm & AIS_MODE_MASK(adapter->isc)) { 3770 trace_kvm_s390_airq_suppressed(adapter->id, adapter->isc); 3771 spin_unlock(&fi->ais_lock); 3772 kfree(inti); 3773 kvm->stat.io_390_inatomic_no_inject++; 3774 return 0; 3775 } 3776 3777 ret = kvm_s390_inject_vm(kvm, &s390int, inti); 3778 if (!ret && (fi->simm & AIS_MODE_MASK(adapter->isc))) { 3779 fi->nimm |= AIS_MODE_MASK(adapter->isc); 3780 trace_kvm_s390_modify_ais_mode(adapter->isc, 3781 KVM_S390_AIS_MODE_SINGLE, 2); 3782 } else if (ret) { 3783 spin_unlock(&fi->ais_lock); 3784 setbit = 0; 3785 adapter_indicators_set_fast(kvm, adapter, &e->adapter, setbit); 3786 kfree(inti); 3787 return -EWOULDBLOCK; 3788 } 3789 3790 spin_unlock(&fi->ais_lock); 3791 return 0; 3792 } 3793