1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * handling privileged instructions 4 * 5 * Copyright IBM Corp. 2008, 2020 6 * 7 * Author(s): Carsten Otte <cotte@de.ibm.com> 8 * Christian Borntraeger <borntraeger@de.ibm.com> 9 */ 10 11 #include <linux/kvm.h> 12 #include <linux/gfp.h> 13 #include <linux/errno.h> 14 #include <linux/mm_types.h> 15 #include <linux/pgtable.h> 16 #include <linux/io.h> 17 #include <asm/asm-offsets.h> 18 #include <asm/facility.h> 19 #include <asm/current.h> 20 #include <asm/debug.h> 21 #include <asm/ebcdic.h> 22 #include <asm/sysinfo.h> 23 #include <asm/page-states.h> 24 #include <asm/gmap.h> 25 #include <asm/ptrace.h> 26 #include <asm/sclp.h> 27 #include <asm/ap.h> 28 #include "gaccess.h" 29 #include "kvm-s390.h" 30 #include "trace.h" 31 32 static int handle_ri(struct kvm_vcpu *vcpu) 33 { 34 vcpu->stat.instruction_ri++; 35 36 if (test_kvm_facility(vcpu->kvm, 64)) { 37 VCPU_EVENT(vcpu, 3, "%s", "ENABLE: RI (lazy)"); 38 vcpu->arch.sie_block->ecb3 |= ECB3_RI; 39 kvm_s390_retry_instr(vcpu); 40 return 0; 41 } else 42 return kvm_s390_inject_program_int(vcpu, PGM_OPERATION); 43 } 44 45 int kvm_s390_handle_aa(struct kvm_vcpu *vcpu) 46 { 47 if ((vcpu->arch.sie_block->ipa & 0xf) <= 4) 48 return handle_ri(vcpu); 49 else 50 return -EOPNOTSUPP; 51 } 52 53 static int handle_gs(struct kvm_vcpu *vcpu) 54 { 55 vcpu->stat.instruction_gs++; 56 57 if (test_kvm_facility(vcpu->kvm, 133)) { 58 VCPU_EVENT(vcpu, 3, "%s", "ENABLE: GS (lazy)"); 59 preempt_disable(); 60 local_ctl_set_bit(2, CR2_GUARDED_STORAGE_BIT); 61 current->thread.gs_cb = (struct gs_cb *)&vcpu->run->s.regs.gscb; 62 restore_gs_cb(current->thread.gs_cb); 63 preempt_enable(); 64 vcpu->arch.sie_block->ecb |= ECB_GS; 65 vcpu->arch.sie_block->ecd |= ECD_HOSTREGMGMT; 66 vcpu->arch.gs_enabled = 1; 67 kvm_s390_retry_instr(vcpu); 68 return 0; 69 } else 70 return kvm_s390_inject_program_int(vcpu, PGM_OPERATION); 71 } 72 73 int kvm_s390_handle_e3(struct kvm_vcpu *vcpu) 74 { 75 int code = vcpu->arch.sie_block->ipb & 0xff; 76 77 if (code == 0x49 || code == 0x4d) 78 return handle_gs(vcpu); 79 else 80 return -EOPNOTSUPP; 81 } 82 /* Handle SCK (SET CLOCK) interception */ 83 static int handle_set_clock(struct kvm_vcpu *vcpu) 84 { 85 struct kvm_s390_vm_tod_clock gtod = { 0 }; 86 int rc; 87 u8 ar; 88 u64 op2; 89 90 vcpu->stat.instruction_sck++; 91 92 if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE) 93 return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP); 94 95 op2 = kvm_s390_get_base_disp_s(vcpu, &ar); 96 if (op2 & 7) /* Operand must be on a doubleword boundary */ 97 return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION); 98 rc = read_guest(vcpu, op2, ar, >od.tod, sizeof(gtod.tod)); 99 if (rc) 100 return kvm_s390_inject_prog_cond(vcpu, rc); 101 102 VCPU_EVENT(vcpu, 3, "SCK: setting guest TOD to 0x%llx", gtod.tod); 103 /* 104 * To set the TOD clock the kvm lock must be taken, but the vcpu lock 105 * is already held in handle_set_clock. The usual lock order is the 106 * opposite. As SCK is deprecated and should not be used in several 107 * cases, for example when the multiple epoch facility or TOD clock 108 * steering facility is installed (see Principles of Operation), a 109 * slow path can be used. If the lock can not be taken via try_lock, 110 * the instruction will be retried via -EAGAIN at a later point in 111 * time. 112 */ 113 if (!kvm_s390_try_set_tod_clock(vcpu->kvm, >od)) { 114 kvm_s390_retry_instr(vcpu); 115 return -EAGAIN; 116 } 117 118 kvm_s390_set_psw_cc(vcpu, 0); 119 return 0; 120 } 121 122 static int handle_set_prefix(struct kvm_vcpu *vcpu) 123 { 124 u64 operand2; 125 u32 address; 126 int rc; 127 u8 ar; 128 129 vcpu->stat.instruction_spx++; 130 131 if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE) 132 return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP); 133 134 operand2 = kvm_s390_get_base_disp_s(vcpu, &ar); 135 136 /* must be word boundary */ 137 if (operand2 & 3) 138 return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION); 139 140 /* get the value */ 141 rc = read_guest(vcpu, operand2, ar, &address, sizeof(address)); 142 if (rc) 143 return kvm_s390_inject_prog_cond(vcpu, rc); 144 145 address &= 0x7fffe000u; 146 147 /* 148 * Make sure the new value is valid memory. We only need to check the 149 * first page, since address is 8k aligned and memory pieces are always 150 * at least 1MB aligned and have at least a size of 1MB. 151 */ 152 if (kvm_is_error_gpa(vcpu->kvm, address)) 153 return kvm_s390_inject_program_int(vcpu, PGM_ADDRESSING); 154 155 kvm_s390_set_prefix(vcpu, address); 156 trace_kvm_s390_handle_prefix(vcpu, 1, address); 157 return 0; 158 } 159 160 static int handle_store_prefix(struct kvm_vcpu *vcpu) 161 { 162 u64 operand2; 163 u32 address; 164 int rc; 165 u8 ar; 166 167 vcpu->stat.instruction_stpx++; 168 169 if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE) 170 return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP); 171 172 operand2 = kvm_s390_get_base_disp_s(vcpu, &ar); 173 174 /* must be word boundary */ 175 if (operand2 & 3) 176 return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION); 177 178 address = kvm_s390_get_prefix(vcpu); 179 180 /* get the value */ 181 rc = write_guest(vcpu, operand2, ar, &address, sizeof(address)); 182 if (rc) 183 return kvm_s390_inject_prog_cond(vcpu, rc); 184 185 VCPU_EVENT(vcpu, 3, "STPX: storing prefix 0x%x into 0x%llx", address, operand2); 186 trace_kvm_s390_handle_prefix(vcpu, 0, address); 187 return 0; 188 } 189 190 static int handle_store_cpu_address(struct kvm_vcpu *vcpu) 191 { 192 u16 vcpu_id = vcpu->vcpu_id; 193 u64 ga; 194 int rc; 195 u8 ar; 196 197 vcpu->stat.instruction_stap++; 198 199 if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE) 200 return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP); 201 202 ga = kvm_s390_get_base_disp_s(vcpu, &ar); 203 204 if (ga & 1) 205 return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION); 206 207 rc = write_guest(vcpu, ga, ar, &vcpu_id, sizeof(vcpu_id)); 208 if (rc) 209 return kvm_s390_inject_prog_cond(vcpu, rc); 210 211 VCPU_EVENT(vcpu, 3, "STAP: storing cpu address (%u) to 0x%llx", vcpu_id, ga); 212 trace_kvm_s390_handle_stap(vcpu, ga); 213 return 0; 214 } 215 216 int kvm_s390_skey_check_enable(struct kvm_vcpu *vcpu) 217 { 218 int rc; 219 220 trace_kvm_s390_skey_related_inst(vcpu); 221 /* Already enabled? */ 222 if (vcpu->arch.skey_enabled) 223 return 0; 224 225 rc = s390_enable_skey(); 226 VCPU_EVENT(vcpu, 3, "enabling storage keys for guest: %d", rc); 227 if (rc) 228 return rc; 229 230 if (kvm_s390_test_cpuflags(vcpu, CPUSTAT_KSS)) 231 kvm_s390_clear_cpuflags(vcpu, CPUSTAT_KSS); 232 if (!vcpu->kvm->arch.use_skf) 233 vcpu->arch.sie_block->ictl |= ICTL_ISKE | ICTL_SSKE | ICTL_RRBE; 234 else 235 vcpu->arch.sie_block->ictl &= ~(ICTL_ISKE | ICTL_SSKE | ICTL_RRBE); 236 vcpu->arch.skey_enabled = true; 237 return 0; 238 } 239 240 static int try_handle_skey(struct kvm_vcpu *vcpu) 241 { 242 int rc; 243 244 rc = kvm_s390_skey_check_enable(vcpu); 245 if (rc) 246 return rc; 247 if (vcpu->kvm->arch.use_skf) { 248 /* with storage-key facility, SIE interprets it for us */ 249 kvm_s390_retry_instr(vcpu); 250 VCPU_EVENT(vcpu, 4, "%s", "retrying storage key operation"); 251 return -EAGAIN; 252 } 253 return 0; 254 } 255 256 static int handle_iske(struct kvm_vcpu *vcpu) 257 { 258 unsigned long gaddr, vmaddr; 259 unsigned char key; 260 int reg1, reg2; 261 bool unlocked; 262 int rc; 263 264 vcpu->stat.instruction_iske++; 265 266 if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE) 267 return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP); 268 269 rc = try_handle_skey(vcpu); 270 if (rc) 271 return rc != -EAGAIN ? rc : 0; 272 273 kvm_s390_get_regs_rre(vcpu, ®1, ®2); 274 275 gaddr = vcpu->run->s.regs.gprs[reg2] & PAGE_MASK; 276 gaddr = kvm_s390_logical_to_effective(vcpu, gaddr); 277 gaddr = kvm_s390_real_to_abs(vcpu, gaddr); 278 vmaddr = gfn_to_hva(vcpu->kvm, gpa_to_gfn(gaddr)); 279 if (kvm_is_error_hva(vmaddr)) 280 return kvm_s390_inject_program_int(vcpu, PGM_ADDRESSING); 281 retry: 282 unlocked = false; 283 mmap_read_lock(current->mm); 284 rc = get_guest_storage_key(current->mm, vmaddr, &key); 285 286 if (rc) { 287 rc = fixup_user_fault(current->mm, vmaddr, 288 FAULT_FLAG_WRITE, &unlocked); 289 if (!rc) { 290 mmap_read_unlock(current->mm); 291 goto retry; 292 } 293 } 294 mmap_read_unlock(current->mm); 295 if (rc == -EFAULT) 296 return kvm_s390_inject_program_int(vcpu, PGM_ADDRESSING); 297 if (rc < 0) 298 return rc; 299 vcpu->run->s.regs.gprs[reg1] &= ~0xff; 300 vcpu->run->s.regs.gprs[reg1] |= key; 301 return 0; 302 } 303 304 static int handle_rrbe(struct kvm_vcpu *vcpu) 305 { 306 unsigned long vmaddr, gaddr; 307 int reg1, reg2; 308 bool unlocked; 309 int rc; 310 311 vcpu->stat.instruction_rrbe++; 312 313 if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE) 314 return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP); 315 316 rc = try_handle_skey(vcpu); 317 if (rc) 318 return rc != -EAGAIN ? rc : 0; 319 320 kvm_s390_get_regs_rre(vcpu, ®1, ®2); 321 322 gaddr = vcpu->run->s.regs.gprs[reg2] & PAGE_MASK; 323 gaddr = kvm_s390_logical_to_effective(vcpu, gaddr); 324 gaddr = kvm_s390_real_to_abs(vcpu, gaddr); 325 vmaddr = gfn_to_hva(vcpu->kvm, gpa_to_gfn(gaddr)); 326 if (kvm_is_error_hva(vmaddr)) 327 return kvm_s390_inject_program_int(vcpu, PGM_ADDRESSING); 328 retry: 329 unlocked = false; 330 mmap_read_lock(current->mm); 331 rc = reset_guest_reference_bit(current->mm, vmaddr); 332 if (rc < 0) { 333 rc = fixup_user_fault(current->mm, vmaddr, 334 FAULT_FLAG_WRITE, &unlocked); 335 if (!rc) { 336 mmap_read_unlock(current->mm); 337 goto retry; 338 } 339 } 340 mmap_read_unlock(current->mm); 341 if (rc == -EFAULT) 342 return kvm_s390_inject_program_int(vcpu, PGM_ADDRESSING); 343 if (rc < 0) 344 return rc; 345 kvm_s390_set_psw_cc(vcpu, rc); 346 return 0; 347 } 348 349 #define SSKE_NQ 0x8 350 #define SSKE_MR 0x4 351 #define SSKE_MC 0x2 352 #define SSKE_MB 0x1 353 static int handle_sske(struct kvm_vcpu *vcpu) 354 { 355 unsigned char m3 = vcpu->arch.sie_block->ipb >> 28; 356 unsigned long start, end; 357 unsigned char key, oldkey; 358 int reg1, reg2; 359 bool unlocked; 360 int rc; 361 362 vcpu->stat.instruction_sske++; 363 364 if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE) 365 return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP); 366 367 rc = try_handle_skey(vcpu); 368 if (rc) 369 return rc != -EAGAIN ? rc : 0; 370 371 if (!test_kvm_facility(vcpu->kvm, 8)) 372 m3 &= ~SSKE_MB; 373 if (!test_kvm_facility(vcpu->kvm, 10)) 374 m3 &= ~(SSKE_MC | SSKE_MR); 375 if (!test_kvm_facility(vcpu->kvm, 14)) 376 m3 &= ~SSKE_NQ; 377 378 kvm_s390_get_regs_rre(vcpu, ®1, ®2); 379 380 key = vcpu->run->s.regs.gprs[reg1] & 0xfe; 381 start = vcpu->run->s.regs.gprs[reg2] & PAGE_MASK; 382 start = kvm_s390_logical_to_effective(vcpu, start); 383 if (m3 & SSKE_MB) { 384 /* start already designates an absolute address */ 385 end = (start + _SEGMENT_SIZE) & ~(_SEGMENT_SIZE - 1); 386 } else { 387 start = kvm_s390_real_to_abs(vcpu, start); 388 end = start + PAGE_SIZE; 389 } 390 391 while (start != end) { 392 unsigned long vmaddr = gfn_to_hva(vcpu->kvm, gpa_to_gfn(start)); 393 unlocked = false; 394 395 if (kvm_is_error_hva(vmaddr)) 396 return kvm_s390_inject_program_int(vcpu, PGM_ADDRESSING); 397 398 mmap_read_lock(current->mm); 399 rc = cond_set_guest_storage_key(current->mm, vmaddr, key, &oldkey, 400 m3 & SSKE_NQ, m3 & SSKE_MR, 401 m3 & SSKE_MC); 402 403 if (rc < 0) { 404 rc = fixup_user_fault(current->mm, vmaddr, 405 FAULT_FLAG_WRITE, &unlocked); 406 rc = !rc ? -EAGAIN : rc; 407 } 408 mmap_read_unlock(current->mm); 409 if (rc == -EFAULT) 410 return kvm_s390_inject_program_int(vcpu, PGM_ADDRESSING); 411 if (rc == -EAGAIN) 412 continue; 413 if (rc < 0) 414 return rc; 415 start += PAGE_SIZE; 416 } 417 418 if (m3 & (SSKE_MC | SSKE_MR)) { 419 if (m3 & SSKE_MB) { 420 /* skey in reg1 is unpredictable */ 421 kvm_s390_set_psw_cc(vcpu, 3); 422 } else { 423 kvm_s390_set_psw_cc(vcpu, rc); 424 vcpu->run->s.regs.gprs[reg1] &= ~0xff00UL; 425 vcpu->run->s.regs.gprs[reg1] |= (u64) oldkey << 8; 426 } 427 } 428 if (m3 & SSKE_MB) { 429 if (psw_bits(vcpu->arch.sie_block->gpsw).eaba == PSW_BITS_AMODE_64BIT) 430 vcpu->run->s.regs.gprs[reg2] &= ~PAGE_MASK; 431 else 432 vcpu->run->s.regs.gprs[reg2] &= ~0xfffff000UL; 433 end = kvm_s390_logical_to_effective(vcpu, end); 434 vcpu->run->s.regs.gprs[reg2] |= end; 435 } 436 return 0; 437 } 438 439 static int handle_ipte_interlock(struct kvm_vcpu *vcpu) 440 { 441 vcpu->stat.instruction_ipte_interlock++; 442 if (psw_bits(vcpu->arch.sie_block->gpsw).pstate) 443 return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP); 444 wait_event(vcpu->kvm->arch.ipte_wq, !ipte_lock_held(vcpu->kvm)); 445 kvm_s390_retry_instr(vcpu); 446 VCPU_EVENT(vcpu, 4, "%s", "retrying ipte interlock operation"); 447 return 0; 448 } 449 450 static int handle_test_block(struct kvm_vcpu *vcpu) 451 { 452 gpa_t addr; 453 int reg2; 454 455 vcpu->stat.instruction_tb++; 456 457 if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE) 458 return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP); 459 460 kvm_s390_get_regs_rre(vcpu, NULL, ®2); 461 addr = vcpu->run->s.regs.gprs[reg2] & PAGE_MASK; 462 addr = kvm_s390_logical_to_effective(vcpu, addr); 463 if (kvm_s390_check_low_addr_prot_real(vcpu, addr)) 464 return kvm_s390_inject_prog_irq(vcpu, &vcpu->arch.pgm); 465 addr = kvm_s390_real_to_abs(vcpu, addr); 466 467 if (kvm_is_error_gpa(vcpu->kvm, addr)) 468 return kvm_s390_inject_program_int(vcpu, PGM_ADDRESSING); 469 /* 470 * We don't expect errors on modern systems, and do not care 471 * about storage keys (yet), so let's just clear the page. 472 */ 473 if (kvm_clear_guest(vcpu->kvm, addr, PAGE_SIZE)) 474 return -EFAULT; 475 kvm_s390_set_psw_cc(vcpu, 0); 476 vcpu->run->s.regs.gprs[0] = 0; 477 return 0; 478 } 479 480 static int handle_tpi(struct kvm_vcpu *vcpu) 481 { 482 struct kvm_s390_interrupt_info *inti; 483 unsigned long len; 484 u32 tpi_data[3]; 485 int rc; 486 u64 addr; 487 u8 ar; 488 489 vcpu->stat.instruction_tpi++; 490 491 addr = kvm_s390_get_base_disp_s(vcpu, &ar); 492 if (addr & 3) 493 return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION); 494 495 inti = kvm_s390_get_io_int(vcpu->kvm, vcpu->arch.sie_block->gcr[6], 0); 496 if (!inti) { 497 kvm_s390_set_psw_cc(vcpu, 0); 498 return 0; 499 } 500 501 tpi_data[0] = inti->io.subchannel_id << 16 | inti->io.subchannel_nr; 502 tpi_data[1] = inti->io.io_int_parm; 503 tpi_data[2] = inti->io.io_int_word; 504 if (addr) { 505 /* 506 * Store the two-word I/O interruption code into the 507 * provided area. 508 */ 509 len = sizeof(tpi_data) - 4; 510 rc = write_guest(vcpu, addr, ar, &tpi_data, len); 511 if (rc) { 512 rc = kvm_s390_inject_prog_cond(vcpu, rc); 513 goto reinject_interrupt; 514 } 515 } else { 516 /* 517 * Store the three-word I/O interruption code into 518 * the appropriate lowcore area. 519 */ 520 len = sizeof(tpi_data); 521 if (write_guest_lc(vcpu, __LC_SUBCHANNEL_ID, &tpi_data, len)) { 522 /* failed writes to the low core are not recoverable */ 523 rc = -EFAULT; 524 goto reinject_interrupt; 525 } 526 } 527 528 /* irq was successfully handed to the guest */ 529 kfree(inti); 530 kvm_s390_set_psw_cc(vcpu, 1); 531 return 0; 532 reinject_interrupt: 533 /* 534 * If we encounter a problem storing the interruption code, the 535 * instruction is suppressed from the guest's view: reinject the 536 * interrupt. 537 */ 538 if (kvm_s390_reinject_io_int(vcpu->kvm, inti)) { 539 kfree(inti); 540 rc = -EFAULT; 541 } 542 /* don't set the cc, a pgm irq was injected or we drop to user space */ 543 return rc ? -EFAULT : 0; 544 } 545 546 static int handle_tsch(struct kvm_vcpu *vcpu) 547 { 548 struct kvm_s390_interrupt_info *inti = NULL; 549 const u64 isc_mask = 0xffUL << 24; /* all iscs set */ 550 551 vcpu->stat.instruction_tsch++; 552 553 /* a valid schid has at least one bit set */ 554 if (vcpu->run->s.regs.gprs[1]) 555 inti = kvm_s390_get_io_int(vcpu->kvm, isc_mask, 556 vcpu->run->s.regs.gprs[1]); 557 558 /* 559 * Prepare exit to userspace. 560 * We indicate whether we dequeued a pending I/O interrupt 561 * so that userspace can re-inject it if the instruction gets 562 * a program check. While this may re-order the pending I/O 563 * interrupts, this is no problem since the priority is kept 564 * intact. 565 */ 566 vcpu->run->exit_reason = KVM_EXIT_S390_TSCH; 567 vcpu->run->s390_tsch.dequeued = !!inti; 568 if (inti) { 569 vcpu->run->s390_tsch.subchannel_id = inti->io.subchannel_id; 570 vcpu->run->s390_tsch.subchannel_nr = inti->io.subchannel_nr; 571 vcpu->run->s390_tsch.io_int_parm = inti->io.io_int_parm; 572 vcpu->run->s390_tsch.io_int_word = inti->io.io_int_word; 573 } 574 vcpu->run->s390_tsch.ipb = vcpu->arch.sie_block->ipb; 575 kfree(inti); 576 return -EREMOTE; 577 } 578 579 static int handle_io_inst(struct kvm_vcpu *vcpu) 580 { 581 VCPU_EVENT(vcpu, 4, "%s", "I/O instruction"); 582 583 if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE) 584 return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP); 585 586 if (vcpu->kvm->arch.css_support) { 587 /* 588 * Most I/O instructions will be handled by userspace. 589 * Exceptions are tpi and the interrupt portion of tsch. 590 */ 591 if (vcpu->arch.sie_block->ipa == 0xb236) 592 return handle_tpi(vcpu); 593 if (vcpu->arch.sie_block->ipa == 0xb235) 594 return handle_tsch(vcpu); 595 /* Handle in userspace. */ 596 vcpu->stat.instruction_io_other++; 597 return -EOPNOTSUPP; 598 } else { 599 /* 600 * Set condition code 3 to stop the guest from issuing channel 601 * I/O instructions. 602 */ 603 kvm_s390_set_psw_cc(vcpu, 3); 604 return 0; 605 } 606 } 607 608 /* 609 * handle_pqap: Handling pqap interception 610 * @vcpu: the vcpu having issue the pqap instruction 611 * 612 * We now support PQAP/AQIC instructions and we need to correctly 613 * answer the guest even if no dedicated driver's hook is available. 614 * 615 * The intercepting code calls a dedicated callback for this instruction 616 * if a driver did register one in the CRYPTO satellite of the 617 * SIE block. 618 * 619 * If no callback is available, the queues are not available, return this 620 * response code to the caller and set CC to 3. 621 * Else return the response code returned by the callback. 622 */ 623 static int handle_pqap(struct kvm_vcpu *vcpu) 624 { 625 struct ap_queue_status status = {}; 626 crypto_hook pqap_hook; 627 unsigned long reg0; 628 int ret; 629 uint8_t fc; 630 631 /* Verify that the AP instruction are available */ 632 if (!ap_instructions_available()) 633 return -EOPNOTSUPP; 634 /* Verify that the guest is allowed to use AP instructions */ 635 if (!(vcpu->arch.sie_block->eca & ECA_APIE)) 636 return -EOPNOTSUPP; 637 /* 638 * The only possibly intercepted functions when AP instructions are 639 * available for the guest are AQIC and TAPQ with the t bit set 640 * since we do not set IC.3 (FIII) we currently will only intercept 641 * the AQIC function code. 642 * Note: running nested under z/VM can result in intercepts for other 643 * function codes, e.g. PQAP(QCI). We do not support this and bail out. 644 */ 645 reg0 = vcpu->run->s.regs.gprs[0]; 646 fc = (reg0 >> 24) & 0xff; 647 if (fc != 0x03) 648 return -EOPNOTSUPP; 649 650 /* PQAP instruction is allowed for guest kernel only */ 651 if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE) 652 return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP); 653 654 /* Common PQAP instruction specification exceptions */ 655 /* bits 41-47 must all be zeros */ 656 if (reg0 & 0x007f0000UL) 657 return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION); 658 /* APFT not install and T bit set */ 659 if (!test_kvm_facility(vcpu->kvm, 15) && (reg0 & 0x00800000UL)) 660 return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION); 661 /* APXA not installed and APID greater 64 or APQI greater 16 */ 662 if (!(vcpu->kvm->arch.crypto.crycbd & 0x02) && (reg0 & 0x0000c0f0UL)) 663 return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION); 664 665 /* AQIC function code specific exception */ 666 /* facility 65 not present for AQIC function code */ 667 if (!test_kvm_facility(vcpu->kvm, 65)) 668 return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION); 669 670 /* 671 * If the hook callback is registered, there will be a pointer to the 672 * hook function pointer in the kvm_s390_crypto structure. Lock the 673 * owner, retrieve the hook function pointer and call the hook. 674 */ 675 down_read(&vcpu->kvm->arch.crypto.pqap_hook_rwsem); 676 if (vcpu->kvm->arch.crypto.pqap_hook) { 677 pqap_hook = *vcpu->kvm->arch.crypto.pqap_hook; 678 ret = pqap_hook(vcpu); 679 if (!ret) { 680 if (vcpu->run->s.regs.gprs[1] & 0x00ff0000) 681 kvm_s390_set_psw_cc(vcpu, 3); 682 else 683 kvm_s390_set_psw_cc(vcpu, 0); 684 } 685 up_read(&vcpu->kvm->arch.crypto.pqap_hook_rwsem); 686 return ret; 687 } 688 up_read(&vcpu->kvm->arch.crypto.pqap_hook_rwsem); 689 /* 690 * A vfio_driver must register a hook. 691 * No hook means no driver to enable the SIE CRYCB and no queues. 692 * We send this response to the guest. 693 */ 694 status.response_code = 0x01; 695 memcpy(&vcpu->run->s.regs.gprs[1], &status, sizeof(status)); 696 kvm_s390_set_psw_cc(vcpu, 3); 697 return 0; 698 } 699 700 static int handle_stfl(struct kvm_vcpu *vcpu) 701 { 702 int rc; 703 unsigned int fac; 704 705 vcpu->stat.instruction_stfl++; 706 707 if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE) 708 return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP); 709 710 /* 711 * We need to shift the lower 32 facility bits (bit 0-31) from a u64 712 * into a u32 memory representation. They will remain bits 0-31. 713 */ 714 fac = *vcpu->kvm->arch.model.fac_list >> 32; 715 rc = write_guest_lc(vcpu, offsetof(struct lowcore, stfl_fac_list), 716 &fac, sizeof(fac)); 717 if (rc) 718 return rc; 719 VCPU_EVENT(vcpu, 3, "STFL: store facility list 0x%x", fac); 720 trace_kvm_s390_handle_stfl(vcpu, fac); 721 return 0; 722 } 723 724 #define PSW_MASK_ADDR_MODE (PSW_MASK_EA | PSW_MASK_BA) 725 #define PSW_MASK_UNASSIGNED 0xb80800fe7fffffffUL 726 #define PSW_ADDR_24 0x0000000000ffffffUL 727 #define PSW_ADDR_31 0x000000007fffffffUL 728 729 int is_valid_psw(psw_t *psw) 730 { 731 if (psw->mask & PSW_MASK_UNASSIGNED) 732 return 0; 733 if ((psw->mask & PSW_MASK_ADDR_MODE) == PSW_MASK_BA) { 734 if (psw->addr & ~PSW_ADDR_31) 735 return 0; 736 } 737 if (!(psw->mask & PSW_MASK_ADDR_MODE) && (psw->addr & ~PSW_ADDR_24)) 738 return 0; 739 if ((psw->mask & PSW_MASK_ADDR_MODE) == PSW_MASK_EA) 740 return 0; 741 if (psw->addr & 1) 742 return 0; 743 return 1; 744 } 745 746 int kvm_s390_handle_lpsw(struct kvm_vcpu *vcpu) 747 { 748 psw_t *gpsw = &vcpu->arch.sie_block->gpsw; 749 psw_compat_t new_psw; 750 u64 addr; 751 int rc; 752 u8 ar; 753 754 vcpu->stat.instruction_lpsw++; 755 756 if (gpsw->mask & PSW_MASK_PSTATE) 757 return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP); 758 759 addr = kvm_s390_get_base_disp_s(vcpu, &ar); 760 if (addr & 7) 761 return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION); 762 763 rc = read_guest(vcpu, addr, ar, &new_psw, sizeof(new_psw)); 764 if (rc) 765 return kvm_s390_inject_prog_cond(vcpu, rc); 766 if (!(new_psw.mask & PSW32_MASK_BASE)) 767 return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION); 768 gpsw->mask = (new_psw.mask & ~PSW32_MASK_BASE) << 32; 769 gpsw->mask |= new_psw.addr & PSW32_ADDR_AMODE; 770 gpsw->addr = new_psw.addr & ~PSW32_ADDR_AMODE; 771 if (!is_valid_psw(gpsw)) 772 return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION); 773 return 0; 774 } 775 776 static int handle_lpswe(struct kvm_vcpu *vcpu) 777 { 778 psw_t new_psw; 779 u64 addr; 780 int rc; 781 u8 ar; 782 783 vcpu->stat.instruction_lpswe++; 784 785 if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE) 786 return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP); 787 788 addr = kvm_s390_get_base_disp_s(vcpu, &ar); 789 if (addr & 7) 790 return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION); 791 rc = read_guest(vcpu, addr, ar, &new_psw, sizeof(new_psw)); 792 if (rc) 793 return kvm_s390_inject_prog_cond(vcpu, rc); 794 vcpu->arch.sie_block->gpsw = new_psw; 795 if (!is_valid_psw(&vcpu->arch.sie_block->gpsw)) 796 return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION); 797 return 0; 798 } 799 800 static int handle_stidp(struct kvm_vcpu *vcpu) 801 { 802 u64 stidp_data = vcpu->kvm->arch.model.cpuid; 803 u64 operand2; 804 int rc; 805 u8 ar; 806 807 vcpu->stat.instruction_stidp++; 808 809 if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE) 810 return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP); 811 812 operand2 = kvm_s390_get_base_disp_s(vcpu, &ar); 813 814 if (operand2 & 7) 815 return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION); 816 817 rc = write_guest(vcpu, operand2, ar, &stidp_data, sizeof(stidp_data)); 818 if (rc) 819 return kvm_s390_inject_prog_cond(vcpu, rc); 820 821 VCPU_EVENT(vcpu, 3, "STIDP: store cpu id 0x%llx", stidp_data); 822 return 0; 823 } 824 825 static void handle_stsi_3_2_2(struct kvm_vcpu *vcpu, struct sysinfo_3_2_2 *mem) 826 { 827 int cpus = 0; 828 int n; 829 830 cpus = atomic_read(&vcpu->kvm->online_vcpus); 831 832 /* deal with other level 3 hypervisors */ 833 if (stsi(mem, 3, 2, 2)) 834 mem->count = 0; 835 if (mem->count < 8) 836 mem->count++; 837 for (n = mem->count - 1; n > 0 ; n--) 838 memcpy(&mem->vm[n], &mem->vm[n - 1], sizeof(mem->vm[0])); 839 840 memset(&mem->vm[0], 0, sizeof(mem->vm[0])); 841 mem->vm[0].cpus_total = cpus; 842 mem->vm[0].cpus_configured = cpus; 843 mem->vm[0].cpus_standby = 0; 844 mem->vm[0].cpus_reserved = 0; 845 mem->vm[0].caf = 1000; 846 memcpy(mem->vm[0].name, "KVMguest", 8); 847 ASCEBC(mem->vm[0].name, 8); 848 memcpy(mem->vm[0].cpi, "KVM/Linux ", 16); 849 ASCEBC(mem->vm[0].cpi, 16); 850 } 851 852 static void insert_stsi_usr_data(struct kvm_vcpu *vcpu, u64 addr, u8 ar, 853 u8 fc, u8 sel1, u16 sel2) 854 { 855 vcpu->run->exit_reason = KVM_EXIT_S390_STSI; 856 vcpu->run->s390_stsi.addr = addr; 857 vcpu->run->s390_stsi.ar = ar; 858 vcpu->run->s390_stsi.fc = fc; 859 vcpu->run->s390_stsi.sel1 = sel1; 860 vcpu->run->s390_stsi.sel2 = sel2; 861 } 862 863 static int handle_stsi(struct kvm_vcpu *vcpu) 864 { 865 int fc = (vcpu->run->s.regs.gprs[0] & 0xf0000000) >> 28; 866 int sel1 = vcpu->run->s.regs.gprs[0] & 0xff; 867 int sel2 = vcpu->run->s.regs.gprs[1] & 0xffff; 868 unsigned long mem = 0; 869 u64 operand2; 870 int rc = 0; 871 u8 ar; 872 873 vcpu->stat.instruction_stsi++; 874 VCPU_EVENT(vcpu, 3, "STSI: fc: %u sel1: %u sel2: %u", fc, sel1, sel2); 875 876 if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE) 877 return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP); 878 879 /* Bailout forbidden function codes */ 880 if (fc > 3 && fc != 15) 881 goto out_no_data; 882 883 /* 884 * fc 15 is provided only with 885 * - PTF/CPU topology support through facility 15 886 * - KVM_CAP_S390_USER_STSI 887 */ 888 if (fc == 15 && (!test_kvm_facility(vcpu->kvm, 11) || 889 !vcpu->kvm->arch.user_stsi)) 890 goto out_no_data; 891 892 if (vcpu->run->s.regs.gprs[0] & 0x0fffff00 893 || vcpu->run->s.regs.gprs[1] & 0xffff0000) 894 return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION); 895 896 if (fc == 0) { 897 vcpu->run->s.regs.gprs[0] = 3 << 28; 898 kvm_s390_set_psw_cc(vcpu, 0); 899 return 0; 900 } 901 902 operand2 = kvm_s390_get_base_disp_s(vcpu, &ar); 903 904 if (!kvm_s390_pv_cpu_is_protected(vcpu) && (operand2 & 0xfff)) 905 return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION); 906 907 switch (fc) { 908 case 1: /* same handling for 1 and 2 */ 909 case 2: 910 mem = get_zeroed_page(GFP_KERNEL_ACCOUNT); 911 if (!mem) 912 goto out_no_data; 913 if (stsi((void *) mem, fc, sel1, sel2)) 914 goto out_no_data; 915 break; 916 case 3: 917 if (sel1 != 2 || sel2 != 2) 918 goto out_no_data; 919 mem = get_zeroed_page(GFP_KERNEL_ACCOUNT); 920 if (!mem) 921 goto out_no_data; 922 handle_stsi_3_2_2(vcpu, (void *) mem); 923 break; 924 case 15: /* fc 15 is fully handled in userspace */ 925 insert_stsi_usr_data(vcpu, operand2, ar, fc, sel1, sel2); 926 trace_kvm_s390_handle_stsi(vcpu, fc, sel1, sel2, operand2); 927 return -EREMOTE; 928 } 929 if (kvm_s390_pv_cpu_is_protected(vcpu)) { 930 memcpy(sida_addr(vcpu->arch.sie_block), (void *)mem, PAGE_SIZE); 931 rc = 0; 932 } else { 933 rc = write_guest(vcpu, operand2, ar, (void *)mem, PAGE_SIZE); 934 } 935 if (rc) { 936 rc = kvm_s390_inject_prog_cond(vcpu, rc); 937 goto out; 938 } 939 if (vcpu->kvm->arch.user_stsi) { 940 insert_stsi_usr_data(vcpu, operand2, ar, fc, sel1, sel2); 941 rc = -EREMOTE; 942 } 943 trace_kvm_s390_handle_stsi(vcpu, fc, sel1, sel2, operand2); 944 free_page(mem); 945 kvm_s390_set_psw_cc(vcpu, 0); 946 vcpu->run->s.regs.gprs[0] = 0; 947 return rc; 948 out_no_data: 949 kvm_s390_set_psw_cc(vcpu, 3); 950 out: 951 free_page(mem); 952 return rc; 953 } 954 955 int kvm_s390_handle_b2(struct kvm_vcpu *vcpu) 956 { 957 switch (vcpu->arch.sie_block->ipa & 0x00ff) { 958 case 0x02: 959 return handle_stidp(vcpu); 960 case 0x04: 961 return handle_set_clock(vcpu); 962 case 0x10: 963 return handle_set_prefix(vcpu); 964 case 0x11: 965 return handle_store_prefix(vcpu); 966 case 0x12: 967 return handle_store_cpu_address(vcpu); 968 case 0x14: 969 return kvm_s390_handle_vsie(vcpu); 970 case 0x21: 971 case 0x50: 972 return handle_ipte_interlock(vcpu); 973 case 0x29: 974 return handle_iske(vcpu); 975 case 0x2a: 976 return handle_rrbe(vcpu); 977 case 0x2b: 978 return handle_sske(vcpu); 979 case 0x2c: 980 return handle_test_block(vcpu); 981 case 0x30: 982 case 0x31: 983 case 0x32: 984 case 0x33: 985 case 0x34: 986 case 0x35: 987 case 0x36: 988 case 0x37: 989 case 0x38: 990 case 0x39: 991 case 0x3a: 992 case 0x3b: 993 case 0x3c: 994 case 0x5f: 995 case 0x74: 996 case 0x76: 997 return handle_io_inst(vcpu); 998 case 0x56: 999 return handle_sthyi(vcpu); 1000 case 0x7d: 1001 return handle_stsi(vcpu); 1002 case 0xaf: 1003 return handle_pqap(vcpu); 1004 case 0xb1: 1005 return handle_stfl(vcpu); 1006 case 0xb2: 1007 return handle_lpswe(vcpu); 1008 default: 1009 return -EOPNOTSUPP; 1010 } 1011 } 1012 1013 static int handle_epsw(struct kvm_vcpu *vcpu) 1014 { 1015 int reg1, reg2; 1016 1017 vcpu->stat.instruction_epsw++; 1018 1019 kvm_s390_get_regs_rre(vcpu, ®1, ®2); 1020 1021 /* This basically extracts the mask half of the psw. */ 1022 vcpu->run->s.regs.gprs[reg1] &= 0xffffffff00000000UL; 1023 vcpu->run->s.regs.gprs[reg1] |= vcpu->arch.sie_block->gpsw.mask >> 32; 1024 if (reg2) { 1025 vcpu->run->s.regs.gprs[reg2] &= 0xffffffff00000000UL; 1026 vcpu->run->s.regs.gprs[reg2] |= 1027 vcpu->arch.sie_block->gpsw.mask & 0x00000000ffffffffUL; 1028 } 1029 return 0; 1030 } 1031 1032 #define PFMF_RESERVED 0xfffc0101UL 1033 #define PFMF_SK 0x00020000UL 1034 #define PFMF_CF 0x00010000UL 1035 #define PFMF_UI 0x00008000UL 1036 #define PFMF_FSC 0x00007000UL 1037 #define PFMF_NQ 0x00000800UL 1038 #define PFMF_MR 0x00000400UL 1039 #define PFMF_MC 0x00000200UL 1040 #define PFMF_KEY 0x000000feUL 1041 1042 static int handle_pfmf(struct kvm_vcpu *vcpu) 1043 { 1044 bool mr = false, mc = false, nq; 1045 int reg1, reg2; 1046 unsigned long start, end; 1047 unsigned char key; 1048 1049 vcpu->stat.instruction_pfmf++; 1050 1051 kvm_s390_get_regs_rre(vcpu, ®1, ®2); 1052 1053 if (!test_kvm_facility(vcpu->kvm, 8)) 1054 return kvm_s390_inject_program_int(vcpu, PGM_OPERATION); 1055 1056 if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE) 1057 return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP); 1058 1059 if (vcpu->run->s.regs.gprs[reg1] & PFMF_RESERVED) 1060 return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION); 1061 1062 /* Only provide non-quiescing support if enabled for the guest */ 1063 if (vcpu->run->s.regs.gprs[reg1] & PFMF_NQ && 1064 !test_kvm_facility(vcpu->kvm, 14)) 1065 return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION); 1066 1067 /* Only provide conditional-SSKE support if enabled for the guest */ 1068 if (vcpu->run->s.regs.gprs[reg1] & PFMF_SK && 1069 test_kvm_facility(vcpu->kvm, 10)) { 1070 mr = vcpu->run->s.regs.gprs[reg1] & PFMF_MR; 1071 mc = vcpu->run->s.regs.gprs[reg1] & PFMF_MC; 1072 } 1073 1074 nq = vcpu->run->s.regs.gprs[reg1] & PFMF_NQ; 1075 key = vcpu->run->s.regs.gprs[reg1] & PFMF_KEY; 1076 start = vcpu->run->s.regs.gprs[reg2] & PAGE_MASK; 1077 start = kvm_s390_logical_to_effective(vcpu, start); 1078 1079 if (vcpu->run->s.regs.gprs[reg1] & PFMF_CF) { 1080 if (kvm_s390_check_low_addr_prot_real(vcpu, start)) 1081 return kvm_s390_inject_prog_irq(vcpu, &vcpu->arch.pgm); 1082 } 1083 1084 switch (vcpu->run->s.regs.gprs[reg1] & PFMF_FSC) { 1085 case 0x00000000: 1086 /* only 4k frames specify a real address */ 1087 start = kvm_s390_real_to_abs(vcpu, start); 1088 end = (start + PAGE_SIZE) & ~(PAGE_SIZE - 1); 1089 break; 1090 case 0x00001000: 1091 end = (start + _SEGMENT_SIZE) & ~(_SEGMENT_SIZE - 1); 1092 break; 1093 case 0x00002000: 1094 /* only support 2G frame size if EDAT2 is available and we are 1095 not in 24-bit addressing mode */ 1096 if (!test_kvm_facility(vcpu->kvm, 78) || 1097 psw_bits(vcpu->arch.sie_block->gpsw).eaba == PSW_BITS_AMODE_24BIT) 1098 return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION); 1099 end = (start + _REGION3_SIZE) & ~(_REGION3_SIZE - 1); 1100 break; 1101 default: 1102 return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION); 1103 } 1104 1105 while (start != end) { 1106 unsigned long vmaddr; 1107 bool unlocked = false; 1108 1109 /* Translate guest address to host address */ 1110 vmaddr = gfn_to_hva(vcpu->kvm, gpa_to_gfn(start)); 1111 if (kvm_is_error_hva(vmaddr)) 1112 return kvm_s390_inject_program_int(vcpu, PGM_ADDRESSING); 1113 1114 if (vcpu->run->s.regs.gprs[reg1] & PFMF_CF) { 1115 if (kvm_clear_guest(vcpu->kvm, start, PAGE_SIZE)) 1116 return kvm_s390_inject_program_int(vcpu, PGM_ADDRESSING); 1117 } 1118 1119 if (vcpu->run->s.regs.gprs[reg1] & PFMF_SK) { 1120 int rc = kvm_s390_skey_check_enable(vcpu); 1121 1122 if (rc) 1123 return rc; 1124 mmap_read_lock(current->mm); 1125 rc = cond_set_guest_storage_key(current->mm, vmaddr, 1126 key, NULL, nq, mr, mc); 1127 if (rc < 0) { 1128 rc = fixup_user_fault(current->mm, vmaddr, 1129 FAULT_FLAG_WRITE, &unlocked); 1130 rc = !rc ? -EAGAIN : rc; 1131 } 1132 mmap_read_unlock(current->mm); 1133 if (rc == -EFAULT) 1134 return kvm_s390_inject_program_int(vcpu, PGM_ADDRESSING); 1135 if (rc == -EAGAIN) 1136 continue; 1137 if (rc < 0) 1138 return rc; 1139 } 1140 start += PAGE_SIZE; 1141 } 1142 if (vcpu->run->s.regs.gprs[reg1] & PFMF_FSC) { 1143 if (psw_bits(vcpu->arch.sie_block->gpsw).eaba == PSW_BITS_AMODE_64BIT) { 1144 vcpu->run->s.regs.gprs[reg2] = end; 1145 } else { 1146 vcpu->run->s.regs.gprs[reg2] &= ~0xffffffffUL; 1147 end = kvm_s390_logical_to_effective(vcpu, end); 1148 vcpu->run->s.regs.gprs[reg2] |= end; 1149 } 1150 } 1151 return 0; 1152 } 1153 1154 /* 1155 * Must be called with relevant read locks held (kvm->mm->mmap_lock, kvm->srcu) 1156 */ 1157 static inline int __do_essa(struct kvm_vcpu *vcpu, const int orc) 1158 { 1159 int r1, r2, nappended, entries; 1160 unsigned long gfn, hva, res, pgstev, ptev; 1161 unsigned long *cbrlo; 1162 1163 /* 1164 * We don't need to set SD.FPF.SK to 1 here, because if we have a 1165 * machine check here we either handle it or crash 1166 */ 1167 1168 kvm_s390_get_regs_rre(vcpu, &r1, &r2); 1169 gfn = vcpu->run->s.regs.gprs[r2] >> PAGE_SHIFT; 1170 hva = gfn_to_hva(vcpu->kvm, gfn); 1171 entries = (vcpu->arch.sie_block->cbrlo & ~PAGE_MASK) >> 3; 1172 1173 if (kvm_is_error_hva(hva)) 1174 return kvm_s390_inject_program_int(vcpu, PGM_ADDRESSING); 1175 1176 nappended = pgste_perform_essa(vcpu->kvm->mm, hva, orc, &ptev, &pgstev); 1177 if (nappended < 0) { 1178 res = orc ? 0x10 : 0; 1179 vcpu->run->s.regs.gprs[r1] = res; /* Exception Indication */ 1180 return 0; 1181 } 1182 res = (pgstev & _PGSTE_GPS_USAGE_MASK) >> 22; 1183 /* 1184 * Set the block-content state part of the result. 0 means resident, so 1185 * nothing to do if the page is valid. 2 is for preserved pages 1186 * (non-present and non-zero), and 3 for zero pages (non-present and 1187 * zero). 1188 */ 1189 if (ptev & _PAGE_INVALID) { 1190 res |= 2; 1191 if (pgstev & _PGSTE_GPS_ZERO) 1192 res |= 1; 1193 } 1194 if (pgstev & _PGSTE_GPS_NODAT) 1195 res |= 0x20; 1196 vcpu->run->s.regs.gprs[r1] = res; 1197 /* 1198 * It is possible that all the normal 511 slots were full, in which case 1199 * we will now write in the 512th slot, which is reserved for host use. 1200 * In both cases we let the normal essa handling code process all the 1201 * slots, including the reserved one, if needed. 1202 */ 1203 if (nappended > 0) { 1204 cbrlo = phys_to_virt(vcpu->arch.sie_block->cbrlo & PAGE_MASK); 1205 cbrlo[entries] = gfn << PAGE_SHIFT; 1206 } 1207 1208 if (orc) { 1209 struct kvm_memory_slot *ms = gfn_to_memslot(vcpu->kvm, gfn); 1210 1211 /* Increment only if we are really flipping the bit */ 1212 if (ms && !test_and_set_bit(gfn - ms->base_gfn, kvm_second_dirty_bitmap(ms))) 1213 atomic64_inc(&vcpu->kvm->arch.cmma_dirty_pages); 1214 } 1215 1216 return nappended; 1217 } 1218 1219 static int handle_essa(struct kvm_vcpu *vcpu) 1220 { 1221 /* entries expected to be 1FF */ 1222 int entries = (vcpu->arch.sie_block->cbrlo & ~PAGE_MASK) >> 3; 1223 unsigned long *cbrlo; 1224 struct gmap *gmap; 1225 int i, orc; 1226 1227 VCPU_EVENT(vcpu, 4, "ESSA: release %d pages", entries); 1228 gmap = vcpu->arch.gmap; 1229 vcpu->stat.instruction_essa++; 1230 if (!vcpu->kvm->arch.use_cmma) 1231 return kvm_s390_inject_program_int(vcpu, PGM_OPERATION); 1232 1233 if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE) 1234 return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP); 1235 /* Check for invalid operation request code */ 1236 orc = (vcpu->arch.sie_block->ipb & 0xf0000000) >> 28; 1237 /* ORCs 0-6 are always valid */ 1238 if (orc > (test_kvm_facility(vcpu->kvm, 147) ? ESSA_SET_STABLE_NODAT 1239 : ESSA_SET_STABLE_IF_RESIDENT)) 1240 return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION); 1241 1242 if (!vcpu->kvm->arch.migration_mode) { 1243 /* 1244 * CMMA is enabled in the KVM settings, but is disabled in 1245 * the SIE block and in the mm_context, and we are not doing 1246 * a migration. Enable CMMA in the mm_context. 1247 * Since we need to take a write lock to write to the context 1248 * to avoid races with storage keys handling, we check if the 1249 * value really needs to be written to; if the value is 1250 * already correct, we do nothing and avoid the lock. 1251 */ 1252 if (vcpu->kvm->mm->context.uses_cmm == 0) { 1253 mmap_write_lock(vcpu->kvm->mm); 1254 vcpu->kvm->mm->context.uses_cmm = 1; 1255 mmap_write_unlock(vcpu->kvm->mm); 1256 } 1257 /* 1258 * If we are here, we are supposed to have CMMA enabled in 1259 * the SIE block. Enabling CMMA works on a per-CPU basis, 1260 * while the context use_cmma flag is per process. 1261 * It's possible that the context flag is enabled and the 1262 * SIE flag is not, so we set the flag always; if it was 1263 * already set, nothing changes, otherwise we enable it 1264 * on this CPU too. 1265 */ 1266 vcpu->arch.sie_block->ecb2 |= ECB2_CMMA; 1267 /* Retry the ESSA instruction */ 1268 kvm_s390_retry_instr(vcpu); 1269 } else { 1270 int srcu_idx; 1271 1272 mmap_read_lock(vcpu->kvm->mm); 1273 srcu_idx = srcu_read_lock(&vcpu->kvm->srcu); 1274 i = __do_essa(vcpu, orc); 1275 srcu_read_unlock(&vcpu->kvm->srcu, srcu_idx); 1276 mmap_read_unlock(vcpu->kvm->mm); 1277 if (i < 0) 1278 return i; 1279 /* Account for the possible extra cbrl entry */ 1280 entries += i; 1281 } 1282 vcpu->arch.sie_block->cbrlo &= PAGE_MASK; /* reset nceo */ 1283 cbrlo = phys_to_virt(vcpu->arch.sie_block->cbrlo); 1284 mmap_read_lock(gmap->mm); 1285 for (i = 0; i < entries; ++i) 1286 __gmap_zap(gmap, cbrlo[i]); 1287 mmap_read_unlock(gmap->mm); 1288 return 0; 1289 } 1290 1291 int kvm_s390_handle_b9(struct kvm_vcpu *vcpu) 1292 { 1293 switch (vcpu->arch.sie_block->ipa & 0x00ff) { 1294 case 0x8a: 1295 case 0x8e: 1296 case 0x8f: 1297 return handle_ipte_interlock(vcpu); 1298 case 0x8d: 1299 return handle_epsw(vcpu); 1300 case 0xab: 1301 return handle_essa(vcpu); 1302 case 0xaf: 1303 return handle_pfmf(vcpu); 1304 default: 1305 return -EOPNOTSUPP; 1306 } 1307 } 1308 1309 int kvm_s390_handle_lctl(struct kvm_vcpu *vcpu) 1310 { 1311 int reg1 = (vcpu->arch.sie_block->ipa & 0x00f0) >> 4; 1312 int reg3 = vcpu->arch.sie_block->ipa & 0x000f; 1313 int reg, rc, nr_regs; 1314 u32 ctl_array[16]; 1315 u64 ga; 1316 u8 ar; 1317 1318 vcpu->stat.instruction_lctl++; 1319 1320 if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE) 1321 return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP); 1322 1323 ga = kvm_s390_get_base_disp_rs(vcpu, &ar); 1324 1325 if (ga & 3) 1326 return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION); 1327 1328 VCPU_EVENT(vcpu, 4, "LCTL: r1:%d, r3:%d, addr: 0x%llx", reg1, reg3, ga); 1329 trace_kvm_s390_handle_lctl(vcpu, 0, reg1, reg3, ga); 1330 1331 nr_regs = ((reg3 - reg1) & 0xf) + 1; 1332 rc = read_guest(vcpu, ga, ar, ctl_array, nr_regs * sizeof(u32)); 1333 if (rc) 1334 return kvm_s390_inject_prog_cond(vcpu, rc); 1335 reg = reg1; 1336 nr_regs = 0; 1337 do { 1338 vcpu->arch.sie_block->gcr[reg] &= 0xffffffff00000000ul; 1339 vcpu->arch.sie_block->gcr[reg] |= ctl_array[nr_regs++]; 1340 if (reg == reg3) 1341 break; 1342 reg = (reg + 1) % 16; 1343 } while (1); 1344 kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu); 1345 return 0; 1346 } 1347 1348 int kvm_s390_handle_stctl(struct kvm_vcpu *vcpu) 1349 { 1350 int reg1 = (vcpu->arch.sie_block->ipa & 0x00f0) >> 4; 1351 int reg3 = vcpu->arch.sie_block->ipa & 0x000f; 1352 int reg, rc, nr_regs; 1353 u32 ctl_array[16]; 1354 u64 ga; 1355 u8 ar; 1356 1357 vcpu->stat.instruction_stctl++; 1358 1359 if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE) 1360 return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP); 1361 1362 ga = kvm_s390_get_base_disp_rs(vcpu, &ar); 1363 1364 if (ga & 3) 1365 return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION); 1366 1367 VCPU_EVENT(vcpu, 4, "STCTL r1:%d, r3:%d, addr: 0x%llx", reg1, reg3, ga); 1368 trace_kvm_s390_handle_stctl(vcpu, 0, reg1, reg3, ga); 1369 1370 reg = reg1; 1371 nr_regs = 0; 1372 do { 1373 ctl_array[nr_regs++] = vcpu->arch.sie_block->gcr[reg]; 1374 if (reg == reg3) 1375 break; 1376 reg = (reg + 1) % 16; 1377 } while (1); 1378 rc = write_guest(vcpu, ga, ar, ctl_array, nr_regs * sizeof(u32)); 1379 return rc ? kvm_s390_inject_prog_cond(vcpu, rc) : 0; 1380 } 1381 1382 static int handle_lctlg(struct kvm_vcpu *vcpu) 1383 { 1384 int reg1 = (vcpu->arch.sie_block->ipa & 0x00f0) >> 4; 1385 int reg3 = vcpu->arch.sie_block->ipa & 0x000f; 1386 int reg, rc, nr_regs; 1387 u64 ctl_array[16]; 1388 u64 ga; 1389 u8 ar; 1390 1391 vcpu->stat.instruction_lctlg++; 1392 1393 if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE) 1394 return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP); 1395 1396 ga = kvm_s390_get_base_disp_rsy(vcpu, &ar); 1397 1398 if (ga & 7) 1399 return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION); 1400 1401 VCPU_EVENT(vcpu, 4, "LCTLG: r1:%d, r3:%d, addr: 0x%llx", reg1, reg3, ga); 1402 trace_kvm_s390_handle_lctl(vcpu, 1, reg1, reg3, ga); 1403 1404 nr_regs = ((reg3 - reg1) & 0xf) + 1; 1405 rc = read_guest(vcpu, ga, ar, ctl_array, nr_regs * sizeof(u64)); 1406 if (rc) 1407 return kvm_s390_inject_prog_cond(vcpu, rc); 1408 reg = reg1; 1409 nr_regs = 0; 1410 do { 1411 vcpu->arch.sie_block->gcr[reg] = ctl_array[nr_regs++]; 1412 if (reg == reg3) 1413 break; 1414 reg = (reg + 1) % 16; 1415 } while (1); 1416 kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu); 1417 return 0; 1418 } 1419 1420 static int handle_stctg(struct kvm_vcpu *vcpu) 1421 { 1422 int reg1 = (vcpu->arch.sie_block->ipa & 0x00f0) >> 4; 1423 int reg3 = vcpu->arch.sie_block->ipa & 0x000f; 1424 int reg, rc, nr_regs; 1425 u64 ctl_array[16]; 1426 u64 ga; 1427 u8 ar; 1428 1429 vcpu->stat.instruction_stctg++; 1430 1431 if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE) 1432 return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP); 1433 1434 ga = kvm_s390_get_base_disp_rsy(vcpu, &ar); 1435 1436 if (ga & 7) 1437 return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION); 1438 1439 VCPU_EVENT(vcpu, 4, "STCTG r1:%d, r3:%d, addr: 0x%llx", reg1, reg3, ga); 1440 trace_kvm_s390_handle_stctl(vcpu, 1, reg1, reg3, ga); 1441 1442 reg = reg1; 1443 nr_regs = 0; 1444 do { 1445 ctl_array[nr_regs++] = vcpu->arch.sie_block->gcr[reg]; 1446 if (reg == reg3) 1447 break; 1448 reg = (reg + 1) % 16; 1449 } while (1); 1450 rc = write_guest(vcpu, ga, ar, ctl_array, nr_regs * sizeof(u64)); 1451 return rc ? kvm_s390_inject_prog_cond(vcpu, rc) : 0; 1452 } 1453 1454 int kvm_s390_handle_eb(struct kvm_vcpu *vcpu) 1455 { 1456 switch (vcpu->arch.sie_block->ipb & 0x000000ff) { 1457 case 0x25: 1458 return handle_stctg(vcpu); 1459 case 0x2f: 1460 return handle_lctlg(vcpu); 1461 case 0x60: 1462 case 0x61: 1463 case 0x62: 1464 return handle_ri(vcpu); 1465 default: 1466 return -EOPNOTSUPP; 1467 } 1468 } 1469 1470 static int handle_tprot(struct kvm_vcpu *vcpu) 1471 { 1472 u64 address, operand2; 1473 unsigned long gpa; 1474 u8 access_key; 1475 bool writable; 1476 int ret, cc; 1477 u8 ar; 1478 1479 vcpu->stat.instruction_tprot++; 1480 1481 if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE) 1482 return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP); 1483 1484 kvm_s390_get_base_disp_sse(vcpu, &address, &operand2, &ar, NULL); 1485 access_key = (operand2 & 0xf0) >> 4; 1486 1487 if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_DAT) 1488 ipte_lock(vcpu->kvm); 1489 1490 ret = guest_translate_address_with_key(vcpu, address, ar, &gpa, 1491 GACC_STORE, access_key); 1492 if (ret == 0) { 1493 gfn_to_hva_prot(vcpu->kvm, gpa_to_gfn(gpa), &writable); 1494 } else if (ret == PGM_PROTECTION) { 1495 writable = false; 1496 /* Write protected? Try again with read-only... */ 1497 ret = guest_translate_address_with_key(vcpu, address, ar, &gpa, 1498 GACC_FETCH, access_key); 1499 } 1500 if (ret >= 0) { 1501 cc = -1; 1502 1503 /* Fetching permitted; storing permitted */ 1504 if (ret == 0 && writable) 1505 cc = 0; 1506 /* Fetching permitted; storing not permitted */ 1507 else if (ret == 0 && !writable) 1508 cc = 1; 1509 /* Fetching not permitted; storing not permitted */ 1510 else if (ret == PGM_PROTECTION) 1511 cc = 2; 1512 /* Translation not available */ 1513 else if (ret != PGM_ADDRESSING && ret != PGM_TRANSLATION_SPEC) 1514 cc = 3; 1515 1516 if (cc != -1) { 1517 kvm_s390_set_psw_cc(vcpu, cc); 1518 ret = 0; 1519 } else { 1520 ret = kvm_s390_inject_program_int(vcpu, ret); 1521 } 1522 } 1523 1524 if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_DAT) 1525 ipte_unlock(vcpu->kvm); 1526 return ret; 1527 } 1528 1529 int kvm_s390_handle_e5(struct kvm_vcpu *vcpu) 1530 { 1531 switch (vcpu->arch.sie_block->ipa & 0x00ff) { 1532 case 0x01: 1533 return handle_tprot(vcpu); 1534 default: 1535 return -EOPNOTSUPP; 1536 } 1537 } 1538 1539 static int handle_sckpf(struct kvm_vcpu *vcpu) 1540 { 1541 u32 value; 1542 1543 vcpu->stat.instruction_sckpf++; 1544 1545 if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE) 1546 return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP); 1547 1548 if (vcpu->run->s.regs.gprs[0] & 0x00000000ffff0000) 1549 return kvm_s390_inject_program_int(vcpu, 1550 PGM_SPECIFICATION); 1551 1552 value = vcpu->run->s.regs.gprs[0] & 0x000000000000ffff; 1553 vcpu->arch.sie_block->todpr = value; 1554 1555 return 0; 1556 } 1557 1558 static int handle_ptff(struct kvm_vcpu *vcpu) 1559 { 1560 vcpu->stat.instruction_ptff++; 1561 1562 /* we don't emulate any control instructions yet */ 1563 kvm_s390_set_psw_cc(vcpu, 3); 1564 return 0; 1565 } 1566 1567 int kvm_s390_handle_01(struct kvm_vcpu *vcpu) 1568 { 1569 switch (vcpu->arch.sie_block->ipa & 0x00ff) { 1570 case 0x04: 1571 return handle_ptff(vcpu); 1572 case 0x07: 1573 return handle_sckpf(vcpu); 1574 default: 1575 return -EOPNOTSUPP; 1576 } 1577 } 1578