1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * handling privileged instructions 4 * 5 * Copyright IBM Corp. 2008, 2020 6 * 7 * Author(s): Carsten Otte <cotte@de.ibm.com> 8 * Christian Borntraeger <borntraeger@de.ibm.com> 9 */ 10 11 #include <linux/kvm.h> 12 #include <linux/gfp.h> 13 #include <linux/errno.h> 14 #include <linux/mm_types.h> 15 #include <linux/pgtable.h> 16 #include <linux/io.h> 17 #include <asm/asm-offsets.h> 18 #include <asm/facility.h> 19 #include <asm/current.h> 20 #include <asm/debug.h> 21 #include <asm/ebcdic.h> 22 #include <asm/sysinfo.h> 23 #include <asm/page-states.h> 24 #include <asm/gmap.h> 25 #include <asm/ptrace.h> 26 #include <asm/sclp.h> 27 #include <asm/ap.h> 28 #include "gaccess.h" 29 #include "kvm-s390.h" 30 #include "trace.h" 31 32 static int handle_ri(struct kvm_vcpu *vcpu) 33 { 34 vcpu->stat.instruction_ri++; 35 36 if (test_kvm_facility(vcpu->kvm, 64)) { 37 VCPU_EVENT(vcpu, 3, "%s", "ENABLE: RI (lazy)"); 38 vcpu->arch.sie_block->ecb3 |= ECB3_RI; 39 kvm_s390_retry_instr(vcpu); 40 return 0; 41 } else 42 return kvm_s390_inject_program_int(vcpu, PGM_OPERATION); 43 } 44 45 int kvm_s390_handle_aa(struct kvm_vcpu *vcpu) 46 { 47 if ((vcpu->arch.sie_block->ipa & 0xf) <= 4) 48 return handle_ri(vcpu); 49 else 50 return -EOPNOTSUPP; 51 } 52 53 static int handle_gs(struct kvm_vcpu *vcpu) 54 { 55 vcpu->stat.instruction_gs++; 56 57 if (test_kvm_facility(vcpu->kvm, 133)) { 58 VCPU_EVENT(vcpu, 3, "%s", "ENABLE: GS (lazy)"); 59 preempt_disable(); 60 local_ctl_set_bit(2, CR2_GUARDED_STORAGE_BIT); 61 current->thread.gs_cb = (struct gs_cb *)&vcpu->run->s.regs.gscb; 62 restore_gs_cb(current->thread.gs_cb); 63 preempt_enable(); 64 vcpu->arch.sie_block->ecb |= ECB_GS; 65 vcpu->arch.sie_block->ecd |= ECD_HOSTREGMGMT; 66 vcpu->arch.gs_enabled = 1; 67 kvm_s390_retry_instr(vcpu); 68 return 0; 69 } else 70 return kvm_s390_inject_program_int(vcpu, PGM_OPERATION); 71 } 72 73 int kvm_s390_handle_e3(struct kvm_vcpu *vcpu) 74 { 75 int code = vcpu->arch.sie_block->ipb & 0xff; 76 77 if (code == 0x49 || code == 0x4d) 78 return handle_gs(vcpu); 79 else 80 return -EOPNOTSUPP; 81 } 82 /* Handle SCK (SET CLOCK) interception */ 83 static int handle_set_clock(struct kvm_vcpu *vcpu) 84 { 85 struct kvm_s390_vm_tod_clock gtod = { 0 }; 86 int rc; 87 u8 ar; 88 u64 op2; 89 90 vcpu->stat.instruction_sck++; 91 92 if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE) 93 return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP); 94 95 op2 = kvm_s390_get_base_disp_s(vcpu, &ar); 96 if (op2 & 7) /* Operand must be on a doubleword boundary */ 97 return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION); 98 rc = read_guest(vcpu, op2, ar, >od.tod, sizeof(gtod.tod)); 99 if (rc) 100 return kvm_s390_inject_prog_cond(vcpu, rc); 101 102 VCPU_EVENT(vcpu, 3, "SCK: setting guest TOD to 0x%llx", gtod.tod); 103 /* 104 * To set the TOD clock the kvm lock must be taken, but the vcpu lock 105 * is already held in handle_set_clock. The usual lock order is the 106 * opposite. As SCK is deprecated and should not be used in several 107 * cases, for example when the multiple epoch facility or TOD clock 108 * steering facility is installed (see Principles of Operation), a 109 * slow path can be used. If the lock can not be taken via try_lock, 110 * the instruction will be retried via -EAGAIN at a later point in 111 * time. 112 */ 113 if (!kvm_s390_try_set_tod_clock(vcpu->kvm, >od)) { 114 kvm_s390_retry_instr(vcpu); 115 return -EAGAIN; 116 } 117 118 kvm_s390_set_psw_cc(vcpu, 0); 119 return 0; 120 } 121 122 static int handle_set_prefix(struct kvm_vcpu *vcpu) 123 { 124 u64 operand2; 125 u32 address; 126 int rc; 127 u8 ar; 128 129 vcpu->stat.instruction_spx++; 130 131 if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE) 132 return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP); 133 134 operand2 = kvm_s390_get_base_disp_s(vcpu, &ar); 135 136 /* must be word boundary */ 137 if (operand2 & 3) 138 return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION); 139 140 /* get the value */ 141 rc = read_guest(vcpu, operand2, ar, &address, sizeof(address)); 142 if (rc) 143 return kvm_s390_inject_prog_cond(vcpu, rc); 144 145 address &= 0x7fffe000u; 146 147 /* 148 * Make sure the new value is valid memory. We only need to check the 149 * first page, since address is 8k aligned and memory pieces are always 150 * at least 1MB aligned and have at least a size of 1MB. 151 */ 152 if (kvm_is_error_gpa(vcpu->kvm, address)) 153 return kvm_s390_inject_program_int(vcpu, PGM_ADDRESSING); 154 155 kvm_s390_set_prefix(vcpu, address); 156 trace_kvm_s390_handle_prefix(vcpu, 1, address); 157 return 0; 158 } 159 160 static int handle_store_prefix(struct kvm_vcpu *vcpu) 161 { 162 u64 operand2; 163 u32 address; 164 int rc; 165 u8 ar; 166 167 vcpu->stat.instruction_stpx++; 168 169 if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE) 170 return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP); 171 172 operand2 = kvm_s390_get_base_disp_s(vcpu, &ar); 173 174 /* must be word boundary */ 175 if (operand2 & 3) 176 return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION); 177 178 address = kvm_s390_get_prefix(vcpu); 179 180 /* get the value */ 181 rc = write_guest(vcpu, operand2, ar, &address, sizeof(address)); 182 if (rc) 183 return kvm_s390_inject_prog_cond(vcpu, rc); 184 185 VCPU_EVENT(vcpu, 3, "STPX: storing prefix 0x%x into 0x%llx", address, operand2); 186 trace_kvm_s390_handle_prefix(vcpu, 0, address); 187 return 0; 188 } 189 190 static int handle_store_cpu_address(struct kvm_vcpu *vcpu) 191 { 192 u16 vcpu_id = vcpu->vcpu_id; 193 u64 ga; 194 int rc; 195 u8 ar; 196 197 vcpu->stat.instruction_stap++; 198 199 if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE) 200 return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP); 201 202 ga = kvm_s390_get_base_disp_s(vcpu, &ar); 203 204 if (ga & 1) 205 return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION); 206 207 rc = write_guest(vcpu, ga, ar, &vcpu_id, sizeof(vcpu_id)); 208 if (rc) 209 return kvm_s390_inject_prog_cond(vcpu, rc); 210 211 VCPU_EVENT(vcpu, 3, "STAP: storing cpu address (%u) to 0x%llx", vcpu_id, ga); 212 trace_kvm_s390_handle_stap(vcpu, ga); 213 return 0; 214 } 215 216 int kvm_s390_skey_check_enable(struct kvm_vcpu *vcpu) 217 { 218 int rc; 219 220 trace_kvm_s390_skey_related_inst(vcpu); 221 /* Already enabled? */ 222 if (vcpu->arch.skey_enabled) 223 return 0; 224 225 rc = s390_enable_skey(); 226 VCPU_EVENT(vcpu, 3, "enabling storage keys for guest: %d", rc); 227 if (rc) 228 return rc; 229 230 if (kvm_s390_test_cpuflags(vcpu, CPUSTAT_KSS)) 231 kvm_s390_clear_cpuflags(vcpu, CPUSTAT_KSS); 232 if (!vcpu->kvm->arch.use_skf) 233 vcpu->arch.sie_block->ictl |= ICTL_ISKE | ICTL_SSKE | ICTL_RRBE; 234 else 235 vcpu->arch.sie_block->ictl &= ~(ICTL_ISKE | ICTL_SSKE | ICTL_RRBE); 236 vcpu->arch.skey_enabled = true; 237 return 0; 238 } 239 240 static int try_handle_skey(struct kvm_vcpu *vcpu) 241 { 242 int rc; 243 244 rc = kvm_s390_skey_check_enable(vcpu); 245 if (rc) 246 return rc; 247 if (vcpu->kvm->arch.use_skf) { 248 /* with storage-key facility, SIE interprets it for us */ 249 kvm_s390_retry_instr(vcpu); 250 VCPU_EVENT(vcpu, 4, "%s", "retrying storage key operation"); 251 return -EAGAIN; 252 } 253 return 0; 254 } 255 256 static int handle_iske(struct kvm_vcpu *vcpu) 257 { 258 unsigned long gaddr, vmaddr; 259 unsigned char key; 260 int reg1, reg2; 261 bool unlocked; 262 int rc; 263 264 vcpu->stat.instruction_iske++; 265 266 if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE) 267 return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP); 268 269 rc = try_handle_skey(vcpu); 270 if (rc) 271 return rc != -EAGAIN ? rc : 0; 272 273 kvm_s390_get_regs_rre(vcpu, ®1, ®2); 274 275 gaddr = vcpu->run->s.regs.gprs[reg2] & PAGE_MASK; 276 gaddr = kvm_s390_logical_to_effective(vcpu, gaddr); 277 gaddr = kvm_s390_real_to_abs(vcpu, gaddr); 278 vmaddr = gfn_to_hva(vcpu->kvm, gpa_to_gfn(gaddr)); 279 if (kvm_is_error_hva(vmaddr)) 280 return kvm_s390_inject_program_int(vcpu, PGM_ADDRESSING); 281 retry: 282 unlocked = false; 283 mmap_read_lock(current->mm); 284 rc = get_guest_storage_key(current->mm, vmaddr, &key); 285 286 if (rc) { 287 rc = fixup_user_fault(current->mm, vmaddr, 288 FAULT_FLAG_WRITE, &unlocked); 289 if (!rc) { 290 mmap_read_unlock(current->mm); 291 goto retry; 292 } 293 } 294 mmap_read_unlock(current->mm); 295 if (rc == -EFAULT) 296 return kvm_s390_inject_program_int(vcpu, PGM_ADDRESSING); 297 if (rc < 0) 298 return rc; 299 vcpu->run->s.regs.gprs[reg1] &= ~0xff; 300 vcpu->run->s.regs.gprs[reg1] |= key; 301 return 0; 302 } 303 304 static int handle_rrbe(struct kvm_vcpu *vcpu) 305 { 306 unsigned long vmaddr, gaddr; 307 int reg1, reg2; 308 bool unlocked; 309 int rc; 310 311 vcpu->stat.instruction_rrbe++; 312 313 if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE) 314 return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP); 315 316 rc = try_handle_skey(vcpu); 317 if (rc) 318 return rc != -EAGAIN ? rc : 0; 319 320 kvm_s390_get_regs_rre(vcpu, ®1, ®2); 321 322 gaddr = vcpu->run->s.regs.gprs[reg2] & PAGE_MASK; 323 gaddr = kvm_s390_logical_to_effective(vcpu, gaddr); 324 gaddr = kvm_s390_real_to_abs(vcpu, gaddr); 325 vmaddr = gfn_to_hva(vcpu->kvm, gpa_to_gfn(gaddr)); 326 if (kvm_is_error_hva(vmaddr)) 327 return kvm_s390_inject_program_int(vcpu, PGM_ADDRESSING); 328 retry: 329 unlocked = false; 330 mmap_read_lock(current->mm); 331 rc = reset_guest_reference_bit(current->mm, vmaddr); 332 if (rc < 0) { 333 rc = fixup_user_fault(current->mm, vmaddr, 334 FAULT_FLAG_WRITE, &unlocked); 335 if (!rc) { 336 mmap_read_unlock(current->mm); 337 goto retry; 338 } 339 } 340 mmap_read_unlock(current->mm); 341 if (rc == -EFAULT) 342 return kvm_s390_inject_program_int(vcpu, PGM_ADDRESSING); 343 if (rc < 0) 344 return rc; 345 kvm_s390_set_psw_cc(vcpu, rc); 346 return 0; 347 } 348 349 #define SSKE_NQ 0x8 350 #define SSKE_MR 0x4 351 #define SSKE_MC 0x2 352 #define SSKE_MB 0x1 353 static int handle_sske(struct kvm_vcpu *vcpu) 354 { 355 unsigned char m3 = vcpu->arch.sie_block->ipb >> 28; 356 unsigned long start, end; 357 unsigned char key, oldkey; 358 int reg1, reg2; 359 bool unlocked; 360 int rc; 361 362 vcpu->stat.instruction_sske++; 363 364 if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE) 365 return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP); 366 367 rc = try_handle_skey(vcpu); 368 if (rc) 369 return rc != -EAGAIN ? rc : 0; 370 371 if (!test_kvm_facility(vcpu->kvm, 8)) 372 m3 &= ~SSKE_MB; 373 if (!test_kvm_facility(vcpu->kvm, 10)) 374 m3 &= ~(SSKE_MC | SSKE_MR); 375 if (!test_kvm_facility(vcpu->kvm, 14)) 376 m3 &= ~SSKE_NQ; 377 378 kvm_s390_get_regs_rre(vcpu, ®1, ®2); 379 380 key = vcpu->run->s.regs.gprs[reg1] & 0xfe; 381 start = vcpu->run->s.regs.gprs[reg2] & PAGE_MASK; 382 start = kvm_s390_logical_to_effective(vcpu, start); 383 if (m3 & SSKE_MB) { 384 /* start already designates an absolute address */ 385 end = (start + _SEGMENT_SIZE) & ~(_SEGMENT_SIZE - 1); 386 } else { 387 start = kvm_s390_real_to_abs(vcpu, start); 388 end = start + PAGE_SIZE; 389 } 390 391 while (start != end) { 392 unsigned long vmaddr = gfn_to_hva(vcpu->kvm, gpa_to_gfn(start)); 393 unlocked = false; 394 395 if (kvm_is_error_hva(vmaddr)) 396 return kvm_s390_inject_program_int(vcpu, PGM_ADDRESSING); 397 398 mmap_read_lock(current->mm); 399 rc = cond_set_guest_storage_key(current->mm, vmaddr, key, &oldkey, 400 m3 & SSKE_NQ, m3 & SSKE_MR, 401 m3 & SSKE_MC); 402 403 if (rc < 0) { 404 rc = fixup_user_fault(current->mm, vmaddr, 405 FAULT_FLAG_WRITE, &unlocked); 406 rc = !rc ? -EAGAIN : rc; 407 } 408 mmap_read_unlock(current->mm); 409 if (rc == -EFAULT) 410 return kvm_s390_inject_program_int(vcpu, PGM_ADDRESSING); 411 if (rc == -EAGAIN) 412 continue; 413 if (rc < 0) 414 return rc; 415 start += PAGE_SIZE; 416 } 417 418 if (m3 & (SSKE_MC | SSKE_MR)) { 419 if (m3 & SSKE_MB) { 420 /* skey in reg1 is unpredictable */ 421 kvm_s390_set_psw_cc(vcpu, 3); 422 } else { 423 kvm_s390_set_psw_cc(vcpu, rc); 424 vcpu->run->s.regs.gprs[reg1] &= ~0xff00UL; 425 vcpu->run->s.regs.gprs[reg1] |= (u64) oldkey << 8; 426 } 427 } 428 if (m3 & SSKE_MB) { 429 if (psw_bits(vcpu->arch.sie_block->gpsw).eaba == PSW_BITS_AMODE_64BIT) 430 vcpu->run->s.regs.gprs[reg2] &= ~PAGE_MASK; 431 else 432 vcpu->run->s.regs.gprs[reg2] &= ~0xfffff000UL; 433 end = kvm_s390_logical_to_effective(vcpu, end); 434 vcpu->run->s.regs.gprs[reg2] |= end; 435 } 436 return 0; 437 } 438 439 static int handle_ipte_interlock(struct kvm_vcpu *vcpu) 440 { 441 vcpu->stat.instruction_ipte_interlock++; 442 if (psw_bits(vcpu->arch.sie_block->gpsw).pstate) 443 return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP); 444 wait_event(vcpu->kvm->arch.ipte_wq, !ipte_lock_held(vcpu->kvm)); 445 kvm_s390_retry_instr(vcpu); 446 VCPU_EVENT(vcpu, 4, "%s", "retrying ipte interlock operation"); 447 return 0; 448 } 449 450 static int handle_test_block(struct kvm_vcpu *vcpu) 451 { 452 gpa_t addr; 453 int reg2; 454 455 vcpu->stat.instruction_tb++; 456 457 if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE) 458 return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP); 459 460 kvm_s390_get_regs_rre(vcpu, NULL, ®2); 461 addr = vcpu->run->s.regs.gprs[reg2] & PAGE_MASK; 462 addr = kvm_s390_logical_to_effective(vcpu, addr); 463 if (kvm_s390_check_low_addr_prot_real(vcpu, addr)) 464 return kvm_s390_inject_prog_irq(vcpu, &vcpu->arch.pgm); 465 addr = kvm_s390_real_to_abs(vcpu, addr); 466 467 if (kvm_is_error_gpa(vcpu->kvm, addr)) 468 return kvm_s390_inject_program_int(vcpu, PGM_ADDRESSING); 469 /* 470 * We don't expect errors on modern systems, and do not care 471 * about storage keys (yet), so let's just clear the page. 472 */ 473 if (kvm_clear_guest(vcpu->kvm, addr, PAGE_SIZE)) 474 return -EFAULT; 475 kvm_s390_set_psw_cc(vcpu, 0); 476 vcpu->run->s.regs.gprs[0] = 0; 477 return 0; 478 } 479 480 static int handle_tpi(struct kvm_vcpu *vcpu) 481 { 482 struct kvm_s390_interrupt_info *inti; 483 unsigned long len; 484 u32 tpi_data[3]; 485 int rc; 486 u64 addr; 487 u8 ar; 488 489 vcpu->stat.instruction_tpi++; 490 491 addr = kvm_s390_get_base_disp_s(vcpu, &ar); 492 if (addr & 3) 493 return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION); 494 495 inti = kvm_s390_get_io_int(vcpu->kvm, vcpu->arch.sie_block->gcr[6], 0); 496 if (!inti) { 497 kvm_s390_set_psw_cc(vcpu, 0); 498 return 0; 499 } 500 501 tpi_data[0] = inti->io.subchannel_id << 16 | inti->io.subchannel_nr; 502 tpi_data[1] = inti->io.io_int_parm; 503 tpi_data[2] = inti->io.io_int_word; 504 if (addr) { 505 /* 506 * Store the two-word I/O interruption code into the 507 * provided area. 508 */ 509 len = sizeof(tpi_data) - 4; 510 rc = write_guest(vcpu, addr, ar, &tpi_data, len); 511 if (rc) { 512 rc = kvm_s390_inject_prog_cond(vcpu, rc); 513 goto reinject_interrupt; 514 } 515 } else { 516 /* 517 * Store the three-word I/O interruption code into 518 * the appropriate lowcore area. 519 */ 520 len = sizeof(tpi_data); 521 if (write_guest_lc(vcpu, __LC_SUBCHANNEL_ID, &tpi_data, len)) { 522 /* failed writes to the low core are not recoverable */ 523 rc = -EFAULT; 524 goto reinject_interrupt; 525 } 526 } 527 528 /* irq was successfully handed to the guest */ 529 kfree(inti); 530 kvm_s390_set_psw_cc(vcpu, 1); 531 return 0; 532 reinject_interrupt: 533 /* 534 * If we encounter a problem storing the interruption code, the 535 * instruction is suppressed from the guest's view: reinject the 536 * interrupt. 537 */ 538 if (kvm_s390_reinject_io_int(vcpu->kvm, inti)) { 539 kfree(inti); 540 rc = -EFAULT; 541 } 542 /* don't set the cc, a pgm irq was injected or we drop to user space */ 543 return rc ? -EFAULT : 0; 544 } 545 546 static int handle_tsch(struct kvm_vcpu *vcpu) 547 { 548 struct kvm_s390_interrupt_info *inti = NULL; 549 const u64 isc_mask = 0xffUL << 24; /* all iscs set */ 550 551 vcpu->stat.instruction_tsch++; 552 553 /* a valid schid has at least one bit set */ 554 if (vcpu->run->s.regs.gprs[1]) 555 inti = kvm_s390_get_io_int(vcpu->kvm, isc_mask, 556 vcpu->run->s.regs.gprs[1]); 557 558 /* 559 * Prepare exit to userspace. 560 * We indicate whether we dequeued a pending I/O interrupt 561 * so that userspace can re-inject it if the instruction gets 562 * a program check. While this may re-order the pending I/O 563 * interrupts, this is no problem since the priority is kept 564 * intact. 565 */ 566 vcpu->run->exit_reason = KVM_EXIT_S390_TSCH; 567 vcpu->run->s390_tsch.dequeued = !!inti; 568 if (inti) { 569 vcpu->run->s390_tsch.subchannel_id = inti->io.subchannel_id; 570 vcpu->run->s390_tsch.subchannel_nr = inti->io.subchannel_nr; 571 vcpu->run->s390_tsch.io_int_parm = inti->io.io_int_parm; 572 vcpu->run->s390_tsch.io_int_word = inti->io.io_int_word; 573 } 574 vcpu->run->s390_tsch.ipb = vcpu->arch.sie_block->ipb; 575 kfree(inti); 576 return -EREMOTE; 577 } 578 579 static int handle_io_inst(struct kvm_vcpu *vcpu) 580 { 581 VCPU_EVENT(vcpu, 4, "%s", "I/O instruction"); 582 583 if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE) 584 return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP); 585 586 if (vcpu->kvm->arch.css_support) { 587 /* 588 * Most I/O instructions will be handled by userspace. 589 * Exceptions are tpi and the interrupt portion of tsch. 590 */ 591 if (vcpu->arch.sie_block->ipa == 0xb236) 592 return handle_tpi(vcpu); 593 if (vcpu->arch.sie_block->ipa == 0xb235) 594 return handle_tsch(vcpu); 595 /* Handle in userspace. */ 596 vcpu->stat.instruction_io_other++; 597 return -EOPNOTSUPP; 598 } else { 599 /* 600 * Set condition code 3 to stop the guest from issuing channel 601 * I/O instructions. 602 */ 603 kvm_s390_set_psw_cc(vcpu, 3); 604 return 0; 605 } 606 } 607 608 /* 609 * handle_pqap: Handling pqap interception 610 * @vcpu: the vcpu having issue the pqap instruction 611 * 612 * We now support PQAP/AQIC instructions and we need to correctly 613 * answer the guest even if no dedicated driver's hook is available. 614 * 615 * The intercepting code calls a dedicated callback for this instruction 616 * if a driver did register one in the CRYPTO satellite of the 617 * SIE block. 618 * 619 * If no callback is available, the queues are not available, return this 620 * response code to the caller and set CC to 3. 621 * Else return the response code returned by the callback. 622 */ 623 static int handle_pqap(struct kvm_vcpu *vcpu) 624 { 625 struct ap_queue_status status = {}; 626 crypto_hook pqap_hook; 627 unsigned long reg0; 628 int ret; 629 uint8_t fc; 630 631 /* Verify that the AP instruction are available */ 632 if (!ap_instructions_available()) 633 return -EOPNOTSUPP; 634 /* Verify that the guest is allowed to use AP instructions */ 635 if (!(vcpu->arch.sie_block->eca & ECA_APIE)) 636 return -EOPNOTSUPP; 637 /* 638 * The only possibly intercepted functions when AP instructions are 639 * available for the guest are AQIC and TAPQ with the t bit set 640 * since we do not set IC.3 (FIII) we currently will only intercept 641 * the AQIC function code. 642 * Note: running nested under z/VM can result in intercepts for other 643 * function codes, e.g. PQAP(QCI). We do not support this and bail out. 644 */ 645 reg0 = vcpu->run->s.regs.gprs[0]; 646 fc = (reg0 >> 24) & 0xff; 647 if (fc != 0x03) 648 return -EOPNOTSUPP; 649 650 /* PQAP instruction is allowed for guest kernel only */ 651 if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE) 652 return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP); 653 654 /* Common PQAP instruction specification exceptions */ 655 /* bits 41-47 must all be zeros */ 656 if (reg0 & 0x007f0000UL) 657 return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION); 658 /* APFT not install and T bit set */ 659 if (!test_kvm_facility(vcpu->kvm, 15) && (reg0 & 0x00800000UL)) 660 return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION); 661 /* APXA not installed and APID greater 64 or APQI greater 16 */ 662 if (!(vcpu->kvm->arch.crypto.crycbd & 0x02) && (reg0 & 0x0000c0f0UL)) 663 return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION); 664 665 /* AQIC function code specific exception */ 666 /* facility 65 not present for AQIC function code */ 667 if (!test_kvm_facility(vcpu->kvm, 65)) 668 return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION); 669 670 /* 671 * If the hook callback is registered, there will be a pointer to the 672 * hook function pointer in the kvm_s390_crypto structure. Lock the 673 * owner, retrieve the hook function pointer and call the hook. 674 */ 675 down_read(&vcpu->kvm->arch.crypto.pqap_hook_rwsem); 676 if (vcpu->kvm->arch.crypto.pqap_hook) { 677 pqap_hook = *vcpu->kvm->arch.crypto.pqap_hook; 678 ret = pqap_hook(vcpu); 679 if (!ret && vcpu->run->s.regs.gprs[1] & 0x00ff0000) 680 kvm_s390_set_psw_cc(vcpu, 3); 681 up_read(&vcpu->kvm->arch.crypto.pqap_hook_rwsem); 682 return ret; 683 } 684 up_read(&vcpu->kvm->arch.crypto.pqap_hook_rwsem); 685 /* 686 * A vfio_driver must register a hook. 687 * No hook means no driver to enable the SIE CRYCB and no queues. 688 * We send this response to the guest. 689 */ 690 status.response_code = 0x01; 691 memcpy(&vcpu->run->s.regs.gprs[1], &status, sizeof(status)); 692 kvm_s390_set_psw_cc(vcpu, 3); 693 return 0; 694 } 695 696 static int handle_stfl(struct kvm_vcpu *vcpu) 697 { 698 int rc; 699 unsigned int fac; 700 701 vcpu->stat.instruction_stfl++; 702 703 if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE) 704 return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP); 705 706 /* 707 * We need to shift the lower 32 facility bits (bit 0-31) from a u64 708 * into a u32 memory representation. They will remain bits 0-31. 709 */ 710 fac = *vcpu->kvm->arch.model.fac_list >> 32; 711 rc = write_guest_lc(vcpu, offsetof(struct lowcore, stfl_fac_list), 712 &fac, sizeof(fac)); 713 if (rc) 714 return rc; 715 VCPU_EVENT(vcpu, 3, "STFL: store facility list 0x%x", fac); 716 trace_kvm_s390_handle_stfl(vcpu, fac); 717 return 0; 718 } 719 720 #define PSW_MASK_ADDR_MODE (PSW_MASK_EA | PSW_MASK_BA) 721 #define PSW_MASK_UNASSIGNED 0xb80800fe7fffffffUL 722 #define PSW_ADDR_24 0x0000000000ffffffUL 723 #define PSW_ADDR_31 0x000000007fffffffUL 724 725 int is_valid_psw(psw_t *psw) 726 { 727 if (psw->mask & PSW_MASK_UNASSIGNED) 728 return 0; 729 if ((psw->mask & PSW_MASK_ADDR_MODE) == PSW_MASK_BA) { 730 if (psw->addr & ~PSW_ADDR_31) 731 return 0; 732 } 733 if (!(psw->mask & PSW_MASK_ADDR_MODE) && (psw->addr & ~PSW_ADDR_24)) 734 return 0; 735 if ((psw->mask & PSW_MASK_ADDR_MODE) == PSW_MASK_EA) 736 return 0; 737 if (psw->addr & 1) 738 return 0; 739 return 1; 740 } 741 742 int kvm_s390_handle_lpsw(struct kvm_vcpu *vcpu) 743 { 744 psw_t *gpsw = &vcpu->arch.sie_block->gpsw; 745 psw_compat_t new_psw; 746 u64 addr; 747 int rc; 748 u8 ar; 749 750 vcpu->stat.instruction_lpsw++; 751 752 if (gpsw->mask & PSW_MASK_PSTATE) 753 return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP); 754 755 addr = kvm_s390_get_base_disp_s(vcpu, &ar); 756 if (addr & 7) 757 return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION); 758 759 rc = read_guest(vcpu, addr, ar, &new_psw, sizeof(new_psw)); 760 if (rc) 761 return kvm_s390_inject_prog_cond(vcpu, rc); 762 if (!(new_psw.mask & PSW32_MASK_BASE)) 763 return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION); 764 gpsw->mask = (new_psw.mask & ~PSW32_MASK_BASE) << 32; 765 gpsw->mask |= new_psw.addr & PSW32_ADDR_AMODE; 766 gpsw->addr = new_psw.addr & ~PSW32_ADDR_AMODE; 767 if (!is_valid_psw(gpsw)) 768 return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION); 769 return 0; 770 } 771 772 static int handle_lpswe(struct kvm_vcpu *vcpu) 773 { 774 psw_t new_psw; 775 u64 addr; 776 int rc; 777 u8 ar; 778 779 vcpu->stat.instruction_lpswe++; 780 781 if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE) 782 return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP); 783 784 addr = kvm_s390_get_base_disp_s(vcpu, &ar); 785 if (addr & 7) 786 return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION); 787 rc = read_guest(vcpu, addr, ar, &new_psw, sizeof(new_psw)); 788 if (rc) 789 return kvm_s390_inject_prog_cond(vcpu, rc); 790 vcpu->arch.sie_block->gpsw = new_psw; 791 if (!is_valid_psw(&vcpu->arch.sie_block->gpsw)) 792 return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION); 793 return 0; 794 } 795 796 static int handle_stidp(struct kvm_vcpu *vcpu) 797 { 798 u64 stidp_data = vcpu->kvm->arch.model.cpuid; 799 u64 operand2; 800 int rc; 801 u8 ar; 802 803 vcpu->stat.instruction_stidp++; 804 805 if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE) 806 return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP); 807 808 operand2 = kvm_s390_get_base_disp_s(vcpu, &ar); 809 810 if (operand2 & 7) 811 return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION); 812 813 rc = write_guest(vcpu, operand2, ar, &stidp_data, sizeof(stidp_data)); 814 if (rc) 815 return kvm_s390_inject_prog_cond(vcpu, rc); 816 817 VCPU_EVENT(vcpu, 3, "STIDP: store cpu id 0x%llx", stidp_data); 818 return 0; 819 } 820 821 static void handle_stsi_3_2_2(struct kvm_vcpu *vcpu, struct sysinfo_3_2_2 *mem) 822 { 823 int cpus = 0; 824 int n; 825 826 cpus = atomic_read(&vcpu->kvm->online_vcpus); 827 828 /* deal with other level 3 hypervisors */ 829 if (stsi(mem, 3, 2, 2)) 830 mem->count = 0; 831 if (mem->count < 8) 832 mem->count++; 833 for (n = mem->count - 1; n > 0 ; n--) 834 memcpy(&mem->vm[n], &mem->vm[n - 1], sizeof(mem->vm[0])); 835 836 memset(&mem->vm[0], 0, sizeof(mem->vm[0])); 837 mem->vm[0].cpus_total = cpus; 838 mem->vm[0].cpus_configured = cpus; 839 mem->vm[0].cpus_standby = 0; 840 mem->vm[0].cpus_reserved = 0; 841 mem->vm[0].caf = 1000; 842 memcpy(mem->vm[0].name, "KVMguest", 8); 843 ASCEBC(mem->vm[0].name, 8); 844 memcpy(mem->vm[0].cpi, "KVM/Linux ", 16); 845 ASCEBC(mem->vm[0].cpi, 16); 846 } 847 848 static void insert_stsi_usr_data(struct kvm_vcpu *vcpu, u64 addr, u8 ar, 849 u8 fc, u8 sel1, u16 sel2) 850 { 851 vcpu->run->exit_reason = KVM_EXIT_S390_STSI; 852 vcpu->run->s390_stsi.addr = addr; 853 vcpu->run->s390_stsi.ar = ar; 854 vcpu->run->s390_stsi.fc = fc; 855 vcpu->run->s390_stsi.sel1 = sel1; 856 vcpu->run->s390_stsi.sel2 = sel2; 857 } 858 859 static int handle_stsi(struct kvm_vcpu *vcpu) 860 { 861 int fc = (vcpu->run->s.regs.gprs[0] & 0xf0000000) >> 28; 862 int sel1 = vcpu->run->s.regs.gprs[0] & 0xff; 863 int sel2 = vcpu->run->s.regs.gprs[1] & 0xffff; 864 unsigned long mem = 0; 865 u64 operand2; 866 int rc = 0; 867 u8 ar; 868 869 vcpu->stat.instruction_stsi++; 870 VCPU_EVENT(vcpu, 3, "STSI: fc: %u sel1: %u sel2: %u", fc, sel1, sel2); 871 872 if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE) 873 return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP); 874 875 /* Bailout forbidden function codes */ 876 if (fc > 3 && fc != 15) 877 goto out_no_data; 878 879 /* 880 * fc 15 is provided only with 881 * - PTF/CPU topology support through facility 15 882 * - KVM_CAP_S390_USER_STSI 883 */ 884 if (fc == 15 && (!test_kvm_facility(vcpu->kvm, 11) || 885 !vcpu->kvm->arch.user_stsi)) 886 goto out_no_data; 887 888 if (vcpu->run->s.regs.gprs[0] & 0x0fffff00 889 || vcpu->run->s.regs.gprs[1] & 0xffff0000) 890 return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION); 891 892 if (fc == 0) { 893 vcpu->run->s.regs.gprs[0] = 3 << 28; 894 kvm_s390_set_psw_cc(vcpu, 0); 895 return 0; 896 } 897 898 operand2 = kvm_s390_get_base_disp_s(vcpu, &ar); 899 900 if (!kvm_s390_pv_cpu_is_protected(vcpu) && (operand2 & 0xfff)) 901 return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION); 902 903 switch (fc) { 904 case 1: /* same handling for 1 and 2 */ 905 case 2: 906 mem = get_zeroed_page(GFP_KERNEL_ACCOUNT); 907 if (!mem) 908 goto out_no_data; 909 if (stsi((void *) mem, fc, sel1, sel2)) 910 goto out_no_data; 911 break; 912 case 3: 913 if (sel1 != 2 || sel2 != 2) 914 goto out_no_data; 915 mem = get_zeroed_page(GFP_KERNEL_ACCOUNT); 916 if (!mem) 917 goto out_no_data; 918 handle_stsi_3_2_2(vcpu, (void *) mem); 919 break; 920 case 15: /* fc 15 is fully handled in userspace */ 921 insert_stsi_usr_data(vcpu, operand2, ar, fc, sel1, sel2); 922 trace_kvm_s390_handle_stsi(vcpu, fc, sel1, sel2, operand2); 923 return -EREMOTE; 924 } 925 if (kvm_s390_pv_cpu_is_protected(vcpu)) { 926 memcpy(sida_addr(vcpu->arch.sie_block), (void *)mem, PAGE_SIZE); 927 rc = 0; 928 } else { 929 rc = write_guest(vcpu, operand2, ar, (void *)mem, PAGE_SIZE); 930 } 931 if (rc) { 932 rc = kvm_s390_inject_prog_cond(vcpu, rc); 933 goto out; 934 } 935 if (vcpu->kvm->arch.user_stsi) { 936 insert_stsi_usr_data(vcpu, operand2, ar, fc, sel1, sel2); 937 rc = -EREMOTE; 938 } 939 trace_kvm_s390_handle_stsi(vcpu, fc, sel1, sel2, operand2); 940 free_page(mem); 941 kvm_s390_set_psw_cc(vcpu, 0); 942 vcpu->run->s.regs.gprs[0] = 0; 943 return rc; 944 out_no_data: 945 kvm_s390_set_psw_cc(vcpu, 3); 946 out: 947 free_page(mem); 948 return rc; 949 } 950 951 int kvm_s390_handle_b2(struct kvm_vcpu *vcpu) 952 { 953 switch (vcpu->arch.sie_block->ipa & 0x00ff) { 954 case 0x02: 955 return handle_stidp(vcpu); 956 case 0x04: 957 return handle_set_clock(vcpu); 958 case 0x10: 959 return handle_set_prefix(vcpu); 960 case 0x11: 961 return handle_store_prefix(vcpu); 962 case 0x12: 963 return handle_store_cpu_address(vcpu); 964 case 0x14: 965 return kvm_s390_handle_vsie(vcpu); 966 case 0x21: 967 case 0x50: 968 return handle_ipte_interlock(vcpu); 969 case 0x29: 970 return handle_iske(vcpu); 971 case 0x2a: 972 return handle_rrbe(vcpu); 973 case 0x2b: 974 return handle_sske(vcpu); 975 case 0x2c: 976 return handle_test_block(vcpu); 977 case 0x30: 978 case 0x31: 979 case 0x32: 980 case 0x33: 981 case 0x34: 982 case 0x35: 983 case 0x36: 984 case 0x37: 985 case 0x38: 986 case 0x39: 987 case 0x3a: 988 case 0x3b: 989 case 0x3c: 990 case 0x5f: 991 case 0x74: 992 case 0x76: 993 return handle_io_inst(vcpu); 994 case 0x56: 995 return handle_sthyi(vcpu); 996 case 0x7d: 997 return handle_stsi(vcpu); 998 case 0xaf: 999 return handle_pqap(vcpu); 1000 case 0xb1: 1001 return handle_stfl(vcpu); 1002 case 0xb2: 1003 return handle_lpswe(vcpu); 1004 default: 1005 return -EOPNOTSUPP; 1006 } 1007 } 1008 1009 static int handle_epsw(struct kvm_vcpu *vcpu) 1010 { 1011 int reg1, reg2; 1012 1013 vcpu->stat.instruction_epsw++; 1014 1015 kvm_s390_get_regs_rre(vcpu, ®1, ®2); 1016 1017 /* This basically extracts the mask half of the psw. */ 1018 vcpu->run->s.regs.gprs[reg1] &= 0xffffffff00000000UL; 1019 vcpu->run->s.regs.gprs[reg1] |= vcpu->arch.sie_block->gpsw.mask >> 32; 1020 if (reg2) { 1021 vcpu->run->s.regs.gprs[reg2] &= 0xffffffff00000000UL; 1022 vcpu->run->s.regs.gprs[reg2] |= 1023 vcpu->arch.sie_block->gpsw.mask & 0x00000000ffffffffUL; 1024 } 1025 return 0; 1026 } 1027 1028 #define PFMF_RESERVED 0xfffc0101UL 1029 #define PFMF_SK 0x00020000UL 1030 #define PFMF_CF 0x00010000UL 1031 #define PFMF_UI 0x00008000UL 1032 #define PFMF_FSC 0x00007000UL 1033 #define PFMF_NQ 0x00000800UL 1034 #define PFMF_MR 0x00000400UL 1035 #define PFMF_MC 0x00000200UL 1036 #define PFMF_KEY 0x000000feUL 1037 1038 static int handle_pfmf(struct kvm_vcpu *vcpu) 1039 { 1040 bool mr = false, mc = false, nq; 1041 int reg1, reg2; 1042 unsigned long start, end; 1043 unsigned char key; 1044 1045 vcpu->stat.instruction_pfmf++; 1046 1047 kvm_s390_get_regs_rre(vcpu, ®1, ®2); 1048 1049 if (!test_kvm_facility(vcpu->kvm, 8)) 1050 return kvm_s390_inject_program_int(vcpu, PGM_OPERATION); 1051 1052 if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE) 1053 return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP); 1054 1055 if (vcpu->run->s.regs.gprs[reg1] & PFMF_RESERVED) 1056 return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION); 1057 1058 /* Only provide non-quiescing support if enabled for the guest */ 1059 if (vcpu->run->s.regs.gprs[reg1] & PFMF_NQ && 1060 !test_kvm_facility(vcpu->kvm, 14)) 1061 return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION); 1062 1063 /* Only provide conditional-SSKE support if enabled for the guest */ 1064 if (vcpu->run->s.regs.gprs[reg1] & PFMF_SK && 1065 test_kvm_facility(vcpu->kvm, 10)) { 1066 mr = vcpu->run->s.regs.gprs[reg1] & PFMF_MR; 1067 mc = vcpu->run->s.regs.gprs[reg1] & PFMF_MC; 1068 } 1069 1070 nq = vcpu->run->s.regs.gprs[reg1] & PFMF_NQ; 1071 key = vcpu->run->s.regs.gprs[reg1] & PFMF_KEY; 1072 start = vcpu->run->s.regs.gprs[reg2] & PAGE_MASK; 1073 start = kvm_s390_logical_to_effective(vcpu, start); 1074 1075 if (vcpu->run->s.regs.gprs[reg1] & PFMF_CF) { 1076 if (kvm_s390_check_low_addr_prot_real(vcpu, start)) 1077 return kvm_s390_inject_prog_irq(vcpu, &vcpu->arch.pgm); 1078 } 1079 1080 switch (vcpu->run->s.regs.gprs[reg1] & PFMF_FSC) { 1081 case 0x00000000: 1082 /* only 4k frames specify a real address */ 1083 start = kvm_s390_real_to_abs(vcpu, start); 1084 end = (start + PAGE_SIZE) & ~(PAGE_SIZE - 1); 1085 break; 1086 case 0x00001000: 1087 end = (start + _SEGMENT_SIZE) & ~(_SEGMENT_SIZE - 1); 1088 break; 1089 case 0x00002000: 1090 /* only support 2G frame size if EDAT2 is available and we are 1091 not in 24-bit addressing mode */ 1092 if (!test_kvm_facility(vcpu->kvm, 78) || 1093 psw_bits(vcpu->arch.sie_block->gpsw).eaba == PSW_BITS_AMODE_24BIT) 1094 return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION); 1095 end = (start + _REGION3_SIZE) & ~(_REGION3_SIZE - 1); 1096 break; 1097 default: 1098 return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION); 1099 } 1100 1101 while (start != end) { 1102 unsigned long vmaddr; 1103 bool unlocked = false; 1104 1105 /* Translate guest address to host address */ 1106 vmaddr = gfn_to_hva(vcpu->kvm, gpa_to_gfn(start)); 1107 if (kvm_is_error_hva(vmaddr)) 1108 return kvm_s390_inject_program_int(vcpu, PGM_ADDRESSING); 1109 1110 if (vcpu->run->s.regs.gprs[reg1] & PFMF_CF) { 1111 if (kvm_clear_guest(vcpu->kvm, start, PAGE_SIZE)) 1112 return kvm_s390_inject_program_int(vcpu, PGM_ADDRESSING); 1113 } 1114 1115 if (vcpu->run->s.regs.gprs[reg1] & PFMF_SK) { 1116 int rc = kvm_s390_skey_check_enable(vcpu); 1117 1118 if (rc) 1119 return rc; 1120 mmap_read_lock(current->mm); 1121 rc = cond_set_guest_storage_key(current->mm, vmaddr, 1122 key, NULL, nq, mr, mc); 1123 if (rc < 0) { 1124 rc = fixup_user_fault(current->mm, vmaddr, 1125 FAULT_FLAG_WRITE, &unlocked); 1126 rc = !rc ? -EAGAIN : rc; 1127 } 1128 mmap_read_unlock(current->mm); 1129 if (rc == -EFAULT) 1130 return kvm_s390_inject_program_int(vcpu, PGM_ADDRESSING); 1131 if (rc == -EAGAIN) 1132 continue; 1133 if (rc < 0) 1134 return rc; 1135 } 1136 start += PAGE_SIZE; 1137 } 1138 if (vcpu->run->s.regs.gprs[reg1] & PFMF_FSC) { 1139 if (psw_bits(vcpu->arch.sie_block->gpsw).eaba == PSW_BITS_AMODE_64BIT) { 1140 vcpu->run->s.regs.gprs[reg2] = end; 1141 } else { 1142 vcpu->run->s.regs.gprs[reg2] &= ~0xffffffffUL; 1143 end = kvm_s390_logical_to_effective(vcpu, end); 1144 vcpu->run->s.regs.gprs[reg2] |= end; 1145 } 1146 } 1147 return 0; 1148 } 1149 1150 /* 1151 * Must be called with relevant read locks held (kvm->mm->mmap_lock, kvm->srcu) 1152 */ 1153 static inline int __do_essa(struct kvm_vcpu *vcpu, const int orc) 1154 { 1155 int r1, r2, nappended, entries; 1156 unsigned long gfn, hva, res, pgstev, ptev; 1157 unsigned long *cbrlo; 1158 1159 /* 1160 * We don't need to set SD.FPF.SK to 1 here, because if we have a 1161 * machine check here we either handle it or crash 1162 */ 1163 1164 kvm_s390_get_regs_rre(vcpu, &r1, &r2); 1165 gfn = vcpu->run->s.regs.gprs[r2] >> PAGE_SHIFT; 1166 hva = gfn_to_hva(vcpu->kvm, gfn); 1167 entries = (vcpu->arch.sie_block->cbrlo & ~PAGE_MASK) >> 3; 1168 1169 if (kvm_is_error_hva(hva)) 1170 return kvm_s390_inject_program_int(vcpu, PGM_ADDRESSING); 1171 1172 nappended = pgste_perform_essa(vcpu->kvm->mm, hva, orc, &ptev, &pgstev); 1173 if (nappended < 0) { 1174 res = orc ? 0x10 : 0; 1175 vcpu->run->s.regs.gprs[r1] = res; /* Exception Indication */ 1176 return 0; 1177 } 1178 res = (pgstev & _PGSTE_GPS_USAGE_MASK) >> 22; 1179 /* 1180 * Set the block-content state part of the result. 0 means resident, so 1181 * nothing to do if the page is valid. 2 is for preserved pages 1182 * (non-present and non-zero), and 3 for zero pages (non-present and 1183 * zero). 1184 */ 1185 if (ptev & _PAGE_INVALID) { 1186 res |= 2; 1187 if (pgstev & _PGSTE_GPS_ZERO) 1188 res |= 1; 1189 } 1190 if (pgstev & _PGSTE_GPS_NODAT) 1191 res |= 0x20; 1192 vcpu->run->s.regs.gprs[r1] = res; 1193 /* 1194 * It is possible that all the normal 511 slots were full, in which case 1195 * we will now write in the 512th slot, which is reserved for host use. 1196 * In both cases we let the normal essa handling code process all the 1197 * slots, including the reserved one, if needed. 1198 */ 1199 if (nappended > 0) { 1200 cbrlo = phys_to_virt(vcpu->arch.sie_block->cbrlo & PAGE_MASK); 1201 cbrlo[entries] = gfn << PAGE_SHIFT; 1202 } 1203 1204 if (orc) { 1205 struct kvm_memory_slot *ms = gfn_to_memslot(vcpu->kvm, gfn); 1206 1207 /* Increment only if we are really flipping the bit */ 1208 if (ms && !test_and_set_bit(gfn - ms->base_gfn, kvm_second_dirty_bitmap(ms))) 1209 atomic64_inc(&vcpu->kvm->arch.cmma_dirty_pages); 1210 } 1211 1212 return nappended; 1213 } 1214 1215 static int handle_essa(struct kvm_vcpu *vcpu) 1216 { 1217 /* entries expected to be 1FF */ 1218 int entries = (vcpu->arch.sie_block->cbrlo & ~PAGE_MASK) >> 3; 1219 unsigned long *cbrlo; 1220 struct gmap *gmap; 1221 int i, orc; 1222 1223 VCPU_EVENT(vcpu, 4, "ESSA: release %d pages", entries); 1224 gmap = vcpu->arch.gmap; 1225 vcpu->stat.instruction_essa++; 1226 if (!vcpu->kvm->arch.use_cmma) 1227 return kvm_s390_inject_program_int(vcpu, PGM_OPERATION); 1228 1229 if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE) 1230 return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP); 1231 /* Check for invalid operation request code */ 1232 orc = (vcpu->arch.sie_block->ipb & 0xf0000000) >> 28; 1233 /* ORCs 0-6 are always valid */ 1234 if (orc > (test_kvm_facility(vcpu->kvm, 147) ? ESSA_SET_STABLE_NODAT 1235 : ESSA_SET_STABLE_IF_RESIDENT)) 1236 return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION); 1237 1238 if (!vcpu->kvm->arch.migration_mode) { 1239 /* 1240 * CMMA is enabled in the KVM settings, but is disabled in 1241 * the SIE block and in the mm_context, and we are not doing 1242 * a migration. Enable CMMA in the mm_context. 1243 * Since we need to take a write lock to write to the context 1244 * to avoid races with storage keys handling, we check if the 1245 * value really needs to be written to; if the value is 1246 * already correct, we do nothing and avoid the lock. 1247 */ 1248 if (vcpu->kvm->mm->context.uses_cmm == 0) { 1249 mmap_write_lock(vcpu->kvm->mm); 1250 vcpu->kvm->mm->context.uses_cmm = 1; 1251 mmap_write_unlock(vcpu->kvm->mm); 1252 } 1253 /* 1254 * If we are here, we are supposed to have CMMA enabled in 1255 * the SIE block. Enabling CMMA works on a per-CPU basis, 1256 * while the context use_cmma flag is per process. 1257 * It's possible that the context flag is enabled and the 1258 * SIE flag is not, so we set the flag always; if it was 1259 * already set, nothing changes, otherwise we enable it 1260 * on this CPU too. 1261 */ 1262 vcpu->arch.sie_block->ecb2 |= ECB2_CMMA; 1263 /* Retry the ESSA instruction */ 1264 kvm_s390_retry_instr(vcpu); 1265 } else { 1266 int srcu_idx; 1267 1268 mmap_read_lock(vcpu->kvm->mm); 1269 srcu_idx = srcu_read_lock(&vcpu->kvm->srcu); 1270 i = __do_essa(vcpu, orc); 1271 srcu_read_unlock(&vcpu->kvm->srcu, srcu_idx); 1272 mmap_read_unlock(vcpu->kvm->mm); 1273 if (i < 0) 1274 return i; 1275 /* Account for the possible extra cbrl entry */ 1276 entries += i; 1277 } 1278 vcpu->arch.sie_block->cbrlo &= PAGE_MASK; /* reset nceo */ 1279 cbrlo = phys_to_virt(vcpu->arch.sie_block->cbrlo); 1280 mmap_read_lock(gmap->mm); 1281 for (i = 0; i < entries; ++i) 1282 __gmap_zap(gmap, cbrlo[i]); 1283 mmap_read_unlock(gmap->mm); 1284 return 0; 1285 } 1286 1287 int kvm_s390_handle_b9(struct kvm_vcpu *vcpu) 1288 { 1289 switch (vcpu->arch.sie_block->ipa & 0x00ff) { 1290 case 0x8a: 1291 case 0x8e: 1292 case 0x8f: 1293 return handle_ipte_interlock(vcpu); 1294 case 0x8d: 1295 return handle_epsw(vcpu); 1296 case 0xab: 1297 return handle_essa(vcpu); 1298 case 0xaf: 1299 return handle_pfmf(vcpu); 1300 default: 1301 return -EOPNOTSUPP; 1302 } 1303 } 1304 1305 int kvm_s390_handle_lctl(struct kvm_vcpu *vcpu) 1306 { 1307 int reg1 = (vcpu->arch.sie_block->ipa & 0x00f0) >> 4; 1308 int reg3 = vcpu->arch.sie_block->ipa & 0x000f; 1309 int reg, rc, nr_regs; 1310 u32 ctl_array[16]; 1311 u64 ga; 1312 u8 ar; 1313 1314 vcpu->stat.instruction_lctl++; 1315 1316 if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE) 1317 return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP); 1318 1319 ga = kvm_s390_get_base_disp_rs(vcpu, &ar); 1320 1321 if (ga & 3) 1322 return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION); 1323 1324 VCPU_EVENT(vcpu, 4, "LCTL: r1:%d, r3:%d, addr: 0x%llx", reg1, reg3, ga); 1325 trace_kvm_s390_handle_lctl(vcpu, 0, reg1, reg3, ga); 1326 1327 nr_regs = ((reg3 - reg1) & 0xf) + 1; 1328 rc = read_guest(vcpu, ga, ar, ctl_array, nr_regs * sizeof(u32)); 1329 if (rc) 1330 return kvm_s390_inject_prog_cond(vcpu, rc); 1331 reg = reg1; 1332 nr_regs = 0; 1333 do { 1334 vcpu->arch.sie_block->gcr[reg] &= 0xffffffff00000000ul; 1335 vcpu->arch.sie_block->gcr[reg] |= ctl_array[nr_regs++]; 1336 if (reg == reg3) 1337 break; 1338 reg = (reg + 1) % 16; 1339 } while (1); 1340 kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu); 1341 return 0; 1342 } 1343 1344 int kvm_s390_handle_stctl(struct kvm_vcpu *vcpu) 1345 { 1346 int reg1 = (vcpu->arch.sie_block->ipa & 0x00f0) >> 4; 1347 int reg3 = vcpu->arch.sie_block->ipa & 0x000f; 1348 int reg, rc, nr_regs; 1349 u32 ctl_array[16]; 1350 u64 ga; 1351 u8 ar; 1352 1353 vcpu->stat.instruction_stctl++; 1354 1355 if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE) 1356 return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP); 1357 1358 ga = kvm_s390_get_base_disp_rs(vcpu, &ar); 1359 1360 if (ga & 3) 1361 return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION); 1362 1363 VCPU_EVENT(vcpu, 4, "STCTL r1:%d, r3:%d, addr: 0x%llx", reg1, reg3, ga); 1364 trace_kvm_s390_handle_stctl(vcpu, 0, reg1, reg3, ga); 1365 1366 reg = reg1; 1367 nr_regs = 0; 1368 do { 1369 ctl_array[nr_regs++] = vcpu->arch.sie_block->gcr[reg]; 1370 if (reg == reg3) 1371 break; 1372 reg = (reg + 1) % 16; 1373 } while (1); 1374 rc = write_guest(vcpu, ga, ar, ctl_array, nr_regs * sizeof(u32)); 1375 return rc ? kvm_s390_inject_prog_cond(vcpu, rc) : 0; 1376 } 1377 1378 static int handle_lctlg(struct kvm_vcpu *vcpu) 1379 { 1380 int reg1 = (vcpu->arch.sie_block->ipa & 0x00f0) >> 4; 1381 int reg3 = vcpu->arch.sie_block->ipa & 0x000f; 1382 int reg, rc, nr_regs; 1383 u64 ctl_array[16]; 1384 u64 ga; 1385 u8 ar; 1386 1387 vcpu->stat.instruction_lctlg++; 1388 1389 if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE) 1390 return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP); 1391 1392 ga = kvm_s390_get_base_disp_rsy(vcpu, &ar); 1393 1394 if (ga & 7) 1395 return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION); 1396 1397 VCPU_EVENT(vcpu, 4, "LCTLG: r1:%d, r3:%d, addr: 0x%llx", reg1, reg3, ga); 1398 trace_kvm_s390_handle_lctl(vcpu, 1, reg1, reg3, ga); 1399 1400 nr_regs = ((reg3 - reg1) & 0xf) + 1; 1401 rc = read_guest(vcpu, ga, ar, ctl_array, nr_regs * sizeof(u64)); 1402 if (rc) 1403 return kvm_s390_inject_prog_cond(vcpu, rc); 1404 reg = reg1; 1405 nr_regs = 0; 1406 do { 1407 vcpu->arch.sie_block->gcr[reg] = ctl_array[nr_regs++]; 1408 if (reg == reg3) 1409 break; 1410 reg = (reg + 1) % 16; 1411 } while (1); 1412 kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu); 1413 return 0; 1414 } 1415 1416 static int handle_stctg(struct kvm_vcpu *vcpu) 1417 { 1418 int reg1 = (vcpu->arch.sie_block->ipa & 0x00f0) >> 4; 1419 int reg3 = vcpu->arch.sie_block->ipa & 0x000f; 1420 int reg, rc, nr_regs; 1421 u64 ctl_array[16]; 1422 u64 ga; 1423 u8 ar; 1424 1425 vcpu->stat.instruction_stctg++; 1426 1427 if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE) 1428 return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP); 1429 1430 ga = kvm_s390_get_base_disp_rsy(vcpu, &ar); 1431 1432 if (ga & 7) 1433 return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION); 1434 1435 VCPU_EVENT(vcpu, 4, "STCTG r1:%d, r3:%d, addr: 0x%llx", reg1, reg3, ga); 1436 trace_kvm_s390_handle_stctl(vcpu, 1, reg1, reg3, ga); 1437 1438 reg = reg1; 1439 nr_regs = 0; 1440 do { 1441 ctl_array[nr_regs++] = vcpu->arch.sie_block->gcr[reg]; 1442 if (reg == reg3) 1443 break; 1444 reg = (reg + 1) % 16; 1445 } while (1); 1446 rc = write_guest(vcpu, ga, ar, ctl_array, nr_regs * sizeof(u64)); 1447 return rc ? kvm_s390_inject_prog_cond(vcpu, rc) : 0; 1448 } 1449 1450 int kvm_s390_handle_eb(struct kvm_vcpu *vcpu) 1451 { 1452 switch (vcpu->arch.sie_block->ipb & 0x000000ff) { 1453 case 0x25: 1454 return handle_stctg(vcpu); 1455 case 0x2f: 1456 return handle_lctlg(vcpu); 1457 case 0x60: 1458 case 0x61: 1459 case 0x62: 1460 return handle_ri(vcpu); 1461 default: 1462 return -EOPNOTSUPP; 1463 } 1464 } 1465 1466 static int handle_tprot(struct kvm_vcpu *vcpu) 1467 { 1468 u64 address, operand2; 1469 unsigned long gpa; 1470 u8 access_key; 1471 bool writable; 1472 int ret, cc; 1473 u8 ar; 1474 1475 vcpu->stat.instruction_tprot++; 1476 1477 if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE) 1478 return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP); 1479 1480 kvm_s390_get_base_disp_sse(vcpu, &address, &operand2, &ar, NULL); 1481 access_key = (operand2 & 0xf0) >> 4; 1482 1483 if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_DAT) 1484 ipte_lock(vcpu->kvm); 1485 1486 ret = guest_translate_address_with_key(vcpu, address, ar, &gpa, 1487 GACC_STORE, access_key); 1488 if (ret == 0) { 1489 gfn_to_hva_prot(vcpu->kvm, gpa_to_gfn(gpa), &writable); 1490 } else if (ret == PGM_PROTECTION) { 1491 writable = false; 1492 /* Write protected? Try again with read-only... */ 1493 ret = guest_translate_address_with_key(vcpu, address, ar, &gpa, 1494 GACC_FETCH, access_key); 1495 } 1496 if (ret >= 0) { 1497 cc = -1; 1498 1499 /* Fetching permitted; storing permitted */ 1500 if (ret == 0 && writable) 1501 cc = 0; 1502 /* Fetching permitted; storing not permitted */ 1503 else if (ret == 0 && !writable) 1504 cc = 1; 1505 /* Fetching not permitted; storing not permitted */ 1506 else if (ret == PGM_PROTECTION) 1507 cc = 2; 1508 /* Translation not available */ 1509 else if (ret != PGM_ADDRESSING && ret != PGM_TRANSLATION_SPEC) 1510 cc = 3; 1511 1512 if (cc != -1) { 1513 kvm_s390_set_psw_cc(vcpu, cc); 1514 ret = 0; 1515 } else { 1516 ret = kvm_s390_inject_program_int(vcpu, ret); 1517 } 1518 } 1519 1520 if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_DAT) 1521 ipte_unlock(vcpu->kvm); 1522 return ret; 1523 } 1524 1525 int kvm_s390_handle_e5(struct kvm_vcpu *vcpu) 1526 { 1527 switch (vcpu->arch.sie_block->ipa & 0x00ff) { 1528 case 0x01: 1529 return handle_tprot(vcpu); 1530 default: 1531 return -EOPNOTSUPP; 1532 } 1533 } 1534 1535 static int handle_sckpf(struct kvm_vcpu *vcpu) 1536 { 1537 u32 value; 1538 1539 vcpu->stat.instruction_sckpf++; 1540 1541 if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE) 1542 return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP); 1543 1544 if (vcpu->run->s.regs.gprs[0] & 0x00000000ffff0000) 1545 return kvm_s390_inject_program_int(vcpu, 1546 PGM_SPECIFICATION); 1547 1548 value = vcpu->run->s.regs.gprs[0] & 0x000000000000ffff; 1549 vcpu->arch.sie_block->todpr = value; 1550 1551 return 0; 1552 } 1553 1554 static int handle_ptff(struct kvm_vcpu *vcpu) 1555 { 1556 vcpu->stat.instruction_ptff++; 1557 1558 /* we don't emulate any control instructions yet */ 1559 kvm_s390_set_psw_cc(vcpu, 3); 1560 return 0; 1561 } 1562 1563 int kvm_s390_handle_01(struct kvm_vcpu *vcpu) 1564 { 1565 switch (vcpu->arch.sie_block->ipa & 0x00ff) { 1566 case 0x04: 1567 return handle_ptff(vcpu); 1568 case 0x07: 1569 return handle_sckpf(vcpu); 1570 default: 1571 return -EOPNOTSUPP; 1572 } 1573 } 1574