1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * handling privileged instructions 4 * 5 * Copyright IBM Corp. 2008, 2020 6 * 7 * Author(s): Carsten Otte <cotte@de.ibm.com> 8 * Christian Borntraeger <borntraeger@de.ibm.com> 9 */ 10 11 #include <linux/kvm.h> 12 #include <linux/gfp.h> 13 #include <linux/errno.h> 14 #include <linux/mm_types.h> 15 #include <linux/pgtable.h> 16 #include <linux/io.h> 17 #include <asm/asm-offsets.h> 18 #include <asm/facility.h> 19 #include <asm/current.h> 20 #include <asm/debug.h> 21 #include <asm/ebcdic.h> 22 #include <asm/sysinfo.h> 23 #include <asm/page-states.h> 24 #include <asm/gmap.h> 25 #include <asm/ptrace.h> 26 #include <asm/sclp.h> 27 #include <asm/ap.h> 28 #include "gaccess.h" 29 #include "kvm-s390.h" 30 #include "trace.h" 31 32 static int handle_ri(struct kvm_vcpu *vcpu) 33 { 34 vcpu->stat.instruction_ri++; 35 36 if (test_kvm_facility(vcpu->kvm, 64)) { 37 VCPU_EVENT(vcpu, 3, "%s", "ENABLE: RI (lazy)"); 38 vcpu->arch.sie_block->ecb3 |= ECB3_RI; 39 kvm_s390_retry_instr(vcpu); 40 return 0; 41 } else 42 return kvm_s390_inject_program_int(vcpu, PGM_OPERATION); 43 } 44 45 int kvm_s390_handle_aa(struct kvm_vcpu *vcpu) 46 { 47 if ((vcpu->arch.sie_block->ipa & 0xf) <= 4) 48 return handle_ri(vcpu); 49 else 50 return -EOPNOTSUPP; 51 } 52 53 static int handle_gs(struct kvm_vcpu *vcpu) 54 { 55 vcpu->stat.instruction_gs++; 56 57 if (test_kvm_facility(vcpu->kvm, 133)) { 58 VCPU_EVENT(vcpu, 3, "%s", "ENABLE: GS (lazy)"); 59 preempt_disable(); 60 local_ctl_set_bit(2, CR2_GUARDED_STORAGE_BIT); 61 current->thread.gs_cb = (struct gs_cb *)&vcpu->run->s.regs.gscb; 62 restore_gs_cb(current->thread.gs_cb); 63 preempt_enable(); 64 vcpu->arch.sie_block->ecb |= ECB_GS; 65 vcpu->arch.sie_block->ecd |= ECD_HOSTREGMGMT; 66 vcpu->arch.gs_enabled = 1; 67 kvm_s390_retry_instr(vcpu); 68 return 0; 69 } else 70 return kvm_s390_inject_program_int(vcpu, PGM_OPERATION); 71 } 72 73 int kvm_s390_handle_e3(struct kvm_vcpu *vcpu) 74 { 75 int code = vcpu->arch.sie_block->ipb & 0xff; 76 77 if (code == 0x49 || code == 0x4d) 78 return handle_gs(vcpu); 79 else 80 return -EOPNOTSUPP; 81 } 82 /* Handle SCK (SET CLOCK) interception */ 83 static int handle_set_clock(struct kvm_vcpu *vcpu) 84 { 85 struct kvm_s390_vm_tod_clock gtod = { 0 }; 86 int rc; 87 u8 ar; 88 u64 op2; 89 90 vcpu->stat.instruction_sck++; 91 92 if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE) 93 return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP); 94 95 op2 = kvm_s390_get_base_disp_s(vcpu, &ar); 96 if (op2 & 7) /* Operand must be on a doubleword boundary */ 97 return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION); 98 rc = read_guest(vcpu, op2, ar, >od.tod, sizeof(gtod.tod)); 99 if (rc) 100 return kvm_s390_inject_prog_cond(vcpu, rc); 101 102 VCPU_EVENT(vcpu, 3, "SCK: setting guest TOD to 0x%llx", gtod.tod); 103 /* 104 * To set the TOD clock the kvm lock must be taken, but the vcpu lock 105 * is already held in handle_set_clock. The usual lock order is the 106 * opposite. As SCK is deprecated and should not be used in several 107 * cases, for example when the multiple epoch facility or TOD clock 108 * steering facility is installed (see Principles of Operation), a 109 * slow path can be used. If the lock can not be taken via try_lock, 110 * the instruction will be retried via -EAGAIN at a later point in 111 * time. 112 */ 113 if (!kvm_s390_try_set_tod_clock(vcpu->kvm, >od)) { 114 kvm_s390_retry_instr(vcpu); 115 return -EAGAIN; 116 } 117 118 kvm_s390_set_psw_cc(vcpu, 0); 119 return 0; 120 } 121 122 static int handle_set_prefix(struct kvm_vcpu *vcpu) 123 { 124 u64 operand2; 125 u32 address; 126 int rc; 127 u8 ar; 128 129 vcpu->stat.instruction_spx++; 130 131 if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE) 132 return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP); 133 134 operand2 = kvm_s390_get_base_disp_s(vcpu, &ar); 135 136 /* must be word boundary */ 137 if (operand2 & 3) 138 return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION); 139 140 /* get the value */ 141 rc = read_guest(vcpu, operand2, ar, &address, sizeof(address)); 142 if (rc) 143 return kvm_s390_inject_prog_cond(vcpu, rc); 144 145 address &= 0x7fffe000u; 146 147 /* 148 * Make sure the new value is valid memory. We only need to check the 149 * first page, since address is 8k aligned and memory pieces are always 150 * at least 1MB aligned and have at least a size of 1MB. 151 */ 152 if (!kvm_is_gpa_in_memslot(vcpu->kvm, address)) 153 return kvm_s390_inject_program_int(vcpu, PGM_ADDRESSING); 154 155 kvm_s390_set_prefix(vcpu, address); 156 trace_kvm_s390_handle_prefix(vcpu, 1, address); 157 return 0; 158 } 159 160 static int handle_store_prefix(struct kvm_vcpu *vcpu) 161 { 162 u64 operand2; 163 u32 address; 164 int rc; 165 u8 ar; 166 167 vcpu->stat.instruction_stpx++; 168 169 if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE) 170 return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP); 171 172 operand2 = kvm_s390_get_base_disp_s(vcpu, &ar); 173 174 /* must be word boundary */ 175 if (operand2 & 3) 176 return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION); 177 178 address = kvm_s390_get_prefix(vcpu); 179 180 /* get the value */ 181 rc = write_guest(vcpu, operand2, ar, &address, sizeof(address)); 182 if (rc) 183 return kvm_s390_inject_prog_cond(vcpu, rc); 184 185 VCPU_EVENT(vcpu, 3, "STPX: storing prefix 0x%x into 0x%llx", address, operand2); 186 trace_kvm_s390_handle_prefix(vcpu, 0, address); 187 return 0; 188 } 189 190 static int handle_store_cpu_address(struct kvm_vcpu *vcpu) 191 { 192 u16 vcpu_id = vcpu->vcpu_id; 193 u64 ga; 194 int rc; 195 u8 ar; 196 197 vcpu->stat.instruction_stap++; 198 199 if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE) 200 return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP); 201 202 ga = kvm_s390_get_base_disp_s(vcpu, &ar); 203 204 if (ga & 1) 205 return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION); 206 207 rc = write_guest(vcpu, ga, ar, &vcpu_id, sizeof(vcpu_id)); 208 if (rc) 209 return kvm_s390_inject_prog_cond(vcpu, rc); 210 211 VCPU_EVENT(vcpu, 3, "STAP: storing cpu address (%u) to 0x%llx", vcpu_id, ga); 212 trace_kvm_s390_handle_stap(vcpu, ga); 213 return 0; 214 } 215 216 int kvm_s390_skey_check_enable(struct kvm_vcpu *vcpu) 217 { 218 int rc; 219 220 trace_kvm_s390_skey_related_inst(vcpu); 221 /* Already enabled? */ 222 if (vcpu->arch.skey_enabled) 223 return 0; 224 225 rc = s390_enable_skey(); 226 VCPU_EVENT(vcpu, 3, "enabling storage keys for guest: %d", rc); 227 if (rc) 228 return rc; 229 230 if (kvm_s390_test_cpuflags(vcpu, CPUSTAT_KSS)) 231 kvm_s390_clear_cpuflags(vcpu, CPUSTAT_KSS); 232 if (!vcpu->kvm->arch.use_skf) 233 vcpu->arch.sie_block->ictl |= ICTL_ISKE | ICTL_SSKE | ICTL_RRBE; 234 else 235 vcpu->arch.sie_block->ictl &= ~(ICTL_ISKE | ICTL_SSKE | ICTL_RRBE); 236 vcpu->arch.skey_enabled = true; 237 return 0; 238 } 239 240 static int try_handle_skey(struct kvm_vcpu *vcpu) 241 { 242 int rc; 243 244 rc = kvm_s390_skey_check_enable(vcpu); 245 if (rc) 246 return rc; 247 if (vcpu->kvm->arch.use_skf) { 248 /* with storage-key facility, SIE interprets it for us */ 249 kvm_s390_retry_instr(vcpu); 250 VCPU_EVENT(vcpu, 4, "%s", "retrying storage key operation"); 251 return -EAGAIN; 252 } 253 return 0; 254 } 255 256 static int handle_iske(struct kvm_vcpu *vcpu) 257 { 258 unsigned long gaddr, vmaddr; 259 unsigned char key; 260 int reg1, reg2; 261 bool unlocked; 262 int rc; 263 264 vcpu->stat.instruction_iske++; 265 266 if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE) 267 return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP); 268 269 rc = try_handle_skey(vcpu); 270 if (rc) 271 return rc != -EAGAIN ? rc : 0; 272 273 kvm_s390_get_regs_rre(vcpu, ®1, ®2); 274 275 gaddr = vcpu->run->s.regs.gprs[reg2] & PAGE_MASK; 276 gaddr = kvm_s390_logical_to_effective(vcpu, gaddr); 277 gaddr = kvm_s390_real_to_abs(vcpu, gaddr); 278 vmaddr = gfn_to_hva(vcpu->kvm, gpa_to_gfn(gaddr)); 279 if (kvm_is_error_hva(vmaddr)) 280 return kvm_s390_inject_program_int(vcpu, PGM_ADDRESSING); 281 retry: 282 unlocked = false; 283 mmap_read_lock(current->mm); 284 rc = get_guest_storage_key(current->mm, vmaddr, &key); 285 286 if (rc) { 287 rc = fixup_user_fault(current->mm, vmaddr, 288 FAULT_FLAG_WRITE, &unlocked); 289 if (!rc) { 290 mmap_read_unlock(current->mm); 291 goto retry; 292 } 293 } 294 mmap_read_unlock(current->mm); 295 if (rc == -EFAULT) 296 return kvm_s390_inject_program_int(vcpu, PGM_ADDRESSING); 297 if (rc < 0) 298 return rc; 299 vcpu->run->s.regs.gprs[reg1] &= ~0xff; 300 vcpu->run->s.regs.gprs[reg1] |= key; 301 return 0; 302 } 303 304 static int handle_rrbe(struct kvm_vcpu *vcpu) 305 { 306 unsigned long vmaddr, gaddr; 307 int reg1, reg2; 308 bool unlocked; 309 int rc; 310 311 vcpu->stat.instruction_rrbe++; 312 313 if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE) 314 return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP); 315 316 rc = try_handle_skey(vcpu); 317 if (rc) 318 return rc != -EAGAIN ? rc : 0; 319 320 kvm_s390_get_regs_rre(vcpu, ®1, ®2); 321 322 gaddr = vcpu->run->s.regs.gprs[reg2] & PAGE_MASK; 323 gaddr = kvm_s390_logical_to_effective(vcpu, gaddr); 324 gaddr = kvm_s390_real_to_abs(vcpu, gaddr); 325 vmaddr = gfn_to_hva(vcpu->kvm, gpa_to_gfn(gaddr)); 326 if (kvm_is_error_hva(vmaddr)) 327 return kvm_s390_inject_program_int(vcpu, PGM_ADDRESSING); 328 retry: 329 unlocked = false; 330 mmap_read_lock(current->mm); 331 rc = reset_guest_reference_bit(current->mm, vmaddr); 332 if (rc < 0) { 333 rc = fixup_user_fault(current->mm, vmaddr, 334 FAULT_FLAG_WRITE, &unlocked); 335 if (!rc) { 336 mmap_read_unlock(current->mm); 337 goto retry; 338 } 339 } 340 mmap_read_unlock(current->mm); 341 if (rc == -EFAULT) 342 return kvm_s390_inject_program_int(vcpu, PGM_ADDRESSING); 343 if (rc < 0) 344 return rc; 345 kvm_s390_set_psw_cc(vcpu, rc); 346 return 0; 347 } 348 349 #define SSKE_NQ 0x8 350 #define SSKE_MR 0x4 351 #define SSKE_MC 0x2 352 #define SSKE_MB 0x1 353 static int handle_sske(struct kvm_vcpu *vcpu) 354 { 355 unsigned char m3 = vcpu->arch.sie_block->ipb >> 28; 356 unsigned long start, end; 357 unsigned char key, oldkey; 358 int reg1, reg2; 359 bool unlocked; 360 int rc; 361 362 vcpu->stat.instruction_sske++; 363 364 if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE) 365 return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP); 366 367 rc = try_handle_skey(vcpu); 368 if (rc) 369 return rc != -EAGAIN ? rc : 0; 370 371 if (!test_kvm_facility(vcpu->kvm, 8)) 372 m3 &= ~SSKE_MB; 373 if (!test_kvm_facility(vcpu->kvm, 10)) 374 m3 &= ~(SSKE_MC | SSKE_MR); 375 if (!test_kvm_facility(vcpu->kvm, 14)) 376 m3 &= ~SSKE_NQ; 377 378 kvm_s390_get_regs_rre(vcpu, ®1, ®2); 379 380 key = vcpu->run->s.regs.gprs[reg1] & 0xfe; 381 start = vcpu->run->s.regs.gprs[reg2] & PAGE_MASK; 382 start = kvm_s390_logical_to_effective(vcpu, start); 383 if (m3 & SSKE_MB) { 384 /* start already designates an absolute address */ 385 end = (start + _SEGMENT_SIZE) & ~(_SEGMENT_SIZE - 1); 386 } else { 387 start = kvm_s390_real_to_abs(vcpu, start); 388 end = start + PAGE_SIZE; 389 } 390 391 while (start != end) { 392 unsigned long vmaddr = gfn_to_hva(vcpu->kvm, gpa_to_gfn(start)); 393 unlocked = false; 394 395 if (kvm_is_error_hva(vmaddr)) 396 return kvm_s390_inject_program_int(vcpu, PGM_ADDRESSING); 397 398 mmap_read_lock(current->mm); 399 rc = cond_set_guest_storage_key(current->mm, vmaddr, key, &oldkey, 400 m3 & SSKE_NQ, m3 & SSKE_MR, 401 m3 & SSKE_MC); 402 403 if (rc < 0) { 404 rc = fixup_user_fault(current->mm, vmaddr, 405 FAULT_FLAG_WRITE, &unlocked); 406 rc = !rc ? -EAGAIN : rc; 407 } 408 mmap_read_unlock(current->mm); 409 if (rc == -EFAULT) 410 return kvm_s390_inject_program_int(vcpu, PGM_ADDRESSING); 411 if (rc == -EAGAIN) 412 continue; 413 if (rc < 0) 414 return rc; 415 start += PAGE_SIZE; 416 } 417 418 if (m3 & (SSKE_MC | SSKE_MR)) { 419 if (m3 & SSKE_MB) { 420 /* skey in reg1 is unpredictable */ 421 kvm_s390_set_psw_cc(vcpu, 3); 422 } else { 423 kvm_s390_set_psw_cc(vcpu, rc); 424 vcpu->run->s.regs.gprs[reg1] &= ~0xff00UL; 425 vcpu->run->s.regs.gprs[reg1] |= (u64) oldkey << 8; 426 } 427 } 428 if (m3 & SSKE_MB) { 429 if (psw_bits(vcpu->arch.sie_block->gpsw).eaba == PSW_BITS_AMODE_64BIT) 430 vcpu->run->s.regs.gprs[reg2] &= ~PAGE_MASK; 431 else 432 vcpu->run->s.regs.gprs[reg2] &= ~0xfffff000UL; 433 end = kvm_s390_logical_to_effective(vcpu, end); 434 vcpu->run->s.regs.gprs[reg2] |= end; 435 } 436 return 0; 437 } 438 439 static int handle_ipte_interlock(struct kvm_vcpu *vcpu) 440 { 441 vcpu->stat.instruction_ipte_interlock++; 442 if (psw_bits(vcpu->arch.sie_block->gpsw).pstate) 443 return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP); 444 wait_event(vcpu->kvm->arch.ipte_wq, !ipte_lock_held(vcpu->kvm)); 445 kvm_s390_retry_instr(vcpu); 446 VCPU_EVENT(vcpu, 4, "%s", "retrying ipte interlock operation"); 447 return 0; 448 } 449 450 static int handle_test_block(struct kvm_vcpu *vcpu) 451 { 452 gpa_t addr; 453 int reg2; 454 455 vcpu->stat.instruction_tb++; 456 457 if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE) 458 return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP); 459 460 kvm_s390_get_regs_rre(vcpu, NULL, ®2); 461 addr = vcpu->run->s.regs.gprs[reg2] & PAGE_MASK; 462 addr = kvm_s390_logical_to_effective(vcpu, addr); 463 if (kvm_s390_check_low_addr_prot_real(vcpu, addr)) 464 return kvm_s390_inject_prog_irq(vcpu, &vcpu->arch.pgm); 465 addr = kvm_s390_real_to_abs(vcpu, addr); 466 467 if (!kvm_is_gpa_in_memslot(vcpu->kvm, addr)) 468 return kvm_s390_inject_program_int(vcpu, PGM_ADDRESSING); 469 /* 470 * We don't expect errors on modern systems, and do not care 471 * about storage keys (yet), so let's just clear the page. 472 */ 473 if (kvm_clear_guest(vcpu->kvm, addr, PAGE_SIZE)) 474 return -EFAULT; 475 kvm_s390_set_psw_cc(vcpu, 0); 476 vcpu->run->s.regs.gprs[0] = 0; 477 return 0; 478 } 479 480 static int handle_tpi(struct kvm_vcpu *vcpu) 481 { 482 struct kvm_s390_interrupt_info *inti; 483 unsigned long len; 484 u32 tpi_data[3]; 485 int rc; 486 u64 addr; 487 u8 ar; 488 489 vcpu->stat.instruction_tpi++; 490 491 addr = kvm_s390_get_base_disp_s(vcpu, &ar); 492 if (addr & 3) 493 return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION); 494 495 inti = kvm_s390_get_io_int(vcpu->kvm, vcpu->arch.sie_block->gcr[6], 0); 496 if (!inti) { 497 kvm_s390_set_psw_cc(vcpu, 0); 498 return 0; 499 } 500 501 tpi_data[0] = inti->io.subchannel_id << 16 | inti->io.subchannel_nr; 502 tpi_data[1] = inti->io.io_int_parm; 503 tpi_data[2] = inti->io.io_int_word; 504 if (addr) { 505 /* 506 * Store the two-word I/O interruption code into the 507 * provided area. 508 */ 509 len = sizeof(tpi_data) - 4; 510 rc = write_guest(vcpu, addr, ar, &tpi_data, len); 511 if (rc) { 512 rc = kvm_s390_inject_prog_cond(vcpu, rc); 513 goto reinject_interrupt; 514 } 515 } else { 516 /* 517 * Store the three-word I/O interruption code into 518 * the appropriate lowcore area. 519 */ 520 len = sizeof(tpi_data); 521 if (write_guest_lc(vcpu, __LC_SUBCHANNEL_ID, &tpi_data, len)) { 522 /* failed writes to the low core are not recoverable */ 523 rc = -EFAULT; 524 goto reinject_interrupt; 525 } 526 } 527 528 /* irq was successfully handed to the guest */ 529 kfree(inti); 530 kvm_s390_set_psw_cc(vcpu, 1); 531 return 0; 532 reinject_interrupt: 533 /* 534 * If we encounter a problem storing the interruption code, the 535 * instruction is suppressed from the guest's view: reinject the 536 * interrupt. 537 */ 538 if (kvm_s390_reinject_io_int(vcpu->kvm, inti)) { 539 kfree(inti); 540 rc = -EFAULT; 541 } 542 /* don't set the cc, a pgm irq was injected or we drop to user space */ 543 return rc ? -EFAULT : 0; 544 } 545 546 static int handle_tsch(struct kvm_vcpu *vcpu) 547 { 548 struct kvm_s390_interrupt_info *inti = NULL; 549 const u64 isc_mask = 0xffUL << 24; /* all iscs set */ 550 551 vcpu->stat.instruction_tsch++; 552 553 /* a valid schid has at least one bit set */ 554 if (vcpu->run->s.regs.gprs[1]) 555 inti = kvm_s390_get_io_int(vcpu->kvm, isc_mask, 556 vcpu->run->s.regs.gprs[1]); 557 558 /* 559 * Prepare exit to userspace. 560 * We indicate whether we dequeued a pending I/O interrupt 561 * so that userspace can re-inject it if the instruction gets 562 * a program check. While this may re-order the pending I/O 563 * interrupts, this is no problem since the priority is kept 564 * intact. 565 */ 566 vcpu->run->exit_reason = KVM_EXIT_S390_TSCH; 567 vcpu->run->s390_tsch.dequeued = !!inti; 568 if (inti) { 569 vcpu->run->s390_tsch.subchannel_id = inti->io.subchannel_id; 570 vcpu->run->s390_tsch.subchannel_nr = inti->io.subchannel_nr; 571 vcpu->run->s390_tsch.io_int_parm = inti->io.io_int_parm; 572 vcpu->run->s390_tsch.io_int_word = inti->io.io_int_word; 573 } 574 vcpu->run->s390_tsch.ipb = vcpu->arch.sie_block->ipb; 575 kfree(inti); 576 return -EREMOTE; 577 } 578 579 static int handle_io_inst(struct kvm_vcpu *vcpu) 580 { 581 VCPU_EVENT(vcpu, 4, "%s", "I/O instruction"); 582 583 if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE) 584 return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP); 585 586 if (vcpu->kvm->arch.css_support) { 587 /* 588 * Most I/O instructions will be handled by userspace. 589 * Exceptions are tpi and the interrupt portion of tsch. 590 */ 591 if (vcpu->arch.sie_block->ipa == 0xb236) 592 return handle_tpi(vcpu); 593 if (vcpu->arch.sie_block->ipa == 0xb235) 594 return handle_tsch(vcpu); 595 /* Handle in userspace. */ 596 vcpu->stat.instruction_io_other++; 597 return -EOPNOTSUPP; 598 } else { 599 /* 600 * Set condition code 3 to stop the guest from issuing channel 601 * I/O instructions. 602 */ 603 kvm_s390_set_psw_cc(vcpu, 3); 604 return 0; 605 } 606 } 607 608 #if IS_ENABLED(CONFIG_VFIO_AP) 609 bool kvm_s390_is_gpa_in_memslot(struct kvm *kvm, gpa_t gpa) 610 { 611 return kvm_is_gpa_in_memslot(kvm, gpa); 612 } 613 EXPORT_SYMBOL_FOR_MODULES(kvm_s390_is_gpa_in_memslot, "vfio_ap"); 614 #endif 615 616 /* 617 * handle_pqap: Handling pqap interception 618 * @vcpu: the vcpu having issue the pqap instruction 619 * 620 * We now support PQAP/AQIC instructions and we need to correctly 621 * answer the guest even if no dedicated driver's hook is available. 622 * 623 * The intercepting code calls a dedicated callback for this instruction 624 * if a driver did register one in the CRYPTO satellite of the 625 * SIE block. 626 * 627 * If no callback is available, the queues are not available, return this 628 * response code to the caller and set CC to 3. 629 * Else return the response code returned by the callback. 630 */ 631 static int handle_pqap(struct kvm_vcpu *vcpu) 632 { 633 struct ap_queue_status status = {}; 634 crypto_hook pqap_hook; 635 unsigned long reg0; 636 int ret; 637 uint8_t fc; 638 639 /* Verify that the AP instruction are available */ 640 if (!ap_instructions_available()) 641 return -EOPNOTSUPP; 642 /* Verify that the guest is allowed to use AP instructions */ 643 if (!(vcpu->arch.sie_block->eca & ECA_APIE)) 644 return -EOPNOTSUPP; 645 /* 646 * The only possibly intercepted functions when AP instructions are 647 * available for the guest are AQIC and TAPQ with the t bit set 648 * since we do not set IC.3 (FIII) we currently will only intercept 649 * the AQIC function code. 650 * Note: running nested under z/VM can result in intercepts for other 651 * function codes, e.g. PQAP(QCI). We do not support this and bail out. 652 */ 653 reg0 = vcpu->run->s.regs.gprs[0]; 654 fc = (reg0 >> 24) & 0xff; 655 if (fc != 0x03) 656 return -EOPNOTSUPP; 657 658 /* PQAP instruction is allowed for guest kernel only */ 659 if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE) 660 return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP); 661 662 /* Common PQAP instruction specification exceptions */ 663 /* bits 41-47 must all be zeros */ 664 if (reg0 & 0x007f0000UL) 665 return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION); 666 /* APFT not install and T bit set */ 667 if (!test_kvm_facility(vcpu->kvm, 15) && (reg0 & 0x00800000UL)) 668 return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION); 669 /* APXA not installed and APID greater 64 or APQI greater 16 */ 670 if (!(vcpu->kvm->arch.crypto.crycbd & 0x02) && (reg0 & 0x0000c0f0UL)) 671 return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION); 672 673 /* AQIC function code specific exception */ 674 /* facility 65 not present for AQIC function code */ 675 if (!test_kvm_facility(vcpu->kvm, 65)) 676 return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION); 677 678 /* 679 * If the hook callback is registered, there will be a pointer to the 680 * hook function pointer in the kvm_s390_crypto structure. Lock the 681 * owner, retrieve the hook function pointer and call the hook. 682 */ 683 down_read(&vcpu->kvm->arch.crypto.pqap_hook_rwsem); 684 if (vcpu->kvm->arch.crypto.pqap_hook) { 685 pqap_hook = *vcpu->kvm->arch.crypto.pqap_hook; 686 ret = pqap_hook(vcpu); 687 if (!ret) { 688 if (vcpu->run->s.regs.gprs[1] & 0x00ff0000) 689 kvm_s390_set_psw_cc(vcpu, 3); 690 else 691 kvm_s390_set_psw_cc(vcpu, 0); 692 } 693 up_read(&vcpu->kvm->arch.crypto.pqap_hook_rwsem); 694 return ret; 695 } 696 up_read(&vcpu->kvm->arch.crypto.pqap_hook_rwsem); 697 /* 698 * A vfio_driver must register a hook. 699 * No hook means no driver to enable the SIE CRYCB and no queues. 700 * We send this response to the guest. 701 */ 702 status.response_code = 0x01; 703 memcpy(&vcpu->run->s.regs.gprs[1], &status, sizeof(status)); 704 kvm_s390_set_psw_cc(vcpu, 3); 705 return 0; 706 } 707 708 static int handle_stfl(struct kvm_vcpu *vcpu) 709 { 710 int rc; 711 unsigned int fac; 712 713 vcpu->stat.instruction_stfl++; 714 715 if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE) 716 return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP); 717 718 /* 719 * We need to shift the lower 32 facility bits (bit 0-31) from a u64 720 * into a u32 memory representation. They will remain bits 0-31. 721 */ 722 fac = *vcpu->kvm->arch.model.fac_list >> 32; 723 rc = write_guest_lc(vcpu, offsetof(struct lowcore, stfl_fac_list), 724 &fac, sizeof(fac)); 725 if (rc) 726 return rc; 727 VCPU_EVENT(vcpu, 3, "STFL: store facility list 0x%x", fac); 728 trace_kvm_s390_handle_stfl(vcpu, fac); 729 return 0; 730 } 731 732 #define PSW_MASK_ADDR_MODE (PSW_MASK_EA | PSW_MASK_BA) 733 #define PSW_MASK_UNASSIGNED 0xb80800fe7fffffffUL 734 #define PSW_ADDR_24 0x0000000000ffffffUL 735 #define PSW_ADDR_31 0x000000007fffffffUL 736 737 int is_valid_psw(psw_t *psw) 738 { 739 if (psw->mask & PSW_MASK_UNASSIGNED) 740 return 0; 741 if ((psw->mask & PSW_MASK_ADDR_MODE) == PSW_MASK_BA) { 742 if (psw->addr & ~PSW_ADDR_31) 743 return 0; 744 } 745 if (!(psw->mask & PSW_MASK_ADDR_MODE) && (psw->addr & ~PSW_ADDR_24)) 746 return 0; 747 if ((psw->mask & PSW_MASK_ADDR_MODE) == PSW_MASK_EA) 748 return 0; 749 if (psw->addr & 1) 750 return 0; 751 return 1; 752 } 753 754 int kvm_s390_handle_lpsw(struct kvm_vcpu *vcpu) 755 { 756 psw_t *gpsw = &vcpu->arch.sie_block->gpsw; 757 psw_compat_t new_psw; 758 u64 addr; 759 int rc; 760 u8 ar; 761 762 vcpu->stat.instruction_lpsw++; 763 764 if (gpsw->mask & PSW_MASK_PSTATE) 765 return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP); 766 767 addr = kvm_s390_get_base_disp_s(vcpu, &ar); 768 if (addr & 7) 769 return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION); 770 771 rc = read_guest(vcpu, addr, ar, &new_psw, sizeof(new_psw)); 772 if (rc) 773 return kvm_s390_inject_prog_cond(vcpu, rc); 774 if (!(new_psw.mask & PSW32_MASK_BASE)) 775 return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION); 776 gpsw->mask = (new_psw.mask & ~PSW32_MASK_BASE) << 32; 777 gpsw->mask |= new_psw.addr & PSW32_ADDR_AMODE; 778 gpsw->addr = new_psw.addr & ~PSW32_ADDR_AMODE; 779 if (!is_valid_psw(gpsw)) 780 return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION); 781 return 0; 782 } 783 784 static int handle_lpswe(struct kvm_vcpu *vcpu) 785 { 786 psw_t new_psw; 787 u64 addr; 788 int rc; 789 u8 ar; 790 791 vcpu->stat.instruction_lpswe++; 792 793 if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE) 794 return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP); 795 796 addr = kvm_s390_get_base_disp_s(vcpu, &ar); 797 if (addr & 7) 798 return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION); 799 rc = read_guest(vcpu, addr, ar, &new_psw, sizeof(new_psw)); 800 if (rc) 801 return kvm_s390_inject_prog_cond(vcpu, rc); 802 vcpu->arch.sie_block->gpsw = new_psw; 803 if (!is_valid_psw(&vcpu->arch.sie_block->gpsw)) 804 return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION); 805 return 0; 806 } 807 808 static int handle_lpswey(struct kvm_vcpu *vcpu) 809 { 810 psw_t new_psw; 811 u64 addr; 812 int rc; 813 u8 ar; 814 815 vcpu->stat.instruction_lpswey++; 816 817 if (!test_kvm_facility(vcpu->kvm, 193)) 818 return kvm_s390_inject_program_int(vcpu, PGM_OPERATION); 819 820 if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE) 821 return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP); 822 823 addr = kvm_s390_get_base_disp_siy(vcpu, &ar); 824 if (addr & 7) 825 return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION); 826 827 rc = read_guest(vcpu, addr, ar, &new_psw, sizeof(new_psw)); 828 if (rc) 829 return kvm_s390_inject_prog_cond(vcpu, rc); 830 831 vcpu->arch.sie_block->gpsw = new_psw; 832 if (!is_valid_psw(&vcpu->arch.sie_block->gpsw)) 833 return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION); 834 835 return 0; 836 } 837 838 static int handle_stidp(struct kvm_vcpu *vcpu) 839 { 840 u64 stidp_data = vcpu->kvm->arch.model.cpuid; 841 u64 operand2; 842 int rc; 843 u8 ar; 844 845 vcpu->stat.instruction_stidp++; 846 847 if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE) 848 return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP); 849 850 operand2 = kvm_s390_get_base_disp_s(vcpu, &ar); 851 852 if (operand2 & 7) 853 return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION); 854 855 rc = write_guest(vcpu, operand2, ar, &stidp_data, sizeof(stidp_data)); 856 if (rc) 857 return kvm_s390_inject_prog_cond(vcpu, rc); 858 859 VCPU_EVENT(vcpu, 3, "STIDP: store cpu id 0x%llx", stidp_data); 860 return 0; 861 } 862 863 static void handle_stsi_3_2_2(struct kvm_vcpu *vcpu, struct sysinfo_3_2_2 *mem) 864 { 865 int cpus = 0; 866 int n; 867 868 cpus = atomic_read(&vcpu->kvm->online_vcpus); 869 870 /* deal with other level 3 hypervisors */ 871 if (stsi(mem, 3, 2, 2)) 872 mem->count = 0; 873 if (mem->count < 8) 874 mem->count++; 875 for (n = mem->count - 1; n > 0 ; n--) 876 memcpy(&mem->vm[n], &mem->vm[n - 1], sizeof(mem->vm[0])); 877 878 memset(&mem->vm[0], 0, sizeof(mem->vm[0])); 879 mem->vm[0].cpus_total = cpus; 880 mem->vm[0].cpus_configured = cpus; 881 mem->vm[0].cpus_standby = 0; 882 mem->vm[0].cpus_reserved = 0; 883 mem->vm[0].caf = 1000; 884 memcpy(mem->vm[0].name, "KVMguest", 8); 885 ASCEBC(mem->vm[0].name, 8); 886 memcpy(mem->vm[0].cpi, "KVM/Linux ", 16); 887 ASCEBC(mem->vm[0].cpi, 16); 888 } 889 890 static void insert_stsi_usr_data(struct kvm_vcpu *vcpu, u64 addr, u8 ar, 891 u8 fc, u8 sel1, u16 sel2) 892 { 893 vcpu->run->exit_reason = KVM_EXIT_S390_STSI; 894 vcpu->run->s390_stsi.addr = addr; 895 vcpu->run->s390_stsi.ar = ar; 896 vcpu->run->s390_stsi.fc = fc; 897 vcpu->run->s390_stsi.sel1 = sel1; 898 vcpu->run->s390_stsi.sel2 = sel2; 899 } 900 901 static int handle_stsi(struct kvm_vcpu *vcpu) 902 { 903 int fc = (vcpu->run->s.regs.gprs[0] & 0xf0000000) >> 28; 904 int sel1 = vcpu->run->s.regs.gprs[0] & 0xff; 905 int sel2 = vcpu->run->s.regs.gprs[1] & 0xffff; 906 unsigned long mem = 0; 907 u64 operand2; 908 int rc = 0; 909 u8 ar; 910 911 vcpu->stat.instruction_stsi++; 912 VCPU_EVENT(vcpu, 3, "STSI: fc: %u sel1: %u sel2: %u", fc, sel1, sel2); 913 914 if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE) 915 return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP); 916 917 /* Bailout forbidden function codes */ 918 if (fc > 3 && fc != 15) 919 goto out_no_data; 920 921 /* 922 * fc 15 is provided only with 923 * - PTF/CPU topology support through facility 15 924 * - KVM_CAP_S390_USER_STSI 925 */ 926 if (fc == 15 && (!test_kvm_facility(vcpu->kvm, 11) || 927 !vcpu->kvm->arch.user_stsi)) 928 goto out_no_data; 929 930 if (vcpu->run->s.regs.gprs[0] & 0x0fffff00 931 || vcpu->run->s.regs.gprs[1] & 0xffff0000) 932 return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION); 933 934 if (fc == 0) { 935 vcpu->run->s.regs.gprs[0] = 3 << 28; 936 kvm_s390_set_psw_cc(vcpu, 0); 937 return 0; 938 } 939 940 operand2 = kvm_s390_get_base_disp_s(vcpu, &ar); 941 942 if (!kvm_s390_pv_cpu_is_protected(vcpu) && (operand2 & 0xfff)) 943 return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION); 944 945 switch (fc) { 946 case 1: /* same handling for 1 and 2 */ 947 case 2: 948 mem = get_zeroed_page(GFP_KERNEL_ACCOUNT); 949 if (!mem) 950 goto out_no_data; 951 if (stsi((void *) mem, fc, sel1, sel2)) 952 goto out_no_data; 953 break; 954 case 3: 955 if (sel1 != 2 || sel2 != 2) 956 goto out_no_data; 957 mem = get_zeroed_page(GFP_KERNEL_ACCOUNT); 958 if (!mem) 959 goto out_no_data; 960 handle_stsi_3_2_2(vcpu, (void *) mem); 961 break; 962 case 15: /* fc 15 is fully handled in userspace */ 963 insert_stsi_usr_data(vcpu, operand2, ar, fc, sel1, sel2); 964 trace_kvm_s390_handle_stsi(vcpu, fc, sel1, sel2, operand2); 965 return -EREMOTE; 966 } 967 if (kvm_s390_pv_cpu_is_protected(vcpu)) { 968 memcpy(sida_addr(vcpu->arch.sie_block), (void *)mem, PAGE_SIZE); 969 rc = 0; 970 } else { 971 rc = write_guest(vcpu, operand2, ar, (void *)mem, PAGE_SIZE); 972 } 973 if (rc) { 974 rc = kvm_s390_inject_prog_cond(vcpu, rc); 975 goto out; 976 } 977 if (vcpu->kvm->arch.user_stsi) { 978 insert_stsi_usr_data(vcpu, operand2, ar, fc, sel1, sel2); 979 rc = -EREMOTE; 980 } 981 trace_kvm_s390_handle_stsi(vcpu, fc, sel1, sel2, operand2); 982 free_page(mem); 983 kvm_s390_set_psw_cc(vcpu, 0); 984 vcpu->run->s.regs.gprs[0] = 0; 985 return rc; 986 out_no_data: 987 kvm_s390_set_psw_cc(vcpu, 3); 988 out: 989 free_page(mem); 990 return rc; 991 } 992 993 int kvm_s390_handle_b2(struct kvm_vcpu *vcpu) 994 { 995 switch (vcpu->arch.sie_block->ipa & 0x00ff) { 996 case 0x02: 997 return handle_stidp(vcpu); 998 case 0x04: 999 return handle_set_clock(vcpu); 1000 case 0x10: 1001 return handle_set_prefix(vcpu); 1002 case 0x11: 1003 return handle_store_prefix(vcpu); 1004 case 0x12: 1005 return handle_store_cpu_address(vcpu); 1006 case 0x14: 1007 return kvm_s390_handle_vsie(vcpu); 1008 case 0x21: 1009 case 0x50: 1010 return handle_ipte_interlock(vcpu); 1011 case 0x29: 1012 return handle_iske(vcpu); 1013 case 0x2a: 1014 return handle_rrbe(vcpu); 1015 case 0x2b: 1016 return handle_sske(vcpu); 1017 case 0x2c: 1018 return handle_test_block(vcpu); 1019 case 0x30: 1020 case 0x31: 1021 case 0x32: 1022 case 0x33: 1023 case 0x34: 1024 case 0x35: 1025 case 0x36: 1026 case 0x37: 1027 case 0x38: 1028 case 0x39: 1029 case 0x3a: 1030 case 0x3b: 1031 case 0x3c: 1032 case 0x5f: 1033 case 0x74: 1034 case 0x76: 1035 return handle_io_inst(vcpu); 1036 case 0x56: 1037 return handle_sthyi(vcpu); 1038 case 0x7d: 1039 return handle_stsi(vcpu); 1040 case 0xaf: 1041 return handle_pqap(vcpu); 1042 case 0xb1: 1043 return handle_stfl(vcpu); 1044 case 0xb2: 1045 return handle_lpswe(vcpu); 1046 default: 1047 return -EOPNOTSUPP; 1048 } 1049 } 1050 1051 static int handle_epsw(struct kvm_vcpu *vcpu) 1052 { 1053 int reg1, reg2; 1054 1055 vcpu->stat.instruction_epsw++; 1056 1057 kvm_s390_get_regs_rre(vcpu, ®1, ®2); 1058 1059 /* This basically extracts the mask half of the psw. */ 1060 vcpu->run->s.regs.gprs[reg1] &= 0xffffffff00000000UL; 1061 vcpu->run->s.regs.gprs[reg1] |= vcpu->arch.sie_block->gpsw.mask >> 32; 1062 if (reg2) { 1063 vcpu->run->s.regs.gprs[reg2] &= 0xffffffff00000000UL; 1064 vcpu->run->s.regs.gprs[reg2] |= 1065 vcpu->arch.sie_block->gpsw.mask & 0x00000000ffffffffUL; 1066 } 1067 return 0; 1068 } 1069 1070 #define PFMF_RESERVED 0xfffc0101UL 1071 #define PFMF_SK 0x00020000UL 1072 #define PFMF_CF 0x00010000UL 1073 #define PFMF_UI 0x00008000UL 1074 #define PFMF_FSC 0x00007000UL 1075 #define PFMF_NQ 0x00000800UL 1076 #define PFMF_MR 0x00000400UL 1077 #define PFMF_MC 0x00000200UL 1078 #define PFMF_KEY 0x000000feUL 1079 1080 static int handle_pfmf(struct kvm_vcpu *vcpu) 1081 { 1082 bool mr = false, mc = false, nq; 1083 int reg1, reg2; 1084 unsigned long start, end; 1085 unsigned char key; 1086 1087 vcpu->stat.instruction_pfmf++; 1088 1089 kvm_s390_get_regs_rre(vcpu, ®1, ®2); 1090 1091 if (!test_kvm_facility(vcpu->kvm, 8)) 1092 return kvm_s390_inject_program_int(vcpu, PGM_OPERATION); 1093 1094 if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE) 1095 return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP); 1096 1097 if (vcpu->run->s.regs.gprs[reg1] & PFMF_RESERVED) 1098 return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION); 1099 1100 /* Only provide non-quiescing support if enabled for the guest */ 1101 if (vcpu->run->s.regs.gprs[reg1] & PFMF_NQ && 1102 !test_kvm_facility(vcpu->kvm, 14)) 1103 return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION); 1104 1105 /* Only provide conditional-SSKE support if enabled for the guest */ 1106 if (vcpu->run->s.regs.gprs[reg1] & PFMF_SK && 1107 test_kvm_facility(vcpu->kvm, 10)) { 1108 mr = vcpu->run->s.regs.gprs[reg1] & PFMF_MR; 1109 mc = vcpu->run->s.regs.gprs[reg1] & PFMF_MC; 1110 } 1111 1112 nq = vcpu->run->s.regs.gprs[reg1] & PFMF_NQ; 1113 key = vcpu->run->s.regs.gprs[reg1] & PFMF_KEY; 1114 start = vcpu->run->s.regs.gprs[reg2] & PAGE_MASK; 1115 start = kvm_s390_logical_to_effective(vcpu, start); 1116 1117 if (vcpu->run->s.regs.gprs[reg1] & PFMF_CF) { 1118 if (kvm_s390_check_low_addr_prot_real(vcpu, start)) 1119 return kvm_s390_inject_prog_irq(vcpu, &vcpu->arch.pgm); 1120 } 1121 1122 switch (vcpu->run->s.regs.gprs[reg1] & PFMF_FSC) { 1123 case 0x00000000: 1124 /* only 4k frames specify a real address */ 1125 start = kvm_s390_real_to_abs(vcpu, start); 1126 end = (start + PAGE_SIZE) & ~(PAGE_SIZE - 1); 1127 break; 1128 case 0x00001000: 1129 end = (start + _SEGMENT_SIZE) & ~(_SEGMENT_SIZE - 1); 1130 break; 1131 case 0x00002000: 1132 /* only support 2G frame size if EDAT2 is available and we are 1133 not in 24-bit addressing mode */ 1134 if (!test_kvm_facility(vcpu->kvm, 78) || 1135 psw_bits(vcpu->arch.sie_block->gpsw).eaba == PSW_BITS_AMODE_24BIT) 1136 return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION); 1137 end = (start + _REGION3_SIZE) & ~(_REGION3_SIZE - 1); 1138 break; 1139 default: 1140 return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION); 1141 } 1142 1143 while (start != end) { 1144 unsigned long vmaddr; 1145 bool unlocked = false; 1146 1147 /* Translate guest address to host address */ 1148 vmaddr = gfn_to_hva(vcpu->kvm, gpa_to_gfn(start)); 1149 if (kvm_is_error_hva(vmaddr)) 1150 return kvm_s390_inject_program_int(vcpu, PGM_ADDRESSING); 1151 1152 if (vcpu->run->s.regs.gprs[reg1] & PFMF_CF) { 1153 if (kvm_clear_guest(vcpu->kvm, start, PAGE_SIZE)) 1154 return kvm_s390_inject_program_int(vcpu, PGM_ADDRESSING); 1155 } 1156 1157 if (vcpu->run->s.regs.gprs[reg1] & PFMF_SK) { 1158 int rc = kvm_s390_skey_check_enable(vcpu); 1159 1160 if (rc) 1161 return rc; 1162 mmap_read_lock(current->mm); 1163 rc = cond_set_guest_storage_key(current->mm, vmaddr, 1164 key, NULL, nq, mr, mc); 1165 if (rc < 0) { 1166 rc = fixup_user_fault(current->mm, vmaddr, 1167 FAULT_FLAG_WRITE, &unlocked); 1168 rc = !rc ? -EAGAIN : rc; 1169 } 1170 mmap_read_unlock(current->mm); 1171 if (rc == -EFAULT) 1172 return kvm_s390_inject_program_int(vcpu, PGM_ADDRESSING); 1173 if (rc == -EAGAIN) 1174 continue; 1175 if (rc < 0) 1176 return rc; 1177 } 1178 start += PAGE_SIZE; 1179 } 1180 if (vcpu->run->s.regs.gprs[reg1] & PFMF_FSC) { 1181 if (psw_bits(vcpu->arch.sie_block->gpsw).eaba == PSW_BITS_AMODE_64BIT) { 1182 vcpu->run->s.regs.gprs[reg2] = end; 1183 } else { 1184 vcpu->run->s.regs.gprs[reg2] &= ~0xffffffffUL; 1185 end = kvm_s390_logical_to_effective(vcpu, end); 1186 vcpu->run->s.regs.gprs[reg2] |= end; 1187 } 1188 } 1189 return 0; 1190 } 1191 1192 /* 1193 * Must be called with relevant read locks held (kvm->mm->mmap_lock, kvm->srcu) 1194 */ 1195 static inline int __do_essa(struct kvm_vcpu *vcpu, const int orc) 1196 { 1197 int r1, r2, nappended, entries; 1198 unsigned long gfn, hva, res, pgstev, ptev; 1199 unsigned long *cbrlo; 1200 1201 /* 1202 * We don't need to set SD.FPF.SK to 1 here, because if we have a 1203 * machine check here we either handle it or crash 1204 */ 1205 1206 kvm_s390_get_regs_rre(vcpu, &r1, &r2); 1207 gfn = vcpu->run->s.regs.gprs[r2] >> PAGE_SHIFT; 1208 hva = gfn_to_hva(vcpu->kvm, gfn); 1209 entries = (vcpu->arch.sie_block->cbrlo & ~PAGE_MASK) >> 3; 1210 1211 if (kvm_is_error_hva(hva)) 1212 return kvm_s390_inject_program_int(vcpu, PGM_ADDRESSING); 1213 1214 nappended = pgste_perform_essa(vcpu->kvm->mm, hva, orc, &ptev, &pgstev); 1215 if (nappended < 0) { 1216 res = orc ? 0x10 : 0; 1217 vcpu->run->s.regs.gprs[r1] = res; /* Exception Indication */ 1218 return 0; 1219 } 1220 res = (pgstev & _PGSTE_GPS_USAGE_MASK) >> 22; 1221 /* 1222 * Set the block-content state part of the result. 0 means resident, so 1223 * nothing to do if the page is valid. 2 is for preserved pages 1224 * (non-present and non-zero), and 3 for zero pages (non-present and 1225 * zero). 1226 */ 1227 if (ptev & _PAGE_INVALID) { 1228 res |= 2; 1229 if (pgstev & _PGSTE_GPS_ZERO) 1230 res |= 1; 1231 } 1232 if (pgstev & _PGSTE_GPS_NODAT) 1233 res |= 0x20; 1234 vcpu->run->s.regs.gprs[r1] = res; 1235 /* 1236 * It is possible that all the normal 511 slots were full, in which case 1237 * we will now write in the 512th slot, which is reserved for host use. 1238 * In both cases we let the normal essa handling code process all the 1239 * slots, including the reserved one, if needed. 1240 */ 1241 if (nappended > 0) { 1242 cbrlo = phys_to_virt(vcpu->arch.sie_block->cbrlo & PAGE_MASK); 1243 cbrlo[entries] = gfn << PAGE_SHIFT; 1244 } 1245 1246 if (orc) { 1247 struct kvm_memory_slot *ms = gfn_to_memslot(vcpu->kvm, gfn); 1248 1249 /* Increment only if we are really flipping the bit */ 1250 if (ms && !test_and_set_bit(gfn - ms->base_gfn, kvm_second_dirty_bitmap(ms))) 1251 atomic64_inc(&vcpu->kvm->arch.cmma_dirty_pages); 1252 } 1253 1254 return nappended; 1255 } 1256 1257 static int handle_essa(struct kvm_vcpu *vcpu) 1258 { 1259 lockdep_assert_held(&vcpu->kvm->srcu); 1260 1261 /* entries expected to be 1FF */ 1262 int entries = (vcpu->arch.sie_block->cbrlo & ~PAGE_MASK) >> 3; 1263 unsigned long *cbrlo; 1264 struct gmap *gmap; 1265 int i, orc; 1266 1267 VCPU_EVENT(vcpu, 4, "ESSA: release %d pages", entries); 1268 gmap = vcpu->arch.gmap; 1269 vcpu->stat.instruction_essa++; 1270 if (!vcpu->kvm->arch.use_cmma) 1271 return kvm_s390_inject_program_int(vcpu, PGM_OPERATION); 1272 1273 if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE) 1274 return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP); 1275 /* Check for invalid operation request code */ 1276 orc = (vcpu->arch.sie_block->ipb & 0xf0000000) >> 28; 1277 /* ORCs 0-6 are always valid */ 1278 if (orc > (test_kvm_facility(vcpu->kvm, 147) ? ESSA_SET_STABLE_NODAT 1279 : ESSA_SET_STABLE_IF_RESIDENT)) 1280 return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION); 1281 1282 if (!vcpu->kvm->arch.migration_mode) { 1283 /* 1284 * CMMA is enabled in the KVM settings, but is disabled in 1285 * the SIE block and in the mm_context, and we are not doing 1286 * a migration. Enable CMMA in the mm_context. 1287 * Since we need to take a write lock to write to the context 1288 * to avoid races with storage keys handling, we check if the 1289 * value really needs to be written to; if the value is 1290 * already correct, we do nothing and avoid the lock. 1291 */ 1292 if (vcpu->kvm->mm->context.uses_cmm == 0) { 1293 mmap_write_lock(vcpu->kvm->mm); 1294 vcpu->kvm->mm->context.uses_cmm = 1; 1295 mmap_write_unlock(vcpu->kvm->mm); 1296 } 1297 /* 1298 * If we are here, we are supposed to have CMMA enabled in 1299 * the SIE block. Enabling CMMA works on a per-CPU basis, 1300 * while the context use_cmma flag is per process. 1301 * It's possible that the context flag is enabled and the 1302 * SIE flag is not, so we set the flag always; if it was 1303 * already set, nothing changes, otherwise we enable it 1304 * on this CPU too. 1305 */ 1306 vcpu->arch.sie_block->ecb2 |= ECB2_CMMA; 1307 /* Retry the ESSA instruction */ 1308 kvm_s390_retry_instr(vcpu); 1309 } else { 1310 mmap_read_lock(vcpu->kvm->mm); 1311 i = __do_essa(vcpu, orc); 1312 mmap_read_unlock(vcpu->kvm->mm); 1313 if (i < 0) 1314 return i; 1315 /* Account for the possible extra cbrl entry */ 1316 entries += i; 1317 } 1318 vcpu->arch.sie_block->cbrlo &= PAGE_MASK; /* reset nceo */ 1319 cbrlo = phys_to_virt(vcpu->arch.sie_block->cbrlo); 1320 mmap_read_lock(gmap->mm); 1321 for (i = 0; i < entries; ++i) 1322 __gmap_zap(gmap, cbrlo[i]); 1323 mmap_read_unlock(gmap->mm); 1324 return 0; 1325 } 1326 1327 int kvm_s390_handle_b9(struct kvm_vcpu *vcpu) 1328 { 1329 switch (vcpu->arch.sie_block->ipa & 0x00ff) { 1330 case 0x8a: 1331 case 0x8e: 1332 case 0x8f: 1333 return handle_ipte_interlock(vcpu); 1334 case 0x8d: 1335 return handle_epsw(vcpu); 1336 case 0xab: 1337 return handle_essa(vcpu); 1338 case 0xaf: 1339 return handle_pfmf(vcpu); 1340 default: 1341 return -EOPNOTSUPP; 1342 } 1343 } 1344 1345 int kvm_s390_handle_lctl(struct kvm_vcpu *vcpu) 1346 { 1347 int reg1 = (vcpu->arch.sie_block->ipa & 0x00f0) >> 4; 1348 int reg3 = vcpu->arch.sie_block->ipa & 0x000f; 1349 int reg, rc, nr_regs; 1350 u32 ctl_array[16]; 1351 u64 ga; 1352 u8 ar; 1353 1354 vcpu->stat.instruction_lctl++; 1355 1356 if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE) 1357 return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP); 1358 1359 ga = kvm_s390_get_base_disp_rs(vcpu, &ar); 1360 1361 if (ga & 3) 1362 return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION); 1363 1364 VCPU_EVENT(vcpu, 4, "LCTL: r1:%d, r3:%d, addr: 0x%llx", reg1, reg3, ga); 1365 trace_kvm_s390_handle_lctl(vcpu, 0, reg1, reg3, ga); 1366 1367 nr_regs = ((reg3 - reg1) & 0xf) + 1; 1368 rc = read_guest(vcpu, ga, ar, ctl_array, nr_regs * sizeof(u32)); 1369 if (rc) 1370 return kvm_s390_inject_prog_cond(vcpu, rc); 1371 reg = reg1; 1372 nr_regs = 0; 1373 do { 1374 vcpu->arch.sie_block->gcr[reg] &= 0xffffffff00000000ul; 1375 vcpu->arch.sie_block->gcr[reg] |= ctl_array[nr_regs++]; 1376 if (reg == reg3) 1377 break; 1378 reg = (reg + 1) % 16; 1379 } while (1); 1380 kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu); 1381 return 0; 1382 } 1383 1384 int kvm_s390_handle_stctl(struct kvm_vcpu *vcpu) 1385 { 1386 int reg1 = (vcpu->arch.sie_block->ipa & 0x00f0) >> 4; 1387 int reg3 = vcpu->arch.sie_block->ipa & 0x000f; 1388 int reg, rc, nr_regs; 1389 u32 ctl_array[16]; 1390 u64 ga; 1391 u8 ar; 1392 1393 vcpu->stat.instruction_stctl++; 1394 1395 if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE) 1396 return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP); 1397 1398 ga = kvm_s390_get_base_disp_rs(vcpu, &ar); 1399 1400 if (ga & 3) 1401 return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION); 1402 1403 VCPU_EVENT(vcpu, 4, "STCTL r1:%d, r3:%d, addr: 0x%llx", reg1, reg3, ga); 1404 trace_kvm_s390_handle_stctl(vcpu, 0, reg1, reg3, ga); 1405 1406 reg = reg1; 1407 nr_regs = 0; 1408 do { 1409 ctl_array[nr_regs++] = vcpu->arch.sie_block->gcr[reg]; 1410 if (reg == reg3) 1411 break; 1412 reg = (reg + 1) % 16; 1413 } while (1); 1414 rc = write_guest(vcpu, ga, ar, ctl_array, nr_regs * sizeof(u32)); 1415 return rc ? kvm_s390_inject_prog_cond(vcpu, rc) : 0; 1416 } 1417 1418 static int handle_lctlg(struct kvm_vcpu *vcpu) 1419 { 1420 int reg1 = (vcpu->arch.sie_block->ipa & 0x00f0) >> 4; 1421 int reg3 = vcpu->arch.sie_block->ipa & 0x000f; 1422 int reg, rc, nr_regs; 1423 u64 ctl_array[16]; 1424 u64 ga; 1425 u8 ar; 1426 1427 vcpu->stat.instruction_lctlg++; 1428 1429 if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE) 1430 return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP); 1431 1432 ga = kvm_s390_get_base_disp_rsy(vcpu, &ar); 1433 1434 if (ga & 7) 1435 return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION); 1436 1437 VCPU_EVENT(vcpu, 4, "LCTLG: r1:%d, r3:%d, addr: 0x%llx", reg1, reg3, ga); 1438 trace_kvm_s390_handle_lctl(vcpu, 1, reg1, reg3, ga); 1439 1440 nr_regs = ((reg3 - reg1) & 0xf) + 1; 1441 rc = read_guest(vcpu, ga, ar, ctl_array, nr_regs * sizeof(u64)); 1442 if (rc) 1443 return kvm_s390_inject_prog_cond(vcpu, rc); 1444 reg = reg1; 1445 nr_regs = 0; 1446 do { 1447 vcpu->arch.sie_block->gcr[reg] = ctl_array[nr_regs++]; 1448 if (reg == reg3) 1449 break; 1450 reg = (reg + 1) % 16; 1451 } while (1); 1452 kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu); 1453 return 0; 1454 } 1455 1456 static int handle_stctg(struct kvm_vcpu *vcpu) 1457 { 1458 int reg1 = (vcpu->arch.sie_block->ipa & 0x00f0) >> 4; 1459 int reg3 = vcpu->arch.sie_block->ipa & 0x000f; 1460 int reg, rc, nr_regs; 1461 u64 ctl_array[16]; 1462 u64 ga; 1463 u8 ar; 1464 1465 vcpu->stat.instruction_stctg++; 1466 1467 if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE) 1468 return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP); 1469 1470 ga = kvm_s390_get_base_disp_rsy(vcpu, &ar); 1471 1472 if (ga & 7) 1473 return kvm_s390_inject_program_int(vcpu, PGM_SPECIFICATION); 1474 1475 VCPU_EVENT(vcpu, 4, "STCTG r1:%d, r3:%d, addr: 0x%llx", reg1, reg3, ga); 1476 trace_kvm_s390_handle_stctl(vcpu, 1, reg1, reg3, ga); 1477 1478 reg = reg1; 1479 nr_regs = 0; 1480 do { 1481 ctl_array[nr_regs++] = vcpu->arch.sie_block->gcr[reg]; 1482 if (reg == reg3) 1483 break; 1484 reg = (reg + 1) % 16; 1485 } while (1); 1486 rc = write_guest(vcpu, ga, ar, ctl_array, nr_regs * sizeof(u64)); 1487 return rc ? kvm_s390_inject_prog_cond(vcpu, rc) : 0; 1488 } 1489 1490 int kvm_s390_handle_eb(struct kvm_vcpu *vcpu) 1491 { 1492 switch (vcpu->arch.sie_block->ipb & 0x000000ff) { 1493 case 0x25: 1494 return handle_stctg(vcpu); 1495 case 0x2f: 1496 return handle_lctlg(vcpu); 1497 case 0x60: 1498 case 0x61: 1499 case 0x62: 1500 return handle_ri(vcpu); 1501 case 0x71: 1502 return handle_lpswey(vcpu); 1503 default: 1504 return -EOPNOTSUPP; 1505 } 1506 } 1507 1508 static int handle_tprot(struct kvm_vcpu *vcpu) 1509 { 1510 u64 address, operand2; 1511 unsigned long gpa; 1512 u8 access_key; 1513 bool writable; 1514 int ret, cc; 1515 u8 ar; 1516 1517 vcpu->stat.instruction_tprot++; 1518 1519 if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE) 1520 return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP); 1521 1522 kvm_s390_get_base_disp_sse(vcpu, &address, &operand2, &ar, NULL); 1523 access_key = (operand2 & 0xf0) >> 4; 1524 1525 if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_DAT) 1526 ipte_lock(vcpu->kvm); 1527 1528 ret = guest_translate_address_with_key(vcpu, address, ar, &gpa, 1529 GACC_STORE, access_key); 1530 if (ret == 0) { 1531 gfn_to_hva_prot(vcpu->kvm, gpa_to_gfn(gpa), &writable); 1532 } else if (ret == PGM_PROTECTION) { 1533 writable = false; 1534 /* Write protected? Try again with read-only... */ 1535 ret = guest_translate_address_with_key(vcpu, address, ar, &gpa, 1536 GACC_FETCH, access_key); 1537 } 1538 if (ret >= 0) { 1539 cc = -1; 1540 1541 /* Fetching permitted; storing permitted */ 1542 if (ret == 0 && writable) 1543 cc = 0; 1544 /* Fetching permitted; storing not permitted */ 1545 else if (ret == 0 && !writable) 1546 cc = 1; 1547 /* Fetching not permitted; storing not permitted */ 1548 else if (ret == PGM_PROTECTION) 1549 cc = 2; 1550 /* Translation not available */ 1551 else if (ret != PGM_ADDRESSING && ret != PGM_TRANSLATION_SPEC) 1552 cc = 3; 1553 1554 if (cc != -1) { 1555 kvm_s390_set_psw_cc(vcpu, cc); 1556 ret = 0; 1557 } else { 1558 ret = kvm_s390_inject_program_int(vcpu, ret); 1559 } 1560 } 1561 1562 if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_DAT) 1563 ipte_unlock(vcpu->kvm); 1564 return ret; 1565 } 1566 1567 int kvm_s390_handle_e5(struct kvm_vcpu *vcpu) 1568 { 1569 switch (vcpu->arch.sie_block->ipa & 0x00ff) { 1570 case 0x01: 1571 return handle_tprot(vcpu); 1572 default: 1573 return -EOPNOTSUPP; 1574 } 1575 } 1576 1577 static int handle_sckpf(struct kvm_vcpu *vcpu) 1578 { 1579 u32 value; 1580 1581 vcpu->stat.instruction_sckpf++; 1582 1583 if (vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE) 1584 return kvm_s390_inject_program_int(vcpu, PGM_PRIVILEGED_OP); 1585 1586 if (vcpu->run->s.regs.gprs[0] & 0x00000000ffff0000) 1587 return kvm_s390_inject_program_int(vcpu, 1588 PGM_SPECIFICATION); 1589 1590 value = vcpu->run->s.regs.gprs[0] & 0x000000000000ffff; 1591 vcpu->arch.sie_block->todpr = value; 1592 1593 return 0; 1594 } 1595 1596 static int handle_ptff(struct kvm_vcpu *vcpu) 1597 { 1598 vcpu->stat.instruction_ptff++; 1599 1600 /* we don't emulate any control instructions yet */ 1601 kvm_s390_set_psw_cc(vcpu, 3); 1602 return 0; 1603 } 1604 1605 int kvm_s390_handle_01(struct kvm_vcpu *vcpu) 1606 { 1607 switch (vcpu->arch.sie_block->ipa & 0x00ff) { 1608 case 0x04: 1609 return handle_ptff(vcpu); 1610 case 0x07: 1611 return handle_sckpf(vcpu); 1612 default: 1613 return -EOPNOTSUPP; 1614 } 1615 } 1616