1 // SPDX-License-Identifier: GPL-2.0 2 /* 3 * hosting IBM Z kernel virtual machines (s390x) 4 * 5 * Copyright IBM Corp. 2008, 2017 6 * 7 * Author(s): Carsten Otte <cotte@de.ibm.com> 8 * Christian Borntraeger <borntraeger@de.ibm.com> 9 * Heiko Carstens <heiko.carstens@de.ibm.com> 10 * Christian Ehrhardt <ehrhardt@de.ibm.com> 11 * Jason J. Herne <jjherne@us.ibm.com> 12 */ 13 14 #include <linux/compiler.h> 15 #include <linux/err.h> 16 #include <linux/fs.h> 17 #include <linux/hrtimer.h> 18 #include <linux/init.h> 19 #include <linux/kvm.h> 20 #include <linux/kvm_host.h> 21 #include <linux/mman.h> 22 #include <linux/module.h> 23 #include <linux/moduleparam.h> 24 #include <linux/random.h> 25 #include <linux/slab.h> 26 #include <linux/timer.h> 27 #include <linux/vmalloc.h> 28 #include <linux/bitmap.h> 29 #include <linux/sched/signal.h> 30 #include <linux/string.h> 31 32 #include <asm/asm-offsets.h> 33 #include <asm/lowcore.h> 34 #include <asm/stp.h> 35 #include <asm/pgtable.h> 36 #include <asm/gmap.h> 37 #include <asm/nmi.h> 38 #include <asm/switch_to.h> 39 #include <asm/isc.h> 40 #include <asm/sclp.h> 41 #include <asm/cpacf.h> 42 #include <asm/timex.h> 43 #include "kvm-s390.h" 44 #include "gaccess.h" 45 46 #define KMSG_COMPONENT "kvm-s390" 47 #undef pr_fmt 48 #define pr_fmt(fmt) KMSG_COMPONENT ": " fmt 49 50 #define CREATE_TRACE_POINTS 51 #include "trace.h" 52 #include "trace-s390.h" 53 54 #define MEM_OP_MAX_SIZE 65536 /* Maximum transfer size for KVM_S390_MEM_OP */ 55 #define LOCAL_IRQS 32 56 #define VCPU_IRQS_MAX_BUF (sizeof(struct kvm_s390_irq) * \ 57 (KVM_MAX_VCPUS + LOCAL_IRQS)) 58 59 #define VCPU_STAT(x) offsetof(struct kvm_vcpu, stat.x), KVM_STAT_VCPU 60 61 struct kvm_stats_debugfs_item debugfs_entries[] = { 62 { "userspace_handled", VCPU_STAT(exit_userspace) }, 63 { "exit_null", VCPU_STAT(exit_null) }, 64 { "exit_validity", VCPU_STAT(exit_validity) }, 65 { "exit_stop_request", VCPU_STAT(exit_stop_request) }, 66 { "exit_external_request", VCPU_STAT(exit_external_request) }, 67 { "exit_external_interrupt", VCPU_STAT(exit_external_interrupt) }, 68 { "exit_instruction", VCPU_STAT(exit_instruction) }, 69 { "exit_pei", VCPU_STAT(exit_pei) }, 70 { "exit_program_interruption", VCPU_STAT(exit_program_interruption) }, 71 { "exit_instr_and_program_int", VCPU_STAT(exit_instr_and_program) }, 72 { "exit_operation_exception", VCPU_STAT(exit_operation_exception) }, 73 { "halt_successful_poll", VCPU_STAT(halt_successful_poll) }, 74 { "halt_attempted_poll", VCPU_STAT(halt_attempted_poll) }, 75 { "halt_poll_invalid", VCPU_STAT(halt_poll_invalid) }, 76 { "halt_wakeup", VCPU_STAT(halt_wakeup) }, 77 { "instruction_lctlg", VCPU_STAT(instruction_lctlg) }, 78 { "instruction_lctl", VCPU_STAT(instruction_lctl) }, 79 { "instruction_stctl", VCPU_STAT(instruction_stctl) }, 80 { "instruction_stctg", VCPU_STAT(instruction_stctg) }, 81 { "deliver_emergency_signal", VCPU_STAT(deliver_emergency_signal) }, 82 { "deliver_external_call", VCPU_STAT(deliver_external_call) }, 83 { "deliver_service_signal", VCPU_STAT(deliver_service_signal) }, 84 { "deliver_virtio_interrupt", VCPU_STAT(deliver_virtio_interrupt) }, 85 { "deliver_stop_signal", VCPU_STAT(deliver_stop_signal) }, 86 { "deliver_prefix_signal", VCPU_STAT(deliver_prefix_signal) }, 87 { "deliver_restart_signal", VCPU_STAT(deliver_restart_signal) }, 88 { "deliver_program_interruption", VCPU_STAT(deliver_program_int) }, 89 { "exit_wait_state", VCPU_STAT(exit_wait_state) }, 90 { "instruction_pfmf", VCPU_STAT(instruction_pfmf) }, 91 { "instruction_stidp", VCPU_STAT(instruction_stidp) }, 92 { "instruction_spx", VCPU_STAT(instruction_spx) }, 93 { "instruction_stpx", VCPU_STAT(instruction_stpx) }, 94 { "instruction_stap", VCPU_STAT(instruction_stap) }, 95 { "instruction_storage_key", VCPU_STAT(instruction_storage_key) }, 96 { "instruction_ipte_interlock", VCPU_STAT(instruction_ipte_interlock) }, 97 { "instruction_stsch", VCPU_STAT(instruction_stsch) }, 98 { "instruction_chsc", VCPU_STAT(instruction_chsc) }, 99 { "instruction_essa", VCPU_STAT(instruction_essa) }, 100 { "instruction_stsi", VCPU_STAT(instruction_stsi) }, 101 { "instruction_stfl", VCPU_STAT(instruction_stfl) }, 102 { "instruction_tprot", VCPU_STAT(instruction_tprot) }, 103 { "instruction_sthyi", VCPU_STAT(instruction_sthyi) }, 104 { "instruction_sie", VCPU_STAT(instruction_sie) }, 105 { "instruction_sigp_sense", VCPU_STAT(instruction_sigp_sense) }, 106 { "instruction_sigp_sense_running", VCPU_STAT(instruction_sigp_sense_running) }, 107 { "instruction_sigp_external_call", VCPU_STAT(instruction_sigp_external_call) }, 108 { "instruction_sigp_emergency", VCPU_STAT(instruction_sigp_emergency) }, 109 { "instruction_sigp_cond_emergency", VCPU_STAT(instruction_sigp_cond_emergency) }, 110 { "instruction_sigp_start", VCPU_STAT(instruction_sigp_start) }, 111 { "instruction_sigp_stop", VCPU_STAT(instruction_sigp_stop) }, 112 { "instruction_sigp_stop_store_status", VCPU_STAT(instruction_sigp_stop_store_status) }, 113 { "instruction_sigp_store_status", VCPU_STAT(instruction_sigp_store_status) }, 114 { "instruction_sigp_store_adtl_status", VCPU_STAT(instruction_sigp_store_adtl_status) }, 115 { "instruction_sigp_set_arch", VCPU_STAT(instruction_sigp_arch) }, 116 { "instruction_sigp_set_prefix", VCPU_STAT(instruction_sigp_prefix) }, 117 { "instruction_sigp_restart", VCPU_STAT(instruction_sigp_restart) }, 118 { "instruction_sigp_cpu_reset", VCPU_STAT(instruction_sigp_cpu_reset) }, 119 { "instruction_sigp_init_cpu_reset", VCPU_STAT(instruction_sigp_init_cpu_reset) }, 120 { "instruction_sigp_unknown", VCPU_STAT(instruction_sigp_unknown) }, 121 { "diagnose_10", VCPU_STAT(diagnose_10) }, 122 { "diagnose_44", VCPU_STAT(diagnose_44) }, 123 { "diagnose_9c", VCPU_STAT(diagnose_9c) }, 124 { "diagnose_258", VCPU_STAT(diagnose_258) }, 125 { "diagnose_308", VCPU_STAT(diagnose_308) }, 126 { "diagnose_500", VCPU_STAT(diagnose_500) }, 127 { NULL } 128 }; 129 130 struct kvm_s390_tod_clock_ext { 131 __u8 epoch_idx; 132 __u64 tod; 133 __u8 reserved[7]; 134 } __packed; 135 136 /* allow nested virtualization in KVM (if enabled by user space) */ 137 static int nested; 138 module_param(nested, int, S_IRUGO); 139 MODULE_PARM_DESC(nested, "Nested virtualization support"); 140 141 /* upper facilities limit for kvm */ 142 unsigned long kvm_s390_fac_list_mask[16] = { FACILITIES_KVM }; 143 144 unsigned long kvm_s390_fac_list_mask_size(void) 145 { 146 BUILD_BUG_ON(ARRAY_SIZE(kvm_s390_fac_list_mask) > S390_ARCH_FAC_MASK_SIZE_U64); 147 return ARRAY_SIZE(kvm_s390_fac_list_mask); 148 } 149 150 /* available cpu features supported by kvm */ 151 static DECLARE_BITMAP(kvm_s390_available_cpu_feat, KVM_S390_VM_CPU_FEAT_NR_BITS); 152 /* available subfunctions indicated via query / "test bit" */ 153 static struct kvm_s390_vm_cpu_subfunc kvm_s390_available_subfunc; 154 155 static struct gmap_notifier gmap_notifier; 156 static struct gmap_notifier vsie_gmap_notifier; 157 debug_info_t *kvm_s390_dbf; 158 159 /* Section: not file related */ 160 int kvm_arch_hardware_enable(void) 161 { 162 /* every s390 is virtualization enabled ;-) */ 163 return 0; 164 } 165 166 static void kvm_gmap_notifier(struct gmap *gmap, unsigned long start, 167 unsigned long end); 168 169 /* 170 * This callback is executed during stop_machine(). All CPUs are therefore 171 * temporarily stopped. In order not to change guest behavior, we have to 172 * disable preemption whenever we touch the epoch of kvm and the VCPUs, 173 * so a CPU won't be stopped while calculating with the epoch. 174 */ 175 static int kvm_clock_sync(struct notifier_block *notifier, unsigned long val, 176 void *v) 177 { 178 struct kvm *kvm; 179 struct kvm_vcpu *vcpu; 180 int i; 181 unsigned long long *delta = v; 182 183 list_for_each_entry(kvm, &vm_list, vm_list) { 184 kvm->arch.epoch -= *delta; 185 kvm_for_each_vcpu(i, vcpu, kvm) { 186 vcpu->arch.sie_block->epoch -= *delta; 187 if (vcpu->arch.cputm_enabled) 188 vcpu->arch.cputm_start += *delta; 189 if (vcpu->arch.vsie_block) 190 vcpu->arch.vsie_block->epoch -= *delta; 191 } 192 } 193 return NOTIFY_OK; 194 } 195 196 static struct notifier_block kvm_clock_notifier = { 197 .notifier_call = kvm_clock_sync, 198 }; 199 200 int kvm_arch_hardware_setup(void) 201 { 202 gmap_notifier.notifier_call = kvm_gmap_notifier; 203 gmap_register_pte_notifier(&gmap_notifier); 204 vsie_gmap_notifier.notifier_call = kvm_s390_vsie_gmap_notifier; 205 gmap_register_pte_notifier(&vsie_gmap_notifier); 206 atomic_notifier_chain_register(&s390_epoch_delta_notifier, 207 &kvm_clock_notifier); 208 return 0; 209 } 210 211 void kvm_arch_hardware_unsetup(void) 212 { 213 gmap_unregister_pte_notifier(&gmap_notifier); 214 gmap_unregister_pte_notifier(&vsie_gmap_notifier); 215 atomic_notifier_chain_unregister(&s390_epoch_delta_notifier, 216 &kvm_clock_notifier); 217 } 218 219 static void allow_cpu_feat(unsigned long nr) 220 { 221 set_bit_inv(nr, kvm_s390_available_cpu_feat); 222 } 223 224 static inline int plo_test_bit(unsigned char nr) 225 { 226 register unsigned long r0 asm("0") = (unsigned long) nr | 0x100; 227 int cc; 228 229 asm volatile( 230 /* Parameter registers are ignored for "test bit" */ 231 " plo 0,0,0,0(0)\n" 232 " ipm %0\n" 233 " srl %0,28\n" 234 : "=d" (cc) 235 : "d" (r0) 236 : "cc"); 237 return cc == 0; 238 } 239 240 static void kvm_s390_cpu_feat_init(void) 241 { 242 int i; 243 244 for (i = 0; i < 256; ++i) { 245 if (plo_test_bit(i)) 246 kvm_s390_available_subfunc.plo[i >> 3] |= 0x80 >> (i & 7); 247 } 248 249 if (test_facility(28)) /* TOD-clock steering */ 250 ptff(kvm_s390_available_subfunc.ptff, 251 sizeof(kvm_s390_available_subfunc.ptff), 252 PTFF_QAF); 253 254 if (test_facility(17)) { /* MSA */ 255 __cpacf_query(CPACF_KMAC, (cpacf_mask_t *) 256 kvm_s390_available_subfunc.kmac); 257 __cpacf_query(CPACF_KMC, (cpacf_mask_t *) 258 kvm_s390_available_subfunc.kmc); 259 __cpacf_query(CPACF_KM, (cpacf_mask_t *) 260 kvm_s390_available_subfunc.km); 261 __cpacf_query(CPACF_KIMD, (cpacf_mask_t *) 262 kvm_s390_available_subfunc.kimd); 263 __cpacf_query(CPACF_KLMD, (cpacf_mask_t *) 264 kvm_s390_available_subfunc.klmd); 265 } 266 if (test_facility(76)) /* MSA3 */ 267 __cpacf_query(CPACF_PCKMO, (cpacf_mask_t *) 268 kvm_s390_available_subfunc.pckmo); 269 if (test_facility(77)) { /* MSA4 */ 270 __cpacf_query(CPACF_KMCTR, (cpacf_mask_t *) 271 kvm_s390_available_subfunc.kmctr); 272 __cpacf_query(CPACF_KMF, (cpacf_mask_t *) 273 kvm_s390_available_subfunc.kmf); 274 __cpacf_query(CPACF_KMO, (cpacf_mask_t *) 275 kvm_s390_available_subfunc.kmo); 276 __cpacf_query(CPACF_PCC, (cpacf_mask_t *) 277 kvm_s390_available_subfunc.pcc); 278 } 279 if (test_facility(57)) /* MSA5 */ 280 __cpacf_query(CPACF_PRNO, (cpacf_mask_t *) 281 kvm_s390_available_subfunc.ppno); 282 283 if (test_facility(146)) /* MSA8 */ 284 __cpacf_query(CPACF_KMA, (cpacf_mask_t *) 285 kvm_s390_available_subfunc.kma); 286 287 if (MACHINE_HAS_ESOP) 288 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_ESOP); 289 /* 290 * We need SIE support, ESOP (PROT_READ protection for gmap_shadow), 291 * 64bit SCAO (SCA passthrough) and IDTE (for gmap_shadow unshadowing). 292 */ 293 if (!sclp.has_sief2 || !MACHINE_HAS_ESOP || !sclp.has_64bscao || 294 !test_facility(3) || !nested) 295 return; 296 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_SIEF2); 297 if (sclp.has_64bscao) 298 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_64BSCAO); 299 if (sclp.has_siif) 300 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_SIIF); 301 if (sclp.has_gpere) 302 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_GPERE); 303 if (sclp.has_gsls) 304 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_GSLS); 305 if (sclp.has_ib) 306 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_IB); 307 if (sclp.has_cei) 308 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_CEI); 309 if (sclp.has_ibs) 310 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_IBS); 311 if (sclp.has_kss) 312 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_KSS); 313 /* 314 * KVM_S390_VM_CPU_FEAT_SKEY: Wrong shadow of PTE.I bits will make 315 * all skey handling functions read/set the skey from the PGSTE 316 * instead of the real storage key. 317 * 318 * KVM_S390_VM_CPU_FEAT_CMMA: Wrong shadow of PTE.I bits will make 319 * pages being detected as preserved although they are resident. 320 * 321 * KVM_S390_VM_CPU_FEAT_PFMFI: Wrong shadow of PTE.I bits will 322 * have the same effect as for KVM_S390_VM_CPU_FEAT_SKEY. 323 * 324 * For KVM_S390_VM_CPU_FEAT_SKEY, KVM_S390_VM_CPU_FEAT_CMMA and 325 * KVM_S390_VM_CPU_FEAT_PFMFI, all PTE.I and PGSTE bits have to be 326 * correctly shadowed. We can do that for the PGSTE but not for PTE.I. 327 * 328 * KVM_S390_VM_CPU_FEAT_SIGPIF: Wrong SCB addresses in the SCA. We 329 * cannot easily shadow the SCA because of the ipte lock. 330 */ 331 } 332 333 int kvm_arch_init(void *opaque) 334 { 335 kvm_s390_dbf = debug_register("kvm-trace", 32, 1, 7 * sizeof(long)); 336 if (!kvm_s390_dbf) 337 return -ENOMEM; 338 339 if (debug_register_view(kvm_s390_dbf, &debug_sprintf_view)) { 340 debug_unregister(kvm_s390_dbf); 341 return -ENOMEM; 342 } 343 344 kvm_s390_cpu_feat_init(); 345 346 /* Register floating interrupt controller interface. */ 347 return kvm_register_device_ops(&kvm_flic_ops, KVM_DEV_TYPE_FLIC); 348 } 349 350 void kvm_arch_exit(void) 351 { 352 debug_unregister(kvm_s390_dbf); 353 } 354 355 /* Section: device related */ 356 long kvm_arch_dev_ioctl(struct file *filp, 357 unsigned int ioctl, unsigned long arg) 358 { 359 if (ioctl == KVM_S390_ENABLE_SIE) 360 return s390_enable_sie(); 361 return -EINVAL; 362 } 363 364 int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext) 365 { 366 int r; 367 368 switch (ext) { 369 case KVM_CAP_S390_PSW: 370 case KVM_CAP_S390_GMAP: 371 case KVM_CAP_SYNC_MMU: 372 #ifdef CONFIG_KVM_S390_UCONTROL 373 case KVM_CAP_S390_UCONTROL: 374 #endif 375 case KVM_CAP_ASYNC_PF: 376 case KVM_CAP_SYNC_REGS: 377 case KVM_CAP_ONE_REG: 378 case KVM_CAP_ENABLE_CAP: 379 case KVM_CAP_S390_CSS_SUPPORT: 380 case KVM_CAP_IOEVENTFD: 381 case KVM_CAP_DEVICE_CTRL: 382 case KVM_CAP_ENABLE_CAP_VM: 383 case KVM_CAP_S390_IRQCHIP: 384 case KVM_CAP_VM_ATTRIBUTES: 385 case KVM_CAP_MP_STATE: 386 case KVM_CAP_IMMEDIATE_EXIT: 387 case KVM_CAP_S390_INJECT_IRQ: 388 case KVM_CAP_S390_USER_SIGP: 389 case KVM_CAP_S390_USER_STSI: 390 case KVM_CAP_S390_SKEYS: 391 case KVM_CAP_S390_IRQ_STATE: 392 case KVM_CAP_S390_USER_INSTR0: 393 case KVM_CAP_S390_CMMA_MIGRATION: 394 case KVM_CAP_S390_AIS: 395 case KVM_CAP_S390_AIS_MIGRATION: 396 r = 1; 397 break; 398 case KVM_CAP_S390_MEM_OP: 399 r = MEM_OP_MAX_SIZE; 400 break; 401 case KVM_CAP_NR_VCPUS: 402 case KVM_CAP_MAX_VCPUS: 403 r = KVM_S390_BSCA_CPU_SLOTS; 404 if (!kvm_s390_use_sca_entries()) 405 r = KVM_MAX_VCPUS; 406 else if (sclp.has_esca && sclp.has_64bscao) 407 r = KVM_S390_ESCA_CPU_SLOTS; 408 break; 409 case KVM_CAP_NR_MEMSLOTS: 410 r = KVM_USER_MEM_SLOTS; 411 break; 412 case KVM_CAP_S390_COW: 413 r = MACHINE_HAS_ESOP; 414 break; 415 case KVM_CAP_S390_VECTOR_REGISTERS: 416 r = MACHINE_HAS_VX; 417 break; 418 case KVM_CAP_S390_RI: 419 r = test_facility(64); 420 break; 421 case KVM_CAP_S390_GS: 422 r = test_facility(133); 423 break; 424 case KVM_CAP_S390_BPB: 425 r = test_facility(82); 426 break; 427 default: 428 r = 0; 429 } 430 return r; 431 } 432 433 static void kvm_s390_sync_dirty_log(struct kvm *kvm, 434 struct kvm_memory_slot *memslot) 435 { 436 gfn_t cur_gfn, last_gfn; 437 unsigned long address; 438 struct gmap *gmap = kvm->arch.gmap; 439 440 /* Loop over all guest pages */ 441 last_gfn = memslot->base_gfn + memslot->npages; 442 for (cur_gfn = memslot->base_gfn; cur_gfn <= last_gfn; cur_gfn++) { 443 address = gfn_to_hva_memslot(memslot, cur_gfn); 444 445 if (test_and_clear_guest_dirty(gmap->mm, address)) 446 mark_page_dirty(kvm, cur_gfn); 447 if (fatal_signal_pending(current)) 448 return; 449 cond_resched(); 450 } 451 } 452 453 /* Section: vm related */ 454 static void sca_del_vcpu(struct kvm_vcpu *vcpu); 455 456 /* 457 * Get (and clear) the dirty memory log for a memory slot. 458 */ 459 int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm, 460 struct kvm_dirty_log *log) 461 { 462 int r; 463 unsigned long n; 464 struct kvm_memslots *slots; 465 struct kvm_memory_slot *memslot; 466 int is_dirty = 0; 467 468 if (kvm_is_ucontrol(kvm)) 469 return -EINVAL; 470 471 mutex_lock(&kvm->slots_lock); 472 473 r = -EINVAL; 474 if (log->slot >= KVM_USER_MEM_SLOTS) 475 goto out; 476 477 slots = kvm_memslots(kvm); 478 memslot = id_to_memslot(slots, log->slot); 479 r = -ENOENT; 480 if (!memslot->dirty_bitmap) 481 goto out; 482 483 kvm_s390_sync_dirty_log(kvm, memslot); 484 r = kvm_get_dirty_log(kvm, log, &is_dirty); 485 if (r) 486 goto out; 487 488 /* Clear the dirty log */ 489 if (is_dirty) { 490 n = kvm_dirty_bitmap_bytes(memslot); 491 memset(memslot->dirty_bitmap, 0, n); 492 } 493 r = 0; 494 out: 495 mutex_unlock(&kvm->slots_lock); 496 return r; 497 } 498 499 static void icpt_operexc_on_all_vcpus(struct kvm *kvm) 500 { 501 unsigned int i; 502 struct kvm_vcpu *vcpu; 503 504 kvm_for_each_vcpu(i, vcpu, kvm) { 505 kvm_s390_sync_request(KVM_REQ_ICPT_OPEREXC, vcpu); 506 } 507 } 508 509 static int kvm_vm_ioctl_enable_cap(struct kvm *kvm, struct kvm_enable_cap *cap) 510 { 511 int r; 512 513 if (cap->flags) 514 return -EINVAL; 515 516 switch (cap->cap) { 517 case KVM_CAP_S390_IRQCHIP: 518 VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_IRQCHIP"); 519 kvm->arch.use_irqchip = 1; 520 r = 0; 521 break; 522 case KVM_CAP_S390_USER_SIGP: 523 VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_USER_SIGP"); 524 kvm->arch.user_sigp = 1; 525 r = 0; 526 break; 527 case KVM_CAP_S390_VECTOR_REGISTERS: 528 mutex_lock(&kvm->lock); 529 if (kvm->created_vcpus) { 530 r = -EBUSY; 531 } else if (MACHINE_HAS_VX) { 532 set_kvm_facility(kvm->arch.model.fac_mask, 129); 533 set_kvm_facility(kvm->arch.model.fac_list, 129); 534 if (test_facility(134)) { 535 set_kvm_facility(kvm->arch.model.fac_mask, 134); 536 set_kvm_facility(kvm->arch.model.fac_list, 134); 537 } 538 if (test_facility(135)) { 539 set_kvm_facility(kvm->arch.model.fac_mask, 135); 540 set_kvm_facility(kvm->arch.model.fac_list, 135); 541 } 542 r = 0; 543 } else 544 r = -EINVAL; 545 mutex_unlock(&kvm->lock); 546 VM_EVENT(kvm, 3, "ENABLE: CAP_S390_VECTOR_REGISTERS %s", 547 r ? "(not available)" : "(success)"); 548 break; 549 case KVM_CAP_S390_RI: 550 r = -EINVAL; 551 mutex_lock(&kvm->lock); 552 if (kvm->created_vcpus) { 553 r = -EBUSY; 554 } else if (test_facility(64)) { 555 set_kvm_facility(kvm->arch.model.fac_mask, 64); 556 set_kvm_facility(kvm->arch.model.fac_list, 64); 557 r = 0; 558 } 559 mutex_unlock(&kvm->lock); 560 VM_EVENT(kvm, 3, "ENABLE: CAP_S390_RI %s", 561 r ? "(not available)" : "(success)"); 562 break; 563 case KVM_CAP_S390_AIS: 564 mutex_lock(&kvm->lock); 565 if (kvm->created_vcpus) { 566 r = -EBUSY; 567 } else { 568 set_kvm_facility(kvm->arch.model.fac_mask, 72); 569 set_kvm_facility(kvm->arch.model.fac_list, 72); 570 r = 0; 571 } 572 mutex_unlock(&kvm->lock); 573 VM_EVENT(kvm, 3, "ENABLE: AIS %s", 574 r ? "(not available)" : "(success)"); 575 break; 576 case KVM_CAP_S390_GS: 577 r = -EINVAL; 578 mutex_lock(&kvm->lock); 579 if (atomic_read(&kvm->online_vcpus)) { 580 r = -EBUSY; 581 } else if (test_facility(133)) { 582 set_kvm_facility(kvm->arch.model.fac_mask, 133); 583 set_kvm_facility(kvm->arch.model.fac_list, 133); 584 r = 0; 585 } 586 mutex_unlock(&kvm->lock); 587 VM_EVENT(kvm, 3, "ENABLE: CAP_S390_GS %s", 588 r ? "(not available)" : "(success)"); 589 break; 590 case KVM_CAP_S390_USER_STSI: 591 VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_USER_STSI"); 592 kvm->arch.user_stsi = 1; 593 r = 0; 594 break; 595 case KVM_CAP_S390_USER_INSTR0: 596 VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_USER_INSTR0"); 597 kvm->arch.user_instr0 = 1; 598 icpt_operexc_on_all_vcpus(kvm); 599 r = 0; 600 break; 601 default: 602 r = -EINVAL; 603 break; 604 } 605 return r; 606 } 607 608 static int kvm_s390_get_mem_control(struct kvm *kvm, struct kvm_device_attr *attr) 609 { 610 int ret; 611 612 switch (attr->attr) { 613 case KVM_S390_VM_MEM_LIMIT_SIZE: 614 ret = 0; 615 VM_EVENT(kvm, 3, "QUERY: max guest memory: %lu bytes", 616 kvm->arch.mem_limit); 617 if (put_user(kvm->arch.mem_limit, (u64 __user *)attr->addr)) 618 ret = -EFAULT; 619 break; 620 default: 621 ret = -ENXIO; 622 break; 623 } 624 return ret; 625 } 626 627 static int kvm_s390_set_mem_control(struct kvm *kvm, struct kvm_device_attr *attr) 628 { 629 int ret; 630 unsigned int idx; 631 switch (attr->attr) { 632 case KVM_S390_VM_MEM_ENABLE_CMMA: 633 ret = -ENXIO; 634 if (!sclp.has_cmma) 635 break; 636 637 ret = -EBUSY; 638 VM_EVENT(kvm, 3, "%s", "ENABLE: CMMA support"); 639 mutex_lock(&kvm->lock); 640 if (!kvm->created_vcpus) { 641 kvm->arch.use_cmma = 1; 642 ret = 0; 643 } 644 mutex_unlock(&kvm->lock); 645 break; 646 case KVM_S390_VM_MEM_CLR_CMMA: 647 ret = -ENXIO; 648 if (!sclp.has_cmma) 649 break; 650 ret = -EINVAL; 651 if (!kvm->arch.use_cmma) 652 break; 653 654 VM_EVENT(kvm, 3, "%s", "RESET: CMMA states"); 655 mutex_lock(&kvm->lock); 656 idx = srcu_read_lock(&kvm->srcu); 657 s390_reset_cmma(kvm->arch.gmap->mm); 658 srcu_read_unlock(&kvm->srcu, idx); 659 mutex_unlock(&kvm->lock); 660 ret = 0; 661 break; 662 case KVM_S390_VM_MEM_LIMIT_SIZE: { 663 unsigned long new_limit; 664 665 if (kvm_is_ucontrol(kvm)) 666 return -EINVAL; 667 668 if (get_user(new_limit, (u64 __user *)attr->addr)) 669 return -EFAULT; 670 671 if (kvm->arch.mem_limit != KVM_S390_NO_MEM_LIMIT && 672 new_limit > kvm->arch.mem_limit) 673 return -E2BIG; 674 675 if (!new_limit) 676 return -EINVAL; 677 678 /* gmap_create takes last usable address */ 679 if (new_limit != KVM_S390_NO_MEM_LIMIT) 680 new_limit -= 1; 681 682 ret = -EBUSY; 683 mutex_lock(&kvm->lock); 684 if (!kvm->created_vcpus) { 685 /* gmap_create will round the limit up */ 686 struct gmap *new = gmap_create(current->mm, new_limit); 687 688 if (!new) { 689 ret = -ENOMEM; 690 } else { 691 gmap_remove(kvm->arch.gmap); 692 new->private = kvm; 693 kvm->arch.gmap = new; 694 ret = 0; 695 } 696 } 697 mutex_unlock(&kvm->lock); 698 VM_EVENT(kvm, 3, "SET: max guest address: %lu", new_limit); 699 VM_EVENT(kvm, 3, "New guest asce: 0x%pK", 700 (void *) kvm->arch.gmap->asce); 701 break; 702 } 703 default: 704 ret = -ENXIO; 705 break; 706 } 707 return ret; 708 } 709 710 static void kvm_s390_vcpu_crypto_setup(struct kvm_vcpu *vcpu); 711 712 static int kvm_s390_vm_set_crypto(struct kvm *kvm, struct kvm_device_attr *attr) 713 { 714 struct kvm_vcpu *vcpu; 715 int i; 716 717 if (!test_kvm_facility(kvm, 76)) 718 return -EINVAL; 719 720 mutex_lock(&kvm->lock); 721 switch (attr->attr) { 722 case KVM_S390_VM_CRYPTO_ENABLE_AES_KW: 723 get_random_bytes( 724 kvm->arch.crypto.crycb->aes_wrapping_key_mask, 725 sizeof(kvm->arch.crypto.crycb->aes_wrapping_key_mask)); 726 kvm->arch.crypto.aes_kw = 1; 727 VM_EVENT(kvm, 3, "%s", "ENABLE: AES keywrapping support"); 728 break; 729 case KVM_S390_VM_CRYPTO_ENABLE_DEA_KW: 730 get_random_bytes( 731 kvm->arch.crypto.crycb->dea_wrapping_key_mask, 732 sizeof(kvm->arch.crypto.crycb->dea_wrapping_key_mask)); 733 kvm->arch.crypto.dea_kw = 1; 734 VM_EVENT(kvm, 3, "%s", "ENABLE: DEA keywrapping support"); 735 break; 736 case KVM_S390_VM_CRYPTO_DISABLE_AES_KW: 737 kvm->arch.crypto.aes_kw = 0; 738 memset(kvm->arch.crypto.crycb->aes_wrapping_key_mask, 0, 739 sizeof(kvm->arch.crypto.crycb->aes_wrapping_key_mask)); 740 VM_EVENT(kvm, 3, "%s", "DISABLE: AES keywrapping support"); 741 break; 742 case KVM_S390_VM_CRYPTO_DISABLE_DEA_KW: 743 kvm->arch.crypto.dea_kw = 0; 744 memset(kvm->arch.crypto.crycb->dea_wrapping_key_mask, 0, 745 sizeof(kvm->arch.crypto.crycb->dea_wrapping_key_mask)); 746 VM_EVENT(kvm, 3, "%s", "DISABLE: DEA keywrapping support"); 747 break; 748 default: 749 mutex_unlock(&kvm->lock); 750 return -ENXIO; 751 } 752 753 kvm_for_each_vcpu(i, vcpu, kvm) { 754 kvm_s390_vcpu_crypto_setup(vcpu); 755 exit_sie(vcpu); 756 } 757 mutex_unlock(&kvm->lock); 758 return 0; 759 } 760 761 static void kvm_s390_sync_request_broadcast(struct kvm *kvm, int req) 762 { 763 int cx; 764 struct kvm_vcpu *vcpu; 765 766 kvm_for_each_vcpu(cx, vcpu, kvm) 767 kvm_s390_sync_request(req, vcpu); 768 } 769 770 /* 771 * Must be called with kvm->srcu held to avoid races on memslots, and with 772 * kvm->slots_lock to avoid races with ourselves and kvm_s390_vm_stop_migration. 773 */ 774 static int kvm_s390_vm_start_migration(struct kvm *kvm) 775 { 776 struct kvm_s390_migration_state *mgs; 777 struct kvm_memory_slot *ms; 778 /* should be the only one */ 779 struct kvm_memslots *slots; 780 unsigned long ram_pages; 781 int slotnr; 782 783 /* migration mode already enabled */ 784 if (kvm->arch.migration_state) 785 return 0; 786 787 slots = kvm_memslots(kvm); 788 if (!slots || !slots->used_slots) 789 return -EINVAL; 790 791 mgs = kzalloc(sizeof(*mgs), GFP_KERNEL); 792 if (!mgs) 793 return -ENOMEM; 794 kvm->arch.migration_state = mgs; 795 796 if (kvm->arch.use_cmma) { 797 /* 798 * Get the first slot. They are reverse sorted by base_gfn, so 799 * the first slot is also the one at the end of the address 800 * space. We have verified above that at least one slot is 801 * present. 802 */ 803 ms = slots->memslots; 804 /* round up so we only use full longs */ 805 ram_pages = roundup(ms->base_gfn + ms->npages, BITS_PER_LONG); 806 /* allocate enough bytes to store all the bits */ 807 mgs->pgste_bitmap = vmalloc(ram_pages / 8); 808 if (!mgs->pgste_bitmap) { 809 kfree(mgs); 810 kvm->arch.migration_state = NULL; 811 return -ENOMEM; 812 } 813 814 mgs->bitmap_size = ram_pages; 815 atomic64_set(&mgs->dirty_pages, ram_pages); 816 /* mark all the pages in active slots as dirty */ 817 for (slotnr = 0; slotnr < slots->used_slots; slotnr++) { 818 ms = slots->memslots + slotnr; 819 bitmap_set(mgs->pgste_bitmap, ms->base_gfn, ms->npages); 820 } 821 822 kvm_s390_sync_request_broadcast(kvm, KVM_REQ_START_MIGRATION); 823 } 824 return 0; 825 } 826 827 /* 828 * Must be called with kvm->slots_lock to avoid races with ourselves and 829 * kvm_s390_vm_start_migration. 830 */ 831 static int kvm_s390_vm_stop_migration(struct kvm *kvm) 832 { 833 struct kvm_s390_migration_state *mgs; 834 835 /* migration mode already disabled */ 836 if (!kvm->arch.migration_state) 837 return 0; 838 mgs = kvm->arch.migration_state; 839 kvm->arch.migration_state = NULL; 840 841 if (kvm->arch.use_cmma) { 842 kvm_s390_sync_request_broadcast(kvm, KVM_REQ_STOP_MIGRATION); 843 /* We have to wait for the essa emulation to finish */ 844 synchronize_srcu(&kvm->srcu); 845 vfree(mgs->pgste_bitmap); 846 } 847 kfree(mgs); 848 return 0; 849 } 850 851 static int kvm_s390_vm_set_migration(struct kvm *kvm, 852 struct kvm_device_attr *attr) 853 { 854 int res = -ENXIO; 855 856 mutex_lock(&kvm->slots_lock); 857 switch (attr->attr) { 858 case KVM_S390_VM_MIGRATION_START: 859 res = kvm_s390_vm_start_migration(kvm); 860 break; 861 case KVM_S390_VM_MIGRATION_STOP: 862 res = kvm_s390_vm_stop_migration(kvm); 863 break; 864 default: 865 break; 866 } 867 mutex_unlock(&kvm->slots_lock); 868 869 return res; 870 } 871 872 static int kvm_s390_vm_get_migration(struct kvm *kvm, 873 struct kvm_device_attr *attr) 874 { 875 u64 mig = (kvm->arch.migration_state != NULL); 876 877 if (attr->attr != KVM_S390_VM_MIGRATION_STATUS) 878 return -ENXIO; 879 880 if (copy_to_user((void __user *)attr->addr, &mig, sizeof(mig))) 881 return -EFAULT; 882 return 0; 883 } 884 885 static int kvm_s390_set_tod_ext(struct kvm *kvm, struct kvm_device_attr *attr) 886 { 887 struct kvm_s390_vm_tod_clock gtod; 888 889 if (copy_from_user(>od, (void __user *)attr->addr, sizeof(gtod))) 890 return -EFAULT; 891 892 if (test_kvm_facility(kvm, 139)) 893 kvm_s390_set_tod_clock_ext(kvm, >od); 894 else if (gtod.epoch_idx == 0) 895 kvm_s390_set_tod_clock(kvm, gtod.tod); 896 else 897 return -EINVAL; 898 899 VM_EVENT(kvm, 3, "SET: TOD extension: 0x%x, TOD base: 0x%llx", 900 gtod.epoch_idx, gtod.tod); 901 902 return 0; 903 } 904 905 static int kvm_s390_set_tod_high(struct kvm *kvm, struct kvm_device_attr *attr) 906 { 907 u8 gtod_high; 908 909 if (copy_from_user(>od_high, (void __user *)attr->addr, 910 sizeof(gtod_high))) 911 return -EFAULT; 912 913 if (gtod_high != 0) 914 return -EINVAL; 915 VM_EVENT(kvm, 3, "SET: TOD extension: 0x%x", gtod_high); 916 917 return 0; 918 } 919 920 static int kvm_s390_set_tod_low(struct kvm *kvm, struct kvm_device_attr *attr) 921 { 922 u64 gtod; 923 924 if (copy_from_user(>od, (void __user *)attr->addr, sizeof(gtod))) 925 return -EFAULT; 926 927 kvm_s390_set_tod_clock(kvm, gtod); 928 VM_EVENT(kvm, 3, "SET: TOD base: 0x%llx", gtod); 929 return 0; 930 } 931 932 static int kvm_s390_set_tod(struct kvm *kvm, struct kvm_device_attr *attr) 933 { 934 int ret; 935 936 if (attr->flags) 937 return -EINVAL; 938 939 switch (attr->attr) { 940 case KVM_S390_VM_TOD_EXT: 941 ret = kvm_s390_set_tod_ext(kvm, attr); 942 break; 943 case KVM_S390_VM_TOD_HIGH: 944 ret = kvm_s390_set_tod_high(kvm, attr); 945 break; 946 case KVM_S390_VM_TOD_LOW: 947 ret = kvm_s390_set_tod_low(kvm, attr); 948 break; 949 default: 950 ret = -ENXIO; 951 break; 952 } 953 return ret; 954 } 955 956 static void kvm_s390_get_tod_clock_ext(struct kvm *kvm, 957 struct kvm_s390_vm_tod_clock *gtod) 958 { 959 struct kvm_s390_tod_clock_ext htod; 960 961 preempt_disable(); 962 963 get_tod_clock_ext((char *)&htod); 964 965 gtod->tod = htod.tod + kvm->arch.epoch; 966 gtod->epoch_idx = htod.epoch_idx + kvm->arch.epdx; 967 968 if (gtod->tod < htod.tod) 969 gtod->epoch_idx += 1; 970 971 preempt_enable(); 972 } 973 974 static int kvm_s390_get_tod_ext(struct kvm *kvm, struct kvm_device_attr *attr) 975 { 976 struct kvm_s390_vm_tod_clock gtod; 977 978 memset(>od, 0, sizeof(gtod)); 979 980 if (test_kvm_facility(kvm, 139)) 981 kvm_s390_get_tod_clock_ext(kvm, >od); 982 else 983 gtod.tod = kvm_s390_get_tod_clock_fast(kvm); 984 985 if (copy_to_user((void __user *)attr->addr, >od, sizeof(gtod))) 986 return -EFAULT; 987 988 VM_EVENT(kvm, 3, "QUERY: TOD extension: 0x%x, TOD base: 0x%llx", 989 gtod.epoch_idx, gtod.tod); 990 return 0; 991 } 992 993 static int kvm_s390_get_tod_high(struct kvm *kvm, struct kvm_device_attr *attr) 994 { 995 u8 gtod_high = 0; 996 997 if (copy_to_user((void __user *)attr->addr, >od_high, 998 sizeof(gtod_high))) 999 return -EFAULT; 1000 VM_EVENT(kvm, 3, "QUERY: TOD extension: 0x%x", gtod_high); 1001 1002 return 0; 1003 } 1004 1005 static int kvm_s390_get_tod_low(struct kvm *kvm, struct kvm_device_attr *attr) 1006 { 1007 u64 gtod; 1008 1009 gtod = kvm_s390_get_tod_clock_fast(kvm); 1010 if (copy_to_user((void __user *)attr->addr, >od, sizeof(gtod))) 1011 return -EFAULT; 1012 VM_EVENT(kvm, 3, "QUERY: TOD base: 0x%llx", gtod); 1013 1014 return 0; 1015 } 1016 1017 static int kvm_s390_get_tod(struct kvm *kvm, struct kvm_device_attr *attr) 1018 { 1019 int ret; 1020 1021 if (attr->flags) 1022 return -EINVAL; 1023 1024 switch (attr->attr) { 1025 case KVM_S390_VM_TOD_EXT: 1026 ret = kvm_s390_get_tod_ext(kvm, attr); 1027 break; 1028 case KVM_S390_VM_TOD_HIGH: 1029 ret = kvm_s390_get_tod_high(kvm, attr); 1030 break; 1031 case KVM_S390_VM_TOD_LOW: 1032 ret = kvm_s390_get_tod_low(kvm, attr); 1033 break; 1034 default: 1035 ret = -ENXIO; 1036 break; 1037 } 1038 return ret; 1039 } 1040 1041 static int kvm_s390_set_processor(struct kvm *kvm, struct kvm_device_attr *attr) 1042 { 1043 struct kvm_s390_vm_cpu_processor *proc; 1044 u16 lowest_ibc, unblocked_ibc; 1045 int ret = 0; 1046 1047 mutex_lock(&kvm->lock); 1048 if (kvm->created_vcpus) { 1049 ret = -EBUSY; 1050 goto out; 1051 } 1052 proc = kzalloc(sizeof(*proc), GFP_KERNEL); 1053 if (!proc) { 1054 ret = -ENOMEM; 1055 goto out; 1056 } 1057 if (!copy_from_user(proc, (void __user *)attr->addr, 1058 sizeof(*proc))) { 1059 kvm->arch.model.cpuid = proc->cpuid; 1060 lowest_ibc = sclp.ibc >> 16 & 0xfff; 1061 unblocked_ibc = sclp.ibc & 0xfff; 1062 if (lowest_ibc && proc->ibc) { 1063 if (proc->ibc > unblocked_ibc) 1064 kvm->arch.model.ibc = unblocked_ibc; 1065 else if (proc->ibc < lowest_ibc) 1066 kvm->arch.model.ibc = lowest_ibc; 1067 else 1068 kvm->arch.model.ibc = proc->ibc; 1069 } 1070 memcpy(kvm->arch.model.fac_list, proc->fac_list, 1071 S390_ARCH_FAC_LIST_SIZE_BYTE); 1072 VM_EVENT(kvm, 3, "SET: guest ibc: 0x%4.4x, guest cpuid: 0x%16.16llx", 1073 kvm->arch.model.ibc, 1074 kvm->arch.model.cpuid); 1075 VM_EVENT(kvm, 3, "SET: guest faclist: 0x%16.16llx.%16.16llx.%16.16llx", 1076 kvm->arch.model.fac_list[0], 1077 kvm->arch.model.fac_list[1], 1078 kvm->arch.model.fac_list[2]); 1079 } else 1080 ret = -EFAULT; 1081 kfree(proc); 1082 out: 1083 mutex_unlock(&kvm->lock); 1084 return ret; 1085 } 1086 1087 static int kvm_s390_set_processor_feat(struct kvm *kvm, 1088 struct kvm_device_attr *attr) 1089 { 1090 struct kvm_s390_vm_cpu_feat data; 1091 int ret = -EBUSY; 1092 1093 if (copy_from_user(&data, (void __user *)attr->addr, sizeof(data))) 1094 return -EFAULT; 1095 if (!bitmap_subset((unsigned long *) data.feat, 1096 kvm_s390_available_cpu_feat, 1097 KVM_S390_VM_CPU_FEAT_NR_BITS)) 1098 return -EINVAL; 1099 1100 mutex_lock(&kvm->lock); 1101 if (!atomic_read(&kvm->online_vcpus)) { 1102 bitmap_copy(kvm->arch.cpu_feat, (unsigned long *) data.feat, 1103 KVM_S390_VM_CPU_FEAT_NR_BITS); 1104 ret = 0; 1105 } 1106 mutex_unlock(&kvm->lock); 1107 return ret; 1108 } 1109 1110 static int kvm_s390_set_processor_subfunc(struct kvm *kvm, 1111 struct kvm_device_attr *attr) 1112 { 1113 /* 1114 * Once supported by kernel + hw, we have to store the subfunctions 1115 * in kvm->arch and remember that user space configured them. 1116 */ 1117 return -ENXIO; 1118 } 1119 1120 static int kvm_s390_set_cpu_model(struct kvm *kvm, struct kvm_device_attr *attr) 1121 { 1122 int ret = -ENXIO; 1123 1124 switch (attr->attr) { 1125 case KVM_S390_VM_CPU_PROCESSOR: 1126 ret = kvm_s390_set_processor(kvm, attr); 1127 break; 1128 case KVM_S390_VM_CPU_PROCESSOR_FEAT: 1129 ret = kvm_s390_set_processor_feat(kvm, attr); 1130 break; 1131 case KVM_S390_VM_CPU_PROCESSOR_SUBFUNC: 1132 ret = kvm_s390_set_processor_subfunc(kvm, attr); 1133 break; 1134 } 1135 return ret; 1136 } 1137 1138 static int kvm_s390_get_processor(struct kvm *kvm, struct kvm_device_attr *attr) 1139 { 1140 struct kvm_s390_vm_cpu_processor *proc; 1141 int ret = 0; 1142 1143 proc = kzalloc(sizeof(*proc), GFP_KERNEL); 1144 if (!proc) { 1145 ret = -ENOMEM; 1146 goto out; 1147 } 1148 proc->cpuid = kvm->arch.model.cpuid; 1149 proc->ibc = kvm->arch.model.ibc; 1150 memcpy(&proc->fac_list, kvm->arch.model.fac_list, 1151 S390_ARCH_FAC_LIST_SIZE_BYTE); 1152 VM_EVENT(kvm, 3, "GET: guest ibc: 0x%4.4x, guest cpuid: 0x%16.16llx", 1153 kvm->arch.model.ibc, 1154 kvm->arch.model.cpuid); 1155 VM_EVENT(kvm, 3, "GET: guest faclist: 0x%16.16llx.%16.16llx.%16.16llx", 1156 kvm->arch.model.fac_list[0], 1157 kvm->arch.model.fac_list[1], 1158 kvm->arch.model.fac_list[2]); 1159 if (copy_to_user((void __user *)attr->addr, proc, sizeof(*proc))) 1160 ret = -EFAULT; 1161 kfree(proc); 1162 out: 1163 return ret; 1164 } 1165 1166 static int kvm_s390_get_machine(struct kvm *kvm, struct kvm_device_attr *attr) 1167 { 1168 struct kvm_s390_vm_cpu_machine *mach; 1169 int ret = 0; 1170 1171 mach = kzalloc(sizeof(*mach), GFP_KERNEL); 1172 if (!mach) { 1173 ret = -ENOMEM; 1174 goto out; 1175 } 1176 get_cpu_id((struct cpuid *) &mach->cpuid); 1177 mach->ibc = sclp.ibc; 1178 memcpy(&mach->fac_mask, kvm->arch.model.fac_mask, 1179 S390_ARCH_FAC_LIST_SIZE_BYTE); 1180 memcpy((unsigned long *)&mach->fac_list, S390_lowcore.stfle_fac_list, 1181 sizeof(S390_lowcore.stfle_fac_list)); 1182 VM_EVENT(kvm, 3, "GET: host ibc: 0x%4.4x, host cpuid: 0x%16.16llx", 1183 kvm->arch.model.ibc, 1184 kvm->arch.model.cpuid); 1185 VM_EVENT(kvm, 3, "GET: host facmask: 0x%16.16llx.%16.16llx.%16.16llx", 1186 mach->fac_mask[0], 1187 mach->fac_mask[1], 1188 mach->fac_mask[2]); 1189 VM_EVENT(kvm, 3, "GET: host faclist: 0x%16.16llx.%16.16llx.%16.16llx", 1190 mach->fac_list[0], 1191 mach->fac_list[1], 1192 mach->fac_list[2]); 1193 if (copy_to_user((void __user *)attr->addr, mach, sizeof(*mach))) 1194 ret = -EFAULT; 1195 kfree(mach); 1196 out: 1197 return ret; 1198 } 1199 1200 static int kvm_s390_get_processor_feat(struct kvm *kvm, 1201 struct kvm_device_attr *attr) 1202 { 1203 struct kvm_s390_vm_cpu_feat data; 1204 1205 bitmap_copy((unsigned long *) data.feat, kvm->arch.cpu_feat, 1206 KVM_S390_VM_CPU_FEAT_NR_BITS); 1207 if (copy_to_user((void __user *)attr->addr, &data, sizeof(data))) 1208 return -EFAULT; 1209 return 0; 1210 } 1211 1212 static int kvm_s390_get_machine_feat(struct kvm *kvm, 1213 struct kvm_device_attr *attr) 1214 { 1215 struct kvm_s390_vm_cpu_feat data; 1216 1217 bitmap_copy((unsigned long *) data.feat, 1218 kvm_s390_available_cpu_feat, 1219 KVM_S390_VM_CPU_FEAT_NR_BITS); 1220 if (copy_to_user((void __user *)attr->addr, &data, sizeof(data))) 1221 return -EFAULT; 1222 return 0; 1223 } 1224 1225 static int kvm_s390_get_processor_subfunc(struct kvm *kvm, 1226 struct kvm_device_attr *attr) 1227 { 1228 /* 1229 * Once we can actually configure subfunctions (kernel + hw support), 1230 * we have to check if they were already set by user space, if so copy 1231 * them from kvm->arch. 1232 */ 1233 return -ENXIO; 1234 } 1235 1236 static int kvm_s390_get_machine_subfunc(struct kvm *kvm, 1237 struct kvm_device_attr *attr) 1238 { 1239 if (copy_to_user((void __user *)attr->addr, &kvm_s390_available_subfunc, 1240 sizeof(struct kvm_s390_vm_cpu_subfunc))) 1241 return -EFAULT; 1242 return 0; 1243 } 1244 static int kvm_s390_get_cpu_model(struct kvm *kvm, struct kvm_device_attr *attr) 1245 { 1246 int ret = -ENXIO; 1247 1248 switch (attr->attr) { 1249 case KVM_S390_VM_CPU_PROCESSOR: 1250 ret = kvm_s390_get_processor(kvm, attr); 1251 break; 1252 case KVM_S390_VM_CPU_MACHINE: 1253 ret = kvm_s390_get_machine(kvm, attr); 1254 break; 1255 case KVM_S390_VM_CPU_PROCESSOR_FEAT: 1256 ret = kvm_s390_get_processor_feat(kvm, attr); 1257 break; 1258 case KVM_S390_VM_CPU_MACHINE_FEAT: 1259 ret = kvm_s390_get_machine_feat(kvm, attr); 1260 break; 1261 case KVM_S390_VM_CPU_PROCESSOR_SUBFUNC: 1262 ret = kvm_s390_get_processor_subfunc(kvm, attr); 1263 break; 1264 case KVM_S390_VM_CPU_MACHINE_SUBFUNC: 1265 ret = kvm_s390_get_machine_subfunc(kvm, attr); 1266 break; 1267 } 1268 return ret; 1269 } 1270 1271 static int kvm_s390_vm_set_attr(struct kvm *kvm, struct kvm_device_attr *attr) 1272 { 1273 int ret; 1274 1275 switch (attr->group) { 1276 case KVM_S390_VM_MEM_CTRL: 1277 ret = kvm_s390_set_mem_control(kvm, attr); 1278 break; 1279 case KVM_S390_VM_TOD: 1280 ret = kvm_s390_set_tod(kvm, attr); 1281 break; 1282 case KVM_S390_VM_CPU_MODEL: 1283 ret = kvm_s390_set_cpu_model(kvm, attr); 1284 break; 1285 case KVM_S390_VM_CRYPTO: 1286 ret = kvm_s390_vm_set_crypto(kvm, attr); 1287 break; 1288 case KVM_S390_VM_MIGRATION: 1289 ret = kvm_s390_vm_set_migration(kvm, attr); 1290 break; 1291 default: 1292 ret = -ENXIO; 1293 break; 1294 } 1295 1296 return ret; 1297 } 1298 1299 static int kvm_s390_vm_get_attr(struct kvm *kvm, struct kvm_device_attr *attr) 1300 { 1301 int ret; 1302 1303 switch (attr->group) { 1304 case KVM_S390_VM_MEM_CTRL: 1305 ret = kvm_s390_get_mem_control(kvm, attr); 1306 break; 1307 case KVM_S390_VM_TOD: 1308 ret = kvm_s390_get_tod(kvm, attr); 1309 break; 1310 case KVM_S390_VM_CPU_MODEL: 1311 ret = kvm_s390_get_cpu_model(kvm, attr); 1312 break; 1313 case KVM_S390_VM_MIGRATION: 1314 ret = kvm_s390_vm_get_migration(kvm, attr); 1315 break; 1316 default: 1317 ret = -ENXIO; 1318 break; 1319 } 1320 1321 return ret; 1322 } 1323 1324 static int kvm_s390_vm_has_attr(struct kvm *kvm, struct kvm_device_attr *attr) 1325 { 1326 int ret; 1327 1328 switch (attr->group) { 1329 case KVM_S390_VM_MEM_CTRL: 1330 switch (attr->attr) { 1331 case KVM_S390_VM_MEM_ENABLE_CMMA: 1332 case KVM_S390_VM_MEM_CLR_CMMA: 1333 ret = sclp.has_cmma ? 0 : -ENXIO; 1334 break; 1335 case KVM_S390_VM_MEM_LIMIT_SIZE: 1336 ret = 0; 1337 break; 1338 default: 1339 ret = -ENXIO; 1340 break; 1341 } 1342 break; 1343 case KVM_S390_VM_TOD: 1344 switch (attr->attr) { 1345 case KVM_S390_VM_TOD_LOW: 1346 case KVM_S390_VM_TOD_HIGH: 1347 ret = 0; 1348 break; 1349 default: 1350 ret = -ENXIO; 1351 break; 1352 } 1353 break; 1354 case KVM_S390_VM_CPU_MODEL: 1355 switch (attr->attr) { 1356 case KVM_S390_VM_CPU_PROCESSOR: 1357 case KVM_S390_VM_CPU_MACHINE: 1358 case KVM_S390_VM_CPU_PROCESSOR_FEAT: 1359 case KVM_S390_VM_CPU_MACHINE_FEAT: 1360 case KVM_S390_VM_CPU_MACHINE_SUBFUNC: 1361 ret = 0; 1362 break; 1363 /* configuring subfunctions is not supported yet */ 1364 case KVM_S390_VM_CPU_PROCESSOR_SUBFUNC: 1365 default: 1366 ret = -ENXIO; 1367 break; 1368 } 1369 break; 1370 case KVM_S390_VM_CRYPTO: 1371 switch (attr->attr) { 1372 case KVM_S390_VM_CRYPTO_ENABLE_AES_KW: 1373 case KVM_S390_VM_CRYPTO_ENABLE_DEA_KW: 1374 case KVM_S390_VM_CRYPTO_DISABLE_AES_KW: 1375 case KVM_S390_VM_CRYPTO_DISABLE_DEA_KW: 1376 ret = 0; 1377 break; 1378 default: 1379 ret = -ENXIO; 1380 break; 1381 } 1382 break; 1383 case KVM_S390_VM_MIGRATION: 1384 ret = 0; 1385 break; 1386 default: 1387 ret = -ENXIO; 1388 break; 1389 } 1390 1391 return ret; 1392 } 1393 1394 static long kvm_s390_get_skeys(struct kvm *kvm, struct kvm_s390_skeys *args) 1395 { 1396 uint8_t *keys; 1397 uint64_t hva; 1398 int srcu_idx, i, r = 0; 1399 1400 if (args->flags != 0) 1401 return -EINVAL; 1402 1403 /* Is this guest using storage keys? */ 1404 if (!mm_use_skey(current->mm)) 1405 return KVM_S390_GET_SKEYS_NONE; 1406 1407 /* Enforce sane limit on memory allocation */ 1408 if (args->count < 1 || args->count > KVM_S390_SKEYS_MAX) 1409 return -EINVAL; 1410 1411 keys = kvmalloc_array(args->count, sizeof(uint8_t), GFP_KERNEL); 1412 if (!keys) 1413 return -ENOMEM; 1414 1415 down_read(¤t->mm->mmap_sem); 1416 srcu_idx = srcu_read_lock(&kvm->srcu); 1417 for (i = 0; i < args->count; i++) { 1418 hva = gfn_to_hva(kvm, args->start_gfn + i); 1419 if (kvm_is_error_hva(hva)) { 1420 r = -EFAULT; 1421 break; 1422 } 1423 1424 r = get_guest_storage_key(current->mm, hva, &keys[i]); 1425 if (r) 1426 break; 1427 } 1428 srcu_read_unlock(&kvm->srcu, srcu_idx); 1429 up_read(¤t->mm->mmap_sem); 1430 1431 if (!r) { 1432 r = copy_to_user((uint8_t __user *)args->skeydata_addr, keys, 1433 sizeof(uint8_t) * args->count); 1434 if (r) 1435 r = -EFAULT; 1436 } 1437 1438 kvfree(keys); 1439 return r; 1440 } 1441 1442 static long kvm_s390_set_skeys(struct kvm *kvm, struct kvm_s390_skeys *args) 1443 { 1444 uint8_t *keys; 1445 uint64_t hva; 1446 int srcu_idx, i, r = 0; 1447 1448 if (args->flags != 0) 1449 return -EINVAL; 1450 1451 /* Enforce sane limit on memory allocation */ 1452 if (args->count < 1 || args->count > KVM_S390_SKEYS_MAX) 1453 return -EINVAL; 1454 1455 keys = kvmalloc_array(args->count, sizeof(uint8_t), GFP_KERNEL); 1456 if (!keys) 1457 return -ENOMEM; 1458 1459 r = copy_from_user(keys, (uint8_t __user *)args->skeydata_addr, 1460 sizeof(uint8_t) * args->count); 1461 if (r) { 1462 r = -EFAULT; 1463 goto out; 1464 } 1465 1466 /* Enable storage key handling for the guest */ 1467 r = s390_enable_skey(); 1468 if (r) 1469 goto out; 1470 1471 down_read(¤t->mm->mmap_sem); 1472 srcu_idx = srcu_read_lock(&kvm->srcu); 1473 for (i = 0; i < args->count; i++) { 1474 hva = gfn_to_hva(kvm, args->start_gfn + i); 1475 if (kvm_is_error_hva(hva)) { 1476 r = -EFAULT; 1477 break; 1478 } 1479 1480 /* Lowest order bit is reserved */ 1481 if (keys[i] & 0x01) { 1482 r = -EINVAL; 1483 break; 1484 } 1485 1486 r = set_guest_storage_key(current->mm, hva, keys[i], 0); 1487 if (r) 1488 break; 1489 } 1490 srcu_read_unlock(&kvm->srcu, srcu_idx); 1491 up_read(¤t->mm->mmap_sem); 1492 out: 1493 kvfree(keys); 1494 return r; 1495 } 1496 1497 /* 1498 * Base address and length must be sent at the start of each block, therefore 1499 * it's cheaper to send some clean data, as long as it's less than the size of 1500 * two longs. 1501 */ 1502 #define KVM_S390_MAX_BIT_DISTANCE (2 * sizeof(void *)) 1503 /* for consistency */ 1504 #define KVM_S390_CMMA_SIZE_MAX ((u32)KVM_S390_SKEYS_MAX) 1505 1506 /* 1507 * This function searches for the next page with dirty CMMA attributes, and 1508 * saves the attributes in the buffer up to either the end of the buffer or 1509 * until a block of at least KVM_S390_MAX_BIT_DISTANCE clean bits is found; 1510 * no trailing clean bytes are saved. 1511 * In case no dirty bits were found, or if CMMA was not enabled or used, the 1512 * output buffer will indicate 0 as length. 1513 */ 1514 static int kvm_s390_get_cmma_bits(struct kvm *kvm, 1515 struct kvm_s390_cmma_log *args) 1516 { 1517 struct kvm_s390_migration_state *s = kvm->arch.migration_state; 1518 unsigned long bufsize, hva, pgstev, i, next, cur; 1519 int srcu_idx, peek, r = 0, rr; 1520 u8 *res; 1521 1522 cur = args->start_gfn; 1523 i = next = pgstev = 0; 1524 1525 if (unlikely(!kvm->arch.use_cmma)) 1526 return -ENXIO; 1527 /* Invalid/unsupported flags were specified */ 1528 if (args->flags & ~KVM_S390_CMMA_PEEK) 1529 return -EINVAL; 1530 /* Migration mode query, and we are not doing a migration */ 1531 peek = !!(args->flags & KVM_S390_CMMA_PEEK); 1532 if (!peek && !s) 1533 return -EINVAL; 1534 /* CMMA is disabled or was not used, or the buffer has length zero */ 1535 bufsize = min(args->count, KVM_S390_CMMA_SIZE_MAX); 1536 if (!bufsize || !kvm->mm->context.use_cmma) { 1537 memset(args, 0, sizeof(*args)); 1538 return 0; 1539 } 1540 1541 if (!peek) { 1542 /* We are not peeking, and there are no dirty pages */ 1543 if (!atomic64_read(&s->dirty_pages)) { 1544 memset(args, 0, sizeof(*args)); 1545 return 0; 1546 } 1547 cur = find_next_bit(s->pgste_bitmap, s->bitmap_size, 1548 args->start_gfn); 1549 if (cur >= s->bitmap_size) /* nothing found, loop back */ 1550 cur = find_next_bit(s->pgste_bitmap, s->bitmap_size, 0); 1551 if (cur >= s->bitmap_size) { /* again! (very unlikely) */ 1552 memset(args, 0, sizeof(*args)); 1553 return 0; 1554 } 1555 next = find_next_bit(s->pgste_bitmap, s->bitmap_size, cur + 1); 1556 } 1557 1558 res = vmalloc(bufsize); 1559 if (!res) 1560 return -ENOMEM; 1561 1562 args->start_gfn = cur; 1563 1564 down_read(&kvm->mm->mmap_sem); 1565 srcu_idx = srcu_read_lock(&kvm->srcu); 1566 while (i < bufsize) { 1567 hva = gfn_to_hva(kvm, cur); 1568 if (kvm_is_error_hva(hva)) { 1569 r = -EFAULT; 1570 break; 1571 } 1572 /* decrement only if we actually flipped the bit to 0 */ 1573 if (!peek && test_and_clear_bit(cur, s->pgste_bitmap)) 1574 atomic64_dec(&s->dirty_pages); 1575 r = get_pgste(kvm->mm, hva, &pgstev); 1576 if (r < 0) 1577 pgstev = 0; 1578 /* save the value */ 1579 res[i++] = (pgstev >> 24) & 0x43; 1580 /* 1581 * if the next bit is too far away, stop. 1582 * if we reached the previous "next", find the next one 1583 */ 1584 if (!peek) { 1585 if (next > cur + KVM_S390_MAX_BIT_DISTANCE) 1586 break; 1587 if (cur == next) 1588 next = find_next_bit(s->pgste_bitmap, 1589 s->bitmap_size, cur + 1); 1590 /* reached the end of the bitmap or of the buffer, stop */ 1591 if ((next >= s->bitmap_size) || 1592 (next >= args->start_gfn + bufsize)) 1593 break; 1594 } 1595 cur++; 1596 } 1597 srcu_read_unlock(&kvm->srcu, srcu_idx); 1598 up_read(&kvm->mm->mmap_sem); 1599 args->count = i; 1600 args->remaining = s ? atomic64_read(&s->dirty_pages) : 0; 1601 1602 rr = copy_to_user((void __user *)args->values, res, args->count); 1603 if (rr) 1604 r = -EFAULT; 1605 1606 vfree(res); 1607 return r; 1608 } 1609 1610 /* 1611 * This function sets the CMMA attributes for the given pages. If the input 1612 * buffer has zero length, no action is taken, otherwise the attributes are 1613 * set and the mm->context.use_cmma flag is set. 1614 */ 1615 static int kvm_s390_set_cmma_bits(struct kvm *kvm, 1616 const struct kvm_s390_cmma_log *args) 1617 { 1618 unsigned long hva, mask, pgstev, i; 1619 uint8_t *bits; 1620 int srcu_idx, r = 0; 1621 1622 mask = args->mask; 1623 1624 if (!kvm->arch.use_cmma) 1625 return -ENXIO; 1626 /* invalid/unsupported flags */ 1627 if (args->flags != 0) 1628 return -EINVAL; 1629 /* Enforce sane limit on memory allocation */ 1630 if (args->count > KVM_S390_CMMA_SIZE_MAX) 1631 return -EINVAL; 1632 /* Nothing to do */ 1633 if (args->count == 0) 1634 return 0; 1635 1636 bits = vmalloc(sizeof(*bits) * args->count); 1637 if (!bits) 1638 return -ENOMEM; 1639 1640 r = copy_from_user(bits, (void __user *)args->values, args->count); 1641 if (r) { 1642 r = -EFAULT; 1643 goto out; 1644 } 1645 1646 down_read(&kvm->mm->mmap_sem); 1647 srcu_idx = srcu_read_lock(&kvm->srcu); 1648 for (i = 0; i < args->count; i++) { 1649 hva = gfn_to_hva(kvm, args->start_gfn + i); 1650 if (kvm_is_error_hva(hva)) { 1651 r = -EFAULT; 1652 break; 1653 } 1654 1655 pgstev = bits[i]; 1656 pgstev = pgstev << 24; 1657 mask &= _PGSTE_GPS_USAGE_MASK | _PGSTE_GPS_NODAT; 1658 set_pgste_bits(kvm->mm, hva, mask, pgstev); 1659 } 1660 srcu_read_unlock(&kvm->srcu, srcu_idx); 1661 up_read(&kvm->mm->mmap_sem); 1662 1663 if (!kvm->mm->context.use_cmma) { 1664 down_write(&kvm->mm->mmap_sem); 1665 kvm->mm->context.use_cmma = 1; 1666 up_write(&kvm->mm->mmap_sem); 1667 } 1668 out: 1669 vfree(bits); 1670 return r; 1671 } 1672 1673 long kvm_arch_vm_ioctl(struct file *filp, 1674 unsigned int ioctl, unsigned long arg) 1675 { 1676 struct kvm *kvm = filp->private_data; 1677 void __user *argp = (void __user *)arg; 1678 struct kvm_device_attr attr; 1679 int r; 1680 1681 switch (ioctl) { 1682 case KVM_S390_INTERRUPT: { 1683 struct kvm_s390_interrupt s390int; 1684 1685 r = -EFAULT; 1686 if (copy_from_user(&s390int, argp, sizeof(s390int))) 1687 break; 1688 r = kvm_s390_inject_vm(kvm, &s390int); 1689 break; 1690 } 1691 case KVM_ENABLE_CAP: { 1692 struct kvm_enable_cap cap; 1693 r = -EFAULT; 1694 if (copy_from_user(&cap, argp, sizeof(cap))) 1695 break; 1696 r = kvm_vm_ioctl_enable_cap(kvm, &cap); 1697 break; 1698 } 1699 case KVM_CREATE_IRQCHIP: { 1700 struct kvm_irq_routing_entry routing; 1701 1702 r = -EINVAL; 1703 if (kvm->arch.use_irqchip) { 1704 /* Set up dummy routing. */ 1705 memset(&routing, 0, sizeof(routing)); 1706 r = kvm_set_irq_routing(kvm, &routing, 0, 0); 1707 } 1708 break; 1709 } 1710 case KVM_SET_DEVICE_ATTR: { 1711 r = -EFAULT; 1712 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr))) 1713 break; 1714 r = kvm_s390_vm_set_attr(kvm, &attr); 1715 break; 1716 } 1717 case KVM_GET_DEVICE_ATTR: { 1718 r = -EFAULT; 1719 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr))) 1720 break; 1721 r = kvm_s390_vm_get_attr(kvm, &attr); 1722 break; 1723 } 1724 case KVM_HAS_DEVICE_ATTR: { 1725 r = -EFAULT; 1726 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr))) 1727 break; 1728 r = kvm_s390_vm_has_attr(kvm, &attr); 1729 break; 1730 } 1731 case KVM_S390_GET_SKEYS: { 1732 struct kvm_s390_skeys args; 1733 1734 r = -EFAULT; 1735 if (copy_from_user(&args, argp, 1736 sizeof(struct kvm_s390_skeys))) 1737 break; 1738 r = kvm_s390_get_skeys(kvm, &args); 1739 break; 1740 } 1741 case KVM_S390_SET_SKEYS: { 1742 struct kvm_s390_skeys args; 1743 1744 r = -EFAULT; 1745 if (copy_from_user(&args, argp, 1746 sizeof(struct kvm_s390_skeys))) 1747 break; 1748 r = kvm_s390_set_skeys(kvm, &args); 1749 break; 1750 } 1751 case KVM_S390_GET_CMMA_BITS: { 1752 struct kvm_s390_cmma_log args; 1753 1754 r = -EFAULT; 1755 if (copy_from_user(&args, argp, sizeof(args))) 1756 break; 1757 mutex_lock(&kvm->slots_lock); 1758 r = kvm_s390_get_cmma_bits(kvm, &args); 1759 mutex_unlock(&kvm->slots_lock); 1760 if (!r) { 1761 r = copy_to_user(argp, &args, sizeof(args)); 1762 if (r) 1763 r = -EFAULT; 1764 } 1765 break; 1766 } 1767 case KVM_S390_SET_CMMA_BITS: { 1768 struct kvm_s390_cmma_log args; 1769 1770 r = -EFAULT; 1771 if (copy_from_user(&args, argp, sizeof(args))) 1772 break; 1773 mutex_lock(&kvm->slots_lock); 1774 r = kvm_s390_set_cmma_bits(kvm, &args); 1775 mutex_unlock(&kvm->slots_lock); 1776 break; 1777 } 1778 default: 1779 r = -ENOTTY; 1780 } 1781 1782 return r; 1783 } 1784 1785 static int kvm_s390_query_ap_config(u8 *config) 1786 { 1787 u32 fcn_code = 0x04000000UL; 1788 u32 cc = 0; 1789 1790 memset(config, 0, 128); 1791 asm volatile( 1792 "lgr 0,%1\n" 1793 "lgr 2,%2\n" 1794 ".long 0xb2af0000\n" /* PQAP(QCI) */ 1795 "0: ipm %0\n" 1796 "srl %0,28\n" 1797 "1:\n" 1798 EX_TABLE(0b, 1b) 1799 : "+r" (cc) 1800 : "r" (fcn_code), "r" (config) 1801 : "cc", "0", "2", "memory" 1802 ); 1803 1804 return cc; 1805 } 1806 1807 static int kvm_s390_apxa_installed(void) 1808 { 1809 u8 config[128]; 1810 int cc; 1811 1812 if (test_facility(12)) { 1813 cc = kvm_s390_query_ap_config(config); 1814 1815 if (cc) 1816 pr_err("PQAP(QCI) failed with cc=%d", cc); 1817 else 1818 return config[0] & 0x40; 1819 } 1820 1821 return 0; 1822 } 1823 1824 static void kvm_s390_set_crycb_format(struct kvm *kvm) 1825 { 1826 kvm->arch.crypto.crycbd = (__u32)(unsigned long) kvm->arch.crypto.crycb; 1827 1828 if (kvm_s390_apxa_installed()) 1829 kvm->arch.crypto.crycbd |= CRYCB_FORMAT2; 1830 else 1831 kvm->arch.crypto.crycbd |= CRYCB_FORMAT1; 1832 } 1833 1834 static u64 kvm_s390_get_initial_cpuid(void) 1835 { 1836 struct cpuid cpuid; 1837 1838 get_cpu_id(&cpuid); 1839 cpuid.version = 0xff; 1840 return *((u64 *) &cpuid); 1841 } 1842 1843 static void kvm_s390_crypto_init(struct kvm *kvm) 1844 { 1845 if (!test_kvm_facility(kvm, 76)) 1846 return; 1847 1848 kvm->arch.crypto.crycb = &kvm->arch.sie_page2->crycb; 1849 kvm_s390_set_crycb_format(kvm); 1850 1851 /* Enable AES/DEA protected key functions by default */ 1852 kvm->arch.crypto.aes_kw = 1; 1853 kvm->arch.crypto.dea_kw = 1; 1854 get_random_bytes(kvm->arch.crypto.crycb->aes_wrapping_key_mask, 1855 sizeof(kvm->arch.crypto.crycb->aes_wrapping_key_mask)); 1856 get_random_bytes(kvm->arch.crypto.crycb->dea_wrapping_key_mask, 1857 sizeof(kvm->arch.crypto.crycb->dea_wrapping_key_mask)); 1858 } 1859 1860 static void sca_dispose(struct kvm *kvm) 1861 { 1862 if (kvm->arch.use_esca) 1863 free_pages_exact(kvm->arch.sca, sizeof(struct esca_block)); 1864 else 1865 free_page((unsigned long)(kvm->arch.sca)); 1866 kvm->arch.sca = NULL; 1867 } 1868 1869 int kvm_arch_init_vm(struct kvm *kvm, unsigned long type) 1870 { 1871 gfp_t alloc_flags = GFP_KERNEL; 1872 int i, rc; 1873 char debug_name[16]; 1874 static unsigned long sca_offset; 1875 1876 rc = -EINVAL; 1877 #ifdef CONFIG_KVM_S390_UCONTROL 1878 if (type & ~KVM_VM_S390_UCONTROL) 1879 goto out_err; 1880 if ((type & KVM_VM_S390_UCONTROL) && (!capable(CAP_SYS_ADMIN))) 1881 goto out_err; 1882 #else 1883 if (type) 1884 goto out_err; 1885 #endif 1886 1887 rc = s390_enable_sie(); 1888 if (rc) 1889 goto out_err; 1890 1891 rc = -ENOMEM; 1892 1893 kvm->arch.use_esca = 0; /* start with basic SCA */ 1894 if (!sclp.has_64bscao) 1895 alloc_flags |= GFP_DMA; 1896 rwlock_init(&kvm->arch.sca_lock); 1897 kvm->arch.sca = (struct bsca_block *) get_zeroed_page(alloc_flags); 1898 if (!kvm->arch.sca) 1899 goto out_err; 1900 spin_lock(&kvm_lock); 1901 sca_offset += 16; 1902 if (sca_offset + sizeof(struct bsca_block) > PAGE_SIZE) 1903 sca_offset = 0; 1904 kvm->arch.sca = (struct bsca_block *) 1905 ((char *) kvm->arch.sca + sca_offset); 1906 spin_unlock(&kvm_lock); 1907 1908 sprintf(debug_name, "kvm-%u", current->pid); 1909 1910 kvm->arch.dbf = debug_register(debug_name, 32, 1, 7 * sizeof(long)); 1911 if (!kvm->arch.dbf) 1912 goto out_err; 1913 1914 kvm->arch.sie_page2 = 1915 (struct sie_page2 *) get_zeroed_page(GFP_KERNEL | GFP_DMA); 1916 if (!kvm->arch.sie_page2) 1917 goto out_err; 1918 1919 /* Populate the facility mask initially. */ 1920 memcpy(kvm->arch.model.fac_mask, S390_lowcore.stfle_fac_list, 1921 sizeof(S390_lowcore.stfle_fac_list)); 1922 for (i = 0; i < S390_ARCH_FAC_LIST_SIZE_U64; i++) { 1923 if (i < kvm_s390_fac_list_mask_size()) 1924 kvm->arch.model.fac_mask[i] &= kvm_s390_fac_list_mask[i]; 1925 else 1926 kvm->arch.model.fac_mask[i] = 0UL; 1927 } 1928 1929 /* Populate the facility list initially. */ 1930 kvm->arch.model.fac_list = kvm->arch.sie_page2->fac_list; 1931 memcpy(kvm->arch.model.fac_list, kvm->arch.model.fac_mask, 1932 S390_ARCH_FAC_LIST_SIZE_BYTE); 1933 1934 /* we are always in czam mode - even on pre z14 machines */ 1935 set_kvm_facility(kvm->arch.model.fac_mask, 138); 1936 set_kvm_facility(kvm->arch.model.fac_list, 138); 1937 /* we emulate STHYI in kvm */ 1938 set_kvm_facility(kvm->arch.model.fac_mask, 74); 1939 set_kvm_facility(kvm->arch.model.fac_list, 74); 1940 if (MACHINE_HAS_TLB_GUEST) { 1941 set_kvm_facility(kvm->arch.model.fac_mask, 147); 1942 set_kvm_facility(kvm->arch.model.fac_list, 147); 1943 } 1944 1945 kvm->arch.model.cpuid = kvm_s390_get_initial_cpuid(); 1946 kvm->arch.model.ibc = sclp.ibc & 0x0fff; 1947 1948 kvm_s390_crypto_init(kvm); 1949 1950 mutex_init(&kvm->arch.float_int.ais_lock); 1951 kvm->arch.float_int.simm = 0; 1952 kvm->arch.float_int.nimm = 0; 1953 spin_lock_init(&kvm->arch.float_int.lock); 1954 for (i = 0; i < FIRQ_LIST_COUNT; i++) 1955 INIT_LIST_HEAD(&kvm->arch.float_int.lists[i]); 1956 init_waitqueue_head(&kvm->arch.ipte_wq); 1957 mutex_init(&kvm->arch.ipte_mutex); 1958 1959 debug_register_view(kvm->arch.dbf, &debug_sprintf_view); 1960 VM_EVENT(kvm, 3, "vm created with type %lu", type); 1961 1962 if (type & KVM_VM_S390_UCONTROL) { 1963 kvm->arch.gmap = NULL; 1964 kvm->arch.mem_limit = KVM_S390_NO_MEM_LIMIT; 1965 } else { 1966 if (sclp.hamax == U64_MAX) 1967 kvm->arch.mem_limit = TASK_SIZE_MAX; 1968 else 1969 kvm->arch.mem_limit = min_t(unsigned long, TASK_SIZE_MAX, 1970 sclp.hamax + 1); 1971 kvm->arch.gmap = gmap_create(current->mm, kvm->arch.mem_limit - 1); 1972 if (!kvm->arch.gmap) 1973 goto out_err; 1974 kvm->arch.gmap->private = kvm; 1975 kvm->arch.gmap->pfault_enabled = 0; 1976 } 1977 1978 kvm->arch.css_support = 0; 1979 kvm->arch.use_irqchip = 0; 1980 kvm->arch.epoch = 0; 1981 1982 spin_lock_init(&kvm->arch.start_stop_lock); 1983 kvm_s390_vsie_init(kvm); 1984 KVM_EVENT(3, "vm 0x%pK created by pid %u", kvm, current->pid); 1985 1986 return 0; 1987 out_err: 1988 free_page((unsigned long)kvm->arch.sie_page2); 1989 debug_unregister(kvm->arch.dbf); 1990 sca_dispose(kvm); 1991 KVM_EVENT(3, "creation of vm failed: %d", rc); 1992 return rc; 1993 } 1994 1995 bool kvm_arch_has_vcpu_debugfs(void) 1996 { 1997 return false; 1998 } 1999 2000 int kvm_arch_create_vcpu_debugfs(struct kvm_vcpu *vcpu) 2001 { 2002 return 0; 2003 } 2004 2005 void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu) 2006 { 2007 VCPU_EVENT(vcpu, 3, "%s", "free cpu"); 2008 trace_kvm_s390_destroy_vcpu(vcpu->vcpu_id); 2009 kvm_s390_clear_local_irqs(vcpu); 2010 kvm_clear_async_pf_completion_queue(vcpu); 2011 if (!kvm_is_ucontrol(vcpu->kvm)) 2012 sca_del_vcpu(vcpu); 2013 2014 if (kvm_is_ucontrol(vcpu->kvm)) 2015 gmap_remove(vcpu->arch.gmap); 2016 2017 if (vcpu->kvm->arch.use_cmma) 2018 kvm_s390_vcpu_unsetup_cmma(vcpu); 2019 free_page((unsigned long)(vcpu->arch.sie_block)); 2020 2021 kvm_vcpu_uninit(vcpu); 2022 kmem_cache_free(kvm_vcpu_cache, vcpu); 2023 } 2024 2025 static void kvm_free_vcpus(struct kvm *kvm) 2026 { 2027 unsigned int i; 2028 struct kvm_vcpu *vcpu; 2029 2030 kvm_for_each_vcpu(i, vcpu, kvm) 2031 kvm_arch_vcpu_destroy(vcpu); 2032 2033 mutex_lock(&kvm->lock); 2034 for (i = 0; i < atomic_read(&kvm->online_vcpus); i++) 2035 kvm->vcpus[i] = NULL; 2036 2037 atomic_set(&kvm->online_vcpus, 0); 2038 mutex_unlock(&kvm->lock); 2039 } 2040 2041 void kvm_arch_destroy_vm(struct kvm *kvm) 2042 { 2043 kvm_free_vcpus(kvm); 2044 sca_dispose(kvm); 2045 debug_unregister(kvm->arch.dbf); 2046 free_page((unsigned long)kvm->arch.sie_page2); 2047 if (!kvm_is_ucontrol(kvm)) 2048 gmap_remove(kvm->arch.gmap); 2049 kvm_s390_destroy_adapters(kvm); 2050 kvm_s390_clear_float_irqs(kvm); 2051 kvm_s390_vsie_destroy(kvm); 2052 if (kvm->arch.migration_state) { 2053 vfree(kvm->arch.migration_state->pgste_bitmap); 2054 kfree(kvm->arch.migration_state); 2055 } 2056 KVM_EVENT(3, "vm 0x%pK destroyed", kvm); 2057 } 2058 2059 /* Section: vcpu related */ 2060 static int __kvm_ucontrol_vcpu_init(struct kvm_vcpu *vcpu) 2061 { 2062 vcpu->arch.gmap = gmap_create(current->mm, -1UL); 2063 if (!vcpu->arch.gmap) 2064 return -ENOMEM; 2065 vcpu->arch.gmap->private = vcpu->kvm; 2066 2067 return 0; 2068 } 2069 2070 static void sca_del_vcpu(struct kvm_vcpu *vcpu) 2071 { 2072 if (!kvm_s390_use_sca_entries()) 2073 return; 2074 read_lock(&vcpu->kvm->arch.sca_lock); 2075 if (vcpu->kvm->arch.use_esca) { 2076 struct esca_block *sca = vcpu->kvm->arch.sca; 2077 2078 clear_bit_inv(vcpu->vcpu_id, (unsigned long *) sca->mcn); 2079 sca->cpu[vcpu->vcpu_id].sda = 0; 2080 } else { 2081 struct bsca_block *sca = vcpu->kvm->arch.sca; 2082 2083 clear_bit_inv(vcpu->vcpu_id, (unsigned long *) &sca->mcn); 2084 sca->cpu[vcpu->vcpu_id].sda = 0; 2085 } 2086 read_unlock(&vcpu->kvm->arch.sca_lock); 2087 } 2088 2089 static void sca_add_vcpu(struct kvm_vcpu *vcpu) 2090 { 2091 if (!kvm_s390_use_sca_entries()) { 2092 struct bsca_block *sca = vcpu->kvm->arch.sca; 2093 2094 /* we still need the basic sca for the ipte control */ 2095 vcpu->arch.sie_block->scaoh = (__u32)(((__u64)sca) >> 32); 2096 vcpu->arch.sie_block->scaol = (__u32)(__u64)sca; 2097 } 2098 read_lock(&vcpu->kvm->arch.sca_lock); 2099 if (vcpu->kvm->arch.use_esca) { 2100 struct esca_block *sca = vcpu->kvm->arch.sca; 2101 2102 sca->cpu[vcpu->vcpu_id].sda = (__u64) vcpu->arch.sie_block; 2103 vcpu->arch.sie_block->scaoh = (__u32)(((__u64)sca) >> 32); 2104 vcpu->arch.sie_block->scaol = (__u32)(__u64)sca & ~0x3fU; 2105 vcpu->arch.sie_block->ecb2 |= ECB2_ESCA; 2106 set_bit_inv(vcpu->vcpu_id, (unsigned long *) sca->mcn); 2107 } else { 2108 struct bsca_block *sca = vcpu->kvm->arch.sca; 2109 2110 sca->cpu[vcpu->vcpu_id].sda = (__u64) vcpu->arch.sie_block; 2111 vcpu->arch.sie_block->scaoh = (__u32)(((__u64)sca) >> 32); 2112 vcpu->arch.sie_block->scaol = (__u32)(__u64)sca; 2113 set_bit_inv(vcpu->vcpu_id, (unsigned long *) &sca->mcn); 2114 } 2115 read_unlock(&vcpu->kvm->arch.sca_lock); 2116 } 2117 2118 /* Basic SCA to Extended SCA data copy routines */ 2119 static inline void sca_copy_entry(struct esca_entry *d, struct bsca_entry *s) 2120 { 2121 d->sda = s->sda; 2122 d->sigp_ctrl.c = s->sigp_ctrl.c; 2123 d->sigp_ctrl.scn = s->sigp_ctrl.scn; 2124 } 2125 2126 static void sca_copy_b_to_e(struct esca_block *d, struct bsca_block *s) 2127 { 2128 int i; 2129 2130 d->ipte_control = s->ipte_control; 2131 d->mcn[0] = s->mcn; 2132 for (i = 0; i < KVM_S390_BSCA_CPU_SLOTS; i++) 2133 sca_copy_entry(&d->cpu[i], &s->cpu[i]); 2134 } 2135 2136 static int sca_switch_to_extended(struct kvm *kvm) 2137 { 2138 struct bsca_block *old_sca = kvm->arch.sca; 2139 struct esca_block *new_sca; 2140 struct kvm_vcpu *vcpu; 2141 unsigned int vcpu_idx; 2142 u32 scaol, scaoh; 2143 2144 new_sca = alloc_pages_exact(sizeof(*new_sca), GFP_KERNEL|__GFP_ZERO); 2145 if (!new_sca) 2146 return -ENOMEM; 2147 2148 scaoh = (u32)((u64)(new_sca) >> 32); 2149 scaol = (u32)(u64)(new_sca) & ~0x3fU; 2150 2151 kvm_s390_vcpu_block_all(kvm); 2152 write_lock(&kvm->arch.sca_lock); 2153 2154 sca_copy_b_to_e(new_sca, old_sca); 2155 2156 kvm_for_each_vcpu(vcpu_idx, vcpu, kvm) { 2157 vcpu->arch.sie_block->scaoh = scaoh; 2158 vcpu->arch.sie_block->scaol = scaol; 2159 vcpu->arch.sie_block->ecb2 |= ECB2_ESCA; 2160 } 2161 kvm->arch.sca = new_sca; 2162 kvm->arch.use_esca = 1; 2163 2164 write_unlock(&kvm->arch.sca_lock); 2165 kvm_s390_vcpu_unblock_all(kvm); 2166 2167 free_page((unsigned long)old_sca); 2168 2169 VM_EVENT(kvm, 2, "Switched to ESCA (0x%pK -> 0x%pK)", 2170 old_sca, kvm->arch.sca); 2171 return 0; 2172 } 2173 2174 static int sca_can_add_vcpu(struct kvm *kvm, unsigned int id) 2175 { 2176 int rc; 2177 2178 if (!kvm_s390_use_sca_entries()) { 2179 if (id < KVM_MAX_VCPUS) 2180 return true; 2181 return false; 2182 } 2183 if (id < KVM_S390_BSCA_CPU_SLOTS) 2184 return true; 2185 if (!sclp.has_esca || !sclp.has_64bscao) 2186 return false; 2187 2188 mutex_lock(&kvm->lock); 2189 rc = kvm->arch.use_esca ? 0 : sca_switch_to_extended(kvm); 2190 mutex_unlock(&kvm->lock); 2191 2192 return rc == 0 && id < KVM_S390_ESCA_CPU_SLOTS; 2193 } 2194 2195 int kvm_arch_vcpu_init(struct kvm_vcpu *vcpu) 2196 { 2197 vcpu->arch.pfault_token = KVM_S390_PFAULT_TOKEN_INVALID; 2198 kvm_clear_async_pf_completion_queue(vcpu); 2199 vcpu->run->kvm_valid_regs = KVM_SYNC_PREFIX | 2200 KVM_SYNC_GPRS | 2201 KVM_SYNC_ACRS | 2202 KVM_SYNC_CRS | 2203 KVM_SYNC_ARCH0 | 2204 KVM_SYNC_PFAULT; 2205 kvm_s390_set_prefix(vcpu, 0); 2206 if (test_kvm_facility(vcpu->kvm, 64)) 2207 vcpu->run->kvm_valid_regs |= KVM_SYNC_RICCB; 2208 if (test_kvm_facility(vcpu->kvm, 82)) 2209 vcpu->run->kvm_valid_regs |= KVM_SYNC_BPBC; 2210 if (test_kvm_facility(vcpu->kvm, 133)) 2211 vcpu->run->kvm_valid_regs |= KVM_SYNC_GSCB; 2212 /* fprs can be synchronized via vrs, even if the guest has no vx. With 2213 * MACHINE_HAS_VX, (load|store)_fpu_regs() will work with vrs format. 2214 */ 2215 if (MACHINE_HAS_VX) 2216 vcpu->run->kvm_valid_regs |= KVM_SYNC_VRS; 2217 else 2218 vcpu->run->kvm_valid_regs |= KVM_SYNC_FPRS; 2219 2220 if (kvm_is_ucontrol(vcpu->kvm)) 2221 return __kvm_ucontrol_vcpu_init(vcpu); 2222 2223 return 0; 2224 } 2225 2226 /* needs disabled preemption to protect from TOD sync and vcpu_load/put */ 2227 static void __start_cpu_timer_accounting(struct kvm_vcpu *vcpu) 2228 { 2229 WARN_ON_ONCE(vcpu->arch.cputm_start != 0); 2230 raw_write_seqcount_begin(&vcpu->arch.cputm_seqcount); 2231 vcpu->arch.cputm_start = get_tod_clock_fast(); 2232 raw_write_seqcount_end(&vcpu->arch.cputm_seqcount); 2233 } 2234 2235 /* needs disabled preemption to protect from TOD sync and vcpu_load/put */ 2236 static void __stop_cpu_timer_accounting(struct kvm_vcpu *vcpu) 2237 { 2238 WARN_ON_ONCE(vcpu->arch.cputm_start == 0); 2239 raw_write_seqcount_begin(&vcpu->arch.cputm_seqcount); 2240 vcpu->arch.sie_block->cputm -= get_tod_clock_fast() - vcpu->arch.cputm_start; 2241 vcpu->arch.cputm_start = 0; 2242 raw_write_seqcount_end(&vcpu->arch.cputm_seqcount); 2243 } 2244 2245 /* needs disabled preemption to protect from TOD sync and vcpu_load/put */ 2246 static void __enable_cpu_timer_accounting(struct kvm_vcpu *vcpu) 2247 { 2248 WARN_ON_ONCE(vcpu->arch.cputm_enabled); 2249 vcpu->arch.cputm_enabled = true; 2250 __start_cpu_timer_accounting(vcpu); 2251 } 2252 2253 /* needs disabled preemption to protect from TOD sync and vcpu_load/put */ 2254 static void __disable_cpu_timer_accounting(struct kvm_vcpu *vcpu) 2255 { 2256 WARN_ON_ONCE(!vcpu->arch.cputm_enabled); 2257 __stop_cpu_timer_accounting(vcpu); 2258 vcpu->arch.cputm_enabled = false; 2259 } 2260 2261 static void enable_cpu_timer_accounting(struct kvm_vcpu *vcpu) 2262 { 2263 preempt_disable(); /* protect from TOD sync and vcpu_load/put */ 2264 __enable_cpu_timer_accounting(vcpu); 2265 preempt_enable(); 2266 } 2267 2268 static void disable_cpu_timer_accounting(struct kvm_vcpu *vcpu) 2269 { 2270 preempt_disable(); /* protect from TOD sync and vcpu_load/put */ 2271 __disable_cpu_timer_accounting(vcpu); 2272 preempt_enable(); 2273 } 2274 2275 /* set the cpu timer - may only be called from the VCPU thread itself */ 2276 void kvm_s390_set_cpu_timer(struct kvm_vcpu *vcpu, __u64 cputm) 2277 { 2278 preempt_disable(); /* protect from TOD sync and vcpu_load/put */ 2279 raw_write_seqcount_begin(&vcpu->arch.cputm_seqcount); 2280 if (vcpu->arch.cputm_enabled) 2281 vcpu->arch.cputm_start = get_tod_clock_fast(); 2282 vcpu->arch.sie_block->cputm = cputm; 2283 raw_write_seqcount_end(&vcpu->arch.cputm_seqcount); 2284 preempt_enable(); 2285 } 2286 2287 /* update and get the cpu timer - can also be called from other VCPU threads */ 2288 __u64 kvm_s390_get_cpu_timer(struct kvm_vcpu *vcpu) 2289 { 2290 unsigned int seq; 2291 __u64 value; 2292 2293 if (unlikely(!vcpu->arch.cputm_enabled)) 2294 return vcpu->arch.sie_block->cputm; 2295 2296 preempt_disable(); /* protect from TOD sync and vcpu_load/put */ 2297 do { 2298 seq = raw_read_seqcount(&vcpu->arch.cputm_seqcount); 2299 /* 2300 * If the writer would ever execute a read in the critical 2301 * section, e.g. in irq context, we have a deadlock. 2302 */ 2303 WARN_ON_ONCE((seq & 1) && smp_processor_id() == vcpu->cpu); 2304 value = vcpu->arch.sie_block->cputm; 2305 /* if cputm_start is 0, accounting is being started/stopped */ 2306 if (likely(vcpu->arch.cputm_start)) 2307 value -= get_tod_clock_fast() - vcpu->arch.cputm_start; 2308 } while (read_seqcount_retry(&vcpu->arch.cputm_seqcount, seq & ~1)); 2309 preempt_enable(); 2310 return value; 2311 } 2312 2313 void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu) 2314 { 2315 2316 gmap_enable(vcpu->arch.enabled_gmap); 2317 atomic_or(CPUSTAT_RUNNING, &vcpu->arch.sie_block->cpuflags); 2318 if (vcpu->arch.cputm_enabled && !is_vcpu_idle(vcpu)) 2319 __start_cpu_timer_accounting(vcpu); 2320 vcpu->cpu = cpu; 2321 } 2322 2323 void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu) 2324 { 2325 vcpu->cpu = -1; 2326 if (vcpu->arch.cputm_enabled && !is_vcpu_idle(vcpu)) 2327 __stop_cpu_timer_accounting(vcpu); 2328 atomic_andnot(CPUSTAT_RUNNING, &vcpu->arch.sie_block->cpuflags); 2329 vcpu->arch.enabled_gmap = gmap_get_enabled(); 2330 gmap_disable(vcpu->arch.enabled_gmap); 2331 2332 } 2333 2334 static void kvm_s390_vcpu_initial_reset(struct kvm_vcpu *vcpu) 2335 { 2336 /* this equals initial cpu reset in pop, but we don't switch to ESA */ 2337 vcpu->arch.sie_block->gpsw.mask = 0UL; 2338 vcpu->arch.sie_block->gpsw.addr = 0UL; 2339 kvm_s390_set_prefix(vcpu, 0); 2340 kvm_s390_set_cpu_timer(vcpu, 0); 2341 vcpu->arch.sie_block->ckc = 0UL; 2342 vcpu->arch.sie_block->todpr = 0; 2343 memset(vcpu->arch.sie_block->gcr, 0, 16 * sizeof(__u64)); 2344 vcpu->arch.sie_block->gcr[0] = 0xE0UL; 2345 vcpu->arch.sie_block->gcr[14] = 0xC2000000UL; 2346 /* make sure the new fpc will be lazily loaded */ 2347 save_fpu_regs(); 2348 current->thread.fpu.fpc = 0; 2349 vcpu->arch.sie_block->gbea = 1; 2350 vcpu->arch.sie_block->pp = 0; 2351 vcpu->arch.sie_block->fpf &= ~FPF_BPBC; 2352 vcpu->arch.pfault_token = KVM_S390_PFAULT_TOKEN_INVALID; 2353 kvm_clear_async_pf_completion_queue(vcpu); 2354 if (!kvm_s390_user_cpu_state_ctrl(vcpu->kvm)) 2355 kvm_s390_vcpu_stop(vcpu); 2356 kvm_s390_clear_local_irqs(vcpu); 2357 } 2358 2359 void kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu) 2360 { 2361 mutex_lock(&vcpu->kvm->lock); 2362 preempt_disable(); 2363 vcpu->arch.sie_block->epoch = vcpu->kvm->arch.epoch; 2364 preempt_enable(); 2365 mutex_unlock(&vcpu->kvm->lock); 2366 if (!kvm_is_ucontrol(vcpu->kvm)) { 2367 vcpu->arch.gmap = vcpu->kvm->arch.gmap; 2368 sca_add_vcpu(vcpu); 2369 } 2370 if (test_kvm_facility(vcpu->kvm, 74) || vcpu->kvm->arch.user_instr0) 2371 vcpu->arch.sie_block->ictl |= ICTL_OPEREXC; 2372 /* make vcpu_load load the right gmap on the first trigger */ 2373 vcpu->arch.enabled_gmap = vcpu->arch.gmap; 2374 } 2375 2376 static void kvm_s390_vcpu_crypto_setup(struct kvm_vcpu *vcpu) 2377 { 2378 if (!test_kvm_facility(vcpu->kvm, 76)) 2379 return; 2380 2381 vcpu->arch.sie_block->ecb3 &= ~(ECB3_AES | ECB3_DEA); 2382 2383 if (vcpu->kvm->arch.crypto.aes_kw) 2384 vcpu->arch.sie_block->ecb3 |= ECB3_AES; 2385 if (vcpu->kvm->arch.crypto.dea_kw) 2386 vcpu->arch.sie_block->ecb3 |= ECB3_DEA; 2387 2388 vcpu->arch.sie_block->crycbd = vcpu->kvm->arch.crypto.crycbd; 2389 } 2390 2391 void kvm_s390_vcpu_unsetup_cmma(struct kvm_vcpu *vcpu) 2392 { 2393 free_page(vcpu->arch.sie_block->cbrlo); 2394 vcpu->arch.sie_block->cbrlo = 0; 2395 } 2396 2397 int kvm_s390_vcpu_setup_cmma(struct kvm_vcpu *vcpu) 2398 { 2399 vcpu->arch.sie_block->cbrlo = get_zeroed_page(GFP_KERNEL); 2400 if (!vcpu->arch.sie_block->cbrlo) 2401 return -ENOMEM; 2402 2403 vcpu->arch.sie_block->ecb2 &= ~ECB2_PFMFI; 2404 return 0; 2405 } 2406 2407 static void kvm_s390_vcpu_setup_model(struct kvm_vcpu *vcpu) 2408 { 2409 struct kvm_s390_cpu_model *model = &vcpu->kvm->arch.model; 2410 2411 vcpu->arch.sie_block->ibc = model->ibc; 2412 if (test_kvm_facility(vcpu->kvm, 7)) 2413 vcpu->arch.sie_block->fac = (u32)(u64) model->fac_list; 2414 } 2415 2416 int kvm_arch_vcpu_setup(struct kvm_vcpu *vcpu) 2417 { 2418 int rc = 0; 2419 2420 atomic_set(&vcpu->arch.sie_block->cpuflags, CPUSTAT_ZARCH | 2421 CPUSTAT_SM | 2422 CPUSTAT_STOPPED); 2423 2424 if (test_kvm_facility(vcpu->kvm, 78)) 2425 atomic_or(CPUSTAT_GED2, &vcpu->arch.sie_block->cpuflags); 2426 else if (test_kvm_facility(vcpu->kvm, 8)) 2427 atomic_or(CPUSTAT_GED, &vcpu->arch.sie_block->cpuflags); 2428 2429 kvm_s390_vcpu_setup_model(vcpu); 2430 2431 /* pgste_set_pte has special handling for !MACHINE_HAS_ESOP */ 2432 if (MACHINE_HAS_ESOP) 2433 vcpu->arch.sie_block->ecb |= ECB_HOSTPROTINT; 2434 if (test_kvm_facility(vcpu->kvm, 9)) 2435 vcpu->arch.sie_block->ecb |= ECB_SRSI; 2436 if (test_kvm_facility(vcpu->kvm, 73)) 2437 vcpu->arch.sie_block->ecb |= ECB_TE; 2438 2439 if (test_kvm_facility(vcpu->kvm, 8) && sclp.has_pfmfi) 2440 vcpu->arch.sie_block->ecb2 |= ECB2_PFMFI; 2441 if (test_kvm_facility(vcpu->kvm, 130)) 2442 vcpu->arch.sie_block->ecb2 |= ECB2_IEP; 2443 vcpu->arch.sie_block->eca = ECA_MVPGI | ECA_PROTEXCI; 2444 if (sclp.has_cei) 2445 vcpu->arch.sie_block->eca |= ECA_CEI; 2446 if (sclp.has_ib) 2447 vcpu->arch.sie_block->eca |= ECA_IB; 2448 if (sclp.has_siif) 2449 vcpu->arch.sie_block->eca |= ECA_SII; 2450 if (sclp.has_sigpif) 2451 vcpu->arch.sie_block->eca |= ECA_SIGPI; 2452 if (test_kvm_facility(vcpu->kvm, 129)) { 2453 vcpu->arch.sie_block->eca |= ECA_VX; 2454 vcpu->arch.sie_block->ecd |= ECD_HOSTREGMGMT; 2455 } 2456 if (test_kvm_facility(vcpu->kvm, 139)) 2457 vcpu->arch.sie_block->ecd |= ECD_MEF; 2458 2459 vcpu->arch.sie_block->sdnxo = ((unsigned long) &vcpu->run->s.regs.sdnx) 2460 | SDNXC; 2461 vcpu->arch.sie_block->riccbd = (unsigned long) &vcpu->run->s.regs.riccb; 2462 2463 if (sclp.has_kss) 2464 atomic_or(CPUSTAT_KSS, &vcpu->arch.sie_block->cpuflags); 2465 else 2466 vcpu->arch.sie_block->ictl |= ICTL_ISKE | ICTL_SSKE | ICTL_RRBE; 2467 2468 if (vcpu->kvm->arch.use_cmma) { 2469 rc = kvm_s390_vcpu_setup_cmma(vcpu); 2470 if (rc) 2471 return rc; 2472 } 2473 hrtimer_init(&vcpu->arch.ckc_timer, CLOCK_MONOTONIC, HRTIMER_MODE_REL); 2474 vcpu->arch.ckc_timer.function = kvm_s390_idle_wakeup; 2475 2476 kvm_s390_vcpu_crypto_setup(vcpu); 2477 2478 return rc; 2479 } 2480 2481 struct kvm_vcpu *kvm_arch_vcpu_create(struct kvm *kvm, 2482 unsigned int id) 2483 { 2484 struct kvm_vcpu *vcpu; 2485 struct sie_page *sie_page; 2486 int rc = -EINVAL; 2487 2488 if (!kvm_is_ucontrol(kvm) && !sca_can_add_vcpu(kvm, id)) 2489 goto out; 2490 2491 rc = -ENOMEM; 2492 2493 vcpu = kmem_cache_zalloc(kvm_vcpu_cache, GFP_KERNEL); 2494 if (!vcpu) 2495 goto out; 2496 2497 BUILD_BUG_ON(sizeof(struct sie_page) != 4096); 2498 sie_page = (struct sie_page *) get_zeroed_page(GFP_KERNEL); 2499 if (!sie_page) 2500 goto out_free_cpu; 2501 2502 vcpu->arch.sie_block = &sie_page->sie_block; 2503 vcpu->arch.sie_block->itdba = (unsigned long) &sie_page->itdb; 2504 2505 /* the real guest size will always be smaller than msl */ 2506 vcpu->arch.sie_block->mso = 0; 2507 vcpu->arch.sie_block->msl = sclp.hamax; 2508 2509 vcpu->arch.sie_block->icpua = id; 2510 spin_lock_init(&vcpu->arch.local_int.lock); 2511 vcpu->arch.local_int.float_int = &kvm->arch.float_int; 2512 vcpu->arch.local_int.wq = &vcpu->wq; 2513 vcpu->arch.local_int.cpuflags = &vcpu->arch.sie_block->cpuflags; 2514 seqcount_init(&vcpu->arch.cputm_seqcount); 2515 2516 rc = kvm_vcpu_init(vcpu, kvm, id); 2517 if (rc) 2518 goto out_free_sie_block; 2519 VM_EVENT(kvm, 3, "create cpu %d at 0x%pK, sie block at 0x%pK", id, vcpu, 2520 vcpu->arch.sie_block); 2521 trace_kvm_s390_create_vcpu(id, vcpu, vcpu->arch.sie_block); 2522 2523 return vcpu; 2524 out_free_sie_block: 2525 free_page((unsigned long)(vcpu->arch.sie_block)); 2526 out_free_cpu: 2527 kmem_cache_free(kvm_vcpu_cache, vcpu); 2528 out: 2529 return ERR_PTR(rc); 2530 } 2531 2532 int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu) 2533 { 2534 return kvm_s390_vcpu_has_irq(vcpu, 0); 2535 } 2536 2537 bool kvm_arch_vcpu_in_kernel(struct kvm_vcpu *vcpu) 2538 { 2539 return !(vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE); 2540 } 2541 2542 void kvm_s390_vcpu_block(struct kvm_vcpu *vcpu) 2543 { 2544 atomic_or(PROG_BLOCK_SIE, &vcpu->arch.sie_block->prog20); 2545 exit_sie(vcpu); 2546 } 2547 2548 void kvm_s390_vcpu_unblock(struct kvm_vcpu *vcpu) 2549 { 2550 atomic_andnot(PROG_BLOCK_SIE, &vcpu->arch.sie_block->prog20); 2551 } 2552 2553 static void kvm_s390_vcpu_request(struct kvm_vcpu *vcpu) 2554 { 2555 atomic_or(PROG_REQUEST, &vcpu->arch.sie_block->prog20); 2556 exit_sie(vcpu); 2557 } 2558 2559 static void kvm_s390_vcpu_request_handled(struct kvm_vcpu *vcpu) 2560 { 2561 atomic_andnot(PROG_REQUEST, &vcpu->arch.sie_block->prog20); 2562 } 2563 2564 /* 2565 * Kick a guest cpu out of SIE and wait until SIE is not running. 2566 * If the CPU is not running (e.g. waiting as idle) the function will 2567 * return immediately. */ 2568 void exit_sie(struct kvm_vcpu *vcpu) 2569 { 2570 atomic_or(CPUSTAT_STOP_INT, &vcpu->arch.sie_block->cpuflags); 2571 while (vcpu->arch.sie_block->prog0c & PROG_IN_SIE) 2572 cpu_relax(); 2573 } 2574 2575 /* Kick a guest cpu out of SIE to process a request synchronously */ 2576 void kvm_s390_sync_request(int req, struct kvm_vcpu *vcpu) 2577 { 2578 kvm_make_request(req, vcpu); 2579 kvm_s390_vcpu_request(vcpu); 2580 } 2581 2582 static void kvm_gmap_notifier(struct gmap *gmap, unsigned long start, 2583 unsigned long end) 2584 { 2585 struct kvm *kvm = gmap->private; 2586 struct kvm_vcpu *vcpu; 2587 unsigned long prefix; 2588 int i; 2589 2590 if (gmap_is_shadow(gmap)) 2591 return; 2592 if (start >= 1UL << 31) 2593 /* We are only interested in prefix pages */ 2594 return; 2595 kvm_for_each_vcpu(i, vcpu, kvm) { 2596 /* match against both prefix pages */ 2597 prefix = kvm_s390_get_prefix(vcpu); 2598 if (prefix <= end && start <= prefix + 2*PAGE_SIZE - 1) { 2599 VCPU_EVENT(vcpu, 2, "gmap notifier for %lx-%lx", 2600 start, end); 2601 kvm_s390_sync_request(KVM_REQ_MMU_RELOAD, vcpu); 2602 } 2603 } 2604 } 2605 2606 int kvm_arch_vcpu_should_kick(struct kvm_vcpu *vcpu) 2607 { 2608 /* kvm common code refers to this, but never calls it */ 2609 BUG(); 2610 return 0; 2611 } 2612 2613 static int kvm_arch_vcpu_ioctl_get_one_reg(struct kvm_vcpu *vcpu, 2614 struct kvm_one_reg *reg) 2615 { 2616 int r = -EINVAL; 2617 2618 switch (reg->id) { 2619 case KVM_REG_S390_TODPR: 2620 r = put_user(vcpu->arch.sie_block->todpr, 2621 (u32 __user *)reg->addr); 2622 break; 2623 case KVM_REG_S390_EPOCHDIFF: 2624 r = put_user(vcpu->arch.sie_block->epoch, 2625 (u64 __user *)reg->addr); 2626 break; 2627 case KVM_REG_S390_CPU_TIMER: 2628 r = put_user(kvm_s390_get_cpu_timer(vcpu), 2629 (u64 __user *)reg->addr); 2630 break; 2631 case KVM_REG_S390_CLOCK_COMP: 2632 r = put_user(vcpu->arch.sie_block->ckc, 2633 (u64 __user *)reg->addr); 2634 break; 2635 case KVM_REG_S390_PFTOKEN: 2636 r = put_user(vcpu->arch.pfault_token, 2637 (u64 __user *)reg->addr); 2638 break; 2639 case KVM_REG_S390_PFCOMPARE: 2640 r = put_user(vcpu->arch.pfault_compare, 2641 (u64 __user *)reg->addr); 2642 break; 2643 case KVM_REG_S390_PFSELECT: 2644 r = put_user(vcpu->arch.pfault_select, 2645 (u64 __user *)reg->addr); 2646 break; 2647 case KVM_REG_S390_PP: 2648 r = put_user(vcpu->arch.sie_block->pp, 2649 (u64 __user *)reg->addr); 2650 break; 2651 case KVM_REG_S390_GBEA: 2652 r = put_user(vcpu->arch.sie_block->gbea, 2653 (u64 __user *)reg->addr); 2654 break; 2655 default: 2656 break; 2657 } 2658 2659 return r; 2660 } 2661 2662 static int kvm_arch_vcpu_ioctl_set_one_reg(struct kvm_vcpu *vcpu, 2663 struct kvm_one_reg *reg) 2664 { 2665 int r = -EINVAL; 2666 __u64 val; 2667 2668 switch (reg->id) { 2669 case KVM_REG_S390_TODPR: 2670 r = get_user(vcpu->arch.sie_block->todpr, 2671 (u32 __user *)reg->addr); 2672 break; 2673 case KVM_REG_S390_EPOCHDIFF: 2674 r = get_user(vcpu->arch.sie_block->epoch, 2675 (u64 __user *)reg->addr); 2676 break; 2677 case KVM_REG_S390_CPU_TIMER: 2678 r = get_user(val, (u64 __user *)reg->addr); 2679 if (!r) 2680 kvm_s390_set_cpu_timer(vcpu, val); 2681 break; 2682 case KVM_REG_S390_CLOCK_COMP: 2683 r = get_user(vcpu->arch.sie_block->ckc, 2684 (u64 __user *)reg->addr); 2685 break; 2686 case KVM_REG_S390_PFTOKEN: 2687 r = get_user(vcpu->arch.pfault_token, 2688 (u64 __user *)reg->addr); 2689 if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID) 2690 kvm_clear_async_pf_completion_queue(vcpu); 2691 break; 2692 case KVM_REG_S390_PFCOMPARE: 2693 r = get_user(vcpu->arch.pfault_compare, 2694 (u64 __user *)reg->addr); 2695 break; 2696 case KVM_REG_S390_PFSELECT: 2697 r = get_user(vcpu->arch.pfault_select, 2698 (u64 __user *)reg->addr); 2699 break; 2700 case KVM_REG_S390_PP: 2701 r = get_user(vcpu->arch.sie_block->pp, 2702 (u64 __user *)reg->addr); 2703 break; 2704 case KVM_REG_S390_GBEA: 2705 r = get_user(vcpu->arch.sie_block->gbea, 2706 (u64 __user *)reg->addr); 2707 break; 2708 default: 2709 break; 2710 } 2711 2712 return r; 2713 } 2714 2715 static int kvm_arch_vcpu_ioctl_initial_reset(struct kvm_vcpu *vcpu) 2716 { 2717 kvm_s390_vcpu_initial_reset(vcpu); 2718 return 0; 2719 } 2720 2721 int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs) 2722 { 2723 memcpy(&vcpu->run->s.regs.gprs, ®s->gprs, sizeof(regs->gprs)); 2724 return 0; 2725 } 2726 2727 int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs) 2728 { 2729 memcpy(®s->gprs, &vcpu->run->s.regs.gprs, sizeof(regs->gprs)); 2730 return 0; 2731 } 2732 2733 int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu, 2734 struct kvm_sregs *sregs) 2735 { 2736 memcpy(&vcpu->run->s.regs.acrs, &sregs->acrs, sizeof(sregs->acrs)); 2737 memcpy(&vcpu->arch.sie_block->gcr, &sregs->crs, sizeof(sregs->crs)); 2738 return 0; 2739 } 2740 2741 int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu, 2742 struct kvm_sregs *sregs) 2743 { 2744 memcpy(&sregs->acrs, &vcpu->run->s.regs.acrs, sizeof(sregs->acrs)); 2745 memcpy(&sregs->crs, &vcpu->arch.sie_block->gcr, sizeof(sregs->crs)); 2746 return 0; 2747 } 2748 2749 int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu) 2750 { 2751 if (test_fp_ctl(fpu->fpc)) 2752 return -EINVAL; 2753 vcpu->run->s.regs.fpc = fpu->fpc; 2754 if (MACHINE_HAS_VX) 2755 convert_fp_to_vx((__vector128 *) vcpu->run->s.regs.vrs, 2756 (freg_t *) fpu->fprs); 2757 else 2758 memcpy(vcpu->run->s.regs.fprs, &fpu->fprs, sizeof(fpu->fprs)); 2759 return 0; 2760 } 2761 2762 int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu) 2763 { 2764 /* make sure we have the latest values */ 2765 save_fpu_regs(); 2766 if (MACHINE_HAS_VX) 2767 convert_vx_to_fp((freg_t *) fpu->fprs, 2768 (__vector128 *) vcpu->run->s.regs.vrs); 2769 else 2770 memcpy(fpu->fprs, vcpu->run->s.regs.fprs, sizeof(fpu->fprs)); 2771 fpu->fpc = vcpu->run->s.regs.fpc; 2772 return 0; 2773 } 2774 2775 static int kvm_arch_vcpu_ioctl_set_initial_psw(struct kvm_vcpu *vcpu, psw_t psw) 2776 { 2777 int rc = 0; 2778 2779 if (!is_vcpu_stopped(vcpu)) 2780 rc = -EBUSY; 2781 else { 2782 vcpu->run->psw_mask = psw.mask; 2783 vcpu->run->psw_addr = psw.addr; 2784 } 2785 return rc; 2786 } 2787 2788 int kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu *vcpu, 2789 struct kvm_translation *tr) 2790 { 2791 return -EINVAL; /* not implemented yet */ 2792 } 2793 2794 #define VALID_GUESTDBG_FLAGS (KVM_GUESTDBG_SINGLESTEP | \ 2795 KVM_GUESTDBG_USE_HW_BP | \ 2796 KVM_GUESTDBG_ENABLE) 2797 2798 int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu, 2799 struct kvm_guest_debug *dbg) 2800 { 2801 int rc = 0; 2802 2803 vcpu->guest_debug = 0; 2804 kvm_s390_clear_bp_data(vcpu); 2805 2806 if (dbg->control & ~VALID_GUESTDBG_FLAGS) 2807 return -EINVAL; 2808 if (!sclp.has_gpere) 2809 return -EINVAL; 2810 2811 if (dbg->control & KVM_GUESTDBG_ENABLE) { 2812 vcpu->guest_debug = dbg->control; 2813 /* enforce guest PER */ 2814 atomic_or(CPUSTAT_P, &vcpu->arch.sie_block->cpuflags); 2815 2816 if (dbg->control & KVM_GUESTDBG_USE_HW_BP) 2817 rc = kvm_s390_import_bp_data(vcpu, dbg); 2818 } else { 2819 atomic_andnot(CPUSTAT_P, &vcpu->arch.sie_block->cpuflags); 2820 vcpu->arch.guestdbg.last_bp = 0; 2821 } 2822 2823 if (rc) { 2824 vcpu->guest_debug = 0; 2825 kvm_s390_clear_bp_data(vcpu); 2826 atomic_andnot(CPUSTAT_P, &vcpu->arch.sie_block->cpuflags); 2827 } 2828 2829 return rc; 2830 } 2831 2832 int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu, 2833 struct kvm_mp_state *mp_state) 2834 { 2835 /* CHECK_STOP and LOAD are not supported yet */ 2836 return is_vcpu_stopped(vcpu) ? KVM_MP_STATE_STOPPED : 2837 KVM_MP_STATE_OPERATING; 2838 } 2839 2840 int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu, 2841 struct kvm_mp_state *mp_state) 2842 { 2843 int rc = 0; 2844 2845 /* user space knows about this interface - let it control the state */ 2846 vcpu->kvm->arch.user_cpu_state_ctrl = 1; 2847 2848 switch (mp_state->mp_state) { 2849 case KVM_MP_STATE_STOPPED: 2850 kvm_s390_vcpu_stop(vcpu); 2851 break; 2852 case KVM_MP_STATE_OPERATING: 2853 kvm_s390_vcpu_start(vcpu); 2854 break; 2855 case KVM_MP_STATE_LOAD: 2856 case KVM_MP_STATE_CHECK_STOP: 2857 /* fall through - CHECK_STOP and LOAD are not supported yet */ 2858 default: 2859 rc = -ENXIO; 2860 } 2861 2862 return rc; 2863 } 2864 2865 static bool ibs_enabled(struct kvm_vcpu *vcpu) 2866 { 2867 return atomic_read(&vcpu->arch.sie_block->cpuflags) & CPUSTAT_IBS; 2868 } 2869 2870 static int kvm_s390_handle_requests(struct kvm_vcpu *vcpu) 2871 { 2872 retry: 2873 kvm_s390_vcpu_request_handled(vcpu); 2874 if (!kvm_request_pending(vcpu)) 2875 return 0; 2876 /* 2877 * We use MMU_RELOAD just to re-arm the ipte notifier for the 2878 * guest prefix page. gmap_mprotect_notify will wait on the ptl lock. 2879 * This ensures that the ipte instruction for this request has 2880 * already finished. We might race against a second unmapper that 2881 * wants to set the blocking bit. Lets just retry the request loop. 2882 */ 2883 if (kvm_check_request(KVM_REQ_MMU_RELOAD, vcpu)) { 2884 int rc; 2885 rc = gmap_mprotect_notify(vcpu->arch.gmap, 2886 kvm_s390_get_prefix(vcpu), 2887 PAGE_SIZE * 2, PROT_WRITE); 2888 if (rc) { 2889 kvm_make_request(KVM_REQ_MMU_RELOAD, vcpu); 2890 return rc; 2891 } 2892 goto retry; 2893 } 2894 2895 if (kvm_check_request(KVM_REQ_TLB_FLUSH, vcpu)) { 2896 vcpu->arch.sie_block->ihcpu = 0xffff; 2897 goto retry; 2898 } 2899 2900 if (kvm_check_request(KVM_REQ_ENABLE_IBS, vcpu)) { 2901 if (!ibs_enabled(vcpu)) { 2902 trace_kvm_s390_enable_disable_ibs(vcpu->vcpu_id, 1); 2903 atomic_or(CPUSTAT_IBS, 2904 &vcpu->arch.sie_block->cpuflags); 2905 } 2906 goto retry; 2907 } 2908 2909 if (kvm_check_request(KVM_REQ_DISABLE_IBS, vcpu)) { 2910 if (ibs_enabled(vcpu)) { 2911 trace_kvm_s390_enable_disable_ibs(vcpu->vcpu_id, 0); 2912 atomic_andnot(CPUSTAT_IBS, 2913 &vcpu->arch.sie_block->cpuflags); 2914 } 2915 goto retry; 2916 } 2917 2918 if (kvm_check_request(KVM_REQ_ICPT_OPEREXC, vcpu)) { 2919 vcpu->arch.sie_block->ictl |= ICTL_OPEREXC; 2920 goto retry; 2921 } 2922 2923 if (kvm_check_request(KVM_REQ_START_MIGRATION, vcpu)) { 2924 /* 2925 * Disable CMMA virtualization; we will emulate the ESSA 2926 * instruction manually, in order to provide additional 2927 * functionalities needed for live migration. 2928 */ 2929 vcpu->arch.sie_block->ecb2 &= ~ECB2_CMMA; 2930 goto retry; 2931 } 2932 2933 if (kvm_check_request(KVM_REQ_STOP_MIGRATION, vcpu)) { 2934 /* 2935 * Re-enable CMMA virtualization if CMMA is available and 2936 * was used. 2937 */ 2938 if ((vcpu->kvm->arch.use_cmma) && 2939 (vcpu->kvm->mm->context.use_cmma)) 2940 vcpu->arch.sie_block->ecb2 |= ECB2_CMMA; 2941 goto retry; 2942 } 2943 2944 /* nothing to do, just clear the request */ 2945 kvm_clear_request(KVM_REQ_UNHALT, vcpu); 2946 2947 return 0; 2948 } 2949 2950 void kvm_s390_set_tod_clock_ext(struct kvm *kvm, 2951 const struct kvm_s390_vm_tod_clock *gtod) 2952 { 2953 struct kvm_vcpu *vcpu; 2954 struct kvm_s390_tod_clock_ext htod; 2955 int i; 2956 2957 mutex_lock(&kvm->lock); 2958 preempt_disable(); 2959 2960 get_tod_clock_ext((char *)&htod); 2961 2962 kvm->arch.epoch = gtod->tod - htod.tod; 2963 kvm->arch.epdx = gtod->epoch_idx - htod.epoch_idx; 2964 2965 if (kvm->arch.epoch > gtod->tod) 2966 kvm->arch.epdx -= 1; 2967 2968 kvm_s390_vcpu_block_all(kvm); 2969 kvm_for_each_vcpu(i, vcpu, kvm) { 2970 vcpu->arch.sie_block->epoch = kvm->arch.epoch; 2971 vcpu->arch.sie_block->epdx = kvm->arch.epdx; 2972 } 2973 2974 kvm_s390_vcpu_unblock_all(kvm); 2975 preempt_enable(); 2976 mutex_unlock(&kvm->lock); 2977 } 2978 2979 void kvm_s390_set_tod_clock(struct kvm *kvm, u64 tod) 2980 { 2981 struct kvm_vcpu *vcpu; 2982 int i; 2983 2984 mutex_lock(&kvm->lock); 2985 preempt_disable(); 2986 kvm->arch.epoch = tod - get_tod_clock(); 2987 kvm_s390_vcpu_block_all(kvm); 2988 kvm_for_each_vcpu(i, vcpu, kvm) 2989 vcpu->arch.sie_block->epoch = kvm->arch.epoch; 2990 kvm_s390_vcpu_unblock_all(kvm); 2991 preempt_enable(); 2992 mutex_unlock(&kvm->lock); 2993 } 2994 2995 /** 2996 * kvm_arch_fault_in_page - fault-in guest page if necessary 2997 * @vcpu: The corresponding virtual cpu 2998 * @gpa: Guest physical address 2999 * @writable: Whether the page should be writable or not 3000 * 3001 * Make sure that a guest page has been faulted-in on the host. 3002 * 3003 * Return: Zero on success, negative error code otherwise. 3004 */ 3005 long kvm_arch_fault_in_page(struct kvm_vcpu *vcpu, gpa_t gpa, int writable) 3006 { 3007 return gmap_fault(vcpu->arch.gmap, gpa, 3008 writable ? FAULT_FLAG_WRITE : 0); 3009 } 3010 3011 static void __kvm_inject_pfault_token(struct kvm_vcpu *vcpu, bool start_token, 3012 unsigned long token) 3013 { 3014 struct kvm_s390_interrupt inti; 3015 struct kvm_s390_irq irq; 3016 3017 if (start_token) { 3018 irq.u.ext.ext_params2 = token; 3019 irq.type = KVM_S390_INT_PFAULT_INIT; 3020 WARN_ON_ONCE(kvm_s390_inject_vcpu(vcpu, &irq)); 3021 } else { 3022 inti.type = KVM_S390_INT_PFAULT_DONE; 3023 inti.parm64 = token; 3024 WARN_ON_ONCE(kvm_s390_inject_vm(vcpu->kvm, &inti)); 3025 } 3026 } 3027 3028 void kvm_arch_async_page_not_present(struct kvm_vcpu *vcpu, 3029 struct kvm_async_pf *work) 3030 { 3031 trace_kvm_s390_pfault_init(vcpu, work->arch.pfault_token); 3032 __kvm_inject_pfault_token(vcpu, true, work->arch.pfault_token); 3033 } 3034 3035 void kvm_arch_async_page_present(struct kvm_vcpu *vcpu, 3036 struct kvm_async_pf *work) 3037 { 3038 trace_kvm_s390_pfault_done(vcpu, work->arch.pfault_token); 3039 __kvm_inject_pfault_token(vcpu, false, work->arch.pfault_token); 3040 } 3041 3042 void kvm_arch_async_page_ready(struct kvm_vcpu *vcpu, 3043 struct kvm_async_pf *work) 3044 { 3045 /* s390 will always inject the page directly */ 3046 } 3047 3048 bool kvm_arch_can_inject_async_page_present(struct kvm_vcpu *vcpu) 3049 { 3050 /* 3051 * s390 will always inject the page directly, 3052 * but we still want check_async_completion to cleanup 3053 */ 3054 return true; 3055 } 3056 3057 static int kvm_arch_setup_async_pf(struct kvm_vcpu *vcpu) 3058 { 3059 hva_t hva; 3060 struct kvm_arch_async_pf arch; 3061 int rc; 3062 3063 if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID) 3064 return 0; 3065 if ((vcpu->arch.sie_block->gpsw.mask & vcpu->arch.pfault_select) != 3066 vcpu->arch.pfault_compare) 3067 return 0; 3068 if (psw_extint_disabled(vcpu)) 3069 return 0; 3070 if (kvm_s390_vcpu_has_irq(vcpu, 0)) 3071 return 0; 3072 if (!(vcpu->arch.sie_block->gcr[0] & 0x200ul)) 3073 return 0; 3074 if (!vcpu->arch.gmap->pfault_enabled) 3075 return 0; 3076 3077 hva = gfn_to_hva(vcpu->kvm, gpa_to_gfn(current->thread.gmap_addr)); 3078 hva += current->thread.gmap_addr & ~PAGE_MASK; 3079 if (read_guest_real(vcpu, vcpu->arch.pfault_token, &arch.pfault_token, 8)) 3080 return 0; 3081 3082 rc = kvm_setup_async_pf(vcpu, current->thread.gmap_addr, hva, &arch); 3083 return rc; 3084 } 3085 3086 static int vcpu_pre_run(struct kvm_vcpu *vcpu) 3087 { 3088 int rc, cpuflags; 3089 3090 /* 3091 * On s390 notifications for arriving pages will be delivered directly 3092 * to the guest but the house keeping for completed pfaults is 3093 * handled outside the worker. 3094 */ 3095 kvm_check_async_pf_completion(vcpu); 3096 3097 vcpu->arch.sie_block->gg14 = vcpu->run->s.regs.gprs[14]; 3098 vcpu->arch.sie_block->gg15 = vcpu->run->s.regs.gprs[15]; 3099 3100 if (need_resched()) 3101 schedule(); 3102 3103 if (test_cpu_flag(CIF_MCCK_PENDING)) 3104 s390_handle_mcck(); 3105 3106 if (!kvm_is_ucontrol(vcpu->kvm)) { 3107 rc = kvm_s390_deliver_pending_interrupts(vcpu); 3108 if (rc) 3109 return rc; 3110 } 3111 3112 rc = kvm_s390_handle_requests(vcpu); 3113 if (rc) 3114 return rc; 3115 3116 if (guestdbg_enabled(vcpu)) { 3117 kvm_s390_backup_guest_per_regs(vcpu); 3118 kvm_s390_patch_guest_per_regs(vcpu); 3119 } 3120 3121 vcpu->arch.sie_block->icptcode = 0; 3122 cpuflags = atomic_read(&vcpu->arch.sie_block->cpuflags); 3123 VCPU_EVENT(vcpu, 6, "entering sie flags %x", cpuflags); 3124 trace_kvm_s390_sie_enter(vcpu, cpuflags); 3125 3126 return 0; 3127 } 3128 3129 static int vcpu_post_run_fault_in_sie(struct kvm_vcpu *vcpu) 3130 { 3131 struct kvm_s390_pgm_info pgm_info = { 3132 .code = PGM_ADDRESSING, 3133 }; 3134 u8 opcode, ilen; 3135 int rc; 3136 3137 VCPU_EVENT(vcpu, 3, "%s", "fault in sie instruction"); 3138 trace_kvm_s390_sie_fault(vcpu); 3139 3140 /* 3141 * We want to inject an addressing exception, which is defined as a 3142 * suppressing or terminating exception. However, since we came here 3143 * by a DAT access exception, the PSW still points to the faulting 3144 * instruction since DAT exceptions are nullifying. So we've got 3145 * to look up the current opcode to get the length of the instruction 3146 * to be able to forward the PSW. 3147 */ 3148 rc = read_guest_instr(vcpu, vcpu->arch.sie_block->gpsw.addr, &opcode, 1); 3149 ilen = insn_length(opcode); 3150 if (rc < 0) { 3151 return rc; 3152 } else if (rc) { 3153 /* Instruction-Fetching Exceptions - we can't detect the ilen. 3154 * Forward by arbitrary ilc, injection will take care of 3155 * nullification if necessary. 3156 */ 3157 pgm_info = vcpu->arch.pgm; 3158 ilen = 4; 3159 } 3160 pgm_info.flags = ilen | KVM_S390_PGM_FLAGS_ILC_VALID; 3161 kvm_s390_forward_psw(vcpu, ilen); 3162 return kvm_s390_inject_prog_irq(vcpu, &pgm_info); 3163 } 3164 3165 static int vcpu_post_run(struct kvm_vcpu *vcpu, int exit_reason) 3166 { 3167 struct mcck_volatile_info *mcck_info; 3168 struct sie_page *sie_page; 3169 3170 VCPU_EVENT(vcpu, 6, "exit sie icptcode %d", 3171 vcpu->arch.sie_block->icptcode); 3172 trace_kvm_s390_sie_exit(vcpu, vcpu->arch.sie_block->icptcode); 3173 3174 if (guestdbg_enabled(vcpu)) 3175 kvm_s390_restore_guest_per_regs(vcpu); 3176 3177 vcpu->run->s.regs.gprs[14] = vcpu->arch.sie_block->gg14; 3178 vcpu->run->s.regs.gprs[15] = vcpu->arch.sie_block->gg15; 3179 3180 if (exit_reason == -EINTR) { 3181 VCPU_EVENT(vcpu, 3, "%s", "machine check"); 3182 sie_page = container_of(vcpu->arch.sie_block, 3183 struct sie_page, sie_block); 3184 mcck_info = &sie_page->mcck_info; 3185 kvm_s390_reinject_machine_check(vcpu, mcck_info); 3186 return 0; 3187 } 3188 3189 if (vcpu->arch.sie_block->icptcode > 0) { 3190 int rc = kvm_handle_sie_intercept(vcpu); 3191 3192 if (rc != -EOPNOTSUPP) 3193 return rc; 3194 vcpu->run->exit_reason = KVM_EXIT_S390_SIEIC; 3195 vcpu->run->s390_sieic.icptcode = vcpu->arch.sie_block->icptcode; 3196 vcpu->run->s390_sieic.ipa = vcpu->arch.sie_block->ipa; 3197 vcpu->run->s390_sieic.ipb = vcpu->arch.sie_block->ipb; 3198 return -EREMOTE; 3199 } else if (exit_reason != -EFAULT) { 3200 vcpu->stat.exit_null++; 3201 return 0; 3202 } else if (kvm_is_ucontrol(vcpu->kvm)) { 3203 vcpu->run->exit_reason = KVM_EXIT_S390_UCONTROL; 3204 vcpu->run->s390_ucontrol.trans_exc_code = 3205 current->thread.gmap_addr; 3206 vcpu->run->s390_ucontrol.pgm_code = 0x10; 3207 return -EREMOTE; 3208 } else if (current->thread.gmap_pfault) { 3209 trace_kvm_s390_major_guest_pfault(vcpu); 3210 current->thread.gmap_pfault = 0; 3211 if (kvm_arch_setup_async_pf(vcpu)) 3212 return 0; 3213 return kvm_arch_fault_in_page(vcpu, current->thread.gmap_addr, 1); 3214 } 3215 return vcpu_post_run_fault_in_sie(vcpu); 3216 } 3217 3218 static int __vcpu_run(struct kvm_vcpu *vcpu) 3219 { 3220 int rc, exit_reason; 3221 3222 /* 3223 * We try to hold kvm->srcu during most of vcpu_run (except when run- 3224 * ning the guest), so that memslots (and other stuff) are protected 3225 */ 3226 vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu); 3227 3228 do { 3229 rc = vcpu_pre_run(vcpu); 3230 if (rc) 3231 break; 3232 3233 srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx); 3234 /* 3235 * As PF_VCPU will be used in fault handler, between 3236 * guest_enter and guest_exit should be no uaccess. 3237 */ 3238 local_irq_disable(); 3239 guest_enter_irqoff(); 3240 __disable_cpu_timer_accounting(vcpu); 3241 local_irq_enable(); 3242 exit_reason = sie64a(vcpu->arch.sie_block, 3243 vcpu->run->s.regs.gprs); 3244 local_irq_disable(); 3245 __enable_cpu_timer_accounting(vcpu); 3246 guest_exit_irqoff(); 3247 local_irq_enable(); 3248 vcpu->srcu_idx = srcu_read_lock(&vcpu->kvm->srcu); 3249 3250 rc = vcpu_post_run(vcpu, exit_reason); 3251 } while (!signal_pending(current) && !guestdbg_exit_pending(vcpu) && !rc); 3252 3253 srcu_read_unlock(&vcpu->kvm->srcu, vcpu->srcu_idx); 3254 return rc; 3255 } 3256 3257 static void sync_regs(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) 3258 { 3259 struct runtime_instr_cb *riccb; 3260 struct gs_cb *gscb; 3261 3262 riccb = (struct runtime_instr_cb *) &kvm_run->s.regs.riccb; 3263 gscb = (struct gs_cb *) &kvm_run->s.regs.gscb; 3264 vcpu->arch.sie_block->gpsw.mask = kvm_run->psw_mask; 3265 vcpu->arch.sie_block->gpsw.addr = kvm_run->psw_addr; 3266 if (kvm_run->kvm_dirty_regs & KVM_SYNC_PREFIX) 3267 kvm_s390_set_prefix(vcpu, kvm_run->s.regs.prefix); 3268 if (kvm_run->kvm_dirty_regs & KVM_SYNC_CRS) { 3269 memcpy(&vcpu->arch.sie_block->gcr, &kvm_run->s.regs.crs, 128); 3270 /* some control register changes require a tlb flush */ 3271 kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu); 3272 } 3273 if (kvm_run->kvm_dirty_regs & KVM_SYNC_ARCH0) { 3274 kvm_s390_set_cpu_timer(vcpu, kvm_run->s.regs.cputm); 3275 vcpu->arch.sie_block->ckc = kvm_run->s.regs.ckc; 3276 vcpu->arch.sie_block->todpr = kvm_run->s.regs.todpr; 3277 vcpu->arch.sie_block->pp = kvm_run->s.regs.pp; 3278 vcpu->arch.sie_block->gbea = kvm_run->s.regs.gbea; 3279 } 3280 if (kvm_run->kvm_dirty_regs & KVM_SYNC_PFAULT) { 3281 vcpu->arch.pfault_token = kvm_run->s.regs.pft; 3282 vcpu->arch.pfault_select = kvm_run->s.regs.pfs; 3283 vcpu->arch.pfault_compare = kvm_run->s.regs.pfc; 3284 if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID) 3285 kvm_clear_async_pf_completion_queue(vcpu); 3286 } 3287 /* 3288 * If userspace sets the riccb (e.g. after migration) to a valid state, 3289 * we should enable RI here instead of doing the lazy enablement. 3290 */ 3291 if ((kvm_run->kvm_dirty_regs & KVM_SYNC_RICCB) && 3292 test_kvm_facility(vcpu->kvm, 64) && 3293 riccb->v && 3294 !(vcpu->arch.sie_block->ecb3 & ECB3_RI)) { 3295 VCPU_EVENT(vcpu, 3, "%s", "ENABLE: RI (sync_regs)"); 3296 vcpu->arch.sie_block->ecb3 |= ECB3_RI; 3297 } 3298 /* 3299 * If userspace sets the gscb (e.g. after migration) to non-zero, 3300 * we should enable GS here instead of doing the lazy enablement. 3301 */ 3302 if ((kvm_run->kvm_dirty_regs & KVM_SYNC_GSCB) && 3303 test_kvm_facility(vcpu->kvm, 133) && 3304 gscb->gssm && 3305 !vcpu->arch.gs_enabled) { 3306 VCPU_EVENT(vcpu, 3, "%s", "ENABLE: GS (sync_regs)"); 3307 vcpu->arch.sie_block->ecb |= ECB_GS; 3308 vcpu->arch.sie_block->ecd |= ECD_HOSTREGMGMT; 3309 vcpu->arch.gs_enabled = 1; 3310 } 3311 if ((kvm_run->kvm_dirty_regs & KVM_SYNC_BPBC) && 3312 test_kvm_facility(vcpu->kvm, 82)) { 3313 vcpu->arch.sie_block->fpf &= ~FPF_BPBC; 3314 vcpu->arch.sie_block->fpf |= kvm_run->s.regs.bpbc ? FPF_BPBC : 0; 3315 } 3316 save_access_regs(vcpu->arch.host_acrs); 3317 restore_access_regs(vcpu->run->s.regs.acrs); 3318 /* save host (userspace) fprs/vrs */ 3319 save_fpu_regs(); 3320 vcpu->arch.host_fpregs.fpc = current->thread.fpu.fpc; 3321 vcpu->arch.host_fpregs.regs = current->thread.fpu.regs; 3322 if (MACHINE_HAS_VX) 3323 current->thread.fpu.regs = vcpu->run->s.regs.vrs; 3324 else 3325 current->thread.fpu.regs = vcpu->run->s.regs.fprs; 3326 current->thread.fpu.fpc = vcpu->run->s.regs.fpc; 3327 if (test_fp_ctl(current->thread.fpu.fpc)) 3328 /* User space provided an invalid FPC, let's clear it */ 3329 current->thread.fpu.fpc = 0; 3330 if (MACHINE_HAS_GS) { 3331 preempt_disable(); 3332 __ctl_set_bit(2, 4); 3333 if (current->thread.gs_cb) { 3334 vcpu->arch.host_gscb = current->thread.gs_cb; 3335 save_gs_cb(vcpu->arch.host_gscb); 3336 } 3337 if (vcpu->arch.gs_enabled) { 3338 current->thread.gs_cb = (struct gs_cb *) 3339 &vcpu->run->s.regs.gscb; 3340 restore_gs_cb(current->thread.gs_cb); 3341 } 3342 preempt_enable(); 3343 } 3344 3345 kvm_run->kvm_dirty_regs = 0; 3346 } 3347 3348 static void store_regs(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) 3349 { 3350 kvm_run->psw_mask = vcpu->arch.sie_block->gpsw.mask; 3351 kvm_run->psw_addr = vcpu->arch.sie_block->gpsw.addr; 3352 kvm_run->s.regs.prefix = kvm_s390_get_prefix(vcpu); 3353 memcpy(&kvm_run->s.regs.crs, &vcpu->arch.sie_block->gcr, 128); 3354 kvm_run->s.regs.cputm = kvm_s390_get_cpu_timer(vcpu); 3355 kvm_run->s.regs.ckc = vcpu->arch.sie_block->ckc; 3356 kvm_run->s.regs.todpr = vcpu->arch.sie_block->todpr; 3357 kvm_run->s.regs.pp = vcpu->arch.sie_block->pp; 3358 kvm_run->s.regs.gbea = vcpu->arch.sie_block->gbea; 3359 kvm_run->s.regs.pft = vcpu->arch.pfault_token; 3360 kvm_run->s.regs.pfs = vcpu->arch.pfault_select; 3361 kvm_run->s.regs.pfc = vcpu->arch.pfault_compare; 3362 kvm_run->s.regs.bpbc = (vcpu->arch.sie_block->fpf & FPF_BPBC) == FPF_BPBC; 3363 save_access_regs(vcpu->run->s.regs.acrs); 3364 restore_access_regs(vcpu->arch.host_acrs); 3365 /* Save guest register state */ 3366 save_fpu_regs(); 3367 vcpu->run->s.regs.fpc = current->thread.fpu.fpc; 3368 /* Restore will be done lazily at return */ 3369 current->thread.fpu.fpc = vcpu->arch.host_fpregs.fpc; 3370 current->thread.fpu.regs = vcpu->arch.host_fpregs.regs; 3371 if (MACHINE_HAS_GS) { 3372 __ctl_set_bit(2, 4); 3373 if (vcpu->arch.gs_enabled) 3374 save_gs_cb(current->thread.gs_cb); 3375 preempt_disable(); 3376 current->thread.gs_cb = vcpu->arch.host_gscb; 3377 restore_gs_cb(vcpu->arch.host_gscb); 3378 preempt_enable(); 3379 if (!vcpu->arch.host_gscb) 3380 __ctl_clear_bit(2, 4); 3381 vcpu->arch.host_gscb = NULL; 3382 } 3383 3384 } 3385 3386 int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu, struct kvm_run *kvm_run) 3387 { 3388 int rc; 3389 3390 if (kvm_run->immediate_exit) 3391 return -EINTR; 3392 3393 if (guestdbg_exit_pending(vcpu)) { 3394 kvm_s390_prepare_debug_exit(vcpu); 3395 return 0; 3396 } 3397 3398 kvm_sigset_activate(vcpu); 3399 3400 if (!kvm_s390_user_cpu_state_ctrl(vcpu->kvm)) { 3401 kvm_s390_vcpu_start(vcpu); 3402 } else if (is_vcpu_stopped(vcpu)) { 3403 pr_err_ratelimited("can't run stopped vcpu %d\n", 3404 vcpu->vcpu_id); 3405 return -EINVAL; 3406 } 3407 3408 sync_regs(vcpu, kvm_run); 3409 enable_cpu_timer_accounting(vcpu); 3410 3411 might_fault(); 3412 rc = __vcpu_run(vcpu); 3413 3414 if (signal_pending(current) && !rc) { 3415 kvm_run->exit_reason = KVM_EXIT_INTR; 3416 rc = -EINTR; 3417 } 3418 3419 if (guestdbg_exit_pending(vcpu) && !rc) { 3420 kvm_s390_prepare_debug_exit(vcpu); 3421 rc = 0; 3422 } 3423 3424 if (rc == -EREMOTE) { 3425 /* userspace support is needed, kvm_run has been prepared */ 3426 rc = 0; 3427 } 3428 3429 disable_cpu_timer_accounting(vcpu); 3430 store_regs(vcpu, kvm_run); 3431 3432 kvm_sigset_deactivate(vcpu); 3433 3434 vcpu->stat.exit_userspace++; 3435 return rc; 3436 } 3437 3438 /* 3439 * store status at address 3440 * we use have two special cases: 3441 * KVM_S390_STORE_STATUS_NOADDR: -> 0x1200 on 64 bit 3442 * KVM_S390_STORE_STATUS_PREFIXED: -> prefix 3443 */ 3444 int kvm_s390_store_status_unloaded(struct kvm_vcpu *vcpu, unsigned long gpa) 3445 { 3446 unsigned char archmode = 1; 3447 freg_t fprs[NUM_FPRS]; 3448 unsigned int px; 3449 u64 clkcomp, cputm; 3450 int rc; 3451 3452 px = kvm_s390_get_prefix(vcpu); 3453 if (gpa == KVM_S390_STORE_STATUS_NOADDR) { 3454 if (write_guest_abs(vcpu, 163, &archmode, 1)) 3455 return -EFAULT; 3456 gpa = 0; 3457 } else if (gpa == KVM_S390_STORE_STATUS_PREFIXED) { 3458 if (write_guest_real(vcpu, 163, &archmode, 1)) 3459 return -EFAULT; 3460 gpa = px; 3461 } else 3462 gpa -= __LC_FPREGS_SAVE_AREA; 3463 3464 /* manually convert vector registers if necessary */ 3465 if (MACHINE_HAS_VX) { 3466 convert_vx_to_fp(fprs, (__vector128 *) vcpu->run->s.regs.vrs); 3467 rc = write_guest_abs(vcpu, gpa + __LC_FPREGS_SAVE_AREA, 3468 fprs, 128); 3469 } else { 3470 rc = write_guest_abs(vcpu, gpa + __LC_FPREGS_SAVE_AREA, 3471 vcpu->run->s.regs.fprs, 128); 3472 } 3473 rc |= write_guest_abs(vcpu, gpa + __LC_GPREGS_SAVE_AREA, 3474 vcpu->run->s.regs.gprs, 128); 3475 rc |= write_guest_abs(vcpu, gpa + __LC_PSW_SAVE_AREA, 3476 &vcpu->arch.sie_block->gpsw, 16); 3477 rc |= write_guest_abs(vcpu, gpa + __LC_PREFIX_SAVE_AREA, 3478 &px, 4); 3479 rc |= write_guest_abs(vcpu, gpa + __LC_FP_CREG_SAVE_AREA, 3480 &vcpu->run->s.regs.fpc, 4); 3481 rc |= write_guest_abs(vcpu, gpa + __LC_TOD_PROGREG_SAVE_AREA, 3482 &vcpu->arch.sie_block->todpr, 4); 3483 cputm = kvm_s390_get_cpu_timer(vcpu); 3484 rc |= write_guest_abs(vcpu, gpa + __LC_CPU_TIMER_SAVE_AREA, 3485 &cputm, 8); 3486 clkcomp = vcpu->arch.sie_block->ckc >> 8; 3487 rc |= write_guest_abs(vcpu, gpa + __LC_CLOCK_COMP_SAVE_AREA, 3488 &clkcomp, 8); 3489 rc |= write_guest_abs(vcpu, gpa + __LC_AREGS_SAVE_AREA, 3490 &vcpu->run->s.regs.acrs, 64); 3491 rc |= write_guest_abs(vcpu, gpa + __LC_CREGS_SAVE_AREA, 3492 &vcpu->arch.sie_block->gcr, 128); 3493 return rc ? -EFAULT : 0; 3494 } 3495 3496 int kvm_s390_vcpu_store_status(struct kvm_vcpu *vcpu, unsigned long addr) 3497 { 3498 /* 3499 * The guest FPRS and ACRS are in the host FPRS/ACRS due to the lazy 3500 * switch in the run ioctl. Let's update our copies before we save 3501 * it into the save area 3502 */ 3503 save_fpu_regs(); 3504 vcpu->run->s.regs.fpc = current->thread.fpu.fpc; 3505 save_access_regs(vcpu->run->s.regs.acrs); 3506 3507 return kvm_s390_store_status_unloaded(vcpu, addr); 3508 } 3509 3510 static void __disable_ibs_on_vcpu(struct kvm_vcpu *vcpu) 3511 { 3512 kvm_check_request(KVM_REQ_ENABLE_IBS, vcpu); 3513 kvm_s390_sync_request(KVM_REQ_DISABLE_IBS, vcpu); 3514 } 3515 3516 static void __disable_ibs_on_all_vcpus(struct kvm *kvm) 3517 { 3518 unsigned int i; 3519 struct kvm_vcpu *vcpu; 3520 3521 kvm_for_each_vcpu(i, vcpu, kvm) { 3522 __disable_ibs_on_vcpu(vcpu); 3523 } 3524 } 3525 3526 static void __enable_ibs_on_vcpu(struct kvm_vcpu *vcpu) 3527 { 3528 if (!sclp.has_ibs) 3529 return; 3530 kvm_check_request(KVM_REQ_DISABLE_IBS, vcpu); 3531 kvm_s390_sync_request(KVM_REQ_ENABLE_IBS, vcpu); 3532 } 3533 3534 void kvm_s390_vcpu_start(struct kvm_vcpu *vcpu) 3535 { 3536 int i, online_vcpus, started_vcpus = 0; 3537 3538 if (!is_vcpu_stopped(vcpu)) 3539 return; 3540 3541 trace_kvm_s390_vcpu_start_stop(vcpu->vcpu_id, 1); 3542 /* Only one cpu at a time may enter/leave the STOPPED state. */ 3543 spin_lock(&vcpu->kvm->arch.start_stop_lock); 3544 online_vcpus = atomic_read(&vcpu->kvm->online_vcpus); 3545 3546 for (i = 0; i < online_vcpus; i++) { 3547 if (!is_vcpu_stopped(vcpu->kvm->vcpus[i])) 3548 started_vcpus++; 3549 } 3550 3551 if (started_vcpus == 0) { 3552 /* we're the only active VCPU -> speed it up */ 3553 __enable_ibs_on_vcpu(vcpu); 3554 } else if (started_vcpus == 1) { 3555 /* 3556 * As we are starting a second VCPU, we have to disable 3557 * the IBS facility on all VCPUs to remove potentially 3558 * oustanding ENABLE requests. 3559 */ 3560 __disable_ibs_on_all_vcpus(vcpu->kvm); 3561 } 3562 3563 atomic_andnot(CPUSTAT_STOPPED, &vcpu->arch.sie_block->cpuflags); 3564 /* 3565 * Another VCPU might have used IBS while we were offline. 3566 * Let's play safe and flush the VCPU at startup. 3567 */ 3568 kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu); 3569 spin_unlock(&vcpu->kvm->arch.start_stop_lock); 3570 return; 3571 } 3572 3573 void kvm_s390_vcpu_stop(struct kvm_vcpu *vcpu) 3574 { 3575 int i, online_vcpus, started_vcpus = 0; 3576 struct kvm_vcpu *started_vcpu = NULL; 3577 3578 if (is_vcpu_stopped(vcpu)) 3579 return; 3580 3581 trace_kvm_s390_vcpu_start_stop(vcpu->vcpu_id, 0); 3582 /* Only one cpu at a time may enter/leave the STOPPED state. */ 3583 spin_lock(&vcpu->kvm->arch.start_stop_lock); 3584 online_vcpus = atomic_read(&vcpu->kvm->online_vcpus); 3585 3586 /* SIGP STOP and SIGP STOP AND STORE STATUS has been fully processed */ 3587 kvm_s390_clear_stop_irq(vcpu); 3588 3589 atomic_or(CPUSTAT_STOPPED, &vcpu->arch.sie_block->cpuflags); 3590 __disable_ibs_on_vcpu(vcpu); 3591 3592 for (i = 0; i < online_vcpus; i++) { 3593 if (!is_vcpu_stopped(vcpu->kvm->vcpus[i])) { 3594 started_vcpus++; 3595 started_vcpu = vcpu->kvm->vcpus[i]; 3596 } 3597 } 3598 3599 if (started_vcpus == 1) { 3600 /* 3601 * As we only have one VCPU left, we want to enable the 3602 * IBS facility for that VCPU to speed it up. 3603 */ 3604 __enable_ibs_on_vcpu(started_vcpu); 3605 } 3606 3607 spin_unlock(&vcpu->kvm->arch.start_stop_lock); 3608 return; 3609 } 3610 3611 static int kvm_vcpu_ioctl_enable_cap(struct kvm_vcpu *vcpu, 3612 struct kvm_enable_cap *cap) 3613 { 3614 int r; 3615 3616 if (cap->flags) 3617 return -EINVAL; 3618 3619 switch (cap->cap) { 3620 case KVM_CAP_S390_CSS_SUPPORT: 3621 if (!vcpu->kvm->arch.css_support) { 3622 vcpu->kvm->arch.css_support = 1; 3623 VM_EVENT(vcpu->kvm, 3, "%s", "ENABLE: CSS support"); 3624 trace_kvm_s390_enable_css(vcpu->kvm); 3625 } 3626 r = 0; 3627 break; 3628 default: 3629 r = -EINVAL; 3630 break; 3631 } 3632 return r; 3633 } 3634 3635 static long kvm_s390_guest_mem_op(struct kvm_vcpu *vcpu, 3636 struct kvm_s390_mem_op *mop) 3637 { 3638 void __user *uaddr = (void __user *)mop->buf; 3639 void *tmpbuf = NULL; 3640 int r, srcu_idx; 3641 const u64 supported_flags = KVM_S390_MEMOP_F_INJECT_EXCEPTION 3642 | KVM_S390_MEMOP_F_CHECK_ONLY; 3643 3644 if (mop->flags & ~supported_flags) 3645 return -EINVAL; 3646 3647 if (mop->size > MEM_OP_MAX_SIZE) 3648 return -E2BIG; 3649 3650 if (!(mop->flags & KVM_S390_MEMOP_F_CHECK_ONLY)) { 3651 tmpbuf = vmalloc(mop->size); 3652 if (!tmpbuf) 3653 return -ENOMEM; 3654 } 3655 3656 srcu_idx = srcu_read_lock(&vcpu->kvm->srcu); 3657 3658 switch (mop->op) { 3659 case KVM_S390_MEMOP_LOGICAL_READ: 3660 if (mop->flags & KVM_S390_MEMOP_F_CHECK_ONLY) { 3661 r = check_gva_range(vcpu, mop->gaddr, mop->ar, 3662 mop->size, GACC_FETCH); 3663 break; 3664 } 3665 r = read_guest(vcpu, mop->gaddr, mop->ar, tmpbuf, mop->size); 3666 if (r == 0) { 3667 if (copy_to_user(uaddr, tmpbuf, mop->size)) 3668 r = -EFAULT; 3669 } 3670 break; 3671 case KVM_S390_MEMOP_LOGICAL_WRITE: 3672 if (mop->flags & KVM_S390_MEMOP_F_CHECK_ONLY) { 3673 r = check_gva_range(vcpu, mop->gaddr, mop->ar, 3674 mop->size, GACC_STORE); 3675 break; 3676 } 3677 if (copy_from_user(tmpbuf, uaddr, mop->size)) { 3678 r = -EFAULT; 3679 break; 3680 } 3681 r = write_guest(vcpu, mop->gaddr, mop->ar, tmpbuf, mop->size); 3682 break; 3683 default: 3684 r = -EINVAL; 3685 } 3686 3687 srcu_read_unlock(&vcpu->kvm->srcu, srcu_idx); 3688 3689 if (r > 0 && (mop->flags & KVM_S390_MEMOP_F_INJECT_EXCEPTION) != 0) 3690 kvm_s390_inject_prog_irq(vcpu, &vcpu->arch.pgm); 3691 3692 vfree(tmpbuf); 3693 return r; 3694 } 3695 3696 long kvm_arch_vcpu_ioctl(struct file *filp, 3697 unsigned int ioctl, unsigned long arg) 3698 { 3699 struct kvm_vcpu *vcpu = filp->private_data; 3700 void __user *argp = (void __user *)arg; 3701 int idx; 3702 long r; 3703 3704 switch (ioctl) { 3705 case KVM_S390_IRQ: { 3706 struct kvm_s390_irq s390irq; 3707 3708 r = -EFAULT; 3709 if (copy_from_user(&s390irq, argp, sizeof(s390irq))) 3710 break; 3711 r = kvm_s390_inject_vcpu(vcpu, &s390irq); 3712 break; 3713 } 3714 case KVM_S390_INTERRUPT: { 3715 struct kvm_s390_interrupt s390int; 3716 struct kvm_s390_irq s390irq; 3717 3718 r = -EFAULT; 3719 if (copy_from_user(&s390int, argp, sizeof(s390int))) 3720 break; 3721 if (s390int_to_s390irq(&s390int, &s390irq)) 3722 return -EINVAL; 3723 r = kvm_s390_inject_vcpu(vcpu, &s390irq); 3724 break; 3725 } 3726 case KVM_S390_STORE_STATUS: 3727 idx = srcu_read_lock(&vcpu->kvm->srcu); 3728 r = kvm_s390_vcpu_store_status(vcpu, arg); 3729 srcu_read_unlock(&vcpu->kvm->srcu, idx); 3730 break; 3731 case KVM_S390_SET_INITIAL_PSW: { 3732 psw_t psw; 3733 3734 r = -EFAULT; 3735 if (copy_from_user(&psw, argp, sizeof(psw))) 3736 break; 3737 r = kvm_arch_vcpu_ioctl_set_initial_psw(vcpu, psw); 3738 break; 3739 } 3740 case KVM_S390_INITIAL_RESET: 3741 r = kvm_arch_vcpu_ioctl_initial_reset(vcpu); 3742 break; 3743 case KVM_SET_ONE_REG: 3744 case KVM_GET_ONE_REG: { 3745 struct kvm_one_reg reg; 3746 r = -EFAULT; 3747 if (copy_from_user(®, argp, sizeof(reg))) 3748 break; 3749 if (ioctl == KVM_SET_ONE_REG) 3750 r = kvm_arch_vcpu_ioctl_set_one_reg(vcpu, ®); 3751 else 3752 r = kvm_arch_vcpu_ioctl_get_one_reg(vcpu, ®); 3753 break; 3754 } 3755 #ifdef CONFIG_KVM_S390_UCONTROL 3756 case KVM_S390_UCAS_MAP: { 3757 struct kvm_s390_ucas_mapping ucasmap; 3758 3759 if (copy_from_user(&ucasmap, argp, sizeof(ucasmap))) { 3760 r = -EFAULT; 3761 break; 3762 } 3763 3764 if (!kvm_is_ucontrol(vcpu->kvm)) { 3765 r = -EINVAL; 3766 break; 3767 } 3768 3769 r = gmap_map_segment(vcpu->arch.gmap, ucasmap.user_addr, 3770 ucasmap.vcpu_addr, ucasmap.length); 3771 break; 3772 } 3773 case KVM_S390_UCAS_UNMAP: { 3774 struct kvm_s390_ucas_mapping ucasmap; 3775 3776 if (copy_from_user(&ucasmap, argp, sizeof(ucasmap))) { 3777 r = -EFAULT; 3778 break; 3779 } 3780 3781 if (!kvm_is_ucontrol(vcpu->kvm)) { 3782 r = -EINVAL; 3783 break; 3784 } 3785 3786 r = gmap_unmap_segment(vcpu->arch.gmap, ucasmap.vcpu_addr, 3787 ucasmap.length); 3788 break; 3789 } 3790 #endif 3791 case KVM_S390_VCPU_FAULT: { 3792 r = gmap_fault(vcpu->arch.gmap, arg, 0); 3793 break; 3794 } 3795 case KVM_ENABLE_CAP: 3796 { 3797 struct kvm_enable_cap cap; 3798 r = -EFAULT; 3799 if (copy_from_user(&cap, argp, sizeof(cap))) 3800 break; 3801 r = kvm_vcpu_ioctl_enable_cap(vcpu, &cap); 3802 break; 3803 } 3804 case KVM_S390_MEM_OP: { 3805 struct kvm_s390_mem_op mem_op; 3806 3807 if (copy_from_user(&mem_op, argp, sizeof(mem_op)) == 0) 3808 r = kvm_s390_guest_mem_op(vcpu, &mem_op); 3809 else 3810 r = -EFAULT; 3811 break; 3812 } 3813 case KVM_S390_SET_IRQ_STATE: { 3814 struct kvm_s390_irq_state irq_state; 3815 3816 r = -EFAULT; 3817 if (copy_from_user(&irq_state, argp, sizeof(irq_state))) 3818 break; 3819 if (irq_state.len > VCPU_IRQS_MAX_BUF || 3820 irq_state.len == 0 || 3821 irq_state.len % sizeof(struct kvm_s390_irq) > 0) { 3822 r = -EINVAL; 3823 break; 3824 } 3825 /* do not use irq_state.flags, it will break old QEMUs */ 3826 r = kvm_s390_set_irq_state(vcpu, 3827 (void __user *) irq_state.buf, 3828 irq_state.len); 3829 break; 3830 } 3831 case KVM_S390_GET_IRQ_STATE: { 3832 struct kvm_s390_irq_state irq_state; 3833 3834 r = -EFAULT; 3835 if (copy_from_user(&irq_state, argp, sizeof(irq_state))) 3836 break; 3837 if (irq_state.len == 0) { 3838 r = -EINVAL; 3839 break; 3840 } 3841 /* do not use irq_state.flags, it will break old QEMUs */ 3842 r = kvm_s390_get_irq_state(vcpu, 3843 (__u8 __user *) irq_state.buf, 3844 irq_state.len); 3845 break; 3846 } 3847 default: 3848 r = -ENOTTY; 3849 } 3850 return r; 3851 } 3852 3853 int kvm_arch_vcpu_fault(struct kvm_vcpu *vcpu, struct vm_fault *vmf) 3854 { 3855 #ifdef CONFIG_KVM_S390_UCONTROL 3856 if ((vmf->pgoff == KVM_S390_SIE_PAGE_OFFSET) 3857 && (kvm_is_ucontrol(vcpu->kvm))) { 3858 vmf->page = virt_to_page(vcpu->arch.sie_block); 3859 get_page(vmf->page); 3860 return 0; 3861 } 3862 #endif 3863 return VM_FAULT_SIGBUS; 3864 } 3865 3866 int kvm_arch_create_memslot(struct kvm *kvm, struct kvm_memory_slot *slot, 3867 unsigned long npages) 3868 { 3869 return 0; 3870 } 3871 3872 /* Section: memory related */ 3873 int kvm_arch_prepare_memory_region(struct kvm *kvm, 3874 struct kvm_memory_slot *memslot, 3875 const struct kvm_userspace_memory_region *mem, 3876 enum kvm_mr_change change) 3877 { 3878 /* A few sanity checks. We can have memory slots which have to be 3879 located/ended at a segment boundary (1MB). The memory in userland is 3880 ok to be fragmented into various different vmas. It is okay to mmap() 3881 and munmap() stuff in this slot after doing this call at any time */ 3882 3883 if (mem->userspace_addr & 0xffffful) 3884 return -EINVAL; 3885 3886 if (mem->memory_size & 0xffffful) 3887 return -EINVAL; 3888 3889 if (mem->guest_phys_addr + mem->memory_size > kvm->arch.mem_limit) 3890 return -EINVAL; 3891 3892 return 0; 3893 } 3894 3895 void kvm_arch_commit_memory_region(struct kvm *kvm, 3896 const struct kvm_userspace_memory_region *mem, 3897 const struct kvm_memory_slot *old, 3898 const struct kvm_memory_slot *new, 3899 enum kvm_mr_change change) 3900 { 3901 int rc; 3902 3903 /* If the basics of the memslot do not change, we do not want 3904 * to update the gmap. Every update causes several unnecessary 3905 * segment translation exceptions. This is usually handled just 3906 * fine by the normal fault handler + gmap, but it will also 3907 * cause faults on the prefix page of running guest CPUs. 3908 */ 3909 if (old->userspace_addr == mem->userspace_addr && 3910 old->base_gfn * PAGE_SIZE == mem->guest_phys_addr && 3911 old->npages * PAGE_SIZE == mem->memory_size) 3912 return; 3913 3914 rc = gmap_map_segment(kvm->arch.gmap, mem->userspace_addr, 3915 mem->guest_phys_addr, mem->memory_size); 3916 if (rc) 3917 pr_warn("failed to commit memory region\n"); 3918 return; 3919 } 3920 3921 static inline unsigned long nonhyp_mask(int i) 3922 { 3923 unsigned int nonhyp_fai = (sclp.hmfai << i * 2) >> 30; 3924 3925 return 0x0000ffffffffffffUL >> (nonhyp_fai << 4); 3926 } 3927 3928 void kvm_arch_vcpu_block_finish(struct kvm_vcpu *vcpu) 3929 { 3930 vcpu->valid_wakeup = false; 3931 } 3932 3933 static int __init kvm_s390_init(void) 3934 { 3935 int i; 3936 3937 if (!sclp.has_sief2) { 3938 pr_info("SIE not available\n"); 3939 return -ENODEV; 3940 } 3941 3942 for (i = 0; i < 16; i++) 3943 kvm_s390_fac_list_mask[i] |= 3944 S390_lowcore.stfle_fac_list[i] & nonhyp_mask(i); 3945 3946 return kvm_init(NULL, sizeof(struct kvm_vcpu), 0, THIS_MODULE); 3947 } 3948 3949 static void __exit kvm_s390_exit(void) 3950 { 3951 kvm_exit(); 3952 } 3953 3954 module_init(kvm_s390_init); 3955 module_exit(kvm_s390_exit); 3956 3957 /* 3958 * Enable autoloading of the kvm module. 3959 * Note that we add the module alias here instead of virt/kvm/kvm_main.c 3960 * since x86 takes a different approach. 3961 */ 3962 #include <linux/miscdevice.h> 3963 MODULE_ALIAS_MISCDEV(KVM_MINOR); 3964 MODULE_ALIAS("devname:kvm"); 3965