1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * hosting IBM Z kernel virtual machines (s390x)
4 *
5 * Copyright IBM Corp. 2008, 2020
6 *
7 * Author(s): Carsten Otte <cotte@de.ibm.com>
8 * Christian Borntraeger <borntraeger@de.ibm.com>
9 * Christian Ehrhardt <ehrhardt@de.ibm.com>
10 * Jason J. Herne <jjherne@us.ibm.com>
11 */
12
13 #define pr_fmt(fmt) "kvm-s390: " fmt
14
15 #include <linux/compiler.h>
16 #include <linux/entry-virt.h>
17 #include <linux/export.h>
18 #include <linux/err.h>
19 #include <linux/fs.h>
20 #include <linux/hrtimer.h>
21 #include <linux/init.h>
22 #include <linux/kvm.h>
23 #include <linux/kvm_host.h>
24 #include <linux/mman.h>
25 #include <linux/module.h>
26 #include <linux/moduleparam.h>
27 #include <linux/cpufeature.h>
28 #include <linux/random.h>
29 #include <linux/slab.h>
30 #include <linux/timer.h>
31 #include <linux/vmalloc.h>
32 #include <linux/bitmap.h>
33 #include <linux/sched/signal.h>
34 #include <linux/string.h>
35 #include <linux/pgtable.h>
36 #include <linux/mmu_notifier.h>
37
38 #include <asm/access-regs.h>
39 #include <asm/asm-offsets.h>
40 #include <asm/lowcore.h>
41 #include <asm/machine.h>
42 #include <asm/stp.h>
43 #include <asm/gmap_helpers.h>
44 #include <asm/nmi.h>
45 #include <asm/isc.h>
46 #include <asm/sclp.h>
47 #include <asm/cpacf.h>
48 #include <asm/timex.h>
49 #include <asm/asm.h>
50 #include <asm/fpu.h>
51 #include <asm/ap.h>
52 #include <asm/uv.h>
53 #include "kvm-s390.h"
54 #include "gaccess.h"
55 #include "gmap.h"
56 #include "faultin.h"
57 #include "pci.h"
58
59 #define CREATE_TRACE_POINTS
60 #include "trace.h"
61 #include "trace-s390.h"
62
63 #define MEM_OP_MAX_SIZE 65536 /* Maximum transfer size for KVM_S390_MEM_OP */
64 #define LOCAL_IRQS 32
65 #define VCPU_IRQS_MAX_BUF (sizeof(struct kvm_s390_irq) * \
66 (KVM_MAX_VCPUS + LOCAL_IRQS))
67
68 const struct kvm_stats_desc kvm_vm_stats_desc[] = {
69 KVM_GENERIC_VM_STATS(),
70 STATS_DESC_COUNTER(VM, inject_io),
71 STATS_DESC_COUNTER(VM, io_390_adapter_map),
72 STATS_DESC_COUNTER(VM, io_390_adapter_unmap),
73 STATS_DESC_COUNTER(VM, io_390_inatomic),
74 STATS_DESC_COUNTER(VM, io_flic_inject_airq),
75 STATS_DESC_COUNTER(VM, io_set_adapter_int),
76 STATS_DESC_COUNTER(VM, io_390_inatomic_no_inject),
77 STATS_DESC_COUNTER(VM, inject_float_mchk),
78 STATS_DESC_COUNTER(VM, inject_pfault_done),
79 STATS_DESC_COUNTER(VM, inject_service_signal),
80 STATS_DESC_COUNTER(VM, inject_virtio),
81 STATS_DESC_COUNTER(VM, aen_forward),
82 STATS_DESC_COUNTER(VM, gmap_shadow_reuse),
83 STATS_DESC_COUNTER(VM, gmap_shadow_create),
84 STATS_DESC_COUNTER(VM, gmap_shadow_r1_entry),
85 STATS_DESC_COUNTER(VM, gmap_shadow_r2_entry),
86 STATS_DESC_COUNTER(VM, gmap_shadow_r3_entry),
87 STATS_DESC_COUNTER(VM, gmap_shadow_sg_entry),
88 STATS_DESC_COUNTER(VM, gmap_shadow_pg_entry),
89 };
90
91 const struct kvm_stats_header kvm_vm_stats_header = {
92 .name_size = KVM_STATS_NAME_SIZE,
93 .num_desc = ARRAY_SIZE(kvm_vm_stats_desc),
94 .id_offset = sizeof(struct kvm_stats_header),
95 .desc_offset = sizeof(struct kvm_stats_header) + KVM_STATS_NAME_SIZE,
96 .data_offset = sizeof(struct kvm_stats_header) + KVM_STATS_NAME_SIZE +
97 sizeof(kvm_vm_stats_desc),
98 };
99
100 const struct kvm_stats_desc kvm_vcpu_stats_desc[] = {
101 KVM_GENERIC_VCPU_STATS(),
102 STATS_DESC_COUNTER(VCPU, exit_userspace),
103 STATS_DESC_COUNTER(VCPU, exit_null),
104 STATS_DESC_COUNTER(VCPU, exit_external_request),
105 STATS_DESC_COUNTER(VCPU, exit_io_request),
106 STATS_DESC_COUNTER(VCPU, exit_external_interrupt),
107 STATS_DESC_COUNTER(VCPU, exit_stop_request),
108 STATS_DESC_COUNTER(VCPU, exit_validity),
109 STATS_DESC_COUNTER(VCPU, exit_instruction),
110 STATS_DESC_COUNTER(VCPU, exit_pei),
111 STATS_DESC_COUNTER(VCPU, halt_no_poll_steal),
112 STATS_DESC_COUNTER(VCPU, instruction_lctl),
113 STATS_DESC_COUNTER(VCPU, instruction_lctlg),
114 STATS_DESC_COUNTER(VCPU, instruction_stctl),
115 STATS_DESC_COUNTER(VCPU, instruction_stctg),
116 STATS_DESC_COUNTER(VCPU, exit_program_interruption),
117 STATS_DESC_COUNTER(VCPU, exit_instr_and_program),
118 STATS_DESC_COUNTER(VCPU, exit_operation_exception),
119 STATS_DESC_COUNTER(VCPU, deliver_ckc),
120 STATS_DESC_COUNTER(VCPU, deliver_cputm),
121 STATS_DESC_COUNTER(VCPU, deliver_external_call),
122 STATS_DESC_COUNTER(VCPU, deliver_emergency_signal),
123 STATS_DESC_COUNTER(VCPU, deliver_service_signal),
124 STATS_DESC_COUNTER(VCPU, deliver_virtio),
125 STATS_DESC_COUNTER(VCPU, deliver_stop_signal),
126 STATS_DESC_COUNTER(VCPU, deliver_prefix_signal),
127 STATS_DESC_COUNTER(VCPU, deliver_restart_signal),
128 STATS_DESC_COUNTER(VCPU, deliver_program),
129 STATS_DESC_COUNTER(VCPU, deliver_io),
130 STATS_DESC_COUNTER(VCPU, deliver_machine_check),
131 STATS_DESC_COUNTER(VCPU, exit_wait_state),
132 STATS_DESC_COUNTER(VCPU, inject_ckc),
133 STATS_DESC_COUNTER(VCPU, inject_cputm),
134 STATS_DESC_COUNTER(VCPU, inject_external_call),
135 STATS_DESC_COUNTER(VCPU, inject_emergency_signal),
136 STATS_DESC_COUNTER(VCPU, inject_mchk),
137 STATS_DESC_COUNTER(VCPU, inject_pfault_init),
138 STATS_DESC_COUNTER(VCPU, inject_program),
139 STATS_DESC_COUNTER(VCPU, inject_restart),
140 STATS_DESC_COUNTER(VCPU, inject_set_prefix),
141 STATS_DESC_COUNTER(VCPU, inject_stop_signal),
142 STATS_DESC_COUNTER(VCPU, instruction_epsw),
143 STATS_DESC_COUNTER(VCPU, instruction_gs),
144 STATS_DESC_COUNTER(VCPU, instruction_io_other),
145 STATS_DESC_COUNTER(VCPU, instruction_lpsw),
146 STATS_DESC_COUNTER(VCPU, instruction_lpswe),
147 STATS_DESC_COUNTER(VCPU, instruction_lpswey),
148 STATS_DESC_COUNTER(VCPU, instruction_pfmf),
149 STATS_DESC_COUNTER(VCPU, instruction_ptff),
150 STATS_DESC_COUNTER(VCPU, instruction_sck),
151 STATS_DESC_COUNTER(VCPU, instruction_sckpf),
152 STATS_DESC_COUNTER(VCPU, instruction_stidp),
153 STATS_DESC_COUNTER(VCPU, instruction_spx),
154 STATS_DESC_COUNTER(VCPU, instruction_stpx),
155 STATS_DESC_COUNTER(VCPU, instruction_stap),
156 STATS_DESC_COUNTER(VCPU, instruction_iske),
157 STATS_DESC_COUNTER(VCPU, instruction_ri),
158 STATS_DESC_COUNTER(VCPU, instruction_rrbe),
159 STATS_DESC_COUNTER(VCPU, instruction_sske),
160 STATS_DESC_COUNTER(VCPU, instruction_ipte_interlock),
161 STATS_DESC_COUNTER(VCPU, instruction_stsi),
162 STATS_DESC_COUNTER(VCPU, instruction_stfl),
163 STATS_DESC_COUNTER(VCPU, instruction_tb),
164 STATS_DESC_COUNTER(VCPU, instruction_tpi),
165 STATS_DESC_COUNTER(VCPU, instruction_tprot),
166 STATS_DESC_COUNTER(VCPU, instruction_tsch),
167 STATS_DESC_COUNTER(VCPU, instruction_sie),
168 STATS_DESC_COUNTER(VCPU, instruction_essa),
169 STATS_DESC_COUNTER(VCPU, instruction_sthyi),
170 STATS_DESC_COUNTER(VCPU, instruction_sigp_sense),
171 STATS_DESC_COUNTER(VCPU, instruction_sigp_sense_running),
172 STATS_DESC_COUNTER(VCPU, instruction_sigp_external_call),
173 STATS_DESC_COUNTER(VCPU, instruction_sigp_emergency),
174 STATS_DESC_COUNTER(VCPU, instruction_sigp_cond_emergency),
175 STATS_DESC_COUNTER(VCPU, instruction_sigp_start),
176 STATS_DESC_COUNTER(VCPU, instruction_sigp_stop),
177 STATS_DESC_COUNTER(VCPU, instruction_sigp_stop_store_status),
178 STATS_DESC_COUNTER(VCPU, instruction_sigp_store_status),
179 STATS_DESC_COUNTER(VCPU, instruction_sigp_store_adtl_status),
180 STATS_DESC_COUNTER(VCPU, instruction_sigp_arch),
181 STATS_DESC_COUNTER(VCPU, instruction_sigp_prefix),
182 STATS_DESC_COUNTER(VCPU, instruction_sigp_restart),
183 STATS_DESC_COUNTER(VCPU, instruction_sigp_init_cpu_reset),
184 STATS_DESC_COUNTER(VCPU, instruction_sigp_cpu_reset),
185 STATS_DESC_COUNTER(VCPU, instruction_sigp_unknown),
186 STATS_DESC_COUNTER(VCPU, instruction_diagnose_10),
187 STATS_DESC_COUNTER(VCPU, instruction_diagnose_44),
188 STATS_DESC_COUNTER(VCPU, instruction_diagnose_9c),
189 STATS_DESC_COUNTER(VCPU, diag_9c_ignored),
190 STATS_DESC_COUNTER(VCPU, diag_9c_forward),
191 STATS_DESC_COUNTER(VCPU, instruction_diagnose_258),
192 STATS_DESC_COUNTER(VCPU, instruction_diagnose_308),
193 STATS_DESC_COUNTER(VCPU, instruction_diagnose_500),
194 STATS_DESC_COUNTER(VCPU, instruction_diagnose_other),
195 STATS_DESC_COUNTER(VCPU, pfault_sync),
196 STATS_DESC_COUNTER(VCPU, signal_exits)
197 };
198
199 const struct kvm_stats_header kvm_vcpu_stats_header = {
200 .name_size = KVM_STATS_NAME_SIZE,
201 .num_desc = ARRAY_SIZE(kvm_vcpu_stats_desc),
202 .id_offset = sizeof(struct kvm_stats_header),
203 .desc_offset = sizeof(struct kvm_stats_header) + KVM_STATS_NAME_SIZE,
204 .data_offset = sizeof(struct kvm_stats_header) + KVM_STATS_NAME_SIZE +
205 sizeof(kvm_vcpu_stats_desc),
206 };
207
208 /* allow nested virtualization in KVM (if enabled by user space) */
209 static int nested;
210 module_param(nested, int, S_IRUGO);
211 MODULE_PARM_DESC(nested, "Nested virtualization support");
212
213 /* allow 1m huge page guest backing */
214 static int hpage;
215 module_param(hpage, int, 0444);
216 MODULE_PARM_DESC(hpage, "1m huge page backing support");
217
218 /* allow 2g huge page guest backing */
219 static int hpage_2g;
220 module_param(hpage_2g, int, 0444);
221 MODULE_PARM_DESC(hpage_2g, "2g huge page backing support");
222
223 /* maximum percentage of steal time for polling. >100 is treated like 100 */
224 static u8 halt_poll_max_steal = 10;
225 module_param(halt_poll_max_steal, byte, 0644);
226 MODULE_PARM_DESC(halt_poll_max_steal, "Maximum percentage of steal time to allow polling");
227
228 /* if set to true, the GISA will be initialized and used if available */
229 static bool use_gisa = true;
230 module_param(use_gisa, bool, 0644);
231 MODULE_PARM_DESC(use_gisa, "Use the GISA if the host supports it.");
232
233 /* maximum diag9c forwarding per second */
234 unsigned int diag9c_forwarding_hz;
235 module_param(diag9c_forwarding_hz, uint, 0644);
236 MODULE_PARM_DESC(diag9c_forwarding_hz, "Maximum diag9c forwarding per second, 0 to turn off");
237
238 /*
239 * allow asynchronous deinit for protected guests; enable by default since
240 * the feature is opt-in anyway
241 */
242 static int async_destroy = 1;
243 module_param(async_destroy, int, 0444);
244 MODULE_PARM_DESC(async_destroy, "Asynchronous destroy for protected guests");
245
246 #define HMFAI_DWORDS 16
247 /*
248 * Base feature mask that defines default mask for facilities. Consists of the
249 * defines in FACILITIES_KVM and the non-hypervisor managed bits.
250 */
251 static unsigned long kvm_s390_fac_base[HMFAI_DWORDS] = { FACILITIES_KVM };
252 static_assert(ARRAY_SIZE(((long[]){ FACILITIES_KVM })) <= HMFAI_DWORDS);
253 static_assert(ARRAY_SIZE(kvm_s390_fac_base) <= S390_ARCH_FAC_MASK_SIZE_U64);
254 static_assert(ARRAY_SIZE(kvm_s390_fac_base) <= S390_ARCH_FAC_LIST_SIZE_U64);
255 static_assert(ARRAY_SIZE(kvm_s390_fac_base) <= ARRAY_SIZE(stfle_fac_list));
256
257 /*
258 * Extended feature mask. Consists of the defines in FACILITIES_KVM_CPUMODEL
259 * and defines the facilities that can be enabled via a cpu model.
260 */
261 static const unsigned long kvm_s390_fac_ext[] = { FACILITIES_KVM_CPUMODEL };
262 static_assert(ARRAY_SIZE(kvm_s390_fac_ext) <= S390_ARCH_FAC_MASK_SIZE_U64);
263 static_assert(ARRAY_SIZE(kvm_s390_fac_ext) <= S390_ARCH_FAC_LIST_SIZE_U64);
264 static_assert(ARRAY_SIZE(kvm_s390_fac_ext) <= ARRAY_SIZE(stfle_fac_list));
265
266 /* available cpu features supported by kvm */
267 static DECLARE_BITMAP(kvm_s390_available_cpu_feat, KVM_S390_VM_CPU_FEAT_NR_BITS);
268 /* available subfunctions indicated via query / "test bit" */
269 static struct kvm_s390_vm_cpu_subfunc kvm_s390_available_subfunc;
270
271 debug_info_t *kvm_s390_dbf;
272 debug_info_t *kvm_s390_dbf_uv;
273
274 /* Section: not file related */
275 /* forward declarations */
kvm_clock_sync_scb(struct kvm_s390_sie_block * scb,u64 delta)276 static void kvm_clock_sync_scb(struct kvm_s390_sie_block *scb, u64 delta)
277 {
278 u8 delta_idx = 0;
279
280 /*
281 * The TOD jumps by delta, we have to compensate this by adding
282 * -delta to the epoch.
283 */
284 delta = -delta;
285
286 /* sign-extension - we're adding to signed values below */
287 if ((s64)delta < 0)
288 delta_idx = -1;
289
290 scb->epoch += delta;
291 if (scb->ecd & ECD_MEF) {
292 scb->epdx += delta_idx;
293 if (scb->epoch < delta)
294 scb->epdx += 1;
295 }
296 }
297
298 /*
299 * This callback is executed during stop_machine(). All CPUs are therefore
300 * temporarily stopped. In order not to change guest behavior, we have to
301 * disable preemption whenever we touch the epoch of kvm and the VCPUs,
302 * so a CPU won't be stopped while calculating with the epoch.
303 */
kvm_clock_sync(struct notifier_block * notifier,unsigned long val,void * v)304 static int kvm_clock_sync(struct notifier_block *notifier, unsigned long val,
305 void *v)
306 {
307 struct kvm *kvm;
308 struct kvm_vcpu *vcpu;
309 unsigned long i;
310 unsigned long long *delta = v;
311
312 list_for_each_entry(kvm, &vm_list, vm_list) {
313 kvm_for_each_vcpu(i, vcpu, kvm) {
314 kvm_clock_sync_scb(vcpu->arch.sie_block, *delta);
315 if (i == 0) {
316 kvm->arch.epoch = vcpu->arch.sie_block->epoch;
317 kvm->arch.epdx = vcpu->arch.sie_block->epdx;
318 }
319 if (vcpu->arch.cputm_enabled)
320 vcpu->arch.cputm_start += *delta;
321 if (vcpu->arch.vsie_block)
322 kvm_clock_sync_scb(vcpu->arch.vsie_block,
323 *delta);
324 }
325 }
326 return NOTIFY_OK;
327 }
328
329 static struct notifier_block kvm_clock_notifier = {
330 .notifier_call = kvm_clock_sync,
331 };
332
allow_cpu_feat(unsigned long nr)333 static void allow_cpu_feat(unsigned long nr)
334 {
335 set_bit_inv(nr, kvm_s390_available_cpu_feat);
336 }
337
plo_test_bit(unsigned char nr)338 static inline int plo_test_bit(unsigned char nr)
339 {
340 unsigned long function = (unsigned long)nr | 0x100;
341 int cc;
342
343 asm volatile(
344 " lgr 0,%[function]\n"
345 /* Parameter registers are ignored for "test bit" */
346 " plo 0,0,0,0(0)\n"
347 CC_IPM(cc)
348 : CC_OUT(cc, cc)
349 : [function] "d" (function)
350 : CC_CLOBBER_LIST("0"));
351 return CC_TRANSFORM(cc) == 0;
352 }
353
pfcr_query(u8 (* query)[16])354 static __always_inline void pfcr_query(u8 (*query)[16])
355 {
356 asm volatile(
357 " lghi 0,0\n"
358 " .insn rsy,0xeb0000000016,0,0,%[query]"
359 : [query] "=QS" (*query)
360 :
361 : "cc", "0");
362 }
363
__sortl_query(u8 (* query)[32])364 static __always_inline void __sortl_query(u8 (*query)[32])
365 {
366 asm volatile(
367 " lghi 0,0\n"
368 " la 1,%[query]\n"
369 /* Parameter registers are ignored */
370 " .insn rre,0xb9380000,2,4"
371 : [query] "=R" (*query)
372 :
373 : "cc", "0", "1");
374 }
375
__dfltcc_query(u8 (* query)[32])376 static __always_inline void __dfltcc_query(u8 (*query)[32])
377 {
378 asm volatile(
379 " lghi 0,0\n"
380 " la 1,%[query]\n"
381 /* Parameter registers are ignored */
382 " .insn rrf,0xb9390000,2,4,6,0"
383 : [query] "=R" (*query)
384 :
385 : "cc", "0", "1");
386 }
387
kvm_s390_cpu_feat_init(void)388 static void __init kvm_s390_cpu_feat_init(void)
389 {
390 int i;
391
392 for (i = 0; i < 256; ++i) {
393 if (plo_test_bit(i))
394 kvm_s390_available_subfunc.plo[i >> 3] |= 0x80 >> (i & 7);
395 }
396
397 if (test_facility(28)) /* TOD-clock steering */
398 ptff(kvm_s390_available_subfunc.ptff,
399 sizeof(kvm_s390_available_subfunc.ptff),
400 PTFF_QAF);
401
402 if (test_facility(17)) { /* MSA */
403 __cpacf_query(CPACF_KMAC, (cpacf_mask_t *)
404 kvm_s390_available_subfunc.kmac);
405 __cpacf_query(CPACF_KMC, (cpacf_mask_t *)
406 kvm_s390_available_subfunc.kmc);
407 __cpacf_query(CPACF_KM, (cpacf_mask_t *)
408 kvm_s390_available_subfunc.km);
409 __cpacf_query(CPACF_KIMD, (cpacf_mask_t *)
410 kvm_s390_available_subfunc.kimd);
411 __cpacf_query(CPACF_KLMD, (cpacf_mask_t *)
412 kvm_s390_available_subfunc.klmd);
413 }
414 if (test_facility(76)) /* MSA3 */
415 __cpacf_query(CPACF_PCKMO, (cpacf_mask_t *)
416 kvm_s390_available_subfunc.pckmo);
417 if (test_facility(77)) { /* MSA4 */
418 __cpacf_query(CPACF_KMCTR, (cpacf_mask_t *)
419 kvm_s390_available_subfunc.kmctr);
420 __cpacf_query(CPACF_KMF, (cpacf_mask_t *)
421 kvm_s390_available_subfunc.kmf);
422 __cpacf_query(CPACF_KMO, (cpacf_mask_t *)
423 kvm_s390_available_subfunc.kmo);
424 __cpacf_query(CPACF_PCC, (cpacf_mask_t *)
425 kvm_s390_available_subfunc.pcc);
426 }
427 if (test_facility(57)) /* MSA5 */
428 __cpacf_query(CPACF_PRNO, (cpacf_mask_t *)
429 kvm_s390_available_subfunc.ppno);
430
431 if (test_facility(146)) /* MSA8 */
432 __cpacf_query(CPACF_KMA, (cpacf_mask_t *)
433 kvm_s390_available_subfunc.kma);
434
435 if (test_facility(155)) /* MSA9 */
436 __cpacf_query(CPACF_KDSA, (cpacf_mask_t *)
437 kvm_s390_available_subfunc.kdsa);
438
439 if (test_facility(150)) /* SORTL */
440 __sortl_query(&kvm_s390_available_subfunc.sortl);
441
442 if (test_facility(151)) /* DFLTCC */
443 __dfltcc_query(&kvm_s390_available_subfunc.dfltcc);
444
445 if (test_facility(201)) /* PFCR */
446 pfcr_query(&kvm_s390_available_subfunc.pfcr);
447
448 if (machine_has_esop())
449 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_ESOP);
450 /*
451 * We need SIE support, ESOP (PROT_READ protection for gmap_shadow),
452 * 64bit SCAO (SCA passthrough) and IDTE (for gmap_shadow unshadowing).
453 */
454 if (!sclp.has_sief2 || !machine_has_esop() || !sclp.has_64bscao ||
455 !test_facility(3) || !nested)
456 return;
457 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_SIEF2);
458 if (sclp.has_64bscao)
459 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_64BSCAO);
460 if (sclp.has_siif)
461 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_SIIF);
462 if (sclp.has_gpere)
463 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_GPERE);
464 if (sclp.has_gsls)
465 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_GSLS);
466 if (sclp.has_ib)
467 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_IB);
468 if (sclp.has_cei)
469 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_CEI);
470 if (sclp.has_ibs)
471 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_IBS);
472 if (sclp.has_kss)
473 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_KSS);
474 if (sclp.has_astfleie2)
475 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_ASTFLEIE2);
476 /*
477 * KVM_S390_VM_CPU_FEAT_SKEY: Wrong shadow of PTE.I bits will make
478 * all skey handling functions read/set the skey from the PGSTE
479 * instead of the real storage key.
480 *
481 * KVM_S390_VM_CPU_FEAT_CMMA: Wrong shadow of PTE.I bits will make
482 * pages being detected as preserved although they are resident.
483 *
484 * KVM_S390_VM_CPU_FEAT_PFMFI: Wrong shadow of PTE.I bits will
485 * have the same effect as for KVM_S390_VM_CPU_FEAT_SKEY.
486 *
487 * For KVM_S390_VM_CPU_FEAT_SKEY, KVM_S390_VM_CPU_FEAT_CMMA and
488 * KVM_S390_VM_CPU_FEAT_PFMFI, all PTE.I and PGSTE bits have to be
489 * correctly shadowed. We can do that for the PGSTE but not for PTE.I.
490 *
491 * KVM_S390_VM_CPU_FEAT_SIGPIF: Wrong SCB addresses in the SCA. We
492 * cannot easily shadow the SCA because of the ipte lock.
493 */
494 }
495
__kvm_s390_init(void)496 static int __init __kvm_s390_init(void)
497 {
498 int rc = -ENOMEM;
499
500 kvm_s390_dbf = debug_register("kvm-trace", 32, 1, 7 * sizeof(long));
501 if (!kvm_s390_dbf)
502 return -ENOMEM;
503
504 kvm_s390_dbf_uv = debug_register("kvm-uv", 32, 1, 7 * sizeof(long));
505 if (!kvm_s390_dbf_uv)
506 goto err_kvm_uv;
507
508 if (debug_register_view(kvm_s390_dbf, &debug_sprintf_view) ||
509 debug_register_view(kvm_s390_dbf_uv, &debug_sprintf_view))
510 goto err_debug_view;
511
512 kvm_s390_cpu_feat_init();
513
514 /* Register floating interrupt controller interface. */
515 rc = kvm_register_device_ops(&kvm_flic_ops, KVM_DEV_TYPE_FLIC);
516 if (rc) {
517 pr_err("A FLIC registration call failed with rc=%d\n", rc);
518 goto err_flic;
519 }
520
521 if (IS_ENABLED(CONFIG_VFIO_PCI_ZDEV_KVM)) {
522 rc = kvm_s390_pci_init();
523 if (rc) {
524 pr_err("Unable to allocate AIFT for PCI\n");
525 goto err_pci;
526 }
527 }
528
529 rc = kvm_s390_gib_init(GAL_ISC);
530 if (rc)
531 goto err_gib;
532
533 atomic_notifier_chain_register(&s390_epoch_delta_notifier,
534 &kvm_clock_notifier);
535
536 return 0;
537
538 err_gib:
539 if (IS_ENABLED(CONFIG_VFIO_PCI_ZDEV_KVM))
540 kvm_s390_pci_exit();
541 err_pci:
542 err_flic:
543 err_debug_view:
544 debug_unregister(kvm_s390_dbf_uv);
545 err_kvm_uv:
546 debug_unregister(kvm_s390_dbf);
547 return rc;
548 }
549
__kvm_s390_exit(void)550 static void __kvm_s390_exit(void)
551 {
552 atomic_notifier_chain_unregister(&s390_epoch_delta_notifier,
553 &kvm_clock_notifier);
554
555 kvm_s390_gib_destroy();
556 if (IS_ENABLED(CONFIG_VFIO_PCI_ZDEV_KVM))
557 kvm_s390_pci_exit();
558 debug_unregister(kvm_s390_dbf);
559 debug_unregister(kvm_s390_dbf_uv);
560 }
561
kvm_s390_keyop(struct kvm_s390_mmu_cache * mc,struct kvm * kvm,int op,unsigned long addr,union skey skey)562 static int kvm_s390_keyop(struct kvm_s390_mmu_cache *mc, struct kvm *kvm, int op,
563 unsigned long addr, union skey skey)
564 {
565 union asce asce = kvm->arch.gmap->asce;
566 gfn_t gfn = gpa_to_gfn(addr);
567 int r;
568
569 guard(read_lock)(&kvm->mmu_lock);
570
571 switch (op) {
572 case KVM_S390_KEYOP_SSKE:
573 r = dat_cond_set_storage_key(mc, asce, gfn, skey, &skey, 0, 0, 0);
574 if (r == 0 || r == 1)
575 return skey.skey;
576 break;
577 case KVM_S390_KEYOP_ISKE:
578 r = dat_get_storage_key(asce, gfn, &skey);
579 if (!r)
580 return skey.skey;
581 break;
582 case KVM_S390_KEYOP_RRBE:
583 r = dat_reset_reference_bit(asce, gfn, &skey);
584 if (!r)
585 return skey.skey;
586 break;
587 default:
588 return -EINVAL;
589 }
590 return r > 0 ? -EFAULT : r;
591 }
592
593 /* Section: device related */
kvm_arch_dev_ioctl(struct file * filp,unsigned int ioctl,unsigned long arg)594 long kvm_arch_dev_ioctl(struct file *filp,
595 unsigned int ioctl, unsigned long arg)
596 {
597 if (ioctl == KVM_S390_ENABLE_SIE)
598 return 0;
599 return -EINVAL;
600 }
601
kvm_vm_ioctl_check_extension(struct kvm * kvm,long ext)602 int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext)
603 {
604 int r;
605
606 switch (ext) {
607 case KVM_CAP_S390_PSW:
608 case KVM_CAP_S390_GMAP:
609 #ifdef CONFIG_KVM_S390_UCONTROL
610 case KVM_CAP_S390_UCONTROL:
611 #endif
612 case KVM_CAP_ASYNC_PF:
613 case KVM_CAP_SYNC_REGS:
614 case KVM_CAP_ONE_REG:
615 case KVM_CAP_ENABLE_CAP:
616 case KVM_CAP_S390_CSS_SUPPORT:
617 case KVM_CAP_IOEVENTFD:
618 case KVM_CAP_S390_IRQCHIP:
619 case KVM_CAP_VM_ATTRIBUTES:
620 case KVM_CAP_MP_STATE:
621 case KVM_CAP_IMMEDIATE_EXIT:
622 case KVM_CAP_S390_INJECT_IRQ:
623 case KVM_CAP_S390_USER_SIGP:
624 case KVM_CAP_S390_USER_STSI:
625 case KVM_CAP_S390_SKEYS:
626 case KVM_CAP_S390_IRQ_STATE:
627 case KVM_CAP_S390_USER_INSTR0:
628 case KVM_CAP_S390_CMMA_MIGRATION:
629 case KVM_CAP_S390_AIS:
630 case KVM_CAP_S390_AIS_MIGRATION:
631 case KVM_CAP_S390_VCPU_RESETS:
632 case KVM_CAP_SET_GUEST_DEBUG:
633 case KVM_CAP_S390_DIAG318:
634 case KVM_CAP_IRQFD_RESAMPLE:
635 case KVM_CAP_S390_USER_OPEREXEC:
636 case KVM_CAP_S390_KEYOP:
637 case KVM_CAP_S390_VSIE_ESAMODE:
638 case KVM_CAP_PRE_FAULT_MEMORY:
639 r = 1;
640 break;
641 case KVM_CAP_SET_GUEST_DEBUG2:
642 r = KVM_GUESTDBG_VALID_MASK;
643 break;
644 case KVM_CAP_S390_HPAGE_1M:
645 r = 0;
646 if (hpage && !(kvm && kvm_is_ucontrol(kvm)))
647 r = 1;
648 break;
649 case KVM_CAP_S390_HPAGE_2G:
650 r = 0;
651 if (hpage_2g && !(kvm && kvm_is_ucontrol(kvm)))
652 r = 1;
653 break;
654 case KVM_CAP_S390_MEM_OP:
655 r = MEM_OP_MAX_SIZE;
656 break;
657 case KVM_CAP_S390_MEM_OP_EXTENSION:
658 /*
659 * Flag bits indicating which extensions are supported.
660 * If r > 0, the base extension must also be supported/indicated,
661 * in order to maintain backwards compatibility.
662 */
663 r = KVM_S390_MEMOP_EXTENSION_CAP_BASE |
664 KVM_S390_MEMOP_EXTENSION_CAP_CMPXCHG;
665 break;
666 case KVM_CAP_NR_VCPUS:
667 case KVM_CAP_MAX_VCPUS:
668 case KVM_CAP_MAX_VCPU_ID:
669 /*
670 * Return the same value for KVM_CAP_MAX_VCPUS and
671 * KVM_CAP_MAX_VCPU_ID to conform with the KVM API.
672 */
673 r = KVM_S390_ESCA_CPU_SLOTS;
674 if (!kvm_s390_use_sca_entries())
675 r = KVM_MAX_VCPUS;
676 if (ext == KVM_CAP_NR_VCPUS)
677 r = min_t(unsigned int, num_online_cpus(), r);
678 break;
679 case KVM_CAP_S390_COW:
680 r = machine_has_esop();
681 break;
682 case KVM_CAP_S390_VECTOR_REGISTERS:
683 r = test_facility(129);
684 break;
685 case KVM_CAP_S390_RI:
686 r = test_facility(64);
687 break;
688 case KVM_CAP_S390_GS:
689 r = test_facility(133);
690 break;
691 case KVM_CAP_S390_BPB:
692 r = test_facility(82);
693 break;
694 case KVM_CAP_S390_PROTECTED_ASYNC_DISABLE:
695 r = async_destroy && is_prot_virt_host();
696 break;
697 case KVM_CAP_S390_PROTECTED:
698 r = is_prot_virt_host();
699 break;
700 case KVM_CAP_S390_PROTECTED_DUMP: {
701 u64 pv_cmds_dump[] = {
702 BIT_UVC_CMD_DUMP_INIT,
703 BIT_UVC_CMD_DUMP_CONFIG_STOR_STATE,
704 BIT_UVC_CMD_DUMP_CPU,
705 BIT_UVC_CMD_DUMP_COMPLETE,
706 };
707 int i;
708
709 r = is_prot_virt_host();
710
711 for (i = 0; i < ARRAY_SIZE(pv_cmds_dump); i++) {
712 if (!test_bit_inv(pv_cmds_dump[i],
713 (unsigned long *)&uv_info.inst_calls_list)) {
714 r = 0;
715 break;
716 }
717 }
718 break;
719 }
720 case KVM_CAP_S390_ZPCI_OP:
721 r = kvm_s390_pci_interp_allowed();
722 break;
723 case KVM_CAP_S390_CPU_TOPOLOGY:
724 r = test_facility(11);
725 break;
726 default:
727 r = 0;
728 }
729 return r;
730 }
731
kvm_arch_sync_dirty_log(struct kvm * kvm,struct kvm_memory_slot * memslot)732 void kvm_arch_sync_dirty_log(struct kvm *kvm, struct kvm_memory_slot *memslot)
733 {
734 gfn_t last_gfn = memslot->base_gfn + memslot->npages;
735
736 scoped_guard(read_lock, &kvm->mmu_lock)
737 gmap_sync_dirty_log(kvm->arch.gmap, memslot->base_gfn, last_gfn);
738 }
739
740 /* Section: vm related */
741 static void sca_del_vcpu(struct kvm_vcpu *vcpu);
742
743 /*
744 * Get (and clear) the dirty memory log for a memory slot.
745 */
kvm_vm_ioctl_get_dirty_log(struct kvm * kvm,struct kvm_dirty_log * log)746 int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm,
747 struct kvm_dirty_log *log)
748 {
749 int r;
750 unsigned long n;
751 struct kvm_memory_slot *memslot;
752 int is_dirty;
753
754 if (kvm_is_ucontrol(kvm))
755 return -EINVAL;
756
757 mutex_lock(&kvm->slots_lock);
758
759 r = -EINVAL;
760 if (log->slot >= KVM_USER_MEM_SLOTS)
761 goto out;
762
763 r = kvm_get_dirty_log(kvm, log, &is_dirty, &memslot);
764 if (r)
765 goto out;
766
767 /* Clear the dirty log */
768 if (is_dirty) {
769 n = kvm_dirty_bitmap_bytes(memslot);
770 memset(memslot->dirty_bitmap, 0, n);
771 }
772 r = 0;
773 out:
774 mutex_unlock(&kvm->slots_lock);
775 return r;
776 }
777
icpt_operexc_on_all_vcpus(struct kvm * kvm)778 static void icpt_operexc_on_all_vcpus(struct kvm *kvm)
779 {
780 unsigned long i;
781 struct kvm_vcpu *vcpu;
782
783 kvm_for_each_vcpu(i, vcpu, kvm) {
784 kvm_s390_sync_request(KVM_REQ_ICPT_OPEREXC, vcpu);
785 }
786 }
787
kvm_vm_ioctl_enable_cap(struct kvm * kvm,struct kvm_enable_cap * cap)788 int kvm_vm_ioctl_enable_cap(struct kvm *kvm, struct kvm_enable_cap *cap)
789 {
790 int r;
791
792 if (cap->flags)
793 return -EINVAL;
794
795 switch (cap->cap) {
796 case KVM_CAP_S390_IRQCHIP:
797 VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_IRQCHIP");
798 kvm->arch.use_irqchip = 1;
799 r = 0;
800 break;
801 case KVM_CAP_S390_USER_SIGP:
802 VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_USER_SIGP");
803 kvm->arch.user_sigp = 1;
804 r = 0;
805 break;
806 case KVM_CAP_S390_VECTOR_REGISTERS:
807 mutex_lock(&kvm->lock);
808 if (kvm->created_vcpus) {
809 r = -EBUSY;
810 } else if (cpu_has_vx()) {
811 set_kvm_facility(kvm->arch.model.fac_mask, 129);
812 set_kvm_facility(kvm->arch.model.fac_list, 129);
813 if (test_facility(134)) {
814 set_kvm_facility(kvm->arch.model.fac_mask, 134);
815 set_kvm_facility(kvm->arch.model.fac_list, 134);
816 }
817 if (test_facility(135)) {
818 set_kvm_facility(kvm->arch.model.fac_mask, 135);
819 set_kvm_facility(kvm->arch.model.fac_list, 135);
820 }
821 if (test_facility(148)) {
822 set_kvm_facility(kvm->arch.model.fac_mask, 148);
823 set_kvm_facility(kvm->arch.model.fac_list, 148);
824 }
825 if (test_facility(152)) {
826 set_kvm_facility(kvm->arch.model.fac_mask, 152);
827 set_kvm_facility(kvm->arch.model.fac_list, 152);
828 }
829 if (test_facility(192)) {
830 set_kvm_facility(kvm->arch.model.fac_mask, 192);
831 set_kvm_facility(kvm->arch.model.fac_list, 192);
832 }
833 if (test_facility(198)) {
834 set_kvm_facility(kvm->arch.model.fac_mask, 198);
835 set_kvm_facility(kvm->arch.model.fac_list, 198);
836 }
837 if (test_facility(199)) {
838 set_kvm_facility(kvm->arch.model.fac_mask, 199);
839 set_kvm_facility(kvm->arch.model.fac_list, 199);
840 }
841 r = 0;
842 } else
843 r = -EINVAL;
844 mutex_unlock(&kvm->lock);
845 VM_EVENT(kvm, 3, "ENABLE: CAP_S390_VECTOR_REGISTERS %s",
846 r ? "(not available)" : "(success)");
847 break;
848 case KVM_CAP_S390_RI:
849 r = -EINVAL;
850 mutex_lock(&kvm->lock);
851 if (kvm->created_vcpus) {
852 r = -EBUSY;
853 } else if (test_facility(64)) {
854 set_kvm_facility(kvm->arch.model.fac_mask, 64);
855 set_kvm_facility(kvm->arch.model.fac_list, 64);
856 r = 0;
857 }
858 mutex_unlock(&kvm->lock);
859 VM_EVENT(kvm, 3, "ENABLE: CAP_S390_RI %s",
860 r ? "(not available)" : "(success)");
861 break;
862 case KVM_CAP_S390_AIS:
863 mutex_lock(&kvm->lock);
864 if (kvm->created_vcpus) {
865 r = -EBUSY;
866 } else {
867 set_kvm_facility(kvm->arch.model.fac_mask, 72);
868 set_kvm_facility(kvm->arch.model.fac_list, 72);
869 r = 0;
870 }
871 mutex_unlock(&kvm->lock);
872 VM_EVENT(kvm, 3, "ENABLE: AIS %s",
873 r ? "(not available)" : "(success)");
874 break;
875 case KVM_CAP_S390_GS:
876 r = -EINVAL;
877 mutex_lock(&kvm->lock);
878 if (kvm->created_vcpus) {
879 r = -EBUSY;
880 } else if (test_facility(133)) {
881 set_kvm_facility(kvm->arch.model.fac_mask, 133);
882 set_kvm_facility(kvm->arch.model.fac_list, 133);
883 r = 0;
884 }
885 mutex_unlock(&kvm->lock);
886 VM_EVENT(kvm, 3, "ENABLE: CAP_S390_GS %s",
887 r ? "(not available)" : "(success)");
888 break;
889 case KVM_CAP_S390_HPAGE_1M:
890 mutex_lock(&kvm->lock);
891 if (kvm->created_vcpus)
892 r = -EBUSY;
893 else if (!hpage || kvm->arch.use_cmma || kvm_is_ucontrol(kvm))
894 r = -EINVAL;
895 else {
896 r = 0;
897 set_bit(GMAP_FLAG_ALLOW_HPAGE_1M, &kvm->arch.gmap->flags);
898 /*
899 * We might have to create fake 4k page
900 * tables. To avoid that the hardware works on
901 * stale PGSTEs, we emulate these instructions.
902 */
903 kvm->arch.use_skf = 0;
904 kvm->arch.use_pfmfi = 0;
905 }
906 mutex_unlock(&kvm->lock);
907 VM_EVENT(kvm, 3, "ENABLE: CAP_S390_HPAGE %s",
908 r ? "(not available)" : "(success)");
909 break;
910 case KVM_CAP_S390_HPAGE_2G:
911 mutex_lock(&kvm->lock);
912 if (kvm->created_vcpus) {
913 r = -EBUSY;
914 } else if (!hpage_2g || kvm->arch.use_cmma || kvm_is_ucontrol(kvm)) {
915 r = -EINVAL;
916 } else {
917 r = 0;
918 set_bit(GMAP_FLAG_ALLOW_HPAGE_2G, &kvm->arch.gmap->flags);
919 /*
920 * We might have to create fake 4k page
921 * tables. To avoid that the hardware works on
922 * stale PGSTEs, we emulate these instructions.
923 */
924 kvm->arch.use_skf = 0;
925 kvm->arch.use_pfmfi = 0;
926 }
927 mutex_unlock(&kvm->lock);
928 VM_EVENT(kvm, 3, "ENABLE: CAP_S390_HPAGE_2G %s",
929 r ? "(not available)" : "(success)");
930 break;
931 case KVM_CAP_S390_USER_STSI:
932 VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_USER_STSI");
933 kvm->arch.user_stsi = 1;
934 r = 0;
935 break;
936 case KVM_CAP_S390_USER_INSTR0:
937 VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_USER_INSTR0");
938 kvm->arch.user_instr0 = 1;
939 icpt_operexc_on_all_vcpus(kvm);
940 r = 0;
941 break;
942 case KVM_CAP_S390_CPU_TOPOLOGY:
943 r = -EINVAL;
944 mutex_lock(&kvm->lock);
945 if (kvm->created_vcpus) {
946 r = -EBUSY;
947 } else if (test_facility(11)) {
948 set_kvm_facility(kvm->arch.model.fac_mask, 11);
949 set_kvm_facility(kvm->arch.model.fac_list, 11);
950 r = 0;
951 }
952 mutex_unlock(&kvm->lock);
953 VM_EVENT(kvm, 3, "ENABLE: CAP_S390_CPU_TOPOLOGY %s",
954 r ? "(not available)" : "(success)");
955 break;
956 case KVM_CAP_S390_USER_OPEREXEC:
957 VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_USER_OPEREXEC");
958 kvm->arch.user_operexec = 1;
959 icpt_operexc_on_all_vcpus(kvm);
960 r = 0;
961 break;
962 case KVM_CAP_S390_VSIE_ESAMODE:
963 VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_VSIE_ESAMODE");
964 kvm->arch.allow_vsie_esamode = 1;
965 r = 0;
966 break;
967 default:
968 r = -EINVAL;
969 break;
970 }
971 return r;
972 }
973
kvm_s390_get_mem_control(struct kvm * kvm,struct kvm_device_attr * attr)974 static int kvm_s390_get_mem_control(struct kvm *kvm, struct kvm_device_attr *attr)
975 {
976 int ret;
977
978 switch (attr->attr) {
979 case KVM_S390_VM_MEM_LIMIT_SIZE:
980 ret = 0;
981 VM_EVENT(kvm, 3, "QUERY: max guest memory: %lu bytes",
982 kvm->arch.mem_limit);
983 if (put_user(kvm->arch.mem_limit, (u64 __user *)attr->addr))
984 ret = -EFAULT;
985 break;
986 default:
987 ret = -ENXIO;
988 break;
989 }
990 return ret;
991 }
992
kvm_s390_set_mem_control(struct kvm * kvm,struct kvm_device_attr * attr)993 static int kvm_s390_set_mem_control(struct kvm *kvm, struct kvm_device_attr *attr)
994 {
995 int ret;
996
997 switch (attr->attr) {
998 case KVM_S390_VM_MEM_ENABLE_CMMA:
999 ret = -ENXIO;
1000 if (!sclp.has_cmma)
1001 break;
1002
1003 VM_EVENT(kvm, 3, "%s", "ENABLE: CMMA support");
1004 mutex_lock(&kvm->lock);
1005 if (kvm->created_vcpus)
1006 ret = -EBUSY;
1007 else {
1008 kvm->arch.use_cmma = 1;
1009 /* Not compatible with cmma. */
1010 kvm->arch.use_pfmfi = 0;
1011 ret = 0;
1012 }
1013 mutex_unlock(&kvm->lock);
1014 break;
1015 case KVM_S390_VM_MEM_CLR_CMMA: {
1016 gfn_t start_gfn = 0;
1017
1018 ret = -ENXIO;
1019 if (!sclp.has_cmma)
1020 break;
1021 ret = -EINVAL;
1022 if (!kvm->arch.use_cmma)
1023 break;
1024
1025 guard(mutex)(&kvm->lock);
1026 VM_EVENT(kvm, 3, "%s", "RESET: CMMA states");
1027 do {
1028 scoped_guard(read_lock, &kvm->mmu_lock)
1029 start_gfn = dat_reset_cmma(kvm->arch.gmap->asce, start_gfn);
1030 cond_resched();
1031 } while (start_gfn);
1032 ret = 0;
1033 break;
1034 }
1035 case KVM_S390_VM_MEM_LIMIT_SIZE: {
1036 struct kvm_memslots *slots;
1037 struct kvm_memory_slot *ms;
1038 unsigned long new_limit;
1039 int bkt;
1040
1041 if (kvm_is_ucontrol(kvm))
1042 return -EINVAL;
1043
1044 if (get_user(new_limit, (u64 __user *)attr->addr))
1045 return -EFAULT;
1046
1047 guard(mutex)(&kvm->lock);
1048
1049 new_limit = ALIGN(new_limit, HPAGE_SIZE);
1050 if (kvm->arch.mem_limit != KVM_S390_NO_MEM_LIMIT &&
1051 new_limit > kvm->arch.mem_limit)
1052 return -E2BIG;
1053
1054 if (!new_limit)
1055 return -EINVAL;
1056
1057 if (kvm->created_vcpus)
1058 return -EBUSY;
1059
1060 ret = 0;
1061 scoped_guard(mutex, &kvm->slots_lock) {
1062 slots = kvm_memslots(kvm);
1063 if (slots && !kvm_memslots_empty(slots)) {
1064 kvm_for_each_memslot(ms, bkt, slots) {
1065 if (gpa_to_gfn(new_limit) < ms->base_gfn + ms->npages) {
1066 ret = -EBUSY;
1067 break;
1068 }
1069 }
1070 }
1071 if (!ret)
1072 ret = gmap_set_limit(kvm->arch.gmap, gpa_to_gfn(new_limit));
1073 }
1074 if (ret)
1075 break;
1076 VM_EVENT(kvm, 3, "SET: max guest address: %lu", new_limit);
1077 VM_EVENT(kvm, 3, "New guest asce: 0x%p", (void *)kvm->arch.gmap->asce.val);
1078 break;
1079 }
1080 default:
1081 ret = -ENXIO;
1082 break;
1083 }
1084 return ret;
1085 }
1086
1087 static void kvm_s390_vcpu_crypto_setup(struct kvm_vcpu *vcpu);
1088
kvm_s390_vcpu_crypto_reset_all(struct kvm * kvm)1089 void kvm_s390_vcpu_crypto_reset_all(struct kvm *kvm)
1090 {
1091 struct kvm_vcpu *vcpu;
1092 unsigned long i;
1093
1094 kvm_s390_vcpu_block_all(kvm);
1095
1096 kvm_for_each_vcpu(i, vcpu, kvm) {
1097 kvm_s390_vcpu_crypto_setup(vcpu);
1098 /* recreate the shadow crycb by leaving the VSIE handler */
1099 kvm_s390_sync_request(KVM_REQ_VSIE_RESTART, vcpu);
1100 }
1101
1102 kvm_s390_vcpu_unblock_all(kvm);
1103 }
1104
kvm_s390_vm_set_crypto(struct kvm * kvm,struct kvm_device_attr * attr)1105 static int kvm_s390_vm_set_crypto(struct kvm *kvm, struct kvm_device_attr *attr)
1106 {
1107 mutex_lock(&kvm->lock);
1108 switch (attr->attr) {
1109 case KVM_S390_VM_CRYPTO_ENABLE_AES_KW:
1110 if (!test_kvm_facility(kvm, 76)) {
1111 mutex_unlock(&kvm->lock);
1112 return -EINVAL;
1113 }
1114 get_random_bytes(
1115 kvm->arch.crypto.crycb->aes_wrapping_key_mask,
1116 sizeof(kvm->arch.crypto.crycb->aes_wrapping_key_mask));
1117 kvm->arch.crypto.aes_kw = 1;
1118 VM_EVENT(kvm, 3, "%s", "ENABLE: AES keywrapping support");
1119 break;
1120 case KVM_S390_VM_CRYPTO_ENABLE_DEA_KW:
1121 if (!test_kvm_facility(kvm, 76)) {
1122 mutex_unlock(&kvm->lock);
1123 return -EINVAL;
1124 }
1125 get_random_bytes(
1126 kvm->arch.crypto.crycb->dea_wrapping_key_mask,
1127 sizeof(kvm->arch.crypto.crycb->dea_wrapping_key_mask));
1128 kvm->arch.crypto.dea_kw = 1;
1129 VM_EVENT(kvm, 3, "%s", "ENABLE: DEA keywrapping support");
1130 break;
1131 case KVM_S390_VM_CRYPTO_DISABLE_AES_KW:
1132 if (!test_kvm_facility(kvm, 76)) {
1133 mutex_unlock(&kvm->lock);
1134 return -EINVAL;
1135 }
1136 kvm->arch.crypto.aes_kw = 0;
1137 memset(kvm->arch.crypto.crycb->aes_wrapping_key_mask, 0,
1138 sizeof(kvm->arch.crypto.crycb->aes_wrapping_key_mask));
1139 VM_EVENT(kvm, 3, "%s", "DISABLE: AES keywrapping support");
1140 break;
1141 case KVM_S390_VM_CRYPTO_DISABLE_DEA_KW:
1142 if (!test_kvm_facility(kvm, 76)) {
1143 mutex_unlock(&kvm->lock);
1144 return -EINVAL;
1145 }
1146 kvm->arch.crypto.dea_kw = 0;
1147 memset(kvm->arch.crypto.crycb->dea_wrapping_key_mask, 0,
1148 sizeof(kvm->arch.crypto.crycb->dea_wrapping_key_mask));
1149 VM_EVENT(kvm, 3, "%s", "DISABLE: DEA keywrapping support");
1150 break;
1151 case KVM_S390_VM_CRYPTO_ENABLE_APIE:
1152 if (!ap_instructions_available()) {
1153 mutex_unlock(&kvm->lock);
1154 return -EOPNOTSUPP;
1155 }
1156 kvm->arch.crypto.apie = 1;
1157 break;
1158 case KVM_S390_VM_CRYPTO_DISABLE_APIE:
1159 if (!ap_instructions_available()) {
1160 mutex_unlock(&kvm->lock);
1161 return -EOPNOTSUPP;
1162 }
1163 kvm->arch.crypto.apie = 0;
1164 break;
1165 default:
1166 mutex_unlock(&kvm->lock);
1167 return -ENXIO;
1168 }
1169
1170 kvm_s390_vcpu_crypto_reset_all(kvm);
1171 mutex_unlock(&kvm->lock);
1172 return 0;
1173 }
1174
kvm_s390_vcpu_pci_setup(struct kvm_vcpu * vcpu)1175 static void kvm_s390_vcpu_pci_setup(struct kvm_vcpu *vcpu)
1176 {
1177 /* Only set the ECB bits after guest requests zPCI interpretation */
1178 if (!vcpu->kvm->arch.use_zpci_interp)
1179 return;
1180
1181 vcpu->arch.sie_block->ecb2 |= ECB2_ZPCI_LSI;
1182 vcpu->arch.sie_block->ecb3 |= ECB3_AISII + ECB3_AISI;
1183 }
1184
kvm_s390_vcpu_pci_enable_interp(struct kvm * kvm)1185 void kvm_s390_vcpu_pci_enable_interp(struct kvm *kvm)
1186 {
1187 struct kvm_vcpu *vcpu;
1188 unsigned long i;
1189
1190 lockdep_assert_held(&kvm->lock);
1191
1192 if (!kvm_s390_pci_interp_allowed())
1193 return;
1194
1195 /*
1196 * If host is configured for PCI and the necessary facilities are
1197 * available, turn on interpretation for the life of this guest
1198 */
1199 kvm->arch.use_zpci_interp = 1;
1200
1201 kvm_s390_vcpu_block_all(kvm);
1202
1203 kvm_for_each_vcpu(i, vcpu, kvm) {
1204 kvm_s390_vcpu_pci_setup(vcpu);
1205 kvm_s390_sync_request(KVM_REQ_VSIE_RESTART, vcpu);
1206 }
1207
1208 kvm_s390_vcpu_unblock_all(kvm);
1209 }
1210
kvm_s390_sync_request_broadcast(struct kvm * kvm,int req)1211 static void kvm_s390_sync_request_broadcast(struct kvm *kvm, int req)
1212 {
1213 unsigned long cx;
1214 struct kvm_vcpu *vcpu;
1215
1216 kvm_for_each_vcpu(cx, vcpu, kvm)
1217 kvm_s390_sync_request(req, vcpu);
1218 }
1219
1220 /*
1221 * Must be called with kvm->srcu held to avoid races on memslots, and with
1222 * kvm->slots_arch_lock to avoid races with ourselves,
1223 * kvm_s390_vm_stop_migration(), and kvm_s390_get_cmma_bits().
1224 */
kvm_s390_vm_start_migration(struct kvm * kvm)1225 static int kvm_s390_vm_start_migration(struct kvm *kvm)
1226 {
1227 struct kvm_memory_slot *ms;
1228 struct kvm_memslots *slots;
1229 int bkt;
1230
1231 /* migration mode already enabled */
1232 if (kvm->arch.migration_mode)
1233 return 0;
1234 slots = kvm_memslots(kvm);
1235 if (!slots || kvm_memslots_empty(slots))
1236 return -EINVAL;
1237
1238 if (!kvm->arch.use_cmma) {
1239 kvm->arch.migration_mode = 1;
1240 return 0;
1241 }
1242 kvm_for_each_memslot(ms, bkt, slots) {
1243 if (!ms->dirty_bitmap)
1244 return -EINVAL;
1245 }
1246 /*
1247 * Set the flag and let KVM handle ESSA manually, potentially setting
1248 * the cmma_d bit in some PGSTEs and increasing cmma_dirty_pages.
1249 * At this point cmma_dirty_pages is still 0, and all existing PGSTEs
1250 * have their cmma_d bit set to 0.
1251 * Any newly allocated page table has its entries marked as cmma-clean,
1252 * which is fine because the CMMA values are not dirty.
1253 */
1254 WRITE_ONCE(kvm->arch.migration_mode, 1);
1255 kvm_s390_sync_request_broadcast(kvm, KVM_REQ_START_MIGRATION);
1256 /*
1257 * Mark all PGSTEs as cmma-dirty, increasing cmma_dirty_pages as needed,
1258 * but without double-counting pages that have become dirty on their own
1259 * in the meantime.
1260 * At this point some pages might have become dirty on their own already
1261 * and cmma_dirty_pages might therefore be non-zero.
1262 */
1263 gmap_set_cmma_all_dirty(kvm->arch.gmap);
1264 return 0;
1265 }
1266
1267 /*
1268 * Must be called with kvm->slots_arch_lock to avoid races with ourselves,
1269 * kvm_s390_vm_start_migration() and kvm_s390_get_cmma_bits().
1270 */
kvm_s390_vm_stop_migration(struct kvm * kvm)1271 static int kvm_s390_vm_stop_migration(struct kvm *kvm)
1272 {
1273 /* migration mode already disabled */
1274 if (!kvm->arch.migration_mode)
1275 return 0;
1276 /*
1277 * Unset the flag and propagate to all vCPUs. From now on the cmma_d
1278 * bit will not be touched on any PGSTE.
1279 * At this point cmma_dirty_pages is possibly non-zero, and thus some
1280 * PGSTEs might have cmma_d set.
1281 */
1282 WRITE_ONCE(kvm->arch.migration_mode, 0);
1283 if (!kvm->arch.use_cmma)
1284 return 0;
1285
1286 kvm_s390_sync_request_broadcast(kvm, KVM_REQ_STOP_MIGRATION);
1287 /* Clear cmma_d on all existing PGSTEs and set cmma_dirty_pages to 0. */
1288 gmap_set_cmma_all_clean(kvm->arch.gmap);
1289 atomic64_set(&kvm->arch.cmma_dirty_pages, 0);
1290 /*
1291 * At this point the system has the expected state: migration_mode is 0,
1292 * cmma_dirty_pages is 0, and all existing PGSTEs have their cmma_d bit
1293 * set to 0.
1294 */
1295 return 0;
1296 }
1297
kvm_s390_vm_set_migration(struct kvm * kvm,struct kvm_device_attr * attr)1298 static int kvm_s390_vm_set_migration(struct kvm *kvm,
1299 struct kvm_device_attr *attr)
1300 {
1301 int res = -ENXIO;
1302
1303 guard(srcu)(&kvm->srcu);
1304 guard(mutex)(&kvm->slots_arch_lock);
1305
1306 switch (attr->attr) {
1307 case KVM_S390_VM_MIGRATION_START:
1308 res = kvm_s390_vm_start_migration(kvm);
1309 break;
1310 case KVM_S390_VM_MIGRATION_STOP:
1311 res = kvm_s390_vm_stop_migration(kvm);
1312 break;
1313 default:
1314 break;
1315 }
1316
1317 return res;
1318 }
1319
kvm_s390_vm_get_migration(struct kvm * kvm,struct kvm_device_attr * attr)1320 static int kvm_s390_vm_get_migration(struct kvm *kvm,
1321 struct kvm_device_attr *attr)
1322 {
1323 u64 mig = kvm->arch.migration_mode;
1324
1325 if (attr->attr != KVM_S390_VM_MIGRATION_STATUS)
1326 return -ENXIO;
1327
1328 if (copy_to_user((void __user *)attr->addr, &mig, sizeof(mig)))
1329 return -EFAULT;
1330 return 0;
1331 }
1332
1333 static void __kvm_s390_set_tod_clock(struct kvm *kvm, const struct kvm_s390_vm_tod_clock *gtod);
1334
kvm_s390_set_tod_ext(struct kvm * kvm,struct kvm_device_attr * attr)1335 static int kvm_s390_set_tod_ext(struct kvm *kvm, struct kvm_device_attr *attr)
1336 {
1337 struct kvm_s390_vm_tod_clock gtod;
1338
1339 if (copy_from_user(>od, (void __user *)attr->addr, sizeof(gtod)))
1340 return -EFAULT;
1341
1342 if (!test_kvm_facility(kvm, 139) && gtod.epoch_idx)
1343 return -EINVAL;
1344 __kvm_s390_set_tod_clock(kvm, >od);
1345
1346 VM_EVENT(kvm, 3, "SET: TOD extension: 0x%x, TOD base: 0x%llx",
1347 gtod.epoch_idx, gtod.tod);
1348
1349 return 0;
1350 }
1351
kvm_s390_set_tod_high(struct kvm * kvm,struct kvm_device_attr * attr)1352 static int kvm_s390_set_tod_high(struct kvm *kvm, struct kvm_device_attr *attr)
1353 {
1354 u8 gtod_high;
1355
1356 if (copy_from_user(>od_high, (void __user *)attr->addr,
1357 sizeof(gtod_high)))
1358 return -EFAULT;
1359
1360 if (gtod_high != 0)
1361 return -EINVAL;
1362 VM_EVENT(kvm, 3, "SET: TOD extension: 0x%x", gtod_high);
1363
1364 return 0;
1365 }
1366
kvm_s390_set_tod_low(struct kvm * kvm,struct kvm_device_attr * attr)1367 static int kvm_s390_set_tod_low(struct kvm *kvm, struct kvm_device_attr *attr)
1368 {
1369 struct kvm_s390_vm_tod_clock gtod = { 0 };
1370
1371 if (copy_from_user(>od.tod, (void __user *)attr->addr,
1372 sizeof(gtod.tod)))
1373 return -EFAULT;
1374
1375 __kvm_s390_set_tod_clock(kvm, >od);
1376 VM_EVENT(kvm, 3, "SET: TOD base: 0x%llx", gtod.tod);
1377 return 0;
1378 }
1379
kvm_s390_set_tod(struct kvm * kvm,struct kvm_device_attr * attr)1380 static int kvm_s390_set_tod(struct kvm *kvm, struct kvm_device_attr *attr)
1381 {
1382 int ret;
1383
1384 if (attr->flags)
1385 return -EINVAL;
1386
1387 mutex_lock(&kvm->lock);
1388 /*
1389 * For protected guests, the TOD is managed by the ultravisor, so trying
1390 * to change it will never bring the expected results.
1391 */
1392 if (kvm_s390_pv_is_protected(kvm)) {
1393 ret = -EOPNOTSUPP;
1394 goto out_unlock;
1395 }
1396
1397 switch (attr->attr) {
1398 case KVM_S390_VM_TOD_EXT:
1399 ret = kvm_s390_set_tod_ext(kvm, attr);
1400 break;
1401 case KVM_S390_VM_TOD_HIGH:
1402 ret = kvm_s390_set_tod_high(kvm, attr);
1403 break;
1404 case KVM_S390_VM_TOD_LOW:
1405 ret = kvm_s390_set_tod_low(kvm, attr);
1406 break;
1407 default:
1408 ret = -ENXIO;
1409 break;
1410 }
1411
1412 out_unlock:
1413 mutex_unlock(&kvm->lock);
1414 return ret;
1415 }
1416
kvm_s390_get_tod_clock(struct kvm * kvm,struct kvm_s390_vm_tod_clock * gtod)1417 static void kvm_s390_get_tod_clock(struct kvm *kvm,
1418 struct kvm_s390_vm_tod_clock *gtod)
1419 {
1420 union tod_clock clk;
1421
1422 preempt_disable();
1423
1424 store_tod_clock_ext(&clk);
1425
1426 gtod->tod = clk.tod + kvm->arch.epoch;
1427 gtod->epoch_idx = 0;
1428 if (test_kvm_facility(kvm, 139)) {
1429 gtod->epoch_idx = clk.ei + kvm->arch.epdx;
1430 if (gtod->tod < clk.tod)
1431 gtod->epoch_idx += 1;
1432 }
1433
1434 preempt_enable();
1435 }
1436
kvm_s390_get_tod_ext(struct kvm * kvm,struct kvm_device_attr * attr)1437 static int kvm_s390_get_tod_ext(struct kvm *kvm, struct kvm_device_attr *attr)
1438 {
1439 struct kvm_s390_vm_tod_clock gtod;
1440
1441 memset(>od, 0, sizeof(gtod));
1442 kvm_s390_get_tod_clock(kvm, >od);
1443 if (copy_to_user((void __user *)attr->addr, >od, sizeof(gtod)))
1444 return -EFAULT;
1445
1446 VM_EVENT(kvm, 3, "QUERY: TOD extension: 0x%x, TOD base: 0x%llx",
1447 gtod.epoch_idx, gtod.tod);
1448 return 0;
1449 }
1450
kvm_s390_get_tod_high(struct kvm * kvm,struct kvm_device_attr * attr)1451 static int kvm_s390_get_tod_high(struct kvm *kvm, struct kvm_device_attr *attr)
1452 {
1453 u8 gtod_high = 0;
1454
1455 if (copy_to_user((void __user *)attr->addr, >od_high,
1456 sizeof(gtod_high)))
1457 return -EFAULT;
1458 VM_EVENT(kvm, 3, "QUERY: TOD extension: 0x%x", gtod_high);
1459
1460 return 0;
1461 }
1462
kvm_s390_get_tod_low(struct kvm * kvm,struct kvm_device_attr * attr)1463 static int kvm_s390_get_tod_low(struct kvm *kvm, struct kvm_device_attr *attr)
1464 {
1465 u64 gtod;
1466
1467 gtod = kvm_s390_get_tod_clock_fast(kvm);
1468 if (copy_to_user((void __user *)attr->addr, >od, sizeof(gtod)))
1469 return -EFAULT;
1470 VM_EVENT(kvm, 3, "QUERY: TOD base: 0x%llx", gtod);
1471
1472 return 0;
1473 }
1474
kvm_s390_get_tod(struct kvm * kvm,struct kvm_device_attr * attr)1475 static int kvm_s390_get_tod(struct kvm *kvm, struct kvm_device_attr *attr)
1476 {
1477 int ret;
1478
1479 if (attr->flags)
1480 return -EINVAL;
1481
1482 switch (attr->attr) {
1483 case KVM_S390_VM_TOD_EXT:
1484 ret = kvm_s390_get_tod_ext(kvm, attr);
1485 break;
1486 case KVM_S390_VM_TOD_HIGH:
1487 ret = kvm_s390_get_tod_high(kvm, attr);
1488 break;
1489 case KVM_S390_VM_TOD_LOW:
1490 ret = kvm_s390_get_tod_low(kvm, attr);
1491 break;
1492 default:
1493 ret = -ENXIO;
1494 break;
1495 }
1496 return ret;
1497 }
1498
kvm_s390_set_processor(struct kvm * kvm,struct kvm_device_attr * attr)1499 static int kvm_s390_set_processor(struct kvm *kvm, struct kvm_device_attr *attr)
1500 {
1501 struct kvm_s390_vm_cpu_processor *proc;
1502 u16 lowest_ibc, unblocked_ibc;
1503 int ret = 0;
1504
1505 mutex_lock(&kvm->lock);
1506 if (kvm->created_vcpus) {
1507 ret = -EBUSY;
1508 goto out;
1509 }
1510 proc = kzalloc_obj(*proc, GFP_KERNEL_ACCOUNT);
1511 if (!proc) {
1512 ret = -ENOMEM;
1513 goto out;
1514 }
1515 if (!copy_from_user(proc, (void __user *)attr->addr,
1516 sizeof(*proc))) {
1517 kvm->arch.model.cpuid = proc->cpuid;
1518 lowest_ibc = sclp.ibc >> 16 & 0xfff;
1519 unblocked_ibc = sclp.ibc & 0xfff;
1520 if (lowest_ibc && proc->ibc) {
1521 if (proc->ibc > unblocked_ibc)
1522 kvm->arch.model.ibc = unblocked_ibc;
1523 else if (proc->ibc < lowest_ibc)
1524 kvm->arch.model.ibc = lowest_ibc;
1525 else
1526 kvm->arch.model.ibc = proc->ibc;
1527 }
1528 memcpy(kvm->arch.model.fac_list, proc->fac_list,
1529 S390_ARCH_FAC_LIST_SIZE_BYTE);
1530 VM_EVENT(kvm, 3, "SET: guest ibc: 0x%4.4x, guest cpuid: 0x%16.16llx",
1531 kvm->arch.model.ibc,
1532 kvm->arch.model.cpuid);
1533 VM_EVENT(kvm, 3, "SET: guest faclist: 0x%16.16llx.%16.16llx.%16.16llx",
1534 kvm->arch.model.fac_list[0],
1535 kvm->arch.model.fac_list[1],
1536 kvm->arch.model.fac_list[2]);
1537 } else
1538 ret = -EFAULT;
1539 kfree(proc);
1540 out:
1541 mutex_unlock(&kvm->lock);
1542 return ret;
1543 }
1544
kvm_s390_set_processor_feat(struct kvm * kvm,struct kvm_device_attr * attr)1545 static int kvm_s390_set_processor_feat(struct kvm *kvm,
1546 struct kvm_device_attr *attr)
1547 {
1548 struct kvm_s390_vm_cpu_feat data;
1549
1550 if (copy_from_user(&data, (void __user *)attr->addr, sizeof(data)))
1551 return -EFAULT;
1552 if (!bitmap_subset((unsigned long *) data.feat,
1553 kvm_s390_available_cpu_feat,
1554 KVM_S390_VM_CPU_FEAT_NR_BITS))
1555 return -EINVAL;
1556
1557 mutex_lock(&kvm->lock);
1558 if (kvm->created_vcpus) {
1559 mutex_unlock(&kvm->lock);
1560 return -EBUSY;
1561 }
1562 bitmap_from_arr64(kvm->arch.cpu_feat, data.feat, KVM_S390_VM_CPU_FEAT_NR_BITS);
1563 mutex_unlock(&kvm->lock);
1564 VM_EVENT(kvm, 3, "SET: guest feat: 0x%16.16llx.0x%16.16llx.0x%16.16llx",
1565 data.feat[0],
1566 data.feat[1],
1567 data.feat[2]);
1568 return 0;
1569 }
1570
kvm_s390_set_processor_subfunc(struct kvm * kvm,struct kvm_device_attr * attr)1571 static int kvm_s390_set_processor_subfunc(struct kvm *kvm,
1572 struct kvm_device_attr *attr)
1573 {
1574 mutex_lock(&kvm->lock);
1575 if (kvm->created_vcpus) {
1576 mutex_unlock(&kvm->lock);
1577 return -EBUSY;
1578 }
1579
1580 if (copy_from_user(&kvm->arch.model.subfuncs, (void __user *)attr->addr,
1581 sizeof(struct kvm_s390_vm_cpu_subfunc))) {
1582 mutex_unlock(&kvm->lock);
1583 return -EFAULT;
1584 }
1585 mutex_unlock(&kvm->lock);
1586
1587 VM_EVENT(kvm, 3, "SET: guest PLO subfunc 0x%16.16lx.%16.16lx.%16.16lx.%16.16lx",
1588 ((unsigned long *) &kvm->arch.model.subfuncs.plo)[0],
1589 ((unsigned long *) &kvm->arch.model.subfuncs.plo)[1],
1590 ((unsigned long *) &kvm->arch.model.subfuncs.plo)[2],
1591 ((unsigned long *) &kvm->arch.model.subfuncs.plo)[3]);
1592 VM_EVENT(kvm, 3, "SET: guest PTFF subfunc 0x%16.16lx.%16.16lx",
1593 ((unsigned long *) &kvm->arch.model.subfuncs.ptff)[0],
1594 ((unsigned long *) &kvm->arch.model.subfuncs.ptff)[1]);
1595 VM_EVENT(kvm, 3, "SET: guest KMAC subfunc 0x%16.16lx.%16.16lx",
1596 ((unsigned long *) &kvm->arch.model.subfuncs.kmac)[0],
1597 ((unsigned long *) &kvm->arch.model.subfuncs.kmac)[1]);
1598 VM_EVENT(kvm, 3, "SET: guest KMC subfunc 0x%16.16lx.%16.16lx",
1599 ((unsigned long *) &kvm->arch.model.subfuncs.kmc)[0],
1600 ((unsigned long *) &kvm->arch.model.subfuncs.kmc)[1]);
1601 VM_EVENT(kvm, 3, "SET: guest KM subfunc 0x%16.16lx.%16.16lx",
1602 ((unsigned long *) &kvm->arch.model.subfuncs.km)[0],
1603 ((unsigned long *) &kvm->arch.model.subfuncs.km)[1]);
1604 VM_EVENT(kvm, 3, "SET: guest KIMD subfunc 0x%16.16lx.%16.16lx",
1605 ((unsigned long *) &kvm->arch.model.subfuncs.kimd)[0],
1606 ((unsigned long *) &kvm->arch.model.subfuncs.kimd)[1]);
1607 VM_EVENT(kvm, 3, "SET: guest KLMD subfunc 0x%16.16lx.%16.16lx",
1608 ((unsigned long *) &kvm->arch.model.subfuncs.klmd)[0],
1609 ((unsigned long *) &kvm->arch.model.subfuncs.klmd)[1]);
1610 VM_EVENT(kvm, 3, "SET: guest PCKMO subfunc 0x%16.16lx.%16.16lx",
1611 ((unsigned long *) &kvm->arch.model.subfuncs.pckmo)[0],
1612 ((unsigned long *) &kvm->arch.model.subfuncs.pckmo)[1]);
1613 VM_EVENT(kvm, 3, "SET: guest KMCTR subfunc 0x%16.16lx.%16.16lx",
1614 ((unsigned long *) &kvm->arch.model.subfuncs.kmctr)[0],
1615 ((unsigned long *) &kvm->arch.model.subfuncs.kmctr)[1]);
1616 VM_EVENT(kvm, 3, "SET: guest KMF subfunc 0x%16.16lx.%16.16lx",
1617 ((unsigned long *) &kvm->arch.model.subfuncs.kmf)[0],
1618 ((unsigned long *) &kvm->arch.model.subfuncs.kmf)[1]);
1619 VM_EVENT(kvm, 3, "SET: guest KMO subfunc 0x%16.16lx.%16.16lx",
1620 ((unsigned long *) &kvm->arch.model.subfuncs.kmo)[0],
1621 ((unsigned long *) &kvm->arch.model.subfuncs.kmo)[1]);
1622 VM_EVENT(kvm, 3, "SET: guest PCC subfunc 0x%16.16lx.%16.16lx",
1623 ((unsigned long *) &kvm->arch.model.subfuncs.pcc)[0],
1624 ((unsigned long *) &kvm->arch.model.subfuncs.pcc)[1]);
1625 VM_EVENT(kvm, 3, "SET: guest PPNO subfunc 0x%16.16lx.%16.16lx",
1626 ((unsigned long *) &kvm->arch.model.subfuncs.ppno)[0],
1627 ((unsigned long *) &kvm->arch.model.subfuncs.ppno)[1]);
1628 VM_EVENT(kvm, 3, "SET: guest KMA subfunc 0x%16.16lx.%16.16lx",
1629 ((unsigned long *) &kvm->arch.model.subfuncs.kma)[0],
1630 ((unsigned long *) &kvm->arch.model.subfuncs.kma)[1]);
1631 VM_EVENT(kvm, 3, "SET: guest KDSA subfunc 0x%16.16lx.%16.16lx",
1632 ((unsigned long *) &kvm->arch.model.subfuncs.kdsa)[0],
1633 ((unsigned long *) &kvm->arch.model.subfuncs.kdsa)[1]);
1634 VM_EVENT(kvm, 3, "SET: guest SORTL subfunc 0x%16.16lx.%16.16lx.%16.16lx.%16.16lx",
1635 ((unsigned long *) &kvm->arch.model.subfuncs.sortl)[0],
1636 ((unsigned long *) &kvm->arch.model.subfuncs.sortl)[1],
1637 ((unsigned long *) &kvm->arch.model.subfuncs.sortl)[2],
1638 ((unsigned long *) &kvm->arch.model.subfuncs.sortl)[3]);
1639 VM_EVENT(kvm, 3, "SET: guest DFLTCC subfunc 0x%16.16lx.%16.16lx.%16.16lx.%16.16lx",
1640 ((unsigned long *) &kvm->arch.model.subfuncs.dfltcc)[0],
1641 ((unsigned long *) &kvm->arch.model.subfuncs.dfltcc)[1],
1642 ((unsigned long *) &kvm->arch.model.subfuncs.dfltcc)[2],
1643 ((unsigned long *) &kvm->arch.model.subfuncs.dfltcc)[3]);
1644 VM_EVENT(kvm, 3, "GET: guest PFCR subfunc 0x%16.16lx.%16.16lx",
1645 ((unsigned long *) &kvm_s390_available_subfunc.pfcr)[0],
1646 ((unsigned long *) &kvm_s390_available_subfunc.pfcr)[1]);
1647
1648 return 0;
1649 }
1650
1651 #define KVM_S390_VM_CPU_UV_FEAT_GUEST_MASK \
1652 ( \
1653 ((struct kvm_s390_vm_cpu_uv_feat){ \
1654 .ap = 1, \
1655 .ap_intr = 1, \
1656 }) \
1657 .feat \
1658 )
1659
kvm_s390_set_uv_feat(struct kvm * kvm,struct kvm_device_attr * attr)1660 static int kvm_s390_set_uv_feat(struct kvm *kvm, struct kvm_device_attr *attr)
1661 {
1662 struct kvm_s390_vm_cpu_uv_feat __user *ptr = (void __user *)attr->addr;
1663 unsigned long data, filter;
1664
1665 filter = uv_info.uv_feature_indications & KVM_S390_VM_CPU_UV_FEAT_GUEST_MASK;
1666 if (get_user(data, &ptr->feat))
1667 return -EFAULT;
1668 if (!bitmap_subset(&data, &filter, KVM_S390_VM_CPU_UV_FEAT_NR_BITS))
1669 return -EINVAL;
1670
1671 mutex_lock(&kvm->lock);
1672 if (kvm->created_vcpus) {
1673 mutex_unlock(&kvm->lock);
1674 return -EBUSY;
1675 }
1676 kvm->arch.model.uv_feat_guest.feat = data;
1677 mutex_unlock(&kvm->lock);
1678
1679 VM_EVENT(kvm, 3, "SET: guest UV-feat: 0x%16.16lx", data);
1680
1681 return 0;
1682 }
1683
kvm_s390_set_cpu_model(struct kvm * kvm,struct kvm_device_attr * attr)1684 static int kvm_s390_set_cpu_model(struct kvm *kvm, struct kvm_device_attr *attr)
1685 {
1686 int ret = -ENXIO;
1687
1688 switch (attr->attr) {
1689 case KVM_S390_VM_CPU_PROCESSOR:
1690 ret = kvm_s390_set_processor(kvm, attr);
1691 break;
1692 case KVM_S390_VM_CPU_PROCESSOR_FEAT:
1693 ret = kvm_s390_set_processor_feat(kvm, attr);
1694 break;
1695 case KVM_S390_VM_CPU_PROCESSOR_SUBFUNC:
1696 ret = kvm_s390_set_processor_subfunc(kvm, attr);
1697 break;
1698 case KVM_S390_VM_CPU_PROCESSOR_UV_FEAT_GUEST:
1699 ret = kvm_s390_set_uv_feat(kvm, attr);
1700 break;
1701 }
1702 return ret;
1703 }
1704
kvm_s390_get_processor(struct kvm * kvm,struct kvm_device_attr * attr)1705 static int kvm_s390_get_processor(struct kvm *kvm, struct kvm_device_attr *attr)
1706 {
1707 struct kvm_s390_vm_cpu_processor *proc;
1708 int ret = 0;
1709
1710 proc = kzalloc_obj(*proc, GFP_KERNEL_ACCOUNT);
1711 if (!proc) {
1712 ret = -ENOMEM;
1713 goto out;
1714 }
1715 proc->cpuid = kvm->arch.model.cpuid;
1716 proc->ibc = kvm->arch.model.ibc;
1717 memcpy(&proc->fac_list, kvm->arch.model.fac_list,
1718 S390_ARCH_FAC_LIST_SIZE_BYTE);
1719 VM_EVENT(kvm, 3, "GET: guest ibc: 0x%4.4x, guest cpuid: 0x%16.16llx",
1720 kvm->arch.model.ibc,
1721 kvm->arch.model.cpuid);
1722 VM_EVENT(kvm, 3, "GET: guest faclist: 0x%16.16llx.%16.16llx.%16.16llx",
1723 kvm->arch.model.fac_list[0],
1724 kvm->arch.model.fac_list[1],
1725 kvm->arch.model.fac_list[2]);
1726 if (copy_to_user((void __user *)attr->addr, proc, sizeof(*proc)))
1727 ret = -EFAULT;
1728 kfree(proc);
1729 out:
1730 return ret;
1731 }
1732
kvm_s390_get_machine(struct kvm * kvm,struct kvm_device_attr * attr)1733 static int kvm_s390_get_machine(struct kvm *kvm, struct kvm_device_attr *attr)
1734 {
1735 struct kvm_s390_vm_cpu_machine *mach;
1736 int ret = 0;
1737
1738 mach = kzalloc_obj(*mach, GFP_KERNEL_ACCOUNT);
1739 if (!mach) {
1740 ret = -ENOMEM;
1741 goto out;
1742 }
1743 get_cpu_id((struct cpuid *) &mach->cpuid);
1744 mach->ibc = sclp.ibc;
1745 memcpy(&mach->fac_mask, kvm->arch.model.fac_mask,
1746 S390_ARCH_FAC_LIST_SIZE_BYTE);
1747 memcpy((unsigned long *)&mach->fac_list, stfle_fac_list,
1748 sizeof(stfle_fac_list));
1749 VM_EVENT(kvm, 3, "GET: host ibc: 0x%4.4x, host cpuid: 0x%16.16llx",
1750 kvm->arch.model.ibc,
1751 kvm->arch.model.cpuid);
1752 VM_EVENT(kvm, 3, "GET: host facmask: 0x%16.16llx.%16.16llx.%16.16llx",
1753 mach->fac_mask[0],
1754 mach->fac_mask[1],
1755 mach->fac_mask[2]);
1756 VM_EVENT(kvm, 3, "GET: host faclist: 0x%16.16llx.%16.16llx.%16.16llx",
1757 mach->fac_list[0],
1758 mach->fac_list[1],
1759 mach->fac_list[2]);
1760 if (copy_to_user((void __user *)attr->addr, mach, sizeof(*mach)))
1761 ret = -EFAULT;
1762 kfree(mach);
1763 out:
1764 return ret;
1765 }
1766
kvm_s390_get_processor_feat(struct kvm * kvm,struct kvm_device_attr * attr)1767 static int kvm_s390_get_processor_feat(struct kvm *kvm,
1768 struct kvm_device_attr *attr)
1769 {
1770 struct kvm_s390_vm_cpu_feat data;
1771
1772 bitmap_to_arr64(data.feat, kvm->arch.cpu_feat, KVM_S390_VM_CPU_FEAT_NR_BITS);
1773 if (copy_to_user((void __user *)attr->addr, &data, sizeof(data)))
1774 return -EFAULT;
1775 VM_EVENT(kvm, 3, "GET: guest feat: 0x%16.16llx.0x%16.16llx.0x%16.16llx",
1776 data.feat[0],
1777 data.feat[1],
1778 data.feat[2]);
1779 return 0;
1780 }
1781
kvm_s390_get_machine_feat(struct kvm * kvm,struct kvm_device_attr * attr)1782 static int kvm_s390_get_machine_feat(struct kvm *kvm,
1783 struct kvm_device_attr *attr)
1784 {
1785 struct kvm_s390_vm_cpu_feat data;
1786
1787 bitmap_to_arr64(data.feat, kvm_s390_available_cpu_feat, KVM_S390_VM_CPU_FEAT_NR_BITS);
1788 if (copy_to_user((void __user *)attr->addr, &data, sizeof(data)))
1789 return -EFAULT;
1790 VM_EVENT(kvm, 3, "GET: host feat: 0x%16.16llx.0x%16.16llx.0x%16.16llx",
1791 data.feat[0],
1792 data.feat[1],
1793 data.feat[2]);
1794 return 0;
1795 }
1796
kvm_s390_get_processor_subfunc(struct kvm * kvm,struct kvm_device_attr * attr)1797 static int kvm_s390_get_processor_subfunc(struct kvm *kvm,
1798 struct kvm_device_attr *attr)
1799 {
1800 if (copy_to_user((void __user *)attr->addr, &kvm->arch.model.subfuncs,
1801 sizeof(struct kvm_s390_vm_cpu_subfunc)))
1802 return -EFAULT;
1803
1804 VM_EVENT(kvm, 3, "GET: guest PLO subfunc 0x%16.16lx.%16.16lx.%16.16lx.%16.16lx",
1805 ((unsigned long *) &kvm->arch.model.subfuncs.plo)[0],
1806 ((unsigned long *) &kvm->arch.model.subfuncs.plo)[1],
1807 ((unsigned long *) &kvm->arch.model.subfuncs.plo)[2],
1808 ((unsigned long *) &kvm->arch.model.subfuncs.plo)[3]);
1809 VM_EVENT(kvm, 3, "GET: guest PTFF subfunc 0x%16.16lx.%16.16lx",
1810 ((unsigned long *) &kvm->arch.model.subfuncs.ptff)[0],
1811 ((unsigned long *) &kvm->arch.model.subfuncs.ptff)[1]);
1812 VM_EVENT(kvm, 3, "GET: guest KMAC subfunc 0x%16.16lx.%16.16lx",
1813 ((unsigned long *) &kvm->arch.model.subfuncs.kmac)[0],
1814 ((unsigned long *) &kvm->arch.model.subfuncs.kmac)[1]);
1815 VM_EVENT(kvm, 3, "GET: guest KMC subfunc 0x%16.16lx.%16.16lx",
1816 ((unsigned long *) &kvm->arch.model.subfuncs.kmc)[0],
1817 ((unsigned long *) &kvm->arch.model.subfuncs.kmc)[1]);
1818 VM_EVENT(kvm, 3, "GET: guest KM subfunc 0x%16.16lx.%16.16lx",
1819 ((unsigned long *) &kvm->arch.model.subfuncs.km)[0],
1820 ((unsigned long *) &kvm->arch.model.subfuncs.km)[1]);
1821 VM_EVENT(kvm, 3, "GET: guest KIMD subfunc 0x%16.16lx.%16.16lx",
1822 ((unsigned long *) &kvm->arch.model.subfuncs.kimd)[0],
1823 ((unsigned long *) &kvm->arch.model.subfuncs.kimd)[1]);
1824 VM_EVENT(kvm, 3, "GET: guest KLMD subfunc 0x%16.16lx.%16.16lx",
1825 ((unsigned long *) &kvm->arch.model.subfuncs.klmd)[0],
1826 ((unsigned long *) &kvm->arch.model.subfuncs.klmd)[1]);
1827 VM_EVENT(kvm, 3, "GET: guest PCKMO subfunc 0x%16.16lx.%16.16lx",
1828 ((unsigned long *) &kvm->arch.model.subfuncs.pckmo)[0],
1829 ((unsigned long *) &kvm->arch.model.subfuncs.pckmo)[1]);
1830 VM_EVENT(kvm, 3, "GET: guest KMCTR subfunc 0x%16.16lx.%16.16lx",
1831 ((unsigned long *) &kvm->arch.model.subfuncs.kmctr)[0],
1832 ((unsigned long *) &kvm->arch.model.subfuncs.kmctr)[1]);
1833 VM_EVENT(kvm, 3, "GET: guest KMF subfunc 0x%16.16lx.%16.16lx",
1834 ((unsigned long *) &kvm->arch.model.subfuncs.kmf)[0],
1835 ((unsigned long *) &kvm->arch.model.subfuncs.kmf)[1]);
1836 VM_EVENT(kvm, 3, "GET: guest KMO subfunc 0x%16.16lx.%16.16lx",
1837 ((unsigned long *) &kvm->arch.model.subfuncs.kmo)[0],
1838 ((unsigned long *) &kvm->arch.model.subfuncs.kmo)[1]);
1839 VM_EVENT(kvm, 3, "GET: guest PCC subfunc 0x%16.16lx.%16.16lx",
1840 ((unsigned long *) &kvm->arch.model.subfuncs.pcc)[0],
1841 ((unsigned long *) &kvm->arch.model.subfuncs.pcc)[1]);
1842 VM_EVENT(kvm, 3, "GET: guest PPNO subfunc 0x%16.16lx.%16.16lx",
1843 ((unsigned long *) &kvm->arch.model.subfuncs.ppno)[0],
1844 ((unsigned long *) &kvm->arch.model.subfuncs.ppno)[1]);
1845 VM_EVENT(kvm, 3, "GET: guest KMA subfunc 0x%16.16lx.%16.16lx",
1846 ((unsigned long *) &kvm->arch.model.subfuncs.kma)[0],
1847 ((unsigned long *) &kvm->arch.model.subfuncs.kma)[1]);
1848 VM_EVENT(kvm, 3, "GET: guest KDSA subfunc 0x%16.16lx.%16.16lx",
1849 ((unsigned long *) &kvm->arch.model.subfuncs.kdsa)[0],
1850 ((unsigned long *) &kvm->arch.model.subfuncs.kdsa)[1]);
1851 VM_EVENT(kvm, 3, "GET: guest SORTL subfunc 0x%16.16lx.%16.16lx.%16.16lx.%16.16lx",
1852 ((unsigned long *) &kvm->arch.model.subfuncs.sortl)[0],
1853 ((unsigned long *) &kvm->arch.model.subfuncs.sortl)[1],
1854 ((unsigned long *) &kvm->arch.model.subfuncs.sortl)[2],
1855 ((unsigned long *) &kvm->arch.model.subfuncs.sortl)[3]);
1856 VM_EVENT(kvm, 3, "GET: guest DFLTCC subfunc 0x%16.16lx.%16.16lx.%16.16lx.%16.16lx",
1857 ((unsigned long *) &kvm->arch.model.subfuncs.dfltcc)[0],
1858 ((unsigned long *) &kvm->arch.model.subfuncs.dfltcc)[1],
1859 ((unsigned long *) &kvm->arch.model.subfuncs.dfltcc)[2],
1860 ((unsigned long *) &kvm->arch.model.subfuncs.dfltcc)[3]);
1861 VM_EVENT(kvm, 3, "GET: guest PFCR subfunc 0x%16.16lx.%16.16lx",
1862 ((unsigned long *) &kvm_s390_available_subfunc.pfcr)[0],
1863 ((unsigned long *) &kvm_s390_available_subfunc.pfcr)[1]);
1864
1865 return 0;
1866 }
1867
kvm_s390_get_machine_subfunc(struct kvm * kvm,struct kvm_device_attr * attr)1868 static int kvm_s390_get_machine_subfunc(struct kvm *kvm,
1869 struct kvm_device_attr *attr)
1870 {
1871 if (copy_to_user((void __user *)attr->addr, &kvm_s390_available_subfunc,
1872 sizeof(struct kvm_s390_vm_cpu_subfunc)))
1873 return -EFAULT;
1874
1875 VM_EVENT(kvm, 3, "GET: host PLO subfunc 0x%16.16lx.%16.16lx.%16.16lx.%16.16lx",
1876 ((unsigned long *) &kvm_s390_available_subfunc.plo)[0],
1877 ((unsigned long *) &kvm_s390_available_subfunc.plo)[1],
1878 ((unsigned long *) &kvm_s390_available_subfunc.plo)[2],
1879 ((unsigned long *) &kvm_s390_available_subfunc.plo)[3]);
1880 VM_EVENT(kvm, 3, "GET: host PTFF subfunc 0x%16.16lx.%16.16lx",
1881 ((unsigned long *) &kvm_s390_available_subfunc.ptff)[0],
1882 ((unsigned long *) &kvm_s390_available_subfunc.ptff)[1]);
1883 VM_EVENT(kvm, 3, "GET: host KMAC subfunc 0x%16.16lx.%16.16lx",
1884 ((unsigned long *) &kvm_s390_available_subfunc.kmac)[0],
1885 ((unsigned long *) &kvm_s390_available_subfunc.kmac)[1]);
1886 VM_EVENT(kvm, 3, "GET: host KMC subfunc 0x%16.16lx.%16.16lx",
1887 ((unsigned long *) &kvm_s390_available_subfunc.kmc)[0],
1888 ((unsigned long *) &kvm_s390_available_subfunc.kmc)[1]);
1889 VM_EVENT(kvm, 3, "GET: host KM subfunc 0x%16.16lx.%16.16lx",
1890 ((unsigned long *) &kvm_s390_available_subfunc.km)[0],
1891 ((unsigned long *) &kvm_s390_available_subfunc.km)[1]);
1892 VM_EVENT(kvm, 3, "GET: host KIMD subfunc 0x%16.16lx.%16.16lx",
1893 ((unsigned long *) &kvm_s390_available_subfunc.kimd)[0],
1894 ((unsigned long *) &kvm_s390_available_subfunc.kimd)[1]);
1895 VM_EVENT(kvm, 3, "GET: host KLMD subfunc 0x%16.16lx.%16.16lx",
1896 ((unsigned long *) &kvm_s390_available_subfunc.klmd)[0],
1897 ((unsigned long *) &kvm_s390_available_subfunc.klmd)[1]);
1898 VM_EVENT(kvm, 3, "GET: host PCKMO subfunc 0x%16.16lx.%16.16lx",
1899 ((unsigned long *) &kvm_s390_available_subfunc.pckmo)[0],
1900 ((unsigned long *) &kvm_s390_available_subfunc.pckmo)[1]);
1901 VM_EVENT(kvm, 3, "GET: host KMCTR subfunc 0x%16.16lx.%16.16lx",
1902 ((unsigned long *) &kvm_s390_available_subfunc.kmctr)[0],
1903 ((unsigned long *) &kvm_s390_available_subfunc.kmctr)[1]);
1904 VM_EVENT(kvm, 3, "GET: host KMF subfunc 0x%16.16lx.%16.16lx",
1905 ((unsigned long *) &kvm_s390_available_subfunc.kmf)[0],
1906 ((unsigned long *) &kvm_s390_available_subfunc.kmf)[1]);
1907 VM_EVENT(kvm, 3, "GET: host KMO subfunc 0x%16.16lx.%16.16lx",
1908 ((unsigned long *) &kvm_s390_available_subfunc.kmo)[0],
1909 ((unsigned long *) &kvm_s390_available_subfunc.kmo)[1]);
1910 VM_EVENT(kvm, 3, "GET: host PCC subfunc 0x%16.16lx.%16.16lx",
1911 ((unsigned long *) &kvm_s390_available_subfunc.pcc)[0],
1912 ((unsigned long *) &kvm_s390_available_subfunc.pcc)[1]);
1913 VM_EVENT(kvm, 3, "GET: host PPNO subfunc 0x%16.16lx.%16.16lx",
1914 ((unsigned long *) &kvm_s390_available_subfunc.ppno)[0],
1915 ((unsigned long *) &kvm_s390_available_subfunc.ppno)[1]);
1916 VM_EVENT(kvm, 3, "GET: host KMA subfunc 0x%16.16lx.%16.16lx",
1917 ((unsigned long *) &kvm_s390_available_subfunc.kma)[0],
1918 ((unsigned long *) &kvm_s390_available_subfunc.kma)[1]);
1919 VM_EVENT(kvm, 3, "GET: host KDSA subfunc 0x%16.16lx.%16.16lx",
1920 ((unsigned long *) &kvm_s390_available_subfunc.kdsa)[0],
1921 ((unsigned long *) &kvm_s390_available_subfunc.kdsa)[1]);
1922 VM_EVENT(kvm, 3, "GET: host SORTL subfunc 0x%16.16lx.%16.16lx.%16.16lx.%16.16lx",
1923 ((unsigned long *) &kvm_s390_available_subfunc.sortl)[0],
1924 ((unsigned long *) &kvm_s390_available_subfunc.sortl)[1],
1925 ((unsigned long *) &kvm_s390_available_subfunc.sortl)[2],
1926 ((unsigned long *) &kvm_s390_available_subfunc.sortl)[3]);
1927 VM_EVENT(kvm, 3, "GET: host DFLTCC subfunc 0x%16.16lx.%16.16lx.%16.16lx.%16.16lx",
1928 ((unsigned long *) &kvm_s390_available_subfunc.dfltcc)[0],
1929 ((unsigned long *) &kvm_s390_available_subfunc.dfltcc)[1],
1930 ((unsigned long *) &kvm_s390_available_subfunc.dfltcc)[2],
1931 ((unsigned long *) &kvm_s390_available_subfunc.dfltcc)[3]);
1932 VM_EVENT(kvm, 3, "GET: host PFCR subfunc 0x%16.16lx.%16.16lx",
1933 ((unsigned long *) &kvm_s390_available_subfunc.pfcr)[0],
1934 ((unsigned long *) &kvm_s390_available_subfunc.pfcr)[1]);
1935
1936 return 0;
1937 }
1938
kvm_s390_get_processor_uv_feat(struct kvm * kvm,struct kvm_device_attr * attr)1939 static int kvm_s390_get_processor_uv_feat(struct kvm *kvm, struct kvm_device_attr *attr)
1940 {
1941 struct kvm_s390_vm_cpu_uv_feat __user *dst = (void __user *)attr->addr;
1942 unsigned long feat = kvm->arch.model.uv_feat_guest.feat;
1943
1944 if (put_user(feat, &dst->feat))
1945 return -EFAULT;
1946 VM_EVENT(kvm, 3, "GET: guest UV-feat: 0x%16.16lx", feat);
1947
1948 return 0;
1949 }
1950
kvm_s390_get_machine_uv_feat(struct kvm * kvm,struct kvm_device_attr * attr)1951 static int kvm_s390_get_machine_uv_feat(struct kvm *kvm, struct kvm_device_attr *attr)
1952 {
1953 struct kvm_s390_vm_cpu_uv_feat __user *dst = (void __user *)attr->addr;
1954 unsigned long feat;
1955
1956 BUILD_BUG_ON(sizeof(*dst) != sizeof(uv_info.uv_feature_indications));
1957
1958 feat = uv_info.uv_feature_indications & KVM_S390_VM_CPU_UV_FEAT_GUEST_MASK;
1959 if (put_user(feat, &dst->feat))
1960 return -EFAULT;
1961 VM_EVENT(kvm, 3, "GET: guest UV-feat: 0x%16.16lx", feat);
1962
1963 return 0;
1964 }
1965
kvm_s390_get_cpu_model(struct kvm * kvm,struct kvm_device_attr * attr)1966 static int kvm_s390_get_cpu_model(struct kvm *kvm, struct kvm_device_attr *attr)
1967 {
1968 int ret = -ENXIO;
1969
1970 switch (attr->attr) {
1971 case KVM_S390_VM_CPU_PROCESSOR:
1972 ret = kvm_s390_get_processor(kvm, attr);
1973 break;
1974 case KVM_S390_VM_CPU_MACHINE:
1975 ret = kvm_s390_get_machine(kvm, attr);
1976 break;
1977 case KVM_S390_VM_CPU_PROCESSOR_FEAT:
1978 ret = kvm_s390_get_processor_feat(kvm, attr);
1979 break;
1980 case KVM_S390_VM_CPU_MACHINE_FEAT:
1981 ret = kvm_s390_get_machine_feat(kvm, attr);
1982 break;
1983 case KVM_S390_VM_CPU_PROCESSOR_SUBFUNC:
1984 ret = kvm_s390_get_processor_subfunc(kvm, attr);
1985 break;
1986 case KVM_S390_VM_CPU_MACHINE_SUBFUNC:
1987 ret = kvm_s390_get_machine_subfunc(kvm, attr);
1988 break;
1989 case KVM_S390_VM_CPU_PROCESSOR_UV_FEAT_GUEST:
1990 ret = kvm_s390_get_processor_uv_feat(kvm, attr);
1991 break;
1992 case KVM_S390_VM_CPU_MACHINE_UV_FEAT_GUEST:
1993 ret = kvm_s390_get_machine_uv_feat(kvm, attr);
1994 break;
1995 }
1996 return ret;
1997 }
1998
1999 /**
2000 * kvm_s390_update_topology_change_report - update CPU topology change report
2001 * @kvm: guest KVM description
2002 * @val: set or clear the MTCR bit
2003 *
2004 * Updates the Multiprocessor Topology-Change-Report bit to signal
2005 * the guest with a topology change.
2006 * This is only relevant if the topology facility is present.
2007 */
kvm_s390_update_topology_change_report(struct kvm * kvm,bool val)2008 static void kvm_s390_update_topology_change_report(struct kvm *kvm, bool val)
2009 {
2010 union sca_utility new, old;
2011 struct esca_block *sca;
2012
2013 sca = kvm->arch.sca;
2014 old = READ_ONCE(sca->utility);
2015 do {
2016 new = old;
2017 new.mtcr = val;
2018 } while (!try_cmpxchg(&sca->utility.val, &old.val, new.val));
2019 }
2020
kvm_s390_set_topo_change_indication(struct kvm * kvm,struct kvm_device_attr * attr)2021 static int kvm_s390_set_topo_change_indication(struct kvm *kvm,
2022 struct kvm_device_attr *attr)
2023 {
2024 if (!test_kvm_facility(kvm, 11))
2025 return -ENXIO;
2026
2027 kvm_s390_update_topology_change_report(kvm, !!attr->attr);
2028 return 0;
2029 }
2030
kvm_s390_get_topo_change_indication(struct kvm * kvm,struct kvm_device_attr * attr)2031 static int kvm_s390_get_topo_change_indication(struct kvm *kvm,
2032 struct kvm_device_attr *attr)
2033 {
2034 u8 topo;
2035
2036 if (!test_kvm_facility(kvm, 11))
2037 return -ENXIO;
2038
2039 topo = kvm->arch.sca->utility.mtcr;
2040
2041 return put_user(topo, (u8 __user *)attr->addr);
2042 }
2043
kvm_s390_vm_set_attr(struct kvm * kvm,struct kvm_device_attr * attr)2044 static int kvm_s390_vm_set_attr(struct kvm *kvm, struct kvm_device_attr *attr)
2045 {
2046 int ret;
2047
2048 switch (attr->group) {
2049 case KVM_S390_VM_MEM_CTRL:
2050 ret = kvm_s390_set_mem_control(kvm, attr);
2051 break;
2052 case KVM_S390_VM_TOD:
2053 ret = kvm_s390_set_tod(kvm, attr);
2054 break;
2055 case KVM_S390_VM_CPU_MODEL:
2056 ret = kvm_s390_set_cpu_model(kvm, attr);
2057 break;
2058 case KVM_S390_VM_CRYPTO:
2059 ret = kvm_s390_vm_set_crypto(kvm, attr);
2060 break;
2061 case KVM_S390_VM_MIGRATION:
2062 ret = kvm_s390_vm_set_migration(kvm, attr);
2063 break;
2064 case KVM_S390_VM_CPU_TOPOLOGY:
2065 ret = kvm_s390_set_topo_change_indication(kvm, attr);
2066 break;
2067 default:
2068 ret = -ENXIO;
2069 break;
2070 }
2071
2072 return ret;
2073 }
2074
kvm_s390_vm_get_attr(struct kvm * kvm,struct kvm_device_attr * attr)2075 static int kvm_s390_vm_get_attr(struct kvm *kvm, struct kvm_device_attr *attr)
2076 {
2077 int ret;
2078
2079 switch (attr->group) {
2080 case KVM_S390_VM_MEM_CTRL:
2081 ret = kvm_s390_get_mem_control(kvm, attr);
2082 break;
2083 case KVM_S390_VM_TOD:
2084 ret = kvm_s390_get_tod(kvm, attr);
2085 break;
2086 case KVM_S390_VM_CPU_MODEL:
2087 ret = kvm_s390_get_cpu_model(kvm, attr);
2088 break;
2089 case KVM_S390_VM_MIGRATION:
2090 ret = kvm_s390_vm_get_migration(kvm, attr);
2091 break;
2092 case KVM_S390_VM_CPU_TOPOLOGY:
2093 ret = kvm_s390_get_topo_change_indication(kvm, attr);
2094 break;
2095 default:
2096 ret = -ENXIO;
2097 break;
2098 }
2099
2100 return ret;
2101 }
2102
kvm_s390_vm_has_attr(struct kvm * kvm,struct kvm_device_attr * attr)2103 static int kvm_s390_vm_has_attr(struct kvm *kvm, struct kvm_device_attr *attr)
2104 {
2105 int ret;
2106
2107 switch (attr->group) {
2108 case KVM_S390_VM_MEM_CTRL:
2109 switch (attr->attr) {
2110 case KVM_S390_VM_MEM_ENABLE_CMMA:
2111 case KVM_S390_VM_MEM_CLR_CMMA:
2112 ret = sclp.has_cmma ? 0 : -ENXIO;
2113 break;
2114 case KVM_S390_VM_MEM_LIMIT_SIZE:
2115 ret = 0;
2116 break;
2117 default:
2118 ret = -ENXIO;
2119 break;
2120 }
2121 break;
2122 case KVM_S390_VM_TOD:
2123 switch (attr->attr) {
2124 case KVM_S390_VM_TOD_LOW:
2125 case KVM_S390_VM_TOD_HIGH:
2126 ret = 0;
2127 break;
2128 default:
2129 ret = -ENXIO;
2130 break;
2131 }
2132 break;
2133 case KVM_S390_VM_CPU_MODEL:
2134 switch (attr->attr) {
2135 case KVM_S390_VM_CPU_PROCESSOR:
2136 case KVM_S390_VM_CPU_MACHINE:
2137 case KVM_S390_VM_CPU_PROCESSOR_FEAT:
2138 case KVM_S390_VM_CPU_MACHINE_FEAT:
2139 case KVM_S390_VM_CPU_MACHINE_SUBFUNC:
2140 case KVM_S390_VM_CPU_PROCESSOR_SUBFUNC:
2141 case KVM_S390_VM_CPU_MACHINE_UV_FEAT_GUEST:
2142 case KVM_S390_VM_CPU_PROCESSOR_UV_FEAT_GUEST:
2143 ret = 0;
2144 break;
2145 default:
2146 ret = -ENXIO;
2147 break;
2148 }
2149 break;
2150 case KVM_S390_VM_CRYPTO:
2151 switch (attr->attr) {
2152 case KVM_S390_VM_CRYPTO_ENABLE_AES_KW:
2153 case KVM_S390_VM_CRYPTO_ENABLE_DEA_KW:
2154 case KVM_S390_VM_CRYPTO_DISABLE_AES_KW:
2155 case KVM_S390_VM_CRYPTO_DISABLE_DEA_KW:
2156 ret = 0;
2157 break;
2158 case KVM_S390_VM_CRYPTO_ENABLE_APIE:
2159 case KVM_S390_VM_CRYPTO_DISABLE_APIE:
2160 ret = ap_instructions_available() ? 0 : -ENXIO;
2161 break;
2162 default:
2163 ret = -ENXIO;
2164 break;
2165 }
2166 break;
2167 case KVM_S390_VM_MIGRATION:
2168 ret = 0;
2169 break;
2170 case KVM_S390_VM_CPU_TOPOLOGY:
2171 ret = test_kvm_facility(kvm, 11) ? 0 : -ENXIO;
2172 break;
2173 default:
2174 ret = -ENXIO;
2175 break;
2176 }
2177
2178 return ret;
2179 }
2180
kvm_s390_get_skeys(struct kvm * kvm,struct kvm_s390_skeys * args)2181 static int kvm_s390_get_skeys(struct kvm *kvm, struct kvm_s390_skeys *args)
2182 {
2183 union skey *keys;
2184 int i, r = 0;
2185
2186 if (args->flags != 0)
2187 return -EINVAL;
2188
2189 /* Is this guest using storage keys? */
2190 if (!uses_skeys(kvm->arch.gmap))
2191 return KVM_S390_GET_SKEYS_NONE;
2192
2193 /* Enforce sane limit on memory allocation */
2194 if (args->count < 1 || args->count > KVM_S390_SKEYS_MAX)
2195 return -EINVAL;
2196
2197 keys = kvmalloc_array(args->count, sizeof(*keys), GFP_KERNEL_ACCOUNT);
2198 if (!keys)
2199 return -ENOMEM;
2200
2201 scoped_guard(read_lock, &kvm->mmu_lock) {
2202 for (i = 0; i < args->count; i++) {
2203 r = dat_get_storage_key(kvm->arch.gmap->asce,
2204 args->start_gfn + i, keys + i);
2205 if (r)
2206 break;
2207 }
2208 }
2209
2210 if (!r) {
2211 r = copy_to_user((uint8_t __user *)args->skeydata_addr, keys,
2212 sizeof(uint8_t) * args->count);
2213 if (r)
2214 r = -EFAULT;
2215 }
2216
2217 kvfree(keys);
2218 return r <= 0 ? r : -EFAULT;
2219 }
2220
kvm_s390_set_skeys(struct kvm * kvm,struct kvm_s390_skeys * args)2221 static int kvm_s390_set_skeys(struct kvm *kvm, struct kvm_s390_skeys *args)
2222 {
2223 struct kvm_s390_mmu_cache *mc;
2224 union skey *keys;
2225 int i, r = 0;
2226
2227 if (args->flags != 0)
2228 return -EINVAL;
2229
2230 /* Enforce sane limit on memory allocation */
2231 if (args->count < 1 || args->count > KVM_S390_SKEYS_MAX)
2232 return -EINVAL;
2233
2234 keys = kvmalloc_array(args->count, sizeof(*keys), GFP_KERNEL_ACCOUNT);
2235 if (!keys)
2236 return -ENOMEM;
2237
2238 r = copy_from_user(keys, (uint8_t __user *)args->skeydata_addr,
2239 sizeof(uint8_t) * args->count);
2240 if (r) {
2241 r = -EFAULT;
2242 goto out;
2243 }
2244
2245 /* Enable storage key handling for the guest */
2246 r = gmap_enable_skeys(kvm->arch.gmap);
2247 if (r)
2248 goto out;
2249
2250 r = -EINVAL;
2251 for (i = 0; i < args->count; i++) {
2252 /* Lowest order bit is reserved */
2253 if (keys[i].zero)
2254 goto out;
2255 }
2256
2257 mc = kvm_s390_new_mmu_cache();
2258 if (!mc) {
2259 r = -ENOMEM;
2260 goto out;
2261 }
2262
2263 r = 0;
2264 do {
2265 r = kvm_s390_mmu_cache_topup(mc);
2266 if (r == -ENOMEM)
2267 break;
2268 scoped_guard(read_lock, &kvm->mmu_lock) {
2269 for (i = 0 ; i < args->count; i++) {
2270 r = dat_set_storage_key(mc, kvm->arch.gmap->asce,
2271 args->start_gfn + i, keys[i], 0);
2272 if (r)
2273 break;
2274 }
2275 }
2276 } while (r == -ENOMEM);
2277 kvm_s390_free_mmu_cache(mc);
2278 out:
2279 kvfree(keys);
2280 return r <= 0 ? r : -EFAULT;
2281 }
2282
2283 /*
2284 * This function searches for the next page with dirty CMMA attributes, and
2285 * saves the attributes in the buffer up to either the end of the buffer or
2286 * until a block of at least KVM_S390_MAX_BIT_DISTANCE clean bits is found;
2287 * no trailing clean bytes are saved.
2288 * In case no dirty bits were found, or if CMMA was not enabled or used, the
2289 * output buffer will indicate 0 as length.
2290 */
kvm_s390_get_cmma_bits(struct kvm * kvm,struct kvm_s390_cmma_log * args)2291 static int kvm_s390_get_cmma_bits(struct kvm *kvm,
2292 struct kvm_s390_cmma_log *args)
2293 {
2294 int peek, ret;
2295 u8 *values;
2296
2297 if (!kvm->arch.use_cmma)
2298 return -ENXIO;
2299 /* Invalid/unsupported flags were specified */
2300 if (args->flags & ~KVM_S390_CMMA_PEEK)
2301 return -EINVAL;
2302 /* Migration mode query, and we are not doing a migration */
2303 peek = !!(args->flags & KVM_S390_CMMA_PEEK);
2304 if (!peek && !kvm->arch.migration_mode)
2305 return -EINVAL;
2306 /* CMMA is disabled or was not used, or the buffer has length zero */
2307 args->count = min(args->count, KVM_S390_CMMA_SIZE_MAX);
2308 if (!args->count || !uses_cmm(kvm->arch.gmap)) {
2309 memset(args, 0, sizeof(*args));
2310 return 0;
2311 }
2312 /* We are not peeking, and there are no dirty pages */
2313 if (!peek && !atomic64_read(&kvm->arch.cmma_dirty_pages)) {
2314 memset(args, 0, sizeof(*args));
2315 return 0;
2316 }
2317
2318 values = vzalloc(args->count);
2319 if (!values)
2320 return -ENOMEM;
2321
2322 scoped_guard(read_lock, &kvm->mmu_lock) {
2323 if (peek)
2324 ret = dat_peek_cmma(args->start_gfn, kvm->arch.gmap->asce, &args->count,
2325 values);
2326 else
2327 ret = dat_get_cmma(kvm->arch.gmap->asce, &args->start_gfn, &args->count,
2328 values, &kvm->arch.cmma_dirty_pages);
2329 }
2330
2331 if (kvm->arch.migration_mode)
2332 args->remaining = atomic64_read(&kvm->arch.cmma_dirty_pages);
2333 else
2334 args->remaining = 0;
2335
2336 if (copy_to_user((void __user *)args->values, values, args->count))
2337 ret = -EFAULT;
2338
2339 vfree(values);
2340 return ret;
2341 }
2342
2343 /*
2344 * This function sets the CMMA attributes for the given pages. If the input
2345 * buffer has zero length, no action is taken, otherwise the attributes are
2346 * set and the mm->context.uses_cmm flag is set.
2347 */
kvm_s390_set_cmma_bits(struct kvm * kvm,const struct kvm_s390_cmma_log * args)2348 static int kvm_s390_set_cmma_bits(struct kvm *kvm,
2349 const struct kvm_s390_cmma_log *args)
2350 {
2351 struct kvm_s390_mmu_cache *mc __free(kvm_s390_mmu_cache) = NULL;
2352 u8 *bits __free(kvfree) = NULL;
2353 int r = 0;
2354
2355 if (!kvm->arch.use_cmma)
2356 return -ENXIO;
2357 /* invalid/unsupported flags */
2358 if (args->flags != 0)
2359 return -EINVAL;
2360 /* Enforce sane limit on memory allocation */
2361 if (args->count > KVM_S390_CMMA_SIZE_MAX)
2362 return -EINVAL;
2363 /* Nothing to do */
2364 if (args->count == 0)
2365 return 0;
2366
2367 mc = kvm_s390_new_mmu_cache();
2368 if (!mc)
2369 return -ENOMEM;
2370 bits = vmalloc(array_size(sizeof(*bits), args->count));
2371 if (!bits)
2372 return -ENOMEM;
2373
2374 r = copy_from_user(bits, (void __user *)args->values, args->count);
2375 if (r)
2376 return -EFAULT;
2377
2378 do {
2379 r = kvm_s390_mmu_cache_topup(mc);
2380 if (r)
2381 return r;
2382 scoped_guard(read_lock, &kvm->mmu_lock) {
2383 r = dat_set_cmma_bits(mc, kvm->arch.gmap->asce, args->start_gfn,
2384 args->count, args->mask, bits);
2385 }
2386 } while (r == -ENOMEM);
2387
2388 set_bit(GMAP_FLAG_USES_CMM, &kvm->arch.gmap->flags);
2389
2390 return r <= 0 ? r : -EFAULT;
2391 }
2392
2393 /**
2394 * kvm_s390_cpus_from_pv - Convert all protected vCPUs in a protected VM to
2395 * non protected.
2396 * @kvm: the VM whose protected vCPUs are to be converted
2397 * @rc: return value for the RC field of the UVC (in case of error)
2398 * @rrc: return value for the RRC field of the UVC (in case of error)
2399 *
2400 * Does not stop in case of error, tries to convert as many
2401 * CPUs as possible. In case of error, the RC and RRC of the last error are
2402 * returned.
2403 *
2404 * Return: 0 in case of success, otherwise -EIO
2405 */
kvm_s390_cpus_from_pv(struct kvm * kvm,u16 * rc,u16 * rrc)2406 int kvm_s390_cpus_from_pv(struct kvm *kvm, u16 *rc, u16 *rrc)
2407 {
2408 struct kvm_vcpu *vcpu;
2409 unsigned long i;
2410 u16 _rc, _rrc;
2411 int ret = 0;
2412
2413 /*
2414 * We ignore failures and try to destroy as many CPUs as possible.
2415 * At the same time we must not free the assigned resources when
2416 * this fails, as the ultravisor has still access to that memory.
2417 * So kvm_s390_pv_destroy_cpu can leave a "wanted" memory leak
2418 * behind.
2419 * We want to return the first failure rc and rrc, though.
2420 */
2421 kvm_for_each_vcpu(i, vcpu, kvm) {
2422 mutex_lock(&vcpu->mutex);
2423 if (kvm_s390_pv_destroy_cpu(vcpu, &_rc, &_rrc) && !ret) {
2424 *rc = _rc;
2425 *rrc = _rrc;
2426 ret = -EIO;
2427 }
2428 mutex_unlock(&vcpu->mutex);
2429 }
2430 /* Ensure that we re-enable gisa if the non-PV guest used it but the PV guest did not. */
2431 if (use_gisa)
2432 kvm_s390_gisa_enable(kvm);
2433 return ret;
2434 }
2435
2436 /**
2437 * kvm_s390_cpus_to_pv - Convert all non-protected vCPUs in a protected VM
2438 * to protected.
2439 * @kvm: the VM whose protected vCPUs are to be converted
2440 * @rc: return value for the RC field of the UVC (in case of error)
2441 * @rrc: return value for the RRC field of the UVC (in case of error)
2442 *
2443 * Tries to undo the conversion in case of error.
2444 *
2445 * Return: 0 in case of success, otherwise -EIO
2446 */
kvm_s390_cpus_to_pv(struct kvm * kvm,u16 * rc,u16 * rrc)2447 static int kvm_s390_cpus_to_pv(struct kvm *kvm, u16 *rc, u16 *rrc)
2448 {
2449 unsigned long i;
2450 int r = 0;
2451 u16 dummy;
2452
2453 struct kvm_vcpu *vcpu;
2454
2455 /* Disable the GISA if the ultravisor does not support AIV. */
2456 if (!uv_has_feature(BIT_UV_FEAT_AIV))
2457 kvm_s390_gisa_disable(kvm);
2458
2459 kvm_for_each_vcpu(i, vcpu, kvm) {
2460 mutex_lock(&vcpu->mutex);
2461 r = kvm_s390_pv_create_cpu(vcpu, rc, rrc);
2462 mutex_unlock(&vcpu->mutex);
2463 if (r)
2464 break;
2465 }
2466 if (r)
2467 kvm_s390_cpus_from_pv(kvm, &dummy, &dummy);
2468 return r;
2469 }
2470
2471 /*
2472 * Here we provide user space with a direct interface to query UV
2473 * related data like UV maxima and available features as well as
2474 * feature specific data.
2475 *
2476 * To facilitate future extension of the data structures we'll try to
2477 * write data up to the maximum requested length.
2478 */
kvm_s390_handle_pv_info(struct kvm_s390_pv_info * info)2479 static ssize_t kvm_s390_handle_pv_info(struct kvm_s390_pv_info *info)
2480 {
2481 ssize_t len_min;
2482
2483 switch (info->header.id) {
2484 case KVM_PV_INFO_VM: {
2485 len_min = sizeof(info->header) + sizeof(info->vm);
2486
2487 if (info->header.len_max < len_min)
2488 return -EINVAL;
2489
2490 memcpy(info->vm.inst_calls_list,
2491 uv_info.inst_calls_list,
2492 sizeof(uv_info.inst_calls_list));
2493
2494 /* It's max cpuid not max cpus, so it's off by one */
2495 info->vm.max_cpus = uv_info.max_guest_cpu_id + 1;
2496 info->vm.max_guests = uv_info.max_num_sec_conf;
2497 info->vm.max_guest_addr = uv_info.max_sec_stor_addr;
2498 info->vm.feature_indication = uv_info.uv_feature_indications;
2499
2500 return len_min;
2501 }
2502 case KVM_PV_INFO_DUMP: {
2503 len_min = sizeof(info->header) + sizeof(info->dump);
2504
2505 if (info->header.len_max < len_min)
2506 return -EINVAL;
2507
2508 info->dump.dump_cpu_buffer_len = uv_info.guest_cpu_stor_len;
2509 info->dump.dump_config_mem_buffer_per_1m = uv_info.conf_dump_storage_state_len;
2510 info->dump.dump_config_finalize_len = uv_info.conf_dump_finalize_len;
2511 return len_min;
2512 }
2513 default:
2514 return -EINVAL;
2515 }
2516 }
2517
kvm_s390_pv_dmp(struct kvm * kvm,struct kvm_pv_cmd * cmd,struct kvm_s390_pv_dmp dmp)2518 static int kvm_s390_pv_dmp(struct kvm *kvm, struct kvm_pv_cmd *cmd,
2519 struct kvm_s390_pv_dmp dmp)
2520 {
2521 int r = -EINVAL;
2522 void __user *result_buff = (void __user *)dmp.buff_addr;
2523
2524 switch (dmp.subcmd) {
2525 case KVM_PV_DUMP_INIT: {
2526 if (kvm->arch.pv.dumping)
2527 break;
2528
2529 /*
2530 * Block SIE entry as concurrent dump UVCs could lead
2531 * to validities.
2532 */
2533 kvm_s390_vcpu_block_all(kvm);
2534
2535 r = uv_cmd_nodata(kvm_s390_pv_get_handle(kvm),
2536 UVC_CMD_DUMP_INIT, &cmd->rc, &cmd->rrc);
2537 KVM_UV_EVENT(kvm, 3, "PROTVIRT DUMP INIT: rc %x rrc %x",
2538 cmd->rc, cmd->rrc);
2539 if (!r) {
2540 kvm->arch.pv.dumping = true;
2541 } else {
2542 kvm_s390_vcpu_unblock_all(kvm);
2543 r = -EINVAL;
2544 }
2545 break;
2546 }
2547 case KVM_PV_DUMP_CONFIG_STOR_STATE: {
2548 if (!kvm->arch.pv.dumping)
2549 break;
2550
2551 /*
2552 * gaddr is an output parameter since we might stop
2553 * early. As dmp will be copied back in our caller, we
2554 * don't need to do it ourselves.
2555 */
2556 r = kvm_s390_pv_dump_stor_state(kvm, result_buff, &dmp.gaddr, dmp.buff_len,
2557 &cmd->rc, &cmd->rrc);
2558 break;
2559 }
2560 case KVM_PV_DUMP_COMPLETE: {
2561 if (!kvm->arch.pv.dumping)
2562 break;
2563
2564 r = -EINVAL;
2565 if (dmp.buff_len < uv_info.conf_dump_finalize_len)
2566 break;
2567
2568 r = kvm_s390_pv_dump_complete(kvm, result_buff,
2569 &cmd->rc, &cmd->rrc);
2570 break;
2571 }
2572 default:
2573 r = -ENOTTY;
2574 break;
2575 }
2576
2577 return r;
2578 }
2579
kvm_s390_handle_pv(struct kvm * kvm,struct kvm_pv_cmd * cmd)2580 static int kvm_s390_handle_pv(struct kvm *kvm, struct kvm_pv_cmd *cmd)
2581 {
2582 const bool need_lock = (cmd->cmd != KVM_PV_ASYNC_CLEANUP_PERFORM);
2583 void __user *argp = (void __user *)cmd->data;
2584 int r = 0;
2585 u16 dummy;
2586
2587 if (need_lock)
2588 mutex_lock(&kvm->lock);
2589
2590 switch (cmd->cmd) {
2591 case KVM_PV_ENABLE: {
2592 r = -EINVAL;
2593 if (kvm_s390_pv_is_protected(kvm))
2594 break;
2595
2596 kvm_s390_unmap_all_adapters(kvm);
2597 mmap_write_lock(kvm->mm);
2598 /*
2599 * Disable creation of new THPs. Existing THPs can stay, they
2600 * will be split when any part of them gets imported.
2601 */
2602 mm_flags_clear(MMF_DISABLE_THP_EXCEPT_ADVISED, kvm->mm);
2603 mm_flags_set(MMF_DISABLE_THP_COMPLETELY, kvm->mm);
2604 set_bit(GMAP_FLAG_EXPORT_ON_UNMAP, &kvm->arch.gmap->flags);
2605 r = gmap_helper_disable_cow_sharing();
2606 mmap_write_unlock(kvm->mm);
2607 if (r)
2608 break;
2609
2610 r = kvm_s390_pv_init_vm(kvm, &cmd->rc, &cmd->rrc);
2611 if (r)
2612 break;
2613
2614 r = kvm_s390_cpus_to_pv(kvm, &cmd->rc, &cmd->rrc);
2615 if (r)
2616 kvm_s390_pv_deinit_vm(kvm, &dummy, &dummy);
2617
2618 /* we need to block service interrupts from now on */
2619 set_bit(IRQ_PEND_EXT_SERVICE, &kvm->arch.float_int.masked_irqs);
2620 break;
2621 }
2622 case KVM_PV_ASYNC_CLEANUP_PREPARE:
2623 r = -EINVAL;
2624 if (!kvm_s390_pv_is_protected(kvm) || !async_destroy)
2625 break;
2626
2627 r = kvm_s390_cpus_from_pv(kvm, &cmd->rc, &cmd->rrc);
2628 /*
2629 * If a CPU could not be destroyed, destroy VM will also fail.
2630 * There is no point in trying to destroy it. Instead return
2631 * the rc and rrc from the first CPU that failed destroying.
2632 */
2633 if (r)
2634 break;
2635 r = kvm_s390_pv_set_aside(kvm, &cmd->rc, &cmd->rrc);
2636
2637 /* no need to block service interrupts any more */
2638 clear_bit(IRQ_PEND_EXT_SERVICE, &kvm->arch.float_int.masked_irqs);
2639 break;
2640 case KVM_PV_ASYNC_CLEANUP_PERFORM:
2641 r = -EINVAL;
2642 if (!async_destroy)
2643 break;
2644 /* kvm->lock must not be held; this is asserted inside the function. */
2645 r = kvm_s390_pv_deinit_aside_vm(kvm, &cmd->rc, &cmd->rrc);
2646 break;
2647 case KVM_PV_DISABLE: {
2648 r = -EINVAL;
2649 if (!kvm_s390_pv_is_protected(kvm))
2650 break;
2651
2652 r = kvm_s390_cpus_from_pv(kvm, &cmd->rc, &cmd->rrc);
2653 /*
2654 * If a CPU could not be destroyed, destroy VM will also fail.
2655 * There is no point in trying to destroy it. Instead return
2656 * the rc and rrc from the first CPU that failed destroying.
2657 */
2658 if (r)
2659 break;
2660 r = kvm_s390_pv_deinit_cleanup_all(kvm, &cmd->rc, &cmd->rrc);
2661
2662 /* no need to block service interrupts any more */
2663 clear_bit(IRQ_PEND_EXT_SERVICE, &kvm->arch.float_int.masked_irqs);
2664 break;
2665 }
2666 case KVM_PV_SET_SEC_PARMS: {
2667 struct kvm_s390_pv_sec_parm parms = {};
2668 void *hdr;
2669
2670 r = -EINVAL;
2671 if (!kvm_s390_pv_is_protected(kvm))
2672 break;
2673
2674 r = -EFAULT;
2675 if (copy_from_user(&parms, argp, sizeof(parms)))
2676 break;
2677
2678 /* Currently restricted to 1MiB */
2679 r = -EINVAL;
2680 if (parms.length > SZ_1M)
2681 break;
2682
2683 r = -ENOMEM;
2684 hdr = vmalloc(parms.length);
2685 if (!hdr)
2686 break;
2687
2688 r = -EFAULT;
2689 if (!copy_from_user(hdr, (void __user *)parms.origin,
2690 parms.length))
2691 r = kvm_s390_pv_set_sec_parms(kvm, hdr, parms.length,
2692 &cmd->rc, &cmd->rrc);
2693
2694 vfree(hdr);
2695 break;
2696 }
2697 case KVM_PV_UNPACK: {
2698 struct kvm_s390_pv_unp unp = {};
2699
2700 r = -EINVAL;
2701 if (!kvm_s390_pv_is_protected(kvm) || !mm_is_protected(kvm->mm))
2702 break;
2703
2704 r = -EFAULT;
2705 if (copy_from_user(&unp, argp, sizeof(unp)))
2706 break;
2707
2708 r = kvm_s390_pv_unpack(kvm, unp.addr, unp.size, unp.tweak,
2709 &cmd->rc, &cmd->rrc);
2710 break;
2711 }
2712 case KVM_PV_VERIFY: {
2713 r = -EINVAL;
2714 if (!kvm_s390_pv_is_protected(kvm))
2715 break;
2716
2717 r = uv_cmd_nodata(kvm_s390_pv_get_handle(kvm),
2718 UVC_CMD_VERIFY_IMG, &cmd->rc, &cmd->rrc);
2719 KVM_UV_EVENT(kvm, 3, "PROTVIRT VERIFY: rc %x rrc %x", cmd->rc,
2720 cmd->rrc);
2721 break;
2722 }
2723 case KVM_PV_PREP_RESET: {
2724 r = -EINVAL;
2725 if (!kvm_s390_pv_is_protected(kvm))
2726 break;
2727
2728 r = uv_cmd_nodata(kvm_s390_pv_get_handle(kvm),
2729 UVC_CMD_PREPARE_RESET, &cmd->rc, &cmd->rrc);
2730 KVM_UV_EVENT(kvm, 3, "PROTVIRT PREP RESET: rc %x rrc %x",
2731 cmd->rc, cmd->rrc);
2732 break;
2733 }
2734 case KVM_PV_UNSHARE_ALL: {
2735 r = -EINVAL;
2736 if (!kvm_s390_pv_is_protected(kvm))
2737 break;
2738
2739 r = uv_cmd_nodata(kvm_s390_pv_get_handle(kvm),
2740 UVC_CMD_SET_UNSHARE_ALL, &cmd->rc, &cmd->rrc);
2741 KVM_UV_EVENT(kvm, 3, "PROTVIRT UNSHARE: rc %x rrc %x",
2742 cmd->rc, cmd->rrc);
2743 break;
2744 }
2745 case KVM_PV_INFO: {
2746 struct kvm_s390_pv_info info = {};
2747 ssize_t data_len;
2748
2749 /*
2750 * No need to check the VM protection here.
2751 *
2752 * Maybe user space wants to query some of the data
2753 * when the VM is still unprotected. If we see the
2754 * need to fence a new data command we can still
2755 * return an error in the info handler.
2756 */
2757
2758 r = -EFAULT;
2759 if (copy_from_user(&info, argp, sizeof(info.header)))
2760 break;
2761
2762 r = -EINVAL;
2763 if (info.header.len_max < sizeof(info.header))
2764 break;
2765
2766 data_len = kvm_s390_handle_pv_info(&info);
2767 if (data_len < 0) {
2768 r = data_len;
2769 break;
2770 }
2771 /*
2772 * If a data command struct is extended (multiple
2773 * times) this can be used to determine how much of it
2774 * is valid.
2775 */
2776 info.header.len_written = data_len;
2777
2778 r = -EFAULT;
2779 if (copy_to_user(argp, &info, data_len))
2780 break;
2781
2782 r = 0;
2783 break;
2784 }
2785 case KVM_PV_DUMP: {
2786 struct kvm_s390_pv_dmp dmp;
2787
2788 r = -EINVAL;
2789 if (!kvm_s390_pv_is_protected(kvm))
2790 break;
2791
2792 r = -EFAULT;
2793 if (copy_from_user(&dmp, argp, sizeof(dmp)))
2794 break;
2795
2796 r = kvm_s390_pv_dmp(kvm, cmd, dmp);
2797 if (r)
2798 break;
2799
2800 if (copy_to_user(argp, &dmp, sizeof(dmp))) {
2801 r = -EFAULT;
2802 break;
2803 }
2804
2805 break;
2806 }
2807 default:
2808 r = -ENOTTY;
2809 }
2810 if (need_lock)
2811 mutex_unlock(&kvm->lock);
2812
2813 return r;
2814 }
2815
mem_op_validate_common(struct kvm_s390_mem_op * mop,u64 supported_flags)2816 static int mem_op_validate_common(struct kvm_s390_mem_op *mop, u64 supported_flags)
2817 {
2818 if (mop->flags & ~supported_flags || !mop->size)
2819 return -EINVAL;
2820 if (mop->size > MEM_OP_MAX_SIZE)
2821 return -E2BIG;
2822 if (mop->flags & KVM_S390_MEMOP_F_SKEY_PROTECTION) {
2823 if (mop->key > 0xf)
2824 return -EINVAL;
2825 } else {
2826 mop->key = 0;
2827 }
2828 return 0;
2829 }
2830
kvm_s390_vm_mem_op_abs(struct kvm * kvm,struct kvm_s390_mem_op * mop)2831 static int kvm_s390_vm_mem_op_abs(struct kvm *kvm, struct kvm_s390_mem_op *mop)
2832 {
2833 void __user *uaddr = (void __user *)mop->buf;
2834 void *tmpbuf __free(kvfree) = NULL;
2835 enum gacc_mode acc_mode;
2836 int r;
2837
2838 r = mem_op_validate_common(mop, KVM_S390_MEMOP_F_SKEY_PROTECTION |
2839 KVM_S390_MEMOP_F_CHECK_ONLY);
2840 if (r)
2841 return r;
2842
2843 if (!(mop->flags & KVM_S390_MEMOP_F_CHECK_ONLY)) {
2844 tmpbuf = vmalloc(mop->size);
2845 if (!tmpbuf)
2846 return -ENOMEM;
2847 }
2848
2849 acc_mode = mop->op == KVM_S390_MEMOP_ABSOLUTE_READ ? GACC_FETCH : GACC_STORE;
2850
2851 scoped_guard(srcu, &kvm->srcu) {
2852 if (mop->flags & KVM_S390_MEMOP_F_CHECK_ONLY)
2853 return check_gpa_range(kvm, mop->gaddr, mop->size, acc_mode, mop->key);
2854
2855 if (acc_mode == GACC_STORE && copy_from_user(tmpbuf, uaddr, mop->size))
2856 return -EFAULT;
2857 r = access_guest_abs_with_key(kvm, mop->gaddr, tmpbuf,
2858 mop->size, acc_mode, mop->key);
2859 if (r)
2860 return r;
2861 if (acc_mode != GACC_STORE && copy_to_user(uaddr, tmpbuf, mop->size))
2862 return -EFAULT;
2863 }
2864 return 0;
2865 }
2866
kvm_s390_vm_mem_op_cmpxchg(struct kvm * kvm,struct kvm_s390_mem_op * mop)2867 static int kvm_s390_vm_mem_op_cmpxchg(struct kvm *kvm, struct kvm_s390_mem_op *mop)
2868 {
2869 void __user *uaddr = (void __user *)mop->buf;
2870 void __user *old_addr = (void __user *)mop->old_addr;
2871 union kvm_s390_quad old = { .sixteen = 0 };
2872 union kvm_s390_quad new = { .sixteen = 0 };
2873 bool success = false;
2874 int r;
2875
2876 r = mem_op_validate_common(mop, KVM_S390_MEMOP_F_SKEY_PROTECTION);
2877 if (r)
2878 return r;
2879 /*
2880 * This validates off_in_quad. Checking that size is a power
2881 * of two is not necessary, as cmpxchg_guest_abs_with_key
2882 * takes care of that
2883 */
2884 if (mop->size > sizeof(new))
2885 return -EINVAL;
2886 if (copy_from_user(&new, uaddr, mop->size))
2887 return -EFAULT;
2888 if (copy_from_user(&old, old_addr, mop->size))
2889 return -EFAULT;
2890
2891 scoped_guard(srcu, &kvm->srcu) {
2892 r = cmpxchg_guest_abs_with_key(kvm, mop->gaddr, mop->size, &old, new,
2893 mop->key, &success);
2894
2895 if (!success && copy_to_user(old_addr, &old, mop->size))
2896 return -EFAULT;
2897 }
2898 return r;
2899 }
2900
kvm_s390_vm_mem_op(struct kvm * kvm,struct kvm_s390_mem_op * mop)2901 static int kvm_s390_vm_mem_op(struct kvm *kvm, struct kvm_s390_mem_op *mop)
2902 {
2903 /*
2904 * This is technically a heuristic only, if the kvm->lock is not
2905 * taken, it is not guaranteed that the vm is/remains non-protected.
2906 * This is ok from a kernel perspective, wrongdoing is detected
2907 * on the access, -EFAULT is returned and the vm may crash the
2908 * next time it accesses the memory in question.
2909 * There is no sane usecase to do switching and a memop on two
2910 * different CPUs at the same time.
2911 */
2912 if (kvm_s390_pv_get_handle(kvm))
2913 return -EINVAL;
2914
2915 switch (mop->op) {
2916 case KVM_S390_MEMOP_ABSOLUTE_READ:
2917 case KVM_S390_MEMOP_ABSOLUTE_WRITE:
2918 return kvm_s390_vm_mem_op_abs(kvm, mop);
2919 case KVM_S390_MEMOP_ABSOLUTE_CMPXCHG:
2920 return kvm_s390_vm_mem_op_cmpxchg(kvm, mop);
2921 default:
2922 return -EINVAL;
2923 }
2924 }
2925
kvm_arch_vm_ioctl(struct file * filp,unsigned int ioctl,unsigned long arg)2926 int kvm_arch_vm_ioctl(struct file *filp, unsigned int ioctl, unsigned long arg)
2927 {
2928 struct kvm *kvm = filp->private_data;
2929 void __user *argp = (void __user *)arg;
2930 struct kvm_device_attr attr;
2931 int r;
2932 struct kvm_s390_interrupt_info *inti;
2933
2934 switch (ioctl) {
2935 case KVM_S390_INTERRUPT: {
2936 struct kvm_s390_interrupt s390int;
2937
2938 r = -EINVAL;
2939 if (kvm_is_ucontrol(kvm))
2940 break;
2941 r = -EFAULT;
2942 if (copy_from_user(&s390int, argp, sizeof(s390int)))
2943 break;
2944 inti = kzalloc_obj(*inti, GFP_KERNEL_ACCOUNT);
2945 if (!inti)
2946 return -ENOMEM;
2947 r = kvm_s390_inject_vm(kvm, &s390int, inti);
2948 if (r)
2949 kfree(inti);
2950 break;
2951 }
2952 case KVM_CREATE_IRQCHIP: {
2953 r = -EINVAL;
2954 if (kvm->arch.use_irqchip)
2955 r = 0;
2956 break;
2957 }
2958 case KVM_SET_DEVICE_ATTR: {
2959 r = -EFAULT;
2960 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
2961 break;
2962 r = kvm_s390_vm_set_attr(kvm, &attr);
2963 break;
2964 }
2965 case KVM_GET_DEVICE_ATTR: {
2966 r = -EFAULT;
2967 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
2968 break;
2969 r = kvm_s390_vm_get_attr(kvm, &attr);
2970 break;
2971 }
2972 case KVM_HAS_DEVICE_ATTR: {
2973 r = -EFAULT;
2974 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
2975 break;
2976 r = kvm_s390_vm_has_attr(kvm, &attr);
2977 break;
2978 }
2979 case KVM_S390_GET_SKEYS: {
2980 struct kvm_s390_skeys args;
2981
2982 r = -EFAULT;
2983 if (copy_from_user(&args, argp,
2984 sizeof(struct kvm_s390_skeys)))
2985 break;
2986 r = kvm_s390_get_skeys(kvm, &args);
2987 break;
2988 }
2989 case KVM_S390_SET_SKEYS: {
2990 struct kvm_s390_skeys args;
2991
2992 r = -EFAULT;
2993 if (copy_from_user(&args, argp,
2994 sizeof(struct kvm_s390_skeys)))
2995 break;
2996 r = kvm_s390_set_skeys(kvm, &args);
2997 break;
2998 }
2999 case KVM_S390_GET_CMMA_BITS: {
3000 struct kvm_s390_cmma_log args;
3001
3002 r = -EFAULT;
3003 if (copy_from_user(&args, argp, sizeof(args)))
3004 break;
3005 scoped_guard(mutex, &kvm->slots_arch_lock)
3006 r = kvm_s390_get_cmma_bits(kvm, &args);
3007 if (!r) {
3008 r = copy_to_user(argp, &args, sizeof(args));
3009 if (r)
3010 r = -EFAULT;
3011 }
3012 break;
3013 }
3014 case KVM_S390_SET_CMMA_BITS: {
3015 struct kvm_s390_cmma_log args;
3016
3017 r = -EFAULT;
3018 if (copy_from_user(&args, argp, sizeof(args)))
3019 break;
3020 mutex_lock(&kvm->slots_arch_lock);
3021 r = kvm_s390_set_cmma_bits(kvm, &args);
3022 mutex_unlock(&kvm->slots_arch_lock);
3023 break;
3024 }
3025 case KVM_S390_PV_COMMAND: {
3026 struct kvm_pv_cmd args;
3027
3028 /* protvirt means user cpu state */
3029 kvm_s390_set_user_cpu_state_ctrl(kvm);
3030 r = 0;
3031 if (!is_prot_virt_host()) {
3032 r = -EINVAL;
3033 break;
3034 }
3035 if (copy_from_user(&args, argp, sizeof(args))) {
3036 r = -EFAULT;
3037 break;
3038 }
3039 if (args.flags) {
3040 r = -EINVAL;
3041 break;
3042 }
3043 /* must be called without kvm->lock */
3044 r = kvm_s390_handle_pv(kvm, &args);
3045 if (copy_to_user(argp, &args, sizeof(args))) {
3046 r = -EFAULT;
3047 break;
3048 }
3049 break;
3050 }
3051 case KVM_S390_MEM_OP: {
3052 struct kvm_s390_mem_op mem_op;
3053
3054 if (copy_from_user(&mem_op, argp, sizeof(mem_op)) == 0)
3055 r = kvm_s390_vm_mem_op(kvm, &mem_op);
3056 else
3057 r = -EFAULT;
3058 break;
3059 }
3060 case KVM_S390_KEYOP: {
3061 struct kvm_s390_mmu_cache *mc;
3062 struct kvm_s390_keyop kop;
3063 union skey skey;
3064
3065 if (copy_from_user(&kop, argp, sizeof(kop))) {
3066 r = -EFAULT;
3067 break;
3068 }
3069 skey.skey = kop.key;
3070
3071 mc = kvm_s390_new_mmu_cache();
3072 if (!mc)
3073 return -ENOMEM;
3074
3075 r = kvm_s390_keyop(mc, kvm, kop.operation, kop.guest_addr, skey);
3076 kvm_s390_free_mmu_cache(mc);
3077 if (r < 0)
3078 break;
3079
3080 kop.key = r;
3081 r = 0;
3082 if (copy_to_user(argp, &kop, sizeof(kop)))
3083 r = -EFAULT;
3084 break;
3085 }
3086 case KVM_S390_ZPCI_OP: {
3087 struct kvm_s390_zpci_op args;
3088
3089 r = -EINVAL;
3090 if (!IS_ENABLED(CONFIG_VFIO_PCI_ZDEV_KVM))
3091 break;
3092 if (copy_from_user(&args, argp, sizeof(args))) {
3093 r = -EFAULT;
3094 break;
3095 }
3096 r = kvm_s390_pci_zpci_op(kvm, &args);
3097 break;
3098 }
3099 default:
3100 r = -ENOTTY;
3101 }
3102
3103 return r;
3104 }
3105
kvm_s390_apxa_installed(void)3106 static int kvm_s390_apxa_installed(void)
3107 {
3108 struct ap_config_info info;
3109
3110 if (ap_instructions_available()) {
3111 if (ap_qci(&info) == 0)
3112 return info.apxa;
3113 }
3114
3115 return 0;
3116 }
3117
3118 /*
3119 * The format of the crypto control block (CRYCB) is specified in the 3 low
3120 * order bits of the CRYCB designation (CRYCBD) field as follows:
3121 * Format 0: Neither the message security assist extension 3 (MSAX3) nor the
3122 * AP extended addressing (APXA) facility are installed.
3123 * Format 1: The APXA facility is not installed but the MSAX3 facility is.
3124 * Format 2: Both the APXA and MSAX3 facilities are installed
3125 */
kvm_s390_set_crycb_format(struct kvm * kvm)3126 static void kvm_s390_set_crycb_format(struct kvm *kvm)
3127 {
3128 kvm->arch.crypto.crycbd = virt_to_phys(kvm->arch.crypto.crycb);
3129
3130 /* Clear the CRYCB format bits - i.e., set format 0 by default */
3131 kvm->arch.crypto.crycbd &= ~(CRYCB_FORMAT_MASK);
3132
3133 /* Check whether MSAX3 is installed */
3134 if (!test_kvm_facility(kvm, 76))
3135 return;
3136
3137 if (kvm_s390_apxa_installed())
3138 kvm->arch.crypto.crycbd |= CRYCB_FORMAT2;
3139 else
3140 kvm->arch.crypto.crycbd |= CRYCB_FORMAT1;
3141 }
3142
3143 /*
3144 * kvm_arch_crypto_set_masks
3145 *
3146 * @kvm: pointer to the target guest's KVM struct containing the crypto masks
3147 * to be set.
3148 * @apm: the mask identifying the accessible AP adapters
3149 * @aqm: the mask identifying the accessible AP domains
3150 * @adm: the mask identifying the accessible AP control domains
3151 *
3152 * Set the masks that identify the adapters, domains and control domains to
3153 * which the KVM guest is granted access.
3154 *
3155 * Note: The kvm->lock mutex must be locked by the caller before invoking this
3156 * function.
3157 */
kvm_arch_crypto_set_masks(struct kvm * kvm,unsigned long * apm,unsigned long * aqm,unsigned long * adm)3158 void kvm_arch_crypto_set_masks(struct kvm *kvm, unsigned long *apm,
3159 unsigned long *aqm, unsigned long *adm)
3160 {
3161 struct kvm_s390_crypto_cb *crycb = kvm->arch.crypto.crycb;
3162
3163 kvm_s390_vcpu_block_all(kvm);
3164
3165 switch (kvm->arch.crypto.crycbd & CRYCB_FORMAT_MASK) {
3166 case CRYCB_FORMAT2: /* APCB1 use 256 bits */
3167 memcpy(crycb->apcb1.apm, apm, 32);
3168 VM_EVENT(kvm, 3, "SET CRYCB: apm %016lx %016lx %016lx %016lx",
3169 apm[0], apm[1], apm[2], apm[3]);
3170 memcpy(crycb->apcb1.aqm, aqm, 32);
3171 VM_EVENT(kvm, 3, "SET CRYCB: aqm %016lx %016lx %016lx %016lx",
3172 aqm[0], aqm[1], aqm[2], aqm[3]);
3173 memcpy(crycb->apcb1.adm, adm, 32);
3174 VM_EVENT(kvm, 3, "SET CRYCB: adm %016lx %016lx %016lx %016lx",
3175 adm[0], adm[1], adm[2], adm[3]);
3176 break;
3177 case CRYCB_FORMAT1:
3178 case CRYCB_FORMAT0: /* Fall through both use APCB0 */
3179 memcpy(crycb->apcb0.apm, apm, 8);
3180 memcpy(crycb->apcb0.aqm, aqm, 2);
3181 memcpy(crycb->apcb0.adm, adm, 2);
3182 VM_EVENT(kvm, 3, "SET CRYCB: apm %016lx aqm %04x adm %04x",
3183 apm[0], *((unsigned short *)aqm),
3184 *((unsigned short *)adm));
3185 break;
3186 default: /* Can not happen */
3187 break;
3188 }
3189
3190 /* recreate the shadow crycb for each vcpu */
3191 kvm_s390_sync_request_broadcast(kvm, KVM_REQ_VSIE_RESTART);
3192 kvm_s390_vcpu_unblock_all(kvm);
3193 }
3194 EXPORT_SYMBOL_GPL(kvm_arch_crypto_set_masks);
3195
3196 /*
3197 * kvm_arch_crypto_clear_masks
3198 *
3199 * @kvm: pointer to the target guest's KVM struct containing the crypto masks
3200 * to be cleared.
3201 *
3202 * Clear the masks that identify the adapters, domains and control domains to
3203 * which the KVM guest is granted access.
3204 *
3205 * Note: The kvm->lock mutex must be locked by the caller before invoking this
3206 * function.
3207 */
kvm_arch_crypto_clear_masks(struct kvm * kvm)3208 void kvm_arch_crypto_clear_masks(struct kvm *kvm)
3209 {
3210 kvm_s390_vcpu_block_all(kvm);
3211
3212 memset(&kvm->arch.crypto.crycb->apcb0, 0,
3213 sizeof(kvm->arch.crypto.crycb->apcb0));
3214 memset(&kvm->arch.crypto.crycb->apcb1, 0,
3215 sizeof(kvm->arch.crypto.crycb->apcb1));
3216
3217 VM_EVENT(kvm, 3, "%s", "CLR CRYCB:");
3218 /* recreate the shadow crycb for each vcpu */
3219 kvm_s390_sync_request_broadcast(kvm, KVM_REQ_VSIE_RESTART);
3220 kvm_s390_vcpu_unblock_all(kvm);
3221 }
3222 EXPORT_SYMBOL_GPL(kvm_arch_crypto_clear_masks);
3223
kvm_s390_get_initial_cpuid(void)3224 static u64 kvm_s390_get_initial_cpuid(void)
3225 {
3226 struct cpuid cpuid;
3227
3228 get_cpu_id(&cpuid);
3229 cpuid.version = 0xff;
3230 return *((u64 *) &cpuid);
3231 }
3232
kvm_s390_crypto_init(struct kvm * kvm)3233 static void kvm_s390_crypto_init(struct kvm *kvm)
3234 {
3235 kvm->arch.crypto.crycb = &kvm->arch.sie_page2->crycb;
3236 kvm_s390_set_crycb_format(kvm);
3237 init_rwsem(&kvm->arch.crypto.pqap_hook_rwsem);
3238
3239 if (!test_kvm_facility(kvm, 76))
3240 return;
3241
3242 /* Enable AES/DEA protected key functions by default */
3243 kvm->arch.crypto.aes_kw = 1;
3244 kvm->arch.crypto.dea_kw = 1;
3245 get_random_bytes(kvm->arch.crypto.crycb->aes_wrapping_key_mask,
3246 sizeof(kvm->arch.crypto.crycb->aes_wrapping_key_mask));
3247 get_random_bytes(kvm->arch.crypto.crycb->dea_wrapping_key_mask,
3248 sizeof(kvm->arch.crypto.crycb->dea_wrapping_key_mask));
3249 }
3250
sca_dispose(struct kvm * kvm)3251 static void sca_dispose(struct kvm *kvm)
3252 {
3253 if (kvm->arch.sca)
3254 free_pages_exact(kvm->arch.sca, sizeof(*kvm->arch.sca));
3255 kvm->arch.sca = NULL;
3256 }
3257
kvm_arch_free_vm(struct kvm * kvm)3258 void kvm_arch_free_vm(struct kvm *kvm)
3259 {
3260 if (IS_ENABLED(CONFIG_VFIO_PCI_ZDEV_KVM))
3261 kvm_s390_pci_clear_list(kvm);
3262
3263 __kvm_arch_free_vm(kvm);
3264 }
3265
kvm_arch_init_vm(struct kvm * kvm,unsigned long type)3266 int kvm_arch_init_vm(struct kvm *kvm, unsigned long type)
3267 {
3268 gfp_t alloc_flags = GFP_KERNEL_ACCOUNT | __GFP_ZERO;
3269 char debug_name[16];
3270 int i, rc;
3271
3272 mutex_init(&kvm->arch.pv.import_lock);
3273
3274 rc = -EINVAL;
3275 #ifdef CONFIG_KVM_S390_UCONTROL
3276 if (type & ~KVM_VM_S390_UCONTROL)
3277 goto out_err;
3278 if ((type & KVM_VM_S390_UCONTROL) && (!capable(CAP_SYS_ADMIN)))
3279 goto out_err;
3280 #else
3281 if (type)
3282 goto out_err;
3283 #endif
3284 rc = -ENOMEM;
3285
3286 if (!sclp.has_64bscao)
3287 alloc_flags |= GFP_DMA;
3288 mutex_lock(&kvm_lock);
3289
3290 kvm->arch.sca = alloc_pages_exact(sizeof(*kvm->arch.sca), alloc_flags);
3291 mutex_unlock(&kvm_lock);
3292 if (!kvm->arch.sca)
3293 goto out_err;
3294
3295 snprintf(debug_name, sizeof(debug_name), "kvm-%u", current->pid);
3296
3297 kvm->arch.dbf = debug_register(debug_name, 32, 1, 7 * sizeof(long));
3298 if (!kvm->arch.dbf)
3299 goto out_err;
3300
3301 BUILD_BUG_ON(sizeof(struct sie_page2) != 4096);
3302 kvm->arch.sie_page2 =
3303 (struct sie_page2 *) get_zeroed_page(GFP_KERNEL_ACCOUNT | GFP_DMA);
3304 if (!kvm->arch.sie_page2)
3305 goto out_err;
3306
3307 kvm->arch.sie_page2->kvm = kvm;
3308 kvm->arch.model.fac_list = kvm->arch.sie_page2->fac_list;
3309
3310 for (i = 0; i < ARRAY_SIZE(kvm_s390_fac_base); i++) {
3311 kvm->arch.model.fac_mask[i] = stfle_fac_list[i] &
3312 kvm_s390_fac_base[i];
3313 kvm->arch.model.fac_list[i] = stfle_fac_list[i] &
3314 kvm_s390_fac_base[i];
3315 }
3316 for (i = 0; i < ARRAY_SIZE(kvm_s390_fac_ext); i++) {
3317 kvm->arch.model.fac_mask[i] |= stfle_fac_list[i] &
3318 kvm_s390_fac_ext[i];
3319 }
3320 kvm->arch.model.subfuncs = kvm_s390_available_subfunc;
3321
3322 /* we are always in czam mode - even on pre z14 machines */
3323 set_kvm_facility(kvm->arch.model.fac_mask, 138);
3324 set_kvm_facility(kvm->arch.model.fac_list, 138);
3325 /* we emulate STHYI in kvm */
3326 set_kvm_facility(kvm->arch.model.fac_mask, 74);
3327 set_kvm_facility(kvm->arch.model.fac_list, 74);
3328 if (machine_has_tlb_guest()) {
3329 set_kvm_facility(kvm->arch.model.fac_mask, 147);
3330 set_kvm_facility(kvm->arch.model.fac_list, 147);
3331 }
3332
3333 if (css_general_characteristics.aiv && test_facility(65))
3334 set_kvm_facility(kvm->arch.model.fac_mask, 65);
3335
3336 kvm->arch.model.cpuid = kvm_s390_get_initial_cpuid();
3337 kvm->arch.model.ibc = sclp.ibc & 0x0fff;
3338
3339 kvm->arch.model.uv_feat_guest.feat = 0;
3340
3341 kvm_s390_crypto_init(kvm);
3342
3343 if (IS_ENABLED(CONFIG_VFIO_PCI_ZDEV_KVM)) {
3344 mutex_lock(&kvm->lock);
3345 kvm_s390_pci_init_list(kvm);
3346 kvm_s390_vcpu_pci_enable_interp(kvm);
3347 mutex_unlock(&kvm->lock);
3348 }
3349
3350 spin_lock_init(&kvm->arch.float_int.ais_lock);
3351 spin_lock_init(&kvm->arch.float_int.lock);
3352 for (i = 0; i < FIRQ_LIST_COUNT; i++)
3353 INIT_LIST_HEAD(&kvm->arch.float_int.lists[i]);
3354 init_waitqueue_head(&kvm->arch.ipte_wq);
3355 mutex_init(&kvm->arch.ipte_mutex);
3356
3357 debug_register_view(kvm->arch.dbf, &debug_sprintf_view);
3358 VM_EVENT(kvm, 3, "vm created with type %lu", type);
3359
3360 kvm->arch.mem_limit = type & KVM_VM_S390_UCONTROL ? KVM_S390_NO_MEM_LIMIT : sclp.hamax + 1;
3361 kvm->arch.gmap = gmap_new(kvm, gpa_to_gfn(kvm->arch.mem_limit));
3362 if (!kvm->arch.gmap)
3363 goto out_err;
3364 clear_bit(GMAP_FLAG_PFAULT_ENABLED, &kvm->arch.gmap->flags);
3365
3366 if (type & KVM_VM_S390_UCONTROL) {
3367 struct kvm_userspace_memory_region2 fake_memslot = {
3368 .slot = KVM_S390_UCONTROL_MEMSLOT,
3369 .guest_phys_addr = 0,
3370 .userspace_addr = 0,
3371 .memory_size = ALIGN_DOWN(TASK_SIZE, _SEGMENT_SIZE),
3372 .flags = 0,
3373 };
3374
3375 /* one flat fake memslot covering the whole address-space */
3376 mutex_lock(&kvm->slots_lock);
3377 KVM_BUG_ON(kvm_set_internal_memslot(kvm, &fake_memslot), kvm);
3378 mutex_unlock(&kvm->slots_lock);
3379 set_bit(GMAP_FLAG_IS_UCONTROL, &kvm->arch.gmap->flags);
3380 } else {
3381 struct crst_table *table = dereference_asce(kvm->arch.gmap->asce);
3382
3383 crst_table_init((void *)table, _CRSTE_HOLE(table->crstes[0].h.tt).val);
3384 }
3385
3386 kvm->arch.use_pfmfi = sclp.has_pfmfi;
3387 kvm->arch.use_skf = sclp.has_skey;
3388 spin_lock_init(&kvm->arch.start_stop_lock);
3389 kvm_s390_vsie_init(kvm);
3390 if (use_gisa)
3391 kvm_s390_gisa_init(kvm);
3392 INIT_LIST_HEAD(&kvm->arch.pv.need_cleanup);
3393 kvm->arch.pv.set_aside = NULL;
3394 KVM_EVENT(3, "vm 0x%p created by pid %u", kvm, current->pid);
3395
3396 return 0;
3397 out_err:
3398 free_page((unsigned long)kvm->arch.sie_page2);
3399 debug_unregister(kvm->arch.dbf);
3400 sca_dispose(kvm);
3401 KVM_EVENT(3, "creation of vm failed: %d", rc);
3402 return rc;
3403 }
3404
kvm_arch_vcpu_destroy(struct kvm_vcpu * vcpu)3405 void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu)
3406 {
3407 u16 rc, rrc;
3408
3409 VCPU_EVENT(vcpu, 3, "%s", "free cpu");
3410 trace_kvm_s390_destroy_vcpu(vcpu->vcpu_id);
3411 kvm_s390_clear_local_irqs(vcpu);
3412 kvm_clear_async_pf_completion_queue(vcpu);
3413 if (!kvm_is_ucontrol(vcpu->kvm))
3414 sca_del_vcpu(vcpu);
3415 kvm_s390_update_topology_change_report(vcpu->kvm, 1);
3416
3417 if (kvm_is_ucontrol(vcpu->kvm)) {
3418 scoped_guard(spinlock, &vcpu->kvm->arch.gmap->children_lock)
3419 gmap_remove_child(vcpu->arch.gmap);
3420 vcpu->arch.gmap = gmap_put(vcpu->arch.gmap);
3421 }
3422
3423 if (vcpu->kvm->arch.use_cmma)
3424 kvm_s390_vcpu_unsetup_cmma(vcpu);
3425 /* We can not hold the vcpu mutex here, we are already dying */
3426 if (kvm_s390_pv_cpu_get_handle(vcpu))
3427 kvm_s390_pv_destroy_cpu(vcpu, &rc, &rrc);
3428 free_page((unsigned long)(vcpu->arch.sie_block));
3429 kvm_s390_free_mmu_cache(vcpu->arch.mc);
3430 }
3431
kvm_arch_destroy_vm(struct kvm * kvm)3432 void kvm_arch_destroy_vm(struct kvm *kvm)
3433 {
3434 u16 rc, rrc;
3435
3436 kvm_destroy_vcpus(kvm);
3437 sca_dispose(kvm);
3438 kvm_s390_gisa_destroy(kvm);
3439 /*
3440 * We are already at the end of life and kvm->lock is not taken.
3441 * This is ok as the file descriptor is closed by now and nobody
3442 * can mess with the pv state.
3443 */
3444 kvm_s390_pv_deinit_cleanup_all(kvm, &rc, &rrc);
3445 /*
3446 * Remove the mmu notifier only when the whole KVM VM is torn down,
3447 * and only if one was registered to begin with. If the VM is
3448 * currently not protected, but has been previously been protected,
3449 * then it's possible that the notifier is still registered.
3450 */
3451 if (kvm->arch.pv.mmu_notifier.ops)
3452 mmu_notifier_unregister(&kvm->arch.pv.mmu_notifier, kvm->mm);
3453
3454 debug_unregister(kvm->arch.dbf);
3455 free_page((unsigned long)kvm->arch.sie_page2);
3456 kvm_s390_destroy_adapters(kvm);
3457 kvm_s390_clear_float_irqs(kvm);
3458 kvm_s390_vsie_destroy(kvm);
3459 kvm->arch.gmap = gmap_put(kvm->arch.gmap);
3460 KVM_EVENT(3, "vm 0x%p destroyed", kvm);
3461 }
3462
3463 /* Section: vcpu related */
sca_del_vcpu(struct kvm_vcpu * vcpu)3464 static void sca_del_vcpu(struct kvm_vcpu *vcpu)
3465 {
3466 struct esca_block *sca = vcpu->kvm->arch.sca;
3467
3468 if (!kvm_s390_use_sca_entries() || !vcpu->arch.initialized)
3469 return;
3470
3471 clear_bit_inv(vcpu->vcpu_id, (unsigned long *)sca->mcn);
3472 sca->cpu[vcpu->vcpu_id].sda = 0;
3473 }
3474
sca_add_vcpu(struct kvm_vcpu * vcpu)3475 static void sca_add_vcpu(struct kvm_vcpu *vcpu)
3476 {
3477 struct esca_block *sca = vcpu->kvm->arch.sca;
3478 phys_addr_t sca_phys = virt_to_phys(sca);
3479
3480 /* we still need the sca header for the ipte control */
3481 vcpu->arch.sie_block->scaoh = sca_phys >> 32;
3482 vcpu->arch.sie_block->scaol = sca_phys & ESCA_SCAOL_MASK;
3483 vcpu->arch.sie_block->ecb2 |= ECB2_ESCA;
3484
3485 if (!kvm_s390_use_sca_entries())
3486 return;
3487
3488 WRITE_ONCE(sca->cpu[vcpu->vcpu_id].sda, virt_to_phys(vcpu->arch.sie_block));
3489 set_bit_inv(vcpu->vcpu_id, (unsigned long *)sca->mcn);
3490 }
3491
sca_can_add_vcpu(struct kvm * kvm,unsigned int id)3492 static int sca_can_add_vcpu(struct kvm *kvm, unsigned int id)
3493 {
3494 if (!kvm_s390_use_sca_entries())
3495 return id < KVM_MAX_VCPUS;
3496
3497 return id < KVM_S390_ESCA_CPU_SLOTS;
3498 }
3499
3500 /* needs disabled preemption to protect from TOD sync and vcpu_load/put */
__start_cpu_timer_accounting(struct kvm_vcpu * vcpu)3501 static void __start_cpu_timer_accounting(struct kvm_vcpu *vcpu)
3502 {
3503 WARN_ON_ONCE(vcpu->arch.cputm_start != 0);
3504 raw_write_seqcount_begin(&vcpu->arch.cputm_seqcount);
3505 vcpu->arch.cputm_start = get_tod_clock_fast();
3506 raw_write_seqcount_end(&vcpu->arch.cputm_seqcount);
3507 }
3508
3509 /* needs disabled preemption to protect from TOD sync and vcpu_load/put */
__stop_cpu_timer_accounting(struct kvm_vcpu * vcpu)3510 static void __stop_cpu_timer_accounting(struct kvm_vcpu *vcpu)
3511 {
3512 WARN_ON_ONCE(vcpu->arch.cputm_start == 0);
3513 raw_write_seqcount_begin(&vcpu->arch.cputm_seqcount);
3514 vcpu->arch.sie_block->cputm -= get_tod_clock_fast() - vcpu->arch.cputm_start;
3515 vcpu->arch.cputm_start = 0;
3516 raw_write_seqcount_end(&vcpu->arch.cputm_seqcount);
3517 }
3518
3519 /* needs disabled preemption to protect from TOD sync and vcpu_load/put */
__enable_cpu_timer_accounting(struct kvm_vcpu * vcpu)3520 static void __enable_cpu_timer_accounting(struct kvm_vcpu *vcpu)
3521 {
3522 WARN_ON_ONCE(vcpu->arch.cputm_enabled);
3523 vcpu->arch.cputm_enabled = true;
3524 __start_cpu_timer_accounting(vcpu);
3525 }
3526
3527 /* needs disabled preemption to protect from TOD sync and vcpu_load/put */
__disable_cpu_timer_accounting(struct kvm_vcpu * vcpu)3528 static void __disable_cpu_timer_accounting(struct kvm_vcpu *vcpu)
3529 {
3530 WARN_ON_ONCE(!vcpu->arch.cputm_enabled);
3531 __stop_cpu_timer_accounting(vcpu);
3532 vcpu->arch.cputm_enabled = false;
3533 }
3534
enable_cpu_timer_accounting(struct kvm_vcpu * vcpu)3535 static void enable_cpu_timer_accounting(struct kvm_vcpu *vcpu)
3536 {
3537 preempt_disable(); /* protect from TOD sync and vcpu_load/put */
3538 __enable_cpu_timer_accounting(vcpu);
3539 preempt_enable();
3540 }
3541
disable_cpu_timer_accounting(struct kvm_vcpu * vcpu)3542 static void disable_cpu_timer_accounting(struct kvm_vcpu *vcpu)
3543 {
3544 preempt_disable(); /* protect from TOD sync and vcpu_load/put */
3545 __disable_cpu_timer_accounting(vcpu);
3546 preempt_enable();
3547 }
3548
3549 /* set the cpu timer - may only be called from the VCPU thread itself */
kvm_s390_set_cpu_timer(struct kvm_vcpu * vcpu,__u64 cputm)3550 void kvm_s390_set_cpu_timer(struct kvm_vcpu *vcpu, __u64 cputm)
3551 {
3552 preempt_disable(); /* protect from TOD sync and vcpu_load/put */
3553 raw_write_seqcount_begin(&vcpu->arch.cputm_seqcount);
3554 if (vcpu->arch.cputm_enabled)
3555 vcpu->arch.cputm_start = get_tod_clock_fast();
3556 vcpu->arch.sie_block->cputm = cputm;
3557 raw_write_seqcount_end(&vcpu->arch.cputm_seqcount);
3558 preempt_enable();
3559 }
3560
3561 /* update and get the cpu timer - can also be called from other VCPU threads */
kvm_s390_get_cpu_timer(struct kvm_vcpu * vcpu)3562 __u64 kvm_s390_get_cpu_timer(struct kvm_vcpu *vcpu)
3563 {
3564 unsigned int seq;
3565 __u64 value;
3566
3567 if (unlikely(!vcpu->arch.cputm_enabled))
3568 return vcpu->arch.sie_block->cputm;
3569
3570 preempt_disable(); /* protect from TOD sync and vcpu_load/put */
3571 do {
3572 seq = raw_read_seqcount(&vcpu->arch.cputm_seqcount);
3573 /*
3574 * If the writer would ever execute a read in the critical
3575 * section, e.g. in irq context, we have a deadlock.
3576 */
3577 WARN_ON_ONCE((seq & 1) && smp_processor_id() == vcpu->cpu);
3578 value = vcpu->arch.sie_block->cputm;
3579 /* if cputm_start is 0, accounting is being started/stopped */
3580 if (likely(vcpu->arch.cputm_start))
3581 value -= get_tod_clock_fast() - vcpu->arch.cputm_start;
3582 } while (read_seqcount_retry(&vcpu->arch.cputm_seqcount, seq & ~1));
3583 preempt_enable();
3584 return value;
3585 }
3586
kvm_arch_vcpu_load(struct kvm_vcpu * vcpu,int cpu)3587 void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
3588 {
3589
3590 kvm_s390_set_cpuflags(vcpu, CPUSTAT_RUNNING);
3591 if (vcpu->arch.cputm_enabled && !is_vcpu_idle(vcpu))
3592 __start_cpu_timer_accounting(vcpu);
3593 vcpu->cpu = cpu;
3594 }
3595
kvm_arch_vcpu_put(struct kvm_vcpu * vcpu)3596 void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu)
3597 {
3598 vcpu->cpu = -1;
3599 if (vcpu->arch.cputm_enabled && !is_vcpu_idle(vcpu))
3600 __stop_cpu_timer_accounting(vcpu);
3601 kvm_s390_clear_cpuflags(vcpu, CPUSTAT_RUNNING);
3602
3603 }
3604
kvm_arch_vcpu_postcreate(struct kvm_vcpu * vcpu)3605 void kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu)
3606 {
3607 mutex_lock(&vcpu->kvm->lock);
3608 preempt_disable();
3609 vcpu->arch.sie_block->epoch = vcpu->kvm->arch.epoch;
3610 vcpu->arch.sie_block->epdx = vcpu->kvm->arch.epdx;
3611 preempt_enable();
3612 mutex_unlock(&vcpu->kvm->lock);
3613 if (!kvm_is_ucontrol(vcpu->kvm)) {
3614 vcpu->arch.gmap = vcpu->kvm->arch.gmap;
3615 sca_add_vcpu(vcpu);
3616 }
3617 if (test_kvm_facility(vcpu->kvm, 74) || vcpu->kvm->arch.user_instr0 ||
3618 vcpu->kvm->arch.user_operexec)
3619 vcpu->arch.sie_block->ictl |= ICTL_OPEREXC;
3620
3621 /* Pairs with smp_load_acquire() in kvm_arch_vcpu_ioctl_run() and kvm_arch_vcpu_ioctl() */
3622 smp_store_release(&vcpu->arch.initialized, true);
3623 }
3624
kvm_has_pckmo_subfunc(struct kvm * kvm,unsigned long nr)3625 static bool kvm_has_pckmo_subfunc(struct kvm *kvm, unsigned long nr)
3626 {
3627 if (test_bit_inv(nr, (unsigned long *)&kvm->arch.model.subfuncs.pckmo) &&
3628 test_bit_inv(nr, (unsigned long *)&kvm_s390_available_subfunc.pckmo))
3629 return true;
3630 return false;
3631 }
3632
kvm_has_pckmo_ecc(struct kvm * kvm)3633 static bool kvm_has_pckmo_ecc(struct kvm *kvm)
3634 {
3635 /* At least one ECC subfunction must be present */
3636 return kvm_has_pckmo_subfunc(kvm, 32) ||
3637 kvm_has_pckmo_subfunc(kvm, 33) ||
3638 kvm_has_pckmo_subfunc(kvm, 34) ||
3639 kvm_has_pckmo_subfunc(kvm, 40) ||
3640 kvm_has_pckmo_subfunc(kvm, 41);
3641
3642 }
3643
kvm_has_pckmo_hmac(struct kvm * kvm)3644 static bool kvm_has_pckmo_hmac(struct kvm *kvm)
3645 {
3646 /* At least one HMAC subfunction must be present */
3647 return kvm_has_pckmo_subfunc(kvm, 118) ||
3648 kvm_has_pckmo_subfunc(kvm, 122);
3649 }
3650
kvm_s390_vcpu_crypto_setup(struct kvm_vcpu * vcpu)3651 static void kvm_s390_vcpu_crypto_setup(struct kvm_vcpu *vcpu)
3652 {
3653 /*
3654 * If the AP instructions are not being interpreted and the MSAX3
3655 * facility is not configured for the guest, there is nothing to set up.
3656 */
3657 if (!vcpu->kvm->arch.crypto.apie && !test_kvm_facility(vcpu->kvm, 76))
3658 return;
3659
3660 vcpu->arch.sie_block->crycbd = vcpu->kvm->arch.crypto.crycbd;
3661 vcpu->arch.sie_block->ecb3 &= ~(ECB3_AES | ECB3_DEA);
3662 vcpu->arch.sie_block->eca &= ~ECA_APIE;
3663 vcpu->arch.sie_block->ecd &= ~(ECD_ECC | ECD_HMAC);
3664
3665 if (vcpu->kvm->arch.crypto.apie)
3666 vcpu->arch.sie_block->eca |= ECA_APIE;
3667
3668 /* Set up protected key support */
3669 if (vcpu->kvm->arch.crypto.aes_kw) {
3670 vcpu->arch.sie_block->ecb3 |= ECB3_AES;
3671 /* ecc/hmac is also wrapped with AES key */
3672 if (kvm_has_pckmo_ecc(vcpu->kvm))
3673 vcpu->arch.sie_block->ecd |= ECD_ECC;
3674 if (kvm_has_pckmo_hmac(vcpu->kvm))
3675 vcpu->arch.sie_block->ecd |= ECD_HMAC;
3676 }
3677
3678 if (vcpu->kvm->arch.crypto.dea_kw)
3679 vcpu->arch.sie_block->ecb3 |= ECB3_DEA;
3680 }
3681
kvm_s390_vcpu_unsetup_cmma(struct kvm_vcpu * vcpu)3682 void kvm_s390_vcpu_unsetup_cmma(struct kvm_vcpu *vcpu)
3683 {
3684 if (vcpu->arch.sie_block->cbrlo)
3685 free_page((unsigned long)phys_to_virt(vcpu->arch.sie_block->cbrlo));
3686 vcpu->arch.sie_block->cbrlo = 0;
3687 }
3688
kvm_s390_vcpu_setup_cmma(struct kvm_vcpu * vcpu)3689 int kvm_s390_vcpu_setup_cmma(struct kvm_vcpu *vcpu)
3690 {
3691 void *cbrlo_page = (void *)get_zeroed_page(GFP_KERNEL_ACCOUNT);
3692
3693 if (!cbrlo_page)
3694 return -ENOMEM;
3695
3696 vcpu->arch.sie_block->cbrlo = virt_to_phys(cbrlo_page);
3697 return 0;
3698 }
3699
kvm_s390_vcpu_setup_model(struct kvm_vcpu * vcpu)3700 static void kvm_s390_vcpu_setup_model(struct kvm_vcpu *vcpu)
3701 {
3702 struct kvm_s390_cpu_model *model = &vcpu->kvm->arch.model;
3703
3704 vcpu->arch.sie_block->ibc = model->ibc;
3705 if (test_kvm_facility(vcpu->kvm, 7))
3706 vcpu->arch.sie_block->fac = virt_to_phys(model->fac_list);
3707 }
3708
kvm_s390_vcpu_setup(struct kvm_vcpu * vcpu)3709 static int kvm_s390_vcpu_setup(struct kvm_vcpu *vcpu)
3710 {
3711 int rc = 0;
3712 u16 uvrc, uvrrc;
3713
3714 atomic_set(&vcpu->arch.sie_block->cpuflags, CPUSTAT_ZARCH |
3715 CPUSTAT_SM |
3716 CPUSTAT_STOPPED);
3717
3718 if (test_kvm_facility(vcpu->kvm, 78))
3719 kvm_s390_set_cpuflags(vcpu, CPUSTAT_GED2);
3720 else if (test_kvm_facility(vcpu->kvm, 8))
3721 kvm_s390_set_cpuflags(vcpu, CPUSTAT_GED);
3722
3723 kvm_s390_vcpu_setup_model(vcpu);
3724
3725 /* pgste_set_pte has special handling for !machine_has_esop() */
3726 if (machine_has_esop())
3727 vcpu->arch.sie_block->ecb |= ECB_HOSTPROTINT;
3728 if (test_kvm_facility(vcpu->kvm, 9))
3729 vcpu->arch.sie_block->ecb |= ECB_SRSI;
3730 if (test_kvm_facility(vcpu->kvm, 11))
3731 vcpu->arch.sie_block->ecb |= ECB_PTF;
3732 if (test_kvm_facility(vcpu->kvm, 73))
3733 vcpu->arch.sie_block->ecb |= ECB_TE;
3734 if (!kvm_is_ucontrol(vcpu->kvm))
3735 vcpu->arch.sie_block->ecb |= ECB_SPECI;
3736
3737 if (test_kvm_facility(vcpu->kvm, 8) && vcpu->kvm->arch.use_pfmfi)
3738 vcpu->arch.sie_block->ecb2 |= ECB2_PFMFI;
3739 if (test_kvm_facility(vcpu->kvm, 130))
3740 vcpu->arch.sie_block->ecb2 |= ECB2_IEP;
3741 vcpu->arch.sie_block->eca = ECA_MVPGI | ECA_PROTEXCI;
3742 if (sclp.has_cei)
3743 vcpu->arch.sie_block->eca |= ECA_CEI;
3744 if (sclp.has_ib)
3745 vcpu->arch.sie_block->eca |= ECA_IB;
3746 if (sclp.has_siif)
3747 vcpu->arch.sie_block->eca |= ECA_SII;
3748 if (kvm_s390_use_sca_entries())
3749 vcpu->arch.sie_block->eca |= ECA_SIGPI;
3750 if (test_kvm_facility(vcpu->kvm, 129)) {
3751 vcpu->arch.sie_block->eca |= ECA_VX;
3752 vcpu->arch.sie_block->ecd |= ECD_HOSTREGMGMT;
3753 }
3754 if (test_kvm_facility(vcpu->kvm, 139))
3755 vcpu->arch.sie_block->ecd |= ECD_MEF;
3756 if (test_kvm_facility(vcpu->kvm, 156))
3757 vcpu->arch.sie_block->ecd |= ECD_ETOKENF;
3758 if (vcpu->arch.sie_block->gd) {
3759 vcpu->arch.sie_block->eca |= ECA_AIV;
3760 VCPU_EVENT(vcpu, 3, "AIV gisa format-%u enabled for cpu %03u",
3761 vcpu->arch.sie_block->gd & 0x3, vcpu->vcpu_id);
3762 }
3763 vcpu->arch.sie_block->sdnxo = virt_to_phys(&vcpu->run->s.regs.sdnx) | SDNXC;
3764 vcpu->arch.sie_block->riccbd = virt_to_phys(&vcpu->run->s.regs.riccb);
3765
3766 if (sclp.has_kss)
3767 kvm_s390_set_cpuflags(vcpu, CPUSTAT_KSS);
3768 else
3769 vcpu->arch.sie_block->ictl |= ICTL_ISKE | ICTL_SSKE | ICTL_RRBE;
3770
3771 if (vcpu->kvm->arch.use_cmma) {
3772 rc = kvm_s390_vcpu_setup_cmma(vcpu);
3773 if (rc)
3774 return rc;
3775 }
3776 hrtimer_setup(&vcpu->arch.ckc_timer, kvm_s390_idle_wakeup, CLOCK_MONOTONIC,
3777 HRTIMER_MODE_REL);
3778
3779 vcpu->arch.sie_block->hpid = HPID_KVM;
3780
3781 kvm_s390_vcpu_crypto_setup(vcpu);
3782
3783 kvm_s390_vcpu_pci_setup(vcpu);
3784
3785 mutex_lock(&vcpu->kvm->lock);
3786 if (kvm_s390_pv_is_protected(vcpu->kvm)) {
3787 rc = kvm_s390_pv_create_cpu(vcpu, &uvrc, &uvrrc);
3788 if (rc)
3789 kvm_s390_vcpu_unsetup_cmma(vcpu);
3790 }
3791 mutex_unlock(&vcpu->kvm->lock);
3792
3793 return rc;
3794 }
3795
kvm_arch_vcpu_precreate(struct kvm * kvm,unsigned int id)3796 int kvm_arch_vcpu_precreate(struct kvm *kvm, unsigned int id)
3797 {
3798 if (!kvm_is_ucontrol(kvm) && !sca_can_add_vcpu(kvm, id))
3799 return -EINVAL;
3800 return 0;
3801 }
3802
DEFINE_FREE(sie_page,struct sie_page *,if (_T)free_page ((unsigned long)(_T)))3803 DEFINE_FREE(sie_page, struct sie_page *, if (_T) free_page((unsigned long)(_T)))
3804
3805 int kvm_arch_vcpu_create(struct kvm_vcpu *vcpu)
3806 {
3807 struct kvm_s390_mmu_cache *mc __free(kvm_s390_mmu_cache) = NULL;
3808 struct sie_page *sie_page __free(sie_page) = NULL;
3809 int rc;
3810
3811 BUILD_BUG_ON(sizeof(struct sie_page) != 4096);
3812 mc = kvm_s390_new_mmu_cache();
3813 if (!mc)
3814 return -ENOMEM;
3815 sie_page = (struct sie_page *) get_zeroed_page(GFP_KERNEL_ACCOUNT);
3816 if (!sie_page)
3817 return -ENOMEM;
3818
3819 vcpu->arch.sie_block = &sie_page->sie_block;
3820 vcpu->arch.sie_block->itdba = virt_to_phys(&sie_page->itdb);
3821
3822 /* the real guest size will always be smaller than msl */
3823 vcpu->arch.sie_block->mso = 0;
3824 vcpu->arch.sie_block->msl = sclp.hamax;
3825
3826 vcpu->arch.sie_block->icpua = vcpu->vcpu_id;
3827 spin_lock_init(&vcpu->arch.local_int.lock);
3828 vcpu->arch.sie_block->gd = kvm_s390_get_gisa_desc(vcpu->kvm);
3829 seqcount_init(&vcpu->arch.cputm_seqcount);
3830
3831 vcpu->arch.pfault_token = KVM_S390_PFAULT_TOKEN_INVALID;
3832 kvm_clear_async_pf_completion_queue(vcpu);
3833 vcpu->run->kvm_valid_regs = KVM_SYNC_PREFIX |
3834 KVM_SYNC_GPRS |
3835 KVM_SYNC_ACRS |
3836 KVM_SYNC_CRS |
3837 KVM_SYNC_ARCH0 |
3838 KVM_SYNC_PFAULT |
3839 KVM_SYNC_DIAG318;
3840 vcpu->arch.acrs_loaded = false;
3841 kvm_s390_set_prefix(vcpu, 0);
3842 if (test_kvm_facility(vcpu->kvm, 64))
3843 vcpu->run->kvm_valid_regs |= KVM_SYNC_RICCB;
3844 if (test_kvm_facility(vcpu->kvm, 82))
3845 vcpu->run->kvm_valid_regs |= KVM_SYNC_BPBC;
3846 if (test_kvm_facility(vcpu->kvm, 133))
3847 vcpu->run->kvm_valid_regs |= KVM_SYNC_GSCB;
3848 if (test_kvm_facility(vcpu->kvm, 156))
3849 vcpu->run->kvm_valid_regs |= KVM_SYNC_ETOKEN;
3850 /* fprs can be synchronized via vrs, even if the guest has no vx. With
3851 * cpu_has_vx(), (load|store)_fpu_regs() will work with vrs format.
3852 */
3853 if (cpu_has_vx())
3854 vcpu->run->kvm_valid_regs |= KVM_SYNC_VRS;
3855 else
3856 vcpu->run->kvm_valid_regs |= KVM_SYNC_FPRS;
3857
3858 if (kvm_is_ucontrol(vcpu->kvm)) {
3859 vcpu->arch.gmap = gmap_new_child(vcpu->kvm->arch.gmap, -1UL);
3860 if (!vcpu->arch.gmap)
3861 return -ENOMEM;
3862 }
3863
3864 VM_EVENT(vcpu->kvm, 3, "create cpu %d at 0x%p, sie block at 0x%p",
3865 vcpu->vcpu_id, vcpu, vcpu->arch.sie_block);
3866 trace_kvm_s390_create_vcpu(vcpu->vcpu_id, vcpu, vcpu->arch.sie_block);
3867
3868 rc = kvm_s390_vcpu_setup(vcpu);
3869 if (rc) {
3870 if (kvm_is_ucontrol(vcpu->kvm)) {
3871 scoped_guard(spinlock, &vcpu->kvm->arch.gmap->children_lock)
3872 gmap_remove_child(vcpu->arch.gmap);
3873 vcpu->arch.gmap = gmap_put(vcpu->arch.gmap);
3874 }
3875 return rc;
3876 }
3877
3878 vcpu->arch.mc = no_free_ptr(mc);
3879 sie_page = NULL;
3880 kvm_s390_update_topology_change_report(vcpu->kvm, 1);
3881 return 0;
3882 }
3883
kvm_arch_vcpu_runnable(struct kvm_vcpu * vcpu)3884 int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu)
3885 {
3886 clear_bit(vcpu->vcpu_idx, vcpu->kvm->arch.gisa_int.kicked_mask);
3887 return kvm_s390_vcpu_has_irq(vcpu, 0);
3888 }
3889
kvm_arch_vcpu_in_kernel(struct kvm_vcpu * vcpu)3890 bool kvm_arch_vcpu_in_kernel(struct kvm_vcpu *vcpu)
3891 {
3892 return !(vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE);
3893 }
3894
kvm_s390_vcpu_block(struct kvm_vcpu * vcpu)3895 void kvm_s390_vcpu_block(struct kvm_vcpu *vcpu)
3896 {
3897 atomic_or(PROG_BLOCK_SIE, &vcpu->arch.sie_block->prog20);
3898 exit_sie(vcpu);
3899 }
3900
kvm_s390_vcpu_unblock(struct kvm_vcpu * vcpu)3901 void kvm_s390_vcpu_unblock(struct kvm_vcpu *vcpu)
3902 {
3903 atomic_andnot(PROG_BLOCK_SIE, &vcpu->arch.sie_block->prog20);
3904 }
3905
kvm_s390_vcpu_request(struct kvm_vcpu * vcpu)3906 static void kvm_s390_vcpu_request(struct kvm_vcpu *vcpu)
3907 {
3908 atomic_or(PROG_REQUEST, &vcpu->arch.sie_block->prog20);
3909 exit_sie(vcpu);
3910 }
3911
kvm_s390_vcpu_sie_inhibited(struct kvm_vcpu * vcpu)3912 bool kvm_s390_vcpu_sie_inhibited(struct kvm_vcpu *vcpu)
3913 {
3914 return atomic_read(&vcpu->arch.sie_block->prog20) &
3915 (PROG_BLOCK_SIE | PROG_REQUEST);
3916 }
3917
kvm_s390_vcpu_request_handled(struct kvm_vcpu * vcpu)3918 static void kvm_s390_vcpu_request_handled(struct kvm_vcpu *vcpu)
3919 {
3920 atomic_andnot(PROG_REQUEST, &vcpu->arch.sie_block->prog20);
3921 }
3922
3923 /*
3924 * Kick a guest cpu out of (v)SIE and wait until (v)SIE is not running.
3925 * If the CPU is not running (e.g. waiting as idle) the function will
3926 * return immediately. */
exit_sie(struct kvm_vcpu * vcpu)3927 void exit_sie(struct kvm_vcpu *vcpu)
3928 {
3929 kvm_s390_set_cpuflags(vcpu, CPUSTAT_STOP_INT);
3930 kvm_s390_vsie_kick(vcpu);
3931 while (vcpu->arch.sie_block->prog0c & PROG_IN_SIE)
3932 cpu_relax();
3933 }
3934
3935 /* Kick a guest cpu out of SIE to process a request synchronously */
kvm_s390_sync_request(int req,struct kvm_vcpu * vcpu)3936 void kvm_s390_sync_request(int req, struct kvm_vcpu *vcpu)
3937 {
3938 __kvm_make_request(req, vcpu);
3939 kvm_s390_vcpu_request(vcpu);
3940 }
3941
kvm_arch_no_poll(struct kvm_vcpu * vcpu)3942 bool kvm_arch_no_poll(struct kvm_vcpu *vcpu)
3943 {
3944 /* do not poll with more than halt_poll_max_steal percent of steal time */
3945 if (get_lowcore()->avg_steal_timer * 100 / (TICK_USEC << 12) >=
3946 READ_ONCE(halt_poll_max_steal)) {
3947 vcpu->stat.halt_no_poll_steal++;
3948 return true;
3949 }
3950 return false;
3951 }
3952
kvm_arch_vcpu_should_kick(struct kvm_vcpu * vcpu)3953 int kvm_arch_vcpu_should_kick(struct kvm_vcpu *vcpu)
3954 {
3955 /* kvm common code refers to this, but never calls it */
3956 BUG();
3957 return 0;
3958 }
3959
kvm_arch_vcpu_ioctl_get_one_reg(struct kvm_vcpu * vcpu,struct kvm_one_reg * reg)3960 static int kvm_arch_vcpu_ioctl_get_one_reg(struct kvm_vcpu *vcpu,
3961 struct kvm_one_reg *reg)
3962 {
3963 int r = -EINVAL;
3964
3965 switch (reg->id) {
3966 case KVM_REG_S390_TODPR:
3967 r = put_user(vcpu->arch.sie_block->todpr,
3968 (u32 __user *)reg->addr);
3969 break;
3970 case KVM_REG_S390_EPOCHDIFF:
3971 r = put_user(vcpu->arch.sie_block->epoch,
3972 (u64 __user *)reg->addr);
3973 break;
3974 case KVM_REG_S390_CPU_TIMER:
3975 r = put_user(kvm_s390_get_cpu_timer(vcpu),
3976 (u64 __user *)reg->addr);
3977 break;
3978 case KVM_REG_S390_CLOCK_COMP:
3979 r = put_user(vcpu->arch.sie_block->ckc,
3980 (u64 __user *)reg->addr);
3981 break;
3982 case KVM_REG_S390_PFTOKEN:
3983 r = put_user(vcpu->arch.pfault_token,
3984 (u64 __user *)reg->addr);
3985 break;
3986 case KVM_REG_S390_PFCOMPARE:
3987 r = put_user(vcpu->arch.pfault_compare,
3988 (u64 __user *)reg->addr);
3989 break;
3990 case KVM_REG_S390_PFSELECT:
3991 r = put_user(vcpu->arch.pfault_select,
3992 (u64 __user *)reg->addr);
3993 break;
3994 case KVM_REG_S390_PP:
3995 r = put_user(vcpu->arch.sie_block->pp,
3996 (u64 __user *)reg->addr);
3997 break;
3998 case KVM_REG_S390_GBEA:
3999 r = put_user(vcpu->arch.sie_block->gbea,
4000 (u64 __user *)reg->addr);
4001 break;
4002 default:
4003 break;
4004 }
4005
4006 return r;
4007 }
4008
kvm_arch_vcpu_ioctl_set_one_reg(struct kvm_vcpu * vcpu,struct kvm_one_reg * reg)4009 static int kvm_arch_vcpu_ioctl_set_one_reg(struct kvm_vcpu *vcpu,
4010 struct kvm_one_reg *reg)
4011 {
4012 int r = -EINVAL;
4013 __u64 val;
4014
4015 switch (reg->id) {
4016 case KVM_REG_S390_TODPR:
4017 r = get_user(vcpu->arch.sie_block->todpr,
4018 (u32 __user *)reg->addr);
4019 break;
4020 case KVM_REG_S390_EPOCHDIFF:
4021 r = get_user(vcpu->arch.sie_block->epoch,
4022 (u64 __user *)reg->addr);
4023 break;
4024 case KVM_REG_S390_CPU_TIMER:
4025 r = get_user(val, (u64 __user *)reg->addr);
4026 if (!r)
4027 kvm_s390_set_cpu_timer(vcpu, val);
4028 break;
4029 case KVM_REG_S390_CLOCK_COMP:
4030 r = get_user(vcpu->arch.sie_block->ckc,
4031 (u64 __user *)reg->addr);
4032 break;
4033 case KVM_REG_S390_PFTOKEN:
4034 r = get_user(vcpu->arch.pfault_token,
4035 (u64 __user *)reg->addr);
4036 if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID)
4037 kvm_clear_async_pf_completion_queue(vcpu);
4038 break;
4039 case KVM_REG_S390_PFCOMPARE:
4040 r = get_user(vcpu->arch.pfault_compare,
4041 (u64 __user *)reg->addr);
4042 break;
4043 case KVM_REG_S390_PFSELECT:
4044 r = get_user(vcpu->arch.pfault_select,
4045 (u64 __user *)reg->addr);
4046 break;
4047 case KVM_REG_S390_PP:
4048 r = get_user(vcpu->arch.sie_block->pp,
4049 (u64 __user *)reg->addr);
4050 break;
4051 case KVM_REG_S390_GBEA:
4052 r = get_user(vcpu->arch.sie_block->gbea,
4053 (u64 __user *)reg->addr);
4054 break;
4055 default:
4056 break;
4057 }
4058
4059 return r;
4060 }
4061
kvm_arch_vcpu_ioctl_normal_reset(struct kvm_vcpu * vcpu)4062 static void kvm_arch_vcpu_ioctl_normal_reset(struct kvm_vcpu *vcpu)
4063 {
4064 vcpu->arch.sie_block->gpsw.mask &= ~PSW_MASK_RI;
4065 vcpu->arch.pfault_token = KVM_S390_PFAULT_TOKEN_INVALID;
4066 memset(vcpu->run->s.regs.riccb, 0, sizeof(vcpu->run->s.regs.riccb));
4067
4068 kvm_clear_async_pf_completion_queue(vcpu);
4069 if (!kvm_s390_user_cpu_state_ctrl(vcpu->kvm))
4070 kvm_s390_vcpu_stop(vcpu);
4071 kvm_s390_clear_local_irqs(vcpu);
4072 }
4073
kvm_arch_vcpu_ioctl_initial_reset(struct kvm_vcpu * vcpu)4074 static void kvm_arch_vcpu_ioctl_initial_reset(struct kvm_vcpu *vcpu)
4075 {
4076 /* Initial reset is a superset of the normal reset */
4077 kvm_arch_vcpu_ioctl_normal_reset(vcpu);
4078
4079 /*
4080 * This equals initial cpu reset in pop, but we don't switch to ESA.
4081 * We do not only reset the internal data, but also ...
4082 */
4083 vcpu->arch.sie_block->gpsw.mask = 0;
4084 vcpu->arch.sie_block->gpsw.addr = 0;
4085 kvm_s390_set_prefix(vcpu, 0);
4086 kvm_s390_set_cpu_timer(vcpu, 0);
4087 vcpu->arch.sie_block->ckc = 0;
4088 memset(vcpu->arch.sie_block->gcr, 0, sizeof(vcpu->arch.sie_block->gcr));
4089 vcpu->arch.sie_block->gcr[0] = CR0_INITIAL_MASK;
4090 vcpu->arch.sie_block->gcr[14] = CR14_INITIAL_MASK;
4091
4092 /* ... the data in sync regs */
4093 memset(vcpu->run->s.regs.crs, 0, sizeof(vcpu->run->s.regs.crs));
4094 vcpu->run->s.regs.ckc = 0;
4095 vcpu->run->s.regs.crs[0] = CR0_INITIAL_MASK;
4096 vcpu->run->s.regs.crs[14] = CR14_INITIAL_MASK;
4097 vcpu->run->psw_addr = 0;
4098 vcpu->run->psw_mask = 0;
4099 vcpu->run->s.regs.todpr = 0;
4100 vcpu->run->s.regs.cputm = 0;
4101 vcpu->run->s.regs.ckc = 0;
4102 vcpu->run->s.regs.pp = 0;
4103 vcpu->run->s.regs.gbea = 1;
4104 vcpu->run->s.regs.fpc = 0;
4105 /*
4106 * Do not reset these registers in the protected case, as some of
4107 * them are overlaid and they are not accessible in this case
4108 * anyway.
4109 */
4110 if (!kvm_s390_pv_cpu_is_protected(vcpu)) {
4111 vcpu->arch.sie_block->gbea = 1;
4112 vcpu->arch.sie_block->pp = 0;
4113 vcpu->arch.sie_block->fpf &= ~FPF_BPBC;
4114 vcpu->arch.sie_block->todpr = 0;
4115 }
4116 }
4117
kvm_arch_vcpu_ioctl_clear_reset(struct kvm_vcpu * vcpu)4118 static void kvm_arch_vcpu_ioctl_clear_reset(struct kvm_vcpu *vcpu)
4119 {
4120 struct kvm_sync_regs *regs = &vcpu->run->s.regs;
4121
4122 /* Clear reset is a superset of the initial reset */
4123 kvm_arch_vcpu_ioctl_initial_reset(vcpu);
4124
4125 memset(®s->gprs, 0, sizeof(regs->gprs));
4126 memset(®s->vrs, 0, sizeof(regs->vrs));
4127 memset(®s->acrs, 0, sizeof(regs->acrs));
4128 memset(®s->gscb, 0, sizeof(regs->gscb));
4129
4130 regs->etoken = 0;
4131 regs->etoken_extension = 0;
4132 }
4133
kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu * vcpu,struct kvm_regs * regs)4134 int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
4135 {
4136 vcpu_load(vcpu);
4137 memcpy(&vcpu->run->s.regs.gprs, ®s->gprs, sizeof(regs->gprs));
4138 vcpu_put(vcpu);
4139 return 0;
4140 }
4141
kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu * vcpu,struct kvm_regs * regs)4142 int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
4143 {
4144 vcpu_load(vcpu);
4145 memcpy(®s->gprs, &vcpu->run->s.regs.gprs, sizeof(regs->gprs));
4146 vcpu_put(vcpu);
4147 return 0;
4148 }
4149
kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu * vcpu,struct kvm_sregs * sregs)4150 int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu,
4151 struct kvm_sregs *sregs)
4152 {
4153 vcpu_load(vcpu);
4154
4155 memcpy(&vcpu->run->s.regs.acrs, &sregs->acrs, sizeof(sregs->acrs));
4156 memcpy(&vcpu->arch.sie_block->gcr, &sregs->crs, sizeof(sregs->crs));
4157
4158 vcpu_put(vcpu);
4159 return 0;
4160 }
4161
kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu * vcpu,struct kvm_sregs * sregs)4162 int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu,
4163 struct kvm_sregs *sregs)
4164 {
4165 vcpu_load(vcpu);
4166
4167 memcpy(&sregs->acrs, &vcpu->run->s.regs.acrs, sizeof(sregs->acrs));
4168 memcpy(&sregs->crs, &vcpu->arch.sie_block->gcr, sizeof(sregs->crs));
4169
4170 vcpu_put(vcpu);
4171 return 0;
4172 }
4173
kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu * vcpu,struct kvm_fpu * fpu)4174 int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
4175 {
4176 vcpu_load(vcpu);
4177
4178 vcpu->run->s.regs.fpc = fpu->fpc;
4179 if (cpu_has_vx())
4180 convert_fp_to_vx((__vector128 *) vcpu->run->s.regs.vrs,
4181 (freg_t *) fpu->fprs);
4182 else
4183 memcpy(vcpu->run->s.regs.fprs, &fpu->fprs, sizeof(fpu->fprs));
4184
4185 vcpu_put(vcpu);
4186 return 0;
4187 }
4188
kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu * vcpu,struct kvm_fpu * fpu)4189 int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
4190 {
4191 vcpu_load(vcpu);
4192
4193 if (cpu_has_vx())
4194 convert_vx_to_fp((freg_t *) fpu->fprs,
4195 (__vector128 *) vcpu->run->s.regs.vrs);
4196 else
4197 memcpy(fpu->fprs, vcpu->run->s.regs.fprs, sizeof(fpu->fprs));
4198 fpu->fpc = vcpu->run->s.regs.fpc;
4199
4200 vcpu_put(vcpu);
4201 return 0;
4202 }
4203
kvm_arch_vcpu_ioctl_set_initial_psw(struct kvm_vcpu * vcpu,psw_t psw)4204 static int kvm_arch_vcpu_ioctl_set_initial_psw(struct kvm_vcpu *vcpu, psw_t psw)
4205 {
4206 int rc = 0;
4207
4208 if (!is_vcpu_stopped(vcpu))
4209 rc = -EBUSY;
4210 else {
4211 vcpu->run->psw_mask = psw.mask;
4212 vcpu->run->psw_addr = psw.addr;
4213 }
4214 return rc;
4215 }
4216
kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu * vcpu,struct kvm_translation * tr)4217 int kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu *vcpu,
4218 struct kvm_translation *tr)
4219 {
4220 return -EINVAL; /* not implemented yet */
4221 }
4222
4223 #define VALID_GUESTDBG_FLAGS (KVM_GUESTDBG_SINGLESTEP | \
4224 KVM_GUESTDBG_USE_HW_BP | \
4225 KVM_GUESTDBG_ENABLE)
4226
kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu * vcpu,struct kvm_guest_debug * dbg)4227 int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu,
4228 struct kvm_guest_debug *dbg)
4229 {
4230 int rc = 0;
4231
4232 vcpu_load(vcpu);
4233
4234 vcpu->guest_debug = 0;
4235 kvm_s390_clear_bp_data(vcpu);
4236
4237 if (dbg->control & ~VALID_GUESTDBG_FLAGS) {
4238 rc = -EINVAL;
4239 goto out;
4240 }
4241 if (!sclp.has_gpere) {
4242 rc = -EINVAL;
4243 goto out;
4244 }
4245
4246 if (dbg->control & KVM_GUESTDBG_ENABLE) {
4247 vcpu->guest_debug = dbg->control;
4248 /* enforce guest PER */
4249 kvm_s390_set_cpuflags(vcpu, CPUSTAT_P);
4250
4251 if (dbg->control & KVM_GUESTDBG_USE_HW_BP)
4252 rc = kvm_s390_import_bp_data(vcpu, dbg);
4253 } else {
4254 kvm_s390_clear_cpuflags(vcpu, CPUSTAT_P);
4255 vcpu->arch.guestdbg.last_bp = 0;
4256 }
4257
4258 if (rc) {
4259 vcpu->guest_debug = 0;
4260 kvm_s390_clear_bp_data(vcpu);
4261 kvm_s390_clear_cpuflags(vcpu, CPUSTAT_P);
4262 }
4263
4264 out:
4265 vcpu_put(vcpu);
4266 return rc;
4267 }
4268
kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu * vcpu,struct kvm_mp_state * mp_state)4269 int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu,
4270 struct kvm_mp_state *mp_state)
4271 {
4272 int ret;
4273
4274 vcpu_load(vcpu);
4275
4276 /* CHECK_STOP and LOAD are not supported yet */
4277 ret = is_vcpu_stopped(vcpu) ? KVM_MP_STATE_STOPPED :
4278 KVM_MP_STATE_OPERATING;
4279
4280 vcpu_put(vcpu);
4281 return ret;
4282 }
4283
kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu * vcpu,struct kvm_mp_state * mp_state)4284 int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu,
4285 struct kvm_mp_state *mp_state)
4286 {
4287 int rc = 0;
4288
4289 vcpu_load(vcpu);
4290
4291 /* user space knows about this interface - let it control the state */
4292 kvm_s390_set_user_cpu_state_ctrl(vcpu->kvm);
4293
4294 switch (mp_state->mp_state) {
4295 case KVM_MP_STATE_STOPPED:
4296 rc = kvm_s390_vcpu_stop(vcpu);
4297 break;
4298 case KVM_MP_STATE_OPERATING:
4299 rc = kvm_s390_vcpu_start(vcpu);
4300 break;
4301 case KVM_MP_STATE_LOAD:
4302 if (!kvm_s390_pv_cpu_is_protected(vcpu)) {
4303 rc = -ENXIO;
4304 break;
4305 }
4306 rc = kvm_s390_pv_set_cpu_state(vcpu, PV_CPU_STATE_OPR_LOAD);
4307 break;
4308 case KVM_MP_STATE_CHECK_STOP:
4309 fallthrough; /* CHECK_STOP and LOAD are not supported yet */
4310 default:
4311 rc = -ENXIO;
4312 }
4313
4314 vcpu_put(vcpu);
4315 return rc;
4316 }
4317
ibs_enabled(struct kvm_vcpu * vcpu)4318 static bool ibs_enabled(struct kvm_vcpu *vcpu)
4319 {
4320 return kvm_s390_test_cpuflags(vcpu, CPUSTAT_IBS);
4321 }
4322
vcpu_ucontrol_translate(struct kvm_vcpu * vcpu,gpa_t * gaddr)4323 static int vcpu_ucontrol_translate(struct kvm_vcpu *vcpu, gpa_t *gaddr)
4324 {
4325 int rc;
4326
4327 if (kvm_is_ucontrol(vcpu->kvm)) {
4328 rc = gmap_ucas_translate(vcpu->arch.mc, vcpu->arch.gmap, gaddr);
4329 if (rc == -EREMOTE) {
4330 vcpu->run->exit_reason = KVM_EXIT_S390_UCONTROL;
4331 vcpu->run->s390_ucontrol.trans_exc_code = *gaddr;
4332 vcpu->run->s390_ucontrol.pgm_code = PGM_SEGMENT_TRANSLATION;
4333 }
4334 return rc;
4335 }
4336 return 0;
4337 }
4338
kvm_s390_fixup_prefix(struct kvm_vcpu * vcpu)4339 static int kvm_s390_fixup_prefix(struct kvm_vcpu *vcpu)
4340 {
4341 gpa_t gaddr = kvm_s390_get_prefix(vcpu);
4342 gfn_t gfn;
4343 int rc;
4344
4345 if (vcpu_ucontrol_translate(vcpu, &gaddr))
4346 return -EREMOTE;
4347 gfn = gpa_to_gfn(gaddr);
4348
4349 rc = kvm_s390_faultin_gfn_simple(vcpu, NULL, gfn, true);
4350 if (rc)
4351 return rc;
4352 rc = kvm_s390_faultin_gfn_simple(vcpu, NULL, gfn + 1, true);
4353 if (rc)
4354 return rc;
4355
4356 scoped_guard(write_lock, &vcpu->kvm->mmu_lock)
4357 rc = dat_set_prefix_notif_bit(vcpu->kvm->arch.gmap->asce, gfn);
4358 return rc;
4359 }
4360
kvm_s390_handle_requests(struct kvm_vcpu * vcpu)4361 static int kvm_s390_handle_requests(struct kvm_vcpu *vcpu)
4362 {
4363 retry:
4364 kvm_s390_vcpu_request_handled(vcpu);
4365 if (!kvm_request_pending(vcpu))
4366 return 0;
4367 /*
4368 * If the guest prefix changed, re-arm the ipte notifier for the
4369 * guest prefix page. gmap_mprotect_notify will wait on the ptl lock.
4370 * This ensures that the ipte instruction for this request has
4371 * already finished. We might race against a second unmapper that
4372 * wants to set the blocking bit. Lets just retry the request loop.
4373 */
4374 if (kvm_check_request(KVM_REQ_REFRESH_GUEST_PREFIX, vcpu)) {
4375 int rc;
4376
4377 rc = kvm_s390_fixup_prefix(vcpu);
4378 if (rc) {
4379 kvm_make_request(KVM_REQ_REFRESH_GUEST_PREFIX, vcpu);
4380 return rc;
4381 }
4382 goto retry;
4383 }
4384
4385 if (kvm_check_request(KVM_REQ_TLB_FLUSH, vcpu)) {
4386 vcpu->arch.sie_block->ihcpu = 0xffff;
4387 goto retry;
4388 }
4389
4390 if (kvm_check_request(KVM_REQ_ENABLE_IBS, vcpu)) {
4391 if (!ibs_enabled(vcpu)) {
4392 trace_kvm_s390_enable_disable_ibs(vcpu->vcpu_id, 1);
4393 kvm_s390_set_cpuflags(vcpu, CPUSTAT_IBS);
4394 }
4395 goto retry;
4396 }
4397
4398 if (kvm_check_request(KVM_REQ_DISABLE_IBS, vcpu)) {
4399 if (ibs_enabled(vcpu)) {
4400 trace_kvm_s390_enable_disable_ibs(vcpu->vcpu_id, 0);
4401 kvm_s390_clear_cpuflags(vcpu, CPUSTAT_IBS);
4402 }
4403 goto retry;
4404 }
4405
4406 if (kvm_check_request(KVM_REQ_ICPT_OPEREXC, vcpu)) {
4407 vcpu->arch.sie_block->ictl |= ICTL_OPEREXC;
4408 goto retry;
4409 }
4410
4411 if (kvm_check_request(KVM_REQ_START_MIGRATION, vcpu)) {
4412 /*
4413 * Disable CMM virtualization; we will emulate the ESSA
4414 * instruction manually, in order to provide additional
4415 * functionalities needed for live migration.
4416 */
4417 vcpu->arch.sie_block->ecb2 &= ~ECB2_CMMA;
4418 goto retry;
4419 }
4420
4421 if (kvm_check_request(KVM_REQ_STOP_MIGRATION, vcpu)) {
4422 /*
4423 * Re-enable CMM virtualization if CMMA is available and
4424 * CMM has been used.
4425 */
4426 if (vcpu->kvm->arch.use_cmma && uses_cmm(vcpu->arch.gmap))
4427 vcpu->arch.sie_block->ecb2 |= ECB2_CMMA;
4428 goto retry;
4429 }
4430
4431 /* we left the vsie handler, nothing to do, just clear the request */
4432 kvm_clear_request(KVM_REQ_VSIE_RESTART, vcpu);
4433
4434 return 0;
4435 }
4436
__kvm_s390_set_tod_clock(struct kvm * kvm,const struct kvm_s390_vm_tod_clock * gtod)4437 static void __kvm_s390_set_tod_clock(struct kvm *kvm, const struct kvm_s390_vm_tod_clock *gtod)
4438 {
4439 struct kvm_vcpu *vcpu;
4440 union tod_clock clk;
4441 unsigned long i;
4442
4443 preempt_disable();
4444
4445 store_tod_clock_ext(&clk);
4446
4447 kvm->arch.epoch = gtod->tod - clk.tod;
4448 kvm->arch.epdx = 0;
4449 if (test_kvm_facility(kvm, 139)) {
4450 kvm->arch.epdx = gtod->epoch_idx - clk.ei;
4451 if (kvm->arch.epoch > gtod->tod)
4452 kvm->arch.epdx -= 1;
4453 }
4454
4455 kvm_s390_vcpu_block_all(kvm);
4456 kvm_for_each_vcpu(i, vcpu, kvm) {
4457 vcpu->arch.sie_block->epoch = kvm->arch.epoch;
4458 vcpu->arch.sie_block->epdx = kvm->arch.epdx;
4459 }
4460
4461 kvm_s390_vcpu_unblock_all(kvm);
4462 preempt_enable();
4463 }
4464
kvm_s390_try_set_tod_clock(struct kvm * kvm,const struct kvm_s390_vm_tod_clock * gtod)4465 int kvm_s390_try_set_tod_clock(struct kvm *kvm, const struct kvm_s390_vm_tod_clock *gtod)
4466 {
4467 if (!mutex_trylock(&kvm->lock))
4468 return 0;
4469 __kvm_s390_set_tod_clock(kvm, gtod);
4470 mutex_unlock(&kvm->lock);
4471 return 1;
4472 }
4473
__kvm_inject_pfault_token(struct kvm_vcpu * vcpu,bool start_token,unsigned long token)4474 static void __kvm_inject_pfault_token(struct kvm_vcpu *vcpu, bool start_token,
4475 unsigned long token)
4476 {
4477 struct kvm_s390_interrupt inti;
4478 struct kvm_s390_irq irq;
4479 struct kvm_s390_interrupt_info *inti_mem = NULL;
4480 int ret = 0;
4481
4482 if (start_token) {
4483 irq.u.ext.ext_params2 = token;
4484 irq.type = KVM_S390_INT_PFAULT_INIT;
4485 WARN_ON_ONCE(kvm_s390_inject_vcpu(vcpu, &irq));
4486 } else {
4487 inti_mem = kzalloc_obj(*inti_mem, GFP_KERNEL_ACCOUNT);
4488 if (WARN_ON_ONCE(!inti_mem))
4489 return;
4490
4491 inti.type = KVM_S390_INT_PFAULT_DONE;
4492 inti.parm64 = token;
4493 ret = kvm_s390_inject_vm(vcpu->kvm, &inti, inti_mem);
4494 if (ret)
4495 kfree(inti_mem);
4496 WARN_ON_ONCE(ret);
4497 }
4498 }
4499
kvm_arch_async_page_not_present(struct kvm_vcpu * vcpu,struct kvm_async_pf * work)4500 bool kvm_arch_async_page_not_present(struct kvm_vcpu *vcpu,
4501 struct kvm_async_pf *work)
4502 {
4503 trace_kvm_s390_pfault_init(vcpu, work->arch.pfault_token);
4504 __kvm_inject_pfault_token(vcpu, true, work->arch.pfault_token);
4505
4506 return true;
4507 }
4508
kvm_arch_async_page_present(struct kvm_vcpu * vcpu,struct kvm_async_pf * work)4509 void kvm_arch_async_page_present(struct kvm_vcpu *vcpu,
4510 struct kvm_async_pf *work)
4511 {
4512 trace_kvm_s390_pfault_done(vcpu, work->arch.pfault_token);
4513 __kvm_inject_pfault_token(vcpu, false, work->arch.pfault_token);
4514 }
4515
kvm_arch_async_page_ready(struct kvm_vcpu * vcpu,struct kvm_async_pf * work)4516 void kvm_arch_async_page_ready(struct kvm_vcpu *vcpu,
4517 struct kvm_async_pf *work)
4518 {
4519 /* s390 will always inject the page directly */
4520 }
4521
kvm_arch_can_dequeue_async_page_present(struct kvm_vcpu * vcpu)4522 bool kvm_arch_can_dequeue_async_page_present(struct kvm_vcpu *vcpu)
4523 {
4524 /*
4525 * s390 will always inject the page directly,
4526 * but we still want check_async_completion to cleanup
4527 */
4528 return true;
4529 }
4530
kvm_arch_setup_async_pf(struct kvm_vcpu * vcpu)4531 bool kvm_arch_setup_async_pf(struct kvm_vcpu *vcpu)
4532 {
4533 hva_t hva;
4534 struct kvm_arch_async_pf arch;
4535
4536 if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID)
4537 return false;
4538 if ((vcpu->arch.sie_block->gpsw.mask & vcpu->arch.pfault_select) !=
4539 vcpu->arch.pfault_compare)
4540 return false;
4541 if (psw_extint_disabled(vcpu))
4542 return false;
4543 if (kvm_s390_vcpu_has_irq(vcpu, 0))
4544 return false;
4545 if (!(vcpu->arch.sie_block->gcr[0] & CR0_SERVICE_SIGNAL_SUBMASK))
4546 return false;
4547 if (!pfault_enabled(vcpu->arch.gmap))
4548 return false;
4549
4550 hva = gfn_to_hva(vcpu->kvm, current->thread.gmap_teid.addr);
4551 if (read_guest_real(vcpu, vcpu->arch.pfault_token, &arch.pfault_token, 8))
4552 return false;
4553
4554 return kvm_setup_async_pf(vcpu, current->thread.gmap_teid.addr * PAGE_SIZE, hva, &arch);
4555 }
4556
vcpu_pre_run(struct kvm_vcpu * vcpu)4557 static int vcpu_pre_run(struct kvm_vcpu *vcpu)
4558 {
4559 int rc, cpuflags;
4560
4561 /*
4562 * On s390 notifications for arriving pages will be delivered directly
4563 * to the guest but the house keeping for completed pfaults is
4564 * handled outside the worker.
4565 */
4566 kvm_check_async_pf_completion(vcpu);
4567
4568 vcpu->arch.sie_block->gg14 = vcpu->run->s.regs.gprs[14];
4569 vcpu->arch.sie_block->gg15 = vcpu->run->s.regs.gprs[15];
4570
4571 if (!kvm_is_ucontrol(vcpu->kvm)) {
4572 rc = kvm_s390_deliver_pending_interrupts(vcpu);
4573 if (rc || guestdbg_exit_pending(vcpu))
4574 return rc;
4575 }
4576
4577 rc = kvm_s390_handle_requests(vcpu);
4578 if (rc)
4579 return rc;
4580
4581 if (guestdbg_enabled(vcpu)) {
4582 kvm_s390_backup_guest_per_regs(vcpu);
4583 kvm_s390_patch_guest_per_regs(vcpu);
4584 }
4585
4586 clear_bit(vcpu->vcpu_idx, vcpu->kvm->arch.gisa_int.kicked_mask);
4587
4588 vcpu->arch.sie_block->icptcode = 0;
4589 current->thread.gmap_int_code = 0;
4590 cpuflags = atomic_read(&vcpu->arch.sie_block->cpuflags);
4591 VCPU_EVENT(vcpu, 6, "entering sie flags %x", cpuflags);
4592 trace_kvm_s390_sie_enter(vcpu, cpuflags);
4593
4594 return 0;
4595 }
4596
vcpu_post_run_addressing_exception(struct kvm_vcpu * vcpu)4597 static int vcpu_post_run_addressing_exception(struct kvm_vcpu *vcpu)
4598 {
4599 struct kvm_s390_pgm_info pgm_info = {
4600 .code = PGM_ADDRESSING,
4601 };
4602 u8 opcode, ilen;
4603 int rc;
4604
4605 VCPU_EVENT(vcpu, 3, "%s", "fault in sie instruction");
4606 trace_kvm_s390_sie_fault(vcpu);
4607
4608 /*
4609 * We want to inject an addressing exception, which is defined as a
4610 * suppressing or terminating exception. However, since we came here
4611 * by a DAT access exception, the PSW still points to the faulting
4612 * instruction since DAT exceptions are nullifying. So we've got
4613 * to look up the current opcode to get the length of the instruction
4614 * to be able to forward the PSW.
4615 */
4616 rc = read_guest_instr(vcpu, vcpu->arch.sie_block->gpsw.addr, &opcode, 1);
4617 ilen = insn_length(opcode);
4618 if (rc < 0) {
4619 return rc;
4620 } else if (rc) {
4621 /* Instruction-Fetching Exceptions - we can't detect the ilen.
4622 * Forward by arbitrary ilc, injection will take care of
4623 * nullification if necessary.
4624 */
4625 pgm_info = vcpu->arch.pgm;
4626 ilen = 4;
4627 }
4628 pgm_info.flags = ilen | KVM_S390_PGM_FLAGS_ILC_VALID;
4629 kvm_s390_forward_psw(vcpu, ilen);
4630 return kvm_s390_inject_prog_irq(vcpu, &pgm_info);
4631 }
4632
kvm_s390_assert_primary_as(struct kvm_vcpu * vcpu)4633 static void kvm_s390_assert_primary_as(struct kvm_vcpu *vcpu)
4634 {
4635 KVM_BUG(current->thread.gmap_teid.as != PSW_BITS_AS_PRIMARY, vcpu->kvm,
4636 "Unexpected program interrupt 0x%x, TEID 0x%016lx",
4637 current->thread.gmap_int_code, current->thread.gmap_teid.val);
4638 }
4639
vcpu_dat_fault_handler(struct kvm_vcpu * vcpu,gpa_t gaddr,bool wr)4640 static int vcpu_dat_fault_handler(struct kvm_vcpu *vcpu, gpa_t gaddr, bool wr)
4641 {
4642 struct guest_fault f = {
4643 .write_attempt = wr,
4644 .attempt_pfault = pfault_enabled(vcpu->arch.gmap),
4645 };
4646 int rc;
4647
4648 if (vcpu_ucontrol_translate(vcpu, &gaddr))
4649 return -EREMOTE;
4650 f.gfn = gpa_to_gfn(gaddr);
4651
4652 rc = kvm_s390_faultin_gfn(vcpu, NULL, &f);
4653 if (rc <= 0)
4654 return rc;
4655 if (rc == PGM_ADDRESSING)
4656 return vcpu_post_run_addressing_exception(vcpu);
4657 KVM_BUG_ON(rc, vcpu->kvm);
4658 return -EINVAL;
4659 }
4660
vcpu_post_run_handle_fault(struct kvm_vcpu * vcpu)4661 static int vcpu_post_run_handle_fault(struct kvm_vcpu *vcpu)
4662 {
4663 unsigned int foll = 0;
4664 unsigned long gaddr;
4665 int rc;
4666
4667 gaddr = current->thread.gmap_teid.addr * PAGE_SIZE;
4668 if (kvm_s390_cur_gmap_fault_is_write())
4669 foll = FOLL_WRITE;
4670
4671 switch (current->thread.gmap_int_code & PGM_INT_CODE_MASK) {
4672 case 0:
4673 vcpu->stat.exit_null++;
4674 break;
4675 case PGM_SECURE_STORAGE_ACCESS:
4676 case PGM_SECURE_STORAGE_VIOLATION:
4677 kvm_s390_assert_primary_as(vcpu);
4678 /*
4679 * This can happen after a reboot with asynchronous teardown;
4680 * the new guest (normal or protected) will run on top of the
4681 * previous protected guest. The old pages need to be destroyed
4682 * so the new guest can use them.
4683 */
4684 if (kvm_s390_pv_destroy_page(vcpu->kvm, gaddr)) {
4685 /*
4686 * Either KVM messed up the secure guest mapping or the
4687 * same page is mapped into multiple secure guests.
4688 *
4689 * This exception is only triggered when a guest 2 is
4690 * running and can therefore never occur in kernel
4691 * context.
4692 */
4693 pr_warn_ratelimited("Secure storage violation (%x) in task: %s, pid %d\n",
4694 current->thread.gmap_int_code, current->comm,
4695 current->pid);
4696 send_sig(SIGSEGV, current, 0);
4697 }
4698 break;
4699 case PGM_NON_SECURE_STORAGE_ACCESS:
4700 kvm_s390_assert_primary_as(vcpu);
4701 /*
4702 * This is normal operation; a page belonging to a protected
4703 * guest has not been imported yet. Try to import the page into
4704 * the protected guest.
4705 */
4706 rc = kvm_s390_pv_convert_to_secure(vcpu->kvm, gaddr);
4707 if (rc == -EINVAL)
4708 send_sig(SIGSEGV, current, 0);
4709 if (rc != -ENXIO)
4710 break;
4711 foll = FOLL_WRITE;
4712 fallthrough;
4713 case PGM_PROTECTION:
4714 case PGM_SEGMENT_TRANSLATION:
4715 case PGM_PAGE_TRANSLATION:
4716 case PGM_ASCE_TYPE:
4717 case PGM_REGION_FIRST_TRANS:
4718 case PGM_REGION_SECOND_TRANS:
4719 case PGM_REGION_THIRD_TRANS:
4720 kvm_s390_assert_primary_as(vcpu);
4721 return vcpu_dat_fault_handler(vcpu, gaddr, foll);
4722 default:
4723 KVM_BUG(1, vcpu->kvm, "Unexpected program interrupt 0x%x, TEID 0x%016lx",
4724 current->thread.gmap_int_code, current->thread.gmap_teid.val);
4725 send_sig(SIGSEGV, current, 0);
4726 break;
4727 }
4728 return 0;
4729 }
4730
vcpu_post_run(struct kvm_vcpu * vcpu,int sie_return)4731 static int vcpu_post_run(struct kvm_vcpu *vcpu, int sie_return)
4732 {
4733 struct mcck_volatile_info *mcck_info;
4734 struct sie_page *sie_page;
4735 int rc;
4736
4737 VCPU_EVENT(vcpu, 6, "exit sie icptcode %d",
4738 vcpu->arch.sie_block->icptcode);
4739 trace_kvm_s390_sie_exit(vcpu, vcpu->arch.sie_block->icptcode);
4740
4741 if (guestdbg_enabled(vcpu))
4742 kvm_s390_restore_guest_per_regs(vcpu);
4743
4744 vcpu->run->s.regs.gprs[14] = vcpu->arch.sie_block->gg14;
4745 vcpu->run->s.regs.gprs[15] = vcpu->arch.sie_block->gg15;
4746
4747 if (sie_return == SIE64_RETURN_MCCK) {
4748 sie_page = container_of(vcpu->arch.sie_block,
4749 struct sie_page, sie_block);
4750 mcck_info = &sie_page->mcck_info;
4751 kvm_s390_reinject_machine_check(vcpu, mcck_info);
4752 return 0;
4753 }
4754 WARN_ON_ONCE(sie_return != SIE64_RETURN_NORMAL);
4755
4756 if (vcpu->arch.sie_block->icptcode > 0) {
4757 rc = kvm_handle_sie_intercept(vcpu);
4758
4759 if (rc != -EOPNOTSUPP)
4760 return rc;
4761 vcpu->run->exit_reason = KVM_EXIT_S390_SIEIC;
4762 vcpu->run->s390_sieic.icptcode = vcpu->arch.sie_block->icptcode;
4763 vcpu->run->s390_sieic.ipa = vcpu->arch.sie_block->ipa;
4764 vcpu->run->s390_sieic.ipb = vcpu->arch.sie_block->ipb;
4765 return -EREMOTE;
4766 }
4767
4768 return vcpu_post_run_handle_fault(vcpu);
4769 }
4770
kvm_s390_enter_exit_sie(struct kvm_s390_sie_block * scb,u64 * gprs,unsigned long gasce)4771 int noinstr kvm_s390_enter_exit_sie(struct kvm_s390_sie_block *scb,
4772 u64 *gprs, unsigned long gasce)
4773 {
4774 int ret;
4775
4776 guest_state_enter_irqoff();
4777
4778 /*
4779 * The guest_state_{enter,exit}_irqoff() functions inform lockdep and
4780 * tracing that entry to the guest will enable host IRQs, and exit from
4781 * the guest will disable host IRQs.
4782 */
4783 ret = sie64a(scb, gprs, gasce);
4784
4785 guest_state_exit_irqoff();
4786
4787 return ret;
4788 }
4789
4790 #define PSW_INT_MASK (PSW_MASK_EXT | PSW_MASK_IO | PSW_MASK_MCHECK)
__vcpu_run(struct kvm_vcpu * vcpu)4791 static int __vcpu_run(struct kvm_vcpu *vcpu)
4792 {
4793 int rc, sie_return;
4794 struct sie_page *sie_page = (struct sie_page *)vcpu->arch.sie_block;
4795
4796 /*
4797 * We try to hold kvm->srcu during most of vcpu_run (except when run-
4798 * ning the guest), so that memslots (and other stuff) are protected
4799 */
4800 kvm_vcpu_srcu_read_lock(vcpu);
4801
4802 while (true) {
4803 rc = vcpu_pre_run(vcpu);
4804 kvm_vcpu_srcu_read_unlock(vcpu);
4805 if (rc || guestdbg_exit_pending(vcpu))
4806 break;
4807
4808 /*
4809 * As PF_VCPU will be used in fault handler, between
4810 * guest_timing_enter_irqoff and guest_timing_exit_irqoff
4811 * should be no uaccess.
4812 */
4813 if (kvm_s390_pv_cpu_is_protected(vcpu)) {
4814 memcpy(sie_page->pv_grregs,
4815 vcpu->run->s.regs.gprs,
4816 sizeof(sie_page->pv_grregs));
4817 }
4818
4819 xfer_to_guest_mode_check:
4820 local_irq_disable();
4821 xfer_to_guest_mode_prepare();
4822 if (xfer_to_guest_mode_work_pending()) {
4823 local_irq_enable();
4824 rc = kvm_xfer_to_guest_mode_handle_work(vcpu);
4825 if (rc)
4826 break;
4827 goto xfer_to_guest_mode_check;
4828 }
4829
4830 guest_timing_enter_irqoff();
4831 __disable_cpu_timer_accounting(vcpu);
4832
4833 sie_return = kvm_s390_enter_exit_sie(vcpu->arch.sie_block,
4834 vcpu->run->s.regs.gprs,
4835 vcpu->arch.gmap->asce.val);
4836
4837 __enable_cpu_timer_accounting(vcpu);
4838 guest_timing_exit_irqoff();
4839 local_irq_enable();
4840
4841 if (kvm_s390_pv_cpu_is_protected(vcpu)) {
4842 memcpy(vcpu->run->s.regs.gprs,
4843 sie_page->pv_grregs,
4844 sizeof(sie_page->pv_grregs));
4845 /*
4846 * We're not allowed to inject interrupts on intercepts
4847 * that leave the guest state in an "in-between" state
4848 * where the next SIE entry will do a continuation.
4849 * Fence interrupts in our "internal" PSW.
4850 */
4851 if (vcpu->arch.sie_block->icptcode == ICPT_PV_INSTR ||
4852 vcpu->arch.sie_block->icptcode == ICPT_PV_PREF) {
4853 vcpu->arch.sie_block->gpsw.mask &= ~PSW_INT_MASK;
4854 }
4855 }
4856 kvm_vcpu_srcu_read_lock(vcpu);
4857
4858 rc = vcpu_post_run(vcpu, sie_return);
4859 if (rc || guestdbg_exit_pending(vcpu)) {
4860 kvm_vcpu_srcu_read_unlock(vcpu);
4861 break;
4862 }
4863 }
4864
4865 return rc;
4866 }
4867
sync_regs_fmt2(struct kvm_vcpu * vcpu)4868 static void sync_regs_fmt2(struct kvm_vcpu *vcpu)
4869 {
4870 struct kvm_run *kvm_run = vcpu->run;
4871 struct runtime_instr_cb *riccb;
4872 struct gs_cb *gscb;
4873
4874 riccb = (struct runtime_instr_cb *) &kvm_run->s.regs.riccb;
4875 gscb = (struct gs_cb *) &kvm_run->s.regs.gscb;
4876 vcpu->arch.sie_block->gpsw.mask = kvm_run->psw_mask;
4877 vcpu->arch.sie_block->gpsw.addr = kvm_run->psw_addr;
4878 if (kvm_run->kvm_dirty_regs & KVM_SYNC_ARCH0) {
4879 vcpu->arch.sie_block->todpr = kvm_run->s.regs.todpr;
4880 vcpu->arch.sie_block->pp = kvm_run->s.regs.pp;
4881 vcpu->arch.sie_block->gbea = kvm_run->s.regs.gbea;
4882 }
4883 if (kvm_run->kvm_dirty_regs & KVM_SYNC_PFAULT) {
4884 vcpu->arch.pfault_token = kvm_run->s.regs.pft;
4885 vcpu->arch.pfault_select = kvm_run->s.regs.pfs;
4886 vcpu->arch.pfault_compare = kvm_run->s.regs.pfc;
4887 if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID)
4888 kvm_clear_async_pf_completion_queue(vcpu);
4889 }
4890 if (kvm_run->kvm_dirty_regs & KVM_SYNC_DIAG318) {
4891 vcpu->arch.diag318_info.val = kvm_run->s.regs.diag318;
4892 vcpu->arch.sie_block->cpnc = vcpu->arch.diag318_info.cpnc;
4893 VCPU_EVENT(vcpu, 3, "setting cpnc to %d", vcpu->arch.diag318_info.cpnc);
4894 }
4895 /*
4896 * If userspace sets the riccb (e.g. after migration) to a valid state,
4897 * we should enable RI here instead of doing the lazy enablement.
4898 */
4899 if ((kvm_run->kvm_dirty_regs & KVM_SYNC_RICCB) &&
4900 test_kvm_facility(vcpu->kvm, 64) &&
4901 riccb->v &&
4902 !(vcpu->arch.sie_block->ecb3 & ECB3_RI)) {
4903 VCPU_EVENT(vcpu, 3, "%s", "ENABLE: RI (sync_regs)");
4904 vcpu->arch.sie_block->ecb3 |= ECB3_RI;
4905 }
4906 /*
4907 * If userspace sets the gscb (e.g. after migration) to non-zero,
4908 * we should enable GS here instead of doing the lazy enablement.
4909 */
4910 if ((kvm_run->kvm_dirty_regs & KVM_SYNC_GSCB) &&
4911 test_kvm_facility(vcpu->kvm, 133) &&
4912 gscb->gssm &&
4913 !vcpu->arch.gs_enabled) {
4914 VCPU_EVENT(vcpu, 3, "%s", "ENABLE: GS (sync_regs)");
4915 vcpu->arch.sie_block->ecb |= ECB_GS;
4916 vcpu->arch.sie_block->ecd |= ECD_HOSTREGMGMT;
4917 vcpu->arch.gs_enabled = 1;
4918 }
4919 if ((kvm_run->kvm_dirty_regs & KVM_SYNC_BPBC) &&
4920 test_kvm_facility(vcpu->kvm, 82)) {
4921 vcpu->arch.sie_block->fpf &= ~FPF_BPBC;
4922 vcpu->arch.sie_block->fpf |= kvm_run->s.regs.bpbc ? FPF_BPBC : 0;
4923 }
4924 if (cpu_has_gs()) {
4925 preempt_disable();
4926 local_ctl_set_bit(2, CR2_GUARDED_STORAGE_BIT);
4927 if (current->thread.gs_cb) {
4928 vcpu->arch.host_gscb = current->thread.gs_cb;
4929 save_gs_cb(vcpu->arch.host_gscb);
4930 }
4931 if (vcpu->arch.gs_enabled) {
4932 current->thread.gs_cb = (struct gs_cb *)
4933 &vcpu->run->s.regs.gscb;
4934 restore_gs_cb(current->thread.gs_cb);
4935 }
4936 preempt_enable();
4937 }
4938 /* SIE will load etoken directly from SDNX and therefore kvm_run */
4939 }
4940
sync_regs(struct kvm_vcpu * vcpu)4941 static void sync_regs(struct kvm_vcpu *vcpu)
4942 {
4943 struct kvm_run *kvm_run = vcpu->run;
4944
4945 if (kvm_run->kvm_dirty_regs & KVM_SYNC_PREFIX)
4946 kvm_s390_set_prefix(vcpu, kvm_run->s.regs.prefix);
4947 if (kvm_run->kvm_dirty_regs & KVM_SYNC_CRS) {
4948 memcpy(&vcpu->arch.sie_block->gcr, &kvm_run->s.regs.crs, 128);
4949 /* some control register changes require a tlb flush */
4950 kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
4951 }
4952 if (kvm_run->kvm_dirty_regs & KVM_SYNC_ARCH0) {
4953 kvm_s390_set_cpu_timer(vcpu, kvm_run->s.regs.cputm);
4954 vcpu->arch.sie_block->ckc = kvm_run->s.regs.ckc;
4955 }
4956 save_access_regs(vcpu->arch.host_acrs);
4957 restore_access_regs(vcpu->run->s.regs.acrs);
4958 vcpu->arch.acrs_loaded = true;
4959 kvm_s390_fpu_load(vcpu->run);
4960 /* Sync fmt2 only data */
4961 if (likely(!kvm_s390_pv_cpu_is_protected(vcpu))) {
4962 sync_regs_fmt2(vcpu);
4963 } else {
4964 /*
4965 * In several places we have to modify our internal view to
4966 * not do things that are disallowed by the ultravisor. For
4967 * example we must not inject interrupts after specific exits
4968 * (e.g. 112 prefix page not secure). We do this by turning
4969 * off the machine check, external and I/O interrupt bits
4970 * of our PSW copy. To avoid getting validity intercepts, we
4971 * do only accept the condition code from userspace.
4972 */
4973 vcpu->arch.sie_block->gpsw.mask &= ~PSW_MASK_CC;
4974 vcpu->arch.sie_block->gpsw.mask |= kvm_run->psw_mask &
4975 PSW_MASK_CC;
4976 }
4977
4978 kvm_run->kvm_dirty_regs = 0;
4979 }
4980
store_regs_fmt2(struct kvm_vcpu * vcpu)4981 static void store_regs_fmt2(struct kvm_vcpu *vcpu)
4982 {
4983 struct kvm_run *kvm_run = vcpu->run;
4984
4985 kvm_run->s.regs.todpr = vcpu->arch.sie_block->todpr;
4986 kvm_run->s.regs.pp = vcpu->arch.sie_block->pp;
4987 kvm_run->s.regs.gbea = vcpu->arch.sie_block->gbea;
4988 kvm_run->s.regs.bpbc = (vcpu->arch.sie_block->fpf & FPF_BPBC) == FPF_BPBC;
4989 kvm_run->s.regs.diag318 = vcpu->arch.diag318_info.val;
4990 if (cpu_has_gs()) {
4991 preempt_disable();
4992 local_ctl_set_bit(2, CR2_GUARDED_STORAGE_BIT);
4993 if (vcpu->arch.gs_enabled)
4994 save_gs_cb(current->thread.gs_cb);
4995 current->thread.gs_cb = vcpu->arch.host_gscb;
4996 restore_gs_cb(vcpu->arch.host_gscb);
4997 if (!vcpu->arch.host_gscb)
4998 local_ctl_clear_bit(2, CR2_GUARDED_STORAGE_BIT);
4999 vcpu->arch.host_gscb = NULL;
5000 preempt_enable();
5001 }
5002 /* SIE will save etoken directly into SDNX and therefore kvm_run */
5003 }
5004
store_regs(struct kvm_vcpu * vcpu)5005 static void store_regs(struct kvm_vcpu *vcpu)
5006 {
5007 struct kvm_run *kvm_run = vcpu->run;
5008
5009 kvm_run->psw_mask = vcpu->arch.sie_block->gpsw.mask;
5010 kvm_run->psw_addr = vcpu->arch.sie_block->gpsw.addr;
5011 kvm_run->s.regs.prefix = kvm_s390_get_prefix(vcpu);
5012 memcpy(&kvm_run->s.regs.crs, &vcpu->arch.sie_block->gcr, 128);
5013 kvm_run->s.regs.cputm = kvm_s390_get_cpu_timer(vcpu);
5014 kvm_run->s.regs.ckc = vcpu->arch.sie_block->ckc;
5015 kvm_run->s.regs.pft = vcpu->arch.pfault_token;
5016 kvm_run->s.regs.pfs = vcpu->arch.pfault_select;
5017 kvm_run->s.regs.pfc = vcpu->arch.pfault_compare;
5018 save_access_regs(vcpu->run->s.regs.acrs);
5019 restore_access_regs(vcpu->arch.host_acrs);
5020 vcpu->arch.acrs_loaded = false;
5021 kvm_s390_fpu_store(vcpu->run);
5022 if (likely(!kvm_s390_pv_cpu_is_protected(vcpu)))
5023 store_regs_fmt2(vcpu);
5024 }
5025
kvm_arch_vcpu_ioctl_run(struct kvm_vcpu * vcpu)5026 int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu)
5027 {
5028 struct kvm_run *kvm_run = vcpu->run;
5029 DECLARE_KERNEL_FPU_ONSTACK32(fpu);
5030 int rc;
5031
5032 /*
5033 * Running a VM while dumping always has the potential to
5034 * produce inconsistent dump data. But for PV vcpus a SIE
5035 * entry while dumping could also lead to a fatal validity
5036 * intercept which we absolutely want to avoid.
5037 */
5038 if (vcpu->kvm->arch.pv.dumping)
5039 return -EINVAL;
5040
5041 if (!vcpu->wants_to_run)
5042 return -EINTR;
5043
5044 if (kvm_run->kvm_valid_regs & ~KVM_SYNC_S390_VALID_FIELDS ||
5045 kvm_run->kvm_dirty_regs & ~KVM_SYNC_S390_VALID_FIELDS)
5046 return -EINVAL;
5047
5048 /* Pairs with smp_store_release() in kvm_arch_vcpu_postcreate() */
5049 if (!smp_load_acquire(&vcpu->arch.initialized))
5050 return -EINVAL;
5051
5052 vcpu_load(vcpu);
5053
5054 if (guestdbg_exit_pending(vcpu)) {
5055 kvm_s390_prepare_debug_exit(vcpu);
5056 rc = 0;
5057 goto out;
5058 }
5059
5060 kvm_sigset_activate(vcpu);
5061
5062 /*
5063 * no need to check the return value of vcpu_start as it can only have
5064 * an error for protvirt, but protvirt means user cpu state
5065 */
5066 if (!kvm_s390_user_cpu_state_ctrl(vcpu->kvm)) {
5067 kvm_s390_vcpu_start(vcpu);
5068 } else if (is_vcpu_stopped(vcpu)) {
5069 pr_err_ratelimited("can't run stopped vcpu %d\n",
5070 vcpu->vcpu_id);
5071 rc = -EINVAL;
5072 goto out;
5073 }
5074
5075 kernel_fpu_begin(&fpu, KERNEL_FPC | KERNEL_VXR);
5076 sync_regs(vcpu);
5077 enable_cpu_timer_accounting(vcpu);
5078
5079 might_fault();
5080 rc = __vcpu_run(vcpu);
5081
5082 if (signal_pending(current) && !rc) {
5083 kvm_run->exit_reason = KVM_EXIT_INTR;
5084 vcpu->stat.signal_exits++;
5085 rc = -EINTR;
5086 }
5087
5088 if (guestdbg_exit_pending(vcpu) && !rc) {
5089 kvm_s390_prepare_debug_exit(vcpu);
5090 rc = 0;
5091 }
5092
5093 if (rc == -EREMOTE) {
5094 /* userspace support is needed, kvm_run has been prepared */
5095 rc = 0;
5096 }
5097
5098 disable_cpu_timer_accounting(vcpu);
5099 store_regs(vcpu);
5100 kernel_fpu_end(&fpu, KERNEL_FPC | KERNEL_VXR);
5101
5102 kvm_sigset_deactivate(vcpu);
5103
5104 vcpu->stat.exit_userspace++;
5105 out:
5106 vcpu_put(vcpu);
5107 return rc;
5108 }
5109
5110 /*
5111 * store status at address
5112 * we use have two special cases:
5113 * KVM_S390_STORE_STATUS_NOADDR: -> 0x1200 on 64 bit
5114 * KVM_S390_STORE_STATUS_PREFIXED: -> prefix
5115 */
kvm_s390_store_status_unloaded(struct kvm_vcpu * vcpu,unsigned long gpa)5116 int kvm_s390_store_status_unloaded(struct kvm_vcpu *vcpu, unsigned long gpa)
5117 {
5118 unsigned char archmode = 1;
5119 freg_t fprs[NUM_FPRS];
5120 unsigned int px;
5121 u64 clkcomp, cputm;
5122 int rc;
5123
5124 px = kvm_s390_get_prefix(vcpu);
5125 if (gpa == KVM_S390_STORE_STATUS_NOADDR) {
5126 if (write_guest_abs(vcpu, 163, &archmode, 1))
5127 return -EFAULT;
5128 gpa = 0;
5129 } else if (gpa == KVM_S390_STORE_STATUS_PREFIXED) {
5130 if (write_guest_real(vcpu, 163, &archmode, 1))
5131 return -EFAULT;
5132 gpa = px;
5133 } else
5134 gpa -= __LC_FPREGS_SAVE_AREA;
5135
5136 /* manually convert vector registers if necessary */
5137 if (cpu_has_vx()) {
5138 convert_vx_to_fp(fprs, (__vector128 *) vcpu->run->s.regs.vrs);
5139 rc = write_guest_abs(vcpu, gpa + __LC_FPREGS_SAVE_AREA,
5140 fprs, 128);
5141 } else {
5142 rc = write_guest_abs(vcpu, gpa + __LC_FPREGS_SAVE_AREA,
5143 vcpu->run->s.regs.fprs, 128);
5144 }
5145 rc |= write_guest_abs(vcpu, gpa + __LC_GPREGS_SAVE_AREA,
5146 vcpu->run->s.regs.gprs, 128);
5147 rc |= write_guest_abs(vcpu, gpa + __LC_PSW_SAVE_AREA,
5148 &vcpu->arch.sie_block->gpsw, 16);
5149 rc |= write_guest_abs(vcpu, gpa + __LC_PREFIX_SAVE_AREA,
5150 &px, 4);
5151 rc |= write_guest_abs(vcpu, gpa + __LC_FP_CREG_SAVE_AREA,
5152 &vcpu->run->s.regs.fpc, 4);
5153 rc |= write_guest_abs(vcpu, gpa + __LC_TOD_PROGREG_SAVE_AREA,
5154 &vcpu->arch.sie_block->todpr, 4);
5155 cputm = kvm_s390_get_cpu_timer(vcpu);
5156 rc |= write_guest_abs(vcpu, gpa + __LC_CPU_TIMER_SAVE_AREA,
5157 &cputm, 8);
5158 clkcomp = vcpu->arch.sie_block->ckc >> 8;
5159 rc |= write_guest_abs(vcpu, gpa + __LC_CLOCK_COMP_SAVE_AREA,
5160 &clkcomp, 8);
5161 rc |= write_guest_abs(vcpu, gpa + __LC_AREGS_SAVE_AREA,
5162 &vcpu->run->s.regs.acrs, 64);
5163 rc |= write_guest_abs(vcpu, gpa + __LC_CREGS_SAVE_AREA,
5164 &vcpu->arch.sie_block->gcr, 128);
5165 return rc ? -EFAULT : 0;
5166 }
5167
kvm_s390_vcpu_store_status(struct kvm_vcpu * vcpu,unsigned long addr)5168 int kvm_s390_vcpu_store_status(struct kvm_vcpu *vcpu, unsigned long addr)
5169 {
5170 /*
5171 * The guest FPRS and ACRS are in the host FPRS/ACRS due to the lazy
5172 * switch in the run ioctl. Let's update our copies before we save
5173 * it into the save area
5174 */
5175 kvm_s390_fpu_store(vcpu->run);
5176 save_access_regs(vcpu->run->s.regs.acrs);
5177
5178 return kvm_s390_store_status_unloaded(vcpu, addr);
5179 }
5180
__disable_ibs_on_vcpu(struct kvm_vcpu * vcpu)5181 static void __disable_ibs_on_vcpu(struct kvm_vcpu *vcpu)
5182 {
5183 kvm_check_request(KVM_REQ_ENABLE_IBS, vcpu);
5184 kvm_s390_sync_request(KVM_REQ_DISABLE_IBS, vcpu);
5185 }
5186
__disable_ibs_on_all_vcpus(struct kvm * kvm)5187 static void __disable_ibs_on_all_vcpus(struct kvm *kvm)
5188 {
5189 unsigned long i;
5190 struct kvm_vcpu *vcpu;
5191
5192 kvm_for_each_vcpu(i, vcpu, kvm) {
5193 __disable_ibs_on_vcpu(vcpu);
5194 }
5195 }
5196
__enable_ibs_on_vcpu(struct kvm_vcpu * vcpu)5197 static void __enable_ibs_on_vcpu(struct kvm_vcpu *vcpu)
5198 {
5199 if (!sclp.has_ibs)
5200 return;
5201 kvm_check_request(KVM_REQ_DISABLE_IBS, vcpu);
5202 kvm_s390_sync_request(KVM_REQ_ENABLE_IBS, vcpu);
5203 }
5204
kvm_s390_vcpu_start(struct kvm_vcpu * vcpu)5205 int kvm_s390_vcpu_start(struct kvm_vcpu *vcpu)
5206 {
5207 int i, online_vcpus, r = 0, started_vcpus = 0;
5208
5209 if (!is_vcpu_stopped(vcpu))
5210 return 0;
5211
5212 trace_kvm_s390_vcpu_start_stop(vcpu->vcpu_id, 1);
5213 /* Only one cpu at a time may enter/leave the STOPPED state. */
5214 spin_lock(&vcpu->kvm->arch.start_stop_lock);
5215 online_vcpus = atomic_read(&vcpu->kvm->online_vcpus);
5216
5217 /* Let's tell the UV that we want to change into the operating state */
5218 if (kvm_s390_pv_cpu_is_protected(vcpu)) {
5219 r = kvm_s390_pv_set_cpu_state(vcpu, PV_CPU_STATE_OPR);
5220 if (r) {
5221 spin_unlock(&vcpu->kvm->arch.start_stop_lock);
5222 return r;
5223 }
5224 }
5225
5226 for (i = 0; i < online_vcpus; i++) {
5227 if (!is_vcpu_stopped(kvm_get_vcpu(vcpu->kvm, i)))
5228 started_vcpus++;
5229 }
5230
5231 if (started_vcpus == 0) {
5232 /* we're the only active VCPU -> speed it up */
5233 __enable_ibs_on_vcpu(vcpu);
5234 } else if (started_vcpus == 1) {
5235 /*
5236 * As we are starting a second VCPU, we have to disable
5237 * the IBS facility on all VCPUs to remove potentially
5238 * outstanding ENABLE requests.
5239 */
5240 __disable_ibs_on_all_vcpus(vcpu->kvm);
5241 }
5242
5243 kvm_s390_clear_cpuflags(vcpu, CPUSTAT_STOPPED);
5244 /*
5245 * The real PSW might have changed due to a RESTART interpreted by the
5246 * ultravisor. We block all interrupts and let the next sie exit
5247 * refresh our view.
5248 */
5249 if (kvm_s390_pv_cpu_is_protected(vcpu))
5250 vcpu->arch.sie_block->gpsw.mask &= ~PSW_INT_MASK;
5251 /*
5252 * Another VCPU might have used IBS while we were offline.
5253 * Let's play safe and flush the VCPU at startup.
5254 */
5255 kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
5256 spin_unlock(&vcpu->kvm->arch.start_stop_lock);
5257 return 0;
5258 }
5259
kvm_s390_vcpu_stop(struct kvm_vcpu * vcpu)5260 int kvm_s390_vcpu_stop(struct kvm_vcpu *vcpu)
5261 {
5262 int i, online_vcpus, r = 0, started_vcpus = 0;
5263 struct kvm_vcpu *started_vcpu = NULL;
5264
5265 if (is_vcpu_stopped(vcpu))
5266 return 0;
5267
5268 trace_kvm_s390_vcpu_start_stop(vcpu->vcpu_id, 0);
5269 /* Only one cpu at a time may enter/leave the STOPPED state. */
5270 spin_lock(&vcpu->kvm->arch.start_stop_lock);
5271 online_vcpus = atomic_read(&vcpu->kvm->online_vcpus);
5272
5273 /* Let's tell the UV that we want to change into the stopped state */
5274 if (kvm_s390_pv_cpu_is_protected(vcpu)) {
5275 r = kvm_s390_pv_set_cpu_state(vcpu, PV_CPU_STATE_STP);
5276 if (r) {
5277 spin_unlock(&vcpu->kvm->arch.start_stop_lock);
5278 return r;
5279 }
5280 }
5281
5282 /*
5283 * Set the VCPU to STOPPED and THEN clear the interrupt flag,
5284 * now that the SIGP STOP and SIGP STOP AND STORE STATUS orders
5285 * have been fully processed. This will ensure that the VCPU
5286 * is kept BUSY if another VCPU is inquiring with SIGP SENSE.
5287 */
5288 kvm_s390_set_cpuflags(vcpu, CPUSTAT_STOPPED);
5289 kvm_s390_clear_stop_irq(vcpu);
5290
5291 __disable_ibs_on_vcpu(vcpu);
5292
5293 for (i = 0; i < online_vcpus; i++) {
5294 struct kvm_vcpu *tmp = kvm_get_vcpu(vcpu->kvm, i);
5295
5296 if (!is_vcpu_stopped(tmp)) {
5297 started_vcpus++;
5298 started_vcpu = tmp;
5299 }
5300 }
5301
5302 if (started_vcpus == 1) {
5303 /*
5304 * As we only have one VCPU left, we want to enable the
5305 * IBS facility for that VCPU to speed it up.
5306 */
5307 __enable_ibs_on_vcpu(started_vcpu);
5308 }
5309
5310 spin_unlock(&vcpu->kvm->arch.start_stop_lock);
5311 return 0;
5312 }
5313
kvm_vcpu_ioctl_enable_cap(struct kvm_vcpu * vcpu,struct kvm_enable_cap * cap)5314 static int kvm_vcpu_ioctl_enable_cap(struct kvm_vcpu *vcpu,
5315 struct kvm_enable_cap *cap)
5316 {
5317 int r;
5318
5319 if (cap->flags)
5320 return -EINVAL;
5321
5322 switch (cap->cap) {
5323 case KVM_CAP_S390_CSS_SUPPORT:
5324 if (!vcpu->kvm->arch.css_support) {
5325 vcpu->kvm->arch.css_support = 1;
5326 VM_EVENT(vcpu->kvm, 3, "%s", "ENABLE: CSS support");
5327 trace_kvm_s390_enable_css(vcpu->kvm);
5328 }
5329 r = 0;
5330 break;
5331 default:
5332 r = -EINVAL;
5333 break;
5334 }
5335 return r;
5336 }
5337
kvm_s390_vcpu_sida_op(struct kvm_vcpu * vcpu,struct kvm_s390_mem_op * mop)5338 static long kvm_s390_vcpu_sida_op(struct kvm_vcpu *vcpu,
5339 struct kvm_s390_mem_op *mop)
5340 {
5341 void __user *uaddr = (void __user *)mop->buf;
5342 void *sida_addr;
5343 int r = 0;
5344
5345 if (mop->flags || !mop->size)
5346 return -EINVAL;
5347 if (mop->size + mop->sida_offset < mop->size)
5348 return -EINVAL;
5349 if (mop->size + mop->sida_offset > sida_size(vcpu->arch.sie_block))
5350 return -E2BIG;
5351 if (!kvm_s390_pv_cpu_is_protected(vcpu))
5352 return -EINVAL;
5353
5354 sida_addr = (char *)sida_addr(vcpu->arch.sie_block) + mop->sida_offset;
5355
5356 switch (mop->op) {
5357 case KVM_S390_MEMOP_SIDA_READ:
5358 if (copy_to_user(uaddr, sida_addr, mop->size))
5359 r = -EFAULT;
5360
5361 break;
5362 case KVM_S390_MEMOP_SIDA_WRITE:
5363 if (copy_from_user(sida_addr, uaddr, mop->size))
5364 r = -EFAULT;
5365 break;
5366 }
5367 return r;
5368 }
5369
kvm_s390_vcpu_mem_op(struct kvm_vcpu * vcpu,struct kvm_s390_mem_op * mop)5370 static long kvm_s390_vcpu_mem_op(struct kvm_vcpu *vcpu,
5371 struct kvm_s390_mem_op *mop)
5372 {
5373 void __user *uaddr = (void __user *)mop->buf;
5374 void *tmpbuf __free(kvfree) = NULL;
5375 enum gacc_mode acc_mode;
5376 int r;
5377
5378 r = mem_op_validate_common(mop, KVM_S390_MEMOP_F_INJECT_EXCEPTION |
5379 KVM_S390_MEMOP_F_CHECK_ONLY |
5380 KVM_S390_MEMOP_F_SKEY_PROTECTION);
5381 if (r)
5382 return r;
5383 if (mop->ar >= NUM_ACRS)
5384 return -EINVAL;
5385 if (kvm_s390_pv_cpu_is_protected(vcpu))
5386 return -EINVAL;
5387 if (!(mop->flags & KVM_S390_MEMOP_F_CHECK_ONLY)) {
5388 tmpbuf = vmalloc(mop->size);
5389 if (!tmpbuf)
5390 return -ENOMEM;
5391 }
5392
5393 acc_mode = mop->op == KVM_S390_MEMOP_LOGICAL_READ ? GACC_FETCH : GACC_STORE;
5394 if (mop->flags & KVM_S390_MEMOP_F_CHECK_ONLY) {
5395 r = check_gva_range(vcpu, mop->gaddr, mop->ar, mop->size,
5396 acc_mode, mop->key);
5397 } else if (acc_mode == GACC_FETCH) {
5398 r = read_guest_with_key(vcpu, mop->gaddr, mop->ar, tmpbuf,
5399 mop->size, mop->key);
5400 if (!r && copy_to_user(uaddr, tmpbuf, mop->size))
5401 return -EFAULT;
5402 } else {
5403 if (copy_from_user(tmpbuf, uaddr, mop->size))
5404 return -EFAULT;
5405 r = write_guest_with_key(vcpu, mop->gaddr, mop->ar, tmpbuf,
5406 mop->size, mop->key);
5407 }
5408
5409 if (r > 0 && (mop->flags & KVM_S390_MEMOP_F_INJECT_EXCEPTION) != 0)
5410 kvm_s390_inject_prog_irq(vcpu, &vcpu->arch.pgm);
5411
5412 return r;
5413 }
5414
kvm_s390_vcpu_memsida_op(struct kvm_vcpu * vcpu,struct kvm_s390_mem_op * mop)5415 static long kvm_s390_vcpu_memsida_op(struct kvm_vcpu *vcpu,
5416 struct kvm_s390_mem_op *mop)
5417 {
5418 int r, srcu_idx;
5419
5420 srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
5421
5422 switch (mop->op) {
5423 case KVM_S390_MEMOP_LOGICAL_READ:
5424 case KVM_S390_MEMOP_LOGICAL_WRITE:
5425 r = kvm_s390_vcpu_mem_op(vcpu, mop);
5426 break;
5427 case KVM_S390_MEMOP_SIDA_READ:
5428 case KVM_S390_MEMOP_SIDA_WRITE:
5429 /* we are locked against sida going away by the vcpu->mutex */
5430 r = kvm_s390_vcpu_sida_op(vcpu, mop);
5431 break;
5432 default:
5433 r = -EINVAL;
5434 }
5435
5436 srcu_read_unlock(&vcpu->kvm->srcu, srcu_idx);
5437 return r;
5438 }
5439
kvm_arch_vcpu_unlocked_ioctl(struct file * filp,unsigned int ioctl,unsigned long arg)5440 long kvm_arch_vcpu_unlocked_ioctl(struct file *filp, unsigned int ioctl,
5441 unsigned long arg)
5442 {
5443 struct kvm_vcpu *vcpu = filp->private_data;
5444 void __user *argp = (void __user *)arg;
5445 int rc;
5446
5447 switch (ioctl) {
5448 case KVM_S390_IRQ: {
5449 struct kvm_s390_irq s390irq;
5450
5451 if (copy_from_user(&s390irq, argp, sizeof(s390irq)))
5452 return -EFAULT;
5453 rc = kvm_s390_inject_vcpu(vcpu, &s390irq);
5454 break;
5455 }
5456 case KVM_S390_INTERRUPT: {
5457 struct kvm_s390_interrupt s390int;
5458 struct kvm_s390_irq s390irq = {};
5459
5460 if (kvm_is_ucontrol(vcpu->kvm))
5461 return -EINVAL;
5462 if (copy_from_user(&s390int, argp, sizeof(s390int)))
5463 return -EFAULT;
5464 if (s390int_to_s390irq(&s390int, &s390irq))
5465 return -EINVAL;
5466 rc = kvm_s390_inject_vcpu(vcpu, &s390irq);
5467 break;
5468 }
5469 default:
5470 rc = -ENOIOCTLCMD;
5471 break;
5472 }
5473
5474 /*
5475 * To simplify single stepping of userspace-emulated instructions,
5476 * KVM_EXIT_S390_SIEIC exit sets KVM_GUESTDBG_EXIT_PENDING (see
5477 * should_handle_per_ifetch()). However, if userspace emulation injects
5478 * an interrupt, it needs to be cleared, so that KVM_EXIT_DEBUG happens
5479 * after (and not before) the interrupt delivery.
5480 */
5481 if (!rc)
5482 vcpu->guest_debug &= ~KVM_GUESTDBG_EXIT_PENDING;
5483
5484 return rc;
5485 }
5486
kvm_s390_handle_pv_vcpu_dump(struct kvm_vcpu * vcpu,struct kvm_pv_cmd * cmd)5487 static int kvm_s390_handle_pv_vcpu_dump(struct kvm_vcpu *vcpu,
5488 struct kvm_pv_cmd *cmd)
5489 {
5490 struct kvm_s390_pv_dmp dmp;
5491 void *data;
5492 int ret;
5493
5494 /* Dump initialization is a prerequisite */
5495 if (!vcpu->kvm->arch.pv.dumping)
5496 return -EINVAL;
5497
5498 if (copy_from_user(&dmp, (__u8 __user *)cmd->data, sizeof(dmp)))
5499 return -EFAULT;
5500
5501 /* We only handle this subcmd right now */
5502 if (dmp.subcmd != KVM_PV_DUMP_CPU)
5503 return -EINVAL;
5504
5505 /* CPU dump length is the same as create cpu storage donation. */
5506 if (dmp.buff_len != uv_info.guest_cpu_stor_len)
5507 return -EINVAL;
5508
5509 data = kvzalloc(uv_info.guest_cpu_stor_len, GFP_KERNEL);
5510 if (!data)
5511 return -ENOMEM;
5512
5513 ret = kvm_s390_pv_dump_cpu(vcpu, data, &cmd->rc, &cmd->rrc);
5514
5515 VCPU_EVENT(vcpu, 3, "PROTVIRT DUMP CPU %d rc %x rrc %x",
5516 vcpu->vcpu_id, cmd->rc, cmd->rrc);
5517
5518 if (ret)
5519 ret = -EINVAL;
5520
5521 /* On success copy over the dump data */
5522 if (!ret && copy_to_user((__u8 __user *)dmp.buff_addr, data, uv_info.guest_cpu_stor_len))
5523 ret = -EFAULT;
5524
5525 kvfree(data);
5526 return ret;
5527 }
5528
kvm_arch_vcpu_ioctl(struct file * filp,unsigned int ioctl,unsigned long arg)5529 long kvm_arch_vcpu_ioctl(struct file *filp,
5530 unsigned int ioctl, unsigned long arg)
5531 {
5532 struct kvm_vcpu *vcpu = filp->private_data;
5533 void __user *argp = (void __user *)arg;
5534 int idx;
5535 long r;
5536 u16 rc, rrc;
5537
5538 /* Pairs with smp_store_release() in kvm_arch_vcpu_postcreate() */
5539 if (!smp_load_acquire(&vcpu->arch.initialized))
5540 return -EINVAL;
5541
5542 vcpu_load(vcpu);
5543
5544 switch (ioctl) {
5545 case KVM_S390_STORE_STATUS:
5546 idx = srcu_read_lock(&vcpu->kvm->srcu);
5547 r = kvm_s390_store_status_unloaded(vcpu, arg);
5548 srcu_read_unlock(&vcpu->kvm->srcu, idx);
5549 break;
5550 case KVM_S390_SET_INITIAL_PSW: {
5551 psw_t psw;
5552
5553 r = -EFAULT;
5554 if (copy_from_user(&psw, argp, sizeof(psw)))
5555 break;
5556 r = kvm_arch_vcpu_ioctl_set_initial_psw(vcpu, psw);
5557 break;
5558 }
5559 case KVM_S390_CLEAR_RESET:
5560 r = 0;
5561 kvm_arch_vcpu_ioctl_clear_reset(vcpu);
5562 if (kvm_s390_pv_cpu_is_protected(vcpu)) {
5563 r = uv_cmd_nodata(kvm_s390_pv_cpu_get_handle(vcpu),
5564 UVC_CMD_CPU_RESET_CLEAR, &rc, &rrc);
5565 VCPU_EVENT(vcpu, 3, "PROTVIRT RESET CLEAR VCPU: rc %x rrc %x",
5566 rc, rrc);
5567 }
5568 break;
5569 case KVM_S390_INITIAL_RESET:
5570 r = 0;
5571 kvm_arch_vcpu_ioctl_initial_reset(vcpu);
5572 if (kvm_s390_pv_cpu_is_protected(vcpu)) {
5573 r = uv_cmd_nodata(kvm_s390_pv_cpu_get_handle(vcpu),
5574 UVC_CMD_CPU_RESET_INITIAL,
5575 &rc, &rrc);
5576 VCPU_EVENT(vcpu, 3, "PROTVIRT RESET INITIAL VCPU: rc %x rrc %x",
5577 rc, rrc);
5578 }
5579 break;
5580 case KVM_S390_NORMAL_RESET:
5581 r = 0;
5582 kvm_arch_vcpu_ioctl_normal_reset(vcpu);
5583 if (kvm_s390_pv_cpu_is_protected(vcpu)) {
5584 r = uv_cmd_nodata(kvm_s390_pv_cpu_get_handle(vcpu),
5585 UVC_CMD_CPU_RESET, &rc, &rrc);
5586 VCPU_EVENT(vcpu, 3, "PROTVIRT RESET NORMAL VCPU: rc %x rrc %x",
5587 rc, rrc);
5588 }
5589 break;
5590 case KVM_SET_ONE_REG:
5591 case KVM_GET_ONE_REG: {
5592 struct kvm_one_reg reg;
5593 r = -EINVAL;
5594 if (kvm_s390_pv_cpu_is_protected(vcpu))
5595 break;
5596 r = -EFAULT;
5597 if (copy_from_user(®, argp, sizeof(reg)))
5598 break;
5599 if (ioctl == KVM_SET_ONE_REG)
5600 r = kvm_arch_vcpu_ioctl_set_one_reg(vcpu, ®);
5601 else
5602 r = kvm_arch_vcpu_ioctl_get_one_reg(vcpu, ®);
5603 break;
5604 }
5605 #ifdef CONFIG_KVM_S390_UCONTROL
5606 case KVM_S390_UCAS_MAP: {
5607 struct kvm_s390_ucas_mapping ucas;
5608
5609 r = -EFAULT;
5610 if (copy_from_user(&ucas, argp, sizeof(ucas)))
5611 break;
5612
5613 r = -EINVAL;
5614 if (!kvm_is_ucontrol(vcpu->kvm))
5615 break;
5616 if (!IS_ALIGNED(ucas.user_addr | ucas.vcpu_addr | ucas.length, _SEGMENT_SIZE))
5617 break;
5618
5619 r = gmap_ucas_map(vcpu->arch.gmap, gpa_to_gfn(ucas.user_addr),
5620 gpa_to_gfn(ucas.vcpu_addr),
5621 ucas.length >> _SEGMENT_SHIFT);
5622 break;
5623 }
5624 case KVM_S390_UCAS_UNMAP: {
5625 struct kvm_s390_ucas_mapping ucas;
5626
5627 r = -EFAULT;
5628 if (copy_from_user(&ucas, argp, sizeof(ucas)))
5629 break;
5630
5631 r = -EINVAL;
5632 if (!kvm_is_ucontrol(vcpu->kvm))
5633 break;
5634 if (!IS_ALIGNED(ucas.vcpu_addr | ucas.length, _SEGMENT_SIZE))
5635 break;
5636
5637 gmap_ucas_unmap(vcpu->arch.gmap, gpa_to_gfn(ucas.vcpu_addr),
5638 ucas.length >> _SEGMENT_SHIFT);
5639 r = 0;
5640 break;
5641 }
5642 #endif
5643 case KVM_S390_VCPU_FAULT: {
5644 gpa_t gaddr = arg;
5645
5646 scoped_guard(srcu, &vcpu->kvm->srcu) {
5647 r = vcpu_ucontrol_translate(vcpu, &gaddr);
5648 if (r)
5649 break;
5650
5651 r = kvm_s390_faultin_gfn_simple(vcpu, NULL, gpa_to_gfn(gaddr), false);
5652 if (r == PGM_ADDRESSING)
5653 r = -EFAULT;
5654 if (r <= 0)
5655 break;
5656 r = -EIO;
5657 KVM_BUG_ON(r, vcpu->kvm);
5658 }
5659 break;
5660 }
5661 case KVM_ENABLE_CAP:
5662 {
5663 struct kvm_enable_cap cap;
5664 r = -EFAULT;
5665 if (copy_from_user(&cap, argp, sizeof(cap)))
5666 break;
5667 r = kvm_vcpu_ioctl_enable_cap(vcpu, &cap);
5668 break;
5669 }
5670 case KVM_S390_MEM_OP: {
5671 struct kvm_s390_mem_op mem_op;
5672
5673 if (copy_from_user(&mem_op, argp, sizeof(mem_op)) == 0)
5674 r = kvm_s390_vcpu_memsida_op(vcpu, &mem_op);
5675 else
5676 r = -EFAULT;
5677 break;
5678 }
5679 case KVM_S390_SET_IRQ_STATE: {
5680 struct kvm_s390_irq_state irq_state;
5681
5682 r = -EFAULT;
5683 if (copy_from_user(&irq_state, argp, sizeof(irq_state)))
5684 break;
5685 if (irq_state.len > VCPU_IRQS_MAX_BUF ||
5686 irq_state.len == 0 ||
5687 irq_state.len % sizeof(struct kvm_s390_irq) > 0) {
5688 r = -EINVAL;
5689 break;
5690 }
5691 /* do not use irq_state.flags, it will break old QEMUs */
5692 r = kvm_s390_set_irq_state(vcpu,
5693 (void __user *) irq_state.buf,
5694 irq_state.len);
5695 break;
5696 }
5697 case KVM_S390_GET_IRQ_STATE: {
5698 struct kvm_s390_irq_state irq_state;
5699
5700 r = -EFAULT;
5701 if (copy_from_user(&irq_state, argp, sizeof(irq_state)))
5702 break;
5703 if (irq_state.len == 0) {
5704 r = -EINVAL;
5705 break;
5706 }
5707 /* do not use irq_state.flags, it will break old QEMUs */
5708 r = kvm_s390_get_irq_state(vcpu,
5709 (__u8 __user *) irq_state.buf,
5710 irq_state.len);
5711 break;
5712 }
5713 case KVM_S390_PV_CPU_COMMAND: {
5714 struct kvm_pv_cmd cmd;
5715
5716 r = -EINVAL;
5717 if (!is_prot_virt_host())
5718 break;
5719
5720 r = -EFAULT;
5721 if (copy_from_user(&cmd, argp, sizeof(cmd)))
5722 break;
5723
5724 r = -EINVAL;
5725 if (cmd.flags)
5726 break;
5727
5728 /* We only handle this cmd right now */
5729 if (cmd.cmd != KVM_PV_DUMP)
5730 break;
5731
5732 r = kvm_s390_handle_pv_vcpu_dump(vcpu, &cmd);
5733
5734 /* Always copy over UV rc / rrc data */
5735 if (copy_to_user((__u8 __user *)argp, &cmd.rc,
5736 sizeof(cmd.rc) + sizeof(cmd.rrc)))
5737 r = -EFAULT;
5738 break;
5739 }
5740 default:
5741 r = -ENOTTY;
5742 }
5743
5744 vcpu_put(vcpu);
5745 return r;
5746 }
5747
kvm_arch_vcpu_fault(struct kvm_vcpu * vcpu,struct vm_fault * vmf)5748 vm_fault_t kvm_arch_vcpu_fault(struct kvm_vcpu *vcpu, struct vm_fault *vmf)
5749 {
5750 #ifdef CONFIG_KVM_S390_UCONTROL
5751 if ((vmf->pgoff == KVM_S390_SIE_PAGE_OFFSET)
5752 && (kvm_is_ucontrol(vcpu->kvm))) {
5753 vmf->page = virt_to_page(vcpu->arch.sie_block);
5754 get_page(vmf->page);
5755 return 0;
5756 }
5757 #endif
5758 return VM_FAULT_SIGBUS;
5759 }
5760
kvm_arch_irqchip_in_kernel(struct kvm * kvm)5761 bool kvm_arch_irqchip_in_kernel(struct kvm *kvm)
5762 {
5763 return true;
5764 }
5765
5766 /* Section: memory related */
kvm_arch_prepare_memory_region(struct kvm * kvm,const struct kvm_memory_slot * old,struct kvm_memory_slot * new,enum kvm_mr_change change)5767 int kvm_arch_prepare_memory_region(struct kvm *kvm,
5768 const struct kvm_memory_slot *old,
5769 struct kvm_memory_slot *new,
5770 enum kvm_mr_change change)
5771 {
5772 if (kvm_is_ucontrol(kvm) && new && new->id < KVM_USER_MEM_SLOTS)
5773 return -EINVAL;
5774
5775 /* When we are protected, we should not change the memory slots */
5776 if (kvm_s390_pv_get_handle(kvm))
5777 return -EINVAL;
5778
5779 if (change != KVM_MR_DELETE && change != KVM_MR_FLAGS_ONLY) {
5780 /*
5781 * A few sanity checks. The memory in userland is ok to be
5782 * fragmented into various different vmas. It is okay to mmap()
5783 * and munmap() stuff in this slot after doing this call at any
5784 * time.
5785 */
5786 if (new->userspace_addr & ~PAGE_MASK)
5787 return -EINVAL;
5788 if ((new->base_gfn + new->npages) * PAGE_SIZE > kvm->arch.mem_limit)
5789 return -EINVAL;
5790 if (!asce_contains_gfn(kvm->arch.gmap->asce, new->base_gfn + new->npages - 1))
5791 return -EINVAL;
5792 }
5793
5794 if (!kvm->arch.migration_mode)
5795 return 0;
5796
5797 /*
5798 * Turn off migration mode when:
5799 * - userspace creates a new memslot with dirty logging off,
5800 * - userspace modifies an existing memslot (MOVE or FLAGS_ONLY) and
5801 * dirty logging is turned off.
5802 * Migration mode expects dirty page logging being enabled to store
5803 * its dirty bitmap.
5804 */
5805 if (change != KVM_MR_DELETE &&
5806 !(new->flags & KVM_MEM_LOG_DIRTY_PAGES))
5807 WARN(kvm_s390_vm_stop_migration(kvm),
5808 "Failed to stop migration mode");
5809
5810 return 0;
5811 }
5812
cmma_d_count_pte(union pte * ptep,gfn_t gfn,gfn_t next,struct dat_walk * walk)5813 static long cmma_d_count_pte(union pte *ptep, gfn_t gfn, gfn_t next, struct dat_walk *walk)
5814 {
5815 union pgste pgste;
5816
5817 pgste = pgste_get_lock(ptep);
5818 if (pgste.cmma_d) {
5819 pgste.cmma_d = 0;
5820 atomic64_dec(walk->priv);
5821 }
5822 pgste_set_unlock(ptep, pgste);
5823 return 0;
5824 }
5825
kvm_arch_commit_memory_region(struct kvm * kvm,struct kvm_memory_slot * old,const struct kvm_memory_slot * new,enum kvm_mr_change change)5826 void kvm_arch_commit_memory_region(struct kvm *kvm,
5827 struct kvm_memory_slot *old,
5828 const struct kvm_memory_slot *new,
5829 enum kvm_mr_change change)
5830 {
5831 const struct dat_walk_ops ops = { .pte_entry = cmma_d_count_pte, };
5832 struct kvm_s390_mmu_cache *mc __free(kvm_s390_mmu_cache) = NULL;
5833 int rc = 0;
5834
5835 guard(mutex)(&kvm->slots_arch_lock);
5836
5837 if (change == KVM_MR_FLAGS_ONLY)
5838 return;
5839
5840 mc = kvm_s390_new_mmu_cache();
5841 if (!mc) {
5842 rc = -ENOMEM;
5843 goto out;
5844 }
5845
5846 scoped_guard(write_lock, &kvm->mmu_lock) {
5847 if (kvm->arch.migration_mode && kvm->arch.use_cmma && old) {
5848 _dat_walk_gfn_range(old->base_gfn, old->base_gfn + old->npages,
5849 kvm->arch.gmap->asce, &ops, DAT_WALK_IGN_HOLES,
5850 &kvm->arch.cmma_dirty_pages);
5851 }
5852
5853 switch (change) {
5854 case KVM_MR_DELETE:
5855 rc = dat_delete_slot(mc, kvm->arch.gmap->asce, old->base_gfn, old->npages);
5856 break;
5857 case KVM_MR_MOVE:
5858 rc = dat_delete_slot(mc, kvm->arch.gmap->asce, old->base_gfn, old->npages);
5859 if (rc)
5860 break;
5861 fallthrough;
5862 case KVM_MR_CREATE:
5863 rc = dat_create_slot(mc, kvm->arch.gmap->asce, new->base_gfn, new->npages);
5864 break;
5865 case KVM_MR_FLAGS_ONLY:
5866 break;
5867 default:
5868 WARN(1, "Unknown KVM MR CHANGE: %d\n", change);
5869 }
5870 }
5871 out:
5872 if (rc)
5873 pr_warn("failed to commit memory region\n");
5874 return;
5875 }
5876
5877 /**
5878 * kvm_arch_vcpu_pre_fault_memory() -- pre-fault and link gmap dat tables
5879 * @vcpu: the vcpu that shall appear to have generated the fault-in.
5880 * @range: the range that needs to be faulted in.
5881 *
5882 * The first page of the given range is faulted in and the corresponding gmap
5883 * page tables are created, as if the given vCPU had performed a read
5884 * operation.
5885 * If the range starts outside any memslots, an error is returned. An error is
5886 * also returned for UCONTROL VMs, which should instead use the
5887 * KVM_S390_VCPU_FAULT ioctl.
5888 *
5889 * Return:
5890 * * %-ENOENT if the range lies outside of a memslot.
5891 * * %-EINVAL in case of invalid state (for example if the VM is UCONTROL).
5892 * * %-EIO if errors happen while faulting-in the page (will trigger a warning
5893 * in the caller).
5894 * * other error codes < 0 in case of other errors.
5895 * * otherwise a number > 0 of bytes that have been faulted in successfully.
5896 */
kvm_arch_vcpu_pre_fault_memory(struct kvm_vcpu * vcpu,struct kvm_pre_fault_memory * range)5897 long kvm_arch_vcpu_pre_fault_memory(struct kvm_vcpu *vcpu, struct kvm_pre_fault_memory *range)
5898 {
5899 struct guest_fault f = { .gfn = gpa_to_gfn(range->gpa), };
5900 gpa_t end;
5901 int rc;
5902
5903 if (kvm_is_ucontrol(vcpu->kvm))
5904 return -EINVAL;
5905
5906 rc = kvm_s390_faultin_gfn(vcpu, NULL, &f);
5907 if (rc == PGM_ADDRESSING)
5908 return -ENOENT;
5909 if (rc > 0)
5910 return -EIO;
5911 if (rc < 0)
5912 return rc;
5913
5914 if (f.ptep)
5915 return PAGE_SIZE;
5916
5917 end = ALIGN(range->gpa + PAGE_SIZE, f.crste_region3 ? _REGION3_SIZE : HPAGE_SIZE);
5918 return min(range->size, end - range->gpa);
5919 }
5920
5921 /**
5922 * kvm_test_age_gfn() - test young
5923 * @kvm: the kvm instance
5924 * @range: the range of guest addresses whose young status needs to be cleared
5925 *
5926 * Context: called by KVM common code without holding the kvm mmu lock
5927 * Return: true if any page in the given range is young, otherwise 0.
5928 */
kvm_test_age_gfn(struct kvm * kvm,struct kvm_gfn_range * range)5929 bool kvm_test_age_gfn(struct kvm *kvm, struct kvm_gfn_range *range)
5930 {
5931 scoped_guard(read_lock, &kvm->mmu_lock)
5932 return dat_test_age_gfn(kvm->arch.gmap->asce, range->start, range->end);
5933 }
5934
5935 /**
5936 * kvm_age_gfn() - clear young
5937 * @kvm: the kvm instance
5938 * @range: the range of guest addresses whose young status needs to be cleared
5939 *
5940 * Context: called by KVM common code without holding the kvm mmu lock
5941 * Return: true if any page in the given range was young, otherwise 0.
5942 */
kvm_age_gfn(struct kvm * kvm,struct kvm_gfn_range * range)5943 bool kvm_age_gfn(struct kvm *kvm, struct kvm_gfn_range *range)
5944 {
5945 scoped_guard(read_lock, &kvm->mmu_lock)
5946 return gmap_age_gfn(kvm->arch.gmap, range->start, range->end);
5947 }
5948
5949 /**
5950 * kvm_unmap_gfn_range() - Unmap a range of guest addresses
5951 * @kvm: the kvm instance
5952 * @range: the range of guest page frames to invalidate
5953 *
5954 * This function always returns false because every DAT table modification
5955 * has to use the appropriate DAT table manipulation instructions, which will
5956 * keep the TLB coherent, hence no additional TLB flush is ever required.
5957 *
5958 * Context: called by KVM common code with the kvm mmu write lock held
5959 * Return: false
5960 */
kvm_unmap_gfn_range(struct kvm * kvm,struct kvm_gfn_range * range)5961 bool kvm_unmap_gfn_range(struct kvm *kvm, struct kvm_gfn_range *range)
5962 {
5963 return gmap_unmap_gfn_range(kvm->arch.gmap, range->slot, range->start, range->end);
5964 }
5965
nonhyp_mask(int i)5966 static inline unsigned long nonhyp_mask(int i)
5967 {
5968 unsigned int nonhyp_fai = (sclp.hmfai << i * 2) >> 30;
5969
5970 return 0x0000ffffffffffffUL >> (nonhyp_fai << 4);
5971 }
5972
kvm_s390_init(void)5973 static int __init kvm_s390_init(void)
5974 {
5975 int i, r;
5976
5977 if (!sclp.has_sief2) {
5978 pr_info("SIE is not available\n");
5979 return -ENODEV;
5980 }
5981
5982 if (hpage_2g && !hpage) {
5983 hpage_2g = 0;
5984 pr_info("Disabling 2G hugepage support, since 1M hugepage support is not enabled.\n");
5985 }
5986
5987 for (i = 0; i < HMFAI_DWORDS; i++)
5988 kvm_s390_fac_base[i] |= nonhyp_mask(i);
5989
5990 r = __kvm_s390_init();
5991 if (r)
5992 return r;
5993
5994 r = kvm_init(sizeof(struct kvm_vcpu), 0, THIS_MODULE);
5995 if (r) {
5996 __kvm_s390_exit();
5997 return r;
5998 }
5999 return 0;
6000 }
6001
kvm_s390_exit(void)6002 static void __exit kvm_s390_exit(void)
6003 {
6004 kvm_exit();
6005
6006 __kvm_s390_exit();
6007 }
6008
6009 module_init(kvm_s390_init);
6010 module_exit(kvm_s390_exit);
6011
6012 /*
6013 * Enable autoloading of the kvm module.
6014 * Note that we add the module alias here instead of virt/kvm/kvm_main.c
6015 * since x86 takes a different approach.
6016 */
6017 #include <linux/miscdevice.h>
6018 MODULE_ALIAS_MISCDEV(KVM_MINOR);
6019 MODULE_ALIAS("devname:kvm");
6020