1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * hosting IBM Z kernel virtual machines (s390x)
4 *
5 * Copyright IBM Corp. 2008, 2020
6 *
7 * Author(s): Carsten Otte <cotte@de.ibm.com>
8 * Christian Borntraeger <borntraeger@de.ibm.com>
9 * Christian Ehrhardt <ehrhardt@de.ibm.com>
10 * Jason J. Herne <jjherne@us.ibm.com>
11 */
12
13 #define pr_fmt(fmt) "kvm-s390: " fmt
14
15 #include <linux/compiler.h>
16 #include <linux/entry-virt.h>
17 #include <linux/export.h>
18 #include <linux/err.h>
19 #include <linux/fs.h>
20 #include <linux/hrtimer.h>
21 #include <linux/init.h>
22 #include <linux/kvm.h>
23 #include <linux/kvm_host.h>
24 #include <linux/mman.h>
25 #include <linux/module.h>
26 #include <linux/moduleparam.h>
27 #include <linux/cpufeature.h>
28 #include <linux/random.h>
29 #include <linux/slab.h>
30 #include <linux/timer.h>
31 #include <linux/vmalloc.h>
32 #include <linux/bitmap.h>
33 #include <linux/sched/signal.h>
34 #include <linux/string.h>
35 #include <linux/pgtable.h>
36 #include <linux/mmu_notifier.h>
37
38 #include <asm/access-regs.h>
39 #include <asm/asm-offsets.h>
40 #include <asm/lowcore.h>
41 #include <asm/machine.h>
42 #include <asm/stp.h>
43 #include <asm/gmap_helpers.h>
44 #include <asm/nmi.h>
45 #include <asm/isc.h>
46 #include <asm/sclp.h>
47 #include <asm/cpacf.h>
48 #include <asm/timex.h>
49 #include <asm/asm.h>
50 #include <asm/fpu.h>
51 #include <asm/ap.h>
52 #include <asm/uv.h>
53 #include "s390.h"
54 #include "gaccess.h"
55 #include "gmap.h"
56 #include "faultin.h"
57 #include "pci.h"
58 #include "kvm_mmu.h"
59
60 #define CREATE_TRACE_POINTS
61 #include "trace.h"
62 #include "trace-s390.h"
63
64 #define MEM_OP_MAX_SIZE 65536 /* Maximum transfer size for KVM_S390_MEM_OP */
65 #define LOCAL_IRQS 32
66 #define VCPU_IRQS_MAX_BUF (sizeof(struct kvm_s390_irq) * \
67 (KVM_MAX_VCPUS + LOCAL_IRQS))
68
69 const struct kvm_stats_desc kvm_vm_stats_desc[] = {
70 KVM_GENERIC_VM_STATS(),
71 STATS_DESC_COUNTER(VM, inject_io),
72 STATS_DESC_COUNTER(VM, io_390_adapter_map),
73 STATS_DESC_COUNTER(VM, io_390_adapter_unmap),
74 STATS_DESC_COUNTER(VM, io_390_inatomic),
75 STATS_DESC_COUNTER(VM, io_flic_inject_airq),
76 STATS_DESC_COUNTER(VM, io_set_adapter_int),
77 STATS_DESC_COUNTER(VM, io_390_inatomic_no_inject),
78 STATS_DESC_COUNTER(VM, inject_float_mchk),
79 STATS_DESC_COUNTER(VM, inject_pfault_done),
80 STATS_DESC_COUNTER(VM, inject_service_signal),
81 STATS_DESC_COUNTER(VM, inject_virtio),
82 STATS_DESC_COUNTER(VM, aen_forward),
83 STATS_DESC_COUNTER(VM, gmap_shadow_reuse),
84 STATS_DESC_COUNTER(VM, gmap_shadow_create),
85 STATS_DESC_COUNTER(VM, gmap_shadow_r1_entry),
86 STATS_DESC_COUNTER(VM, gmap_shadow_r2_entry),
87 STATS_DESC_COUNTER(VM, gmap_shadow_r3_entry),
88 STATS_DESC_COUNTER(VM, gmap_shadow_sg_entry),
89 STATS_DESC_COUNTER(VM, gmap_shadow_pg_entry),
90 };
91
92 const struct kvm_stats_header kvm_vm_stats_header = {
93 .name_size = KVM_STATS_NAME_SIZE,
94 .num_desc = ARRAY_SIZE(kvm_vm_stats_desc),
95 .id_offset = sizeof(struct kvm_stats_header),
96 .desc_offset = sizeof(struct kvm_stats_header) + KVM_STATS_NAME_SIZE,
97 .data_offset = sizeof(struct kvm_stats_header) + KVM_STATS_NAME_SIZE +
98 sizeof(kvm_vm_stats_desc),
99 };
100
101 const struct kvm_stats_desc kvm_vcpu_stats_desc[] = {
102 KVM_GENERIC_VCPU_STATS(),
103 STATS_DESC_COUNTER(VCPU, exit_userspace),
104 STATS_DESC_COUNTER(VCPU, exit_null),
105 STATS_DESC_COUNTER(VCPU, exit_external_request),
106 STATS_DESC_COUNTER(VCPU, exit_io_request),
107 STATS_DESC_COUNTER(VCPU, exit_external_interrupt),
108 STATS_DESC_COUNTER(VCPU, exit_stop_request),
109 STATS_DESC_COUNTER(VCPU, exit_validity),
110 STATS_DESC_COUNTER(VCPU, exit_instruction),
111 STATS_DESC_COUNTER(VCPU, exit_pei),
112 STATS_DESC_COUNTER(VCPU, halt_no_poll_steal),
113 STATS_DESC_COUNTER(VCPU, instruction_lctl),
114 STATS_DESC_COUNTER(VCPU, instruction_lctlg),
115 STATS_DESC_COUNTER(VCPU, instruction_stctl),
116 STATS_DESC_COUNTER(VCPU, instruction_stctg),
117 STATS_DESC_COUNTER(VCPU, exit_program_interruption),
118 STATS_DESC_COUNTER(VCPU, exit_instr_and_program),
119 STATS_DESC_COUNTER(VCPU, exit_operation_exception),
120 STATS_DESC_COUNTER(VCPU, deliver_ckc),
121 STATS_DESC_COUNTER(VCPU, deliver_cputm),
122 STATS_DESC_COUNTER(VCPU, deliver_external_call),
123 STATS_DESC_COUNTER(VCPU, deliver_emergency_signal),
124 STATS_DESC_COUNTER(VCPU, deliver_service_signal),
125 STATS_DESC_COUNTER(VCPU, deliver_virtio),
126 STATS_DESC_COUNTER(VCPU, deliver_stop_signal),
127 STATS_DESC_COUNTER(VCPU, deliver_prefix_signal),
128 STATS_DESC_COUNTER(VCPU, deliver_restart_signal),
129 STATS_DESC_COUNTER(VCPU, deliver_program),
130 STATS_DESC_COUNTER(VCPU, deliver_io),
131 STATS_DESC_COUNTER(VCPU, deliver_machine_check),
132 STATS_DESC_COUNTER(VCPU, exit_wait_state),
133 STATS_DESC_COUNTER(VCPU, inject_ckc),
134 STATS_DESC_COUNTER(VCPU, inject_cputm),
135 STATS_DESC_COUNTER(VCPU, inject_external_call),
136 STATS_DESC_COUNTER(VCPU, inject_emergency_signal),
137 STATS_DESC_COUNTER(VCPU, inject_mchk),
138 STATS_DESC_COUNTER(VCPU, inject_pfault_init),
139 STATS_DESC_COUNTER(VCPU, inject_program),
140 STATS_DESC_COUNTER(VCPU, inject_restart),
141 STATS_DESC_COUNTER(VCPU, inject_set_prefix),
142 STATS_DESC_COUNTER(VCPU, inject_stop_signal),
143 STATS_DESC_COUNTER(VCPU, instruction_epsw),
144 STATS_DESC_COUNTER(VCPU, instruction_gs),
145 STATS_DESC_COUNTER(VCPU, instruction_io_other),
146 STATS_DESC_COUNTER(VCPU, instruction_lpsw),
147 STATS_DESC_COUNTER(VCPU, instruction_lpswe),
148 STATS_DESC_COUNTER(VCPU, instruction_lpswey),
149 STATS_DESC_COUNTER(VCPU, instruction_pfmf),
150 STATS_DESC_COUNTER(VCPU, instruction_ptff),
151 STATS_DESC_COUNTER(VCPU, instruction_sck),
152 STATS_DESC_COUNTER(VCPU, instruction_sckpf),
153 STATS_DESC_COUNTER(VCPU, instruction_stidp),
154 STATS_DESC_COUNTER(VCPU, instruction_spx),
155 STATS_DESC_COUNTER(VCPU, instruction_stpx),
156 STATS_DESC_COUNTER(VCPU, instruction_stap),
157 STATS_DESC_COUNTER(VCPU, instruction_iske),
158 STATS_DESC_COUNTER(VCPU, instruction_ri),
159 STATS_DESC_COUNTER(VCPU, instruction_rrbe),
160 STATS_DESC_COUNTER(VCPU, instruction_sske),
161 STATS_DESC_COUNTER(VCPU, instruction_ipte_interlock),
162 STATS_DESC_COUNTER(VCPU, instruction_stsi),
163 STATS_DESC_COUNTER(VCPU, instruction_stfl),
164 STATS_DESC_COUNTER(VCPU, instruction_tb),
165 STATS_DESC_COUNTER(VCPU, instruction_tpi),
166 STATS_DESC_COUNTER(VCPU, instruction_tprot),
167 STATS_DESC_COUNTER(VCPU, instruction_tsch),
168 STATS_DESC_COUNTER(VCPU, instruction_sie),
169 STATS_DESC_COUNTER(VCPU, instruction_essa),
170 STATS_DESC_COUNTER(VCPU, instruction_sthyi),
171 STATS_DESC_COUNTER(VCPU, instruction_sigp_sense),
172 STATS_DESC_COUNTER(VCPU, instruction_sigp_sense_running),
173 STATS_DESC_COUNTER(VCPU, instruction_sigp_external_call),
174 STATS_DESC_COUNTER(VCPU, instruction_sigp_emergency),
175 STATS_DESC_COUNTER(VCPU, instruction_sigp_cond_emergency),
176 STATS_DESC_COUNTER(VCPU, instruction_sigp_start),
177 STATS_DESC_COUNTER(VCPU, instruction_sigp_stop),
178 STATS_DESC_COUNTER(VCPU, instruction_sigp_stop_store_status),
179 STATS_DESC_COUNTER(VCPU, instruction_sigp_store_status),
180 STATS_DESC_COUNTER(VCPU, instruction_sigp_store_adtl_status),
181 STATS_DESC_COUNTER(VCPU, instruction_sigp_arch),
182 STATS_DESC_COUNTER(VCPU, instruction_sigp_prefix),
183 STATS_DESC_COUNTER(VCPU, instruction_sigp_restart),
184 STATS_DESC_COUNTER(VCPU, instruction_sigp_init_cpu_reset),
185 STATS_DESC_COUNTER(VCPU, instruction_sigp_cpu_reset),
186 STATS_DESC_COUNTER(VCPU, instruction_sigp_unknown),
187 STATS_DESC_COUNTER(VCPU, instruction_diagnose_10),
188 STATS_DESC_COUNTER(VCPU, instruction_diagnose_44),
189 STATS_DESC_COUNTER(VCPU, instruction_diagnose_9c),
190 STATS_DESC_COUNTER(VCPU, diag_9c_ignored),
191 STATS_DESC_COUNTER(VCPU, diag_9c_forward),
192 STATS_DESC_COUNTER(VCPU, instruction_diagnose_258),
193 STATS_DESC_COUNTER(VCPU, instruction_diagnose_308),
194 STATS_DESC_COUNTER(VCPU, instruction_diagnose_500),
195 STATS_DESC_COUNTER(VCPU, instruction_diagnose_other),
196 STATS_DESC_COUNTER(VCPU, pfault_sync),
197 STATS_DESC_COUNTER(VCPU, signal_exits)
198 };
199
200 const struct kvm_stats_header kvm_vcpu_stats_header = {
201 .name_size = KVM_STATS_NAME_SIZE,
202 .num_desc = ARRAY_SIZE(kvm_vcpu_stats_desc),
203 .id_offset = sizeof(struct kvm_stats_header),
204 .desc_offset = sizeof(struct kvm_stats_header) + KVM_STATS_NAME_SIZE,
205 .data_offset = sizeof(struct kvm_stats_header) + KVM_STATS_NAME_SIZE +
206 sizeof(kvm_vcpu_stats_desc),
207 };
208
209 /* allow nested virtualization in KVM (if enabled by user space) */
210 static int nested;
211 module_param(nested, int, S_IRUGO);
212 MODULE_PARM_DESC(nested, "Nested virtualization support");
213
214 /* allow 1m huge page guest backing */
215 static int hpage;
216 module_param(hpage, int, 0444);
217 MODULE_PARM_DESC(hpage, "1m huge page backing support");
218
219 /* allow 2g huge page guest backing */
220 static int hpage_2g;
221 module_param(hpage_2g, int, 0444);
222 MODULE_PARM_DESC(hpage_2g, "2g huge page backing support");
223
224 /* maximum percentage of steal time for polling. >100 is treated like 100 */
225 static u8 halt_poll_max_steal = 10;
226 module_param(halt_poll_max_steal, byte, 0644);
227 MODULE_PARM_DESC(halt_poll_max_steal, "Maximum percentage of steal time to allow polling");
228
229 /* if set to true, the GISA will be initialized and used if available */
230 static bool use_gisa = true;
231 module_param(use_gisa, bool, 0644);
232 MODULE_PARM_DESC(use_gisa, "Use the GISA if the host supports it.");
233
234 /* maximum diag9c forwarding per second */
235 unsigned int diag9c_forwarding_hz;
236 module_param(diag9c_forwarding_hz, uint, 0644);
237 MODULE_PARM_DESC(diag9c_forwarding_hz, "Maximum diag9c forwarding per second, 0 to turn off");
238
239 /*
240 * allow asynchronous deinit for protected guests; enable by default since
241 * the feature is opt-in anyway
242 */
243 static int async_destroy = 1;
244 module_param(async_destroy, int, 0444);
245 MODULE_PARM_DESC(async_destroy, "Asynchronous destroy for protected guests");
246
247 #define HMFAI_DWORDS 16
248 /*
249 * Base feature mask that defines default mask for facilities. Consists of the
250 * defines in FACILITIES_KVM and the non-hypervisor managed bits.
251 */
252 static unsigned long kvm_s390_fac_base[HMFAI_DWORDS] = { FACILITIES_KVM };
253 static_assert(ARRAY_SIZE(((long[]){ FACILITIES_KVM })) <= HMFAI_DWORDS);
254 static_assert(ARRAY_SIZE(kvm_s390_fac_base) <= S390_ARCH_FAC_MASK_SIZE_U64);
255 static_assert(ARRAY_SIZE(kvm_s390_fac_base) <= S390_ARCH_FAC_LIST_SIZE_U64);
256 static_assert(ARRAY_SIZE(kvm_s390_fac_base) <= ARRAY_SIZE(stfle_fac_list));
257
258 /*
259 * Extended feature mask. Consists of the defines in FACILITIES_KVM_CPUMODEL
260 * and defines the facilities that can be enabled via a cpu model.
261 */
262 static const unsigned long kvm_s390_fac_ext[] = { FACILITIES_KVM_CPUMODEL };
263 static_assert(ARRAY_SIZE(kvm_s390_fac_ext) <= S390_ARCH_FAC_MASK_SIZE_U64);
264 static_assert(ARRAY_SIZE(kvm_s390_fac_ext) <= S390_ARCH_FAC_LIST_SIZE_U64);
265 static_assert(ARRAY_SIZE(kvm_s390_fac_ext) <= ARRAY_SIZE(stfle_fac_list));
266
267 /* available cpu features supported by kvm */
268 static DECLARE_BITMAP(kvm_s390_available_cpu_feat, KVM_S390_VM_CPU_FEAT_NR_BITS);
269 /* available subfunctions indicated via query / "test bit" */
270 static struct kvm_s390_vm_cpu_subfunc kvm_s390_available_subfunc;
271
272 debug_info_t *kvm_s390_dbf;
273 debug_info_t *kvm_s390_dbf_uv;
274
275 /* Section: not file related */
276 /* forward declarations */
kvm_clock_sync_scb(struct kvm_s390_sie_block * scb,u64 delta)277 static void kvm_clock_sync_scb(struct kvm_s390_sie_block *scb, u64 delta)
278 {
279 u8 delta_idx = 0;
280
281 /*
282 * The TOD jumps by delta, we have to compensate this by adding
283 * -delta to the epoch.
284 */
285 delta = -delta;
286
287 /* sign-extension - we're adding to signed values below */
288 if ((s64)delta < 0)
289 delta_idx = -1;
290
291 scb->epoch += delta;
292 if (scb->ecd & ECD_MEF) {
293 scb->epdx += delta_idx;
294 if (scb->epoch < delta)
295 scb->epdx += 1;
296 }
297 }
298
299 /*
300 * This callback is executed during stop_machine(). All CPUs are therefore
301 * temporarily stopped. In order not to change guest behavior, we have to
302 * disable preemption whenever we touch the epoch of kvm and the VCPUs,
303 * so a CPU won't be stopped while calculating with the epoch.
304 */
kvm_clock_sync(struct notifier_block * notifier,unsigned long val,void * v)305 static int kvm_clock_sync(struct notifier_block *notifier, unsigned long val,
306 void *v)
307 {
308 struct kvm *kvm;
309 struct kvm_vcpu *vcpu;
310 unsigned long i;
311 unsigned long long *delta = v;
312
313 list_for_each_entry(kvm, &vm_list, vm_list) {
314 kvm_for_each_vcpu(i, vcpu, kvm) {
315 kvm_clock_sync_scb(vcpu->arch.sie_block, *delta);
316 if (i == 0) {
317 kvm->arch.epoch = vcpu->arch.sie_block->epoch;
318 kvm->arch.epdx = vcpu->arch.sie_block->epdx;
319 }
320 if (vcpu->arch.cputm_enabled)
321 vcpu->arch.cputm_start += *delta;
322 if (vcpu->arch.vsie_block)
323 kvm_clock_sync_scb(vcpu->arch.vsie_block,
324 *delta);
325 }
326 }
327 return NOTIFY_OK;
328 }
329
330 static struct notifier_block kvm_clock_notifier = {
331 .notifier_call = kvm_clock_sync,
332 };
333
allow_cpu_feat(unsigned long nr)334 static void allow_cpu_feat(unsigned long nr)
335 {
336 set_bit_inv(nr, kvm_s390_available_cpu_feat);
337 }
338
plo_test_bit(unsigned char nr)339 static inline int plo_test_bit(unsigned char nr)
340 {
341 unsigned long function = (unsigned long)nr | 0x100;
342 int cc;
343
344 asm volatile(
345 " lgr 0,%[function]\n"
346 /* Parameter registers are ignored for "test bit" */
347 " plo 0,0,0,0(0)\n"
348 CC_IPM(cc)
349 : CC_OUT(cc, cc)
350 : [function] "d" (function)
351 : CC_CLOBBER_LIST("0"));
352 return CC_TRANSFORM(cc) == 0;
353 }
354
pfcr_query(u8 (* query)[16])355 static __always_inline void pfcr_query(u8 (*query)[16])
356 {
357 asm volatile(
358 " lghi 0,0\n"
359 " .insn rsy,0xeb0000000016,0,0,%[query]"
360 : [query] "=QS" (*query)
361 :
362 : "cc", "0");
363 }
364
__sortl_query(u8 (* query)[32])365 static __always_inline void __sortl_query(u8 (*query)[32])
366 {
367 asm volatile(
368 " lghi 0,0\n"
369 " la 1,%[query]\n"
370 /* Parameter registers are ignored */
371 " .insn rre,0xb9380000,2,4"
372 : [query] "=R" (*query)
373 :
374 : "cc", "0", "1");
375 }
376
__dfltcc_query(u8 (* query)[32])377 static __always_inline void __dfltcc_query(u8 (*query)[32])
378 {
379 asm volatile(
380 " lghi 0,0\n"
381 " la 1,%[query]\n"
382 /* Parameter registers are ignored */
383 " .insn rrf,0xb9390000,2,4,6,0"
384 : [query] "=R" (*query)
385 :
386 : "cc", "0", "1");
387 }
388
kvm_s390_cpu_feat_init(void)389 static void __init kvm_s390_cpu_feat_init(void)
390 {
391 int i;
392
393 for (i = 0; i < 256; ++i) {
394 if (plo_test_bit(i))
395 kvm_s390_available_subfunc.plo[i >> 3] |= 0x80 >> (i & 7);
396 }
397
398 if (test_facility(28)) /* TOD-clock steering */
399 ptff(kvm_s390_available_subfunc.ptff,
400 sizeof(kvm_s390_available_subfunc.ptff),
401 PTFF_QAF);
402
403 if (test_facility(17)) { /* MSA */
404 __cpacf_query(CPACF_KMAC, (cpacf_mask_t *)
405 kvm_s390_available_subfunc.kmac);
406 __cpacf_query(CPACF_KMC, (cpacf_mask_t *)
407 kvm_s390_available_subfunc.kmc);
408 __cpacf_query(CPACF_KM, (cpacf_mask_t *)
409 kvm_s390_available_subfunc.km);
410 __cpacf_query(CPACF_KIMD, (cpacf_mask_t *)
411 kvm_s390_available_subfunc.kimd);
412 __cpacf_query(CPACF_KLMD, (cpacf_mask_t *)
413 kvm_s390_available_subfunc.klmd);
414 }
415 if (test_facility(76)) /* MSA3 */
416 __cpacf_query(CPACF_PCKMO, (cpacf_mask_t *)
417 kvm_s390_available_subfunc.pckmo);
418 if (test_facility(77)) { /* MSA4 */
419 __cpacf_query(CPACF_KMCTR, (cpacf_mask_t *)
420 kvm_s390_available_subfunc.kmctr);
421 __cpacf_query(CPACF_KMF, (cpacf_mask_t *)
422 kvm_s390_available_subfunc.kmf);
423 __cpacf_query(CPACF_KMO, (cpacf_mask_t *)
424 kvm_s390_available_subfunc.kmo);
425 __cpacf_query(CPACF_PCC, (cpacf_mask_t *)
426 kvm_s390_available_subfunc.pcc);
427 }
428 if (test_facility(57)) /* MSA5 */
429 __cpacf_query(CPACF_PRNO, (cpacf_mask_t *)
430 kvm_s390_available_subfunc.ppno);
431
432 if (test_facility(146)) /* MSA8 */
433 __cpacf_query(CPACF_KMA, (cpacf_mask_t *)
434 kvm_s390_available_subfunc.kma);
435
436 if (test_facility(155)) /* MSA9 */
437 __cpacf_query(CPACF_KDSA, (cpacf_mask_t *)
438 kvm_s390_available_subfunc.kdsa);
439
440 if (test_facility(150)) /* SORTL */
441 __sortl_query(&kvm_s390_available_subfunc.sortl);
442
443 if (test_facility(151)) /* DFLTCC */
444 __dfltcc_query(&kvm_s390_available_subfunc.dfltcc);
445
446 if (test_facility(201)) /* PFCR */
447 pfcr_query(&kvm_s390_available_subfunc.pfcr);
448
449 if (machine_has_esop())
450 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_ESOP);
451 /*
452 * We need SIE support, ESOP (PROT_READ protection for gmap_shadow),
453 * 64bit SCAO (SCA passthrough) and IDTE (for gmap_shadow unshadowing).
454 */
455 if (!sclp.has_sief2 || !machine_has_esop() || !sclp.has_64bscao ||
456 !test_facility(3) || !nested)
457 return;
458 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_SIEF2);
459 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_64BSCAO);
460 if (sclp.has_siif)
461 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_SIIF);
462 if (sclp.has_gpere)
463 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_GPERE);
464 if (sclp.has_gsls)
465 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_GSLS);
466 if (sclp.has_ib)
467 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_IB);
468 if (sclp.has_cei)
469 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_CEI);
470 if (sclp.has_ibs)
471 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_IBS);
472 if (sclp.has_kss)
473 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_KSS);
474 if (sclp.has_astfleie2)
475 allow_cpu_feat(KVM_S390_VM_CPU_FEAT_ASTFLEIE2);
476 /*
477 * KVM_S390_VM_CPU_FEAT_SKEY: Wrong shadow of PTE.I bits will make
478 * all skey handling functions read/set the skey from the PGSTE
479 * instead of the real storage key.
480 *
481 * KVM_S390_VM_CPU_FEAT_CMMA: Wrong shadow of PTE.I bits will make
482 * pages being detected as preserved although they are resident.
483 *
484 * KVM_S390_VM_CPU_FEAT_PFMFI: Wrong shadow of PTE.I bits will
485 * have the same effect as for KVM_S390_VM_CPU_FEAT_SKEY.
486 *
487 * For KVM_S390_VM_CPU_FEAT_SKEY, KVM_S390_VM_CPU_FEAT_CMMA and
488 * KVM_S390_VM_CPU_FEAT_PFMFI, all PTE.I and PGSTE bits have to be
489 * correctly shadowed. We can do that for the PGSTE but not for PTE.I.
490 *
491 * KVM_S390_VM_CPU_FEAT_SIGPIF: Wrong SCB addresses in the SCA. We
492 * cannot easily shadow the SCA because of the ipte lock.
493 */
494 }
495
__kvm_s390_init(void)496 static int __init __kvm_s390_init(void)
497 {
498 int rc = -ENOMEM;
499
500 kvm_s390_dbf = debug_register("kvm-trace", 32, 1, 7 * sizeof(long));
501 if (!kvm_s390_dbf)
502 return -ENOMEM;
503
504 kvm_s390_dbf_uv = debug_register("kvm-uv", 32, 1, 7 * sizeof(long));
505 if (!kvm_s390_dbf_uv)
506 goto err_kvm_uv;
507
508 if (debug_register_view(kvm_s390_dbf, &debug_sprintf_view) ||
509 debug_register_view(kvm_s390_dbf_uv, &debug_sprintf_view))
510 goto err_debug_view;
511
512 kvm_s390_cpu_feat_init();
513
514 /* Register floating interrupt controller interface. */
515 rc = kvm_register_device_ops(&kvm_flic_ops, KVM_DEV_TYPE_FLIC);
516 if (rc) {
517 pr_err("A FLIC registration call failed with rc=%d\n", rc);
518 goto err_flic;
519 }
520
521 if (IS_ENABLED(CONFIG_VFIO_PCI_ZDEV_KVM)) {
522 rc = kvm_s390_pci_init();
523 if (rc) {
524 pr_err("Unable to allocate AIFT for PCI\n");
525 goto err_pci;
526 }
527 }
528
529 rc = kvm_s390_gib_init(GAL_ISC);
530 if (rc)
531 goto err_gib;
532
533 atomic_notifier_chain_register(&s390_epoch_delta_notifier,
534 &kvm_clock_notifier);
535
536 return 0;
537
538 err_gib:
539 if (IS_ENABLED(CONFIG_VFIO_PCI_ZDEV_KVM))
540 kvm_s390_pci_exit();
541 err_pci:
542 err_flic:
543 err_debug_view:
544 debug_unregister(kvm_s390_dbf_uv);
545 err_kvm_uv:
546 debug_unregister(kvm_s390_dbf);
547 return rc;
548 }
549
__kvm_s390_exit(void)550 static void __kvm_s390_exit(void)
551 {
552 atomic_notifier_chain_unregister(&s390_epoch_delta_notifier,
553 &kvm_clock_notifier);
554
555 kvm_s390_gib_destroy();
556 if (IS_ENABLED(CONFIG_VFIO_PCI_ZDEV_KVM))
557 kvm_s390_pci_exit();
558 debug_unregister(kvm_s390_dbf);
559 debug_unregister(kvm_s390_dbf_uv);
560 }
561
kvm_s390_keyop(struct kvm_s390_mmu_cache * mc,struct kvm * kvm,int op,unsigned long addr,union skey skey)562 static int kvm_s390_keyop(struct kvm_s390_mmu_cache *mc, struct kvm *kvm, int op,
563 unsigned long addr, union skey skey)
564 {
565 gfn_t gfn = gpa_to_gfn(addr);
566 union asce asce;
567 int r;
568
569 guard(read_lock)(&kvm->mmu_lock);
570 asce = kvm->arch.gmap->asce;
571
572 switch (op) {
573 case KVM_S390_KEYOP_SSKE:
574 r = dat_cond_set_storage_key(mc, asce, gfn, skey, &skey, 0, 0, 0);
575 if (r == 0 || r == 1)
576 return skey.skey;
577 break;
578 case KVM_S390_KEYOP_ISKE:
579 r = dat_get_storage_key(asce, gfn, &skey);
580 if (!r)
581 return skey.skey;
582 break;
583 case KVM_S390_KEYOP_RRBE:
584 r = dat_reset_reference_bit(asce, gfn, &skey);
585 if (!r)
586 return skey.skey;
587 break;
588 default:
589 return -EINVAL;
590 }
591 return r > 0 ? -EFAULT : r;
592 }
593
594 /* Section: device related */
kvm_arch_dev_ioctl(struct file * filp,unsigned int ioctl,unsigned long arg)595 long kvm_arch_dev_ioctl(struct file *filp,
596 unsigned int ioctl, unsigned long arg)
597 {
598 if (ioctl == KVM_S390_ENABLE_SIE)
599 return 0;
600 return -EINVAL;
601 }
602
kvm_vm_ioctl_check_extension(struct kvm * kvm,long ext)603 int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext)
604 {
605 int r;
606
607 switch (ext) {
608 case KVM_CAP_S390_PSW:
609 case KVM_CAP_S390_GMAP:
610 #ifdef CONFIG_KVM_S390_UCONTROL
611 case KVM_CAP_S390_UCONTROL:
612 #endif
613 case KVM_CAP_ASYNC_PF:
614 case KVM_CAP_SYNC_REGS:
615 case KVM_CAP_ONE_REG:
616 case KVM_CAP_ENABLE_CAP:
617 case KVM_CAP_S390_CSS_SUPPORT:
618 case KVM_CAP_IOEVENTFD:
619 case KVM_CAP_S390_IRQCHIP:
620 case KVM_CAP_VM_ATTRIBUTES:
621 case KVM_CAP_MP_STATE:
622 case KVM_CAP_IMMEDIATE_EXIT:
623 case KVM_CAP_S390_INJECT_IRQ:
624 case KVM_CAP_S390_USER_SIGP:
625 case KVM_CAP_S390_USER_STSI:
626 case KVM_CAP_S390_SKEYS:
627 case KVM_CAP_S390_IRQ_STATE:
628 case KVM_CAP_S390_USER_INSTR0:
629 case KVM_CAP_S390_CMMA_MIGRATION:
630 case KVM_CAP_S390_AIS:
631 case KVM_CAP_S390_AIS_MIGRATION:
632 case KVM_CAP_S390_VCPU_RESETS:
633 case KVM_CAP_SET_GUEST_DEBUG:
634 case KVM_CAP_S390_DIAG318:
635 case KVM_CAP_IRQFD_RESAMPLE:
636 case KVM_CAP_S390_USER_OPEREXEC:
637 case KVM_CAP_S390_KEYOP:
638 case KVM_CAP_S390_VSIE_ESAMODE:
639 case KVM_CAP_PRE_FAULT_MEMORY:
640 r = 1;
641 break;
642 case KVM_CAP_SET_GUEST_DEBUG2:
643 r = KVM_GUESTDBG_VALID_MASK;
644 break;
645 case KVM_CAP_S390_HPAGE_1M:
646 r = 0;
647 if (hpage && !(kvm && kvm_is_ucontrol(kvm)))
648 r = 1;
649 break;
650 case KVM_CAP_S390_HPAGE_2G:
651 r = 0;
652 if (hpage_2g && !(kvm && kvm_is_ucontrol(kvm)))
653 r = 1;
654 break;
655 case KVM_CAP_S390_MEM_OP:
656 r = MEM_OP_MAX_SIZE;
657 break;
658 case KVM_CAP_S390_MEM_OP_EXTENSION:
659 /*
660 * Flag bits indicating which extensions are supported.
661 * If r > 0, the base extension must also be supported/indicated,
662 * in order to maintain backwards compatibility.
663 */
664 r = KVM_S390_MEMOP_EXTENSION_CAP_BASE |
665 KVM_S390_MEMOP_EXTENSION_CAP_CMPXCHG;
666 break;
667 case KVM_CAP_NR_VCPUS:
668 case KVM_CAP_MAX_VCPUS:
669 case KVM_CAP_MAX_VCPU_ID:
670 /*
671 * Return the same value for KVM_CAP_MAX_VCPUS and
672 * KVM_CAP_MAX_VCPU_ID to conform with the KVM API.
673 */
674 r = KVM_S390_ESCA_CPU_SLOTS;
675 if (!kvm_s390_use_sca_entries())
676 r = KVM_MAX_VCPUS;
677 if (ext == KVM_CAP_NR_VCPUS)
678 r = min_t(unsigned int, num_online_cpus(), r);
679 break;
680 case KVM_CAP_S390_COW:
681 r = machine_has_esop();
682 break;
683 case KVM_CAP_S390_VECTOR_REGISTERS:
684 r = test_facility(129);
685 break;
686 case KVM_CAP_S390_RI:
687 r = test_facility(64);
688 break;
689 case KVM_CAP_S390_GS:
690 r = test_facility(133);
691 break;
692 case KVM_CAP_S390_BPB:
693 r = test_facility(82);
694 break;
695 case KVM_CAP_S390_PROTECTED_ASYNC_DISABLE:
696 r = async_destroy && is_prot_virt_host();
697 break;
698 case KVM_CAP_S390_PROTECTED:
699 r = is_prot_virt_host();
700 break;
701 case KVM_CAP_S390_PROTECTED_DUMP: {
702 u64 pv_cmds_dump[] = {
703 BIT_UVC_CMD_DUMP_INIT,
704 BIT_UVC_CMD_DUMP_CONFIG_STOR_STATE,
705 BIT_UVC_CMD_DUMP_CPU,
706 BIT_UVC_CMD_DUMP_COMPLETE,
707 };
708 int i;
709
710 r = is_prot_virt_host();
711
712 for (i = 0; i < ARRAY_SIZE(pv_cmds_dump); i++) {
713 if (!test_bit_inv(pv_cmds_dump[i],
714 (unsigned long *)&uv_info.inst_calls_list)) {
715 r = 0;
716 break;
717 }
718 }
719 break;
720 }
721 case KVM_CAP_S390_ZPCI_OP:
722 r = kvm_s390_pci_interp_allowed();
723 break;
724 case KVM_CAP_S390_CPU_TOPOLOGY:
725 r = test_facility(11);
726 break;
727 default:
728 r = 0;
729 }
730 return r;
731 }
732
kvm_arch_sync_dirty_log(struct kvm * kvm,struct kvm_memory_slot * memslot)733 void kvm_arch_sync_dirty_log(struct kvm *kvm, struct kvm_memory_slot *memslot)
734 {
735 gfn_t last_gfn = memslot->base_gfn + memslot->npages;
736
737 scoped_guard(read_lock, &kvm->mmu_lock)
738 gmap_sync_dirty_log(kvm->arch.gmap, memslot->base_gfn, last_gfn);
739 }
740
741 /* Section: vm related */
742 static void sca_del_vcpu(struct kvm_vcpu *vcpu);
743
744 /*
745 * Get (and clear) the dirty memory log for a memory slot.
746 */
kvm_vm_ioctl_get_dirty_log(struct kvm * kvm,struct kvm_dirty_log * log)747 int kvm_vm_ioctl_get_dirty_log(struct kvm *kvm,
748 struct kvm_dirty_log *log)
749 {
750 return s390_kvm_mmu_get_dirty_log(kvm, log);
751 }
752
icpt_operexc_on_all_vcpus(struct kvm * kvm)753 static void icpt_operexc_on_all_vcpus(struct kvm *kvm)
754 {
755 unsigned long i;
756 struct kvm_vcpu *vcpu;
757
758 kvm_for_each_vcpu(i, vcpu, kvm) {
759 kvm_s390_sync_request(KVM_REQ_ICPT_OPEREXC, vcpu);
760 }
761 }
762
kvm_vm_ioctl_enable_cap(struct kvm * kvm,struct kvm_enable_cap * cap)763 int kvm_vm_ioctl_enable_cap(struct kvm *kvm, struct kvm_enable_cap *cap)
764 {
765 int r;
766
767 if (cap->flags)
768 return -EINVAL;
769
770 switch (cap->cap) {
771 case KVM_CAP_S390_IRQCHIP:
772 VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_IRQCHIP");
773 kvm->arch.use_irqchip = 1;
774 r = 0;
775 break;
776 case KVM_CAP_S390_USER_SIGP:
777 VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_USER_SIGP");
778 kvm->arch.user_sigp = 1;
779 r = 0;
780 break;
781 case KVM_CAP_S390_VECTOR_REGISTERS:
782 mutex_lock(&kvm->lock);
783 if (kvm->created_vcpus) {
784 r = -EBUSY;
785 } else if (cpu_has_vx()) {
786 set_kvm_facility(kvm->arch.model.fac_mask, 129);
787 set_kvm_facility(kvm->arch.model.fac_list, 129);
788 if (test_facility(134)) {
789 set_kvm_facility(kvm->arch.model.fac_mask, 134);
790 set_kvm_facility(kvm->arch.model.fac_list, 134);
791 }
792 if (test_facility(135)) {
793 set_kvm_facility(kvm->arch.model.fac_mask, 135);
794 set_kvm_facility(kvm->arch.model.fac_list, 135);
795 }
796 if (test_facility(148)) {
797 set_kvm_facility(kvm->arch.model.fac_mask, 148);
798 set_kvm_facility(kvm->arch.model.fac_list, 148);
799 }
800 if (test_facility(152)) {
801 set_kvm_facility(kvm->arch.model.fac_mask, 152);
802 set_kvm_facility(kvm->arch.model.fac_list, 152);
803 }
804 if (test_facility(192)) {
805 set_kvm_facility(kvm->arch.model.fac_mask, 192);
806 set_kvm_facility(kvm->arch.model.fac_list, 192);
807 }
808 if (test_facility(198)) {
809 set_kvm_facility(kvm->arch.model.fac_mask, 198);
810 set_kvm_facility(kvm->arch.model.fac_list, 198);
811 }
812 if (test_facility(199)) {
813 set_kvm_facility(kvm->arch.model.fac_mask, 199);
814 set_kvm_facility(kvm->arch.model.fac_list, 199);
815 }
816 r = 0;
817 } else
818 r = -EINVAL;
819 mutex_unlock(&kvm->lock);
820 VM_EVENT(kvm, 3, "ENABLE: CAP_S390_VECTOR_REGISTERS %s",
821 r ? "(not available)" : "(success)");
822 break;
823 case KVM_CAP_S390_RI:
824 r = -EINVAL;
825 mutex_lock(&kvm->lock);
826 if (kvm->created_vcpus) {
827 r = -EBUSY;
828 } else if (test_facility(64)) {
829 set_kvm_facility(kvm->arch.model.fac_mask, 64);
830 set_kvm_facility(kvm->arch.model.fac_list, 64);
831 r = 0;
832 }
833 mutex_unlock(&kvm->lock);
834 VM_EVENT(kvm, 3, "ENABLE: CAP_S390_RI %s",
835 r ? "(not available)" : "(success)");
836 break;
837 case KVM_CAP_S390_AIS:
838 mutex_lock(&kvm->lock);
839 if (kvm->created_vcpus) {
840 r = -EBUSY;
841 } else {
842 set_kvm_facility(kvm->arch.model.fac_mask, 72);
843 set_kvm_facility(kvm->arch.model.fac_list, 72);
844 r = 0;
845 }
846 mutex_unlock(&kvm->lock);
847 VM_EVENT(kvm, 3, "ENABLE: AIS %s",
848 r ? "(not available)" : "(success)");
849 break;
850 case KVM_CAP_S390_GS:
851 r = -EINVAL;
852 mutex_lock(&kvm->lock);
853 if (kvm->created_vcpus) {
854 r = -EBUSY;
855 } else if (test_facility(133)) {
856 set_kvm_facility(kvm->arch.model.fac_mask, 133);
857 set_kvm_facility(kvm->arch.model.fac_list, 133);
858 r = 0;
859 }
860 mutex_unlock(&kvm->lock);
861 VM_EVENT(kvm, 3, "ENABLE: CAP_S390_GS %s",
862 r ? "(not available)" : "(success)");
863 break;
864 case KVM_CAP_S390_HPAGE_1M:
865 mutex_lock(&kvm->lock);
866 if (kvm->created_vcpus)
867 r = -EBUSY;
868 else if (!hpage || kvm->arch.use_cmma || kvm_is_ucontrol(kvm))
869 r = -EINVAL;
870 else {
871 r = 0;
872 set_bit(GMAP_FLAG_ALLOW_HPAGE_1M, &kvm->arch.gmap->flags);
873 /*
874 * We might have to create fake 4k page
875 * tables. To avoid that the hardware works on
876 * stale PGSTEs, we emulate these instructions.
877 */
878 kvm->arch.use_skf = 0;
879 kvm->arch.use_pfmfi = 0;
880 }
881 mutex_unlock(&kvm->lock);
882 VM_EVENT(kvm, 3, "ENABLE: CAP_S390_HPAGE %s",
883 r ? "(not available)" : "(success)");
884 break;
885 case KVM_CAP_S390_HPAGE_2G:
886 mutex_lock(&kvm->lock);
887 if (kvm->created_vcpus) {
888 r = -EBUSY;
889 } else if (!hpage_2g || kvm->arch.use_cmma || kvm_is_ucontrol(kvm)) {
890 r = -EINVAL;
891 } else {
892 r = 0;
893 set_bit(GMAP_FLAG_ALLOW_HPAGE_2G, &kvm->arch.gmap->flags);
894 /*
895 * We might have to create fake 4k page
896 * tables. To avoid that the hardware works on
897 * stale PGSTEs, we emulate these instructions.
898 */
899 kvm->arch.use_skf = 0;
900 kvm->arch.use_pfmfi = 0;
901 }
902 mutex_unlock(&kvm->lock);
903 VM_EVENT(kvm, 3, "ENABLE: CAP_S390_HPAGE_2G %s",
904 r ? "(not available)" : "(success)");
905 break;
906 case KVM_CAP_S390_USER_STSI:
907 VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_USER_STSI");
908 kvm->arch.user_stsi = 1;
909 r = 0;
910 break;
911 case KVM_CAP_S390_USER_INSTR0:
912 VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_USER_INSTR0");
913 kvm->arch.user_instr0 = 1;
914 icpt_operexc_on_all_vcpus(kvm);
915 r = 0;
916 break;
917 case KVM_CAP_S390_CPU_TOPOLOGY:
918 r = -EINVAL;
919 mutex_lock(&kvm->lock);
920 if (kvm->created_vcpus) {
921 r = -EBUSY;
922 } else if (test_facility(11)) {
923 set_kvm_facility(kvm->arch.model.fac_mask, 11);
924 set_kvm_facility(kvm->arch.model.fac_list, 11);
925 r = 0;
926 }
927 mutex_unlock(&kvm->lock);
928 VM_EVENT(kvm, 3, "ENABLE: CAP_S390_CPU_TOPOLOGY %s",
929 r ? "(not available)" : "(success)");
930 break;
931 case KVM_CAP_S390_USER_OPEREXEC:
932 VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_USER_OPEREXEC");
933 kvm->arch.user_operexec = 1;
934 icpt_operexc_on_all_vcpus(kvm);
935 r = 0;
936 break;
937 case KVM_CAP_S390_VSIE_ESAMODE:
938 VM_EVENT(kvm, 3, "%s", "ENABLE: CAP_S390_VSIE_ESAMODE");
939 kvm->arch.allow_vsie_esamode = 1;
940 r = 0;
941 break;
942 default:
943 r = -EINVAL;
944 break;
945 }
946 return r;
947 }
948
kvm_s390_get_mem_control(struct kvm * kvm,struct kvm_device_attr * attr)949 static int kvm_s390_get_mem_control(struct kvm *kvm, struct kvm_device_attr *attr)
950 {
951 int ret;
952
953 switch (attr->attr) {
954 case KVM_S390_VM_MEM_LIMIT_SIZE:
955 ret = 0;
956 VM_EVENT(kvm, 3, "QUERY: max guest memory: %lu bytes",
957 kvm->arch.mem_limit);
958 if (put_user(kvm->arch.mem_limit, (u64 __user *)attr->addr))
959 ret = -EFAULT;
960 break;
961 default:
962 ret = -ENXIO;
963 break;
964 }
965 return ret;
966 }
967
kvm_s390_set_mem_control(struct kvm * kvm,struct kvm_device_attr * attr)968 static int kvm_s390_set_mem_control(struct kvm *kvm, struct kvm_device_attr *attr)
969 {
970 int ret;
971
972 switch (attr->attr) {
973 case KVM_S390_VM_MEM_ENABLE_CMMA:
974 ret = -ENXIO;
975 if (!sclp.has_cmma)
976 break;
977
978 VM_EVENT(kvm, 3, "%s", "ENABLE: CMMA support");
979 mutex_lock(&kvm->lock);
980 if (kvm->created_vcpus)
981 ret = -EBUSY;
982 else {
983 kvm->arch.use_cmma = 1;
984 /* Not compatible with cmma. */
985 kvm->arch.use_pfmfi = 0;
986 ret = 0;
987 }
988 mutex_unlock(&kvm->lock);
989 break;
990 case KVM_S390_VM_MEM_CLR_CMMA: {
991 gfn_t start_gfn = 0;
992
993 ret = -ENXIO;
994 if (!sclp.has_cmma)
995 break;
996 ret = -EINVAL;
997 if (!kvm->arch.use_cmma)
998 break;
999
1000 guard(mutex)(&kvm->lock);
1001 VM_EVENT(kvm, 3, "%s", "RESET: CMMA states");
1002 do {
1003 scoped_guard(read_lock, &kvm->mmu_lock)
1004 start_gfn = dat_reset_cmma(kvm->arch.gmap->asce, start_gfn);
1005 } while (start_gfn);
1006 ret = 0;
1007 break;
1008 }
1009 case KVM_S390_VM_MEM_LIMIT_SIZE: {
1010 struct kvm_memslots *slots;
1011 struct kvm_memory_slot *ms;
1012 unsigned long new_limit;
1013 int bkt;
1014
1015 if (kvm_is_ucontrol(kvm))
1016 return -EINVAL;
1017
1018 if (get_user(new_limit, (u64 __user *)attr->addr))
1019 return -EFAULT;
1020
1021 guard(mutex)(&kvm->lock);
1022
1023 new_limit = ALIGN(new_limit, HPAGE_SIZE);
1024 if (kvm->arch.mem_limit != KVM_S390_NO_MEM_LIMIT &&
1025 new_limit > kvm->arch.mem_limit)
1026 return -E2BIG;
1027
1028 if (!new_limit)
1029 return -EINVAL;
1030
1031 if (kvm->created_vcpus)
1032 return -EBUSY;
1033
1034 ret = 0;
1035 scoped_guard(mutex, &kvm->slots_lock) {
1036 slots = kvm_memslots(kvm);
1037 if (slots && !kvm_memslots_empty(slots)) {
1038 kvm_for_each_memslot(ms, bkt, slots) {
1039 if (gpa_to_gfn(new_limit) < ms->base_gfn + ms->npages) {
1040 ret = -EBUSY;
1041 break;
1042 }
1043 }
1044 }
1045 if (!ret)
1046 ret = gmap_set_limit(kvm->arch.gmap, gpa_to_gfn(new_limit));
1047 }
1048 if (ret)
1049 break;
1050 VM_EVENT(kvm, 3, "SET: max guest address: %lu", new_limit);
1051 VM_EVENT(kvm, 3, "New guest asce: 0x%p", (void *)kvm->arch.gmap->asce.val);
1052 break;
1053 }
1054 default:
1055 ret = -ENXIO;
1056 break;
1057 }
1058 return ret;
1059 }
1060
1061 static void kvm_s390_vcpu_crypto_setup(struct kvm_vcpu *vcpu);
1062
kvm_s390_vcpu_crypto_reset_all(struct kvm * kvm)1063 void kvm_s390_vcpu_crypto_reset_all(struct kvm *kvm)
1064 {
1065 struct kvm_vcpu *vcpu;
1066 unsigned long i;
1067
1068 kvm_s390_vcpu_block_all(kvm);
1069
1070 kvm_for_each_vcpu(i, vcpu, kvm) {
1071 kvm_s390_vcpu_crypto_setup(vcpu);
1072 /* recreate the shadow crycb by leaving the VSIE handler */
1073 kvm_s390_sync_request(KVM_REQ_VSIE_RESTART, vcpu);
1074 }
1075
1076 kvm_s390_vcpu_unblock_all(kvm);
1077 }
1078
kvm_s390_vm_set_crypto(struct kvm * kvm,struct kvm_device_attr * attr)1079 static int kvm_s390_vm_set_crypto(struct kvm *kvm, struct kvm_device_attr *attr)
1080 {
1081 mutex_lock(&kvm->lock);
1082 switch (attr->attr) {
1083 case KVM_S390_VM_CRYPTO_ENABLE_AES_KW:
1084 if (!test_kvm_facility(kvm, 76)) {
1085 mutex_unlock(&kvm->lock);
1086 return -EINVAL;
1087 }
1088 get_random_bytes(
1089 kvm->arch.crypto.crycb->aes_wrapping_key_mask,
1090 sizeof(kvm->arch.crypto.crycb->aes_wrapping_key_mask));
1091 kvm->arch.crypto.aes_kw = 1;
1092 VM_EVENT(kvm, 3, "%s", "ENABLE: AES keywrapping support");
1093 break;
1094 case KVM_S390_VM_CRYPTO_ENABLE_DEA_KW:
1095 if (!test_kvm_facility(kvm, 76)) {
1096 mutex_unlock(&kvm->lock);
1097 return -EINVAL;
1098 }
1099 get_random_bytes(
1100 kvm->arch.crypto.crycb->dea_wrapping_key_mask,
1101 sizeof(kvm->arch.crypto.crycb->dea_wrapping_key_mask));
1102 kvm->arch.crypto.dea_kw = 1;
1103 VM_EVENT(kvm, 3, "%s", "ENABLE: DEA keywrapping support");
1104 break;
1105 case KVM_S390_VM_CRYPTO_DISABLE_AES_KW:
1106 if (!test_kvm_facility(kvm, 76)) {
1107 mutex_unlock(&kvm->lock);
1108 return -EINVAL;
1109 }
1110 kvm->arch.crypto.aes_kw = 0;
1111 memset(kvm->arch.crypto.crycb->aes_wrapping_key_mask, 0,
1112 sizeof(kvm->arch.crypto.crycb->aes_wrapping_key_mask));
1113 VM_EVENT(kvm, 3, "%s", "DISABLE: AES keywrapping support");
1114 break;
1115 case KVM_S390_VM_CRYPTO_DISABLE_DEA_KW:
1116 if (!test_kvm_facility(kvm, 76)) {
1117 mutex_unlock(&kvm->lock);
1118 return -EINVAL;
1119 }
1120 kvm->arch.crypto.dea_kw = 0;
1121 memset(kvm->arch.crypto.crycb->dea_wrapping_key_mask, 0,
1122 sizeof(kvm->arch.crypto.crycb->dea_wrapping_key_mask));
1123 VM_EVENT(kvm, 3, "%s", "DISABLE: DEA keywrapping support");
1124 break;
1125 case KVM_S390_VM_CRYPTO_ENABLE_APIE:
1126 if (!ap_instructions_available()) {
1127 mutex_unlock(&kvm->lock);
1128 return -EOPNOTSUPP;
1129 }
1130 kvm->arch.crypto.apie = 1;
1131 break;
1132 case KVM_S390_VM_CRYPTO_DISABLE_APIE:
1133 if (!ap_instructions_available()) {
1134 mutex_unlock(&kvm->lock);
1135 return -EOPNOTSUPP;
1136 }
1137 kvm->arch.crypto.apie = 0;
1138 break;
1139 default:
1140 mutex_unlock(&kvm->lock);
1141 return -ENXIO;
1142 }
1143
1144 kvm_s390_vcpu_crypto_reset_all(kvm);
1145 mutex_unlock(&kvm->lock);
1146 return 0;
1147 }
1148
kvm_s390_vcpu_pci_setup(struct kvm_vcpu * vcpu)1149 static void kvm_s390_vcpu_pci_setup(struct kvm_vcpu *vcpu)
1150 {
1151 /* Only set the ECB bits after guest requests zPCI interpretation */
1152 if (!vcpu->kvm->arch.use_zpci_interp)
1153 return;
1154
1155 vcpu->arch.sie_block->ecb2 |= ECB2_ZPCI_LSI;
1156 vcpu->arch.sie_block->ecb3 |= ECB3_AISII + ECB3_AISI;
1157 }
1158
kvm_s390_vcpu_pci_enable_interp(struct kvm * kvm)1159 void kvm_s390_vcpu_pci_enable_interp(struct kvm *kvm)
1160 {
1161 struct kvm_vcpu *vcpu;
1162 unsigned long i;
1163
1164 lockdep_assert_held(&kvm->lock);
1165
1166 if (!kvm_s390_pci_interp_allowed())
1167 return;
1168
1169 /*
1170 * If host is configured for PCI and the necessary facilities are
1171 * available, turn on interpretation for the life of this guest
1172 */
1173 kvm->arch.use_zpci_interp = 1;
1174
1175 kvm_s390_vcpu_block_all(kvm);
1176
1177 kvm_for_each_vcpu(i, vcpu, kvm) {
1178 kvm_s390_vcpu_pci_setup(vcpu);
1179 kvm_s390_sync_request(KVM_REQ_VSIE_RESTART, vcpu);
1180 }
1181
1182 kvm_s390_vcpu_unblock_all(kvm);
1183 }
1184
kvm_s390_sync_request_broadcast(struct kvm * kvm,int req)1185 static void kvm_s390_sync_request_broadcast(struct kvm *kvm, int req)
1186 {
1187 unsigned long cx;
1188 struct kvm_vcpu *vcpu;
1189
1190 kvm_for_each_vcpu(cx, vcpu, kvm)
1191 kvm_s390_sync_request(req, vcpu);
1192 }
1193
1194 /*
1195 * Must be called with kvm->srcu held to avoid races on memslots, and with
1196 * kvm->slots_arch_lock to avoid races with ourselves,
1197 * kvm_s390_vm_stop_migration(), and kvm_s390_get_cmma_bits().
1198 */
kvm_s390_vm_start_migration(struct kvm * kvm)1199 static int kvm_s390_vm_start_migration(struct kvm *kvm)
1200 {
1201 struct kvm_memory_slot *ms;
1202 struct kvm_memslots *slots;
1203 int bkt;
1204
1205 /* migration mode already enabled */
1206 if (kvm->arch.migration_mode)
1207 return 0;
1208 slots = kvm_memslots(kvm);
1209 if (!slots || kvm_memslots_empty(slots))
1210 return -EINVAL;
1211
1212 if (!kvm->arch.use_cmma) {
1213 kvm->arch.migration_mode = 1;
1214 return 0;
1215 }
1216 kvm_for_each_memslot(ms, bkt, slots) {
1217 if (!ms->dirty_bitmap)
1218 return -EINVAL;
1219 }
1220 /*
1221 * Set the flag and let KVM handle ESSA manually, potentially setting
1222 * the cmma_d bit in some PGSTEs and increasing cmma_dirty_pages.
1223 * At this point cmma_dirty_pages is still 0, and all existing PGSTEs
1224 * have their cmma_d bit set to 0.
1225 * Any newly allocated page table has its entries marked as cmma-clean,
1226 * which is fine because the CMMA values are not dirty.
1227 */
1228 WRITE_ONCE(kvm->arch.migration_mode, 1);
1229 kvm_s390_sync_request_broadcast(kvm, KVM_REQ_START_MIGRATION);
1230 /*
1231 * Mark all PGSTEs as cmma-dirty, increasing cmma_dirty_pages as needed,
1232 * but without double-counting pages that have become dirty on their own
1233 * in the meantime.
1234 * At this point some pages might have become dirty on their own already
1235 * and cmma_dirty_pages might therefore be non-zero.
1236 */
1237 gmap_set_cmma_all_dirty(kvm->arch.gmap);
1238 return 0;
1239 }
1240
1241 /*
1242 * Must be called with kvm->slots_arch_lock to avoid races with ourselves,
1243 * kvm_s390_vm_start_migration() and kvm_s390_get_cmma_bits().
1244 */
kvm_s390_vm_stop_migration(struct kvm * kvm)1245 int kvm_s390_vm_stop_migration(struct kvm *kvm)
1246 {
1247 /* migration mode already disabled */
1248 if (!kvm->arch.migration_mode)
1249 return 0;
1250 /*
1251 * Unset the flag and propagate to all vCPUs. From now on the cmma_d
1252 * bit will not be touched on any PGSTE.
1253 * At this point cmma_dirty_pages is possibly non-zero, and thus some
1254 * PGSTEs might have cmma_d set.
1255 */
1256 WRITE_ONCE(kvm->arch.migration_mode, 0);
1257 if (!kvm->arch.use_cmma)
1258 return 0;
1259
1260 kvm_s390_sync_request_broadcast(kvm, KVM_REQ_STOP_MIGRATION);
1261 /* Clear cmma_d on all existing PGSTEs and set cmma_dirty_pages to 0. */
1262 gmap_set_cmma_all_clean(kvm->arch.gmap);
1263 atomic64_set(&kvm->arch.cmma_dirty_pages, 0);
1264 /*
1265 * At this point the system has the expected state: migration_mode is 0,
1266 * cmma_dirty_pages is 0, and all existing PGSTEs have their cmma_d bit
1267 * set to 0.
1268 */
1269 return 0;
1270 }
1271
kvm_s390_vm_set_migration(struct kvm * kvm,struct kvm_device_attr * attr)1272 static int kvm_s390_vm_set_migration(struct kvm *kvm,
1273 struct kvm_device_attr *attr)
1274 {
1275 int res = -ENXIO;
1276
1277 guard(srcu)(&kvm->srcu);
1278 guard(mutex)(&kvm->slots_arch_lock);
1279
1280 switch (attr->attr) {
1281 case KVM_S390_VM_MIGRATION_START:
1282 res = kvm_s390_vm_start_migration(kvm);
1283 break;
1284 case KVM_S390_VM_MIGRATION_STOP:
1285 res = kvm_s390_vm_stop_migration(kvm);
1286 break;
1287 default:
1288 break;
1289 }
1290
1291 return res;
1292 }
1293
kvm_s390_vm_get_migration(struct kvm * kvm,struct kvm_device_attr * attr)1294 static int kvm_s390_vm_get_migration(struct kvm *kvm,
1295 struct kvm_device_attr *attr)
1296 {
1297 u64 mig = kvm->arch.migration_mode;
1298
1299 if (attr->attr != KVM_S390_VM_MIGRATION_STATUS)
1300 return -ENXIO;
1301
1302 if (copy_to_user((void __user *)attr->addr, &mig, sizeof(mig)))
1303 return -EFAULT;
1304 return 0;
1305 }
1306
1307 static void __kvm_s390_set_tod_clock(struct kvm *kvm, const struct kvm_s390_vm_tod_clock *gtod);
1308
kvm_s390_set_tod_ext(struct kvm * kvm,struct kvm_device_attr * attr)1309 static int kvm_s390_set_tod_ext(struct kvm *kvm, struct kvm_device_attr *attr)
1310 {
1311 struct kvm_s390_vm_tod_clock gtod;
1312
1313 if (copy_from_user(>od, (void __user *)attr->addr, sizeof(gtod)))
1314 return -EFAULT;
1315
1316 if (!test_kvm_facility(kvm, 139) && gtod.epoch_idx)
1317 return -EINVAL;
1318 __kvm_s390_set_tod_clock(kvm, >od);
1319
1320 VM_EVENT(kvm, 3, "SET: TOD extension: 0x%x, TOD base: 0x%llx",
1321 gtod.epoch_idx, gtod.tod);
1322
1323 return 0;
1324 }
1325
kvm_s390_set_tod_high(struct kvm * kvm,struct kvm_device_attr * attr)1326 static int kvm_s390_set_tod_high(struct kvm *kvm, struct kvm_device_attr *attr)
1327 {
1328 u8 gtod_high;
1329
1330 if (copy_from_user(>od_high, (void __user *)attr->addr,
1331 sizeof(gtod_high)))
1332 return -EFAULT;
1333
1334 if (gtod_high != 0)
1335 return -EINVAL;
1336 VM_EVENT(kvm, 3, "SET: TOD extension: 0x%x", gtod_high);
1337
1338 return 0;
1339 }
1340
kvm_s390_set_tod_low(struct kvm * kvm,struct kvm_device_attr * attr)1341 static int kvm_s390_set_tod_low(struct kvm *kvm, struct kvm_device_attr *attr)
1342 {
1343 struct kvm_s390_vm_tod_clock gtod = { 0 };
1344
1345 if (copy_from_user(>od.tod, (void __user *)attr->addr,
1346 sizeof(gtod.tod)))
1347 return -EFAULT;
1348
1349 __kvm_s390_set_tod_clock(kvm, >od);
1350 VM_EVENT(kvm, 3, "SET: TOD base: 0x%llx", gtod.tod);
1351 return 0;
1352 }
1353
kvm_s390_set_tod(struct kvm * kvm,struct kvm_device_attr * attr)1354 static int kvm_s390_set_tod(struct kvm *kvm, struct kvm_device_attr *attr)
1355 {
1356 int ret;
1357
1358 if (attr->flags)
1359 return -EINVAL;
1360
1361 mutex_lock(&kvm->lock);
1362 /*
1363 * For protected guests, the TOD is managed by the ultravisor, so trying
1364 * to change it will never bring the expected results.
1365 */
1366 if (kvm_s390_pv_is_protected(kvm)) {
1367 ret = -EOPNOTSUPP;
1368 goto out_unlock;
1369 }
1370
1371 switch (attr->attr) {
1372 case KVM_S390_VM_TOD_EXT:
1373 ret = kvm_s390_set_tod_ext(kvm, attr);
1374 break;
1375 case KVM_S390_VM_TOD_HIGH:
1376 ret = kvm_s390_set_tod_high(kvm, attr);
1377 break;
1378 case KVM_S390_VM_TOD_LOW:
1379 ret = kvm_s390_set_tod_low(kvm, attr);
1380 break;
1381 default:
1382 ret = -ENXIO;
1383 break;
1384 }
1385
1386 out_unlock:
1387 mutex_unlock(&kvm->lock);
1388 return ret;
1389 }
1390
kvm_s390_get_tod_clock(struct kvm * kvm,struct kvm_s390_vm_tod_clock * gtod)1391 static void kvm_s390_get_tod_clock(struct kvm *kvm,
1392 struct kvm_s390_vm_tod_clock *gtod)
1393 {
1394 union tod_clock clk;
1395
1396 preempt_disable();
1397
1398 store_tod_clock_ext(&clk);
1399
1400 gtod->tod = clk.tod + kvm->arch.epoch;
1401 gtod->epoch_idx = 0;
1402 if (test_kvm_facility(kvm, 139)) {
1403 gtod->epoch_idx = clk.ei + kvm->arch.epdx;
1404 if (gtod->tod < clk.tod)
1405 gtod->epoch_idx += 1;
1406 }
1407
1408 preempt_enable();
1409 }
1410
kvm_s390_get_tod_ext(struct kvm * kvm,struct kvm_device_attr * attr)1411 static int kvm_s390_get_tod_ext(struct kvm *kvm, struct kvm_device_attr *attr)
1412 {
1413 struct kvm_s390_vm_tod_clock gtod;
1414
1415 memset(>od, 0, sizeof(gtod));
1416 kvm_s390_get_tod_clock(kvm, >od);
1417 if (copy_to_user((void __user *)attr->addr, >od, sizeof(gtod)))
1418 return -EFAULT;
1419
1420 VM_EVENT(kvm, 3, "QUERY: TOD extension: 0x%x, TOD base: 0x%llx",
1421 gtod.epoch_idx, gtod.tod);
1422 return 0;
1423 }
1424
kvm_s390_get_tod_high(struct kvm * kvm,struct kvm_device_attr * attr)1425 static int kvm_s390_get_tod_high(struct kvm *kvm, struct kvm_device_attr *attr)
1426 {
1427 u8 gtod_high = 0;
1428
1429 if (copy_to_user((void __user *)attr->addr, >od_high,
1430 sizeof(gtod_high)))
1431 return -EFAULT;
1432 VM_EVENT(kvm, 3, "QUERY: TOD extension: 0x%x", gtod_high);
1433
1434 return 0;
1435 }
1436
kvm_s390_get_tod_low(struct kvm * kvm,struct kvm_device_attr * attr)1437 static int kvm_s390_get_tod_low(struct kvm *kvm, struct kvm_device_attr *attr)
1438 {
1439 u64 gtod;
1440
1441 gtod = kvm_s390_get_tod_clock_fast(kvm);
1442 if (copy_to_user((void __user *)attr->addr, >od, sizeof(gtod)))
1443 return -EFAULT;
1444 VM_EVENT(kvm, 3, "QUERY: TOD base: 0x%llx", gtod);
1445
1446 return 0;
1447 }
1448
kvm_s390_get_tod(struct kvm * kvm,struct kvm_device_attr * attr)1449 static int kvm_s390_get_tod(struct kvm *kvm, struct kvm_device_attr *attr)
1450 {
1451 int ret;
1452
1453 if (attr->flags)
1454 return -EINVAL;
1455
1456 switch (attr->attr) {
1457 case KVM_S390_VM_TOD_EXT:
1458 ret = kvm_s390_get_tod_ext(kvm, attr);
1459 break;
1460 case KVM_S390_VM_TOD_HIGH:
1461 ret = kvm_s390_get_tod_high(kvm, attr);
1462 break;
1463 case KVM_S390_VM_TOD_LOW:
1464 ret = kvm_s390_get_tod_low(kvm, attr);
1465 break;
1466 default:
1467 ret = -ENXIO;
1468 break;
1469 }
1470 return ret;
1471 }
1472
kvm_s390_set_processor(struct kvm * kvm,struct kvm_device_attr * attr)1473 static int kvm_s390_set_processor(struct kvm *kvm, struct kvm_device_attr *attr)
1474 {
1475 struct kvm_s390_vm_cpu_processor *proc;
1476 u16 lowest_ibc, unblocked_ibc;
1477 int ret = 0;
1478
1479 mutex_lock(&kvm->lock);
1480 if (kvm->created_vcpus) {
1481 ret = -EBUSY;
1482 goto out;
1483 }
1484 proc = kzalloc_obj(*proc, GFP_KERNEL_ACCOUNT);
1485 if (!proc) {
1486 ret = -ENOMEM;
1487 goto out;
1488 }
1489 if (!copy_from_user(proc, (void __user *)attr->addr,
1490 sizeof(*proc))) {
1491 kvm->arch.model.cpuid = proc->cpuid;
1492 lowest_ibc = sclp.ibc >> 16 & 0xfff;
1493 unblocked_ibc = sclp.ibc & 0xfff;
1494 if (lowest_ibc && proc->ibc) {
1495 if (proc->ibc > unblocked_ibc)
1496 kvm->arch.model.ibc = unblocked_ibc;
1497 else if (proc->ibc < lowest_ibc)
1498 kvm->arch.model.ibc = lowest_ibc;
1499 else
1500 kvm->arch.model.ibc = proc->ibc;
1501 }
1502 memcpy(kvm->arch.model.fac_list, proc->fac_list,
1503 S390_ARCH_FAC_LIST_SIZE_BYTE);
1504 VM_EVENT(kvm, 3, "SET: guest ibc: 0x%4.4x, guest cpuid: 0x%16.16llx",
1505 kvm->arch.model.ibc,
1506 kvm->arch.model.cpuid);
1507 VM_EVENT(kvm, 3, "SET: guest faclist: 0x%16.16llx.%16.16llx.%16.16llx",
1508 kvm->arch.model.fac_list[0],
1509 kvm->arch.model.fac_list[1],
1510 kvm->arch.model.fac_list[2]);
1511 } else
1512 ret = -EFAULT;
1513 kfree(proc);
1514 out:
1515 mutex_unlock(&kvm->lock);
1516 return ret;
1517 }
1518
kvm_s390_set_processor_feat(struct kvm * kvm,struct kvm_device_attr * attr)1519 static int kvm_s390_set_processor_feat(struct kvm *kvm,
1520 struct kvm_device_attr *attr)
1521 {
1522 struct kvm_s390_vm_cpu_feat data;
1523
1524 if (copy_from_user(&data, (void __user *)attr->addr, sizeof(data)))
1525 return -EFAULT;
1526 if (!bitmap_subset((unsigned long *) data.feat,
1527 kvm_s390_available_cpu_feat,
1528 KVM_S390_VM_CPU_FEAT_NR_BITS))
1529 return -EINVAL;
1530
1531 mutex_lock(&kvm->lock);
1532 if (kvm->created_vcpus) {
1533 mutex_unlock(&kvm->lock);
1534 return -EBUSY;
1535 }
1536 bitmap_from_arr64(kvm->arch.cpu_feat, data.feat, KVM_S390_VM_CPU_FEAT_NR_BITS);
1537 mutex_unlock(&kvm->lock);
1538 VM_EVENT(kvm, 3, "SET: guest feat: 0x%16.16llx.0x%16.16llx.0x%16.16llx",
1539 data.feat[0],
1540 data.feat[1],
1541 data.feat[2]);
1542 return 0;
1543 }
1544
kvm_s390_set_processor_subfunc(struct kvm * kvm,struct kvm_device_attr * attr)1545 static int kvm_s390_set_processor_subfunc(struct kvm *kvm,
1546 struct kvm_device_attr *attr)
1547 {
1548 mutex_lock(&kvm->lock);
1549 if (kvm->created_vcpus) {
1550 mutex_unlock(&kvm->lock);
1551 return -EBUSY;
1552 }
1553
1554 if (copy_from_user(&kvm->arch.model.subfuncs, (void __user *)attr->addr,
1555 sizeof(struct kvm_s390_vm_cpu_subfunc))) {
1556 mutex_unlock(&kvm->lock);
1557 return -EFAULT;
1558 }
1559 mutex_unlock(&kvm->lock);
1560
1561 VM_EVENT(kvm, 3, "SET: guest PLO subfunc 0x%16.16lx.%16.16lx.%16.16lx.%16.16lx",
1562 ((unsigned long *) &kvm->arch.model.subfuncs.plo)[0],
1563 ((unsigned long *) &kvm->arch.model.subfuncs.plo)[1],
1564 ((unsigned long *) &kvm->arch.model.subfuncs.plo)[2],
1565 ((unsigned long *) &kvm->arch.model.subfuncs.plo)[3]);
1566 VM_EVENT(kvm, 3, "SET: guest PTFF subfunc 0x%16.16lx.%16.16lx",
1567 ((unsigned long *) &kvm->arch.model.subfuncs.ptff)[0],
1568 ((unsigned long *) &kvm->arch.model.subfuncs.ptff)[1]);
1569 VM_EVENT(kvm, 3, "SET: guest KMAC subfunc 0x%16.16lx.%16.16lx",
1570 ((unsigned long *) &kvm->arch.model.subfuncs.kmac)[0],
1571 ((unsigned long *) &kvm->arch.model.subfuncs.kmac)[1]);
1572 VM_EVENT(kvm, 3, "SET: guest KMC subfunc 0x%16.16lx.%16.16lx",
1573 ((unsigned long *) &kvm->arch.model.subfuncs.kmc)[0],
1574 ((unsigned long *) &kvm->arch.model.subfuncs.kmc)[1]);
1575 VM_EVENT(kvm, 3, "SET: guest KM subfunc 0x%16.16lx.%16.16lx",
1576 ((unsigned long *) &kvm->arch.model.subfuncs.km)[0],
1577 ((unsigned long *) &kvm->arch.model.subfuncs.km)[1]);
1578 VM_EVENT(kvm, 3, "SET: guest KIMD subfunc 0x%16.16lx.%16.16lx",
1579 ((unsigned long *) &kvm->arch.model.subfuncs.kimd)[0],
1580 ((unsigned long *) &kvm->arch.model.subfuncs.kimd)[1]);
1581 VM_EVENT(kvm, 3, "SET: guest KLMD subfunc 0x%16.16lx.%16.16lx",
1582 ((unsigned long *) &kvm->arch.model.subfuncs.klmd)[0],
1583 ((unsigned long *) &kvm->arch.model.subfuncs.klmd)[1]);
1584 VM_EVENT(kvm, 3, "SET: guest PCKMO subfunc 0x%16.16lx.%16.16lx",
1585 ((unsigned long *) &kvm->arch.model.subfuncs.pckmo)[0],
1586 ((unsigned long *) &kvm->arch.model.subfuncs.pckmo)[1]);
1587 VM_EVENT(kvm, 3, "SET: guest KMCTR subfunc 0x%16.16lx.%16.16lx",
1588 ((unsigned long *) &kvm->arch.model.subfuncs.kmctr)[0],
1589 ((unsigned long *) &kvm->arch.model.subfuncs.kmctr)[1]);
1590 VM_EVENT(kvm, 3, "SET: guest KMF subfunc 0x%16.16lx.%16.16lx",
1591 ((unsigned long *) &kvm->arch.model.subfuncs.kmf)[0],
1592 ((unsigned long *) &kvm->arch.model.subfuncs.kmf)[1]);
1593 VM_EVENT(kvm, 3, "SET: guest KMO subfunc 0x%16.16lx.%16.16lx",
1594 ((unsigned long *) &kvm->arch.model.subfuncs.kmo)[0],
1595 ((unsigned long *) &kvm->arch.model.subfuncs.kmo)[1]);
1596 VM_EVENT(kvm, 3, "SET: guest PCC subfunc 0x%16.16lx.%16.16lx",
1597 ((unsigned long *) &kvm->arch.model.subfuncs.pcc)[0],
1598 ((unsigned long *) &kvm->arch.model.subfuncs.pcc)[1]);
1599 VM_EVENT(kvm, 3, "SET: guest PPNO subfunc 0x%16.16lx.%16.16lx",
1600 ((unsigned long *) &kvm->arch.model.subfuncs.ppno)[0],
1601 ((unsigned long *) &kvm->arch.model.subfuncs.ppno)[1]);
1602 VM_EVENT(kvm, 3, "SET: guest KMA subfunc 0x%16.16lx.%16.16lx",
1603 ((unsigned long *) &kvm->arch.model.subfuncs.kma)[0],
1604 ((unsigned long *) &kvm->arch.model.subfuncs.kma)[1]);
1605 VM_EVENT(kvm, 3, "SET: guest KDSA subfunc 0x%16.16lx.%16.16lx",
1606 ((unsigned long *) &kvm->arch.model.subfuncs.kdsa)[0],
1607 ((unsigned long *) &kvm->arch.model.subfuncs.kdsa)[1]);
1608 VM_EVENT(kvm, 3, "SET: guest SORTL subfunc 0x%16.16lx.%16.16lx.%16.16lx.%16.16lx",
1609 ((unsigned long *) &kvm->arch.model.subfuncs.sortl)[0],
1610 ((unsigned long *) &kvm->arch.model.subfuncs.sortl)[1],
1611 ((unsigned long *) &kvm->arch.model.subfuncs.sortl)[2],
1612 ((unsigned long *) &kvm->arch.model.subfuncs.sortl)[3]);
1613 VM_EVENT(kvm, 3, "SET: guest DFLTCC subfunc 0x%16.16lx.%16.16lx.%16.16lx.%16.16lx",
1614 ((unsigned long *) &kvm->arch.model.subfuncs.dfltcc)[0],
1615 ((unsigned long *) &kvm->arch.model.subfuncs.dfltcc)[1],
1616 ((unsigned long *) &kvm->arch.model.subfuncs.dfltcc)[2],
1617 ((unsigned long *) &kvm->arch.model.subfuncs.dfltcc)[3]);
1618 VM_EVENT(kvm, 3, "GET: guest PFCR subfunc 0x%16.16lx.%16.16lx",
1619 ((unsigned long *) &kvm_s390_available_subfunc.pfcr)[0],
1620 ((unsigned long *) &kvm_s390_available_subfunc.pfcr)[1]);
1621
1622 return 0;
1623 }
1624
1625 #define KVM_S390_VM_CPU_UV_FEAT_GUEST_MASK \
1626 ( \
1627 ((struct kvm_s390_vm_cpu_uv_feat){ \
1628 .ap = 1, \
1629 .ap_intr = 1, \
1630 }) \
1631 .feat \
1632 )
1633
kvm_s390_set_uv_feat(struct kvm * kvm,struct kvm_device_attr * attr)1634 static int kvm_s390_set_uv_feat(struct kvm *kvm, struct kvm_device_attr *attr)
1635 {
1636 struct kvm_s390_vm_cpu_uv_feat __user *ptr = (void __user *)attr->addr;
1637 unsigned long data, filter;
1638
1639 filter = uv_info.uv_feature_indications & KVM_S390_VM_CPU_UV_FEAT_GUEST_MASK;
1640 if (get_user(data, &ptr->feat))
1641 return -EFAULT;
1642 if (!bitmap_subset(&data, &filter, KVM_S390_VM_CPU_UV_FEAT_NR_BITS))
1643 return -EINVAL;
1644
1645 mutex_lock(&kvm->lock);
1646 if (kvm->created_vcpus) {
1647 mutex_unlock(&kvm->lock);
1648 return -EBUSY;
1649 }
1650 kvm->arch.model.uv_feat_guest.feat = data;
1651 mutex_unlock(&kvm->lock);
1652
1653 VM_EVENT(kvm, 3, "SET: guest UV-feat: 0x%16.16lx", data);
1654
1655 return 0;
1656 }
1657
kvm_s390_set_cpu_model(struct kvm * kvm,struct kvm_device_attr * attr)1658 static int kvm_s390_set_cpu_model(struct kvm *kvm, struct kvm_device_attr *attr)
1659 {
1660 int ret = -ENXIO;
1661
1662 switch (attr->attr) {
1663 case KVM_S390_VM_CPU_PROCESSOR:
1664 ret = kvm_s390_set_processor(kvm, attr);
1665 break;
1666 case KVM_S390_VM_CPU_PROCESSOR_FEAT:
1667 ret = kvm_s390_set_processor_feat(kvm, attr);
1668 break;
1669 case KVM_S390_VM_CPU_PROCESSOR_SUBFUNC:
1670 ret = kvm_s390_set_processor_subfunc(kvm, attr);
1671 break;
1672 case KVM_S390_VM_CPU_PROCESSOR_UV_FEAT_GUEST:
1673 ret = kvm_s390_set_uv_feat(kvm, attr);
1674 break;
1675 }
1676 return ret;
1677 }
1678
kvm_s390_get_processor(struct kvm * kvm,struct kvm_device_attr * attr)1679 static int kvm_s390_get_processor(struct kvm *kvm, struct kvm_device_attr *attr)
1680 {
1681 struct kvm_s390_vm_cpu_processor *proc;
1682 int ret = 0;
1683
1684 proc = kzalloc_obj(*proc, GFP_KERNEL_ACCOUNT);
1685 if (!proc) {
1686 ret = -ENOMEM;
1687 goto out;
1688 }
1689 proc->cpuid = kvm->arch.model.cpuid;
1690 proc->ibc = kvm->arch.model.ibc;
1691 memcpy(&proc->fac_list, kvm->arch.model.fac_list,
1692 S390_ARCH_FAC_LIST_SIZE_BYTE);
1693 VM_EVENT(kvm, 3, "GET: guest ibc: 0x%4.4x, guest cpuid: 0x%16.16llx",
1694 kvm->arch.model.ibc,
1695 kvm->arch.model.cpuid);
1696 VM_EVENT(kvm, 3, "GET: guest faclist: 0x%16.16llx.%16.16llx.%16.16llx",
1697 kvm->arch.model.fac_list[0],
1698 kvm->arch.model.fac_list[1],
1699 kvm->arch.model.fac_list[2]);
1700 if (copy_to_user((void __user *)attr->addr, proc, sizeof(*proc)))
1701 ret = -EFAULT;
1702 kfree(proc);
1703 out:
1704 return ret;
1705 }
1706
kvm_s390_get_machine(struct kvm * kvm,struct kvm_device_attr * attr)1707 static int kvm_s390_get_machine(struct kvm *kvm, struct kvm_device_attr *attr)
1708 {
1709 struct kvm_s390_vm_cpu_machine *mach;
1710 int ret = 0;
1711
1712 mach = kzalloc_obj(*mach, GFP_KERNEL_ACCOUNT);
1713 if (!mach) {
1714 ret = -ENOMEM;
1715 goto out;
1716 }
1717 get_cpu_id((struct cpuid *) &mach->cpuid);
1718 mach->ibc = sclp.ibc;
1719 memcpy(&mach->fac_mask, kvm->arch.model.fac_mask,
1720 S390_ARCH_FAC_LIST_SIZE_BYTE);
1721 memcpy((unsigned long *)&mach->fac_list, stfle_fac_list,
1722 sizeof(stfle_fac_list));
1723 VM_EVENT(kvm, 3, "GET: host ibc: 0x%4.4x, host cpuid: 0x%16.16llx",
1724 kvm->arch.model.ibc,
1725 kvm->arch.model.cpuid);
1726 VM_EVENT(kvm, 3, "GET: host facmask: 0x%16.16llx.%16.16llx.%16.16llx",
1727 mach->fac_mask[0],
1728 mach->fac_mask[1],
1729 mach->fac_mask[2]);
1730 VM_EVENT(kvm, 3, "GET: host faclist: 0x%16.16llx.%16.16llx.%16.16llx",
1731 mach->fac_list[0],
1732 mach->fac_list[1],
1733 mach->fac_list[2]);
1734 if (copy_to_user((void __user *)attr->addr, mach, sizeof(*mach)))
1735 ret = -EFAULT;
1736 kfree(mach);
1737 out:
1738 return ret;
1739 }
1740
kvm_s390_get_processor_feat(struct kvm * kvm,struct kvm_device_attr * attr)1741 static int kvm_s390_get_processor_feat(struct kvm *kvm,
1742 struct kvm_device_attr *attr)
1743 {
1744 struct kvm_s390_vm_cpu_feat data;
1745
1746 bitmap_to_arr64(data.feat, kvm->arch.cpu_feat, KVM_S390_VM_CPU_FEAT_NR_BITS);
1747 if (copy_to_user((void __user *)attr->addr, &data, sizeof(data)))
1748 return -EFAULT;
1749 VM_EVENT(kvm, 3, "GET: guest feat: 0x%16.16llx.0x%16.16llx.0x%16.16llx",
1750 data.feat[0],
1751 data.feat[1],
1752 data.feat[2]);
1753 return 0;
1754 }
1755
kvm_s390_get_machine_feat(struct kvm * kvm,struct kvm_device_attr * attr)1756 static int kvm_s390_get_machine_feat(struct kvm *kvm,
1757 struct kvm_device_attr *attr)
1758 {
1759 struct kvm_s390_vm_cpu_feat data;
1760
1761 bitmap_to_arr64(data.feat, kvm_s390_available_cpu_feat, KVM_S390_VM_CPU_FEAT_NR_BITS);
1762 if (copy_to_user((void __user *)attr->addr, &data, sizeof(data)))
1763 return -EFAULT;
1764 VM_EVENT(kvm, 3, "GET: host feat: 0x%16.16llx.0x%16.16llx.0x%16.16llx",
1765 data.feat[0],
1766 data.feat[1],
1767 data.feat[2]);
1768 return 0;
1769 }
1770
kvm_s390_get_processor_subfunc(struct kvm * kvm,struct kvm_device_attr * attr)1771 static int kvm_s390_get_processor_subfunc(struct kvm *kvm,
1772 struct kvm_device_attr *attr)
1773 {
1774 if (copy_to_user((void __user *)attr->addr, &kvm->arch.model.subfuncs,
1775 sizeof(struct kvm_s390_vm_cpu_subfunc)))
1776 return -EFAULT;
1777
1778 VM_EVENT(kvm, 3, "GET: guest PLO subfunc 0x%16.16lx.%16.16lx.%16.16lx.%16.16lx",
1779 ((unsigned long *) &kvm->arch.model.subfuncs.plo)[0],
1780 ((unsigned long *) &kvm->arch.model.subfuncs.plo)[1],
1781 ((unsigned long *) &kvm->arch.model.subfuncs.plo)[2],
1782 ((unsigned long *) &kvm->arch.model.subfuncs.plo)[3]);
1783 VM_EVENT(kvm, 3, "GET: guest PTFF subfunc 0x%16.16lx.%16.16lx",
1784 ((unsigned long *) &kvm->arch.model.subfuncs.ptff)[0],
1785 ((unsigned long *) &kvm->arch.model.subfuncs.ptff)[1]);
1786 VM_EVENT(kvm, 3, "GET: guest KMAC subfunc 0x%16.16lx.%16.16lx",
1787 ((unsigned long *) &kvm->arch.model.subfuncs.kmac)[0],
1788 ((unsigned long *) &kvm->arch.model.subfuncs.kmac)[1]);
1789 VM_EVENT(kvm, 3, "GET: guest KMC subfunc 0x%16.16lx.%16.16lx",
1790 ((unsigned long *) &kvm->arch.model.subfuncs.kmc)[0],
1791 ((unsigned long *) &kvm->arch.model.subfuncs.kmc)[1]);
1792 VM_EVENT(kvm, 3, "GET: guest KM subfunc 0x%16.16lx.%16.16lx",
1793 ((unsigned long *) &kvm->arch.model.subfuncs.km)[0],
1794 ((unsigned long *) &kvm->arch.model.subfuncs.km)[1]);
1795 VM_EVENT(kvm, 3, "GET: guest KIMD subfunc 0x%16.16lx.%16.16lx",
1796 ((unsigned long *) &kvm->arch.model.subfuncs.kimd)[0],
1797 ((unsigned long *) &kvm->arch.model.subfuncs.kimd)[1]);
1798 VM_EVENT(kvm, 3, "GET: guest KLMD subfunc 0x%16.16lx.%16.16lx",
1799 ((unsigned long *) &kvm->arch.model.subfuncs.klmd)[0],
1800 ((unsigned long *) &kvm->arch.model.subfuncs.klmd)[1]);
1801 VM_EVENT(kvm, 3, "GET: guest PCKMO subfunc 0x%16.16lx.%16.16lx",
1802 ((unsigned long *) &kvm->arch.model.subfuncs.pckmo)[0],
1803 ((unsigned long *) &kvm->arch.model.subfuncs.pckmo)[1]);
1804 VM_EVENT(kvm, 3, "GET: guest KMCTR subfunc 0x%16.16lx.%16.16lx",
1805 ((unsigned long *) &kvm->arch.model.subfuncs.kmctr)[0],
1806 ((unsigned long *) &kvm->arch.model.subfuncs.kmctr)[1]);
1807 VM_EVENT(kvm, 3, "GET: guest KMF subfunc 0x%16.16lx.%16.16lx",
1808 ((unsigned long *) &kvm->arch.model.subfuncs.kmf)[0],
1809 ((unsigned long *) &kvm->arch.model.subfuncs.kmf)[1]);
1810 VM_EVENT(kvm, 3, "GET: guest KMO subfunc 0x%16.16lx.%16.16lx",
1811 ((unsigned long *) &kvm->arch.model.subfuncs.kmo)[0],
1812 ((unsigned long *) &kvm->arch.model.subfuncs.kmo)[1]);
1813 VM_EVENT(kvm, 3, "GET: guest PCC subfunc 0x%16.16lx.%16.16lx",
1814 ((unsigned long *) &kvm->arch.model.subfuncs.pcc)[0],
1815 ((unsigned long *) &kvm->arch.model.subfuncs.pcc)[1]);
1816 VM_EVENT(kvm, 3, "GET: guest PPNO subfunc 0x%16.16lx.%16.16lx",
1817 ((unsigned long *) &kvm->arch.model.subfuncs.ppno)[0],
1818 ((unsigned long *) &kvm->arch.model.subfuncs.ppno)[1]);
1819 VM_EVENT(kvm, 3, "GET: guest KMA subfunc 0x%16.16lx.%16.16lx",
1820 ((unsigned long *) &kvm->arch.model.subfuncs.kma)[0],
1821 ((unsigned long *) &kvm->arch.model.subfuncs.kma)[1]);
1822 VM_EVENT(kvm, 3, "GET: guest KDSA subfunc 0x%16.16lx.%16.16lx",
1823 ((unsigned long *) &kvm->arch.model.subfuncs.kdsa)[0],
1824 ((unsigned long *) &kvm->arch.model.subfuncs.kdsa)[1]);
1825 VM_EVENT(kvm, 3, "GET: guest SORTL subfunc 0x%16.16lx.%16.16lx.%16.16lx.%16.16lx",
1826 ((unsigned long *) &kvm->arch.model.subfuncs.sortl)[0],
1827 ((unsigned long *) &kvm->arch.model.subfuncs.sortl)[1],
1828 ((unsigned long *) &kvm->arch.model.subfuncs.sortl)[2],
1829 ((unsigned long *) &kvm->arch.model.subfuncs.sortl)[3]);
1830 VM_EVENT(kvm, 3, "GET: guest DFLTCC subfunc 0x%16.16lx.%16.16lx.%16.16lx.%16.16lx",
1831 ((unsigned long *) &kvm->arch.model.subfuncs.dfltcc)[0],
1832 ((unsigned long *) &kvm->arch.model.subfuncs.dfltcc)[1],
1833 ((unsigned long *) &kvm->arch.model.subfuncs.dfltcc)[2],
1834 ((unsigned long *) &kvm->arch.model.subfuncs.dfltcc)[3]);
1835 VM_EVENT(kvm, 3, "GET: guest PFCR subfunc 0x%16.16lx.%16.16lx",
1836 ((unsigned long *) &kvm_s390_available_subfunc.pfcr)[0],
1837 ((unsigned long *) &kvm_s390_available_subfunc.pfcr)[1]);
1838
1839 return 0;
1840 }
1841
kvm_s390_get_machine_subfunc(struct kvm * kvm,struct kvm_device_attr * attr)1842 static int kvm_s390_get_machine_subfunc(struct kvm *kvm,
1843 struct kvm_device_attr *attr)
1844 {
1845 if (copy_to_user((void __user *)attr->addr, &kvm_s390_available_subfunc,
1846 sizeof(struct kvm_s390_vm_cpu_subfunc)))
1847 return -EFAULT;
1848
1849 VM_EVENT(kvm, 3, "GET: host PLO subfunc 0x%16.16lx.%16.16lx.%16.16lx.%16.16lx",
1850 ((unsigned long *) &kvm_s390_available_subfunc.plo)[0],
1851 ((unsigned long *) &kvm_s390_available_subfunc.plo)[1],
1852 ((unsigned long *) &kvm_s390_available_subfunc.plo)[2],
1853 ((unsigned long *) &kvm_s390_available_subfunc.plo)[3]);
1854 VM_EVENT(kvm, 3, "GET: host PTFF subfunc 0x%16.16lx.%16.16lx",
1855 ((unsigned long *) &kvm_s390_available_subfunc.ptff)[0],
1856 ((unsigned long *) &kvm_s390_available_subfunc.ptff)[1]);
1857 VM_EVENT(kvm, 3, "GET: host KMAC subfunc 0x%16.16lx.%16.16lx",
1858 ((unsigned long *) &kvm_s390_available_subfunc.kmac)[0],
1859 ((unsigned long *) &kvm_s390_available_subfunc.kmac)[1]);
1860 VM_EVENT(kvm, 3, "GET: host KMC subfunc 0x%16.16lx.%16.16lx",
1861 ((unsigned long *) &kvm_s390_available_subfunc.kmc)[0],
1862 ((unsigned long *) &kvm_s390_available_subfunc.kmc)[1]);
1863 VM_EVENT(kvm, 3, "GET: host KM subfunc 0x%16.16lx.%16.16lx",
1864 ((unsigned long *) &kvm_s390_available_subfunc.km)[0],
1865 ((unsigned long *) &kvm_s390_available_subfunc.km)[1]);
1866 VM_EVENT(kvm, 3, "GET: host KIMD subfunc 0x%16.16lx.%16.16lx",
1867 ((unsigned long *) &kvm_s390_available_subfunc.kimd)[0],
1868 ((unsigned long *) &kvm_s390_available_subfunc.kimd)[1]);
1869 VM_EVENT(kvm, 3, "GET: host KLMD subfunc 0x%16.16lx.%16.16lx",
1870 ((unsigned long *) &kvm_s390_available_subfunc.klmd)[0],
1871 ((unsigned long *) &kvm_s390_available_subfunc.klmd)[1]);
1872 VM_EVENT(kvm, 3, "GET: host PCKMO subfunc 0x%16.16lx.%16.16lx",
1873 ((unsigned long *) &kvm_s390_available_subfunc.pckmo)[0],
1874 ((unsigned long *) &kvm_s390_available_subfunc.pckmo)[1]);
1875 VM_EVENT(kvm, 3, "GET: host KMCTR subfunc 0x%16.16lx.%16.16lx",
1876 ((unsigned long *) &kvm_s390_available_subfunc.kmctr)[0],
1877 ((unsigned long *) &kvm_s390_available_subfunc.kmctr)[1]);
1878 VM_EVENT(kvm, 3, "GET: host KMF subfunc 0x%16.16lx.%16.16lx",
1879 ((unsigned long *) &kvm_s390_available_subfunc.kmf)[0],
1880 ((unsigned long *) &kvm_s390_available_subfunc.kmf)[1]);
1881 VM_EVENT(kvm, 3, "GET: host KMO subfunc 0x%16.16lx.%16.16lx",
1882 ((unsigned long *) &kvm_s390_available_subfunc.kmo)[0],
1883 ((unsigned long *) &kvm_s390_available_subfunc.kmo)[1]);
1884 VM_EVENT(kvm, 3, "GET: host PCC subfunc 0x%16.16lx.%16.16lx",
1885 ((unsigned long *) &kvm_s390_available_subfunc.pcc)[0],
1886 ((unsigned long *) &kvm_s390_available_subfunc.pcc)[1]);
1887 VM_EVENT(kvm, 3, "GET: host PPNO subfunc 0x%16.16lx.%16.16lx",
1888 ((unsigned long *) &kvm_s390_available_subfunc.ppno)[0],
1889 ((unsigned long *) &kvm_s390_available_subfunc.ppno)[1]);
1890 VM_EVENT(kvm, 3, "GET: host KMA subfunc 0x%16.16lx.%16.16lx",
1891 ((unsigned long *) &kvm_s390_available_subfunc.kma)[0],
1892 ((unsigned long *) &kvm_s390_available_subfunc.kma)[1]);
1893 VM_EVENT(kvm, 3, "GET: host KDSA subfunc 0x%16.16lx.%16.16lx",
1894 ((unsigned long *) &kvm_s390_available_subfunc.kdsa)[0],
1895 ((unsigned long *) &kvm_s390_available_subfunc.kdsa)[1]);
1896 VM_EVENT(kvm, 3, "GET: host SORTL subfunc 0x%16.16lx.%16.16lx.%16.16lx.%16.16lx",
1897 ((unsigned long *) &kvm_s390_available_subfunc.sortl)[0],
1898 ((unsigned long *) &kvm_s390_available_subfunc.sortl)[1],
1899 ((unsigned long *) &kvm_s390_available_subfunc.sortl)[2],
1900 ((unsigned long *) &kvm_s390_available_subfunc.sortl)[3]);
1901 VM_EVENT(kvm, 3, "GET: host DFLTCC subfunc 0x%16.16lx.%16.16lx.%16.16lx.%16.16lx",
1902 ((unsigned long *) &kvm_s390_available_subfunc.dfltcc)[0],
1903 ((unsigned long *) &kvm_s390_available_subfunc.dfltcc)[1],
1904 ((unsigned long *) &kvm_s390_available_subfunc.dfltcc)[2],
1905 ((unsigned long *) &kvm_s390_available_subfunc.dfltcc)[3]);
1906 VM_EVENT(kvm, 3, "GET: host PFCR subfunc 0x%16.16lx.%16.16lx",
1907 ((unsigned long *) &kvm_s390_available_subfunc.pfcr)[0],
1908 ((unsigned long *) &kvm_s390_available_subfunc.pfcr)[1]);
1909
1910 return 0;
1911 }
1912
kvm_s390_get_processor_uv_feat(struct kvm * kvm,struct kvm_device_attr * attr)1913 static int kvm_s390_get_processor_uv_feat(struct kvm *kvm, struct kvm_device_attr *attr)
1914 {
1915 struct kvm_s390_vm_cpu_uv_feat __user *dst = (void __user *)attr->addr;
1916 unsigned long feat = kvm->arch.model.uv_feat_guest.feat;
1917
1918 if (put_user(feat, &dst->feat))
1919 return -EFAULT;
1920 VM_EVENT(kvm, 3, "GET: guest UV-feat: 0x%16.16lx", feat);
1921
1922 return 0;
1923 }
1924
kvm_s390_get_machine_uv_feat(struct kvm * kvm,struct kvm_device_attr * attr)1925 static int kvm_s390_get_machine_uv_feat(struct kvm *kvm, struct kvm_device_attr *attr)
1926 {
1927 struct kvm_s390_vm_cpu_uv_feat __user *dst = (void __user *)attr->addr;
1928 unsigned long feat;
1929
1930 BUILD_BUG_ON(sizeof(*dst) != sizeof(uv_info.uv_feature_indications));
1931
1932 feat = uv_info.uv_feature_indications & KVM_S390_VM_CPU_UV_FEAT_GUEST_MASK;
1933 if (put_user(feat, &dst->feat))
1934 return -EFAULT;
1935 VM_EVENT(kvm, 3, "GET: guest UV-feat: 0x%16.16lx", feat);
1936
1937 return 0;
1938 }
1939
kvm_s390_get_cpu_model(struct kvm * kvm,struct kvm_device_attr * attr)1940 static int kvm_s390_get_cpu_model(struct kvm *kvm, struct kvm_device_attr *attr)
1941 {
1942 int ret = -ENXIO;
1943
1944 switch (attr->attr) {
1945 case KVM_S390_VM_CPU_PROCESSOR:
1946 ret = kvm_s390_get_processor(kvm, attr);
1947 break;
1948 case KVM_S390_VM_CPU_MACHINE:
1949 ret = kvm_s390_get_machine(kvm, attr);
1950 break;
1951 case KVM_S390_VM_CPU_PROCESSOR_FEAT:
1952 ret = kvm_s390_get_processor_feat(kvm, attr);
1953 break;
1954 case KVM_S390_VM_CPU_MACHINE_FEAT:
1955 ret = kvm_s390_get_machine_feat(kvm, attr);
1956 break;
1957 case KVM_S390_VM_CPU_PROCESSOR_SUBFUNC:
1958 ret = kvm_s390_get_processor_subfunc(kvm, attr);
1959 break;
1960 case KVM_S390_VM_CPU_MACHINE_SUBFUNC:
1961 ret = kvm_s390_get_machine_subfunc(kvm, attr);
1962 break;
1963 case KVM_S390_VM_CPU_PROCESSOR_UV_FEAT_GUEST:
1964 ret = kvm_s390_get_processor_uv_feat(kvm, attr);
1965 break;
1966 case KVM_S390_VM_CPU_MACHINE_UV_FEAT_GUEST:
1967 ret = kvm_s390_get_machine_uv_feat(kvm, attr);
1968 break;
1969 }
1970 return ret;
1971 }
1972
1973 /**
1974 * kvm_s390_update_topology_change_report - update CPU topology change report
1975 * @kvm: guest KVM description
1976 * @val: set or clear the MTCR bit
1977 *
1978 * Updates the Multiprocessor Topology-Change-Report bit to signal
1979 * the guest with a topology change.
1980 * This is only relevant if the topology facility is present.
1981 */
kvm_s390_update_topology_change_report(struct kvm * kvm,bool val)1982 static void kvm_s390_update_topology_change_report(struct kvm *kvm, bool val)
1983 {
1984 union sca_utility new, old;
1985 struct esca_block *sca;
1986
1987 sca = kvm->arch.sca;
1988 old = READ_ONCE(sca->utility);
1989 do {
1990 new = old;
1991 new.mtcr = val;
1992 } while (!try_cmpxchg(&sca->utility.val, &old.val, new.val));
1993 }
1994
kvm_s390_set_topo_change_indication(struct kvm * kvm,struct kvm_device_attr * attr)1995 static int kvm_s390_set_topo_change_indication(struct kvm *kvm,
1996 struct kvm_device_attr *attr)
1997 {
1998 if (!test_kvm_facility(kvm, 11))
1999 return -ENXIO;
2000
2001 kvm_s390_update_topology_change_report(kvm, !!attr->attr);
2002 return 0;
2003 }
2004
kvm_s390_get_topo_change_indication(struct kvm * kvm,struct kvm_device_attr * attr)2005 static int kvm_s390_get_topo_change_indication(struct kvm *kvm,
2006 struct kvm_device_attr *attr)
2007 {
2008 u8 topo;
2009
2010 if (!test_kvm_facility(kvm, 11))
2011 return -ENXIO;
2012
2013 topo = kvm->arch.sca->utility.mtcr;
2014
2015 return put_user(topo, (u8 __user *)attr->addr);
2016 }
2017
kvm_s390_vm_set_attr(struct kvm * kvm,struct kvm_device_attr * attr)2018 static int kvm_s390_vm_set_attr(struct kvm *kvm, struct kvm_device_attr *attr)
2019 {
2020 int ret;
2021
2022 switch (attr->group) {
2023 case KVM_S390_VM_MEM_CTRL:
2024 ret = kvm_s390_set_mem_control(kvm, attr);
2025 break;
2026 case KVM_S390_VM_TOD:
2027 ret = kvm_s390_set_tod(kvm, attr);
2028 break;
2029 case KVM_S390_VM_CPU_MODEL:
2030 ret = kvm_s390_set_cpu_model(kvm, attr);
2031 break;
2032 case KVM_S390_VM_CRYPTO:
2033 ret = kvm_s390_vm_set_crypto(kvm, attr);
2034 break;
2035 case KVM_S390_VM_MIGRATION:
2036 ret = kvm_s390_vm_set_migration(kvm, attr);
2037 break;
2038 case KVM_S390_VM_CPU_TOPOLOGY:
2039 ret = kvm_s390_set_topo_change_indication(kvm, attr);
2040 break;
2041 default:
2042 ret = -ENXIO;
2043 break;
2044 }
2045
2046 return ret;
2047 }
2048
kvm_s390_vm_get_attr(struct kvm * kvm,struct kvm_device_attr * attr)2049 static int kvm_s390_vm_get_attr(struct kvm *kvm, struct kvm_device_attr *attr)
2050 {
2051 int ret;
2052
2053 switch (attr->group) {
2054 case KVM_S390_VM_MEM_CTRL:
2055 ret = kvm_s390_get_mem_control(kvm, attr);
2056 break;
2057 case KVM_S390_VM_TOD:
2058 ret = kvm_s390_get_tod(kvm, attr);
2059 break;
2060 case KVM_S390_VM_CPU_MODEL:
2061 ret = kvm_s390_get_cpu_model(kvm, attr);
2062 break;
2063 case KVM_S390_VM_MIGRATION:
2064 ret = kvm_s390_vm_get_migration(kvm, attr);
2065 break;
2066 case KVM_S390_VM_CPU_TOPOLOGY:
2067 ret = kvm_s390_get_topo_change_indication(kvm, attr);
2068 break;
2069 default:
2070 ret = -ENXIO;
2071 break;
2072 }
2073
2074 return ret;
2075 }
2076
kvm_s390_vm_has_attr(struct kvm * kvm,struct kvm_device_attr * attr)2077 static int kvm_s390_vm_has_attr(struct kvm *kvm, struct kvm_device_attr *attr)
2078 {
2079 int ret;
2080
2081 switch (attr->group) {
2082 case KVM_S390_VM_MEM_CTRL:
2083 switch (attr->attr) {
2084 case KVM_S390_VM_MEM_ENABLE_CMMA:
2085 case KVM_S390_VM_MEM_CLR_CMMA:
2086 ret = sclp.has_cmma ? 0 : -ENXIO;
2087 break;
2088 case KVM_S390_VM_MEM_LIMIT_SIZE:
2089 ret = 0;
2090 break;
2091 default:
2092 ret = -ENXIO;
2093 break;
2094 }
2095 break;
2096 case KVM_S390_VM_TOD:
2097 switch (attr->attr) {
2098 case KVM_S390_VM_TOD_LOW:
2099 case KVM_S390_VM_TOD_HIGH:
2100 ret = 0;
2101 break;
2102 default:
2103 ret = -ENXIO;
2104 break;
2105 }
2106 break;
2107 case KVM_S390_VM_CPU_MODEL:
2108 switch (attr->attr) {
2109 case KVM_S390_VM_CPU_PROCESSOR:
2110 case KVM_S390_VM_CPU_MACHINE:
2111 case KVM_S390_VM_CPU_PROCESSOR_FEAT:
2112 case KVM_S390_VM_CPU_MACHINE_FEAT:
2113 case KVM_S390_VM_CPU_MACHINE_SUBFUNC:
2114 case KVM_S390_VM_CPU_PROCESSOR_SUBFUNC:
2115 case KVM_S390_VM_CPU_MACHINE_UV_FEAT_GUEST:
2116 case KVM_S390_VM_CPU_PROCESSOR_UV_FEAT_GUEST:
2117 ret = 0;
2118 break;
2119 default:
2120 ret = -ENXIO;
2121 break;
2122 }
2123 break;
2124 case KVM_S390_VM_CRYPTO:
2125 switch (attr->attr) {
2126 case KVM_S390_VM_CRYPTO_ENABLE_AES_KW:
2127 case KVM_S390_VM_CRYPTO_ENABLE_DEA_KW:
2128 case KVM_S390_VM_CRYPTO_DISABLE_AES_KW:
2129 case KVM_S390_VM_CRYPTO_DISABLE_DEA_KW:
2130 ret = 0;
2131 break;
2132 case KVM_S390_VM_CRYPTO_ENABLE_APIE:
2133 case KVM_S390_VM_CRYPTO_DISABLE_APIE:
2134 ret = ap_instructions_available() ? 0 : -ENXIO;
2135 break;
2136 default:
2137 ret = -ENXIO;
2138 break;
2139 }
2140 break;
2141 case KVM_S390_VM_MIGRATION:
2142 ret = 0;
2143 break;
2144 case KVM_S390_VM_CPU_TOPOLOGY:
2145 ret = test_kvm_facility(kvm, 11) ? 0 : -ENXIO;
2146 break;
2147 default:
2148 ret = -ENXIO;
2149 break;
2150 }
2151
2152 return ret;
2153 }
2154
kvm_s390_get_skeys(struct kvm * kvm,struct kvm_s390_skeys * args)2155 static int kvm_s390_get_skeys(struct kvm *kvm, struct kvm_s390_skeys *args)
2156 {
2157 union skey *keys;
2158 int i, r = 0;
2159
2160 if (args->flags != 0)
2161 return -EINVAL;
2162
2163 /* Is this guest using storage keys? */
2164 if (!uses_skeys(kvm->arch.gmap))
2165 return KVM_S390_GET_SKEYS_NONE;
2166
2167 /* Enforce sane limit on memory allocation */
2168 if (args->count < 1 || args->count > KVM_S390_SKEYS_MAX)
2169 return -EINVAL;
2170
2171 keys = kvmalloc_objs(*keys, args->count, GFP_KERNEL_ACCOUNT);
2172 if (!keys)
2173 return -ENOMEM;
2174
2175 scoped_guard(read_lock, &kvm->mmu_lock) {
2176 for (i = 0; i < args->count; i++) {
2177 r = dat_get_storage_key(kvm->arch.gmap->asce,
2178 args->start_gfn + i, keys + i);
2179 if (r)
2180 break;
2181 }
2182 }
2183
2184 if (!r) {
2185 r = copy_to_user((uint8_t __user *)args->skeydata_addr, keys,
2186 sizeof(uint8_t) * args->count);
2187 if (r)
2188 r = -EFAULT;
2189 }
2190
2191 kvfree(keys);
2192 return r <= 0 ? r : -EFAULT;
2193 }
2194
kvm_s390_set_skeys(struct kvm * kvm,struct kvm_s390_skeys * args)2195 static int kvm_s390_set_skeys(struct kvm *kvm, struct kvm_s390_skeys *args)
2196 {
2197 struct kvm_s390_mmu_cache *mc;
2198 union skey *keys;
2199 int i, r = 0;
2200
2201 if (args->flags != 0)
2202 return -EINVAL;
2203
2204 /* Enforce sane limit on memory allocation */
2205 if (args->count < 1 || args->count > KVM_S390_SKEYS_MAX)
2206 return -EINVAL;
2207
2208 keys = kvmalloc_objs(*keys, args->count, GFP_KERNEL_ACCOUNT);
2209 if (!keys)
2210 return -ENOMEM;
2211
2212 r = copy_from_user(keys, (uint8_t __user *)args->skeydata_addr,
2213 sizeof(uint8_t) * args->count);
2214 if (r) {
2215 r = -EFAULT;
2216 goto out;
2217 }
2218
2219 /* Enable storage key handling for the guest */
2220 r = gmap_enable_skeys(kvm->arch.gmap);
2221 if (r)
2222 goto out;
2223
2224 r = -EINVAL;
2225 for (i = 0; i < args->count; i++) {
2226 /* Lowest order bit is reserved */
2227 if (keys[i].zero)
2228 goto out;
2229 }
2230
2231 mc = kvm_s390_new_mmu_cache();
2232 if (!mc) {
2233 r = -ENOMEM;
2234 goto out;
2235 }
2236
2237 r = 0;
2238 do {
2239 r = kvm_s390_mmu_cache_topup(mc);
2240 if (r == -ENOMEM)
2241 break;
2242 scoped_guard(read_lock, &kvm->mmu_lock) {
2243 for (i = 0 ; i < args->count; i++) {
2244 r = dat_set_storage_key(mc, kvm->arch.gmap->asce,
2245 args->start_gfn + i, keys[i], 0);
2246 if (r)
2247 break;
2248 }
2249 }
2250 } while (r == -ENOMEM);
2251 kvm_s390_free_mmu_cache(mc);
2252 out:
2253 kvfree(keys);
2254 return r <= 0 ? r : -EFAULT;
2255 }
2256
2257 /*
2258 * This function searches for the next page with dirty CMMA attributes, and
2259 * saves the attributes in the buffer up to either the end of the buffer or
2260 * until a block of at least KVM_S390_MAX_BIT_DISTANCE clean bits is found;
2261 * no trailing clean bytes are saved.
2262 * In case no dirty bits were found, or if CMMA was not enabled or used, the
2263 * output buffer will indicate 0 as length.
2264 */
kvm_s390_get_cmma_bits(struct kvm * kvm,struct kvm_s390_cmma_log * args)2265 static int kvm_s390_get_cmma_bits(struct kvm *kvm,
2266 struct kvm_s390_cmma_log *args)
2267 {
2268 int peek, ret;
2269 u8 *values;
2270
2271 if (!kvm->arch.use_cmma)
2272 return -ENXIO;
2273 /* Invalid/unsupported flags were specified */
2274 if (args->flags & ~KVM_S390_CMMA_PEEK)
2275 return -EINVAL;
2276 /* Migration mode query, and we are not doing a migration */
2277 peek = !!(args->flags & KVM_S390_CMMA_PEEK);
2278 if (!peek && !kvm->arch.migration_mode)
2279 return -EINVAL;
2280 /* CMMA is disabled or was not used, or the buffer has length zero */
2281 args->count = min(args->count, KVM_S390_CMMA_SIZE_MAX);
2282 if (!args->count || !uses_cmm(kvm->arch.gmap)) {
2283 memset(args, 0, sizeof(*args));
2284 return 0;
2285 }
2286 /* We are not peeking, and there are no dirty pages */
2287 if (!peek && !atomic64_read(&kvm->arch.cmma_dirty_pages)) {
2288 memset(args, 0, sizeof(*args));
2289 return 0;
2290 }
2291
2292 values = vzalloc(args->count);
2293 if (!values)
2294 return -ENOMEM;
2295
2296 scoped_guard(read_lock, &kvm->mmu_lock) {
2297 if (peek)
2298 ret = dat_peek_cmma(args->start_gfn, kvm->arch.gmap->asce, &args->count,
2299 values);
2300 else
2301 ret = dat_get_cmma(kvm->arch.gmap->asce, &args->start_gfn, &args->count,
2302 values, &kvm->arch.cmma_dirty_pages);
2303 }
2304
2305 if (kvm->arch.migration_mode)
2306 args->remaining = atomic64_read(&kvm->arch.cmma_dirty_pages);
2307 else
2308 args->remaining = 0;
2309
2310 if (copy_to_user((void __user *)args->values, values, args->count))
2311 ret = -EFAULT;
2312
2313 vfree(values);
2314 return ret;
2315 }
2316
2317 /*
2318 * This function sets the CMMA attributes for the given pages. If the input
2319 * buffer has zero length, no action is taken, otherwise the attributes are
2320 * set and the mm->context.uses_cmm flag is set.
2321 */
kvm_s390_set_cmma_bits(struct kvm * kvm,const struct kvm_s390_cmma_log * args)2322 static int kvm_s390_set_cmma_bits(struct kvm *kvm,
2323 const struct kvm_s390_cmma_log *args)
2324 {
2325 struct kvm_s390_mmu_cache *mc __free(kvm_s390_mmu_cache) = NULL;
2326 u8 *bits __free(kvfree) = NULL;
2327 int r = 0;
2328
2329 if (!kvm->arch.use_cmma)
2330 return -ENXIO;
2331 /* invalid/unsupported flags */
2332 if (args->flags != 0)
2333 return -EINVAL;
2334 /* Enforce sane limit on memory allocation */
2335 if (args->count > KVM_S390_CMMA_SIZE_MAX)
2336 return -EINVAL;
2337 /* Nothing to do */
2338 if (args->count == 0)
2339 return 0;
2340
2341 mc = kvm_s390_new_mmu_cache();
2342 if (!mc)
2343 return -ENOMEM;
2344 bits = vmalloc(array_size(sizeof(*bits), args->count));
2345 if (!bits)
2346 return -ENOMEM;
2347
2348 r = copy_from_user(bits, (void __user *)args->values, args->count);
2349 if (r)
2350 return -EFAULT;
2351
2352 do {
2353 r = kvm_s390_mmu_cache_topup(mc);
2354 if (r)
2355 return r;
2356 scoped_guard(read_lock, &kvm->mmu_lock) {
2357 r = dat_set_cmma_bits(mc, kvm->arch.gmap->asce, args->start_gfn,
2358 args->count, args->mask, bits);
2359 }
2360 } while (r == -ENOMEM);
2361
2362 set_bit(GMAP_FLAG_USES_CMM, &kvm->arch.gmap->flags);
2363
2364 return r <= 0 ? r : -EFAULT;
2365 }
2366
2367 /**
2368 * kvm_s390_cpus_from_pv - Convert all protected vCPUs in a protected VM to
2369 * non protected.
2370 * @kvm: the VM whose protected vCPUs are to be converted
2371 * @rc: return value for the RC field of the UVC (in case of error)
2372 * @rrc: return value for the RRC field of the UVC (in case of error)
2373 *
2374 * Does not stop in case of error, tries to convert as many
2375 * CPUs as possible. In case of error, the RC and RRC of the last error are
2376 * returned.
2377 *
2378 * Return: 0 in case of success, otherwise -EIO
2379 */
kvm_s390_cpus_from_pv(struct kvm * kvm,u16 * rc,u16 * rrc)2380 int kvm_s390_cpus_from_pv(struct kvm *kvm, u16 *rc, u16 *rrc)
2381 {
2382 struct kvm_vcpu *vcpu;
2383 unsigned long i;
2384 u16 _rc, _rrc;
2385 int ret = 0;
2386
2387 /*
2388 * We ignore failures and try to destroy as many CPUs as possible.
2389 * At the same time we must not free the assigned resources when
2390 * this fails, as the ultravisor has still access to that memory.
2391 * So kvm_s390_pv_destroy_cpu can leave a "wanted" memory leak
2392 * behind.
2393 * We want to return the first failure rc and rrc, though.
2394 */
2395 kvm_for_each_vcpu(i, vcpu, kvm) {
2396 mutex_lock(&vcpu->mutex);
2397 if (kvm_s390_pv_destroy_cpu(vcpu, &_rc, &_rrc) && !ret) {
2398 *rc = _rc;
2399 *rrc = _rrc;
2400 ret = -EIO;
2401 }
2402 mutex_unlock(&vcpu->mutex);
2403 }
2404 /* Ensure that we re-enable gisa if the non-PV guest used it but the PV guest did not. */
2405 if (use_gisa)
2406 kvm_s390_gisa_enable(kvm);
2407 return ret;
2408 }
2409
2410 /**
2411 * kvm_s390_cpus_to_pv - Convert all non-protected vCPUs in a protected VM
2412 * to protected.
2413 * @kvm: the VM whose protected vCPUs are to be converted
2414 * @rc: return value for the RC field of the UVC (in case of error)
2415 * @rrc: return value for the RRC field of the UVC (in case of error)
2416 *
2417 * Tries to undo the conversion in case of error.
2418 *
2419 * Return: 0 in case of success, otherwise -EIO
2420 */
kvm_s390_cpus_to_pv(struct kvm * kvm,u16 * rc,u16 * rrc)2421 static int kvm_s390_cpus_to_pv(struct kvm *kvm, u16 *rc, u16 *rrc)
2422 {
2423 unsigned long i;
2424 int r = 0;
2425 u16 dummy;
2426
2427 struct kvm_vcpu *vcpu;
2428
2429 /* Disable the GISA if the ultravisor does not support AIV. */
2430 if (!uv_has_feature(BIT_UV_FEAT_AIV))
2431 kvm_s390_gisa_disable(kvm);
2432
2433 kvm_for_each_vcpu(i, vcpu, kvm) {
2434 mutex_lock(&vcpu->mutex);
2435 r = kvm_s390_pv_create_cpu(vcpu, rc, rrc);
2436 mutex_unlock(&vcpu->mutex);
2437 if (r)
2438 break;
2439 }
2440 if (r)
2441 kvm_s390_cpus_from_pv(kvm, &dummy, &dummy);
2442 return r;
2443 }
2444
2445 /*
2446 * Here we provide user space with a direct interface to query UV
2447 * related data like UV maxima and available features as well as
2448 * feature specific data.
2449 *
2450 * To facilitate future extension of the data structures we'll try to
2451 * write data up to the maximum requested length.
2452 */
kvm_s390_handle_pv_info(struct kvm_s390_pv_info * info)2453 static ssize_t kvm_s390_handle_pv_info(struct kvm_s390_pv_info *info)
2454 {
2455 ssize_t len_min;
2456
2457 switch (info->header.id) {
2458 case KVM_PV_INFO_VM: {
2459 len_min = sizeof(info->header) + sizeof(info->vm);
2460
2461 if (info->header.len_max < len_min)
2462 return -EINVAL;
2463
2464 memcpy(info->vm.inst_calls_list,
2465 uv_info.inst_calls_list,
2466 sizeof(uv_info.inst_calls_list));
2467
2468 /* It's max cpuid not max cpus, so it's off by one */
2469 info->vm.max_cpus = uv_info.max_guest_cpu_id + 1;
2470 info->vm.max_guests = uv_info.max_num_sec_conf;
2471 info->vm.max_guest_addr = uv_info.max_sec_stor_addr;
2472 info->vm.feature_indication = uv_info.uv_feature_indications;
2473
2474 return len_min;
2475 }
2476 case KVM_PV_INFO_DUMP: {
2477 len_min = sizeof(info->header) + sizeof(info->dump);
2478
2479 if (info->header.len_max < len_min)
2480 return -EINVAL;
2481
2482 info->dump.dump_cpu_buffer_len = uv_info.guest_cpu_stor_len;
2483 info->dump.dump_config_mem_buffer_per_1m = uv_info.conf_dump_storage_state_len;
2484 info->dump.dump_config_finalize_len = uv_info.conf_dump_finalize_len;
2485 return len_min;
2486 }
2487 default:
2488 return -EINVAL;
2489 }
2490 }
2491
kvm_s390_pv_dmp(struct kvm * kvm,struct kvm_pv_cmd * cmd,struct kvm_s390_pv_dmp dmp)2492 static int kvm_s390_pv_dmp(struct kvm *kvm, struct kvm_pv_cmd *cmd,
2493 struct kvm_s390_pv_dmp dmp)
2494 {
2495 int r = -EINVAL;
2496 void __user *result_buff = (void __user *)dmp.buff_addr;
2497
2498 switch (dmp.subcmd) {
2499 case KVM_PV_DUMP_INIT: {
2500 if (kvm->arch.pv.dumping)
2501 break;
2502
2503 /*
2504 * Block SIE entry as concurrent dump UVCs could lead
2505 * to validities.
2506 */
2507 kvm_s390_vcpu_block_all(kvm);
2508
2509 r = uv_cmd_nodata(kvm_s390_pv_get_handle(kvm),
2510 UVC_CMD_DUMP_INIT, &cmd->rc, &cmd->rrc);
2511 KVM_UV_EVENT(kvm, 3, "PROTVIRT DUMP INIT: rc %x rrc %x",
2512 cmd->rc, cmd->rrc);
2513 if (!r) {
2514 kvm->arch.pv.dumping = true;
2515 } else {
2516 kvm_s390_vcpu_unblock_all(kvm);
2517 r = -EINVAL;
2518 }
2519 break;
2520 }
2521 case KVM_PV_DUMP_CONFIG_STOR_STATE: {
2522 if (!kvm->arch.pv.dumping)
2523 break;
2524
2525 /*
2526 * gaddr is an output parameter since we might stop
2527 * early. As dmp will be copied back in our caller, we
2528 * don't need to do it ourselves.
2529 */
2530 r = kvm_s390_pv_dump_stor_state(kvm, result_buff, &dmp.gaddr, dmp.buff_len,
2531 &cmd->rc, &cmd->rrc);
2532 break;
2533 }
2534 case KVM_PV_DUMP_COMPLETE: {
2535 if (!kvm->arch.pv.dumping)
2536 break;
2537
2538 r = -EINVAL;
2539 if (dmp.buff_len < uv_info.conf_dump_finalize_len)
2540 break;
2541
2542 r = kvm_s390_pv_dump_complete(kvm, result_buff,
2543 &cmd->rc, &cmd->rrc);
2544 break;
2545 }
2546 default:
2547 r = -ENOTTY;
2548 break;
2549 }
2550
2551 return r;
2552 }
2553
kvm_s390_handle_pv(struct kvm * kvm,struct kvm_pv_cmd * cmd)2554 static int kvm_s390_handle_pv(struct kvm *kvm, struct kvm_pv_cmd *cmd)
2555 {
2556 const bool need_lock = (cmd->cmd != KVM_PV_ASYNC_CLEANUP_PERFORM);
2557 void __user *argp = (void __user *)cmd->data;
2558 int r = 0;
2559 u16 dummy;
2560
2561 if (need_lock)
2562 mutex_lock(&kvm->lock);
2563
2564 switch (cmd->cmd) {
2565 case KVM_PV_ENABLE: {
2566 r = -EINVAL;
2567 if (kvm_s390_pv_is_protected(kvm))
2568 break;
2569
2570 kvm_s390_unmap_all_adapters(kvm);
2571 mmap_write_lock(kvm->mm);
2572 /*
2573 * Disable creation of new THPs. Existing THPs can stay, they
2574 * will be split when any part of them gets imported.
2575 */
2576 mm_flags_clear(MMF_DISABLE_THP_EXCEPT_ADVISED, kvm->mm);
2577 mm_flags_set(MMF_DISABLE_THP_COMPLETELY, kvm->mm);
2578 set_bit(GMAP_FLAG_EXPORT_ON_UNMAP, &kvm->arch.gmap->flags);
2579 r = gmap_helper_disable_cow_sharing();
2580 mmap_write_unlock(kvm->mm);
2581 if (r)
2582 break;
2583
2584 r = kvm_s390_pv_init_vm(kvm, &cmd->rc, &cmd->rrc);
2585 if (r)
2586 break;
2587
2588 r = kvm_s390_cpus_to_pv(kvm, &cmd->rc, &cmd->rrc);
2589 if (r)
2590 kvm_s390_pv_deinit_vm(kvm, &dummy, &dummy);
2591
2592 /* we need to block service interrupts from now on */
2593 set_bit(IRQ_PEND_EXT_SERVICE, &kvm->arch.float_int.masked_irqs);
2594 break;
2595 }
2596 case KVM_PV_ASYNC_CLEANUP_PREPARE:
2597 r = -EINVAL;
2598 if (!kvm_s390_pv_is_protected(kvm) || !async_destroy)
2599 break;
2600
2601 r = kvm_s390_cpus_from_pv(kvm, &cmd->rc, &cmd->rrc);
2602 /*
2603 * If a CPU could not be destroyed, destroy VM will also fail.
2604 * There is no point in trying to destroy it. Instead return
2605 * the rc and rrc from the first CPU that failed destroying.
2606 */
2607 if (r)
2608 break;
2609 r = kvm_s390_pv_set_aside(kvm, &cmd->rc, &cmd->rrc);
2610
2611 /* no need to block service interrupts any more */
2612 clear_bit(IRQ_PEND_EXT_SERVICE, &kvm->arch.float_int.masked_irqs);
2613 break;
2614 case KVM_PV_ASYNC_CLEANUP_PERFORM:
2615 r = -EINVAL;
2616 if (!async_destroy)
2617 break;
2618 /* kvm->lock must not be held; this is asserted inside the function. */
2619 r = kvm_s390_pv_deinit_aside_vm(kvm, &cmd->rc, &cmd->rrc);
2620 break;
2621 case KVM_PV_DISABLE: {
2622 r = -EINVAL;
2623 if (!kvm_s390_pv_is_protected(kvm))
2624 break;
2625
2626 r = kvm_s390_cpus_from_pv(kvm, &cmd->rc, &cmd->rrc);
2627 /*
2628 * If a CPU could not be destroyed, destroy VM will also fail.
2629 * There is no point in trying to destroy it. Instead return
2630 * the rc and rrc from the first CPU that failed destroying.
2631 */
2632 if (r)
2633 break;
2634 r = kvm_s390_pv_deinit_cleanup_all(kvm, &cmd->rc, &cmd->rrc);
2635
2636 /* no need to block service interrupts any more */
2637 clear_bit(IRQ_PEND_EXT_SERVICE, &kvm->arch.float_int.masked_irqs);
2638 break;
2639 }
2640 case KVM_PV_SET_SEC_PARMS: {
2641 struct kvm_s390_pv_sec_parm parms = {};
2642 void *hdr;
2643
2644 r = -EINVAL;
2645 if (!kvm_s390_pv_is_protected(kvm))
2646 break;
2647
2648 r = -EFAULT;
2649 if (copy_from_user(&parms, argp, sizeof(parms)))
2650 break;
2651
2652 /* Currently restricted to 1MiB */
2653 r = -EINVAL;
2654 if (parms.length > SZ_1M)
2655 break;
2656
2657 r = -ENOMEM;
2658 hdr = vmalloc(parms.length);
2659 if (!hdr)
2660 break;
2661
2662 r = -EFAULT;
2663 if (!copy_from_user(hdr, (void __user *)parms.origin,
2664 parms.length))
2665 r = kvm_s390_pv_set_sec_parms(kvm, hdr, parms.length,
2666 &cmd->rc, &cmd->rrc);
2667
2668 vfree(hdr);
2669 break;
2670 }
2671 case KVM_PV_UNPACK: {
2672 struct kvm_s390_pv_unp unp = {};
2673
2674 r = -EINVAL;
2675 if (!kvm_s390_pv_is_protected(kvm) || !mm_is_protected(kvm->mm))
2676 break;
2677
2678 r = -EFAULT;
2679 if (copy_from_user(&unp, argp, sizeof(unp)))
2680 break;
2681
2682 r = kvm_s390_pv_unpack(kvm, unp.addr, unp.size, unp.tweak,
2683 &cmd->rc, &cmd->rrc);
2684 break;
2685 }
2686 case KVM_PV_VERIFY: {
2687 r = -EINVAL;
2688 if (!kvm_s390_pv_is_protected(kvm))
2689 break;
2690
2691 r = uv_cmd_nodata(kvm_s390_pv_get_handle(kvm),
2692 UVC_CMD_VERIFY_IMG, &cmd->rc, &cmd->rrc);
2693 KVM_UV_EVENT(kvm, 3, "PROTVIRT VERIFY: rc %x rrc %x", cmd->rc,
2694 cmd->rrc);
2695 break;
2696 }
2697 case KVM_PV_PREP_RESET: {
2698 r = -EINVAL;
2699 if (!kvm_s390_pv_is_protected(kvm))
2700 break;
2701
2702 r = uv_cmd_nodata(kvm_s390_pv_get_handle(kvm),
2703 UVC_CMD_PREPARE_RESET, &cmd->rc, &cmd->rrc);
2704 KVM_UV_EVENT(kvm, 3, "PROTVIRT PREP RESET: rc %x rrc %x",
2705 cmd->rc, cmd->rrc);
2706 break;
2707 }
2708 case KVM_PV_UNSHARE_ALL: {
2709 r = -EINVAL;
2710 if (!kvm_s390_pv_is_protected(kvm))
2711 break;
2712
2713 r = uv_cmd_nodata(kvm_s390_pv_get_handle(kvm),
2714 UVC_CMD_SET_UNSHARE_ALL, &cmd->rc, &cmd->rrc);
2715 KVM_UV_EVENT(kvm, 3, "PROTVIRT UNSHARE: rc %x rrc %x",
2716 cmd->rc, cmd->rrc);
2717 break;
2718 }
2719 case KVM_PV_INFO: {
2720 struct kvm_s390_pv_info info = {};
2721 ssize_t data_len;
2722
2723 /*
2724 * No need to check the VM protection here.
2725 *
2726 * Maybe user space wants to query some of the data
2727 * when the VM is still unprotected. If we see the
2728 * need to fence a new data command we can still
2729 * return an error in the info handler.
2730 */
2731
2732 r = -EFAULT;
2733 if (copy_from_user(&info, argp, sizeof(info.header)))
2734 break;
2735
2736 r = -EINVAL;
2737 if (info.header.len_max < sizeof(info.header))
2738 break;
2739
2740 data_len = kvm_s390_handle_pv_info(&info);
2741 if (data_len < 0) {
2742 r = data_len;
2743 break;
2744 }
2745 /*
2746 * If a data command struct is extended (multiple
2747 * times) this can be used to determine how much of it
2748 * is valid.
2749 */
2750 info.header.len_written = data_len;
2751
2752 r = -EFAULT;
2753 if (copy_to_user(argp, &info, data_len))
2754 break;
2755
2756 r = 0;
2757 break;
2758 }
2759 case KVM_PV_DUMP: {
2760 struct kvm_s390_pv_dmp dmp;
2761
2762 r = -EINVAL;
2763 if (!kvm_s390_pv_is_protected(kvm))
2764 break;
2765
2766 r = -EFAULT;
2767 if (copy_from_user(&dmp, argp, sizeof(dmp)))
2768 break;
2769
2770 r = kvm_s390_pv_dmp(kvm, cmd, dmp);
2771 if (r)
2772 break;
2773
2774 if (copy_to_user(argp, &dmp, sizeof(dmp))) {
2775 r = -EFAULT;
2776 break;
2777 }
2778
2779 break;
2780 }
2781 default:
2782 r = -ENOTTY;
2783 }
2784 if (need_lock)
2785 mutex_unlock(&kvm->lock);
2786
2787 return r;
2788 }
2789
mem_op_validate_common(struct kvm_s390_mem_op * mop,u64 supported_flags)2790 static int mem_op_validate_common(struct kvm_s390_mem_op *mop, u64 supported_flags)
2791 {
2792 if (mop->flags & ~supported_flags || !mop->size)
2793 return -EINVAL;
2794 if (mop->size > MEM_OP_MAX_SIZE)
2795 return -E2BIG;
2796 if (mop->flags & KVM_S390_MEMOP_F_SKEY_PROTECTION) {
2797 if (mop->key > 0xf)
2798 return -EINVAL;
2799 } else {
2800 mop->key = 0;
2801 }
2802 return 0;
2803 }
2804
kvm_s390_vm_mem_op_abs(struct kvm * kvm,struct kvm_s390_mem_op * mop)2805 static int kvm_s390_vm_mem_op_abs(struct kvm *kvm, struct kvm_s390_mem_op *mop)
2806 {
2807 void __user *uaddr = (void __user *)mop->buf;
2808 void *tmpbuf __free(kvfree) = NULL;
2809 enum gacc_mode acc_mode;
2810 int r;
2811
2812 r = mem_op_validate_common(mop, KVM_S390_MEMOP_F_SKEY_PROTECTION |
2813 KVM_S390_MEMOP_F_CHECK_ONLY);
2814 if (r)
2815 return r;
2816
2817 if (!(mop->flags & KVM_S390_MEMOP_F_CHECK_ONLY)) {
2818 tmpbuf = vmalloc(mop->size);
2819 if (!tmpbuf)
2820 return -ENOMEM;
2821 }
2822
2823 acc_mode = mop->op == KVM_S390_MEMOP_ABSOLUTE_READ ? GACC_FETCH : GACC_STORE;
2824
2825 scoped_guard(srcu, &kvm->srcu) {
2826 if (mop->flags & KVM_S390_MEMOP_F_CHECK_ONLY)
2827 return check_gpa_range(kvm, mop->gaddr, mop->size, acc_mode, mop->key);
2828
2829 if (acc_mode == GACC_STORE && copy_from_user(tmpbuf, uaddr, mop->size))
2830 return -EFAULT;
2831 r = access_guest_abs_with_key(kvm, mop->gaddr, tmpbuf,
2832 mop->size, acc_mode, mop->key);
2833 if (r)
2834 return r;
2835 if (acc_mode != GACC_STORE && copy_to_user(uaddr, tmpbuf, mop->size))
2836 return -EFAULT;
2837 }
2838 return 0;
2839 }
2840
kvm_s390_vm_mem_op_cmpxchg(struct kvm * kvm,struct kvm_s390_mem_op * mop)2841 static int kvm_s390_vm_mem_op_cmpxchg(struct kvm *kvm, struct kvm_s390_mem_op *mop)
2842 {
2843 void __user *uaddr = (void __user *)mop->buf;
2844 void __user *old_addr = (void __user *)mop->old_addr;
2845 union kvm_s390_quad old = { .sixteen = 0 };
2846 union kvm_s390_quad new = { .sixteen = 0 };
2847 bool success = false;
2848 int r;
2849
2850 r = mem_op_validate_common(mop, KVM_S390_MEMOP_F_SKEY_PROTECTION);
2851 if (r)
2852 return r;
2853 /*
2854 * This validates off_in_quad. Checking that size is a power
2855 * of two is not necessary, as cmpxchg_guest_abs_with_key
2856 * takes care of that
2857 */
2858 if (mop->size > sizeof(new))
2859 return -EINVAL;
2860 if (copy_from_user(&new, uaddr, mop->size))
2861 return -EFAULT;
2862 if (copy_from_user(&old, old_addr, mop->size))
2863 return -EFAULT;
2864
2865 scoped_guard(srcu, &kvm->srcu) {
2866 r = cmpxchg_guest_abs_with_key(kvm, mop->gaddr, mop->size, &old, new,
2867 mop->key, &success);
2868
2869 if (!success && copy_to_user(old_addr, &old, mop->size))
2870 return -EFAULT;
2871 }
2872 return r;
2873 }
2874
kvm_s390_vm_mem_op(struct kvm * kvm,struct kvm_s390_mem_op * mop)2875 static int kvm_s390_vm_mem_op(struct kvm *kvm, struct kvm_s390_mem_op *mop)
2876 {
2877 /*
2878 * This is technically a heuristic only, if the kvm->lock is not
2879 * taken, it is not guaranteed that the vm is/remains non-protected.
2880 * This is ok from a kernel perspective, wrongdoing is detected
2881 * on the access, -EFAULT is returned and the vm may crash the
2882 * next time it accesses the memory in question.
2883 * There is no sane usecase to do switching and a memop on two
2884 * different CPUs at the same time.
2885 */
2886 if (kvm_s390_pv_get_handle(kvm))
2887 return -EINVAL;
2888
2889 switch (mop->op) {
2890 case KVM_S390_MEMOP_ABSOLUTE_READ:
2891 case KVM_S390_MEMOP_ABSOLUTE_WRITE:
2892 return kvm_s390_vm_mem_op_abs(kvm, mop);
2893 case KVM_S390_MEMOP_ABSOLUTE_CMPXCHG:
2894 return kvm_s390_vm_mem_op_cmpxchg(kvm, mop);
2895 default:
2896 return -EINVAL;
2897 }
2898 }
2899
kvm_arch_vm_ioctl(struct file * filp,unsigned int ioctl,unsigned long arg)2900 int kvm_arch_vm_ioctl(struct file *filp, unsigned int ioctl, unsigned long arg)
2901 {
2902 struct kvm *kvm = filp->private_data;
2903 void __user *argp = (void __user *)arg;
2904 struct kvm_device_attr attr;
2905 int r;
2906 struct kvm_s390_interrupt_info *inti;
2907
2908 switch (ioctl) {
2909 case KVM_S390_INTERRUPT: {
2910 struct kvm_s390_interrupt s390int;
2911
2912 r = -EINVAL;
2913 if (kvm_is_ucontrol(kvm))
2914 break;
2915 r = -EFAULT;
2916 if (copy_from_user(&s390int, argp, sizeof(s390int)))
2917 break;
2918 inti = kzalloc_obj(*inti, GFP_KERNEL_ACCOUNT);
2919 if (!inti)
2920 return -ENOMEM;
2921 r = kvm_s390_inject_vm(kvm, &s390int, inti);
2922 if (r)
2923 kfree(inti);
2924 break;
2925 }
2926 case KVM_CREATE_IRQCHIP: {
2927 r = -EINVAL;
2928 if (kvm->arch.use_irqchip)
2929 r = 0;
2930 break;
2931 }
2932 case KVM_SET_DEVICE_ATTR: {
2933 r = -EFAULT;
2934 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
2935 break;
2936 r = kvm_s390_vm_set_attr(kvm, &attr);
2937 break;
2938 }
2939 case KVM_GET_DEVICE_ATTR: {
2940 r = -EFAULT;
2941 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
2942 break;
2943 r = kvm_s390_vm_get_attr(kvm, &attr);
2944 break;
2945 }
2946 case KVM_HAS_DEVICE_ATTR: {
2947 r = -EFAULT;
2948 if (copy_from_user(&attr, (void __user *)arg, sizeof(attr)))
2949 break;
2950 r = kvm_s390_vm_has_attr(kvm, &attr);
2951 break;
2952 }
2953 case KVM_S390_GET_SKEYS: {
2954 struct kvm_s390_skeys args;
2955
2956 r = -EFAULT;
2957 if (copy_from_user(&args, argp,
2958 sizeof(struct kvm_s390_skeys)))
2959 break;
2960 r = kvm_s390_get_skeys(kvm, &args);
2961 break;
2962 }
2963 case KVM_S390_SET_SKEYS: {
2964 struct kvm_s390_skeys args;
2965
2966 r = -EFAULT;
2967 if (copy_from_user(&args, argp,
2968 sizeof(struct kvm_s390_skeys)))
2969 break;
2970 r = kvm_s390_set_skeys(kvm, &args);
2971 break;
2972 }
2973 case KVM_S390_GET_CMMA_BITS: {
2974 struct kvm_s390_cmma_log args;
2975
2976 r = -EFAULT;
2977 if (copy_from_user(&args, argp, sizeof(args)))
2978 break;
2979 scoped_guard(mutex, &kvm->slots_arch_lock)
2980 r = kvm_s390_get_cmma_bits(kvm, &args);
2981 if (!r) {
2982 r = copy_to_user(argp, &args, sizeof(args));
2983 if (r)
2984 r = -EFAULT;
2985 }
2986 break;
2987 }
2988 case KVM_S390_SET_CMMA_BITS: {
2989 struct kvm_s390_cmma_log args;
2990
2991 r = -EFAULT;
2992 if (copy_from_user(&args, argp, sizeof(args)))
2993 break;
2994 mutex_lock(&kvm->slots_arch_lock);
2995 r = kvm_s390_set_cmma_bits(kvm, &args);
2996 mutex_unlock(&kvm->slots_arch_lock);
2997 break;
2998 }
2999 case KVM_S390_PV_COMMAND: {
3000 struct kvm_pv_cmd args;
3001
3002 /* protvirt means user cpu state */
3003 kvm_s390_set_user_cpu_state_ctrl(kvm);
3004 r = 0;
3005 if (!is_prot_virt_host()) {
3006 r = -EINVAL;
3007 break;
3008 }
3009 if (copy_from_user(&args, argp, sizeof(args))) {
3010 r = -EFAULT;
3011 break;
3012 }
3013 if (args.flags) {
3014 r = -EINVAL;
3015 break;
3016 }
3017 /* must be called without kvm->lock */
3018 r = kvm_s390_handle_pv(kvm, &args);
3019 if (copy_to_user(argp, &args, sizeof(args))) {
3020 r = -EFAULT;
3021 break;
3022 }
3023 break;
3024 }
3025 case KVM_S390_MEM_OP: {
3026 struct kvm_s390_mem_op mem_op;
3027
3028 if (copy_from_user(&mem_op, argp, sizeof(mem_op)) == 0)
3029 r = kvm_s390_vm_mem_op(kvm, &mem_op);
3030 else
3031 r = -EFAULT;
3032 break;
3033 }
3034 case KVM_S390_KEYOP: {
3035 struct kvm_s390_mmu_cache *mc;
3036 struct kvm_s390_keyop kop;
3037 union skey skey;
3038
3039 if (copy_from_user(&kop, argp, sizeof(kop))) {
3040 r = -EFAULT;
3041 break;
3042 }
3043 skey.skey = kop.key;
3044
3045 mc = kvm_s390_new_mmu_cache();
3046 if (!mc)
3047 return -ENOMEM;
3048
3049 r = kvm_s390_keyop(mc, kvm, kop.operation, kop.guest_addr, skey);
3050 kvm_s390_free_mmu_cache(mc);
3051 if (r < 0)
3052 break;
3053
3054 kop.key = r;
3055 r = 0;
3056 if (copy_to_user(argp, &kop, sizeof(kop)))
3057 r = -EFAULT;
3058 break;
3059 }
3060 case KVM_S390_ZPCI_OP: {
3061 struct kvm_s390_zpci_op args;
3062
3063 r = -EINVAL;
3064 if (!IS_ENABLED(CONFIG_VFIO_PCI_ZDEV_KVM))
3065 break;
3066 if (copy_from_user(&args, argp, sizeof(args))) {
3067 r = -EFAULT;
3068 break;
3069 }
3070 r = kvm_s390_pci_zpci_op(kvm, &args);
3071 break;
3072 }
3073 default:
3074 r = -ENOTTY;
3075 }
3076
3077 return r;
3078 }
3079
kvm_s390_apxa_installed(void)3080 static int kvm_s390_apxa_installed(void)
3081 {
3082 struct ap_config_info info;
3083
3084 if (ap_instructions_available()) {
3085 if (ap_qci(&info) == 0)
3086 return info.apxa;
3087 }
3088
3089 return 0;
3090 }
3091
3092 /*
3093 * The format of the crypto control block (CRYCB) is specified in the 3 low
3094 * order bits of the CRYCB designation (CRYCBD) field as follows:
3095 * Format 0: Neither the message security assist extension 3 (MSAX3) nor the
3096 * AP extended addressing (APXA) facility are installed.
3097 * Format 1: The APXA facility is not installed but the MSAX3 facility is.
3098 * Format 2: Both the APXA and MSAX3 facilities are installed
3099 */
kvm_s390_set_crycb_format(struct kvm * kvm)3100 static void kvm_s390_set_crycb_format(struct kvm *kvm)
3101 {
3102 kvm->arch.crypto.crycbd = virt_to_phys(kvm->arch.crypto.crycb);
3103
3104 /* Clear the CRYCB format bits - i.e., set format 0 by default */
3105 kvm->arch.crypto.crycbd &= ~(CRYCB_FORMAT_MASK);
3106
3107 /* Check whether MSAX3 is installed */
3108 if (!test_kvm_facility(kvm, 76))
3109 return;
3110
3111 if (kvm_s390_apxa_installed())
3112 kvm->arch.crypto.crycbd |= CRYCB_FORMAT2;
3113 else
3114 kvm->arch.crypto.crycbd |= CRYCB_FORMAT1;
3115 }
3116
3117 /*
3118 * kvm_arch_crypto_set_masks
3119 *
3120 * @kvm: pointer to the target guest's KVM struct containing the crypto masks
3121 * to be set.
3122 * @apm: the mask identifying the accessible AP adapters
3123 * @aqm: the mask identifying the accessible AP domains
3124 * @adm: the mask identifying the accessible AP control domains
3125 *
3126 * Set the masks that identify the adapters, domains and control domains to
3127 * which the KVM guest is granted access.
3128 *
3129 * Note: The kvm->lock mutex must be locked by the caller before invoking this
3130 * function.
3131 */
kvm_arch_crypto_set_masks(struct kvm * kvm,unsigned long * apm,unsigned long * aqm,unsigned long * adm)3132 void kvm_arch_crypto_set_masks(struct kvm *kvm, unsigned long *apm,
3133 unsigned long *aqm, unsigned long *adm)
3134 {
3135 struct kvm_s390_crypto_cb *crycb = kvm->arch.crypto.crycb;
3136
3137 kvm_s390_vcpu_block_all(kvm);
3138
3139 switch (kvm->arch.crypto.crycbd & CRYCB_FORMAT_MASK) {
3140 case CRYCB_FORMAT2: /* APCB1 use 256 bits */
3141 memcpy(crycb->apcb1.apm, apm, 32);
3142 VM_EVENT(kvm, 3, "SET CRYCB: apm %016lx %016lx %016lx %016lx",
3143 apm[0], apm[1], apm[2], apm[3]);
3144 memcpy(crycb->apcb1.aqm, aqm, 32);
3145 VM_EVENT(kvm, 3, "SET CRYCB: aqm %016lx %016lx %016lx %016lx",
3146 aqm[0], aqm[1], aqm[2], aqm[3]);
3147 memcpy(crycb->apcb1.adm, adm, 32);
3148 VM_EVENT(kvm, 3, "SET CRYCB: adm %016lx %016lx %016lx %016lx",
3149 adm[0], adm[1], adm[2], adm[3]);
3150 break;
3151 case CRYCB_FORMAT1:
3152 case CRYCB_FORMAT0: /* Fall through both use APCB0 */
3153 memcpy(crycb->apcb0.apm, apm, 8);
3154 memcpy(crycb->apcb0.aqm, aqm, 2);
3155 memcpy(crycb->apcb0.adm, adm, 2);
3156 VM_EVENT(kvm, 3, "SET CRYCB: apm %016lx aqm %04x adm %04x",
3157 apm[0], *((unsigned short *)aqm),
3158 *((unsigned short *)adm));
3159 break;
3160 default: /* Can not happen */
3161 break;
3162 }
3163
3164 /* recreate the shadow crycb for each vcpu */
3165 kvm_s390_sync_request_broadcast(kvm, KVM_REQ_VSIE_RESTART);
3166 kvm_s390_vcpu_unblock_all(kvm);
3167 }
3168 EXPORT_SYMBOL_GPL(kvm_arch_crypto_set_masks);
3169
3170 /*
3171 * kvm_arch_crypto_clear_masks
3172 *
3173 * @kvm: pointer to the target guest's KVM struct containing the crypto masks
3174 * to be cleared.
3175 *
3176 * Clear the masks that identify the adapters, domains and control domains to
3177 * which the KVM guest is granted access.
3178 *
3179 * Note: The kvm->lock mutex must be locked by the caller before invoking this
3180 * function.
3181 */
kvm_arch_crypto_clear_masks(struct kvm * kvm)3182 void kvm_arch_crypto_clear_masks(struct kvm *kvm)
3183 {
3184 kvm_s390_vcpu_block_all(kvm);
3185
3186 memset(&kvm->arch.crypto.crycb->apcb0, 0,
3187 sizeof(kvm->arch.crypto.crycb->apcb0));
3188 memset(&kvm->arch.crypto.crycb->apcb1, 0,
3189 sizeof(kvm->arch.crypto.crycb->apcb1));
3190
3191 VM_EVENT(kvm, 3, "%s", "CLR CRYCB:");
3192 /* recreate the shadow crycb for each vcpu */
3193 kvm_s390_sync_request_broadcast(kvm, KVM_REQ_VSIE_RESTART);
3194 kvm_s390_vcpu_unblock_all(kvm);
3195 }
3196 EXPORT_SYMBOL_GPL(kvm_arch_crypto_clear_masks);
3197
kvm_s390_get_initial_cpuid(void)3198 static u64 kvm_s390_get_initial_cpuid(void)
3199 {
3200 struct cpuid cpuid;
3201
3202 get_cpu_id(&cpuid);
3203 cpuid.version = 0xff;
3204 return *((u64 *) &cpuid);
3205 }
3206
kvm_s390_crypto_init(struct kvm * kvm)3207 static void kvm_s390_crypto_init(struct kvm *kvm)
3208 {
3209 kvm->arch.crypto.crycb = &kvm->arch.sie_page2->crycb;
3210 kvm_s390_set_crycb_format(kvm);
3211 init_rwsem(&kvm->arch.crypto.pqap_hook_rwsem);
3212
3213 if (!test_kvm_facility(kvm, 76))
3214 return;
3215
3216 /* Enable AES/DEA protected key functions by default */
3217 kvm->arch.crypto.aes_kw = 1;
3218 kvm->arch.crypto.dea_kw = 1;
3219 get_random_bytes(kvm->arch.crypto.crycb->aes_wrapping_key_mask,
3220 sizeof(kvm->arch.crypto.crycb->aes_wrapping_key_mask));
3221 get_random_bytes(kvm->arch.crypto.crycb->dea_wrapping_key_mask,
3222 sizeof(kvm->arch.crypto.crycb->dea_wrapping_key_mask));
3223 }
3224
sca_dispose(struct kvm * kvm)3225 static void sca_dispose(struct kvm *kvm)
3226 {
3227 if (kvm->arch.sca)
3228 free_pages_exact(kvm->arch.sca, sizeof(*kvm->arch.sca));
3229 kvm->arch.sca = NULL;
3230 }
3231
kvm_arch_free_vm(struct kvm * kvm)3232 void kvm_arch_free_vm(struct kvm *kvm)
3233 {
3234 if (IS_ENABLED(CONFIG_VFIO_PCI_ZDEV_KVM))
3235 kvm_s390_pci_clear_list(kvm);
3236
3237 __kvm_arch_free_vm(kvm);
3238 }
3239
kvm_arch_init_vm(struct kvm * kvm,unsigned long type)3240 int kvm_arch_init_vm(struct kvm *kvm, unsigned long type)
3241 {
3242 gfp_t alloc_flags = GFP_KERNEL_ACCOUNT | __GFP_ZERO;
3243 char debug_name[16];
3244 int i, rc;
3245
3246 mutex_init(&kvm->arch.pv.import_lock);
3247
3248 rc = -EINVAL;
3249 #ifdef CONFIG_KVM_S390_UCONTROL
3250 if (type & ~KVM_VM_S390_UCONTROL)
3251 goto out_err;
3252 if ((type & KVM_VM_S390_UCONTROL) && (!capable(CAP_SYS_ADMIN)))
3253 goto out_err;
3254 #else
3255 if (type)
3256 goto out_err;
3257 #endif
3258 rc = -ENOMEM;
3259
3260 if (!sclp.has_64bscao)
3261 alloc_flags |= GFP_DMA;
3262 mutex_lock(&kvm_lock);
3263
3264 kvm->arch.sca = alloc_pages_exact(sizeof(*kvm->arch.sca), alloc_flags);
3265 mutex_unlock(&kvm_lock);
3266 if (!kvm->arch.sca)
3267 goto out_err;
3268
3269 snprintf(debug_name, sizeof(debug_name), "kvm-%u", current->pid);
3270
3271 kvm->arch.dbf = debug_register(debug_name, 32, 1, 7 * sizeof(long));
3272 if (!kvm->arch.dbf)
3273 goto out_err;
3274
3275 BUILD_BUG_ON(sizeof(struct sie_page2) != 4096);
3276 kvm->arch.sie_page2 =
3277 (struct sie_page2 *) get_zeroed_page(GFP_KERNEL_ACCOUNT | GFP_DMA);
3278 if (!kvm->arch.sie_page2)
3279 goto out_err;
3280
3281 kvm->arch.sie_page2->kvm = kvm;
3282 kvm->arch.model.fac_list = kvm->arch.sie_page2->fac_list;
3283
3284 for (i = 0; i < ARRAY_SIZE(kvm_s390_fac_base); i++) {
3285 kvm->arch.model.fac_mask[i] = stfle_fac_list[i] &
3286 kvm_s390_fac_base[i];
3287 kvm->arch.model.fac_list[i] = stfle_fac_list[i] &
3288 kvm_s390_fac_base[i];
3289 }
3290 for (i = 0; i < ARRAY_SIZE(kvm_s390_fac_ext); i++) {
3291 kvm->arch.model.fac_mask[i] |= stfle_fac_list[i] &
3292 kvm_s390_fac_ext[i];
3293 }
3294 kvm->arch.model.subfuncs = kvm_s390_available_subfunc;
3295
3296 /* we are always in czam mode - even on pre z14 machines */
3297 set_kvm_facility(kvm->arch.model.fac_mask, 138);
3298 set_kvm_facility(kvm->arch.model.fac_list, 138);
3299 /* we emulate STHYI in kvm */
3300 set_kvm_facility(kvm->arch.model.fac_mask, 74);
3301 set_kvm_facility(kvm->arch.model.fac_list, 74);
3302 if (machine_has_tlb_guest()) {
3303 set_kvm_facility(kvm->arch.model.fac_mask, 147);
3304 set_kvm_facility(kvm->arch.model.fac_list, 147);
3305 }
3306
3307 if (css_general_characteristics.aiv && test_facility(65))
3308 set_kvm_facility(kvm->arch.model.fac_mask, 65);
3309
3310 kvm->arch.model.cpuid = kvm_s390_get_initial_cpuid();
3311 kvm->arch.model.ibc = sclp.ibc & 0x0fff;
3312
3313 kvm->arch.model.uv_feat_guest.feat = 0;
3314
3315 kvm_s390_crypto_init(kvm);
3316
3317 if (IS_ENABLED(CONFIG_VFIO_PCI_ZDEV_KVM)) {
3318 mutex_lock(&kvm->lock);
3319 kvm_s390_pci_init_list(kvm);
3320 kvm_s390_vcpu_pci_enable_interp(kvm);
3321 mutex_unlock(&kvm->lock);
3322 }
3323
3324 spin_lock_init(&kvm->arch.float_int.ais_lock);
3325 spin_lock_init(&kvm->arch.float_int.lock);
3326 for (i = 0; i < FIRQ_LIST_COUNT; i++)
3327 INIT_LIST_HEAD(&kvm->arch.float_int.lists[i]);
3328 init_waitqueue_head(&kvm->arch.ipte_wq);
3329 mutex_init(&kvm->arch.ipte_mutex);
3330
3331 debug_register_view(kvm->arch.dbf, &debug_sprintf_view);
3332 VM_EVENT(kvm, 3, "vm created with type %lu", type);
3333
3334 kvm->arch.mem_limit = type & KVM_VM_S390_UCONTROL ? KVM_S390_NO_MEM_LIMIT : sclp.hamax + 1;
3335 kvm->arch.gmap = gmap_new(kvm, gpa_to_gfn(kvm->arch.mem_limit));
3336 if (!kvm->arch.gmap)
3337 goto out_err;
3338 clear_bit(GMAP_FLAG_PFAULT_ENABLED, &kvm->arch.gmap->flags);
3339
3340 if (type & KVM_VM_S390_UCONTROL) {
3341 struct kvm_userspace_memory_region2 fake_memslot = {
3342 .slot = KVM_S390_UCONTROL_MEMSLOT,
3343 .guest_phys_addr = 0,
3344 .userspace_addr = 0,
3345 .memory_size = ALIGN_DOWN(TASK_SIZE, _SEGMENT_SIZE),
3346 .flags = 0,
3347 };
3348
3349 /* one flat fake memslot covering the whole address-space */
3350 mutex_lock(&kvm->slots_lock);
3351 KVM_BUG_ON(kvm_set_internal_memslot(kvm, &fake_memslot), kvm);
3352 mutex_unlock(&kvm->slots_lock);
3353 set_bit(GMAP_FLAG_IS_UCONTROL, &kvm->arch.gmap->flags);
3354 } else {
3355 struct crst_table *table = dereference_asce(kvm->arch.gmap->asce);
3356
3357 crst_table_init((void *)table, _CRSTE_HOLE(table->crstes[0].h.tt).val);
3358 }
3359
3360 kvm->arch.use_pfmfi = sclp.has_pfmfi;
3361 kvm->arch.use_skf = sclp.has_skey;
3362 spin_lock_init(&kvm->arch.start_stop_lock);
3363 kvm_s390_vsie_init(kvm);
3364 if (use_gisa)
3365 kvm_s390_gisa_init(kvm);
3366 INIT_LIST_HEAD(&kvm->arch.pv.need_cleanup);
3367 kvm->arch.pv.set_aside = NULL;
3368 KVM_EVENT(3, "vm 0x%p created by pid %u", kvm, current->pid);
3369
3370 return 0;
3371 out_err:
3372 free_page((unsigned long)kvm->arch.sie_page2);
3373 debug_unregister(kvm->arch.dbf);
3374 sca_dispose(kvm);
3375 KVM_EVENT(3, "creation of vm failed: %d", rc);
3376 return rc;
3377 }
3378
kvm_arch_vcpu_destroy(struct kvm_vcpu * vcpu)3379 void kvm_arch_vcpu_destroy(struct kvm_vcpu *vcpu)
3380 {
3381 u16 rc, rrc;
3382
3383 VCPU_EVENT(vcpu, 3, "%s", "free cpu");
3384 trace_kvm_s390_destroy_vcpu(vcpu->vcpu_id);
3385 kvm_s390_clear_local_irqs(vcpu);
3386 kvm_clear_async_pf_completion_queue(vcpu);
3387 kvm_s390_clear_bp_data(vcpu);
3388 if (!kvm_is_ucontrol(vcpu->kvm))
3389 sca_del_vcpu(vcpu);
3390 kvm_s390_update_topology_change_report(vcpu->kvm, 1);
3391
3392 if (kvm_is_ucontrol(vcpu->kvm)) {
3393 scoped_guard(spinlock, &vcpu->kvm->arch.gmap->children_lock)
3394 gmap_remove_child(vcpu->arch.gmap);
3395 vcpu->arch.gmap = gmap_put(vcpu->arch.gmap);
3396 }
3397
3398 if (vcpu->kvm->arch.use_cmma)
3399 kvm_s390_vcpu_unsetup_cmma(vcpu);
3400 /* We can not hold the vcpu mutex here, we are already dying */
3401 if (kvm_s390_pv_cpu_get_handle(vcpu))
3402 kvm_s390_pv_destroy_cpu(vcpu, &rc, &rrc);
3403 free_page((unsigned long)(vcpu->arch.sie_block));
3404 kvm_s390_free_mmu_cache(vcpu->arch.mc);
3405 }
3406
kvm_arch_destroy_vm(struct kvm * kvm)3407 void kvm_arch_destroy_vm(struct kvm *kvm)
3408 {
3409 u16 rc, rrc;
3410
3411 kvm_destroy_vcpus(kvm);
3412 sca_dispose(kvm);
3413 kvm_s390_gisa_destroy(kvm);
3414 /*
3415 * We are already at the end of life and kvm->lock is not taken.
3416 * This is ok as the file descriptor is closed by now and nobody
3417 * can mess with the pv state.
3418 */
3419 kvm_s390_pv_deinit_cleanup_all(kvm, &rc, &rrc);
3420 /*
3421 * Remove the mmu notifier only when the whole KVM VM is torn down,
3422 * and only if one was registered to begin with. If the VM is
3423 * currently not protected, but has been previously been protected,
3424 * then it's possible that the notifier is still registered.
3425 */
3426 if (kvm->arch.pv.mmu_notifier.ops)
3427 mmu_notifier_unregister(&kvm->arch.pv.mmu_notifier, kvm->mm);
3428
3429 debug_unregister(kvm->arch.dbf);
3430 free_page((unsigned long)kvm->arch.sie_page2);
3431 kvm_s390_destroy_adapters(kvm);
3432 kvm_s390_clear_float_irqs(kvm);
3433 kvm_s390_vsie_destroy(kvm);
3434 kvm->arch.gmap = gmap_put(kvm->arch.gmap);
3435 KVM_EVENT(3, "vm 0x%p destroyed", kvm);
3436 }
3437
3438 /* Section: vcpu related */
sca_del_vcpu(struct kvm_vcpu * vcpu)3439 static void sca_del_vcpu(struct kvm_vcpu *vcpu)
3440 {
3441 struct esca_block *sca = vcpu->kvm->arch.sca;
3442
3443 if (!kvm_s390_use_sca_entries() || !vcpu->arch.initialized)
3444 return;
3445
3446 clear_bit_inv(vcpu->vcpu_id, (unsigned long *)sca->mcn);
3447 sca->cpu[vcpu->vcpu_id].sda = 0;
3448 }
3449
sca_add_vcpu(struct kvm_vcpu * vcpu)3450 static void sca_add_vcpu(struct kvm_vcpu *vcpu)
3451 {
3452 struct esca_block *sca = vcpu->kvm->arch.sca;
3453 phys_addr_t sca_phys = virt_to_phys(sca);
3454
3455 /* we still need the sca header for the ipte control */
3456 vcpu->arch.sie_block->scaoh = sca_phys >> 32;
3457 vcpu->arch.sie_block->scaol = sca_phys & ESCA_SCAOL_MASK;
3458 vcpu->arch.sie_block->ecb2 |= ECB2_ESCA;
3459
3460 if (!kvm_s390_use_sca_entries())
3461 return;
3462
3463 WRITE_ONCE(sca->cpu[vcpu->vcpu_id].sda, virt_to_phys(vcpu->arch.sie_block));
3464 set_bit_inv(vcpu->vcpu_id, (unsigned long *)sca->mcn);
3465 }
3466
sca_can_add_vcpu(struct kvm * kvm,unsigned int id)3467 static int sca_can_add_vcpu(struct kvm *kvm, unsigned int id)
3468 {
3469 if (!kvm_s390_use_sca_entries())
3470 return id < KVM_MAX_VCPUS;
3471
3472 return id < KVM_S390_ESCA_CPU_SLOTS;
3473 }
3474
3475 /* needs disabled preemption to protect from TOD sync and vcpu_load/put */
__start_cpu_timer_accounting(struct kvm_vcpu * vcpu)3476 static void __start_cpu_timer_accounting(struct kvm_vcpu *vcpu)
3477 {
3478 WARN_ON_ONCE(vcpu->arch.cputm_start != 0);
3479 raw_write_seqcount_begin(&vcpu->arch.cputm_seqcount);
3480 vcpu->arch.cputm_start = get_tod_clock_fast();
3481 raw_write_seqcount_end(&vcpu->arch.cputm_seqcount);
3482 }
3483
3484 /* needs disabled preemption to protect from TOD sync and vcpu_load/put */
__stop_cpu_timer_accounting(struct kvm_vcpu * vcpu)3485 static void __stop_cpu_timer_accounting(struct kvm_vcpu *vcpu)
3486 {
3487 WARN_ON_ONCE(vcpu->arch.cputm_start == 0);
3488 raw_write_seqcount_begin(&vcpu->arch.cputm_seqcount);
3489 vcpu->arch.sie_block->cputm -= get_tod_clock_fast() - vcpu->arch.cputm_start;
3490 vcpu->arch.cputm_start = 0;
3491 raw_write_seqcount_end(&vcpu->arch.cputm_seqcount);
3492 }
3493
3494 /* needs disabled preemption to protect from TOD sync and vcpu_load/put */
__enable_cpu_timer_accounting(struct kvm_vcpu * vcpu)3495 static void __enable_cpu_timer_accounting(struct kvm_vcpu *vcpu)
3496 {
3497 WARN_ON_ONCE(vcpu->arch.cputm_enabled);
3498 vcpu->arch.cputm_enabled = true;
3499 __start_cpu_timer_accounting(vcpu);
3500 }
3501
3502 /* needs disabled preemption to protect from TOD sync and vcpu_load/put */
__disable_cpu_timer_accounting(struct kvm_vcpu * vcpu)3503 static void __disable_cpu_timer_accounting(struct kvm_vcpu *vcpu)
3504 {
3505 WARN_ON_ONCE(!vcpu->arch.cputm_enabled);
3506 __stop_cpu_timer_accounting(vcpu);
3507 vcpu->arch.cputm_enabled = false;
3508 }
3509
enable_cpu_timer_accounting(struct kvm_vcpu * vcpu)3510 static void enable_cpu_timer_accounting(struct kvm_vcpu *vcpu)
3511 {
3512 preempt_disable(); /* protect from TOD sync and vcpu_load/put */
3513 __enable_cpu_timer_accounting(vcpu);
3514 preempt_enable();
3515 }
3516
disable_cpu_timer_accounting(struct kvm_vcpu * vcpu)3517 static void disable_cpu_timer_accounting(struct kvm_vcpu *vcpu)
3518 {
3519 preempt_disable(); /* protect from TOD sync and vcpu_load/put */
3520 __disable_cpu_timer_accounting(vcpu);
3521 preempt_enable();
3522 }
3523
3524 /* set the cpu timer - may only be called from the VCPU thread itself */
kvm_s390_set_cpu_timer(struct kvm_vcpu * vcpu,__u64 cputm)3525 void kvm_s390_set_cpu_timer(struct kvm_vcpu *vcpu, __u64 cputm)
3526 {
3527 preempt_disable(); /* protect from TOD sync and vcpu_load/put */
3528 raw_write_seqcount_begin(&vcpu->arch.cputm_seqcount);
3529 if (vcpu->arch.cputm_enabled)
3530 vcpu->arch.cputm_start = get_tod_clock_fast();
3531 vcpu->arch.sie_block->cputm = cputm;
3532 raw_write_seqcount_end(&vcpu->arch.cputm_seqcount);
3533 preempt_enable();
3534 }
3535
3536 /* update and get the cpu timer - can also be called from other VCPU threads */
kvm_s390_get_cpu_timer(struct kvm_vcpu * vcpu)3537 __u64 kvm_s390_get_cpu_timer(struct kvm_vcpu *vcpu)
3538 {
3539 unsigned int seq;
3540 __u64 value;
3541
3542 if (unlikely(!vcpu->arch.cputm_enabled))
3543 return vcpu->arch.sie_block->cputm;
3544
3545 preempt_disable(); /* protect from TOD sync and vcpu_load/put */
3546 do {
3547 seq = raw_read_seqcount(&vcpu->arch.cputm_seqcount);
3548 /*
3549 * If the writer would ever execute a read in the critical
3550 * section, e.g. in irq context, we have a deadlock.
3551 */
3552 WARN_ON_ONCE((seq & 1) && smp_processor_id() == vcpu->cpu);
3553 value = vcpu->arch.sie_block->cputm;
3554 /* if cputm_start is 0, accounting is being started/stopped */
3555 if (likely(vcpu->arch.cputm_start))
3556 value -= get_tod_clock_fast() - vcpu->arch.cputm_start;
3557 } while (read_seqcount_retry(&vcpu->arch.cputm_seqcount, seq & ~1));
3558 preempt_enable();
3559 return value;
3560 }
3561
kvm_arch_vcpu_load(struct kvm_vcpu * vcpu,int cpu)3562 void kvm_arch_vcpu_load(struct kvm_vcpu *vcpu, int cpu)
3563 {
3564
3565 kvm_s390_set_cpuflags(vcpu, CPUSTAT_RUNNING);
3566 if (vcpu->arch.cputm_enabled && !is_vcpu_idle(vcpu))
3567 __start_cpu_timer_accounting(vcpu);
3568 vcpu->cpu = cpu;
3569 }
3570
kvm_arch_vcpu_put(struct kvm_vcpu * vcpu)3571 void kvm_arch_vcpu_put(struct kvm_vcpu *vcpu)
3572 {
3573 vcpu->cpu = -1;
3574 if (vcpu->arch.cputm_enabled && !is_vcpu_idle(vcpu))
3575 __stop_cpu_timer_accounting(vcpu);
3576 kvm_s390_clear_cpuflags(vcpu, CPUSTAT_RUNNING);
3577
3578 }
3579
kvm_arch_vcpu_postcreate(struct kvm_vcpu * vcpu)3580 void kvm_arch_vcpu_postcreate(struct kvm_vcpu *vcpu)
3581 {
3582 mutex_lock(&vcpu->kvm->lock);
3583 preempt_disable();
3584 vcpu->arch.sie_block->epoch = vcpu->kvm->arch.epoch;
3585 vcpu->arch.sie_block->epdx = vcpu->kvm->arch.epdx;
3586 preempt_enable();
3587 mutex_unlock(&vcpu->kvm->lock);
3588 if (!kvm_is_ucontrol(vcpu->kvm)) {
3589 vcpu->arch.gmap = vcpu->kvm->arch.gmap;
3590 sca_add_vcpu(vcpu);
3591 }
3592 if (test_kvm_facility(vcpu->kvm, 74) || vcpu->kvm->arch.user_instr0 ||
3593 vcpu->kvm->arch.user_operexec)
3594 vcpu->arch.sie_block->ictl |= ICTL_OPEREXC;
3595
3596 /* Pairs with smp_load_acquire() in kvm_arch_vcpu_ioctl_run() and kvm_arch_vcpu_ioctl() */
3597 smp_store_release(&vcpu->arch.initialized, true);
3598 }
3599
kvm_has_pckmo_subfunc(struct kvm * kvm,unsigned long nr)3600 static bool kvm_has_pckmo_subfunc(struct kvm *kvm, unsigned long nr)
3601 {
3602 if (test_bit_inv(nr, (unsigned long *)&kvm->arch.model.subfuncs.pckmo) &&
3603 test_bit_inv(nr, (unsigned long *)&kvm_s390_available_subfunc.pckmo))
3604 return true;
3605 return false;
3606 }
3607
kvm_has_pckmo_ecc(struct kvm * kvm)3608 static bool kvm_has_pckmo_ecc(struct kvm *kvm)
3609 {
3610 /* At least one ECC subfunction must be present */
3611 return kvm_has_pckmo_subfunc(kvm, 32) ||
3612 kvm_has_pckmo_subfunc(kvm, 33) ||
3613 kvm_has_pckmo_subfunc(kvm, 34) ||
3614 kvm_has_pckmo_subfunc(kvm, 40) ||
3615 kvm_has_pckmo_subfunc(kvm, 41);
3616
3617 }
3618
kvm_has_pckmo_hmac(struct kvm * kvm)3619 static bool kvm_has_pckmo_hmac(struct kvm *kvm)
3620 {
3621 /* At least one HMAC subfunction must be present */
3622 return kvm_has_pckmo_subfunc(kvm, 118) ||
3623 kvm_has_pckmo_subfunc(kvm, 122);
3624 }
3625
kvm_s390_vcpu_crypto_setup(struct kvm_vcpu * vcpu)3626 static void kvm_s390_vcpu_crypto_setup(struct kvm_vcpu *vcpu)
3627 {
3628 /*
3629 * If the AP instructions are not being interpreted and the MSAX3
3630 * facility is not configured for the guest, there is nothing to set up.
3631 */
3632 if (!vcpu->kvm->arch.crypto.apie && !test_kvm_facility(vcpu->kvm, 76))
3633 return;
3634
3635 vcpu->arch.sie_block->crycbd = vcpu->kvm->arch.crypto.crycbd;
3636 vcpu->arch.sie_block->ecb3 &= ~(ECB3_AES | ECB3_DEA);
3637 vcpu->arch.sie_block->eca &= ~ECA_APIE;
3638 vcpu->arch.sie_block->ecd &= ~(ECD_ECC | ECD_HMAC);
3639
3640 if (vcpu->kvm->arch.crypto.apie)
3641 vcpu->arch.sie_block->eca |= ECA_APIE;
3642
3643 /* Set up protected key support */
3644 if (vcpu->kvm->arch.crypto.aes_kw) {
3645 vcpu->arch.sie_block->ecb3 |= ECB3_AES;
3646 /* ecc/hmac is also wrapped with AES key */
3647 if (kvm_has_pckmo_ecc(vcpu->kvm))
3648 vcpu->arch.sie_block->ecd |= ECD_ECC;
3649 if (kvm_has_pckmo_hmac(vcpu->kvm))
3650 vcpu->arch.sie_block->ecd |= ECD_HMAC;
3651 }
3652
3653 if (vcpu->kvm->arch.crypto.dea_kw)
3654 vcpu->arch.sie_block->ecb3 |= ECB3_DEA;
3655 }
3656
kvm_s390_vcpu_unsetup_cmma(struct kvm_vcpu * vcpu)3657 void kvm_s390_vcpu_unsetup_cmma(struct kvm_vcpu *vcpu)
3658 {
3659 if (vcpu->arch.sie_block->cbrlo)
3660 free_page((unsigned long)phys_to_virt(vcpu->arch.sie_block->cbrlo));
3661 vcpu->arch.sie_block->cbrlo = 0;
3662 }
3663
kvm_s390_vcpu_setup_cmma(struct kvm_vcpu * vcpu)3664 int kvm_s390_vcpu_setup_cmma(struct kvm_vcpu *vcpu)
3665 {
3666 void *cbrlo_page = (void *)get_zeroed_page(GFP_KERNEL_ACCOUNT);
3667
3668 if (!cbrlo_page)
3669 return -ENOMEM;
3670
3671 vcpu->arch.sie_block->cbrlo = virt_to_phys(cbrlo_page);
3672 return 0;
3673 }
3674
kvm_s390_vcpu_setup_model(struct kvm_vcpu * vcpu)3675 static void kvm_s390_vcpu_setup_model(struct kvm_vcpu *vcpu)
3676 {
3677 struct kvm_s390_cpu_model *model = &vcpu->kvm->arch.model;
3678
3679 vcpu->arch.sie_block->ibc = model->ibc;
3680 if (test_kvm_facility(vcpu->kvm, 7))
3681 vcpu->arch.sie_block->fac = virt_to_phys(model->fac_list);
3682 }
3683
kvm_s390_vcpu_setup(struct kvm_vcpu * vcpu)3684 static int kvm_s390_vcpu_setup(struct kvm_vcpu *vcpu)
3685 {
3686 int rc = 0;
3687 u16 uvrc, uvrrc;
3688
3689 atomic_set(&vcpu->arch.sie_block->cpuflags, CPUSTAT_ZARCH |
3690 CPUSTAT_SM |
3691 CPUSTAT_STOPPED);
3692
3693 if (test_kvm_facility(vcpu->kvm, 78))
3694 kvm_s390_set_cpuflags(vcpu, CPUSTAT_GED2);
3695 else if (test_kvm_facility(vcpu->kvm, 8))
3696 kvm_s390_set_cpuflags(vcpu, CPUSTAT_GED);
3697
3698 kvm_s390_vcpu_setup_model(vcpu);
3699
3700 /* pgste_set_pte has special handling for !machine_has_esop() */
3701 if (machine_has_esop())
3702 vcpu->arch.sie_block->ecb |= ECB_HOSTPROTINT;
3703 if (test_kvm_facility(vcpu->kvm, 9))
3704 vcpu->arch.sie_block->ecb |= ECB_SRSI;
3705 if (test_kvm_facility(vcpu->kvm, 11))
3706 vcpu->arch.sie_block->ecb |= ECB_PTF;
3707 if (test_kvm_facility(vcpu->kvm, 73))
3708 vcpu->arch.sie_block->ecb |= ECB_TE;
3709 if (!kvm_is_ucontrol(vcpu->kvm))
3710 vcpu->arch.sie_block->ecb |= ECB_SPECI;
3711
3712 if (test_kvm_facility(vcpu->kvm, 8) && vcpu->kvm->arch.use_pfmfi)
3713 vcpu->arch.sie_block->ecb2 |= ECB2_PFMFI;
3714 if (test_kvm_facility(vcpu->kvm, 130))
3715 vcpu->arch.sie_block->ecb2 |= ECB2_IEP;
3716 vcpu->arch.sie_block->eca = ECA_MVPGI | ECA_PROTEXCI;
3717 if (sclp.has_cei)
3718 vcpu->arch.sie_block->eca |= ECA_CEI;
3719 if (sclp.has_ib)
3720 vcpu->arch.sie_block->eca |= ECA_IB;
3721 if (sclp.has_siif)
3722 vcpu->arch.sie_block->eca |= ECA_SII;
3723 if (kvm_s390_use_sca_entries())
3724 vcpu->arch.sie_block->eca |= ECA_SIGPI;
3725 if (test_kvm_facility(vcpu->kvm, 129)) {
3726 vcpu->arch.sie_block->eca |= ECA_VX;
3727 vcpu->arch.sie_block->ecd |= ECD_HOSTREGMGMT;
3728 }
3729 if (test_kvm_facility(vcpu->kvm, 139))
3730 vcpu->arch.sie_block->ecd |= ECD_MEF;
3731 if (test_kvm_facility(vcpu->kvm, 156))
3732 vcpu->arch.sie_block->ecd |= ECD_ETOKENF;
3733 if (vcpu->arch.sie_block->gd) {
3734 vcpu->arch.sie_block->eca |= ECA_AIV;
3735 VCPU_EVENT(vcpu, 3, "AIV gisa format-%u enabled for cpu %03u",
3736 vcpu->arch.sie_block->gd & 0x3, vcpu->vcpu_id);
3737 }
3738 vcpu->arch.sie_block->sdnxo = virt_to_phys(&vcpu->run->s.regs.sdnx) | SDNXC;
3739 vcpu->arch.sie_block->riccbd = virt_to_phys(&vcpu->run->s.regs.riccb);
3740
3741 if (sclp.has_kss)
3742 kvm_s390_set_cpuflags(vcpu, CPUSTAT_KSS);
3743 else
3744 vcpu->arch.sie_block->ictl |= ICTL_ISKE | ICTL_SSKE | ICTL_RRBE;
3745
3746 if (vcpu->kvm->arch.use_cmma) {
3747 rc = kvm_s390_vcpu_setup_cmma(vcpu);
3748 if (rc)
3749 return rc;
3750 }
3751 hrtimer_setup(&vcpu->arch.ckc_timer, kvm_s390_idle_wakeup, CLOCK_MONOTONIC,
3752 HRTIMER_MODE_REL);
3753
3754 vcpu->arch.sie_block->hpid = HPID_KVM;
3755
3756 kvm_s390_vcpu_crypto_setup(vcpu);
3757
3758 kvm_s390_vcpu_pci_setup(vcpu);
3759
3760 mutex_lock(&vcpu->kvm->lock);
3761 if (kvm_s390_pv_is_protected(vcpu->kvm)) {
3762 rc = kvm_s390_pv_create_cpu(vcpu, &uvrc, &uvrrc);
3763 if (rc)
3764 kvm_s390_vcpu_unsetup_cmma(vcpu);
3765 }
3766 mutex_unlock(&vcpu->kvm->lock);
3767
3768 return rc;
3769 }
3770
kvm_arch_vcpu_precreate(struct kvm * kvm,unsigned int id)3771 int kvm_arch_vcpu_precreate(struct kvm *kvm, unsigned int id)
3772 {
3773 if (!kvm_is_ucontrol(kvm) && !sca_can_add_vcpu(kvm, id))
3774 return -EINVAL;
3775 return 0;
3776 }
3777
DEFINE_FREE(sie_page,struct sie_page *,if (_T)free_page ((unsigned long)(_T)))3778 DEFINE_FREE(sie_page, struct sie_page *, if (_T) free_page((unsigned long)(_T)))
3779
3780 int kvm_arch_vcpu_create(struct kvm_vcpu *vcpu)
3781 {
3782 struct kvm_s390_mmu_cache *mc __free(kvm_s390_mmu_cache) = NULL;
3783 struct sie_page *sie_page __free(sie_page) = NULL;
3784 int rc;
3785
3786 BUILD_BUG_ON(sizeof(struct sie_page) != 4096);
3787 mc = kvm_s390_new_mmu_cache();
3788 if (!mc)
3789 return -ENOMEM;
3790 sie_page = (struct sie_page *) get_zeroed_page(GFP_KERNEL_ACCOUNT);
3791 if (!sie_page)
3792 return -ENOMEM;
3793
3794 vcpu->arch.sie_block = &sie_page->sie_block;
3795 vcpu->arch.sie_block->itdba = virt_to_phys(&sie_page->itdb);
3796
3797 /* the real guest size will always be smaller than msl */
3798 vcpu->arch.sie_block->mso = 0;
3799 vcpu->arch.sie_block->msl = sclp.hamax;
3800
3801 vcpu->arch.sie_block->icpua = vcpu->vcpu_id;
3802 spin_lock_init(&vcpu->arch.local_int.lock);
3803 vcpu->arch.sie_block->gd = kvm_s390_get_gisa_desc(vcpu->kvm);
3804 seqcount_init(&vcpu->arch.cputm_seqcount);
3805
3806 vcpu->arch.pfault_token = KVM_S390_PFAULT_TOKEN_INVALID;
3807 kvm_clear_async_pf_completion_queue(vcpu);
3808 vcpu->run->kvm_valid_regs = KVM_SYNC_PREFIX |
3809 KVM_SYNC_GPRS |
3810 KVM_SYNC_ACRS |
3811 KVM_SYNC_CRS |
3812 KVM_SYNC_ARCH0 |
3813 KVM_SYNC_PFAULT |
3814 KVM_SYNC_DIAG318;
3815 vcpu->arch.acrs_loaded = false;
3816 kvm_s390_set_prefix(vcpu, 0);
3817 if (test_kvm_facility(vcpu->kvm, 64))
3818 vcpu->run->kvm_valid_regs |= KVM_SYNC_RICCB;
3819 if (test_kvm_facility(vcpu->kvm, 82))
3820 vcpu->run->kvm_valid_regs |= KVM_SYNC_BPBC;
3821 if (test_kvm_facility(vcpu->kvm, 133))
3822 vcpu->run->kvm_valid_regs |= KVM_SYNC_GSCB;
3823 if (test_kvm_facility(vcpu->kvm, 156))
3824 vcpu->run->kvm_valid_regs |= KVM_SYNC_ETOKEN;
3825 /* fprs can be synchronized via vrs, even if the guest has no vx. With
3826 * cpu_has_vx(), (load|store)_fpu_regs() will work with vrs format.
3827 */
3828 if (cpu_has_vx())
3829 vcpu->run->kvm_valid_regs |= KVM_SYNC_VRS;
3830 else
3831 vcpu->run->kvm_valid_regs |= KVM_SYNC_FPRS;
3832
3833 if (kvm_is_ucontrol(vcpu->kvm)) {
3834 vcpu->arch.gmap = gmap_new_child(vcpu->kvm->arch.gmap, -1UL);
3835 if (!vcpu->arch.gmap)
3836 return -ENOMEM;
3837 }
3838
3839 VM_EVENT(vcpu->kvm, 3, "create cpu %d at 0x%p, sie block at 0x%p",
3840 vcpu->vcpu_id, vcpu, vcpu->arch.sie_block);
3841 trace_kvm_s390_create_vcpu(vcpu->vcpu_id, vcpu, vcpu->arch.sie_block);
3842
3843 rc = kvm_s390_vcpu_setup(vcpu);
3844 if (rc) {
3845 if (kvm_is_ucontrol(vcpu->kvm)) {
3846 scoped_guard(spinlock, &vcpu->kvm->arch.gmap->children_lock)
3847 gmap_remove_child(vcpu->arch.gmap);
3848 vcpu->arch.gmap = gmap_put(vcpu->arch.gmap);
3849 }
3850 return rc;
3851 }
3852
3853 vcpu->arch.mc = no_free_ptr(mc);
3854 sie_page = NULL;
3855 kvm_s390_update_topology_change_report(vcpu->kvm, 1);
3856 return 0;
3857 }
3858
kvm_arch_vcpu_runnable(struct kvm_vcpu * vcpu)3859 int kvm_arch_vcpu_runnable(struct kvm_vcpu *vcpu)
3860 {
3861 clear_bit(vcpu->vcpu_idx, vcpu->kvm->arch.gisa_int.kicked_mask);
3862 return kvm_s390_vcpu_has_irq(vcpu, 0);
3863 }
3864
kvm_arch_vcpu_in_kernel(struct kvm_vcpu * vcpu)3865 bool kvm_arch_vcpu_in_kernel(struct kvm_vcpu *vcpu)
3866 {
3867 return !(vcpu->arch.sie_block->gpsw.mask & PSW_MASK_PSTATE);
3868 }
3869
kvm_s390_vcpu_block(struct kvm_vcpu * vcpu)3870 void kvm_s390_vcpu_block(struct kvm_vcpu *vcpu)
3871 {
3872 atomic_or(PROG_BLOCK_SIE, &vcpu->arch.sie_block->prog20);
3873 exit_sie(vcpu);
3874 }
3875
kvm_s390_vcpu_unblock(struct kvm_vcpu * vcpu)3876 void kvm_s390_vcpu_unblock(struct kvm_vcpu *vcpu)
3877 {
3878 atomic_andnot(PROG_BLOCK_SIE, &vcpu->arch.sie_block->prog20);
3879 }
3880
kvm_s390_vcpu_request(struct kvm_vcpu * vcpu)3881 static void kvm_s390_vcpu_request(struct kvm_vcpu *vcpu)
3882 {
3883 atomic_or(PROG_REQUEST, &vcpu->arch.sie_block->prog20);
3884 exit_sie(vcpu);
3885 }
3886
kvm_s390_vcpu_sie_inhibited(struct kvm_vcpu * vcpu)3887 bool kvm_s390_vcpu_sie_inhibited(struct kvm_vcpu *vcpu)
3888 {
3889 return atomic_read(&vcpu->arch.sie_block->prog20) &
3890 (PROG_BLOCK_SIE | PROG_REQUEST);
3891 }
3892
kvm_s390_vcpu_request_handled(struct kvm_vcpu * vcpu)3893 static void kvm_s390_vcpu_request_handled(struct kvm_vcpu *vcpu)
3894 {
3895 atomic_andnot(PROG_REQUEST, &vcpu->arch.sie_block->prog20);
3896 }
3897
3898 /*
3899 * Kick a guest cpu out of (v)SIE and wait until (v)SIE is not running.
3900 * If the CPU is not running (e.g. waiting as idle) the function will
3901 * return immediately. */
exit_sie(struct kvm_vcpu * vcpu)3902 void exit_sie(struct kvm_vcpu *vcpu)
3903 {
3904 kvm_s390_set_cpuflags(vcpu, CPUSTAT_STOP_INT);
3905 kvm_s390_vsie_kick(vcpu);
3906 while (vcpu->arch.sie_block->prog0c & PROG_IN_SIE)
3907 cpu_relax();
3908 }
3909
3910 /* Kick a guest cpu out of SIE to process a request synchronously */
kvm_s390_sync_request(int req,struct kvm_vcpu * vcpu)3911 void kvm_s390_sync_request(int req, struct kvm_vcpu *vcpu)
3912 {
3913 __kvm_make_request(req, vcpu);
3914 kvm_s390_vcpu_request(vcpu);
3915 }
3916
kvm_arch_no_poll(struct kvm_vcpu * vcpu)3917 bool kvm_arch_no_poll(struct kvm_vcpu *vcpu)
3918 {
3919 /* do not poll with more than halt_poll_max_steal percent of steal time */
3920 if (get_lowcore()->avg_steal_timer * 100 / (TICK_USEC << 12) >=
3921 READ_ONCE(halt_poll_max_steal)) {
3922 vcpu->stat.halt_no_poll_steal++;
3923 return true;
3924 }
3925 return false;
3926 }
3927
kvm_arch_vcpu_should_kick(struct kvm_vcpu * vcpu)3928 int kvm_arch_vcpu_should_kick(struct kvm_vcpu *vcpu)
3929 {
3930 /* kvm common code refers to this, but never calls it */
3931 BUG();
3932 return 0;
3933 }
3934
kvm_arch_vcpu_ioctl_get_one_reg(struct kvm_vcpu * vcpu,struct kvm_one_reg * reg)3935 static int kvm_arch_vcpu_ioctl_get_one_reg(struct kvm_vcpu *vcpu,
3936 struct kvm_one_reg *reg)
3937 {
3938 int r = -EINVAL;
3939
3940 switch (reg->id) {
3941 case KVM_REG_S390_TODPR:
3942 r = put_user(vcpu->arch.sie_block->todpr,
3943 (u32 __user *)reg->addr);
3944 break;
3945 case KVM_REG_S390_EPOCHDIFF:
3946 r = put_user(vcpu->arch.sie_block->epoch,
3947 (u64 __user *)reg->addr);
3948 break;
3949 case KVM_REG_S390_CPU_TIMER:
3950 r = put_user(kvm_s390_get_cpu_timer(vcpu),
3951 (u64 __user *)reg->addr);
3952 break;
3953 case KVM_REG_S390_CLOCK_COMP:
3954 r = put_user(vcpu->arch.sie_block->ckc,
3955 (u64 __user *)reg->addr);
3956 break;
3957 case KVM_REG_S390_PFTOKEN:
3958 r = put_user(vcpu->arch.pfault_token,
3959 (u64 __user *)reg->addr);
3960 break;
3961 case KVM_REG_S390_PFCOMPARE:
3962 r = put_user(vcpu->arch.pfault_compare,
3963 (u64 __user *)reg->addr);
3964 break;
3965 case KVM_REG_S390_PFSELECT:
3966 r = put_user(vcpu->arch.pfault_select,
3967 (u64 __user *)reg->addr);
3968 break;
3969 case KVM_REG_S390_PP:
3970 r = put_user(vcpu->arch.sie_block->pp,
3971 (u64 __user *)reg->addr);
3972 break;
3973 case KVM_REG_S390_GBEA:
3974 r = put_user(vcpu->arch.sie_block->gbea,
3975 (u64 __user *)reg->addr);
3976 break;
3977 default:
3978 break;
3979 }
3980
3981 return r;
3982 }
3983
kvm_arch_vcpu_ioctl_set_one_reg(struct kvm_vcpu * vcpu,struct kvm_one_reg * reg)3984 static int kvm_arch_vcpu_ioctl_set_one_reg(struct kvm_vcpu *vcpu,
3985 struct kvm_one_reg *reg)
3986 {
3987 int r = -EINVAL;
3988 __u64 val;
3989
3990 switch (reg->id) {
3991 case KVM_REG_S390_TODPR:
3992 r = get_user(vcpu->arch.sie_block->todpr,
3993 (u32 __user *)reg->addr);
3994 break;
3995 case KVM_REG_S390_EPOCHDIFF:
3996 r = get_user(vcpu->arch.sie_block->epoch,
3997 (u64 __user *)reg->addr);
3998 break;
3999 case KVM_REG_S390_CPU_TIMER:
4000 r = get_user(val, (u64 __user *)reg->addr);
4001 if (!r)
4002 kvm_s390_set_cpu_timer(vcpu, val);
4003 break;
4004 case KVM_REG_S390_CLOCK_COMP:
4005 r = get_user(vcpu->arch.sie_block->ckc,
4006 (u64 __user *)reg->addr);
4007 break;
4008 case KVM_REG_S390_PFTOKEN:
4009 r = get_user(vcpu->arch.pfault_token,
4010 (u64 __user *)reg->addr);
4011 if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID)
4012 kvm_clear_async_pf_completion_queue(vcpu);
4013 break;
4014 case KVM_REG_S390_PFCOMPARE:
4015 r = get_user(vcpu->arch.pfault_compare,
4016 (u64 __user *)reg->addr);
4017 break;
4018 case KVM_REG_S390_PFSELECT:
4019 r = get_user(vcpu->arch.pfault_select,
4020 (u64 __user *)reg->addr);
4021 break;
4022 case KVM_REG_S390_PP:
4023 r = get_user(vcpu->arch.sie_block->pp,
4024 (u64 __user *)reg->addr);
4025 break;
4026 case KVM_REG_S390_GBEA:
4027 r = get_user(vcpu->arch.sie_block->gbea,
4028 (u64 __user *)reg->addr);
4029 break;
4030 default:
4031 break;
4032 }
4033
4034 return r;
4035 }
4036
kvm_arch_vcpu_ioctl_normal_reset(struct kvm_vcpu * vcpu)4037 static void kvm_arch_vcpu_ioctl_normal_reset(struct kvm_vcpu *vcpu)
4038 {
4039 vcpu->arch.sie_block->gpsw.mask &= ~PSW_MASK_RI;
4040 vcpu->arch.pfault_token = KVM_S390_PFAULT_TOKEN_INVALID;
4041 memset(vcpu->run->s.regs.riccb, 0, sizeof(vcpu->run->s.regs.riccb));
4042
4043 kvm_clear_async_pf_completion_queue(vcpu);
4044 if (!kvm_s390_user_cpu_state_ctrl(vcpu->kvm))
4045 kvm_s390_vcpu_stop(vcpu);
4046 kvm_s390_clear_local_irqs(vcpu);
4047 }
4048
kvm_arch_vcpu_ioctl_initial_reset(struct kvm_vcpu * vcpu)4049 static void kvm_arch_vcpu_ioctl_initial_reset(struct kvm_vcpu *vcpu)
4050 {
4051 /* Initial reset is a superset of the normal reset */
4052 kvm_arch_vcpu_ioctl_normal_reset(vcpu);
4053
4054 /*
4055 * This equals initial cpu reset in pop, but we don't switch to ESA.
4056 * We do not only reset the internal data, but also ...
4057 */
4058 vcpu->arch.sie_block->gpsw.mask = 0;
4059 vcpu->arch.sie_block->gpsw.addr = 0;
4060 kvm_s390_set_prefix(vcpu, 0);
4061 kvm_s390_set_cpu_timer(vcpu, 0);
4062 vcpu->arch.sie_block->ckc = 0;
4063 memset(vcpu->arch.sie_block->gcr, 0, sizeof(vcpu->arch.sie_block->gcr));
4064 vcpu->arch.sie_block->gcr[0] = CR0_INITIAL_MASK;
4065 vcpu->arch.sie_block->gcr[14] = CR14_INITIAL_MASK;
4066
4067 /* ... the data in sync regs */
4068 memset(vcpu->run->s.regs.crs, 0, sizeof(vcpu->run->s.regs.crs));
4069 vcpu->run->s.regs.ckc = 0;
4070 vcpu->run->s.regs.crs[0] = CR0_INITIAL_MASK;
4071 vcpu->run->s.regs.crs[14] = CR14_INITIAL_MASK;
4072 vcpu->run->psw_addr = 0;
4073 vcpu->run->psw_mask = 0;
4074 vcpu->run->s.regs.todpr = 0;
4075 vcpu->run->s.regs.cputm = 0;
4076 vcpu->run->s.regs.ckc = 0;
4077 vcpu->run->s.regs.pp = 0;
4078 vcpu->run->s.regs.gbea = 1;
4079 vcpu->run->s.regs.fpc = 0;
4080 /*
4081 * Do not reset these registers in the protected case, as some of
4082 * them are overlaid and they are not accessible in this case
4083 * anyway.
4084 */
4085 if (!kvm_s390_pv_cpu_is_protected(vcpu)) {
4086 vcpu->arch.sie_block->gbea = 1;
4087 vcpu->arch.sie_block->pp = 0;
4088 vcpu->arch.sie_block->fpf &= ~FPF_BPBC;
4089 vcpu->arch.sie_block->todpr = 0;
4090 }
4091 }
4092
kvm_arch_vcpu_ioctl_clear_reset(struct kvm_vcpu * vcpu)4093 static void kvm_arch_vcpu_ioctl_clear_reset(struct kvm_vcpu *vcpu)
4094 {
4095 struct kvm_sync_regs *regs = &vcpu->run->s.regs;
4096
4097 /* Clear reset is a superset of the initial reset */
4098 kvm_arch_vcpu_ioctl_initial_reset(vcpu);
4099
4100 memset(®s->gprs, 0, sizeof(regs->gprs));
4101 memset(®s->vrs, 0, sizeof(regs->vrs));
4102 memset(®s->acrs, 0, sizeof(regs->acrs));
4103 memset(®s->gscb, 0, sizeof(regs->gscb));
4104
4105 regs->etoken = 0;
4106 regs->etoken_extension = 0;
4107 }
4108
kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu * vcpu,struct kvm_regs * regs)4109 int kvm_arch_vcpu_ioctl_set_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
4110 {
4111 vcpu_load(vcpu);
4112 memcpy(&vcpu->run->s.regs.gprs, ®s->gprs, sizeof(regs->gprs));
4113 vcpu_put(vcpu);
4114 return 0;
4115 }
4116
kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu * vcpu,struct kvm_regs * regs)4117 int kvm_arch_vcpu_ioctl_get_regs(struct kvm_vcpu *vcpu, struct kvm_regs *regs)
4118 {
4119 vcpu_load(vcpu);
4120 memcpy(®s->gprs, &vcpu->run->s.regs.gprs, sizeof(regs->gprs));
4121 vcpu_put(vcpu);
4122 return 0;
4123 }
4124
kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu * vcpu,struct kvm_sregs * sregs)4125 int kvm_arch_vcpu_ioctl_set_sregs(struct kvm_vcpu *vcpu,
4126 struct kvm_sregs *sregs)
4127 {
4128 vcpu_load(vcpu);
4129
4130 memcpy(&vcpu->run->s.regs.acrs, &sregs->acrs, sizeof(sregs->acrs));
4131 memcpy(&vcpu->arch.sie_block->gcr, &sregs->crs, sizeof(sregs->crs));
4132
4133 vcpu_put(vcpu);
4134 return 0;
4135 }
4136
kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu * vcpu,struct kvm_sregs * sregs)4137 int kvm_arch_vcpu_ioctl_get_sregs(struct kvm_vcpu *vcpu,
4138 struct kvm_sregs *sregs)
4139 {
4140 vcpu_load(vcpu);
4141
4142 memcpy(&sregs->acrs, &vcpu->run->s.regs.acrs, sizeof(sregs->acrs));
4143 memcpy(&sregs->crs, &vcpu->arch.sie_block->gcr, sizeof(sregs->crs));
4144
4145 vcpu_put(vcpu);
4146 return 0;
4147 }
4148
kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu * vcpu,struct kvm_fpu * fpu)4149 int kvm_arch_vcpu_ioctl_set_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
4150 {
4151 vcpu_load(vcpu);
4152
4153 vcpu->run->s.regs.fpc = fpu->fpc;
4154 if (cpu_has_vx())
4155 convert_fp_to_vx((__vector128 *) vcpu->run->s.regs.vrs,
4156 (freg_t *) fpu->fprs);
4157 else
4158 memcpy(vcpu->run->s.regs.fprs, &fpu->fprs, sizeof(fpu->fprs));
4159
4160 vcpu_put(vcpu);
4161 return 0;
4162 }
4163
kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu * vcpu,struct kvm_fpu * fpu)4164 int kvm_arch_vcpu_ioctl_get_fpu(struct kvm_vcpu *vcpu, struct kvm_fpu *fpu)
4165 {
4166 vcpu_load(vcpu);
4167
4168 if (cpu_has_vx())
4169 convert_vx_to_fp((freg_t *) fpu->fprs,
4170 (__vector128 *) vcpu->run->s.regs.vrs);
4171 else
4172 memcpy(fpu->fprs, vcpu->run->s.regs.fprs, sizeof(fpu->fprs));
4173 fpu->fpc = vcpu->run->s.regs.fpc;
4174
4175 vcpu_put(vcpu);
4176 return 0;
4177 }
4178
kvm_arch_vcpu_ioctl_set_initial_psw(struct kvm_vcpu * vcpu,psw_t psw)4179 static int kvm_arch_vcpu_ioctl_set_initial_psw(struct kvm_vcpu *vcpu, psw_t psw)
4180 {
4181 int rc = 0;
4182
4183 if (!is_vcpu_stopped(vcpu))
4184 rc = -EBUSY;
4185 else {
4186 vcpu->run->psw_mask = psw.mask;
4187 vcpu->run->psw_addr = psw.addr;
4188 }
4189 return rc;
4190 }
4191
kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu * vcpu,struct kvm_translation * tr)4192 int kvm_arch_vcpu_ioctl_translate(struct kvm_vcpu *vcpu,
4193 struct kvm_translation *tr)
4194 {
4195 return -EINVAL; /* not implemented yet */
4196 }
4197
4198 #define VALID_GUESTDBG_FLAGS (KVM_GUESTDBG_SINGLESTEP | \
4199 KVM_GUESTDBG_USE_HW_BP | \
4200 KVM_GUESTDBG_ENABLE)
4201
kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu * vcpu,struct kvm_guest_debug * dbg)4202 int kvm_arch_vcpu_ioctl_set_guest_debug(struct kvm_vcpu *vcpu,
4203 struct kvm_guest_debug *dbg)
4204 {
4205 int rc = 0;
4206
4207 vcpu_load(vcpu);
4208
4209 vcpu->guest_debug = 0;
4210 kvm_s390_clear_bp_data(vcpu);
4211
4212 if (dbg->control & ~VALID_GUESTDBG_FLAGS) {
4213 rc = -EINVAL;
4214 goto out;
4215 }
4216 if (!sclp.has_gpere) {
4217 rc = -EINVAL;
4218 goto out;
4219 }
4220
4221 if (dbg->control & KVM_GUESTDBG_ENABLE) {
4222 vcpu->guest_debug = dbg->control;
4223 /* enforce guest PER */
4224 kvm_s390_set_cpuflags(vcpu, CPUSTAT_P);
4225
4226 if (dbg->control & KVM_GUESTDBG_USE_HW_BP) {
4227 scoped_guard(srcu, &vcpu->kvm->srcu)
4228 rc = kvm_s390_import_bp_data(vcpu, dbg);
4229 }
4230 } else {
4231 kvm_s390_clear_cpuflags(vcpu, CPUSTAT_P);
4232 vcpu->arch.guestdbg.last_bp = 0;
4233 }
4234
4235 if (rc) {
4236 vcpu->guest_debug = 0;
4237 kvm_s390_clear_bp_data(vcpu);
4238 kvm_s390_clear_cpuflags(vcpu, CPUSTAT_P);
4239 }
4240
4241 out:
4242 vcpu_put(vcpu);
4243 return rc;
4244 }
4245
kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu * vcpu,struct kvm_mp_state * mp_state)4246 int kvm_arch_vcpu_ioctl_get_mpstate(struct kvm_vcpu *vcpu,
4247 struct kvm_mp_state *mp_state)
4248 {
4249 int ret;
4250
4251 vcpu_load(vcpu);
4252
4253 /* CHECK_STOP and LOAD are not supported yet */
4254 ret = is_vcpu_stopped(vcpu) ? KVM_MP_STATE_STOPPED :
4255 KVM_MP_STATE_OPERATING;
4256
4257 vcpu_put(vcpu);
4258 return ret;
4259 }
4260
kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu * vcpu,struct kvm_mp_state * mp_state)4261 int kvm_arch_vcpu_ioctl_set_mpstate(struct kvm_vcpu *vcpu,
4262 struct kvm_mp_state *mp_state)
4263 {
4264 int rc = 0;
4265
4266 vcpu_load(vcpu);
4267
4268 /* user space knows about this interface - let it control the state */
4269 kvm_s390_set_user_cpu_state_ctrl(vcpu->kvm);
4270
4271 switch (mp_state->mp_state) {
4272 case KVM_MP_STATE_STOPPED:
4273 rc = kvm_s390_vcpu_stop(vcpu);
4274 break;
4275 case KVM_MP_STATE_OPERATING:
4276 rc = kvm_s390_vcpu_start(vcpu);
4277 break;
4278 case KVM_MP_STATE_LOAD:
4279 if (!kvm_s390_pv_cpu_is_protected(vcpu)) {
4280 rc = -ENXIO;
4281 break;
4282 }
4283 rc = kvm_s390_pv_set_cpu_state(vcpu, PV_CPU_STATE_OPR_LOAD);
4284 break;
4285 case KVM_MP_STATE_CHECK_STOP:
4286 fallthrough; /* CHECK_STOP and LOAD are not supported yet */
4287 default:
4288 rc = -ENXIO;
4289 }
4290
4291 vcpu_put(vcpu);
4292 return rc;
4293 }
4294
ibs_enabled(struct kvm_vcpu * vcpu)4295 static bool ibs_enabled(struct kvm_vcpu *vcpu)
4296 {
4297 return kvm_s390_test_cpuflags(vcpu, CPUSTAT_IBS);
4298 }
4299
vcpu_ucontrol_translate(struct kvm_vcpu * vcpu,gpa_t * gaddr)4300 static int vcpu_ucontrol_translate(struct kvm_vcpu *vcpu, gpa_t *gaddr)
4301 {
4302 int rc;
4303
4304 if (kvm_is_ucontrol(vcpu->kvm)) {
4305 rc = gmap_ucas_translate(vcpu->arch.mc, vcpu->arch.gmap, gaddr);
4306 if (rc == -EREMOTE) {
4307 vcpu->run->exit_reason = KVM_EXIT_S390_UCONTROL;
4308 vcpu->run->s390_ucontrol.trans_exc_code = *gaddr;
4309 vcpu->run->s390_ucontrol.pgm_code = PGM_SEGMENT_TRANSLATION;
4310 }
4311 return rc;
4312 }
4313 return 0;
4314 }
4315
kvm_s390_fixup_prefix(struct kvm_vcpu * vcpu)4316 static int kvm_s390_fixup_prefix(struct kvm_vcpu *vcpu)
4317 {
4318 gpa_t gaddr = kvm_s390_get_prefix(vcpu);
4319 gfn_t gfn;
4320 int rc;
4321
4322 if (vcpu_ucontrol_translate(vcpu, &gaddr))
4323 return -EREMOTE;
4324 gfn = gpa_to_gfn(gaddr);
4325
4326 rc = kvm_s390_faultin_gfn_simple(vcpu, NULL, gfn, true);
4327 if (rc)
4328 return rc;
4329 rc = kvm_s390_faultin_gfn_simple(vcpu, NULL, gfn + 1, true);
4330 if (rc)
4331 return rc;
4332
4333 scoped_guard(write_lock, &vcpu->kvm->mmu_lock)
4334 rc = dat_set_prefix_notif_bit(vcpu->kvm->arch.gmap->asce, gfn);
4335 return rc;
4336 }
4337
kvm_s390_handle_requests(struct kvm_vcpu * vcpu)4338 static int kvm_s390_handle_requests(struct kvm_vcpu *vcpu)
4339 {
4340 retry:
4341 kvm_s390_vcpu_request_handled(vcpu);
4342 if (!kvm_request_pending(vcpu))
4343 return 0;
4344 /*
4345 * If the guest prefix changed, re-arm the ipte notifier for the
4346 * guest prefix page. gmap_mprotect_notify will wait on the ptl lock.
4347 * This ensures that the ipte instruction for this request has
4348 * already finished. We might race against a second unmapper that
4349 * wants to set the blocking bit. Lets just retry the request loop.
4350 */
4351 if (kvm_check_request(KVM_REQ_REFRESH_GUEST_PREFIX, vcpu)) {
4352 int rc;
4353
4354 rc = kvm_s390_fixup_prefix(vcpu);
4355 if (rc) {
4356 kvm_make_request(KVM_REQ_REFRESH_GUEST_PREFIX, vcpu);
4357 return rc;
4358 }
4359 goto retry;
4360 }
4361
4362 if (kvm_check_request(KVM_REQ_TLB_FLUSH, vcpu)) {
4363 vcpu->arch.sie_block->ihcpu = 0xffff;
4364 goto retry;
4365 }
4366
4367 if (kvm_check_request(KVM_REQ_ENABLE_IBS, vcpu)) {
4368 if (!ibs_enabled(vcpu)) {
4369 trace_kvm_s390_enable_disable_ibs(vcpu->vcpu_id, 1);
4370 kvm_s390_set_cpuflags(vcpu, CPUSTAT_IBS);
4371 }
4372 goto retry;
4373 }
4374
4375 if (kvm_check_request(KVM_REQ_DISABLE_IBS, vcpu)) {
4376 if (ibs_enabled(vcpu)) {
4377 trace_kvm_s390_enable_disable_ibs(vcpu->vcpu_id, 0);
4378 kvm_s390_clear_cpuflags(vcpu, CPUSTAT_IBS);
4379 }
4380 goto retry;
4381 }
4382
4383 if (kvm_check_request(KVM_REQ_ICPT_OPEREXC, vcpu)) {
4384 vcpu->arch.sie_block->ictl |= ICTL_OPEREXC;
4385 goto retry;
4386 }
4387
4388 if (kvm_check_request(KVM_REQ_START_MIGRATION, vcpu)) {
4389 /*
4390 * Disable CMM virtualization; we will emulate the ESSA
4391 * instruction manually, in order to provide additional
4392 * functionalities needed for live migration.
4393 */
4394 vcpu->arch.sie_block->ecb2 &= ~ECB2_CMMA;
4395 goto retry;
4396 }
4397
4398 if (kvm_check_request(KVM_REQ_STOP_MIGRATION, vcpu)) {
4399 /*
4400 * Re-enable CMM virtualization if CMMA is available and
4401 * CMM has been used.
4402 */
4403 if (vcpu->kvm->arch.use_cmma && uses_cmm(vcpu->arch.gmap))
4404 vcpu->arch.sie_block->ecb2 |= ECB2_CMMA;
4405 goto retry;
4406 }
4407
4408 /* we left the vsie handler, nothing to do, just clear the request */
4409 kvm_clear_request(KVM_REQ_VSIE_RESTART, vcpu);
4410
4411 return 0;
4412 }
4413
__kvm_s390_set_tod_clock(struct kvm * kvm,const struct kvm_s390_vm_tod_clock * gtod)4414 static void __kvm_s390_set_tod_clock(struct kvm *kvm, const struct kvm_s390_vm_tod_clock *gtod)
4415 {
4416 struct kvm_vcpu *vcpu;
4417 union tod_clock clk;
4418 unsigned long i;
4419
4420 preempt_disable();
4421
4422 store_tod_clock_ext(&clk);
4423
4424 kvm->arch.epoch = gtod->tod - clk.tod;
4425 kvm->arch.epdx = 0;
4426 if (test_kvm_facility(kvm, 139)) {
4427 kvm->arch.epdx = gtod->epoch_idx - clk.ei;
4428 if (kvm->arch.epoch > gtod->tod)
4429 kvm->arch.epdx -= 1;
4430 }
4431
4432 kvm_s390_vcpu_block_all(kvm);
4433 kvm_for_each_vcpu(i, vcpu, kvm) {
4434 vcpu->arch.sie_block->epoch = kvm->arch.epoch;
4435 vcpu->arch.sie_block->epdx = kvm->arch.epdx;
4436 }
4437
4438 kvm_s390_vcpu_unblock_all(kvm);
4439 preempt_enable();
4440 }
4441
kvm_s390_try_set_tod_clock(struct kvm * kvm,const struct kvm_s390_vm_tod_clock * gtod)4442 int kvm_s390_try_set_tod_clock(struct kvm *kvm, const struct kvm_s390_vm_tod_clock *gtod)
4443 {
4444 if (!mutex_trylock(&kvm->lock))
4445 return 0;
4446 __kvm_s390_set_tod_clock(kvm, gtod);
4447 mutex_unlock(&kvm->lock);
4448 return 1;
4449 }
4450
__kvm_inject_pfault_token(struct kvm_vcpu * vcpu,bool start_token,unsigned long token)4451 static void __kvm_inject_pfault_token(struct kvm_vcpu *vcpu, bool start_token,
4452 unsigned long token)
4453 {
4454 struct kvm_s390_interrupt inti = {};
4455 struct kvm_s390_irq irq = {};
4456 struct kvm_s390_interrupt_info *inti_mem = NULL;
4457 int ret = 0;
4458
4459 if (start_token) {
4460 irq.u.ext.ext_params2 = token;
4461 irq.type = KVM_S390_INT_PFAULT_INIT;
4462 WARN_ON_ONCE(kvm_s390_inject_vcpu(vcpu, &irq));
4463 } else {
4464 inti_mem = kzalloc_obj(*inti_mem, GFP_KERNEL_ACCOUNT);
4465 if (WARN_ON_ONCE(!inti_mem))
4466 return;
4467
4468 inti.type = KVM_S390_INT_PFAULT_DONE;
4469 inti.parm64 = token;
4470 ret = kvm_s390_inject_vm(vcpu->kvm, &inti, inti_mem);
4471 if (ret)
4472 kfree(inti_mem);
4473 WARN_ON_ONCE(ret);
4474 }
4475 }
4476
kvm_arch_async_page_not_present(struct kvm_vcpu * vcpu,struct kvm_async_pf * work)4477 bool kvm_arch_async_page_not_present(struct kvm_vcpu *vcpu,
4478 struct kvm_async_pf *work)
4479 {
4480 trace_kvm_s390_pfault_init(vcpu, work->arch.pfault_token);
4481 __kvm_inject_pfault_token(vcpu, true, work->arch.pfault_token);
4482
4483 return true;
4484 }
4485
kvm_arch_async_page_present(struct kvm_vcpu * vcpu,struct kvm_async_pf * work)4486 void kvm_arch_async_page_present(struct kvm_vcpu *vcpu,
4487 struct kvm_async_pf *work)
4488 {
4489 trace_kvm_s390_pfault_done(vcpu, work->arch.pfault_token);
4490 __kvm_inject_pfault_token(vcpu, false, work->arch.pfault_token);
4491 }
4492
kvm_arch_async_page_ready(struct kvm_vcpu * vcpu,struct kvm_async_pf * work)4493 void kvm_arch_async_page_ready(struct kvm_vcpu *vcpu,
4494 struct kvm_async_pf *work)
4495 {
4496 /* s390 will always inject the page directly */
4497 }
4498
kvm_arch_can_dequeue_async_page_present(struct kvm_vcpu * vcpu)4499 bool kvm_arch_can_dequeue_async_page_present(struct kvm_vcpu *vcpu)
4500 {
4501 /*
4502 * s390 will always inject the page directly,
4503 * but we still want check_async_completion to cleanup
4504 */
4505 return true;
4506 }
4507
kvm_arch_setup_async_pf(struct kvm_vcpu * vcpu)4508 bool kvm_arch_setup_async_pf(struct kvm_vcpu *vcpu)
4509 {
4510 hva_t hva;
4511 struct kvm_arch_async_pf arch;
4512
4513 if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID)
4514 return false;
4515 if ((vcpu->arch.sie_block->gpsw.mask & vcpu->arch.pfault_select) !=
4516 vcpu->arch.pfault_compare)
4517 return false;
4518 if (psw_extint_disabled(vcpu))
4519 return false;
4520 if (kvm_s390_vcpu_has_irq(vcpu, 0))
4521 return false;
4522 if (!(vcpu->arch.sie_block->gcr[0] & CR0_SERVICE_SIGNAL_SUBMASK))
4523 return false;
4524 if (!pfault_enabled(vcpu->arch.gmap))
4525 return false;
4526
4527 hva = gfn_to_hva(vcpu->kvm, current->thread.gmap_teid.addr);
4528 if (read_guest_real(vcpu, vcpu->arch.pfault_token, &arch.pfault_token, 8))
4529 return false;
4530
4531 return kvm_setup_async_pf(vcpu, current->thread.gmap_teid.addr * PAGE_SIZE, hva, &arch);
4532 }
4533
vcpu_pre_run(struct kvm_vcpu * vcpu)4534 static int vcpu_pre_run(struct kvm_vcpu *vcpu)
4535 {
4536 int rc, cpuflags;
4537
4538 /*
4539 * On s390 notifications for arriving pages will be delivered directly
4540 * to the guest but the house keeping for completed pfaults is
4541 * handled outside the worker.
4542 */
4543 kvm_check_async_pf_completion(vcpu);
4544
4545 vcpu->arch.sie_block->gg14 = vcpu->run->s.regs.gprs[14];
4546 vcpu->arch.sie_block->gg15 = vcpu->run->s.regs.gprs[15];
4547
4548 if (!kvm_is_ucontrol(vcpu->kvm)) {
4549 rc = kvm_s390_deliver_pending_interrupts(vcpu);
4550 if (rc || guestdbg_exit_pending(vcpu))
4551 return rc;
4552 }
4553
4554 rc = kvm_s390_handle_requests(vcpu);
4555 if (rc)
4556 return rc;
4557
4558 if (guestdbg_enabled(vcpu)) {
4559 kvm_s390_backup_guest_per_regs(vcpu);
4560 kvm_s390_patch_guest_per_regs(vcpu);
4561 }
4562
4563 clear_bit(vcpu->vcpu_idx, vcpu->kvm->arch.gisa_int.kicked_mask);
4564
4565 vcpu->arch.sie_block->icptcode = 0;
4566 current->thread.gmap_int_code = 0;
4567 cpuflags = atomic_read(&vcpu->arch.sie_block->cpuflags);
4568 VCPU_EVENT(vcpu, 6, "entering sie flags %x", cpuflags);
4569 trace_kvm_s390_sie_enter(vcpu, cpuflags);
4570
4571 return 0;
4572 }
4573
vcpu_post_run_addressing_exception(struct kvm_vcpu * vcpu)4574 static int vcpu_post_run_addressing_exception(struct kvm_vcpu *vcpu)
4575 {
4576 struct kvm_s390_pgm_info pgm_info = {
4577 .code = PGM_ADDRESSING,
4578 };
4579 u8 opcode, ilen;
4580 int rc;
4581
4582 VCPU_EVENT(vcpu, 3, "%s", "fault in sie instruction");
4583 trace_kvm_s390_sie_fault(vcpu);
4584
4585 /*
4586 * We want to inject an addressing exception, which is defined as a
4587 * suppressing or terminating exception. However, since we came here
4588 * by a DAT access exception, the PSW still points to the faulting
4589 * instruction since DAT exceptions are nullifying. So we've got
4590 * to look up the current opcode to get the length of the instruction
4591 * to be able to forward the PSW.
4592 */
4593 rc = read_guest_instr(vcpu, vcpu->arch.sie_block->gpsw.addr, &opcode, 1);
4594 ilen = insn_length(opcode);
4595 if (rc < 0) {
4596 return rc;
4597 } else if (rc) {
4598 /* Instruction-Fetching Exceptions - we can't detect the ilen.
4599 * Forward by arbitrary ilc, injection will take care of
4600 * nullification if necessary.
4601 */
4602 pgm_info = vcpu->arch.pgm;
4603 ilen = 4;
4604 }
4605 pgm_info.flags = ilen | KVM_S390_PGM_FLAGS_ILC_VALID;
4606 kvm_s390_forward_psw(vcpu, ilen);
4607 return kvm_s390_inject_prog_irq(vcpu, &pgm_info);
4608 }
4609
kvm_s390_assert_primary_as(struct kvm_vcpu * vcpu)4610 static void kvm_s390_assert_primary_as(struct kvm_vcpu *vcpu)
4611 {
4612 KVM_BUG(current->thread.gmap_teid.as != PSW_BITS_AS_PRIMARY, vcpu->kvm,
4613 "Unexpected program interrupt 0x%x, TEID 0x%016lx",
4614 current->thread.gmap_int_code, current->thread.gmap_teid.val);
4615 }
4616
vcpu_dat_fault_handler(struct kvm_vcpu * vcpu,gpa_t gaddr,bool wr)4617 static int vcpu_dat_fault_handler(struct kvm_vcpu *vcpu, gpa_t gaddr, bool wr)
4618 {
4619 struct guest_fault f = {
4620 .write_attempt = wr,
4621 .attempt_pfault = pfault_enabled(vcpu->arch.gmap),
4622 };
4623 int rc;
4624
4625 if (vcpu_ucontrol_translate(vcpu, &gaddr))
4626 return -EREMOTE;
4627 f.gfn = gpa_to_gfn(gaddr);
4628
4629 rc = kvm_s390_faultin_gfn(vcpu, NULL, &f);
4630 if (rc <= 0)
4631 return rc;
4632 if (rc == PGM_ADDRESSING)
4633 return vcpu_post_run_addressing_exception(vcpu);
4634 KVM_BUG_ON(rc, vcpu->kvm);
4635 return -EINVAL;
4636 }
4637
vcpu_post_run_handle_fault(struct kvm_vcpu * vcpu)4638 static int vcpu_post_run_handle_fault(struct kvm_vcpu *vcpu)
4639 {
4640 unsigned int foll = 0;
4641 unsigned long gaddr;
4642 int rc;
4643
4644 gaddr = current->thread.gmap_teid.addr * PAGE_SIZE;
4645 if (kvm_s390_cur_gmap_fault_is_write())
4646 foll = FOLL_WRITE;
4647
4648 switch (current->thread.gmap_int_code & PGM_INT_CODE_MASK) {
4649 case 0:
4650 vcpu->stat.exit_null++;
4651 break;
4652 case PGM_SECURE_STORAGE_ACCESS:
4653 case PGM_SECURE_STORAGE_VIOLATION:
4654 kvm_s390_assert_primary_as(vcpu);
4655 /*
4656 * This can happen after a reboot with asynchronous teardown;
4657 * the new guest (normal or protected) will run on top of the
4658 * previous protected guest. The old pages need to be destroyed
4659 * so the new guest can use them.
4660 */
4661 if (kvm_s390_pv_destroy_page(vcpu->kvm, gaddr)) {
4662 /*
4663 * Either KVM messed up the secure guest mapping or the
4664 * same page is mapped into multiple secure guests.
4665 *
4666 * This exception is only triggered when a guest 2 is
4667 * running and can therefore never occur in kernel
4668 * context.
4669 */
4670 pr_warn_ratelimited("Secure storage violation (%x) in task: %s, pid %d\n",
4671 current->thread.gmap_int_code, current->comm,
4672 current->pid);
4673 send_sig(SIGSEGV, current, 0);
4674 }
4675 break;
4676 case PGM_NON_SECURE_STORAGE_ACCESS:
4677 kvm_s390_assert_primary_as(vcpu);
4678 /*
4679 * This is normal operation; a page belonging to a protected
4680 * guest has not been imported yet. Try to import the page into
4681 * the protected guest.
4682 */
4683 rc = kvm_s390_pv_convert_to_secure(vcpu->kvm, gaddr);
4684 if (rc == -EINVAL)
4685 send_sig(SIGSEGV, current, 0);
4686 if (rc != -ENXIO)
4687 break;
4688 foll = FOLL_WRITE;
4689 fallthrough;
4690 case PGM_PROTECTION:
4691 case PGM_SEGMENT_TRANSLATION:
4692 case PGM_PAGE_TRANSLATION:
4693 case PGM_ASCE_TYPE:
4694 case PGM_REGION_FIRST_TRANS:
4695 case PGM_REGION_SECOND_TRANS:
4696 case PGM_REGION_THIRD_TRANS:
4697 kvm_s390_assert_primary_as(vcpu);
4698 return vcpu_dat_fault_handler(vcpu, gaddr, foll);
4699 default:
4700 KVM_BUG(1, vcpu->kvm, "Unexpected program interrupt 0x%x, TEID 0x%016lx",
4701 current->thread.gmap_int_code, current->thread.gmap_teid.val);
4702 send_sig(SIGSEGV, current, 0);
4703 break;
4704 }
4705 return 0;
4706 }
4707
vcpu_post_run(struct kvm_vcpu * vcpu,int sie_return)4708 static int vcpu_post_run(struct kvm_vcpu *vcpu, int sie_return)
4709 {
4710 struct mcck_volatile_info *mcck_info;
4711 struct sie_page *sie_page;
4712 int rc;
4713
4714 VCPU_EVENT(vcpu, 6, "exit sie icptcode %d",
4715 vcpu->arch.sie_block->icptcode);
4716 trace_kvm_s390_sie_exit(vcpu, vcpu->arch.sie_block->icptcode);
4717
4718 if (guestdbg_enabled(vcpu))
4719 kvm_s390_restore_guest_per_regs(vcpu);
4720
4721 vcpu->run->s.regs.gprs[14] = vcpu->arch.sie_block->gg14;
4722 vcpu->run->s.regs.gprs[15] = vcpu->arch.sie_block->gg15;
4723
4724 if (sie_return == SIE64_RETURN_MCCK) {
4725 sie_page = container_of(vcpu->arch.sie_block,
4726 struct sie_page, sie_block);
4727 mcck_info = &sie_page->mcck_info;
4728 kvm_s390_reinject_machine_check(vcpu, mcck_info);
4729 return 0;
4730 }
4731 WARN_ON_ONCE(sie_return != SIE64_RETURN_NORMAL);
4732
4733 if (vcpu->arch.sie_block->icptcode > 0) {
4734 rc = kvm_handle_sie_intercept(vcpu);
4735
4736 if (rc != -EOPNOTSUPP)
4737 return rc;
4738 vcpu->run->exit_reason = KVM_EXIT_S390_SIEIC;
4739 vcpu->run->s390_sieic.icptcode = vcpu->arch.sie_block->icptcode;
4740 vcpu->run->s390_sieic.ipa = vcpu->arch.sie_block->ipa;
4741 vcpu->run->s390_sieic.ipb = vcpu->arch.sie_block->ipb;
4742 return -EREMOTE;
4743 }
4744
4745 return vcpu_post_run_handle_fault(vcpu);
4746 }
4747
kvm_s390_enter_exit_sie(struct kvm_s390_sie_block * scb,u64 * gprs,unsigned long gasce)4748 int noinstr kvm_s390_enter_exit_sie(struct kvm_s390_sie_block *scb,
4749 u64 *gprs, unsigned long gasce)
4750 {
4751 int ret;
4752
4753 guest_state_enter_irqoff();
4754
4755 /*
4756 * The guest_state_{enter,exit}_irqoff() functions inform lockdep and
4757 * tracing that entry to the guest will enable host IRQs, and exit from
4758 * the guest will disable host IRQs.
4759 */
4760 ret = sie64a(scb, gprs, gasce);
4761
4762 guest_state_exit_irqoff();
4763
4764 return ret;
4765 }
4766
4767 #define PSW_INT_MASK (PSW_MASK_EXT | PSW_MASK_IO | PSW_MASK_MCHECK)
__vcpu_run(struct kvm_vcpu * vcpu)4768 static int __vcpu_run(struct kvm_vcpu *vcpu)
4769 {
4770 int rc, sie_return;
4771 struct sie_page *sie_page = (struct sie_page *)vcpu->arch.sie_block;
4772
4773 /*
4774 * We try to hold kvm->srcu during most of vcpu_run (except when run-
4775 * ning the guest), so that memslots (and other stuff) are protected
4776 */
4777 kvm_vcpu_srcu_read_lock(vcpu);
4778
4779 while (true) {
4780 rc = vcpu_pre_run(vcpu);
4781 kvm_vcpu_srcu_read_unlock(vcpu);
4782 if (rc || guestdbg_exit_pending(vcpu))
4783 break;
4784
4785 /*
4786 * As PF_VCPU will be used in fault handler, between
4787 * guest_timing_enter_irqoff and guest_timing_exit_irqoff
4788 * should be no uaccess.
4789 */
4790 if (kvm_s390_pv_cpu_is_protected(vcpu)) {
4791 memcpy(sie_page->pv_grregs,
4792 vcpu->run->s.regs.gprs,
4793 sizeof(sie_page->pv_grregs));
4794 }
4795
4796 xfer_to_guest_mode_check:
4797 local_irq_disable();
4798 xfer_to_guest_mode_prepare();
4799 if (xfer_to_guest_mode_work_pending()) {
4800 local_irq_enable();
4801 rc = kvm_xfer_to_guest_mode_handle_work(vcpu);
4802 if (rc)
4803 break;
4804 goto xfer_to_guest_mode_check;
4805 }
4806
4807 guest_timing_enter_irqoff();
4808 __disable_cpu_timer_accounting(vcpu);
4809
4810 sie_return = kvm_s390_enter_exit_sie(vcpu->arch.sie_block,
4811 vcpu->run->s.regs.gprs,
4812 vcpu->arch.gmap->asce.val);
4813
4814 __enable_cpu_timer_accounting(vcpu);
4815 guest_timing_exit_irqoff();
4816 local_irq_enable();
4817
4818 if (kvm_s390_pv_cpu_is_protected(vcpu)) {
4819 memcpy(vcpu->run->s.regs.gprs,
4820 sie_page->pv_grregs,
4821 sizeof(sie_page->pv_grregs));
4822 /*
4823 * We're not allowed to inject interrupts on intercepts
4824 * that leave the guest state in an "in-between" state
4825 * where the next SIE entry will do a continuation.
4826 * Fence interrupts in our "internal" PSW.
4827 */
4828 if (vcpu->arch.sie_block->icptcode == ICPT_PV_INSTR ||
4829 vcpu->arch.sie_block->icptcode == ICPT_PV_PREF) {
4830 vcpu->arch.sie_block->gpsw.mask &= ~PSW_INT_MASK;
4831 }
4832 }
4833 kvm_vcpu_srcu_read_lock(vcpu);
4834
4835 rc = vcpu_post_run(vcpu, sie_return);
4836 if (rc || guestdbg_exit_pending(vcpu)) {
4837 kvm_vcpu_srcu_read_unlock(vcpu);
4838 break;
4839 }
4840 }
4841
4842 return rc;
4843 }
4844
sync_regs_fmt2(struct kvm_vcpu * vcpu)4845 static void sync_regs_fmt2(struct kvm_vcpu *vcpu)
4846 {
4847 struct kvm_run *kvm_run = vcpu->run;
4848 struct runtime_instr_cb *riccb;
4849 struct gs_cb *gscb;
4850
4851 riccb = (struct runtime_instr_cb *) &kvm_run->s.regs.riccb;
4852 gscb = (struct gs_cb *) &kvm_run->s.regs.gscb;
4853 vcpu->arch.sie_block->gpsw.mask = kvm_run->psw_mask;
4854 vcpu->arch.sie_block->gpsw.addr = kvm_run->psw_addr;
4855 if (kvm_run->kvm_dirty_regs & KVM_SYNC_ARCH0) {
4856 vcpu->arch.sie_block->todpr = kvm_run->s.regs.todpr;
4857 vcpu->arch.sie_block->pp = kvm_run->s.regs.pp;
4858 vcpu->arch.sie_block->gbea = kvm_run->s.regs.gbea;
4859 }
4860 if (kvm_run->kvm_dirty_regs & KVM_SYNC_PFAULT) {
4861 vcpu->arch.pfault_token = kvm_run->s.regs.pft;
4862 vcpu->arch.pfault_select = kvm_run->s.regs.pfs;
4863 vcpu->arch.pfault_compare = kvm_run->s.regs.pfc;
4864 if (vcpu->arch.pfault_token == KVM_S390_PFAULT_TOKEN_INVALID)
4865 kvm_clear_async_pf_completion_queue(vcpu);
4866 }
4867 if (kvm_run->kvm_dirty_regs & KVM_SYNC_DIAG318) {
4868 vcpu->arch.diag318_info.val = kvm_run->s.regs.diag318;
4869 vcpu->arch.sie_block->cpnc = vcpu->arch.diag318_info.cpnc;
4870 VCPU_EVENT(vcpu, 3, "setting cpnc to %d", vcpu->arch.diag318_info.cpnc);
4871 }
4872 /*
4873 * If userspace sets the riccb (e.g. after migration) to a valid state,
4874 * we should enable RI here instead of doing the lazy enablement.
4875 */
4876 if ((kvm_run->kvm_dirty_regs & KVM_SYNC_RICCB) &&
4877 test_kvm_facility(vcpu->kvm, 64) &&
4878 riccb->v &&
4879 !(vcpu->arch.sie_block->ecb3 & ECB3_RI)) {
4880 VCPU_EVENT(vcpu, 3, "%s", "ENABLE: RI (sync_regs)");
4881 vcpu->arch.sie_block->ecb3 |= ECB3_RI;
4882 }
4883 /*
4884 * If userspace sets the gscb (e.g. after migration) to non-zero,
4885 * we should enable GS here instead of doing the lazy enablement.
4886 */
4887 if ((kvm_run->kvm_dirty_regs & KVM_SYNC_GSCB) &&
4888 test_kvm_facility(vcpu->kvm, 133) &&
4889 gscb->gssm &&
4890 !vcpu->arch.gs_enabled) {
4891 VCPU_EVENT(vcpu, 3, "%s", "ENABLE: GS (sync_regs)");
4892 vcpu->arch.sie_block->ecb |= ECB_GS;
4893 vcpu->arch.sie_block->ecd |= ECD_HOSTREGMGMT;
4894 vcpu->arch.gs_enabled = 1;
4895 }
4896 if ((kvm_run->kvm_dirty_regs & KVM_SYNC_BPBC) &&
4897 test_kvm_facility(vcpu->kvm, 82)) {
4898 vcpu->arch.sie_block->fpf &= ~FPF_BPBC;
4899 vcpu->arch.sie_block->fpf |= kvm_run->s.regs.bpbc ? FPF_BPBC : 0;
4900 }
4901 if (cpu_has_gs()) {
4902 preempt_disable();
4903 local_ctl_set_bit(2, CR2_GUARDED_STORAGE_BIT);
4904 if (current->thread.gs_cb) {
4905 vcpu->arch.host_gscb = current->thread.gs_cb;
4906 save_gs_cb(vcpu->arch.host_gscb);
4907 }
4908 if (vcpu->arch.gs_enabled) {
4909 current->thread.gs_cb = (struct gs_cb *)
4910 &vcpu->run->s.regs.gscb;
4911 restore_gs_cb(current->thread.gs_cb);
4912 }
4913 preempt_enable();
4914 }
4915 /* SIE will load etoken directly from SDNX and therefore kvm_run */
4916 }
4917
sync_regs(struct kvm_vcpu * vcpu)4918 static void sync_regs(struct kvm_vcpu *vcpu)
4919 {
4920 struct kvm_run *kvm_run = vcpu->run;
4921
4922 if (kvm_run->kvm_dirty_regs & KVM_SYNC_PREFIX)
4923 kvm_s390_set_prefix(vcpu, kvm_run->s.regs.prefix);
4924 if (kvm_run->kvm_dirty_regs & KVM_SYNC_CRS) {
4925 memcpy(&vcpu->arch.sie_block->gcr, &kvm_run->s.regs.crs, 128);
4926 /* some control register changes require a tlb flush */
4927 kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
4928 }
4929 if (kvm_run->kvm_dirty_regs & KVM_SYNC_ARCH0) {
4930 kvm_s390_set_cpu_timer(vcpu, kvm_run->s.regs.cputm);
4931 vcpu->arch.sie_block->ckc = kvm_run->s.regs.ckc;
4932 }
4933 save_access_regs(vcpu->arch.host_acrs);
4934 restore_access_regs(vcpu->run->s.regs.acrs);
4935 vcpu->arch.acrs_loaded = true;
4936 kvm_s390_fpu_load(vcpu->run);
4937 /* Sync fmt2 only data */
4938 if (likely(!kvm_s390_pv_cpu_is_protected(vcpu))) {
4939 sync_regs_fmt2(vcpu);
4940 } else {
4941 /*
4942 * In several places we have to modify our internal view to
4943 * not do things that are disallowed by the ultravisor. For
4944 * example we must not inject interrupts after specific exits
4945 * (e.g. 112 prefix page not secure). We do this by turning
4946 * off the machine check, external and I/O interrupt bits
4947 * of our PSW copy. To avoid getting validity intercepts, we
4948 * do only accept the condition code from userspace.
4949 */
4950 vcpu->arch.sie_block->gpsw.mask &= ~PSW_MASK_CC;
4951 vcpu->arch.sie_block->gpsw.mask |= kvm_run->psw_mask &
4952 PSW_MASK_CC;
4953 }
4954
4955 kvm_run->kvm_dirty_regs = 0;
4956 }
4957
store_regs_fmt2(struct kvm_vcpu * vcpu)4958 static void store_regs_fmt2(struct kvm_vcpu *vcpu)
4959 {
4960 struct kvm_run *kvm_run = vcpu->run;
4961
4962 kvm_run->s.regs.todpr = vcpu->arch.sie_block->todpr;
4963 kvm_run->s.regs.pp = vcpu->arch.sie_block->pp;
4964 kvm_run->s.regs.gbea = vcpu->arch.sie_block->gbea;
4965 kvm_run->s.regs.bpbc = (vcpu->arch.sie_block->fpf & FPF_BPBC) == FPF_BPBC;
4966 kvm_run->s.regs.diag318 = vcpu->arch.diag318_info.val;
4967 if (cpu_has_gs()) {
4968 preempt_disable();
4969 local_ctl_set_bit(2, CR2_GUARDED_STORAGE_BIT);
4970 if (vcpu->arch.gs_enabled)
4971 save_gs_cb(current->thread.gs_cb);
4972 current->thread.gs_cb = vcpu->arch.host_gscb;
4973 restore_gs_cb(vcpu->arch.host_gscb);
4974 if (!vcpu->arch.host_gscb)
4975 local_ctl_clear_bit(2, CR2_GUARDED_STORAGE_BIT);
4976 vcpu->arch.host_gscb = NULL;
4977 preempt_enable();
4978 }
4979 /* SIE will save etoken directly into SDNX and therefore kvm_run */
4980 }
4981
store_regs(struct kvm_vcpu * vcpu)4982 static void store_regs(struct kvm_vcpu *vcpu)
4983 {
4984 struct kvm_run *kvm_run = vcpu->run;
4985
4986 kvm_run->psw_mask = vcpu->arch.sie_block->gpsw.mask;
4987 kvm_run->psw_addr = vcpu->arch.sie_block->gpsw.addr;
4988 kvm_run->s.regs.prefix = kvm_s390_get_prefix(vcpu);
4989 memcpy(&kvm_run->s.regs.crs, &vcpu->arch.sie_block->gcr, 128);
4990 kvm_run->s.regs.cputm = kvm_s390_get_cpu_timer(vcpu);
4991 kvm_run->s.regs.ckc = vcpu->arch.sie_block->ckc;
4992 kvm_run->s.regs.pft = vcpu->arch.pfault_token;
4993 kvm_run->s.regs.pfs = vcpu->arch.pfault_select;
4994 kvm_run->s.regs.pfc = vcpu->arch.pfault_compare;
4995 save_access_regs(vcpu->run->s.regs.acrs);
4996 restore_access_regs(vcpu->arch.host_acrs);
4997 vcpu->arch.acrs_loaded = false;
4998 kvm_s390_fpu_store(vcpu->run);
4999 if (likely(!kvm_s390_pv_cpu_is_protected(vcpu)))
5000 store_regs_fmt2(vcpu);
5001 }
5002
kvm_arch_vcpu_ioctl_run(struct kvm_vcpu * vcpu)5003 int kvm_arch_vcpu_ioctl_run(struct kvm_vcpu *vcpu)
5004 {
5005 struct kvm_run *kvm_run = vcpu->run;
5006 DECLARE_KERNEL_FPU_ONSTACK32(fpu);
5007 int rc;
5008
5009 /*
5010 * Running a VM while dumping always has the potential to
5011 * produce inconsistent dump data. But for PV vcpus a SIE
5012 * entry while dumping could also lead to a fatal validity
5013 * intercept which we absolutely want to avoid.
5014 */
5015 if (vcpu->kvm->arch.pv.dumping)
5016 return -EINVAL;
5017
5018 if (!vcpu->wants_to_run)
5019 return -EINTR;
5020
5021 if (kvm_run->kvm_valid_regs & ~KVM_SYNC_S390_VALID_FIELDS ||
5022 kvm_run->kvm_dirty_regs & ~KVM_SYNC_S390_VALID_FIELDS)
5023 return -EINVAL;
5024
5025 /* Pairs with smp_store_release() in kvm_arch_vcpu_postcreate() */
5026 if (!smp_load_acquire(&vcpu->arch.initialized))
5027 return -EINVAL;
5028
5029 vcpu_load(vcpu);
5030
5031 if (guestdbg_exit_pending(vcpu)) {
5032 kvm_s390_prepare_debug_exit(vcpu);
5033 rc = 0;
5034 goto out;
5035 }
5036
5037 kvm_sigset_activate(vcpu);
5038
5039 /*
5040 * no need to check the return value of vcpu_start as it can only have
5041 * an error for protvirt, but protvirt means user cpu state
5042 */
5043 if (!kvm_s390_user_cpu_state_ctrl(vcpu->kvm)) {
5044 kvm_s390_vcpu_start(vcpu);
5045 } else if (is_vcpu_stopped(vcpu)) {
5046 pr_err_ratelimited("can't run stopped vcpu %d\n",
5047 vcpu->vcpu_id);
5048 rc = -EINVAL;
5049 goto out_sigset;
5050 }
5051
5052 kernel_fpu_begin(&fpu, KERNEL_FPC | KERNEL_VXR);
5053 sync_regs(vcpu);
5054 enable_cpu_timer_accounting(vcpu);
5055
5056 might_fault();
5057 rc = __vcpu_run(vcpu);
5058
5059 if (signal_pending(current) && !rc) {
5060 kvm_run->exit_reason = KVM_EXIT_INTR;
5061 vcpu->stat.signal_exits++;
5062 rc = -EINTR;
5063 }
5064
5065 if (guestdbg_exit_pending(vcpu) && !rc) {
5066 kvm_s390_prepare_debug_exit(vcpu);
5067 rc = 0;
5068 }
5069
5070 if (rc == -EREMOTE) {
5071 /* userspace support is needed, kvm_run has been prepared */
5072 rc = 0;
5073 }
5074
5075 disable_cpu_timer_accounting(vcpu);
5076 store_regs(vcpu);
5077 kernel_fpu_end(&fpu, KERNEL_FPC | KERNEL_VXR);
5078
5079 vcpu->stat.exit_userspace++;
5080
5081 out_sigset:
5082 kvm_sigset_deactivate(vcpu);
5083
5084 out:
5085 vcpu_put(vcpu);
5086 return rc;
5087 }
5088
5089 /*
5090 * store status at address
5091 * we use have two special cases:
5092 * KVM_S390_STORE_STATUS_NOADDR: -> 0x1200 on 64 bit
5093 * KVM_S390_STORE_STATUS_PREFIXED: -> prefix
5094 */
kvm_s390_store_status_unloaded(struct kvm_vcpu * vcpu,unsigned long gpa)5095 int kvm_s390_store_status_unloaded(struct kvm_vcpu *vcpu, unsigned long gpa)
5096 {
5097 unsigned char archmode = 1;
5098 freg_t fprs[NUM_FPRS];
5099 unsigned int px;
5100 u64 clkcomp, cputm;
5101 int rc;
5102
5103 px = kvm_s390_get_prefix(vcpu);
5104 if (gpa == KVM_S390_STORE_STATUS_NOADDR) {
5105 if (write_guest_abs(vcpu, 163, &archmode, 1))
5106 return -EFAULT;
5107 gpa = 0;
5108 } else if (gpa == KVM_S390_STORE_STATUS_PREFIXED) {
5109 if (write_guest_real(vcpu, 163, &archmode, 1))
5110 return -EFAULT;
5111 gpa = px;
5112 } else
5113 gpa -= __LC_FPREGS_SAVE_AREA;
5114
5115 /* manually convert vector registers if necessary */
5116 if (cpu_has_vx()) {
5117 convert_vx_to_fp(fprs, (__vector128 *) vcpu->run->s.regs.vrs);
5118 rc = write_guest_abs(vcpu, gpa + __LC_FPREGS_SAVE_AREA,
5119 fprs, 128);
5120 } else {
5121 rc = write_guest_abs(vcpu, gpa + __LC_FPREGS_SAVE_AREA,
5122 vcpu->run->s.regs.fprs, 128);
5123 }
5124 rc |= write_guest_abs(vcpu, gpa + __LC_GPREGS_SAVE_AREA,
5125 vcpu->run->s.regs.gprs, 128);
5126 rc |= write_guest_abs(vcpu, gpa + __LC_PSW_SAVE_AREA,
5127 &vcpu->arch.sie_block->gpsw, 16);
5128 rc |= write_guest_abs(vcpu, gpa + __LC_PREFIX_SAVE_AREA,
5129 &px, 4);
5130 rc |= write_guest_abs(vcpu, gpa + __LC_FP_CREG_SAVE_AREA,
5131 &vcpu->run->s.regs.fpc, 4);
5132 rc |= write_guest_abs(vcpu, gpa + __LC_TOD_PROGREG_SAVE_AREA,
5133 &vcpu->arch.sie_block->todpr, 4);
5134 cputm = kvm_s390_get_cpu_timer(vcpu);
5135 rc |= write_guest_abs(vcpu, gpa + __LC_CPU_TIMER_SAVE_AREA,
5136 &cputm, 8);
5137 clkcomp = vcpu->arch.sie_block->ckc >> 8;
5138 rc |= write_guest_abs(vcpu, gpa + __LC_CLOCK_COMP_SAVE_AREA,
5139 &clkcomp, 8);
5140 rc |= write_guest_abs(vcpu, gpa + __LC_AREGS_SAVE_AREA,
5141 &vcpu->run->s.regs.acrs, 64);
5142 rc |= write_guest_abs(vcpu, gpa + __LC_CREGS_SAVE_AREA,
5143 &vcpu->arch.sie_block->gcr, 128);
5144 return rc ? -EFAULT : 0;
5145 }
5146
kvm_s390_vcpu_store_status(struct kvm_vcpu * vcpu,unsigned long addr)5147 int kvm_s390_vcpu_store_status(struct kvm_vcpu *vcpu, unsigned long addr)
5148 {
5149 /*
5150 * The guest FPRS and ACRS are in the host FPRS/ACRS due to the lazy
5151 * switch in the run ioctl. Let's update our copies before we save
5152 * it into the save area
5153 */
5154 kvm_s390_fpu_store(vcpu->run);
5155 save_access_regs(vcpu->run->s.regs.acrs);
5156
5157 return kvm_s390_store_status_unloaded(vcpu, addr);
5158 }
5159
__disable_ibs_on_vcpu(struct kvm_vcpu * vcpu)5160 static void __disable_ibs_on_vcpu(struct kvm_vcpu *vcpu)
5161 {
5162 kvm_check_request(KVM_REQ_ENABLE_IBS, vcpu);
5163 kvm_s390_sync_request(KVM_REQ_DISABLE_IBS, vcpu);
5164 }
5165
__disable_ibs_on_all_vcpus(struct kvm * kvm)5166 static void __disable_ibs_on_all_vcpus(struct kvm *kvm)
5167 {
5168 unsigned long i;
5169 struct kvm_vcpu *vcpu;
5170
5171 kvm_for_each_vcpu(i, vcpu, kvm) {
5172 __disable_ibs_on_vcpu(vcpu);
5173 }
5174 }
5175
__enable_ibs_on_vcpu(struct kvm_vcpu * vcpu)5176 static void __enable_ibs_on_vcpu(struct kvm_vcpu *vcpu)
5177 {
5178 if (!sclp.has_ibs)
5179 return;
5180 kvm_check_request(KVM_REQ_DISABLE_IBS, vcpu);
5181 kvm_s390_sync_request(KVM_REQ_ENABLE_IBS, vcpu);
5182 }
5183
kvm_s390_vcpu_start(struct kvm_vcpu * vcpu)5184 int kvm_s390_vcpu_start(struct kvm_vcpu *vcpu)
5185 {
5186 int i, online_vcpus, r = 0, started_vcpus = 0;
5187
5188 if (!is_vcpu_stopped(vcpu))
5189 return 0;
5190
5191 trace_kvm_s390_vcpu_start_stop(vcpu->vcpu_id, 1);
5192 /* Only one cpu at a time may enter/leave the STOPPED state. */
5193 spin_lock(&vcpu->kvm->arch.start_stop_lock);
5194 online_vcpus = atomic_read(&vcpu->kvm->online_vcpus);
5195
5196 /* Let's tell the UV that we want to change into the operating state */
5197 if (kvm_s390_pv_cpu_is_protected(vcpu)) {
5198 r = kvm_s390_pv_set_cpu_state(vcpu, PV_CPU_STATE_OPR);
5199 if (r) {
5200 spin_unlock(&vcpu->kvm->arch.start_stop_lock);
5201 return r;
5202 }
5203 }
5204
5205 for (i = 0; i < online_vcpus; i++) {
5206 if (!is_vcpu_stopped(kvm_get_vcpu(vcpu->kvm, i)))
5207 started_vcpus++;
5208 }
5209
5210 if (started_vcpus == 0) {
5211 /* we're the only active VCPU -> speed it up */
5212 __enable_ibs_on_vcpu(vcpu);
5213 } else if (started_vcpus == 1) {
5214 /*
5215 * As we are starting a second VCPU, we have to disable
5216 * the IBS facility on all VCPUs to remove potentially
5217 * outstanding ENABLE requests.
5218 */
5219 __disable_ibs_on_all_vcpus(vcpu->kvm);
5220 }
5221
5222 kvm_s390_clear_cpuflags(vcpu, CPUSTAT_STOPPED);
5223 /*
5224 * The real PSW might have changed due to a RESTART interpreted by the
5225 * ultravisor. We block all interrupts and let the next sie exit
5226 * refresh our view.
5227 */
5228 if (kvm_s390_pv_cpu_is_protected(vcpu))
5229 vcpu->arch.sie_block->gpsw.mask &= ~PSW_INT_MASK;
5230 /*
5231 * Another VCPU might have used IBS while we were offline.
5232 * Let's play safe and flush the VCPU at startup.
5233 */
5234 kvm_make_request(KVM_REQ_TLB_FLUSH, vcpu);
5235 spin_unlock(&vcpu->kvm->arch.start_stop_lock);
5236 return 0;
5237 }
5238
kvm_s390_vcpu_stop(struct kvm_vcpu * vcpu)5239 int kvm_s390_vcpu_stop(struct kvm_vcpu *vcpu)
5240 {
5241 int i, online_vcpus, r = 0, started_vcpus = 0;
5242 struct kvm_vcpu *started_vcpu = NULL;
5243
5244 if (is_vcpu_stopped(vcpu))
5245 return 0;
5246
5247 trace_kvm_s390_vcpu_start_stop(vcpu->vcpu_id, 0);
5248 /* Only one cpu at a time may enter/leave the STOPPED state. */
5249 spin_lock(&vcpu->kvm->arch.start_stop_lock);
5250 online_vcpus = atomic_read(&vcpu->kvm->online_vcpus);
5251
5252 /* Let's tell the UV that we want to change into the stopped state */
5253 if (kvm_s390_pv_cpu_is_protected(vcpu)) {
5254 r = kvm_s390_pv_set_cpu_state(vcpu, PV_CPU_STATE_STP);
5255 if (r) {
5256 spin_unlock(&vcpu->kvm->arch.start_stop_lock);
5257 return r;
5258 }
5259 }
5260
5261 /*
5262 * Set the VCPU to STOPPED and THEN clear the interrupt flag,
5263 * now that the SIGP STOP and SIGP STOP AND STORE STATUS orders
5264 * have been fully processed. This will ensure that the VCPU
5265 * is kept BUSY if another VCPU is inquiring with SIGP SENSE.
5266 */
5267 kvm_s390_set_cpuflags(vcpu, CPUSTAT_STOPPED);
5268 kvm_s390_clear_stop_irq(vcpu);
5269
5270 __disable_ibs_on_vcpu(vcpu);
5271
5272 for (i = 0; i < online_vcpus; i++) {
5273 struct kvm_vcpu *tmp = kvm_get_vcpu(vcpu->kvm, i);
5274
5275 if (!is_vcpu_stopped(tmp)) {
5276 started_vcpus++;
5277 started_vcpu = tmp;
5278 }
5279 }
5280
5281 if (started_vcpus == 1) {
5282 /*
5283 * As we only have one VCPU left, we want to enable the
5284 * IBS facility for that VCPU to speed it up.
5285 */
5286 __enable_ibs_on_vcpu(started_vcpu);
5287 }
5288
5289 spin_unlock(&vcpu->kvm->arch.start_stop_lock);
5290 return 0;
5291 }
5292
kvm_vcpu_ioctl_enable_cap(struct kvm_vcpu * vcpu,struct kvm_enable_cap * cap)5293 static int kvm_vcpu_ioctl_enable_cap(struct kvm_vcpu *vcpu,
5294 struct kvm_enable_cap *cap)
5295 {
5296 int r;
5297
5298 if (cap->flags)
5299 return -EINVAL;
5300
5301 switch (cap->cap) {
5302 case KVM_CAP_S390_CSS_SUPPORT:
5303 if (!vcpu->kvm->arch.css_support) {
5304 vcpu->kvm->arch.css_support = 1;
5305 VM_EVENT(vcpu->kvm, 3, "%s", "ENABLE: CSS support");
5306 trace_kvm_s390_enable_css(vcpu->kvm);
5307 }
5308 r = 0;
5309 break;
5310 default:
5311 r = -EINVAL;
5312 break;
5313 }
5314 return r;
5315 }
5316
kvm_s390_vcpu_sida_op(struct kvm_vcpu * vcpu,struct kvm_s390_mem_op * mop)5317 static long kvm_s390_vcpu_sida_op(struct kvm_vcpu *vcpu,
5318 struct kvm_s390_mem_op *mop)
5319 {
5320 void __user *uaddr = (void __user *)mop->buf;
5321 void *sida_addr;
5322 int r = 0;
5323
5324 if (mop->flags || !mop->size)
5325 return -EINVAL;
5326 if (mop->size + mop->sida_offset < mop->size)
5327 return -EINVAL;
5328 if (mop->size + mop->sida_offset > sida_size(vcpu->arch.sie_block))
5329 return -E2BIG;
5330 if (!kvm_s390_pv_cpu_is_protected(vcpu))
5331 return -EINVAL;
5332
5333 sida_addr = (char *)sida_addr(vcpu->arch.sie_block) + mop->sida_offset;
5334
5335 switch (mop->op) {
5336 case KVM_S390_MEMOP_SIDA_READ:
5337 if (copy_to_user(uaddr, sida_addr, mop->size))
5338 r = -EFAULT;
5339
5340 break;
5341 case KVM_S390_MEMOP_SIDA_WRITE:
5342 if (copy_from_user(sida_addr, uaddr, mop->size))
5343 r = -EFAULT;
5344 break;
5345 }
5346 return r;
5347 }
5348
kvm_s390_vcpu_mem_op(struct kvm_vcpu * vcpu,struct kvm_s390_mem_op * mop)5349 static long kvm_s390_vcpu_mem_op(struct kvm_vcpu *vcpu,
5350 struct kvm_s390_mem_op *mop)
5351 {
5352 void __user *uaddr = (void __user *)mop->buf;
5353 void *tmpbuf __free(kvfree) = NULL;
5354 enum gacc_mode acc_mode;
5355 int r;
5356
5357 r = mem_op_validate_common(mop, KVM_S390_MEMOP_F_INJECT_EXCEPTION |
5358 KVM_S390_MEMOP_F_CHECK_ONLY |
5359 KVM_S390_MEMOP_F_SKEY_PROTECTION);
5360 if (r)
5361 return r;
5362 if (mop->ar >= NUM_ACRS)
5363 return -EINVAL;
5364 if (kvm_s390_pv_cpu_is_protected(vcpu))
5365 return -EINVAL;
5366 if (!(mop->flags & KVM_S390_MEMOP_F_CHECK_ONLY)) {
5367 tmpbuf = vmalloc(mop->size);
5368 if (!tmpbuf)
5369 return -ENOMEM;
5370 }
5371
5372 acc_mode = mop->op == KVM_S390_MEMOP_LOGICAL_READ ? GACC_FETCH : GACC_STORE;
5373 if (mop->flags & KVM_S390_MEMOP_F_CHECK_ONLY) {
5374 r = check_gva_range(vcpu, mop->gaddr, mop->ar, mop->size,
5375 acc_mode, mop->key);
5376 } else if (acc_mode == GACC_FETCH) {
5377 r = read_guest_with_key(vcpu, mop->gaddr, mop->ar, tmpbuf,
5378 mop->size, mop->key);
5379 if (!r && copy_to_user(uaddr, tmpbuf, mop->size))
5380 return -EFAULT;
5381 } else {
5382 if (copy_from_user(tmpbuf, uaddr, mop->size))
5383 return -EFAULT;
5384 r = write_guest_with_key(vcpu, mop->gaddr, mop->ar, tmpbuf,
5385 mop->size, mop->key);
5386 }
5387
5388 if (r > 0 && (mop->flags & KVM_S390_MEMOP_F_INJECT_EXCEPTION) != 0)
5389 kvm_s390_inject_prog_irq(vcpu, &vcpu->arch.pgm);
5390
5391 return r;
5392 }
5393
kvm_s390_vcpu_memsida_op(struct kvm_vcpu * vcpu,struct kvm_s390_mem_op * mop)5394 static long kvm_s390_vcpu_memsida_op(struct kvm_vcpu *vcpu,
5395 struct kvm_s390_mem_op *mop)
5396 {
5397 int r, srcu_idx;
5398
5399 srcu_idx = srcu_read_lock(&vcpu->kvm->srcu);
5400
5401 switch (mop->op) {
5402 case KVM_S390_MEMOP_LOGICAL_READ:
5403 case KVM_S390_MEMOP_LOGICAL_WRITE:
5404 r = kvm_s390_vcpu_mem_op(vcpu, mop);
5405 break;
5406 case KVM_S390_MEMOP_SIDA_READ:
5407 case KVM_S390_MEMOP_SIDA_WRITE:
5408 /* we are locked against sida going away by the vcpu->mutex */
5409 r = kvm_s390_vcpu_sida_op(vcpu, mop);
5410 break;
5411 default:
5412 r = -EINVAL;
5413 }
5414
5415 srcu_read_unlock(&vcpu->kvm->srcu, srcu_idx);
5416 return r;
5417 }
5418
kvm_arch_vcpu_unlocked_ioctl(struct file * filp,unsigned int ioctl,unsigned long arg)5419 long kvm_arch_vcpu_unlocked_ioctl(struct file *filp, unsigned int ioctl,
5420 unsigned long arg)
5421 {
5422 struct kvm_vcpu *vcpu = filp->private_data;
5423 void __user *argp = (void __user *)arg;
5424 int rc;
5425
5426 switch (ioctl) {
5427 case KVM_S390_IRQ: {
5428 struct kvm_s390_irq s390irq;
5429
5430 if (copy_from_user(&s390irq, argp, sizeof(s390irq)))
5431 return -EFAULT;
5432 scoped_guard(srcu, &vcpu->kvm->srcu)
5433 rc = kvm_s390_inject_vcpu(vcpu, &s390irq);
5434 break;
5435 }
5436 case KVM_S390_INTERRUPT: {
5437 struct kvm_s390_interrupt s390int;
5438 struct kvm_s390_irq s390irq = {};
5439
5440 if (kvm_is_ucontrol(vcpu->kvm))
5441 return -EINVAL;
5442 if (copy_from_user(&s390int, argp, sizeof(s390int)))
5443 return -EFAULT;
5444 if (s390int_to_s390irq(&s390int, &s390irq))
5445 return -EINVAL;
5446 scoped_guard(srcu, &vcpu->kvm->srcu)
5447 rc = kvm_s390_inject_vcpu(vcpu, &s390irq);
5448 break;
5449 }
5450 default:
5451 rc = -ENOIOCTLCMD;
5452 break;
5453 }
5454
5455 /*
5456 * To simplify single stepping of userspace-emulated instructions,
5457 * KVM_EXIT_S390_SIEIC exit sets KVM_GUESTDBG_EXIT_PENDING (see
5458 * should_handle_per_ifetch()). However, if userspace emulation injects
5459 * an interrupt, it needs to be cleared, so that KVM_EXIT_DEBUG happens
5460 * after (and not before) the interrupt delivery.
5461 */
5462 if (!rc)
5463 vcpu->guest_debug &= ~KVM_GUESTDBG_EXIT_PENDING;
5464
5465 return rc;
5466 }
5467
kvm_s390_handle_pv_vcpu_dump(struct kvm_vcpu * vcpu,struct kvm_pv_cmd * cmd)5468 static int kvm_s390_handle_pv_vcpu_dump(struct kvm_vcpu *vcpu,
5469 struct kvm_pv_cmd *cmd)
5470 {
5471 struct kvm_s390_pv_dmp dmp;
5472 void *data;
5473 int ret;
5474
5475 /* Dump initialization is a prerequisite */
5476 if (!vcpu->kvm->arch.pv.dumping)
5477 return -EINVAL;
5478
5479 if (copy_from_user(&dmp, (__u8 __user *)cmd->data, sizeof(dmp)))
5480 return -EFAULT;
5481
5482 /* We only handle this subcmd right now */
5483 if (dmp.subcmd != KVM_PV_DUMP_CPU)
5484 return -EINVAL;
5485
5486 /* CPU dump length is the same as create cpu storage donation. */
5487 if (dmp.buff_len != uv_info.guest_cpu_stor_len)
5488 return -EINVAL;
5489
5490 data = kvzalloc(uv_info.guest_cpu_stor_len, GFP_KERNEL);
5491 if (!data)
5492 return -ENOMEM;
5493
5494 ret = kvm_s390_pv_dump_cpu(vcpu, data, &cmd->rc, &cmd->rrc);
5495
5496 VCPU_EVENT(vcpu, 3, "PROTVIRT DUMP CPU %d rc %x rrc %x",
5497 vcpu->vcpu_id, cmd->rc, cmd->rrc);
5498
5499 if (ret)
5500 ret = -EINVAL;
5501
5502 /* On success copy over the dump data */
5503 if (!ret && copy_to_user((__u8 __user *)dmp.buff_addr, data, uv_info.guest_cpu_stor_len))
5504 ret = -EFAULT;
5505
5506 kvfree(data);
5507 return ret;
5508 }
5509
kvm_arch_vcpu_ioctl(struct file * filp,unsigned int ioctl,unsigned long arg)5510 long kvm_arch_vcpu_ioctl(struct file *filp,
5511 unsigned int ioctl, unsigned long arg)
5512 {
5513 struct kvm_vcpu *vcpu = filp->private_data;
5514 void __user *argp = (void __user *)arg;
5515 int idx;
5516 long r;
5517 u16 rc, rrc;
5518
5519 /* Pairs with smp_store_release() in kvm_arch_vcpu_postcreate() */
5520 if (!smp_load_acquire(&vcpu->arch.initialized))
5521 return -EINVAL;
5522
5523 vcpu_load(vcpu);
5524
5525 switch (ioctl) {
5526 case KVM_S390_STORE_STATUS:
5527 idx = srcu_read_lock(&vcpu->kvm->srcu);
5528 r = kvm_s390_store_status_unloaded(vcpu, arg);
5529 srcu_read_unlock(&vcpu->kvm->srcu, idx);
5530 break;
5531 case KVM_S390_SET_INITIAL_PSW: {
5532 psw_t psw;
5533
5534 r = -EFAULT;
5535 if (copy_from_user(&psw, argp, sizeof(psw)))
5536 break;
5537 r = kvm_arch_vcpu_ioctl_set_initial_psw(vcpu, psw);
5538 break;
5539 }
5540 case KVM_S390_CLEAR_RESET:
5541 r = 0;
5542 kvm_arch_vcpu_ioctl_clear_reset(vcpu);
5543 if (kvm_s390_pv_cpu_is_protected(vcpu)) {
5544 r = uv_cmd_nodata(kvm_s390_pv_cpu_get_handle(vcpu),
5545 UVC_CMD_CPU_RESET_CLEAR, &rc, &rrc);
5546 VCPU_EVENT(vcpu, 3, "PROTVIRT RESET CLEAR VCPU: rc %x rrc %x",
5547 rc, rrc);
5548 }
5549 break;
5550 case KVM_S390_INITIAL_RESET:
5551 r = 0;
5552 kvm_arch_vcpu_ioctl_initial_reset(vcpu);
5553 if (kvm_s390_pv_cpu_is_protected(vcpu)) {
5554 r = uv_cmd_nodata(kvm_s390_pv_cpu_get_handle(vcpu),
5555 UVC_CMD_CPU_RESET_INITIAL,
5556 &rc, &rrc);
5557 VCPU_EVENT(vcpu, 3, "PROTVIRT RESET INITIAL VCPU: rc %x rrc %x",
5558 rc, rrc);
5559 }
5560 break;
5561 case KVM_S390_NORMAL_RESET:
5562 r = 0;
5563 kvm_arch_vcpu_ioctl_normal_reset(vcpu);
5564 if (kvm_s390_pv_cpu_is_protected(vcpu)) {
5565 r = uv_cmd_nodata(kvm_s390_pv_cpu_get_handle(vcpu),
5566 UVC_CMD_CPU_RESET, &rc, &rrc);
5567 VCPU_EVENT(vcpu, 3, "PROTVIRT RESET NORMAL VCPU: rc %x rrc %x",
5568 rc, rrc);
5569 }
5570 break;
5571 case KVM_SET_ONE_REG:
5572 case KVM_GET_ONE_REG: {
5573 struct kvm_one_reg reg;
5574 r = -EINVAL;
5575 if (kvm_s390_pv_cpu_is_protected(vcpu))
5576 break;
5577 r = -EFAULT;
5578 if (copy_from_user(®, argp, sizeof(reg)))
5579 break;
5580 if (ioctl == KVM_SET_ONE_REG)
5581 r = kvm_arch_vcpu_ioctl_set_one_reg(vcpu, ®);
5582 else
5583 r = kvm_arch_vcpu_ioctl_get_one_reg(vcpu, ®);
5584 break;
5585 }
5586 #ifdef CONFIG_KVM_S390_UCONTROL
5587 case KVM_S390_UCAS_MAP: {
5588 struct kvm_s390_ucas_mapping ucas;
5589
5590 r = -EFAULT;
5591 if (copy_from_user(&ucas, argp, sizeof(ucas)))
5592 break;
5593
5594 r = -EINVAL;
5595 if (!kvm_is_ucontrol(vcpu->kvm))
5596 break;
5597 if (!IS_ALIGNED(ucas.user_addr | ucas.vcpu_addr | ucas.length, _SEGMENT_SIZE))
5598 break;
5599
5600 r = gmap_ucas_map(vcpu->arch.gmap, gpa_to_gfn(ucas.user_addr),
5601 gpa_to_gfn(ucas.vcpu_addr),
5602 ucas.length >> _SEGMENT_SHIFT);
5603 break;
5604 }
5605 case KVM_S390_UCAS_UNMAP: {
5606 struct kvm_s390_ucas_mapping ucas;
5607
5608 r = -EFAULT;
5609 if (copy_from_user(&ucas, argp, sizeof(ucas)))
5610 break;
5611
5612 r = -EINVAL;
5613 if (!kvm_is_ucontrol(vcpu->kvm))
5614 break;
5615 if (!IS_ALIGNED(ucas.vcpu_addr | ucas.length, _SEGMENT_SIZE))
5616 break;
5617
5618 gmap_ucas_unmap(vcpu->arch.gmap, gpa_to_gfn(ucas.vcpu_addr),
5619 ucas.length >> _SEGMENT_SHIFT);
5620 r = 0;
5621 break;
5622 }
5623 #endif
5624 case KVM_S390_VCPU_FAULT: {
5625 gpa_t gaddr = arg;
5626
5627 scoped_guard(srcu, &vcpu->kvm->srcu) {
5628 r = vcpu_ucontrol_translate(vcpu, &gaddr);
5629 if (r)
5630 break;
5631
5632 r = kvm_s390_faultin_gfn_simple(vcpu, NULL, gpa_to_gfn(gaddr), false);
5633 if (r == PGM_ADDRESSING)
5634 r = -EFAULT;
5635 if (r <= 0)
5636 break;
5637 r = -EIO;
5638 KVM_BUG_ON(r, vcpu->kvm);
5639 }
5640 break;
5641 }
5642 case KVM_ENABLE_CAP:
5643 {
5644 struct kvm_enable_cap cap;
5645 r = -EFAULT;
5646 if (copy_from_user(&cap, argp, sizeof(cap)))
5647 break;
5648 r = kvm_vcpu_ioctl_enable_cap(vcpu, &cap);
5649 break;
5650 }
5651 case KVM_S390_MEM_OP: {
5652 struct kvm_s390_mem_op mem_op;
5653
5654 if (copy_from_user(&mem_op, argp, sizeof(mem_op)) == 0)
5655 r = kvm_s390_vcpu_memsida_op(vcpu, &mem_op);
5656 else
5657 r = -EFAULT;
5658 break;
5659 }
5660 case KVM_S390_SET_IRQ_STATE: {
5661 struct kvm_s390_irq_state irq_state;
5662
5663 r = -EFAULT;
5664 if (copy_from_user(&irq_state, argp, sizeof(irq_state)))
5665 break;
5666 if (irq_state.len > VCPU_IRQS_MAX_BUF ||
5667 irq_state.len == 0 ||
5668 irq_state.len % sizeof(struct kvm_s390_irq) > 0) {
5669 r = -EINVAL;
5670 break;
5671 }
5672 /* do not use irq_state.flags, it will break old QEMUs */
5673 r = kvm_s390_set_irq_state(vcpu,
5674 (void __user *) irq_state.buf,
5675 irq_state.len);
5676 break;
5677 }
5678 case KVM_S390_GET_IRQ_STATE: {
5679 struct kvm_s390_irq_state irq_state;
5680
5681 r = -EFAULT;
5682 if (copy_from_user(&irq_state, argp, sizeof(irq_state)))
5683 break;
5684 if (irq_state.len == 0) {
5685 r = -EINVAL;
5686 break;
5687 }
5688 /* do not use irq_state.flags, it will break old QEMUs */
5689 r = kvm_s390_get_irq_state(vcpu,
5690 (__u8 __user *) irq_state.buf,
5691 irq_state.len);
5692 break;
5693 }
5694 case KVM_S390_PV_CPU_COMMAND: {
5695 struct kvm_pv_cmd cmd;
5696
5697 r = -EINVAL;
5698 if (!is_prot_virt_host())
5699 break;
5700
5701 r = -EFAULT;
5702 if (copy_from_user(&cmd, argp, sizeof(cmd)))
5703 break;
5704
5705 r = -EINVAL;
5706 if (cmd.flags)
5707 break;
5708
5709 /* We only handle this cmd right now */
5710 if (cmd.cmd != KVM_PV_DUMP)
5711 break;
5712
5713 r = kvm_s390_handle_pv_vcpu_dump(vcpu, &cmd);
5714
5715 /* Always copy over UV rc / rrc data */
5716 if (copy_to_user(argp + offsetof(struct kvm_pv_cmd, rc), &cmd.rc,
5717 sizeof(cmd.rc) + sizeof(cmd.rrc)))
5718 r = -EFAULT;
5719 break;
5720 }
5721 default:
5722 r = -ENOTTY;
5723 }
5724
5725 vcpu_put(vcpu);
5726 return r;
5727 }
5728
kvm_arch_vcpu_fault(struct kvm_vcpu * vcpu,struct vm_fault * vmf)5729 vm_fault_t kvm_arch_vcpu_fault(struct kvm_vcpu *vcpu, struct vm_fault *vmf)
5730 {
5731 #ifdef CONFIG_KVM_S390_UCONTROL
5732 if ((vmf->pgoff == KVM_S390_SIE_PAGE_OFFSET)
5733 && (kvm_is_ucontrol(vcpu->kvm))) {
5734 vmf->page = virt_to_page(vcpu->arch.sie_block);
5735 get_page(vmf->page);
5736 return 0;
5737 }
5738 #endif
5739 return VM_FAULT_SIGBUS;
5740 }
5741
kvm_arch_irqchip_in_kernel(struct kvm * kvm)5742 bool kvm_arch_irqchip_in_kernel(struct kvm *kvm)
5743 {
5744 return true;
5745 }
5746
5747 /* Section: memory related */
kvm_arch_prepare_memory_region(struct kvm * kvm,const struct kvm_memory_slot * old,struct kvm_memory_slot * new,enum kvm_mr_change change)5748 int kvm_arch_prepare_memory_region(struct kvm *kvm,
5749 const struct kvm_memory_slot *old,
5750 struct kvm_memory_slot *new,
5751 enum kvm_mr_change change)
5752 {
5753 return s390_kvm_mmu_prepare_memory_region(kvm, old, new, change);
5754 }
5755
cmma_d_count_pte(union pte * ptep,gfn_t gfn,gfn_t next,struct dat_walk * walk)5756 static long cmma_d_count_pte(union pte *ptep, gfn_t gfn, gfn_t next, struct dat_walk *walk)
5757 {
5758 union pgste pgste;
5759
5760 pgste = pgste_get_lock(ptep);
5761 if (pgste.cmma_d) {
5762 pgste.cmma_d = 0;
5763 atomic64_dec(walk->priv);
5764 }
5765 pgste_set_unlock(ptep, pgste);
5766 return 0;
5767 }
5768
kvm_s390_update_cmma_dirty(struct kvm * kvm,struct kvm_memory_slot * old)5769 void kvm_s390_update_cmma_dirty(struct kvm *kvm, struct kvm_memory_slot *old)
5770 {
5771 const struct dat_walk_ops ops = { .pte_entry = cmma_d_count_pte, };
5772
5773 if (kvm->arch.migration_mode && kvm->arch.use_cmma && old) {
5774 _dat_walk_gfn_range(old->base_gfn, old->base_gfn + old->npages,
5775 kvm->arch.gmap->asce, &ops, DAT_WALK_IGN_HOLES,
5776 &kvm->arch.cmma_dirty_pages);
5777 }
5778 }
5779
kvm_arch_commit_memory_region(struct kvm * kvm,struct kvm_memory_slot * old,const struct kvm_memory_slot * new,enum kvm_mr_change change)5780 void kvm_arch_commit_memory_region(struct kvm *kvm, struct kvm_memory_slot *old,
5781 const struct kvm_memory_slot *new,
5782 enum kvm_mr_change change)
5783 {
5784 s390_kvm_mmu_commit_memory_region(kvm, old, new, change);
5785 }
5786
5787 /**
5788 * kvm_arch_vcpu_pre_fault_memory() -- pre-fault and link gmap dat tables
5789 * @vcpu: the vcpu that shall appear to have generated the fault-in.
5790 * @range: the range that needs to be faulted in.
5791 *
5792 * The first page of the given range is faulted in and the corresponding gmap
5793 * page tables are created, as if the given vCPU had performed a read
5794 * operation.
5795 * If the range starts outside any memslots, an error is returned. An error is
5796 * also returned for UCONTROL VMs, which should instead use the
5797 * KVM_S390_VCPU_FAULT ioctl.
5798 *
5799 * Return:
5800 * * %-ENOENT if the range lies outside of a memslot.
5801 * * %-EINVAL in case of invalid state (for example if the VM is UCONTROL).
5802 * * %-EIO if errors happen while faulting-in the page (will trigger a warning
5803 * in the caller).
5804 * * other error codes < 0 in case of other errors.
5805 * * otherwise a number > 0 of bytes that have been faulted in successfully.
5806 */
kvm_arch_vcpu_pre_fault_memory(struct kvm_vcpu * vcpu,struct kvm_pre_fault_memory * range)5807 long kvm_arch_vcpu_pre_fault_memory(struct kvm_vcpu *vcpu, struct kvm_pre_fault_memory *range)
5808 {
5809 struct guest_fault f = { .gfn = gpa_to_gfn(range->gpa), };
5810 gpa_t end;
5811 int rc;
5812
5813 if (kvm_is_ucontrol(vcpu->kvm))
5814 return -EINVAL;
5815
5816 rc = kvm_s390_faultin_gfn(vcpu, NULL, &f);
5817 if (rc == PGM_ADDRESSING)
5818 return -ENOENT;
5819 if (rc > 0)
5820 return -EIO;
5821 if (rc < 0)
5822 return rc;
5823
5824 if (f.ptep)
5825 return PAGE_SIZE;
5826
5827 end = ALIGN(range->gpa + PAGE_SIZE, f.crste_region3 ? _REGION3_SIZE : HPAGE_SIZE);
5828 return min(range->size, end - range->gpa);
5829 }
5830
5831 /**
5832 * kvm_test_age_gfn() - test young
5833 * @kvm: the kvm instance
5834 * @range: the range of guest addresses whose young status needs to be cleared
5835 *
5836 * Context: called by KVM common code without holding the kvm mmu lock
5837 * Return: true if any page in the given range is young, otherwise 0.
5838 */
kvm_test_age_gfn(struct kvm * kvm,struct kvm_gfn_range * range)5839 bool kvm_test_age_gfn(struct kvm *kvm, struct kvm_gfn_range *range)
5840 {
5841 scoped_guard(read_lock, &kvm->mmu_lock)
5842 return dat_test_age_gfn(kvm->arch.gmap->asce, range->start, range->end);
5843 }
5844
5845 /**
5846 * kvm_age_gfn() - clear young
5847 * @kvm: the kvm instance
5848 * @range: the range of guest addresses whose young status needs to be cleared
5849 *
5850 * Context: called by KVM common code without holding the kvm mmu lock
5851 * Return: true if any page in the given range was young, otherwise 0.
5852 */
kvm_age_gfn(struct kvm * kvm,struct kvm_gfn_range * range)5853 bool kvm_age_gfn(struct kvm *kvm, struct kvm_gfn_range *range)
5854 {
5855 scoped_guard(read_lock, &kvm->mmu_lock)
5856 return gmap_age_gfn(kvm->arch.gmap, range->start, range->end);
5857 }
5858
5859 /**
5860 * kvm_unmap_gfn_range() - Unmap a range of guest addresses
5861 * @kvm: the kvm instance
5862 * @range: the range of guest page frames to invalidate
5863 *
5864 * This function always returns false because every DAT table modification
5865 * has to use the appropriate DAT table manipulation instructions, which will
5866 * keep the TLB coherent, hence no additional TLB flush is ever required.
5867 *
5868 * Context: called by KVM common code with the kvm mmu write lock held
5869 * Return: false
5870 */
kvm_unmap_gfn_range(struct kvm * kvm,struct kvm_gfn_range * range)5871 bool kvm_unmap_gfn_range(struct kvm *kvm, struct kvm_gfn_range *range)
5872 {
5873 return gmap_unmap_gfn_range(kvm->arch.gmap, range->slot, range->start, range->end);
5874 }
5875
nonhyp_mask(int i)5876 static inline unsigned long nonhyp_mask(int i)
5877 {
5878 unsigned int nonhyp_fai = (sclp.hmfai << i * 2) >> 30;
5879
5880 return 0x0000ffffffffffffUL >> (nonhyp_fai << 4);
5881 }
5882
kvm_s390_init(void)5883 static int __init kvm_s390_init(void)
5884 {
5885 int i, r;
5886
5887 if (!sclp.has_sief2) {
5888 pr_info("SIE is not available\n");
5889 return -ENODEV;
5890 }
5891
5892 if (hpage_2g && !hpage) {
5893 hpage_2g = 0;
5894 pr_info("Disabling 2G hugepage support, since 1M hugepage support is not enabled.\n");
5895 }
5896
5897 for (i = 0; i < HMFAI_DWORDS; i++)
5898 kvm_s390_fac_base[i] |= nonhyp_mask(i);
5899
5900 r = __kvm_s390_init();
5901 if (r)
5902 return r;
5903
5904 r = kvm_init(sizeof(struct kvm_vcpu), 0, THIS_MODULE);
5905 if (r) {
5906 __kvm_s390_exit();
5907 return r;
5908 }
5909 return 0;
5910 }
5911
kvm_s390_exit(void)5912 static void __exit kvm_s390_exit(void)
5913 {
5914 kvm_exit();
5915
5916 __kvm_s390_exit();
5917 }
5918
5919 module_init(kvm_s390_init);
5920 module_exit(kvm_s390_exit);
5921
5922 /*
5923 * Enable autoloading of the kvm module.
5924 * Note that we add the module alias here instead of virt/kvm/kvm_main.c
5925 * since x86 takes a different approach.
5926 */
5927 #include <linux/miscdevice.h>
5928 MODULE_ALIAS_MISCDEV(KVM_MINOR);
5929 MODULE_ALIAS("devname:kvm");
5930